From 07fb6df6bcab0872f586ec8388be88d3b21d722e Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 16 May 2021 20:44:35 -0600 Subject: [PATCH 0001/1654] Add `checked` to dom This allows the ability to set a checkbox as checked programmatically. It's different from `setAttribute` because once an input has been clicked on by the user, `setAttribute` no longer works programmatically. --- lib/js/dom.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 303cc178f..16fc96b68 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1379,6 +1379,9 @@ proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.} proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.} +proc checked*(n: Node): bool {.importcpp: "#.checked", nodecl.} +proc `checked=`*(n: Node; v: bool) {.importcpp: "#.checked = #", nodecl.} + proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.} when defined(nodejs): From e904c6d87ccbb453f6f1d649b2c715bd3cebc815 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 17 May 2021 00:04:39 -0600 Subject: [PATCH 0002/1654] Add `checked` to dom (#18033) This allows the ability to set a checkbox as checked programmatically. It's different from `setAttribute` because once an input has been clicked on by the user, `setAttribute` no longer works programmatically. --- lib/js/dom.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 303cc178f..16fc96b68 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1379,6 +1379,9 @@ proc `class=`*(n: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} proc value*(n: Node): cstring {.importcpp: "#.value", nodecl.} proc `value=`*(n: Node; v: cstring) {.importcpp: "#.value = #", nodecl.} +proc checked*(n: Node): bool {.importcpp: "#.checked", nodecl.} +proc `checked=`*(n: Node; v: bool) {.importcpp: "#.checked = #", nodecl.} + proc `disabled=`*(n: Node; v: bool) {.importcpp: "#.disabled = #", nodecl.} when defined(nodejs): From 8be5344b3bb74d62cff5aa6a18f4223680612ad5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 17 May 2021 01:28:52 -0700 Subject: [PATCH 0003/1654] ./koch --nonexistant now fails (#18036) --- koch.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 15355ec36..3d7276aee 100644 --- a/koch.nim +++ b/koch.nim @@ -634,9 +634,9 @@ proc valgrind(cmd: string) = let supp = getAppDir() / "tools" / "nimgrind.supp" exec("valgrind --suppressions=" & supp & valcmd) -proc showHelp() = +proc showHelp(success: bool) = quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)), - CompileDate, CompileTime], QuitSuccess) + CompileDate, CompileTime], if success: QuitSuccess else: QuitFailure) proc branchDone() = let thisBranch = execProcess("git symbolic-ref --short HEAD").strip() @@ -656,6 +656,7 @@ when isMainModule: case op.kind of cmdLongOption, cmdShortOption: case normalize(op.key) + of "help", "h": showHelp(success = true) of "latest": latest = true of "stable": latest = false of "nim": nimExe = op.val.absolutePath # absolute so still works with changeDir @@ -663,7 +664,7 @@ when isMainModule: localDocsOnly = true if op.val.len > 0: localDocsOut = op.val.absolutePath - else: showHelp() + else: showHelp(success = false) of cmdArgument: case normalize(op.key) of "boot": boot(op.cmdLineRest) @@ -705,6 +706,6 @@ when isMainModule: exec("nimble install -y fusion@$#" % suffix) of "ic": icTest(op.cmdLineRest) of "branchdone": branchDone() - else: showHelp() + else: showHelp(success = false) break of cmdEnd: break From fac5bae7b7d87aeec48c7252029c2852ee157ac9 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 17 May 2021 13:48:10 +0200 Subject: [PATCH 0004/1654] Fix running testament c nimble-packages without batch arg (#18023) * Fix running testament c nimble-packages without batch arg * Fix --- testament/categories.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 1253a2b8c..1f0bd6acf 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -424,11 +424,12 @@ proc listPackages(packageFilter: string): seq[NimblePackage] = # at least should be a regex; a substring match makes no sense. result = pkgs.filterIt(packageFilter in it.name) else: - let pkgs1 = pkgs.filterIt(it.allowFailure) - let pkgs2 = pkgs.filterIt(not it.allowFailure) if testamentData0.batchArg == "allowed_failures": - result = pkgs1 + result = pkgs.filterIt(it.allowFailure) + elif testamentData0.testamentNumBatch == 0: + result = pkgs else: + let pkgs2 = pkgs.filterIt(not it.allowFailure) for i in 0.. Date: Tue, 18 May 2021 09:16:07 +0300 Subject: [PATCH 0005/1654] add sink and lent annotations for xmltree and streams (#18037) --- lib/pure/streams.nim | 2 +- lib/pure/xmltree.nim | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 3cc5ea038..7dc81148f 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1252,7 +1252,7 @@ else: # after 1.3 or JS not defined var s = StringStream(s) s.data = "" - proc newStringStream*(s: string = ""): owned StringStream = + proc newStringStream*(s: sink string = ""): owned StringStream = ## Creates a new stream from the string `s`. ## ## See also: diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index af7e4d508..fdb2c3fc5 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -70,7 +70,7 @@ proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) -proc newElement*(tag: string): XmlNode = +proc newElement*(tag: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnElement`` with the given `tag`. ## ## See also: @@ -89,7 +89,7 @@ proc newElement*(tag: string): XmlNode = result.s = @[] # init attributes lazily to save memory -proc newText*(text: string): XmlNode = +proc newText*(text: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnText`` with the text `text`. runnableExamples: var b = newText("my text") @@ -99,13 +99,13 @@ proc newText*(text: string): XmlNode = result = newXmlNode(xnText) result.fText = text -proc newVerbatimText*(text: string): XmlNode {.since: (1, 3).} = +proc newVerbatimText*(text: sink string): XmlNode {.since: (1, 3).} = ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`. ## **Since**: Version 1.3. result = newXmlNode(xnVerbatimText) result.fText = text -proc newComment*(comment: string): XmlNode = +proc newComment*(comment: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnComment`` with the text `comment`. runnableExamples: var c = newComment("my comment") @@ -115,7 +115,7 @@ proc newComment*(comment: string): XmlNode = result = newXmlNode(xnComment) result.fText = comment -proc newCData*(cdata: string): XmlNode = +proc newCData*(cdata: sink string): XmlNode = ## Creates a new ``XmlNode`` of kind ``xnCData`` with the text `cdata`. runnableExamples: var d = newCData("my cdata") @@ -135,7 +135,7 @@ proc newEntity*(entity: string): XmlNode = result = newXmlNode(xnEntity) result.fText = entity -proc newXmlTree*(tag: string, children: openArray[XmlNode], +proc newXmlTree*(tag: sink string, children: openArray[XmlNode], attributes: XmlAttributes = nil): XmlNode = ## Creates a new XML tree with `tag`, `children` and `attributes`. ## @@ -151,7 +151,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], h.add newEntity("some entity") let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes let k = newXmlTree("treeTag", [g, h], att) - + doAssert $k == """ some text &some entity; @@ -163,7 +163,7 @@ proc newXmlTree*(tag: string, children: openArray[XmlNode], for i in 0..children.len-1: result.s[i] = children[i] result.fAttr = attributes -proc text*(n: XmlNode): string {.inline.} = +proc text*(n: XmlNode): lent string {.inline.} = ## Gets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -181,7 +181,7 @@ proc text*(n: XmlNode): string {.inline.} = assert n.k in {xnText, xnComment, xnCData, xnEntity} result = n.fText -proc `text=`*(n: XmlNode, text: string){.inline.} = +proc `text=`*(n: XmlNode, text: sink string) {.inline.} = ## Sets the associated text with the node `n`. ## ## `n` can be a CDATA, Text, comment, or entity node. @@ -199,7 +199,7 @@ proc `text=`*(n: XmlNode, text: string){.inline.} = assert n.k in {xnText, xnComment, xnCData, xnEntity} n.fText = text -proc tag*(n: XmlNode): string {.inline.} = +proc tag*(n: XmlNode): lent string {.inline.} = ## Gets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -220,7 +220,7 @@ proc tag*(n: XmlNode): string {.inline.} = assert n.k == xnElement result = n.fTag -proc `tag=`*(n: XmlNode, tag: string) {.inline.} = +proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = ## Sets the tag name of `n`. ## ## `n` has to be an ``xnElement`` node. @@ -389,13 +389,13 @@ proc clear*(n: var XmlNode) = var g = newElement("myTag") g.add newText("some text") g.add newComment("this is comment") - + var h = newElement("secondTag") h.add newEntity("some entity") - + let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes var k = newXmlTree("treeTag", [g, h], att) - + doAssert $k == """ some text &some entity; @@ -447,7 +447,7 @@ proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, let att = {"key1": "first value", "key2": "second value"}.toXmlAttributes var j = newElement("myTag") j.attrs = att - + doAssert $j == """""" newStringTable(keyValuePairs) From 7f077a76fea389ce4f55e18857c499fadb3958fa Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 18 May 2021 06:10:19 -0700 Subject: [PATCH 0006/1654] jsonutils: add customization for toJson via `ToJsonOptions`; generalize symbolName; add symbolRank (#18029) * jsonutils: add customization for toJson via `ToJsonOptions` * add enumutils.symbolRank * lookup table implementation for HoleyEnum * cleanup * changelog * fixup * Update lib/std/jsonutils.nim Co-authored-by: Andreas Rumpf --- changelog.md | 5 ++- lib/std/enumutils.nim | 65 +++++++++++++++++++++++++++++++++++-- lib/std/jsonutils.nim | 43 +++++++++++++++++------- tests/stdlib/tjsonutils.nim | 13 ++++++++ 4 files changed, 112 insertions(+), 14 deletions(-) diff --git a/changelog.md b/changelog.md index 369b11917..97e749337 100644 --- a/changelog.md +++ b/changelog.md @@ -118,9 +118,11 @@ - `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64` instead of raising (as in 1.4) or giving wrong results (as in 1.2). +- `jsonutils` now handles `cstring` (including as Table key), and `set`. + - added `jsonutils.jsonTo` overload with `opt = Joptions()` param. -- `jsonutils` now handles `cstring` (including as Table key), and `set`. +- `jsonutils.toJson` now supports customization via `ToJsonOptions`. - Added an overload for the `collect` macro that inferes the container type based on the syntax of the last expression. Works with std seqs, tables and sets. @@ -138,6 +140,7 @@ - Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum. Added `items` for enums with holes. Added `symbolName` to return the enum symbol name ignoring the human readable name. + Added `symbolRank` to return the index in which an enum member is listed in an enum. - Added `typetraits.HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 6195ae07d..09cf24f51 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -86,8 +86,67 @@ iterator items*[T: HoleyEnum](E: typedesc[T]): T = assert B[float].toSeq == [B[float].b0, B[float].b1] for a in enumFullRange(E): yield a -func symbolName*[T: OrdinalEnum](a: T): string = +func span(T: typedesc[HoleyEnum]): int = + (T.high.ord - T.low.ord) + 1 + +const invalidSlot = uint8.high + +proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = + const n = span(T) + var ret: array[n, uint8] + var i = 0 + assert n <= invalidSlot.int + for ai in mitems(ret): ai = invalidSlot + for ai in items(T): + ret[ai.ord - T.low.ord] = uint8(i) + inc(i) + return ret + +func symbolRankImpl[T](a: T): int {.inline.} = + const n = T.span + const thres = 255 # must be <= `invalidSlot`, but this should be tuned. + when n <= thres: + const lookup = genLookup(T) + let lookup2 {.global.} = lookup # xxx improve pending https://github.com/timotheecour/Nim/issues/553 + #[ + This could be optimized using a hash adapted to `T` (possible since it's known at CT) + to get better key distribution before indexing into the lookup table table. + ]# + {.noSideEffect.}: # because it's immutable + let ret = lookup2[ord(a) - T.low.ord] + if ret != invalidSlot: return ret.int + else: + var i = 0 + # we could also generate a case statement as optimization + for ai in items(T): + if ai == a: return i + inc(i) + raise newException(IndexDefect, $ord(a) & " invalid for " & $T) + +template symbolRank*[T: enum](a: T): int = + ## Returns the index in which `a` is listed in `T`. + ## + ## The cost for a `HoleyEnum` is implementation defined, currently optimized + ## for small enums, otherwise is `O(T.enumLen)`. + runnableExamples: + type + A = enum a0 = -3, a1 = 10, a2, a3 = (20, "f3Alt") # HoleyEnum + B = enum b0, b1, b2 # OrdinalEnum + C = enum c0 = 10, c1, c2 # OrdinalEnum + assert a2.symbolRank == 2 + assert b2.symbolRank == 2 + assert c2.symbolRank == 2 + assert c2.ord == 12 + assert a2.ord == 11 + var invalid = 7.A + doAssertRaises(IndexDefect): discard invalid.symbolRank + when T is Ordinal: ord(a) - T.low.ord.static + else: symbolRankImpl(a) + +func symbolName*[T: enum](a: T): string = ## Returns the symbol name of an enum. + ## + ## This uses `symbolRank`. runnableExamples: type B = enum b0 = (10, "kb0") @@ -97,5 +156,7 @@ func symbolName*[T: OrdinalEnum](a: T): string = assert b.symbolName == "b0" assert $b == "kb0" static: assert B.high.symbolName == "b2" + type C = enum c0 = -3, c1 = 4, c2 = 20 # HoleyEnum + assert c1.symbolName == "c1" const names = enumNames(T) - names[a.ord - T.low.ord] + names[a.symbolRank] diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 1f49f60ed..1e222e3a2 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -31,9 +31,11 @@ add a way to customize serialization, for e.g.: ]# import macros +from enumutils import symbolName +from typetraits import OrdinalEnum type - Joptions* = object + Joptions* = object # xxx rename FromJsonOptions ## Options controlling the behavior of `fromJson`. allowExtraKeys*: bool ## If `true` Nim's object to which the JSON is parsed is not required to @@ -42,6 +44,17 @@ type ## If `true` Nim's object to which JSON is parsed is allowed to have ## fields without corresponding JSON keys. # in future work: a key rename could be added + EnumMode* = enum + joptEnumOrd + joptEnumSymbol + joptEnumString + ToJsonOptions* = object + enumMode*: EnumMode + # xxx charMode + +proc initToJsonOptions*(): ToJsonOptions = + ## initializes `ToJsonOptions` with sane options. + ToJsonOptions(enumMode: joptEnumOrd) proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} @@ -261,33 +274,41 @@ proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = ## reverse of `toJson` fromJson(result, b, opt) -proc toJson*[T](a: T): JsonNode = +proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to ## customize serialization, see strtabs.toJsonHook for an example. when compiles(toJsonHook(a)): result = toJsonHook(a) elif T is object | tuple: when T is object or isNamedTuple(T): result = newJObject() - for k, v in a.fieldPairs: result[k] = toJson(v) + for k, v in a.fieldPairs: result[k] = toJson(v, opt) else: result = newJArray() - for v in a.fields: result.add toJson(v) + for v in a.fields: result.add toJson(v, opt) elif T is ref | ptr: if system.`==`(a, nil): result = newJNull() - else: result = toJson(a[]) + else: result = toJson(a[], opt) elif T is array | seq | set: result = newJArray() - for ai in a: result.add toJson(ai) - elif T is pointer: result = toJson(cast[int](a)) + for ai in a: result.add toJson(ai, opt) + elif T is pointer: result = toJson(cast[int](a), opt) # edge case: `a == nil` could've also led to `newJNull()`, but this results # in simpler code for `toJson` and `fromJson`. - elif T is distinct: result = toJson(a.distinctBase) + elif T is distinct: result = toJson(a.distinctBase, opt) elif T is bool: result = %(a) elif T is SomeInteger: result = %a - elif T is Ordinal: result = %(a.ord) elif T is enum: - when defined(nimLegacyJsonutilsHoleyEnum): result = %a - else: result = %(a.ord) + case opt.enumMode + of joptEnumOrd: + when T is Ordinal or not defined(nimLegacyJsonutilsHoleyEnum): %(a.ord) + else: toJson($a, opt) + of joptEnumSymbol: + when T is OrdinalEnum: + toJson(symbolName(a), opt) + else: + toJson($a, opt) + of joptEnumString: toJson($a, opt) + elif T is Ordinal: result = %(a.ord) elif T is cstring: (if a == nil: result = newJNull() else: result = % $a) else: result = %a diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index c826a79b0..a55b0ca1d 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -35,6 +35,13 @@ type Foo = ref object proc `==`(a, b: Foo): bool = a.id == b.id +type MyEnum = enum me0, me1 = "me1Alt", me2, me3, me4 + +proc `$`(a: MyEnum): string = + # putting this here pending https://github.com/nim-lang/Nim/issues/13747 + if a == me2: "me2Modif" + else: system.`$`(a) + template fn() = block: # toJson, jsonTo type Foo = distinct float @@ -83,6 +90,12 @@ template fn() = doAssert b2.ord == 1 # explains the `1` testRoundtrip(a): """[1,2,3]""" + block: # ToJsonOptions + let a = (me1, me2) + doAssert $a.toJson() == "[1,2]" + doAssert $a.toJson(ToJsonOptions(enumMode: joptEnumSymbol)) == """["me1","me2"]""" + doAssert $a.toJson(ToJsonOptions(enumMode: joptEnumString)) == """["me1Alt","me2Modif"]""" + block: # set type Foo = enum f1, f2, f3, f4, f5 type Goo = enum g1 = 10, g2 = 15, g3 = 17, g4 From 31143c68fc7438406df39207b30594e775b4f97e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 18 May 2021 10:48:13 -0700 Subject: [PATCH 0007/1654] disable pkg/fidget refs https://github.com/treeform/fidget/issues/155 (#18043) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 1a61d3bbf..b84155eee 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -61,7 +61,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget" +pkg "fidget", allowFailure = true # pending https://github.com/treeform/fidget/issues/155 pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" From 53935b8b27643215b677cc152f563a7fb7cb84fb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 18 May 2021 21:45:37 +0200 Subject: [PATCH 0008/1654] ARC: fixes memory leaks with newSeq used in a loop [backport:1.4] (#18040) * ARC: fixes memory leaks with newSeq used in a loop [backport:1.4] * Update tests/arc/tnewseq_legacy.nim --- compiler/ccgexprs.nim | 7 ++++++- lib/system/seqs_v2.nim | 4 ++++ tests/arc/tnewseq_legacy.nim | 13 +++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/arc/tnewseq_legacy.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index b5ad24cc1..7cb403a94 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2319,7 +2319,12 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = gcUsage(p.config, e) else: genNewFinalize(p, e) - of mNewSeq: genNewSeq(p, e) + of mNewSeq: + if optSeqDestructors in p.config.globalOptions: + e[1] = makeAddr(e[1], p.module.idgen) + genCall(p, e, d) + else: + genNewSeq(p, e) of mNewSeqOfCap: genNewSeqOfCap(p, e, d) of mSizeOf: let t = e[1].typ.skipTypes({tyTypeDesc}) diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index b7f24ecd5..375eef340 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -124,3 +124,7 @@ proc setLen[T](s: var seq[T], newlen: Natural) = if xu.p == nil or xu.p.cap < newlen: xu.p = cast[typeof(xu.p)](prepareSeqAdd(oldLen, xu.p, newlen - oldLen, sizeof(T), alignof(T))) xu.len = newlen + +proc newSeq[T](s: var seq[T], len: Natural) = + shrink(s, 0) + setLen(s, len) diff --git a/tests/arc/tnewseq_legacy.nim b/tests/arc/tnewseq_legacy.nim new file mode 100644 index 000000000..4730d2c2b --- /dev/null +++ b/tests/arc/tnewseq_legacy.nim @@ -0,0 +1,13 @@ +discard """ + output: "(allocCount: 201, deallocCount: 201)" + cmd: "nim c --gc:orc -d:nimAllocStats $file" +""" + +proc main(prefix: string) = + var c: seq[string] + for i in 0..<100: + newSeq(c, 100) + c[i] = prefix & $i + +main("abc") +echo getAllocStats() From 558644725ddcb72ae4e0b34ad5cfb1d38e2c5385 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 19 May 2021 16:00:14 +0200 Subject: [PATCH 0009/1654] fixes #17943 (#18045) * fixes #17943 * IC: test 'nim check --ic:on' for the full Nim compiler * Update testament/testament.nim Co-authored-by: Clyybber Co-authored-by: Clyybber --- compiler/ic/ic.nim | 3 +++ testament/categories.nim | 33 +++++++++++++++++++-------------- testament/testament.nim | 4 ++-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 6530cb6c2..bd3be27f0 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -507,6 +507,9 @@ proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var Pac of nkStmtList, nkStmtListExpr: for it in n: toPackedNodeIgnoreProcDefs(it, encoder, m) + of nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, + nkFromStmt, nkIncludeStmt: + discard "nothing to do" else: toPackedNode(n, m.topLevel, encoder, m) diff --git a/testament/categories.nim b/testament/categories.nim index 1f0bd6acf..28ede1ba0 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -502,7 +502,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; isNavigatorTest: bool) = const - tooltests = ["compiler/nim.nim", "tools/nimgrep.nim"] + tooltests = ["compiler/nim.nim"] writeOnly = " --incremental:writeonly " readOnly = " --incremental:readonly " incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " @@ -518,6 +518,24 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; test.spec.targets = {getTestSpecTarget()} testSpecWithNimcache(r, test, nimcache) + template checkTest() = + var test = makeRawTest(file, options, cat) + test.spec.cmd = compilerPrefix & " check --hint[Conf]:off --warnings:off --ic:on $options " & file + testSpecWithNimcache(r, test, nimcache) + + if not isNavigatorTest: + for file in tooltests: + let nimcache = nimcacheDir(file, options, getTestSpecTarget()) + removeDir(nimcache) + + let oldPassed = r.passed + checkTest() + + if r.passed == oldPassed+1: + checkTest() + if r.passed == oldPassed+2: + checkTest() + const tempExt = "_temp.nim" for it in walkDirRec(testsDir): if isTestFile(it) and not it.endsWith(tempExt): @@ -532,19 +550,6 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) if r.passed != oldPassed+1: break - if not isNavigatorTest and false: - for file in tooltests: - let nimcache = nimcacheDir(file, options, getTestSpecTarget()) - removeDir(nimcache) - - let oldPassed = r.passed - test writeOnly - - if r.passed == oldPassed+1: - test readOnly - if r.passed == oldPassed+2: - test readOnly - # ---------------------------------------------------------------------------- const AdditionalCategories = ["debugger", "examples", "lib", "ic", "navigator"] diff --git a/testament/testament.nim b/testament/testament.nim index 076123882..a8dc0557c 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -500,7 +500,7 @@ proc equalModuloLastNewline(a, b: string): bool = proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, target: TTarget, nimcache: string, extraOptions = "") = test.startTime = epochTime() - template callNimCompilerImpl(): untyped = + template callNimCompilerImpl(): untyped = # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently # with other branches callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) @@ -649,9 +649,9 @@ proc makeRawTest(test, options: string, cat: Category): TTest {.used.} = result.name = test result.options = options result.spec = initSpec(addFileExt(test, ".nim")) - result.startTime = epochTime() result.spec.action = actionCompile result.spec.targets = {getTestSpecTarget()} + result.startTime = epochTime() # TODO: fix these files const disabledFilesDefault = @[ From 7052503ca8d360f11fabf476f7450dc3f794a602 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 19 May 2021 19:18:16 +0200 Subject: [PATCH 0010/1654] make strformat part of the prelude (#18046) --- changelog.md | 3 +++ lib/pure/prelude.nim | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 97e749337..f9c48e0bf 100644 --- a/changelog.md +++ b/changelog.md @@ -73,6 +73,9 @@ - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json. +- `strformat` is now part of `include std/prelude`. + + ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` diff --git a/lib/pure/prelude.nim b/lib/pure/prelude.nim index f1728b5f7..3c98d9d57 100644 --- a/lib/pure/prelude.nim +++ b/lib/pure/prelude.nim @@ -25,4 +25,4 @@ when defined(nimdoc) and isMainModule: # xxx `nim doc -b:js -d:nodejs --doccmd:-d:nodejs lib/pure/prelude.nim` fails for some reason # specific to `nim doc`, but the code otherwise works with nodejs. -import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt] +import std/[os, strutils, times, parseutils, hashes, tables, sets, sequtils, parseopt, strformat] From a1c82c39af812b54cd3dd1e472c9088457fb7dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 19 May 2021 19:19:11 +0200 Subject: [PATCH 0011/1654] asyncdispatch+stackTraceOverride: fix premature collection (#18039) [backport:1.2] Copying StackTraceEntry instances when nimStackTraceOverride is defined breaks the link between a cstring field that's supposed to point at another string field in the same object. Sometimes, the original object is garbage collected, that memory region reused for storing other strings, so when the StackTraceEntry copy tries to use its cstring pointer to construct a traceback message, it accesses unrelated strings. This only happens for async tracebacks and this patch prevents that by making sure we only use the string fields when nimStackTraceOverride is defined. Async tracebacks also beautified slightly by getting rid of an extra line that was supposed to be commented out, along with the corresponding debugging output. There's also a micro-optimisation to avoid concatenating two strings just to get their combined length. --- changelog.md | 1 + lib/pure/asyncfutures.nim | 46 ++++++++++++++++++++++---------- lib/system/exceptions.nim | 2 +- tests/async/tasync_traceback.nim | 4 +-- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/changelog.md b/changelog.md index f9c48e0bf..992cd89c0 100644 --- a/changelog.md +++ b/changelog.md @@ -316,6 +316,7 @@ - Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin). +- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place. ## Language changes diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 0178ee4d6..af3fbb3ec 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -278,19 +278,33 @@ proc `callback=`*[T](future: Future[T], ## If future has already completed then `cb` will be called immediately. future.callback = proc () = cb(future) +template getFilenameProcname(entry: StackTraceEntry): (string, string) = + when compiles(entry.filenameStr) and compiles(entry.procnameStr): + # We can't rely on "entry.filename" and "entry.procname" still being valid + # cstring pointers, because the "string.data" buffers they pointed to might + # be already garbage collected (this entry being a non-shallow copy, + # "entry.filename" no longer points to "entry.filenameStr.data", but to the + # buffer of the original object). + (entry.filenameStr, entry.procnameStr) + else: + ($entry.filename, $entry.procname) + proc getHint(entry: StackTraceEntry): string = ## We try to provide some hints about stack trace entries that the user ## may not be familiar with, in particular calls inside the stdlib. + + let (filename, procname) = getFilenameProcname(entry) + result = "" - if entry.procname == cstring"processPendingCallbacks": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + if procname == "processPendingCallbacks": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Executes pending callbacks" - elif entry.procname == cstring"poll": - if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0: + elif procname == "poll": + if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0: return "Processes asynchronous completion events" - if entry.procname.endsWith(NimAsyncContinueSuffix): - if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0: + if procname.endsWith(NimAsyncContinueSuffix): + if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0: return "Resumes an async procedure" proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = @@ -303,16 +317,20 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = # Find longest filename & line number combo for alignment purposes. var longestLeft = 0 for entry in entries: - if entry.procname.isNil: continue + let (filename, procname) = getFilenameProcname(entry) - let left = $entry.filename & $entry.line - if left.len > longestLeft: - longestLeft = left.len + if procname == "": continue + + let leftLen = filename.len + len($entry.line) + if leftLen > longestLeft: + longestLeft = leftLen var indent = 2 # Format the entries. for entry in entries: - if entry.procname.isNil: + let (filename, procname) = getFilenameProcname(entry) + + if procname == "": if entry.line == reraisedFromBegin: result.add(spaces(indent) & "#[\n") indent.inc(2) @@ -321,11 +339,11 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = result.add(spaces(indent) & "]#\n") continue - let left = "$#($#)" % [$entry.filename, $entry.line] + let left = "$#($#)" % [filename, $entry.line] result.add((spaces(indent) & "$#$# $#\n") % [ left, spaces(longestLeft - left.len + 2), - $entry.procname + procname ]) let hint = getHint(entry) if hint.len > 0: @@ -349,9 +367,9 @@ proc injectStacktrace[T](future: Future[T]) = newMsg.add($entries) newMsg.add("Exception message: " & exceptionMsg & "\n") - newMsg.add("Exception type:") # # For debugging purposes + # newMsg.add("Exception type:") # for entry in getStackTraceEntries(future.error): # newMsg.add "\n" & $entry future.error.msg = newMsg diff --git a/lib/system/exceptions.nim b/lib/system/exceptions.nim index b5f4fc325..5dcd77bd0 100644 --- a/lib/system/exceptions.nim +++ b/lib/system/exceptions.nim @@ -29,7 +29,7 @@ type programCounter*: uint ## Program counter - will be used to get the rest of the info, ## when `$` is called on this type. We can't use ## "cuintptr_t" in here. - procnameStr*, filenameStr*: string ## GC-ed objects holding the cstrings in "procname" and "filename" + procnameStr*, filenameStr*: string ## GC-ed alternatives to "procname" and "filename" Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \ ## Base exception class. diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index 9f787929b..cd16b2257 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -86,7 +86,7 @@ Async traceback: asyncfutures\.nim\(\d+?\)\s+?read \]# Exception message: b failure -Exception type: + bar failure Async traceback: @@ -114,7 +114,7 @@ Async traceback: asyncfutures\.nim\(\d+?\)\s+?read \]# Exception message: bar failure -Exception type: + """ # TODO: is asyncmacro good enough location for fooIter traceback/debugging? just put the callsite info for all? From df429fa28772e077faa30dd6e3a701abf48c7669 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 20 May 2021 11:52:46 +0200 Subject: [PATCH 0012/1654] config system: special case -d:release and -d:danger [backport:1.4] (#18051) --- changelog.md | 4 ++++ compiler/commands.nim | 19 +++++++++++++++---- compiler/main.nim | 3 ++- config/nim.cfg | 2 ++ tests/newconfig/tfoo.nim | 4 ++-- tests/newconfig/tfoo.nims | 2 ++ 6 files changed, 27 insertions(+), 7 deletions(-) diff --git a/changelog.md b/changelog.md index 992cd89c0..4c30d67a7 100644 --- a/changelog.md +++ b/changelog.md @@ -75,6 +75,10 @@ - `strformat` is now part of `include std/prelude`. +- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. + The downside is that these defines now have custom logic that doesn't apply for + other defines. + ## Standard library additions and changes - Added support for parenthesized expressions in `strformat` diff --git a/compiler/commands.nim b/compiler/commands.nim index b9c8e9eb3..605e2930c 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -482,6 +482,19 @@ proc setCommandEarly*(conf: ConfigRef, command: string) = else: conf.foreignPackageNotes = foreignPackageNotesDefault +proc specialDefine(conf: ConfigRef, key: string) = + # Keep this syncronized with the default config/nim.cfg! + if cmpIgnoreStyle(key, "nimQuirky") == 0: + conf.exc = excQuirky + elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0: + conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize} + conf.globalOptions.excl {optExcessiveStackTrace, optCDebug} + conf.options.incl optOptimizeSpeed + if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0: + conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, + optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} + conf.globalOptions.excl {optCDebug} + proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = var @@ -548,12 +561,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectArg(conf, switch, arg, pass, info) if {':', '='} in arg: splitSwitch(conf, arg, key, val, pass, info) - if cmpIgnoreStyle(key, "nimQuirky") == 0: - conf.exc = excQuirky + specialDefine(conf, key) defineSymbol(conf.symbols, key, val) else: - if cmpIgnoreStyle(arg, "nimQuirky") == 0: - conf.exc = excQuirky + specialDefine(conf, arg) defineSymbol(conf.symbols, arg) of "undef", "u": expectArg(conf, switch, arg, pass, info) diff --git a/compiler/main.nim b/compiler/main.nim index e2b5f4b67..71e549ab8 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -417,7 +417,8 @@ proc mainCommand*(graph: ModuleGraph) = "project", project, "output", output, ]) - rawMessage(conf, hintBuildMode, build) + if conf.cmd in cmdBackends: + rawMessage(conf, hintBuildMode, build) when PrintRopeCacheStats: echo "rope cache stats: " diff --git a/config/nim.cfg b/config/nim.cfg index d3ab75825..3b964d124 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -50,6 +50,7 @@ path="$lib/pure" @end nimblepath="$home/.nimble/pkgs/" +# Syncronize with compiler/commands.specialDefine @if danger or quick: obj_checks:off field_checks:off @@ -63,6 +64,7 @@ nimblepath="$home/.nimble/pkgs/" line_dir:off @end +# Syncronize with compiler/commands.specialDefine @if release or danger: stacktrace:off excessiveStackTrace:off diff --git a/tests/newconfig/tfoo.nim b/tests/newconfig/tfoo.nim index 6654202c5..0c6ded470 100644 --- a/tests/newconfig/tfoo.nim +++ b/tests/newconfig/tfoo.nim @@ -1,6 +1,6 @@ discard """ cmd: "nim default --hint:cc:off --hint:cc $file" - output: '''hello world! 0.5''' + output: '''hello world! 0.5 true''' nimout: '''[NimScript] exec: gcc -v''' """ @@ -10,4 +10,4 @@ when not defined(definedefine): import math, mfriends discard gen[int]() -echo "hello world! ", ln 2.0 +echo "hello world! ", ln 2.0, " ", compileOption("opt", "speed") diff --git a/tests/newconfig/tfoo.nims b/tests/newconfig/tfoo.nims index 6f0048afb..0cb950227 100644 --- a/tests/newconfig/tfoo.nims +++ b/tests/newconfig/tfoo.nims @@ -3,6 +3,8 @@ mode = ScriptMode.Whatif exec "gcc -v" +--define:release + --forceBuild --path: "../friends" From 6a5973882bb0c4134a0e454ea4ae69dc54815f27 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 20 May 2021 10:48:46 -0300 Subject: [PATCH 0013/1654] jsgen improve spacing (#18048) * Fix inconsistent spacings in generated JS of jsgen --- compiler/jsgen.nim | 60 ++++++++++++++++++++++---------------------- compiler/jstypes.nim | 4 +-- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 53ce45e96..3b183b9d6 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -322,7 +322,7 @@ proc isSimpleExpr(p: PProc; n: PNode): bool = proc getTemp(p: PProc, defineInLocals: bool = true): Rope = inc(p.unique) - result = "Tmp$1" % [rope(p.unique)] + result = "Temporary$1" % [rope(p.unique)] if defineInLocals: p.locals.add(p.indentLine("var $1;$n" % [result])) @@ -489,12 +489,12 @@ proc maybeMakeTempAssignable(p: PProc, n: PNode; x: TCompRes): tuple[a, tmp: Rop if optBoundsCheck in p.options: useMagic(p, "chckIndx") if first == 0: # save a couple chars - index.res = "chckIndx($1, 0, ($2).length-1)" % [index.res, tmp1] + index.res = "chckIndx($1, 0, ($2).length - 1)" % [index.res, tmp1] else: - index.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [ + index.res = "chckIndx($1, $2, ($3).length + ($2) - 1) - ($2)" % [ index.res, rope(first), tmp1] elif first != 0: - index.res = "($1)-($2)" % [index.res, rope(first)] + index.res = "($1) - ($2)" % [index.res, rope(first)] else: discard # index.res = index.res let (n1, tmp2) = maybeMakeTemp(p, n[1], index) @@ -607,12 +607,12 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = if n[1].typ.size <= 4: applyFormat("($1 << $2)", "($1 << $2)") else: - applyFormat("($1 * Math.pow(2,$2))", "($1 * Math.pow(2,$2))") + applyFormat("($1 * Math.pow(2, $2))", "($1 * Math.pow(2, $2))") of mAshrI: if n[1].typ.size <= 4: applyFormat("($1 >> $2)", "($1 >> $2)") else: - applyFormat("Math.floor($1 / Math.pow(2,$2))", "Math.floor($1 / Math.pow(2,$2))") + applyFormat("Math.floor($1 / Math.pow(2, $2))", "Math.floor($1 / Math.pow(2, $2))") of mBitandI: applyFormat("($1 & $2)", "($1 & $2)") of mBitorI: applyFormat("($1 | $2)", "($1 | $2)") of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)") @@ -656,8 +656,8 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mUnaryMinusF64: applyFormat("-($1)", "-($1)") of mCharToStr: applyFormat("nimCharToStr($1)", "nimCharToStr($1)") of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)") - of mIntToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")") - of mInt64ToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")") + of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")") + of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")") of mFloatToStr: useMagic(p, "nimFloatToString") applyFormat "cstrToNimstr(nimFloatToString($1))" @@ -704,7 +704,7 @@ proc hasFrameInfo(p: PProc): bool = ((p.prc == nil) or not (sfPure in p.prc.flags)) proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope = - ropes.`%`("// line $2 \"$1\"$n", + ropes.`%`("/* line $2 \"$1\" */$n", [rope(toFullPath(config, info)), rope(line)]) proc genLineDir(p: PProc, n: PNode) = @@ -725,9 +725,9 @@ proc genWhileStmt(p: PProc, n: PNode) = p.blocks[^1].id = -p.unique p.blocks[^1].isLoop = true let labl = p.unique.rope - lineF(p, "L$1: while (true) {$n", [labl]) + lineF(p, "Label$1: while (true) {$n", [labl]) p.nested: gen(p, n[0], cond) - lineF(p, "if (!$1) break L$2;$n", + lineF(p, "if (!$1) break Label$2;$n", [cond.res, labl]) p.nested: genStmt(p, n[1]) lineF(p, "}$n", [labl]) @@ -750,8 +750,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = # try { # stmts; # --excHandler; - # } catch (EXC) { - # var prevJSError = lastJSError; lastJSError = EXC; + # } catch (EXCEPTION) { + # var prevJSError = lastJSError; lastJSError = EXCEPTION; # framePtr = tmpFramePtr; # --excHandler; # if (e.typ && e.typ == NTI433 || e.typ == NTI2321) { @@ -785,8 +785,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = moveInto(p, a, r) var generalCatchBranchExists = false if catchBranchesExist: - p.body.addf("--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" & - " lastJSError = EXC;$n --excHandler;$n", []) + p.body.addf("--excHandler;$n} catch (EXCEPTION) {$n var prevJSError = lastJSError;$n" & + " lastJSError = EXCEPTION;$n --excHandler;$n", []) line(p, "framePtr = $1;$n" % [tmpFramePtr]) while i < n.len and n[i].kind == nkExceptBranch: if n[i].len == 1: @@ -928,12 +928,12 @@ proc genBlock(p: PProc, n: PNode, r: var TCompRes) = sym.loc.k = locOther sym.position = idx+1 let labl = p.unique - lineF(p, "L$1: do {$n", [labl.rope]) + lineF(p, "Label$1: do {$n", [labl.rope]) setLen(p.blocks, idx + 1) p.blocks[idx].id = - p.unique # negative because it isn't used yet gen(p, n[1], r) setLen(p.blocks, idx) - lineF(p, "} while(false);$n", [labl.rope]) + lineF(p, "} while (false);$n", [labl.rope]) proc genBreakStmt(p: PProc, n: PNode) = var idx: int @@ -951,7 +951,7 @@ proc genBreakStmt(p: PProc, n: PNode) = if idx < 0 or not p.blocks[idx].isLoop: internalError(p.config, n.info, "no loop to break") p.blocks[idx].id = abs(p.blocks[idx].id) # label is used - lineF(p, "break L$1;$n", [rope(p.blocks[idx].id)]) + lineF(p, "break Label$1;$n", [rope(p.blocks[idx].id)]) proc genAsmOrEmitStmt(p: PProc, n: PNode) = genLineDir(p, n) @@ -1248,12 +1248,12 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) = if optBoundsCheck in p.options: useMagic(p, "chckIndx") if first == 0: # save a couple chars - r.res = "chckIndx($1, 0, ($2).length-1)" % [b.res, tmp] + r.res = "chckIndx($1, 0, ($2).length - 1)" % [b.res, tmp] else: - r.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [ + r.res = "chckIndx($1, $2, ($3).length + ($2) - 1) - ($2)" % [ b.res, rope(first), tmp] elif first != 0: - r.res = "($1)-($2)" % [b.res, rope(first)] + r.res = "($1) - ($2)" % [b.res, rope(first)] else: r.res = b.res r.kind = resExpr @@ -1876,13 +1876,13 @@ proc genNewSeq(p: PProc, n: PNode) = gen(p, n[1], x) gen(p, n[2], y) let t = skipTypes(n[1].typ, abstractVar)[0] - lineF(p, "$1 = new Array($2); for (var i=0;i<$2;++i) {$1[i]=$3;}", [ + lineF(p, "$1 = new Array($2); for (var i = 0 ; i < $2 ; ++i) { $1[i] = $3; }", [ x.rdLoc, y.rdLoc, createVar(p, t, false)]) proc genOrd(p: PProc, n: PNode, r: var TCompRes) = case skipTypes(n[1].typ, abstractVar + abstractRange).kind of tyEnum, tyInt..tyUInt64, tyChar: gen(p, n[1], r) - of tyBool: unaryExpr(p, n, r, "", "($1 ? 1:0)") + of tyBool: unaryExpr(p, n, r, "", "($1 ? 1 : 0)") else: internalError(p.config, n.info, "genOrd") proc genConStrStr(p: PProc, n: PNode, r: var TCompRes) = @@ -2122,7 +2122,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = let t = skipTypes(n[1].typ, abstractVar)[0] let (a, tmp) = maybeMakeTemp(p, n[1], x) let (b, tmp2) = maybeMakeTemp(p, n[2], y) - r.res = """if ($1.length < $2) { for (var i=$4.length;i<$5;++i) $4.push($3); } + r.res = """if ($1.length < $2) { for (var i = $4.length ; i < $5 ; ++i) $4.push($3); } else { $4.length = $5; }""" % [a, b, createVar(p, t, false), tmp, tmp2] r.kind = resExpr of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)") @@ -2159,7 +2159,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = gen(p, n[1], x) gen(p, n[2], y) gen(p, n[3], z) - r.res = "($1.slice($2, $3+1))" % [x.rdLoc, y.rdLoc, z.rdLoc] + r.res = "($1.slice($2, $3 + 1))" % [x.rdLoc, y.rdLoc, z.rdLoc] r.kind = resExpr of mMove: genMove(p, n, r) @@ -2281,7 +2281,7 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) = r.res = "(!!($1))" % [r.res] r.kind = resExpr elif toInt: - r.res = "(($1)|0)" % [r.res] + r.res = "(($1) | 0)" % [r.res] else: # TODO: What types must we handle here? discard @@ -2337,7 +2337,7 @@ proc genReturnStmt(p: PProc, n: PNode) = proc frameCreate(p: PProc; procname, filename: Rope): Rope = const frameFmt = - "var F={procname:$1,prev:framePtr,filename:$2,line:0};$n" + "var F = {procname: $1, prev: framePtr, filename: $2, line: 0};$n" result = p.indentLine(frameFmt % [procname, filename]) result.add p.indentLine(ropes.`%`("framePtr = F;$n", [])) @@ -2431,10 +2431,10 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = # references will end up calling the reloaded code. var thunkName = name name = name & "IMLP" - result.add("function $#() { return $#.apply(this, arguments); }$n" % + result.add("\Lfunction $#() { return $#.apply(this, arguments); }$n" % [thunkName, name]) - def = "function $#($#) {$n$#$#$#$#$#" % + def = "\Lfunction $#($#) {$n$#$#$#$#$#" % [ name, header, optionalLine(p.globals), @@ -2482,7 +2482,7 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) = elif fromUint: if src.size == 4 and dest.size == 4: # XXX prevent multi evaluations - r.res = "($1|0)" % [r.res] + r.res = "($1 | 0)" % [r.res] else: let trimmer = unsignedTrimmer(dest.size) let minuend = case dest.size diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 81f03dc12..5b684b60c 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -135,14 +135,14 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = prepend(p.g.typeInfo, s) of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet: var s = - "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % + "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" % [result, rope(ord(t.kind))] prepend(p.g.typeInfo, s) p.g.typeInfo.addf("$1.base = $2;$n", [result, genTypeInfo(p, t.lastSon)]) of tyArray: var s = - "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % + "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" % [result, rope(ord(t.kind))] prepend(p.g.typeInfo, s) p.g.typeInfo.addf("$1.base = $2;$n", From 9f7e2e30573a377770fa630f12a47bac09751282 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 21 May 2021 07:54:20 +0300 Subject: [PATCH 0014/1654] docs: make inline markup more compatible with Markdown (#18053) fixes https://github.com/timotheecour/Nim/issues/739 --- doc/regexprs.txt | 2 +- lib/impure/db_sqlite.nim | 2 +- lib/packages/docutils/rst.nim | 147 ++++++++++++++++++++++------------ lib/posix/posix_utils.nim | 2 +- tests/stdlib/trst.nim | 65 +++++++++++++-- tests/stdlib/trstgen.nim | 12 +-- 6 files changed, 168 insertions(+), 62 deletions(-) diff --git a/doc/regexprs.txt b/doc/regexprs.txt index b7370d858..9ec08b810 100644 --- a/doc/regexprs.txt +++ b/doc/regexprs.txt @@ -146,7 +146,7 @@ character meaning After ``\x``, from zero to two hexadecimal digits are read (letters can be in upper or lower case). In UTF-8 mode, any number of hexadecimal digits may appear between ``\x{`` and ``}``, but the value of the character code must be -less than 2**31 (that is, the maximum hexadecimal value is 7FFFFFFF). If +less than 2^31 (that is, the maximum hexadecimal value is 7FFFFFFF). If characters other than hexadecimal digits appear between ``\x{`` and ``}``, or if there is no terminating ``}``, this form of escape is not recognized. Instead, the initial ``\x`` will be interpreted as a basic hexadecimal escape, diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 832407960..7bd807a12 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -152,7 +152,7 @@ ## Instead, a `seq[string]` is returned for each row. ## ## The reasoning is as follows: -## 1. it's close to what many DBs offer natively (char**) +## 1. it's close to what many DBs offer natively (`char**`:c:) ## 2. it hides the number of types that the DB supports ## (int? int64? decimal up to 10 places? geo coords?) ## 3. it's convenient when all you do is to forward the data to somewhere else (echo, log, put the data into a new query) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index da04e9b54..dae692fb7 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -130,6 +130,32 @@ ## .. warning:: Using Nim-specific features can cause other RST implementations ## to fail on your document. ## +## Idiosyncrasies +## -------------- +## +## Currently we do **not** aim at 100% Markdown or RST compatibility in inline +## markup recognition rules because that would provide very little user value. +## This parser has 2 modes for inline markup: +## +## 1) Markdown-like mode which is enabled by `roPreferMarkdown` option +## (turned **on** by default). +## +## .. Note:: RST features like directives are still turned **on** +## +## 2) Compatibility mode which is RST rules. +## +## .. Note:: in both modes the parser interpretes text between single +## backticks (code) identically: +## backslash does not escape; the only exception: ``\`` folowed by ` +## does escape so that we can always input a single backtick ` in +## inline code. However that makes impossible to input code with +## ``\`` at the end in *single* backticks, one must use *double* +## backticks:: +## +## `\` -- WRONG +## ``\`` -- GOOD +## So single backticks can always be input: `\`` will turn to ` code +## ## Limitations ## ----------- ## @@ -994,8 +1020,22 @@ proc expect(p: var RstParser, tok: string) = if currentTok(p).symbol == tok: inc p.idx else: rstMessage(p, meExpected, tok) -proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool = +proc inlineMarkdownEnd(p: RstParser): bool = + result = prevTok(p).kind notin {tkIndent, tkWhite} + ## (For a special case of ` we don't allow spaces surrounding it + ## unlike original Markdown because this behavior confusing/useless) + +proc inlineRstEnd(p: RstParser): bool = # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules + # Rule 2: + result = prevTok(p).kind notin {tkIndent, tkWhite} + if not result: return + # Rule 7: + result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or + nextTok(p).symbol[0] in + {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'} + +proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool = if exact: result = currentTok(p).symbol == markup else: @@ -1004,55 +1044,58 @@ proc isInlineMarkupEnd(p: RstParser, markup: string, exact: bool): bool = # check that escaping may have splitted `` to 2 tokens ` and ` result = currentTok(p).symbol == "`" and prevTok(p).symbol == "`" if not result: return - # Rule 2: - result = prevTok(p).kind notin {tkIndent, tkWhite} - if not result: return - # Rule 7: - result = nextTok(p).kind in {tkIndent, tkWhite, tkEof} or - (roPreferMarkdown in p.s.options and - markup in ["``", "`"] and - nextTok(p).kind in {tkIndent, tkWhite, tkWord, tkEof}) or - nextTok(p).symbol[0] in - {'\'', '\"', ')', ']', '}', '>', '-', '/', '\\', ':', '.', ',', ';', '!', '?', '_'} - if not result: return - # Rule 4: - if p.idx > 0: - # see bug #17260; for now `\` must be written ``\``, likewise with sequences - # ending in an un-escaped `\`; `\\` is legal but not `\\\` for example; - # for this reason we can't use `["``", "`"]` here. - if markup != "``" and prevTok(p).symbol == "\\": - result = false + # surroundings check + if markup in ["_", "__"]: + result = inlineRstEnd(p) + else: + if roPreferMarkdown in p.s.options: result = inlineMarkdownEnd(p) + else: result = inlineRstEnd(p) -proc isInlineMarkupStart(p: RstParser, markup: string): bool = - # rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules - var d: char - if markup != "_`": - result = currentTok(p).symbol == markup - else: # _` is a 2 token case - result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`" +proc rstRuleSurround(p: RstParser): bool = + result = true + # Rules 4 & 5: + if p.idx > 0: + var d: char + var c = prevTok(p).symbol[0] + case c + of '\'', '\"': d = c + of '(': d = ')' + of '[': d = ']' + of '{': d = '}' + of '<': d = '>' + else: d = '\0' + if d != '\0': result = nextTok(p).symbol[0] != d + +proc inlineMarkdownStart(p: RstParser): bool = + result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof} if not result: return - # Rule 6: + # this rst rule is really nice, let us use it in Markdown mode too. + result = rstRuleSurround(p) + +proc inlineRstStart(p: RstParser): bool = + ## rst rules: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#inline-markup-recognition-rules + # Rule 6 result = p.idx == 0 or prevTok(p).kind in {tkIndent, tkWhite} or - (markup in ["``", "`"] and prevTok(p).kind in {tkIndent, tkWhite, tkWord}) or prevTok(p).symbol[0] in {'\'', '\"', '(', '[', '{', '<', '-', '/', ':', '_'} if not result: return # Rule 1: result = nextTok(p).kind notin {tkIndent, tkWhite, tkEof} if not result: return - # Rules 4 & 5: - if p.idx > 0: - if prevTok(p).symbol == "\\": - result = false - else: - var c = prevTok(p).symbol[0] - case c - of '\'', '\"': d = c - of '(': d = ')' - of '[': d = ']' - of '{': d = '}' - of '<': d = '>' - else: d = '\0' - if d != '\0': result = nextTok(p).symbol[0] != d + result = rstRuleSurround(p) + +proc isInlineMarkupStart(p: RstParser, markup: string): bool = + if markup != "_`": + result = currentTok(p).symbol == markup + else: # _` is a 2 token case + result = currentTok(p).symbol == "_" and nextTok(p).symbol == "`" + if not result: return + # surroundings check + if markup in ["_", "__", "[", "|"]: + # Note: we require space/punctuation even before [markdown link](...) + result = inlineRstStart(p) + else: + if roPreferMarkdown in p.s.options: result = inlineMarkdownStart(p) + else: result = inlineRstStart(p) proc match(p: RstParser, start: int, expr: string): bool = # regular expressions are: @@ -1263,10 +1306,7 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) = proc parseBackslash(p: var RstParser, father: PRstNode) = assert(currentTok(p).kind == tkPunct) - if currentTok(p).symbol == "\\\\": - father.add newLeaf("\\") - inc p.idx - elif currentTok(p).symbol == "\\": + if currentTok(p).symbol == "\\": # XXX: Unicode? inc p.idx if currentTok(p).kind != tkWhite: father.add(newLeaf(p)) @@ -1297,11 +1337,20 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, break else: if postfix == "`": - if prevTok(p).symbol == "\\" and currentTok(p).symbol == "`": - father.sons[^1] = newLeaf(p) # instead, we should use lookahead + if currentTok(p).symbol == "\\": + if nextTok(p).symbol == "\\": + father.add newLeaf("\\") + father.add newLeaf("\\") + inc p.idx, 2 + elif nextTok(p).symbol == "`": # escape ` + father.add newLeaf("`") + inc p.idx, 2 + else: + father.add newLeaf("\\") + inc p.idx else: father.add(newLeaf(p)) - inc p.idx + inc p.idx else: if interpretBackslash: parseBackslash(p, father) diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim index aeec73a45..c2d5aab56 100644 --- a/lib/posix/posix_utils.nim +++ b/lib/posix/posix_utils.nim @@ -7,7 +7,7 @@ # ## A set of helpers for the POSIX module. -## Raw interfaces are in the other posix*.nim files. +## Raw interfaces are in the other ``posix*.nim`` files. # Where possible, contribute OS-independent procs in `os `_ instead. diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 71f5a858b..ec34edc91 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -23,7 +23,7 @@ import std/private/miscdollars import os proc toAst(input: string, - rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile}, + rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile}, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -36,10 +36,11 @@ proc toAst(input: string, toLocation(message, filename, line, col + ColRstOffset) message.add " $1: $2" % [$mc, a] if mc == mcError: - doAssert error != nil, "unexpected RST error '" & message & "'" + if error == nil: + raise newException(EParseError, "[unexpected error] " & message) error[] = message # we check only first error because subsequent ones may be meaningless - raise newException(EParseError, message) + raise newException(EParseError, "") else: doAssert warnings != nil, "unexpected RST warning '" & message & "'" warnings[].add message @@ -54,8 +55,9 @@ proc toAst(input: string, var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit, dummyHasToc, rstOptions, myFindFile, testMsgHandler) result = renderRstToStr(rst) - except EParseError: - discard + except EParseError as e: + if e.msg != "": + result = e.msg suite "RST parsing": test "option list has priority over definition list": @@ -326,6 +328,28 @@ suite "RST escaping": """) suite "RST inline markup": + test "* and ** surrounded by spaces are not inline markup": + check("a * b * c ** d ** e".toAst == dedent""" + rnInner + rnLeaf 'a' + rnLeaf ' ' + rnLeaf '*' + rnLeaf ' ' + rnLeaf 'b' + rnLeaf ' ' + rnLeaf '*' + rnLeaf ' ' + rnLeaf 'c' + rnLeaf ' ' + rnLeaf '**' + rnLeaf ' ' + rnLeaf 'd' + rnLeaf ' ' + rnLeaf '**' + rnLeaf ' ' + rnLeaf 'e' + """) + test "end-string has repeating symbols": check("*emphasis content****".toAst == dedent""" rnEmphasis @@ -420,6 +444,37 @@ suite "RST inline markup": rnLeaf 'proc `+`' """) + check("""`\\`""".toAst == + dedent""" + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf '\\' + """) + + test "Markdown-style code/backtick": + # no whitespace is required before ` + check("`try`...`except`".toAst == + dedent""" + rnInner + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'try' + rnLeaf '...' + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'except' + """) + + test """inline literals can contain \ anywhere""": check("""``\``""".toAst == dedent""" rnInlineLiteral diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 667fec780..864728686 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -10,7 +10,7 @@ import unittest, strutils, strtabs import std/private/miscdollars proc toHtml(input: string, - rstOptions: RstParseOptions = {roSupportMarkdown, roNimFile}, + rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile}, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -23,18 +23,20 @@ proc toHtml(input: string, toLocation(message, filename, line, col + ColRstOffset) message.add " $1: $2" % [$mc, a] if mc == mcError: - doAssert error != nil, "unexpected RST error '" & message & "'" + if error == nil: + raise newException(EParseError, "[unexpected error] " & message) error[] = message # we check only first error because subsequent ones may be meaningless - raise newException(EParseError, message) + raise newException(EParseError, "") else: doAssert warnings != nil, "unexpected RST warning '" & message & "'" warnings[].add message try: result = rstToHtml(input, rstOptions, defaultConfig(), msgHandler=testMsgHandler) - except EParseError: - discard + except EParseError as e: + if e.msg != "": + result = e.msg # inline code tags (for parsing originated from highlite.nim) proc id(str: string): string = """""" & str & "" From e12597589f2a8cdaabe09e8b2b92ff0fbe3727ef Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sat, 22 May 2021 08:12:30 -0300 Subject: [PATCH 0015/1654] Error message minor (#18021) * Update compiler/semtypes.nim Co-authored-by: Timothee Cour Co-authored-by: flywind Co-authored-by: Andreas Rumpf --- compiler/lineinfos.nim | 2 +- compiler/semtypes.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 47cddb888..bf81aed02 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -110,7 +110,7 @@ const warnFieldXNotSupported: "field '$1' not supported", warnRstStyle: "RST style: $1", warnCommentXIgnored: "comment '$1' ignored", - warnTypelessParam: "'$1' has no type. Typeless parameters are deprecated; only allowed for 'template'", + warnTypelessParam: "", # deadcode warnUseBase: "use {.base.} for base methods; baseless methods are deprecated", warnWriteToForeignHeap: "write to foreign heap", warnUnsafeCode: "unsafe code: '$1'", diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 97e37f8b7..84556ca1a 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1297,7 +1297,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, let param = strTableGet(c.signatures, arg.name) if param != nil: typ = param.typ else: - localError(c.config, a.info, "typeless parameters are obsolete") + localError(c.config, a.info, "parameter '$1' requires a type" % param.name.s) typ = errorType(c) let lifted = liftParamType(c, kind, genericParams, typ, arg.name.s, arg.info) From 1636c05d138a7a52c5b6cb3cafe6ab84f59f086b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 23 May 2021 01:02:04 -0700 Subject: [PATCH 0016/1654] close #5540 generic object with generic field evaluated too early (#18062) --- tests/misc/t5540.nim | 45 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/misc/t5540.nim diff --git a/tests/misc/t5540.nim b/tests/misc/t5540.nim new file mode 100644 index 000000000..6a19e70e1 --- /dev/null +++ b/tests/misc/t5540.nim @@ -0,0 +1,45 @@ +# bug #5540; works in 1.2.0 +# fails in 1.0 (Error: cannot generate VM code for) +# fails in 0.18.0 (Error: type mismatch: got ) + +block: + type + Fruit = object + Yellow = object + a: int + template getColor(x: typedesc[Fruit]): typedesc = Yellow + type + Banana[T] = object + b: T + a: getColor(Fruit) + Apple[T] = object + a: T + b: getColor(T) + block: + var x: Banana[int] + doAssert x.b == 0 + doAssert x.a is Yellow + block: + var x: Apple[Fruit] + doAssert x.b is Yellow + +block: + type + Fruit = object + Yellow = object + a: int + + template getColor(x: typedesc[Fruit]): typedesc = Yellow + + type + Banana[T] = object + b: T + a: getColor(Fruit) + + Apple[T] = object + a: T + b: getColor(T) + + var x: Banana[int] + x.b = 13 + x.a.a = 17 From 61630c6aee56acf688c48c00f5a6711a59e79616 Mon Sep 17 00:00:00 2001 From: Euan Date: Sun, 23 May 2021 17:23:20 +0100 Subject: [PATCH 0017/1654] Use gmake for NetBSD as well as FreeBSD/OpenBSD (#18064) --- ci/funs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/funs.sh b/ci/funs.sh index cad005246..0e4f0eb3b 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -55,6 +55,8 @@ _nimBuildCsourcesIfNeeded(){ makeX=gmake elif [ "$unamestr" = 'OpenBSD' ]; then makeX=gmake + elif [ "$unamestr" = 'NetBSD' ]; then + makeX=gmake else makeX=make fi From 1421a3bf265e45cea7c26dcaf3be148ea02db5f0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 23 May 2021 12:28:26 -0700 Subject: [PATCH 0018/1654] pkg/pixie: use latest (#18069) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index b84155eee..975dd63ad 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -118,7 +118,7 @@ pkg "optionsutils" pkg "ormin", "nim c -o:orminn ormin.nim" pkg "parsetoml" pkg "patty" -pkg "pixie", useHead = false +pkg "pixie" pkg "plotly", "nim c examples/all.nim" pkg "pnm" pkg "polypbren" From d217888e5679aff063668930bd00892c4f5cb2b3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 23 May 2021 14:06:14 -0700 Subject: [PATCH 0019/1654] close #18009 parseJson JInt vs JFloat; preserve -0.0 as JFloat to distinguish from 0.0 (#18067) --- lib/pure/json.nim | 2 +- tests/stdlib/tjsonutils.nim | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 3a9fa5898..d5bbb86b8 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -938,7 +938,7 @@ when defined(js): of "[object Array]": return JArray of "[object Object]": return JObject of "[object Number]": - if isInteger(x): + if isInteger(x) and 1.0 / cast[float](x) != -Inf: # preserve -0.0 as float if isSafeInteger(x): return JInt else: diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index a55b0ca1d..62486b896 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -4,7 +4,8 @@ discard """ import std/jsonutils import std/json -from std/math import isNaN +from std/math import isNaN, signbit +from stdtest/testutils import whenRuntimeJs proc testRoundtrip[T](t: T, expected: string) = # checks that `T => json => T2 => json2` is such that json2 = json @@ -123,6 +124,20 @@ template fn() = testRoundtripVal((Inf, -Inf, 0.0, -0.0, 1.0)): """["inf","-inf",0.0,-0.0,1.0]""" doAssert ($NaN.toJson).parseJson.jsonTo(float).isNaN + block: # bug #18009; unfixable unless we change parseJson (which would have overhead), + # but at least we can guarantee that the distinction between 0.0 and -0.0 is preserved. + let a = (0, 0.0, -0.0, 0.5, 1, 1.0) + testRoundtripVal(a): "[0,0.0,-0.0,0.5,1,1.0]" + let a2 = $($a.toJson).parseJson + whenRuntimeJs: + doAssert a2 == "[0,0,-0.0,0.5,1,1]" + do: + doAssert a2 == "[0,0.0,-0.0,0.5,1,1.0]" + let b = a2.parseJson.jsonTo(type(a)) + doAssert not b[1].signbit + doAssert b[2].signbit + doAssert not b[3].signbit + block: # case object type Foo = object x0: float From 478f717377c4cd60cfce112b8b21d58031b118b4 Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Mon, 24 May 2021 20:56:31 -0700 Subject: [PATCH 0020/1654] Move async example to asynchronous version of proc (#18078) * improve runnableExamples in std/httpclient * Add synchronous example. * Update lib/pure/httpclient.nim Co-authored-by: Timothee Cour --- lib/pure/httpclient.nim | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 86708dc40..0dbf8a045 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -574,15 +574,11 @@ proc newHttpClient*(userAgent = defUserAgent, maxRedirects = 5, ## ## `headers` specifies the HTTP Headers. runnableExamples: - import std/[asyncdispatch, httpclient, strutils] + import std/strutils - proc asyncProc(): Future[string] {.async.} = - var client = newAsyncHttpClient() - return await client.getContent("http://example.com") - - let exampleHtml = waitFor asyncProc() + let exampleHtml = newHttpClient().getContent("http://example.com") assert "Example Domain" in exampleHtml - assert not ("Pizza" in exampleHtml) + assert "Pizza" notin exampleHtml new result result.headers = headers @@ -616,6 +612,17 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, maxRedirects = 5, ## connections. ## ## `headers` specifies the HTTP Headers. + runnableExamples: + import std/[asyncdispatch, strutils] + + proc asyncProc(): Future[string] {.async.} = + let client = newAsyncHttpClient() + result = await client.getContent("http://example.com") + + let exampleHtml = waitFor asyncProc() + assert "Example Domain" in exampleHtml + assert "Pizza" notin exampleHtml + new result result.headers = headers result.userAgent = userAgent From b59dc3b255d778cefdbef2c0c3ff55d43892c11c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 00:41:50 -0700 Subject: [PATCH 0021/1654] remove some custom logic in testament around flags, testExec (#18090) * remove some custom logic in testament around flags, testExec * remove testExec, custom logic around flags from testament * fixup --- testament/categories.nim | 23 ----------------------- testament/testament.nim | 16 ---------------- tests/flags/tgenscript.nim | 6 ------ tests/misc/trunner.nim | 33 ++++++++++++++++++++++++++++++--- 4 files changed, 30 insertions(+), 48 deletions(-) delete mode 100644 tests/flags/tgenscript.nim diff --git a/testament/categories.nim b/testament/categories.nim index 28ede1ba0..510e90c53 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -23,7 +23,6 @@ const "debugger", "dll", "examples", - "flags", "gc", "io", "js", @@ -48,26 +47,6 @@ proc isTestFile*(file: string): bool = let (_, name, ext) = splitFile(file) result = ext == ".nim" and name.startsWith("t") -# --------------------- flags tests ------------------------------------------- - -proc flagTests(r: var TResults, cat: Category, options: string) = - # --genscript - const filename = testsDir/"flags"/"tgenscript" - const genopts = " --genscript" - let nimcache = nimcacheDir(filename, genopts, targetC) - testSpec r, makeTest(filename, genopts, cat) - - when defined(windows): - testExec r, makeTest(filename, " cmd /c cd " & nimcache & - " && compile_tgenscript.bat", cat) - - elif defined(posix): - testExec r, makeTest(filename, " sh -c \"cd " & nimcache & - " && sh compile_tgenscript.sh\"", cat) - - # Run - testExec r, makeTest(filename, " " & nimcache / "tgenscript", cat) - # --------------------- DLL generation tests ---------------------------------- proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = @@ -705,8 +684,6 @@ proc processCategory(r: var TResults, cat: Category, jsTests(r, cat, options) of "dll": dllTests(r, cat, options) - of "flags": - flagTests(r, cat, options) of "gc": gcTests(r, cat, options) of "longgc": diff --git a/testament/testament.nim b/testament/testament.nim index a8dc0557c..2edefdb01 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -621,22 +621,6 @@ proc testC(r: var TResults, test: TTest, action: TTestAction) = if exitCode != 0: given.err = reExitcodesDiffer if given.err == reSuccess: inc(r.passed) -proc testExec(r: var TResults, test: TTest) = - # runs executable or script, just goes by exit code - if not checkDisabled(r, test): return - - inc(r.total) - let (outp, errC) = execCmdEx(test.options.strip()) - var given: TSpec - if errC == 0: - given.err = reSuccess - else: - given.err = reExitcodesDiffer - given.msg = outp - - if given.err == reSuccess: inc(r.passed) - r.addResult(test, targetC, "", given.msg, given.err) - proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat result.name = test diff --git a/tests/flags/tgenscript.nim b/tests/flags/tgenscript.nim deleted file mode 100644 index d58395a40..000000000 --- a/tests/flags/tgenscript.nim +++ /dev/null @@ -1,6 +0,0 @@ -discard """ - targets: "c" - action: compile -""" - -echo "--genscript" diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 1b679d92c..3297b3a24 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -33,16 +33,23 @@ const proc runNimCmd(file, options = "", rtarg = ""): auto = let fileabs = testsDir / file.unixToNativePath - doAssert fileabs.fileExists, fileabs + # doAssert fileabs.fileExists, fileabs # disabled because this allows passing `nim r --eval:code fakefile` let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}" result = execCmdEx(cmd) - when false: echo result[0] & "\n" & result[1] # for debugging + when false: # for debugging + echo cmd + echo result[0] & "\n" & $result[1] proc runNimCmdChk(file, options = "", rtarg = ""): string = let (ret, status) = runNimCmd(file, options, rtarg = rtarg) doAssert status == 0, $(file, options) & "\n" & ret ret +proc genShellCmd(filename: string): string = + let filename = filename.quoteShell + when defined(windows): "cmd /c " & filename # or "cmd /c " ? + else: "sh " & filename + when defined(nimTrunnerFfi): block: # mevalffi when defined(openbsd): @@ -71,7 +78,9 @@ foo:0.03:asdf:103:105 ret=[s1:foobar s2:foobar age:25 pi:3.14] """, output -else: # don't run twice the same test +elif not defined(nimTestsTrunnerDebugging): + # don't run twice the same test with `nimTrunnerFfi` + # use `-d:nimTestsTrunnerDebugging` for debugging convenience when you want to just run 1 test import std/strutils import std/json template check2(msg) = doAssert msg in output, output @@ -330,3 +339,21 @@ running: v2 doAssert "D20210428T161003" in j["defined_symbols"].to(seq[string]) doAssert j["version"].to(string) == NimVersion doAssert j["nimExe"].to(string) == getCurrentCompilerExe() + + block: # genscript + const nimcache2 = buildDir / "D20210524T212851" + removeDir(nimcache2) + let input = "tgenscript_fakefile" # no need for a real file, --eval is good enough + let output = runNimCmdChk(input, fmt"""--genscript --nimcache:{nimcache2.quoteShell} --eval:"echo(12345)" """) + doAssert output.len == 0, output + let ext = when defined(windows): ".bat" else: ".sh" + let filename = fmt"compile_{input}{ext}" # synchronize with `generateScript` + doAssert fileExists(nimcache2/filename), nimcache2/filename + let (outp, status) = execCmdEx(genShellCmd(filename), options = {poStdErrToStdOut}, workingDir = nimcache2) + doAssert status == 0, outp + let (outp2, status2) = execCmdEx(nimcache2 / input, options = {poStdErrToStdOut}) + doAssert outp2 == "12345\n", outp2 + doAssert status2 == 0 + +else: + discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled From c495628255330a559e3398663cc8872cca18b04d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 00:43:30 -0700 Subject: [PATCH 0022/1654] refactor common code in CI pipelines (#18035) * refactor CI with nimInternalInstallDepsWindows * refactor CI with nimInternalBuildKochAndRunCI * fixup --- .builds/freebsd.yml | 6 +----- .builds/openbsd_0.yml | 6 +----- .builds/openbsd_1.yml | 6 +----- .github/workflows/ci_docs.yml | 8 +++----- .github/workflows/ci_packages.yml | 22 ++++++---------------- azure-pipelines.yml | 16 ++++------------ ci/funs.sh | 17 +++++++++++++++++ tools/ci_generate.nim | 6 +----- 8 files changed, 34 insertions(+), 53 deletions(-) diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 8f336771a..15d09dda0 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -22,15 +22,11 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | set -e cd Nim - if ! ./koch runCI; then - nim r tools/ci_testresults.nim - exit 1 - fi + . ci/funs.sh && nimInternalBuildKochAndRunCI triggers: - action: email condition: failure diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml index c19cd63c6..c3b2fd43e 100644 --- a/.builds/openbsd_0.yml +++ b/.builds/openbsd_0.yml @@ -22,15 +22,11 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | set -e cd Nim - if ! ./koch runCI; then - nim r tools/ci_testresults.nim - exit 1 - fi + . ci/funs.sh && nimInternalBuildKochAndRunCI triggers: - action: email condition: failure diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml index 06f5f2ae5..e98ec4640 100644 --- a/.builds/openbsd_1.yml +++ b/.builds/openbsd_1.yml @@ -22,15 +22,11 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | set -e cd Nim - if ! ./koch runCI; then - nim r tools/ci_testresults.nim - exit 1 - fi + . ci/funs.sh && nimInternalBuildKochAndRunCI triggers: - action: email condition: failure diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index 364c05051..ddef94fbe 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -59,11 +59,9 @@ jobs: if: runner.os == 'Windows' shell: bash run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin + set -e + . ci/funs.sh + nimInternalInstallDepsWindows echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - name: 'Add build binaries to PATH' diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index d033aba10..aeb29e147 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -40,12 +40,10 @@ jobs: if: runner.os == 'Windows' shell: bash run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin - echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" + set -e + . ci/funs.sh + nimInternalInstallDepsWindows + echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - name: 'Add build binaries to PATH' shell: bash @@ -59,14 +57,6 @@ jobs: shell: bash run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' - - name: 'Build koch' + - name: 'koch, Run CI' shell: bash - run: nim c koch - - name: 'Run CI' - shell: bash - run: ./koch runCI - - - name: 'Show failed tests' - if: failure() - shell: bash - run: nim c -r tools/ci_testresults.nim + run: . ci/funs.sh && nimInternalBuildKochAndRunCI diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 112fb0a62..f0c3b2d0a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -133,11 +133,7 @@ jobs: - bash: | set -e . ci/funs.sh - echo_run mkdir dist - echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - echo_run 7z x dist/mingw64.7z -odist - echo_run 7z x dist/dlls.zip -obin + nimInternalInstallDepsWindows echo_run echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin' displayName: 'Install dependencies (Windows)' @@ -163,13 +159,9 @@ jobs: # condition: and(succeeded(), eq(variables['skipci'], 'false')) # displayName: 'Restore built csourcesAny' - - bash: nim c koch + - bash: . ci/funs.sh && nimInternalBuildKochAndRunCI + # would could also show on failure: echo '##vso[task.complete result=Failed]' condition: and(succeeded(), eq(variables['skipci'], 'false')) - displayName: 'Build koch' - - # set result to omit the "bash exited with error code '1'" message - - bash: ./koch runCI || echo '##vso[task.complete result=Failed]' - condition: and(succeeded(), eq(variables['skipci'], 'false')) - displayName: 'Run CI' + displayName: 'koch, Run CI' env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/ci/funs.sh b/ci/funs.sh index 0e4f0eb3b..9772535be 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -31,6 +31,23 @@ nimIsCiSkip(){ fi } +nimInternalInstallDepsWindows(){ + echo_run mkdir dist + echo_run curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z + echo_run curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip + echo_run 7z x dist/mingw64.7z -odist + echo_run 7z x dist/dlls.zip -obin +} + +nimInternalBuildKochAndRunCI(){ + echo_run nim c koch + if ! echo_run ./koch runCI; then + echo_run echo "runCI failed" + echo_run nim r -tools/ci_testresults.nim + return 1 + fi +} + nimDefineVars(){ . config/build_config.txt nim_csources=bin/nim_csources_$nim_csourcesHash diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index f08cd5409..52b84f0d8 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -29,15 +29,11 @@ tasks: set -e cd Nim . ci/funs.sh && nimBuildCsourcesIfNeeded - $nim_csources c --skipUserCfg --skipParentCfg koch echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | set -e cd Nim - if ! ./koch runCI; then - nim r tools/ci_testresults.nim - exit 1 - fi + . ci/funs.sh && nimInternalBuildKochAndRunCI triggers: - action: email condition: failure From 8df55d0ad735277683631bb8e5966ee75545de2b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 00:44:00 -0700 Subject: [PATCH 0023/1654] close #3482 no more cgen error with typed templates (#18094) --- tests/misc/t3482.nim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/misc/t3482.nim diff --git a/tests/misc/t3482.nim b/tests/misc/t3482.nim new file mode 100644 index 000000000..33b3b8f40 --- /dev/null +++ b/tests/misc/t3482.nim @@ -0,0 +1,15 @@ +discard """ + action: reject + nimout: "t3482.nim(13, 8) Error: undeclared identifier: 'output'" +""" +# bug #3482 (correct behavior since 1.4.0, cgen error in 1.2.0) +template foo*(body: typed) = + if true: + body + +proc test = + foo: + var output = "" + echo output.len + +test() From 0b2bbcaa23f91b06bc06dd2cf35d4d26a6d24aa8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 00:46:53 -0700 Subject: [PATCH 0024/1654] fix #18077 testament now parses `cmd` properly (#18086) --- testament/categories.nim | 3 ++- testament/testament.nim | 53 ++++++++++++++++++++++++++++------------ tests/misc/t18077.nim | 21 ++++++++++++++++ 3 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 tests/misc/t18077.nim diff --git a/testament/categories.nim b/testament/categories.nim index 510e90c53..8cf90666c 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -87,8 +87,9 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var hcri = makeTest("tests/dll/nimhcr_integration.nim", options & " --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) - hcri.args = prepareTestArgs(hcri.spec.getCmd, hcri.name, + let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) + hcri.testArgs = cmd.parseCmdLine testSpec r, hcri proc dllTests(r: var TResults, cat: Category, options: string) = diff --git a/testament/testament.nim b/testament/testament.nim index 2edefdb01..1eb925122 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -85,7 +85,7 @@ type name: string cat: Category options: string - args: seq[string] + testArgs: seq[string] spec: TSpec startTime: float debugInfo: string @@ -154,24 +154,23 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = let hashInput = options & $target result = "nimcache" / (filename & '_' & hashInput.getMD5) -proc prepareTestArgs(cmdTemplate, filename, options, nimcache: string, - target: TTarget, extraOptions = ""): seq[string] = +proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, + target: TTarget, extraOptions = ""): string = var options = target.defaultOptions & ' ' & options - # improve pending https://github.com/nim-lang/Nim/issues/14343 - if nimcache.len > 0: options.add ' ' & ("--nimCache:" & nimcache).quoteShell + if nimcache.len > 0: options.add(" --nimCache:$#" % nimcache.quoteShell) options.add ' ' & extraOptions - result = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], + # we avoid using `parseCmdLine` which is buggy, refs bug #14343 + result = cmdTemplate % ["target", targetToCmd[target], "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir(), "nim", compilerPrefix]) + "filedir", filename.getFileDir(), "nim", compilerPrefix] proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, target: TTarget, extraOptions = ""): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache, target, + result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, extraOptions) - result.cmd = quoteShellCommand(c) - verboseCmd(c.quoteShellCommand) - var p = startProcess(command = c[0], args = c[1 .. ^1], - options = {poStdErrToStdOut, poUsePath}) + verboseCmd(result.cmd) + var p = startProcess(command = result.cmd, + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) let outp = p.outputStream var foundSuccessMsg = false var foundErrorMsg = false @@ -224,9 +223,33 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, proc callCCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = - let c = prepareTestArgs(cmdTemplate, filename, options, nimcache = "", target) + let cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache = "", target) + doAssert false + #[ + this code hasn't been run in a while, and should be removed which simplifies code + there are better ways to do this anyways (e.g. running c code from a nim file) + + the only place where this is called is: + `testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun` + which isn't run unless you call: + XDG_CONFIG_HOME= nim r --lib:lib --stacktrace:on testament/testament.nim r longgc + + and this fails since at least nim 1.0 with: + testament/testament.nim(851) testament + testament/testament.nim(822) main + testament/categories.nim(713) processCategory + testament/categories.nim(189) longGCTests + testament/testament.nim(644) makeTest + testament/specs.nim(251) parseSpec + testament/specs.nim(184) extractSpec + lib/system/io.nim(861) readFile + Error: unhandled exception: cannot open: tests/realtimeGC/cmain.nim [IOError] + + Also, `c[5 .. ^1]` is too magical. + ]# + let c = cmd.parseCmdLine var p = startProcess(command = "gcc", args = c[5 .. ^1], - options = {poStdErrToStdOut, poUsePath}) + options = {poStdErrToStdOut, poUsePath, poEvalCommand}) let outp = p.outputStream var x = newStringOfCap(120) result.nimout = "" @@ -525,7 +548,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, reExeNotFound) else: var exeCmd: string - var args = test.args + var args = test.testArgs if isJsTarget: exeCmd = nodejs # see D20210217T215950 diff --git a/tests/misc/t18077.nim b/tests/misc/t18077.nim new file mode 100644 index 000000000..6cd05d575 --- /dev/null +++ b/tests/misc/t18077.nim @@ -0,0 +1,21 @@ +discard """ + cmd: '''nim doc -d:nimTestsT18077b:4 --doccmd:"-d:nimTestsT18077 -d:nimTestsT18077b:3 --hints:off" $file''' + action: compile +""" + +# bug #18077 + +const nimTestsT18077b {.intdefine.} = 1 + +static: + when defined(nimdoc): + doAssert nimTestsT18077b == 4 + doAssert not defined(nimTestsT18077) + else: + doAssert defined(nimTestsT18077) + doAssert nimTestsT18077b == 3 + +runnableExamples: + const nimTestsT18077b {.intdefine.} = 2 + doAssert nimTestsT18077b == 3 + doAssert defined(nimTestsT18077) From 3b1aabdcffafbbf37ad02f013f754f67dfe2461c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 00:51:48 -0700 Subject: [PATCH 0025/1654] change `--hint[X] => --hint:X` in nim repo (avoids shell quoting issues) (#18085) --- compiler/nim.cfg | 4 ++-- compiler/nimfix/nimfix.nim.cfg | 2 +- config/nim.cfg | 4 ++-- doc/advopt.txt | 10 +++++----- doc/filters.rst | 2 +- doc/nimc.rst | 8 ++++---- drnim/nim.cfg | 2 +- nimsuggest/nimsuggest.nim.cfg | 2 +- testament/categories.nim | 4 ++-- tests/effects/toutparam.nim | 2 +- tests/osproc/treadlines.nim | 2 +- tests/pragmas/t8741.nim | 2 +- tests/range/tcompiletime_range_checks.nim | 2 +- tests/tools/tunused_imports.nim | 2 +- tools/nimweb.nim | 2 +- 15 files changed, 25 insertions(+), 25 deletions(-) diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 40e93d523..9ecd00b0b 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -1,6 +1,6 @@ # Special configuration file for the Nim project -hint[XDeclaredButNotUsed]:off +hint:XDeclaredButNotUsed:off define:booting define:nimcore @@ -22,5 +22,5 @@ define:useStdoutAsStdmsg #gc:markAndSweep @if nimHasWarningObservableStores: - warning[ObservableStores]: off + warning:ObservableStores: off @end diff --git a/compiler/nimfix/nimfix.nim.cfg b/compiler/nimfix/nimfix.nim.cfg index 0d9dbfa4b..27ebf10b9 100644 --- a/compiler/nimfix/nimfix.nim.cfg +++ b/compiler/nimfix/nimfix.nim.cfg @@ -1,7 +1,7 @@ # Special configuration file for the Nim project # gc:markAndSweep -hint[XDeclaredButNotUsed]:off +hint:XDeclaredButNotUsed:off path:"$projectPath/.." path:"$lib/packages/docutils" diff --git a/config/nim.cfg b/config/nim.cfg index 3b964d124..c0a844e03 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -13,8 +13,8 @@ cc = gcc # additional options always passed to the compiler: --parallel_build: "0" # 0 to auto-detect number of processors -hint[LineTooLong]=off -#hint[XDeclaredButNotUsed]=off +hint:LineTooLong:off +#hint:XDeclaredButNotUsed:off # Examples of how to setup a cross-compiler: diff --git a/doc/advopt.txt b/doc/advopt.txt index e50fb243b..ea3370880 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -47,12 +47,12 @@ Advanced options: an implementation defined set of suggestions. -w:on|off|list, --warnings:on|off|list turn all warnings on|off or list all available - --warning[X]:on|off turn specific warning X on|off + --warning:X:on|off turn specific warning X on|off. + `warning:X` means `warning:X:on`, as with similar flags. --hints:on|off|list turn all hints on|off or list all available - --hint[X]:on|off turn specific hint X on|off - --warningAsError[X]:on|off - turn specific warning X into an error on|off - --hintAsError[X]:on|off turn specific hint X into an error on|off + --hint:X:on|off turn specific hint X on|off. + --warningAsError:X:on|off turn specific warning X into an error on|off + --hintAsError:X:on|off turn specific hint X into an error on|off --styleCheck:off|hint|error produce hints or errors for Nim identifiers that do not adhere to Nim's official style guide diff --git a/doc/filters.rst b/doc/filters.rst index c6d436194..e237744cb 100644 --- a/doc/filters.rst +++ b/doc/filters.rst @@ -25,7 +25,7 @@ just like an ordinary procedure call with named or positional arguments. The available parameters depend on the invoked filter. Before version 0.12.0 of the language `#!` was used instead of `#?`. -**Hint:** With `--hint[codeBegin]:on`:option: or `--verbosity:2`:option: +**Hint:** With `--hint:codeBegin:on`:option: or `--verbosity:2`:option: (or higher) while compiling or `nim check`:cmd:, Nim lists the processed code after each filter application. diff --git a/doc/nimc.rst b/doc/nimc.rst index 47f7243cc..e467a33af 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -53,8 +53,8 @@ Advanced command-line switches are: List of warnings ---------------- -Each warning can be activated individually with `--warning[NAME]:on|off`:option: or -in a `push` pragma. +Each warning can be activated individually with `--warning:NAME:on|off`:option: or +in a `push` pragma with `{.warning[NAME]:on|off.}`. ========================== ============================================ Name Description @@ -77,8 +77,8 @@ User Some user-defined warning. List of hints ------------- -Each hint can be activated individually with `--hint[NAME]:on|off`:option: or in a -`push` pragma. +Each hint can be activated individually with `--hint:NAME:on|off`:option: or in a +`push` pragma with `{.hint[NAME]:on|off.}`. ========================== ============================================ Name Description diff --git a/drnim/nim.cfg b/drnim/nim.cfg index 58c1725d9..57dca974c 100644 --- a/drnim/nim.cfg +++ b/drnim/nim.cfg @@ -1,6 +1,6 @@ # Special configuration file for the Nim project -hint[XDeclaredButNotUsed]:off +hint:XDeclaredButNotUsed:off define:booting define:nimcore diff --git a/nimsuggest/nimsuggest.nim.cfg b/nimsuggest/nimsuggest.nim.cfg index 394449740..6e5ee8c71 100644 --- a/nimsuggest/nimsuggest.nim.cfg +++ b/nimsuggest/nimsuggest.nim.cfg @@ -2,7 +2,7 @@ gc:markAndSweep -hint[XDeclaredButNotUsed]:off +hint:XDeclaredButNotUsed:off path:"$lib/packages/docutils" diff --git a/testament/categories.nim b/testament/categories.nim index 8cf90666c..0a3595ff4 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -486,7 +486,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; writeOnly = " --incremental:writeonly " readOnly = " --incremental:readonly " incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " - navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint[Conf]:off --warnings:off " + navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " template test(x: untyped) = testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) @@ -500,7 +500,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; template checkTest() = var test = makeRawTest(file, options, cat) - test.spec.cmd = compilerPrefix & " check --hint[Conf]:off --warnings:off --ic:on $options " & file + test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file testSpecWithNimcache(r, test, nimcache) if not isNavigatorTest: diff --git a/tests/effects/toutparam.nim b/tests/effects/toutparam.nim index 00572aa6b..1126aa77e 100644 --- a/tests/effects/toutparam.nim +++ b/tests/effects/toutparam.nim @@ -1,5 +1,5 @@ discard """ - cmd: '''nim c --warningAsError[Uninit]:on --skipCfg --skipParentCfg $file''' + cmd: '''nim c --warningAsError:Uninit:on --skipCfg --skipParentCfg $file''' errormsg: "use explicit initialization of 'x' for clarity [Uninit]" line: 24 disabled: "true" diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim index bcde19d7f..200a7c299 100644 --- a/tests/osproc/treadlines.nim +++ b/tests/osproc/treadlines.nim @@ -13,7 +13,7 @@ var ps: seq[Process] # compile & run 2 progs in parallel const nim = getCurrentCompilerExe() for prog in ["a", "b"]: ps.add startProcess(nim, "", - ["r", "--hint[Conf]=off", "--hint[Processing]=off", prog], + ["r", "--hint:Conf:off", "--hint:Processing:off", prog], options = {poUsePath, poDaemon, poStdErrToStdOut}) for p in ps: diff --git a/tests/pragmas/t8741.nim b/tests/pragmas/t8741.nim index 61a449c01..221c732b0 100644 --- a/tests/pragmas/t8741.nim +++ b/tests/pragmas/t8741.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim check --hint[processing]:off $file" + cmd: "nim check --hint:processing:off $file" errormsg: "3 is not two" nimout: '''t8741.nim(13, 9) Error: cannot attach a custom pragma to 'a' t8741.nim(29, 15) template/generic instantiation of `onlyTwo` from here diff --git a/tests/range/tcompiletime_range_checks.nim b/tests/range/tcompiletime_range_checks.nim index 37095e0b7..29e2c48a8 100644 --- a/tests/range/tcompiletime_range_checks.nim +++ b/tests/range/tcompiletime_range_checks.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim check --hint[Processing]:off --hint[Conf]:off $file" + cmd: "nim check --hint:Processing:off --hint:Conf:off $file" errormsg: "18446744073709551615 can't be converted to int8" nimout: '''tcompiletime_range_checks.nim(36, 21) Error: 2147483648 can't be converted to int32 tcompiletime_range_checks.nim(37, 23) Error: -1 can't be converted to uint64 diff --git a/tests/tools/tunused_imports.nim b/tests/tools/tunused_imports.nim index bce906634..31d6cf7d7 100644 --- a/tests/tools/tunused_imports.nim +++ b/tests/tools/tunused_imports.nim @@ -1,5 +1,5 @@ discard """ - cmd: '''nim c --hint[Processing]:off $file''' + cmd: '''nim c --hint:Processing:off $file''' nimout: ''' tunused_imports.nim(11, 10) Warning: BEGIN [User] tunused_imports.nim(36, 10) Warning: END [User] diff --git a/tools/nimweb.nim b/tools/nimweb.nim index f71b5f3be..ccc80dfcf 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -53,7 +53,7 @@ proc initConfigData(c: var TConfigData) = c.webdoc = @[] c.pdf = @[] c.infile = "" - c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot " + c.nimArgs = "--hint:Conf:off --hint:Path:off --hint:Processing:off -d:boot " c.gitURL = "https://github.com/nim-lang/Nim" c.docHTMLOutput = "doc/html" c.webUploadOutput = "web/upload" From 1e0165186bb8539cfd8aca1a7af8d01dc278bd46 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 26 May 2021 11:49:01 -0700 Subject: [PATCH 0026/1654] testament: remove deadcode related to realtimeGC, testC, callCCompiler (#18087) --- testament/categories.nim | 22 +++--------- testament/testament.nim | 63 --------------------------------- tests/readme.md | 3 +- tests/realtimeGC/cmain.c | 69 ------------------------------------- tests/realtimeGC/readme.txt | 10 ------ tests/realtimeGC/tmain.nim | 10 +++++- 6 files changed, 15 insertions(+), 162 deletions(-) delete mode 100644 tests/realtimeGC/cmain.c delete mode 100644 tests/realtimeGC/readme.txt diff --git a/testament/categories.nim b/testament/categories.nim index 0a3595ff4..7f05797ff 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -28,7 +28,6 @@ const "js", "ic", "lib", - "longgc", "manyloc", "nimble-packages", "niminaction", @@ -39,7 +38,6 @@ const "coroutines", "osproc", "shouldfail", - "dir with space", "destructor" ] @@ -156,19 +154,6 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "cyclecollector" testWithoutBoehm "trace_globals" -proc longGCTests(r: var TResults, cat: Category, options: string) = - when defined(windows): - let cOptions = "-ldl -DWIN" - else: - let cOptions = "-ldl" - - var c = initResults() - # According to ioTests, this should compile the file - testSpec c, makeTest("tests/realtimeGC/shared", options, cat) - # ^- why is this not appended to r? Should this be discarded? - testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun - testSpec r, makeTest("tests/realtimeGC/nmain", options & "--threads: on", cat) - # ------------------------- threading tests ----------------------------------- proc threadTests(r: var TResults, cat: Category, options: string) = @@ -186,6 +171,11 @@ proc ioTests(r: var TResults, cat: Category, options: string) = # dummy compile result: var c = initResults() testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) + # ^- why is this not appended to r? Should this be discarded? + # EDIT: this should be replaced by something like in D20210524T180826, + # likewise in similar instances where `testSpec c` is used, or more generally + # when a test depends on another test, as it makes tests non-independent, + # creating complications for batching and megatest logic. testSpec r, makeTest("tests/system/tio", options, cat) # ------------------------- async tests --------------------------------------- @@ -687,8 +677,6 @@ proc processCategory(r: var TResults, cat: Category, dllTests(r, cat, options) of "gc": gcTests(r, cat, options) - of "longgc": - longGCTests(r, cat, options) of "debugger": debuggerTests(r, cat, options) of "manyloc": diff --git a/testament/testament.nim b/testament/testament.nim index 1eb925122..789c987c0 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -221,53 +221,6 @@ proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, result.msg = matches[0] trimUnitSep result.msg -proc callCCompiler(cmdTemplate, filename, options: string, - target: TTarget): TSpec = - let cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache = "", target) - doAssert false - #[ - this code hasn't been run in a while, and should be removed which simplifies code - there are better ways to do this anyways (e.g. running c code from a nim file) - - the only place where this is called is: - `testC r, makeTest("tests/realtimeGC/cmain", cOptions, cat), actionRun` - which isn't run unless you call: - XDG_CONFIG_HOME= nim r --lib:lib --stacktrace:on testament/testament.nim r longgc - - and this fails since at least nim 1.0 with: - testament/testament.nim(851) testament - testament/testament.nim(822) main - testament/categories.nim(713) processCategory - testament/categories.nim(189) longGCTests - testament/testament.nim(644) makeTest - testament/specs.nim(251) parseSpec - testament/specs.nim(184) extractSpec - lib/system/io.nim(861) readFile - Error: unhandled exception: cannot open: tests/realtimeGC/cmain.nim [IOError] - - Also, `c[5 .. ^1]` is too magical. - ]# - let c = cmd.parseCmdLine - var p = startProcess(command = "gcc", args = c[5 .. ^1], - options = {poStdErrToStdOut, poUsePath, poEvalCommand}) - let outp = p.outputStream - var x = newStringOfCap(120) - result.nimout = "" - result.msg = "" - result.file = "" - result.output = "" - result.line = -1 - while true: - if outp.readLine(x): - result.nimout.add(x & '\n') - elif not running(p): - break - close(p) - if p.peekExitCode == 0: - result.err = reSuccess - else: - result.err = reNimcCrash - proc initResults: TResults = result.total = 0 result.passed = 0 @@ -628,22 +581,6 @@ proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used var testClone = test testSpecHelper(r, testClone, test.spec, target, nimcache) -proc testC(r: var TResults, test: TTest, action: TTestAction) = - # runs C code. Doesn't support any specs, just goes by exit code. - if not checkDisabled(r, test): return - - let tname = test.name.addFileExt(".c") - inc(r.total) - maybeStyledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(getCmd(TSpec()), test.name & ".c", test.options, targetC) - if given.err != reSuccess: - r.addResult(test, targetC, "", given.msg, given.err) - elif action == actionRun: - let exeFile = changeFileExt(test.name, ExeExt) - var (_, exitCode) = execCmdEx(exeFile, options = {poStdErrToStdOut, poUsePath}) - if exitCode != 0: given.err = reExitcodesDiffer - if given.err == reSuccess: inc(r.passed) - proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat result.name = test diff --git a/tests/readme.md b/tests/readme.md index 69be92051..dca7ec152 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -55,5 +55,4 @@ testing, which is slower). The folder ``dll`` contains simple DLL tests. The folder ``realtimeGC`` contains a test for validating that the realtime GC -can run properly without linking against the nimrtl.dll/so. It includes a C -client and platform specific build files for manual compilation. +can run properly without linking against the nimrtl.dll/so. diff --git a/tests/realtimeGC/cmain.c b/tests/realtimeGC/cmain.c deleted file mode 100644 index 4bc5bd026..000000000 --- a/tests/realtimeGC/cmain.c +++ /dev/null @@ -1,69 +0,0 @@ -/* xxx consider removing, this seems redundant with tmain.nim */ - -#ifdef WIN -#include -#else -#include -#include /* for sleep(3) */ -#endif -#include -#include -#include -#include -#include - -#define RUNTIME (15*60) - - -typedef void (*pFunc)(void); - -int main(int argc, char* argv[]) -{ - int i; - void* hndl; - pFunc status; - pFunc count; - pFunc checkOccupiedMem; - -#ifdef WIN - hndl = (void*) LoadLibrary((char const*)"./tests/realtimeGC/shared.dll"); - status = (pFunc)GetProcAddress((HMODULE) hndl, (char const*)"status"); - count = (pFunc)GetProcAddress((HMODULE) hndl, (char const*)"count"); - checkOccupiedMem = (pFunc)GetProcAddress((HMODULE) hndl, (char const*)"checkOccupiedMem"); -#else /* OSX || NIX xxx: OSX assumes dylib*/ - hndl = (void*) dlopen((char const*)"./tests/realtimeGC/libshared.so", RTLD_LAZY); - status = (pFunc) dlsym(hndl, (char const*)"status"); - count = (pFunc) dlsym(hndl, (char const*)"count"); - checkOccupiedMem = (pFunc) dlsym(hndl, (char const*)"checkOccupiedMem"); -#endif - - assert(hndl); - assert(status); - assert(count); - assert(checkOccupiedMem); - - time_t startTime = time((time_t*)0); - time_t runTime = (time_t)(RUNTIME); - time_t accumTime = 0; - while (accumTime < runTime) { - for (i = 0; i < 10; i++) - count(); - /* printf("1. sleeping...\n"); */ - sleep(1); - for (i = 0; i < 10; i++) - status(); - /* printf("2. sleeping...\n"); */ - sleep(1); - checkOccupiedMem(); - accumTime = time((time_t*)0) - startTime; - /* printf("--- Minutes left to run: %d\n", (int)(runTime-accumTime)/60); */ - } - printf("Cleaning up the shared object pointer...\n"); -#ifdef WIN - FreeLibrary((HMODULE)hndl); -#else /* OSX || NIX */ - dlclose(hndl); -#endif - printf("Done\n"); - return 0; -} diff --git a/tests/realtimeGC/readme.txt b/tests/realtimeGC/readme.txt deleted file mode 100644 index c5837ee14..000000000 --- a/tests/realtimeGC/readme.txt +++ /dev/null @@ -1,10 +0,0 @@ -Test the realtime GC without linking nimrtl.dll/so. - -To build by hand and run the test for 35 minutes: - $ nim r --threads:on -d:runtimeSecs:2100 tests/realtimeGC/nmain.nim - -``` -xxx do we still need tests/realtimeGC/cmain.c? -if so, tests/realtimeGC/cmain.c needs to updated and factorized with nmain.nim to avoid duplication (even if it's a C file) -``` - $ gcc -o tests/realtimeGC/cmain tests/realtimeGC/cmain.c -ldl \ No newline at end of file diff --git a/tests/realtimeGC/tmain.nim b/tests/realtimeGC/tmain.nim index f1515a549..ca0177e3d 100644 --- a/tests/realtimeGC/tmain.nim +++ b/tests/realtimeGC/tmain.nim @@ -9,6 +9,13 @@ these dont' seem needed --debuginfo nor these from the previous main.nim.cfg: --app:console ]# +#[ +Test the realtime GC without linking nimrtl.dll/so. + +To build by hand and run the test for 35 minutes: +`nim r --threads:on -d:runtimeSecs:2100 tests/realtimeGC/tmain.nim` +]# + import times, os, strformat, strutils from stdtest/specialpaths import buildDir # import threadpool @@ -16,9 +23,10 @@ from stdtest/specialpaths import buildDir const runtimeSecs {.intdefine.} = 5 const file = "shared.nim" -const dllname = buildDir / (DynlibFormat % file) +const dllname = buildDir / (DynlibFormat % "shared_D20210524T180506") static: + # D20210524T180826:here we compile the dependency on the fly let nim = getCurrentCompilerExe() let (output, exitCode) = gorgeEx(fmt"{nim} c -o:{dllname} --debuginfo --app:lib --threads:on -d:release -d:useRealtimeGC {file}") doAssert exitCode == 0, output From d62a298a2834b9a35aac4d922a810a772cd3bc1d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 27 May 2021 09:04:10 -0700 Subject: [PATCH 0027/1654] fix typo in ci/funs.sh (#18115) --- ci/funs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/funs.sh b/ci/funs.sh index 9772535be..4a8887658 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -43,7 +43,7 @@ nimInternalBuildKochAndRunCI(){ echo_run nim c koch if ! echo_run ./koch runCI; then echo_run echo "runCI failed" - echo_run nim r -tools/ci_testresults.nim + echo_run nim r tools/ci_testresults.nim return 1 fi } From 9eae6b4fe312035daadd8dcb0ca488a3e5897c11 Mon Sep 17 00:00:00 2001 From: vabresto <77133146+vabresto@users.noreply.github.com> Date: Fri, 28 May 2021 18:17:37 -0400 Subject: [PATCH 0028/1654] fix #17456 flaky test tasynchttpserver_transferencoding (#18052) --- .../tasynchttpserver_transferencoding.nim | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/tests/stdlib/tasynchttpserver_transferencoding.nim b/tests/stdlib/tasynchttpserver_transferencoding.nim index bfa0e2f69..fd79ce3a0 100644 --- a/tests/stdlib/tasynchttpserver_transferencoding.nim +++ b/tests/stdlib/tasynchttpserver_transferencoding.nim @@ -1,3 +1,7 @@ +discard """ + matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger; --threads:on" +""" + import httpclient, asynchttpserver, asyncdispatch, asyncfutures import net @@ -10,33 +14,38 @@ Transfer-Encoding:chunked """ -template genTest(input, expected) = - var sanity = false - proc handler(request: Request) {.async.} = - doAssert(request.body == expected) - doAssert(request.headers.hasKey("Transfer-Encoding")) - doAssert(not request.headers.hasKey("Content-Length")) - sanity = true - await request.respond(Http200, "Good") +template genTest(input, expected: string) = + proc handler(request: Request, future: Future[bool]) {.async, gcsafe.} = + doAssert(request.body == expected) + doAssert(request.headers.hasKey("Transfer-Encoding")) + doAssert(not request.headers.hasKey("Content-Length")) + future.complete(true) + await request.respond(Http200, "Good") - proc runSleepLoop(server: AsyncHttpServer) {.async.} = + proc sendData(data: string, port: Port) {.async.} = + var socket = newSocket() + defer: socket.close() + + socket.connect("127.0.0.1", port) + socket.send(data) + + proc runTest(): Future[bool] {.async.} = + var handlerFuture = newFuture[bool]("runTest") + let data = postBegin & input + let server = newAsyncHttpServer() server.listen(Port(0)) - proc wrapper() = - waitFor server.acceptRequest(handler) - asyncdispatch.callSoon wrapper - let server = newAsyncHttpServer() - waitFor runSleepLoop(server) - let data = postBegin & input - var socket = newSocket() - socket.connect("127.0.0.1", server.getPort) - socket.send(data) - waitFor sleepAsync(10) - socket.close() - server.close() + proc wrapper(request: Request): Future[void] {.gcsafe, closure.} = + handler(request, handlerFuture) + + asyncCheck sendData(data, server.getPort) + asyncCheck server.acceptRequest(wrapper) + doAssert await handlerFuture + + server.close() + return true - # Verify we ran the handler and its asserts - doAssert(sanity) + doAssert waitFor runTest() block: const expected = "hello=world" From c792154b030c1bfec9ed0020ddaf985d5f034ca3 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Sat, 29 May 2021 15:39:46 -0400 Subject: [PATCH 0029/1654] Minor doc fixes; follow up to 17258 and 17259 (#18123) Co-authored-by: quantimnot --- doc/manual.rst | 6 +++--- doc/manual_experimental_strictnotnil.rst | 2 +- doc/tut1.rst | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 2b85f5c34..6a0dec120 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -363,7 +363,7 @@ contain the following `escape sequences`:idx:\ : ``\\`` `backslash`:idx: ``\"`` `quotation mark`:idx: ``\'`` `apostrophe`:idx: - ``\\`` '0'..'9'+ `character with decimal value d`:idx:; + ``\`` '0'..'9'+ `character with decimal value d`:idx:; all decimal digits directly following are used for the character ``\a`` `alert`:idx: @@ -473,7 +473,7 @@ literals: ``\\`` `backslash`:idx: ``\"`` `quotation mark`:idx: ``\'`` `apostrophe`:idx: - ``\\`` '0'..'9'+ `character with decimal value d`:idx:; + ``\`` '0'..'9'+ `character with decimal value d`:idx:; all decimal digits directly following are used for the character ``\a`` `alert`:idx: @@ -7825,7 +7825,7 @@ Threads To enable thread support the `--threads:on`:option: command-line switch needs to be used. The system_ module then contains several threading primitives. -See the `threads `_ and `channels `_ modules +See the `threads `_ and `channels `_ modules for the low-level thread API. There are also high-level parallelism constructs available. See `spawn `_ for further details. diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst index c7c045683..cb3bfeca9 100644 --- a/doc/manual_experimental_strictnotnil.rst +++ b/doc/manual_experimental_strictnotnil.rst @@ -143,7 +143,7 @@ Merging usually merges maps and alias sets: nilabilities are merged like this: else: MaybeNil -Special handling is for `.isNil` and ` == nil`, also for `not`, `and` and `or`. +Special handling is for `.isNil` and `== nil`, also for `not`, `and` and `or`. `not` reverses the nilability, `and` is similar to "forking" : the right expression is checked in the layer resulting from the left one and `or` is similar to "merging": the right and left expression should be both checked in the original layer. diff --git a/doc/tut1.rst b/doc/tut1.rst index 2f19ae173..210ea324e 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -114,7 +114,7 @@ String and character literals ----------------------------- String literals are enclosed in double-quotes; character literals in single -quotes. Special characters are escaped with ``\\``: ``\n`` means newline, ``\t`` +quotes. Special characters are escaped with ``\``: ``\n`` means newline, ``\t`` means tabulator, etc. There are also *raw* string literals: .. code-block:: Nim From cfe19247e856950f7facaf7adbc50bca8dec3990 Mon Sep 17 00:00:00 2001 From: vabresto <77133146+vabresto@users.noreply.github.com> Date: Sat, 29 May 2021 15:44:39 -0400 Subject: [PATCH 0030/1654] Disable tasynchttpserver_transferencoding on FreeBSD (#18122) --- tests/stdlib/tasynchttpserver_transferencoding.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/stdlib/tasynchttpserver_transferencoding.nim b/tests/stdlib/tasynchttpserver_transferencoding.nim index fd79ce3a0..cdd30d04b 100644 --- a/tests/stdlib/tasynchttpserver_transferencoding.nim +++ b/tests/stdlib/tasynchttpserver_transferencoding.nim @@ -1,5 +1,6 @@ discard """ matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger; --threads:on" + disabled: "freebsd" """ import httpclient, asynchttpserver, asyncdispatch, asyncfutures From 4a7f2c386cde91b784106905f6e70d3954d8fae9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 30 May 2021 13:37:21 -0700 Subject: [PATCH 0031/1654] close #16569: deprecated `reversed(a, start, last)` overload, use toOpenArray instead (#18047) * close #16569: deprecated reversed overload, use toOpenArray instead * [skip ci] change wording in changelog per review * fixup --- changelog.md | 2 ++ lib/pure/algorithm.nim | 39 ++++++++++----------------------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/changelog.md b/changelog.md index 4c30d67a7..809728b16 100644 --- a/changelog.md +++ b/changelog.md @@ -75,6 +75,8 @@ - `strformat` is now part of `include std/prelude`. +- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. + - The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. The downside is that these defines now have custom logic that doesn't apply for other defines. diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index f4b3f46be..c96f599e8 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -131,40 +131,21 @@ proc reverse*[T](a: var openArray[T]) = # the max is needed, since a.high is -1 if a is empty reverse(a, 0, max(0, a.high)) -proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] = - ## Returns the reverse of the slice `a[first..last]`. - ## - ## If an invalid range is passed, it raises `IndexDefect`. +proc reversed*[T](a: openArray[T]): seq[T] {.inline.} = + ## Returns the elements of `a` in reverse order. ## ## **See also:** - ## * `reverse proc<#reverse,openArray[T],Natural,Natural>`_ reverse a slice ## * `reverse proc<#reverse,openArray[T]>`_ runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = a.reversed(1, 3) - assert b == @[4, 3, 2] - assert last >= first - 1 - var i = last - first - var x = first.int - result = newSeq[T](i + 1) - while i >= 0: - result[i] = a[x] - dec(i) - inc(x) + assert [10, 11, 12].reversed == @[12, 11, 10] + assert seq[string].default.reversed == @[] + let n = a.len + result.setLen(n) + for i in 0..`_ reverse a slice - ## * `reverse proc<#reverse,openArray[T]>`_ - runnableExamples: - let - a = [1, 2, 3, 4, 5, 6] - b = reversed(a) - assert b == @[6, 5, 4, 3, 2, 1] - reversed(a, 0, a.high) +proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] + {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = + reversed(toOpenArray(a, first, last)) proc binarySearch*[T, K](a: openArray[T], key: K, cmp: proc (x: T, y: K): int {.closure.}): int = From a6bd6c7ed803dcef41b49343b5cd60607b984ca8 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Sun, 30 May 2021 23:38:33 +0300 Subject: [PATCH 0032/1654] Fixes #17849 (#18055) [backport:1.2] * Fixes #17849 * Update compiler/closureiters.nim Co-authored-by: Andreas Rumpf --- compiler/closureiters.nim | 6 ++++++ tests/iter/tyieldintry.nim | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index e474e5ba2..adc5a4873 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -606,6 +606,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode = internalError(ctx.g.config, "lowerStmtListExpr(nkCaseStmt): " & $branch.kind) result.add(n) result.add(ctx.newEnvVarAccess(tmp)) + elif n[0].kind == nkStmtListExpr: + result = newNodeI(nkStmtList, n.info) + let (st, ex) = exprToStmtList(n[0]) + result.add(st) + n[0] = ex + result.add(n) of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64: var ns = false diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim index 35df55c24..3d5260800 100644 --- a/tests/iter/tyieldintry.nim +++ b/tests/iter/tyieldintry.nim @@ -483,5 +483,19 @@ block: # nnkChckRange test(it, 1, 2, 3) -echo "ok" +block: #17849 - yield in case subject + template yieldInCase: int = + yield 2 + 3 + iterator it(): int {.closure.} = + yield 1 + case yieldInCase() + of 1: checkpoint(11) + of 3: checkpoint(13) + else: checkpoint(14) + yield 5 + + test(it, 1, 2, 13, 5) + +echo "ok" From 4274e6e4d700ef72275344afed44d0b658729761 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 30 May 2021 13:39:17 -0700 Subject: [PATCH 0033/1654] upgrade ubuntu 16.04 (not supported starting dec 2021) => 18.04; revive Linux_i386 (#18107) --- .gitlab-ci.yml | 4 +++- azure-pipelines.yml | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad9e565a6..31908d223 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,6 @@ -image: ubuntu:16.04 +# xxx unused, out of date + +image: ubuntu:18.04 stages: - pre-build diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f0c3b2d0a..a70a86fae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,14 +19,14 @@ jobs: strategy: matrix: Linux_amd64: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-18.04' CPU: amd64 - # pending bug #17325 (broken again) - # Linux_i386: - # # bug #17325: fails on 'ubuntu-16.04' because it now errors with: - # # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed - # vmImage: 'ubuntu-18.04' - # CPU: i386 + # regularly breaks, refs bug #17325 + Linux_i386: + # on 'ubuntu-16.04' (not supported anymore anyways) it errored with: + # g++-multilib : Depends: gcc-multilib (>= 4:5.3.1-1ubuntu1) but it is not going to be installed + vmImage: 'ubuntu-18.04' + CPU: i386 OSX_amd64: vmImage: 'macOS-10.15' CPU: amd64 From e2ab08603db6499063b3ff0f16776b4ed20c36f0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 30 May 2021 13:40:42 -0700 Subject: [PATCH 0034/1654] fix warnings/hints in nimdoc/tester.nim (#18083) * fix warnings/hints in nimdoc/tester.nim * improve err msg for nimdoc/tester.nim and change flag from fixup to nimTestsNimdocFixup * address comment: put back quit instead of doAssert --- nimdoc/tester.nim | 12 +++++++----- nimdoc/testproject/expected/testproject.html | 8 ++++---- nimdoc/testproject/testproject.nim | 9 +++++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index 9daa0bb51..1e438b97b 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -1,10 +1,12 @@ # Small program that runs the test cases for 'nim doc'. # To run this, cd to the git repo root, and run "nim r nimdoc/tester.nim". -# to change expected results (after carefully verifying everything), use -d:fixup +# to change expected results (after carefully verifying everything), use -d:nimTestsNimdocFixup import strutils, os from std/private/gitutils import diffFiles +const fixup = defined(nimTestsNimdocFixup) + var failures = 0 @@ -62,7 +64,7 @@ let "$1/$2.nim" % [test1Dir, test1PrjName]], buildIndex: @["--out:$1/$2/theindex.html" % [test1Dir, test1DocsDir], "$1/$2" % [test1Dir, test1DocsDir]]) -testNimDoc(test1Dir, test1DocsDir, test1Switches, defined(fixup)) +testNimDoc(test1Dir, test1DocsDir, test1Switches, fixup) # Test "nim doc --out:.. --index:on .." let @@ -75,7 +77,7 @@ let "$1/$2.nim" % [test2Dir, test2PrjName]], buildIndex: @["--out:$1/$2/theindex.html" % [test2Dir, test2DocsDir], "$1/$2" % [test2Dir, test2DocsDir]]) -testNimDoc(test2Dir, test2DocsDir, test2Switches, defined(fixup)) +testNimDoc(test2Dir, test2DocsDir, test2Switches, fixup) -# Check for failures -if failures > 0: quit($failures & " failures occurred.") +if failures > 0: + quit "$# failures occurred; see note in tester.nim regarding -d:nimTestsNimdocFixup" % $failures diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index f23f6bb42..d2fe38021 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -790,7 +790,7 @@ the c printf. etc.
-

Example:

+

Example: cmd: --hint:XDeclaredButNotUsed:off

## mullitline string litterals are tricky as their indentation can span
 ## below that of the runnableExamples
 let s1a = """
@@ -865,21 +865,21 @@ There is no block quote after blank lines at the beginning.
 
Methods

 

 
-
method method1(self: Moo) {....raises: [], tags: [].}

+
method method1(self: Moo) {.base, ...raises: [], tags: [].}

 

 
 foo1
 
 

 
-
method method2(self: Moo): int {....raises: [], tags: [].}

+
method method2(self: Moo): int {.base, ...raises: [], tags: [].}

 

 
 foo2
 
 

 
-
method method3(self: Moo): int {....raises: [], tags: [].}

+
method method3(self: Moo): int {.base, ...raises: [], tags: [].}

 

 
 foo3
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index d435efa66..b88b41098 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -131,6 +131,7 @@ when true:
       # BUG: this currently this won't be run since not exported
       # but probably should
       doAssert false
+  if false: bazNonExported() # silence XDeclaredButNotUsed
 
   proc z17*() =
     # BUG: a comment before 1st doc comment currently doesn't prevent
@@ -212,7 +213,7 @@ when true: # tests RST inside comments
 
 when true: # multiline string litterals
   proc tripleStrLitTest*() =
-    runnableExamples:
+    runnableExamples("--hint:XDeclaredButNotUsed:off"):
       ## mullitline string litterals are tricky as their indentation can span
       ## below that of the runnableExamples
       let s1a = """
@@ -252,12 +253,12 @@ at indent 0
 
 when true: # methods; issue #14691
   type Moo = object
-  method method1*(self: Moo) =
+  method method1*(self: Moo) {.base.} =
     ## foo1
-  method method2*(self: Moo): int =
+  method method2*(self: Moo): int {.base.} =
     ## foo2
     result = 1
-  method method3*(self: Moo): int =
+  method method3*(self: Moo): int {.base.} =
     ## foo3
     1
 

From 50e98e6efaca05b60b33db0c878f04951a43b3b8 Mon Sep 17 00:00:00 2001
From: Miran 
Date: Sun, 30 May 2021 23:55:51 +0200
Subject: [PATCH 0035/1654] hashes for refs should be an opt-in feature
 (#18098)

---
 changelog.md                  | 17 +++++++++++++----
 lib/pure/hashes.nim           |  2 +-
 tests/collections/ttables.nim |  1 +
 tests/stdlib/thashes.nim      |  1 +
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/changelog.md b/changelog.md
index 809728b16..279ba38dd 100644
--- a/changelog.md
+++ b/changelog.md
@@ -50,8 +50,9 @@
 - On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler`
   for previous behavior.
 
-- `hashes.hash` now supports `object` and `ref` (can be overloaded in user code).
-  For a transition period, use `-d:nimLegacyNoHashRef` to avoid defining `hash(ref)`.
+- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code),
+  if `-d:nimEnableHashRef` is used.
+
 - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`,
   `hashes.hash(closure)` has also been improved.
 
@@ -82,7 +83,9 @@
   other defines.
 
 
+
 ## Standard library additions and changes
+
 - Added support for parenthesized expressions in `strformat`
 
 - Fixed buffer overflow bugs in `net`
@@ -99,7 +102,6 @@
   the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
   now also need to ship `cacert.pem` with your `.exe` file.
 
-
 - Make `{.requiresInit.}` pragma to work for `distinct` types.
 
 - Added a macros `enumLen` for returning the number of items in an enum to the
@@ -324,6 +326,8 @@
 
 - Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
+
+
 ## Language changes
 
 - `nimscript` now handles `except Exception as e`.
@@ -368,6 +372,8 @@
 
 - `typeof(voidStmt)` now works and returns `void`.
 
+
+
 ## Compiler changes
 
 - Added `--declaredlocs` to show symbol declaration location in messages.
@@ -394,7 +400,8 @@
 - TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types,
   such types must use `.cppNonPod` and `--tlsEmulation:off`should be used.
 
-- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists, for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
+- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists,
+  for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`.
 
 - docgen: rst files can now use single backticks instead of double backticks and correctly render
   in both rst2html (as before) as well as common tools rendering rst directly (e.g. github), by
@@ -426,6 +433,8 @@
   the official Nim style guide. To be enabled, this has to be combined either
   with `--styleCheck:error` or `--styleCheck:hint`.
 
+
+
 ## Tool changes
 
 - The rst parser now supports markdown table syntax.
diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim
index f61fbdf2b..46b4fc948 100644
--- a/lib/pure/hashes.nim
+++ b/lib/pure/hashes.nim
@@ -236,7 +236,7 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} =
     assert cast[pointer](a[0].addr).hash == a[0].addr.hash
   hash(cast[pointer](x))
 
-when not defined(nimLegacyNoHashRef):
+when defined(nimEnableHashRef):
   proc hash*[T](x: ref[T]): Hash {.inline.} =
     ## Efficient `hash` overload.
     runnableExamples:
diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim
index 638f4241b..751509062 100644
--- a/tests/collections/ttables.nim
+++ b/tests/collections/ttables.nim
@@ -8,6 +8,7 @@ And we get here
 '''
 joinable: false
 targets: "c cpp js"
+matrix: "-d:nimEnableHashRef"
 """
 
 # xxx wrap in a template to test in VM, see https://github.com/timotheecour/Nim/issues/534#issuecomment-769565033
diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim
index 46576ef12..1e9b02d0c 100644
--- a/tests/stdlib/thashes.nim
+++ b/tests/stdlib/thashes.nim
@@ -1,5 +1,6 @@
 discard """
   targets: "c cpp js"
+  matrix: "-d:nimEnableHashRef"
 """
 
 import std/hashes

From 98ea61f09b4bfdc13c911b1ff3fb404dfc0d1338 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 30 May 2021 14:57:25 -0700
Subject: [PATCH 0036/1654] dont silence hints in system.nim (#18034)

---
 lib/system.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/system.nim b/lib/system.nim
index be30665dd..942d29fe8 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -112,7 +112,7 @@ proc compileOption*(option, arg: string): bool {.
       discard "compiled with optimization for size and uses Boehm's GC"
 
 {.push warning[GcMem]: off, warning[Uninit]: off.}
-{.push hints: off.}
+# {.push hints: off.}
 
 proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.}
   ## Constructs an `or` meta class.
@@ -2471,7 +2471,7 @@ proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} =
   quit(errorcode)
 
 {.pop.} # checks: off
-{.pop.} # hints: off
+# {.pop.} # hints: off
 
 proc `/`*(x, y: int): float {.inline, noSideEffect.} =
   ## Division of integers that results in a float.

From 18b477431138d944b90e7a6e6e0412496634b518 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 01:51:20 -0700
Subject: [PATCH 0037/1654] document macros.unpackVarargs (#18106)

* deprecate macros.unpackVarargs

* un-deprecate unpackVarargs and add docs+runnableExamples

* update examples + tests with varargs[typed]
---
 lib/core/macros.nim              | 15 ++++++++++
 tests/destructor/tatomicptrs.nim |  1 +
 tests/stdlib/tmacros.nim         | 50 ++++++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+)

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index 79c3ba28b..89f90a23b 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1682,6 +1682,21 @@ macro getCustomPragmaVal*(n: typed, cp: typed): untyped =
     error(n.repr & " doesn't have a pragma named " & cp.repr, n)
 
 macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
+  ## Calls `callee` with `args` unpacked as individual arguments.
+  ## This is useful in 2 cases:
+  ## * when forwarding `varargs[T]` for some typed `T`
+  ## * when forwarding `varargs[untyped]` when `args` can potentially be empty,
+  ##   due to a compiler limitation
+  runnableExamples:
+    template call1(fun: typed; args: varargs[untyped]): untyped =
+      unpackVarargs(fun, args)
+      # when varargsLen(args) > 0: fun(args) else: fun() # this would also work
+    template call2(fun: typed; args: varargs[typed]): untyped =
+      unpackVarargs(fun, args)
+    proc fn1(a = 0, b = 1) = discard (a, b)
+    call1(fn1, 10, 11)
+    call1(fn1) # `args` is empty in this case
+    if false: call2(echo, 10, 11) # would print 1011
   result = newCall(callee)
   for i in 0 ..< args.len:
     result.add args[i]
diff --git a/tests/destructor/tatomicptrs.nim b/tests/destructor/tatomicptrs.nim
index 7313afbf5..36f0cab8a 100644
--- a/tests/destructor/tatomicptrs.nim
+++ b/tests/destructor/tatomicptrs.nim
@@ -72,6 +72,7 @@ template `.=`*[T](s: SharedPtr[T]; field, value: untyped) =
 from macros import unpackVarargs
 
 template `.()`*[T](s: SharedPtr[T]; field: untyped, args: varargs[untyped]): untyped =
+  # xxx this isn't used, the test should be improved
   unpackVarargs(s.x.field, args)
 
 
diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim
index effe1032f..52ea1e898 100644
--- a/tests/stdlib/tmacros.nim
+++ b/tests/stdlib/tmacros.nim
@@ -10,3 +10,53 @@ block: # hasArgOfName
 
 block: # bug #17454
   proc f(v: NimNode): string {.raises: [].} = $v
+
+block: # unpackVarargs
+  block:
+    proc bar1(a: varargs[int]): string =
+      for ai in a: result.add " " & $ai
+    proc bar2(a: varargs[int]) =
+      let s1 = bar1(a)
+      let s2 = unpackVarargs(bar1, a) # `unpackVarargs` makes no difference here
+      doAssert s1 == s2
+    bar2(1, 2, 3)
+    bar2(1)
+    bar2()
+
+  block:
+    template call1(fun: typed; args: varargs[untyped]): untyped =
+      unpackVarargs(fun, args)
+    template call2(fun: typed; args: varargs[untyped]): untyped =
+      # fun(args) # works except for last case with empty `args`, pending bug #9996
+      when varargsLen(args) > 0: fun(args)
+      else: fun()
+
+    proc fn1(a = 0, b = 1) = discard (a, b)
+
+    call1(fn1)
+    call1(fn1, 10)
+    call1(fn1, 10, 11)
+
+    call2(fn1)
+    call2(fn1, 10)
+    call2(fn1, 10, 11)
+
+  block:
+    template call1(fun: typed; args: varargs[typed]): untyped =
+      unpackVarargs(fun, args)
+    template call2(fun: typed; args: varargs[typed]): untyped =
+      # xxx this would give a confusing error message:
+      # required type for a: varargs[typed] [varargs] but expression '[10]' is of type: varargs[typed] [varargs]
+      when varargsLen(args) > 0: fun(args)
+      else: fun()
+    macro toString(a: varargs[typed, `$`]): string =
+      var msg = genSym(nskVar, "msg")
+      result = newStmtList()
+      result.add quote do:
+        var `msg` = ""
+      for ai in a:
+        result.add quote do: `msg`.add $`ai`
+      result.add quote do: `msg`
+    doAssert call1(toString) == ""
+    doAssert call1(toString, 10) == "10"
+    doAssert call1(toString, 10, 11) == "1011"

From 064fe18de61c2aae5c215077ba1dd0352affa271 Mon Sep 17 00:00:00 2001
From: n5m <72841454+n5m@users.noreply.github.com>
Date: Mon, 31 May 2021 09:20:15 +0000
Subject: [PATCH 0038/1654] improve fast returns of find and rfind (#18127)

---
 lib/pure/strutils.nim      |  4 +++-
 tests/stdlib/tstrutils.nim | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 5bbd850cc..3fb67ead1 100644
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -1890,7 +1890,7 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl,
   ## See also:
   ## * `rfind func<#rfind,string,string,Natural>`_
   ## * `replace func<#replace,string,string,string>`_
-  if sub.len > s.len: return -1
+  if sub.len > s.len - start: return -1
   if sub.len == 1: return find(s, sub[0], start, last)
 
   template useSkipTable {.dirty.} =
@@ -1967,6 +1967,8 @@ func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl,
   ## * `find func<#find,string,string,Natural,int>`_
   if sub.len == 0:
     return -1
+  if sub.len > s.len - start:
+    return -1
   let last = if last == -1: s.high else: last
   result = 0
   for i in countdown(last - sub.len + 1, start):
diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim
index 771dddcaf..637661ba3 100644
--- a/tests/stdlib/tstrutils.nim
+++ b/tests/stdlib/tstrutils.nim
@@ -243,6 +243,34 @@ template main() =
     doAssert "/1/2/3".rfind('/', last=1) == 0
     doAssert "/1/2/3".rfind('0') == -1
 
+    block:
+      const haystack: string = "ABCABABABABCAB"
+      doAssert haystack.len == 14
+      doAssert haystack.rfind("ABC") == 9
+      doAssert haystack.rfind("ABC", last=13) == 9
+      doAssert haystack.rfind("ABC", last=12) == 9
+      doAssert haystack.rfind("ABC", last=11) == 9
+      doAssert haystack.rfind("ABC", last=10) == 0
+
+      doAssert haystack.rfind("ABC", start=0) == 9
+      doAssert haystack.rfind("ABC", start=1) == 9
+      doAssert haystack.rfind("ABC", start=2) == 9
+      doAssert haystack.rfind("ABC", start=9) == 9
+      doAssert haystack.rfind("ABC", start=10) == -1
+      doAssert haystack.rfind("ABC", start=11) == -1
+      doAssert haystack.rfind("ABC", start=12) == -1
+
+      doAssert haystack.rfind("ABC", start=0, last=13) == 9
+      doAssert haystack.rfind("ABC", start=0, last=12) == 9
+      doAssert haystack.rfind("ABC", start=0, last=11) == 9
+      doAssert haystack.rfind("ABC", start=0, last=10) == 0
+      doAssert haystack.rfind("ABC", start=1, last=10) == -1
+
+    doAssert "".rfind("/") == -1
+    doAssert "/".rfind("/") == 0
+    doAssert "/".rfind("//") == -1
+    doAssert "///".rfind("//", start=3) == -1
+
   block: # trimZeros
     var x = "1200"
     x.trimZeros()

From b7ad29e692ffd9d12fdf4d42cd8ef2a63b9340d1 Mon Sep 17 00:00:00 2001
From: Saem Ghani 
Date: Mon, 31 May 2021 04:27:44 -0700
Subject: [PATCH 0039/1654] fix #18113 (#18124)

---
 compiler/semtempl.nim     |  8 ++++++--
 tests/proc/t17157.nim     |  1 +
 tests/template/t18113.nim | 14 ++++++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 tests/template/t18113.nim

diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim
index 40502daf4..181cc01db 100644
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -610,8 +610,6 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
        s.owner.name.s == "vm" and s.name.s == "stackTrace":
       incl(s.flags, sfCallsite)
 
-  s.ast = n
-
   styleCheckDef(c.config, s)
   onDef(n[namePos].info, s)
   # check parameter list:
@@ -664,6 +662,12 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
   semIdeForTemplateOrGeneric(c, n[bodyPos], ctx.cursorInBody)
   closeScope(c)
   popOwner(c)
+
+  # set the symbol AST after pragmas, at least. This stops pragma that have
+  # been pushed (implicit) to be explicitly added to the template definition
+  # and misapplied to the body. see #18113
+  s.ast = n
+
   if sfCustomPragma in s.flags:
     if n[bodyPos].kind != nkEmpty:
       localError(c.config, n[bodyPos].info, errImplOfXNotAllowed % s.name.s)
diff --git a/tests/proc/t17157.nim b/tests/proc/t17157.nim
index 020e93fce..2927eeee8 100644
--- a/tests/proc/t17157.nim
+++ b/tests/proc/t17157.nim
@@ -1,5 +1,6 @@
 discard """
   errormsg: "'untyped' is only allowed in templates and macros or magic procs"
+  disabled: true
 """
 
 template something(op: proc (v: untyped): void): void =
diff --git a/tests/template/t18113.nim b/tests/template/t18113.nim
new file mode 100644
index 000000000..a84b3fd0e
--- /dev/null
+++ b/tests/template/t18113.nim
@@ -0,0 +1,14 @@
+# ensure template pragma handling doesn't eagerly attempt to add an implicit
+# 'pushed' pragma to the evaluation of any intermediate AST prior to
+# substitution.
+ 
+# bug #18113
+
+import sequtils
+
+{.push raises: [Defect].}
+
+var a = toSeq([1, 2, 3, 5, 10]).filterIt(it > 5)
+
+doAssert a.len == 1
+doAssert a[0] == 10

From a36efb59b5a74db6c5bbe6c0997c0221d0f55491 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 04:39:19 -0700
Subject: [PATCH 0040/1654] fix #16256: nimout:  should give error
 (vacuously true); improve a few tests (#18089)

* fix #16256: nimout:  should give error (vacuously true); improve some tests

* renamed:    tests/stdlib/t9710.nim -> tests/misc/t9710.nim

* improve tests

* fix non-DRY tests

* improve $nim_prs_D/tests/stdlib/t9091.nim

* renamed:    tests/stdlib/t9091.nim -> tests/misc/t9091.nim

* fixup

* address comment: doAssert => result.parseErrors
---
 testament/specs.nim              |  5 ++
 tests/{stdlib => misc}/t9091.nim | 15 +++---
 tests/misc/t9710.nim             |  6 +++
 tests/stdlib/t9710.nim           | 11 ----
 tests/stdlib/tgetprotobyname.nim | 10 ----
 tests/stdlib/tnativesockets.nim  | 46 ++++++++---------
 tests/stdlib/tvarints.nim        | 86 +++++++++++++-------------------
 7 files changed, 73 insertions(+), 106 deletions(-)
 rename tests/{stdlib => misc}/t9091.nim (70%)
 create mode 100644 tests/misc/t9710.nim
 delete mode 100644 tests/stdlib/t9710.nim

diff --git a/testament/specs.nim b/testament/specs.nim
index a3271aeaa..6fd0ab22d 100644
--- a/testament/specs.nim
+++ b/testament/specs.nim
@@ -253,6 +253,7 @@ proc parseSpec*(filename: string): TSpec =
   var p: CfgParser
   open(p, ss, filename, 1)
   var flags: HashSet[string]
+  var nimoutFound = false
   while true:
     var e = next(p)
     case e.kind
@@ -309,6 +310,7 @@ proc parseSpec*(filename: string): TSpec =
         result.action = actionReject
       of "nimout":
         result.nimout = e.value
+        nimoutFound = true
       of "nimoutfull":
         result.nimoutFull = parseCfgBool(e.value)
       of "batchable":
@@ -401,6 +403,9 @@ proc parseSpec*(filename: string): TSpec =
   if skips.anyIt(it in result.file):
     result.err = reDisabled
 
+  if nimoutFound and result.nimout.len == 0 and not result.nimoutFull:
+    result.parseErrors.addLine "empty `nimout` is vacuously true, use `nimoutFull:true` if intentional"
+
   result.inCurrentBatch = isCurrentBatch(testamentData0, filename) or result.unbatchable
   if not result.inCurrentBatch:
     result.err = reDisabled
diff --git a/tests/stdlib/t9091.nim b/tests/misc/t9091.nim
similarity index 70%
rename from tests/stdlib/t9091.nim
rename to tests/misc/t9091.nim
index 8419479a7..6e7a98ca5 100644
--- a/tests/stdlib/t9091.nim
+++ b/tests/misc/t9091.nim
@@ -1,10 +1,5 @@
-discard """
-  targets:  "c"
-  output:   "test AObj"
-  action:   "compile"
-  exitcode: 0
-  timeout:  60.0
-"""
+# bug #9091
+
 import streams
 
 block:
@@ -18,6 +13,8 @@ block:
   let mi = new Mine
 
   str.write(mi)
+  str.setPosition 0
+  doAssert str.readAll == "sure"
 
 block:
   type
@@ -27,10 +24,10 @@ block:
   proc foo(a: int): string = ""
 
   proc test(args: varargs[string, foo]) =
-    echo "varargs"
+    doAssert false
 
   proc test(a: AObj) =
-    echo "test AObj"
+    discard
 
   let x = AObj()
   test(x)
diff --git a/tests/misc/t9710.nim b/tests/misc/t9710.nim
new file mode 100644
index 000000000..c65cb7bf4
--- /dev/null
+++ b/tests/misc/t9710.nim
@@ -0,0 +1,6 @@
+discard """
+  matrix: "--debugger:native"
+"""
+# bug #9710
+for i in 1 || 200:
+  discard i
diff --git a/tests/stdlib/t9710.nim b/tests/stdlib/t9710.nim
deleted file mode 100644
index f3ed860df..000000000
--- a/tests/stdlib/t9710.nim
+++ /dev/null
@@ -1,11 +0,0 @@
-discard """
-  cmd:      "nim c -r --debugger:native --panics:on $options $file"
-  targets:  "c"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
-for i in 1 || 200:
-  discard i
diff --git a/tests/stdlib/tgetprotobyname.nim b/tests/stdlib/tgetprotobyname.nim
index b4df05102..0b60d059f 100644
--- a/tests/stdlib/tgetprotobyname.nim
+++ b/tests/stdlib/tgetprotobyname.nim
@@ -1,15 +1,5 @@
-discard """
-  cmd:      "nim c -r --styleCheck:hint --panics:on $options $file"
-  targets:  "c"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
 import nativesockets
 
-
 when not defined(netbsd):
   # Ref: https://github.com/nim-lang/Nim/issues/15452 - NetBSD doesn't define an `ip` protocol
   doAssert getProtoByName("ip") == 0
diff --git a/tests/stdlib/tnativesockets.nim b/tests/stdlib/tnativesockets.nim
index b0cfd09cf..6a1a00881 100644
--- a/tests/stdlib/tnativesockets.nim
+++ b/tests/stdlib/tnativesockets.nim
@@ -1,29 +1,25 @@
-discard """
-  cmd:      "nim c -r --styleCheck:hint --panics:on $options $file"
-  targets:  "c"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
-import nativesockets
+import std/nativesockets
+import stdtest/testutils
 
+block:
+  let hostname = getHostname()
+  doAssert hostname.len > 0
 
 when defined(windows):
-  doAssert toInt(IPPROTO_IP) == 0.cint
-  doAssert toInt(IPPROTO_ICMP) == 1.cint
-  doAssert toInt(IPPROTO_TCP) == 6.cint
-  doAssert toInt(IPPROTO_UDP) == 17.cint
-  doAssert toInt(IPPROTO_IPV6) == 41.cint
-  doAssert toInt(IPPROTO_ICMPV6) == 58.cint
-  doAssert toInt(IPPROTO_RAW) == 20.cint
+  assertAll:
+    toInt(IPPROTO_IP) == 0
+    toInt(IPPROTO_ICMP) == 1
+    toInt(IPPROTO_TCP) == 6
+    toInt(IPPROTO_UDP) == 17
+    toInt(IPPROTO_IPV6) == 41
+    toInt(IPPROTO_ICMPV6) == 58
+    toInt(IPPROTO_RAW) == 20
 
-  # no changes to enum value
-  doAssert ord(IPPROTO_TCP) == 6
-  doAssert ord(IPPROTO_UDP) == 17
-  doAssert ord(IPPROTO_IP) == 18
-  doAssert ord(IPPROTO_IPV6) == 19
-  doAssert ord(IPPROTO_RAW) == 20
-  doAssert ord(IPPROTO_ICMP) == 21
-  doAssert ord(IPPROTO_ICMPV6) == 22
+    # no changes to enum value
+    ord(IPPROTO_TCP) == 6
+    ord(IPPROTO_UDP) == 17
+    ord(IPPROTO_IP) == 18
+    ord(IPPROTO_IPV6) == 19
+    ord(IPPROTO_RAW) == 20
+    ord(IPPROTO_ICMP) == 21
+    ord(IPPROTO_ICMPV6) == 22
diff --git a/tests/stdlib/tvarints.nim b/tests/stdlib/tvarints.nim
index dcdb756ce..772121f9f 100644
--- a/tests/stdlib/tvarints.nim
+++ b/tests/stdlib/tvarints.nim
@@ -1,15 +1,6 @@
-discard """
-  cmd:      "nim c -r --styleCheck:hint --panics:on $options $file"
-  matrix:   "-d:danger; -d:release"
-  targets:  "c cpp"
-  nimout:   ""
-  action:   "run"
-  exitcode: 0
-  timeout:  60.0
-"""
-
 import std/varints
 
+# xxx doesn't work with js: tvarints.nim(18, 14) `wrLen == rdLen`  [AssertionDefect]
 
 block:
   var dest: array[50, byte]
@@ -39,46 +30,39 @@ block:
 block:
   var hugeIntArray: array[50, byte]
   var readedInt: uint64
-  doAssert writeVu64(hugeIntArray, 0.uint64) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == 0.uint64
-  doAssert writeVu64(hugeIntArray, uint64.high) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64.high
-  doAssert writeVu64(hugeIntArray, uint64(int64.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int64.high)
-  doAssert writeVu64(hugeIntArray, uint64(int32.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int32.high)
-  doAssert writeVu64(hugeIntArray, uint64(int16.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int16.high)
-  doAssert writeVu64(hugeIntArray, uint64(int8.high)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == uint64(int8.high)
-  doAssert writeVu64(hugeIntArray, cast[uint64](0.0)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](0.0)
-  doAssert writeVu64(hugeIntArray, cast[uint64](-0.0)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](-0.0)
-  doAssert writeVu64(hugeIntArray, cast[uint64](0.1)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](0.1)
-  doAssert writeVu64(hugeIntArray, cast[uint64](0.9555555555555555555555501)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](0.9555555555555555555555501)
-  doAssert writeVu64(hugeIntArray, cast[uint64](+Inf)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](+Inf)
-  doAssert writeVu64(hugeIntArray, cast[uint64](NegInf)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](NegInf)
-  doAssert writeVu64(hugeIntArray, cast[uint64](Nan)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](Nan)
-  doAssert writeVu64(hugeIntArray, cast[uint64](3.1415926535897932384626433)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](3.1415926535897932384626433)
-  doAssert writeVu64(hugeIntArray, cast[uint64](2.71828182845904523536028747)) == readVu64(hugeIntArray, readedInt)
-  doAssert readedInt == cast[uint64](2.71828182845904523536028747)
+
+  template chk(a) =
+    let b = cast[uint64](a)
+    doAssert writeVu64(hugeIntArray, b) == readVu64(hugeIntArray, readedInt)
+    doAssert readedInt == b
+
+  chk 0
+  chk uint64.high
+  chk int64.high
+  chk int32.high
+  chk int16.high
+  chk int16.high
+  chk int8.high
+  chk 0.0
+  chk -0.0
+  chk 0.1
+  chk Inf
+  chk NegInf
+  chk Nan
+  chk 3.1415926535897932384626433
 
 block:
-  doAssert encodeZigzag(decodeZigzag(0.uint64)) == 0.uint64
-  doAssert encodeZigzag(decodeZigzag(uint64(uint32.high))) == uint64(uint32.high)
-  doAssert encodeZigzag(decodeZigzag(uint64(int32.high))) == uint64(int32.high)
-  doAssert encodeZigzag(decodeZigzag(uint64(int16.high))) == uint64(int16.high)
-  doAssert encodeZigzag(decodeZigzag(uint64(int8.high))) == uint64(int8.high)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](0.0))) == cast[uint64](0.0)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](0.1))) == cast[uint64](0.1)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](0.9555555555555555555555501))) == cast[uint64](0.9555555555555555555555501)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](+Inf))) == cast[uint64](+Inf)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](3.1415926535897932384626433))) == cast[uint64](3.1415926535897932384626433)
-  doAssert encodeZigzag(decodeZigzag(cast[uint64](2.71828182845904523536028747))) == cast[uint64](2.71828182845904523536028747)
+  template chk(a) =
+    let b = cast[uint64](a)
+    doAssert encodeZigzag(decodeZigzag(b)) == b
+  chk 0
+  chk uint32.high
+  chk int32.high
+  chk int16.high
+  chk int8.high
+  chk 0.0
+  chk 0.1
+  chk 0.9555555555555555555555501
+  chk Inf
+  chk 3.1415926535897932384626433
+  chk 2.71828182845904523536028747

From 60cbdbf37ac66624f78edd5b3d575c80c6603679 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 13:14:50 -0700
Subject: [PATCH 0041/1654] close #18092 document elif in case statements
 (#18105)

* close #18092 [skip ci] document elif in case statements

* fixup

* clarify spec; mention special rule for string in case statements

* address comments
---
 doc/manual.rst | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index 6a0dec120..ff34b62ca 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -2981,11 +2981,13 @@ Example:
 
 .. code-block:: nim
 
-  case readline(stdin)
+  let line = readline(stdin)
+  case line
   of "delete-everything", "restart-computer":
     echo "permission denied"
   of "go-for-a-walk":     echo "please yourself"
-  else:                   echo "unknown command"
+  elif line.len == 0:     echo "empty" # optional, must come after `of` branches
+  else:                   echo "unknown command" # ditto
 
   # indentation of the branches is also allowed; and so is an optional colon
   # after the selecting expression:
@@ -2996,19 +2998,23 @@ Example:
     else:                   echo "unknown command"
 
 
-The `case` statement is similar to the if statement, but it represents
+The `case` statement is similar to the `if` statement, but it represents
 a multi-branch selection. The expression after the keyword `case` is
 evaluated and if its value is in a *slicelist* the corresponding statements
 (after the `of` keyword) are executed. If the value is not in any
-given *slicelist* the `else` part is executed. If there is no `else`
-part and not all possible values that `expr` can hold occur in a
-*slicelist*, a static error occurs. This holds only for expressions of
-ordinal types. "All possible values" of `expr` are determined by `expr`'s
-type. To suppress the static error an `else` part with an
-empty `discard` statement should be used.
+given *slicelist*, trailing `elif` and `else` parts are executed using same
+semantics as for `if` statement, and `elif` is handled just like `else: if`.
+If there are no `else` or `elif` parts and not
+all possible values that `expr` can hold occur in a *slicelist*, a static error occurs.
+This holds only for expressions of ordinal types.
+"All possible values" of `expr` are determined by `expr`'s type.
+To suppress the static error an `else: discard` should be used.
 
 For non-ordinal types, it is not possible to list every possible value and so
 these always require an `else` part.
+An exception to this rule is for the `string` type, which currently doesn't
+require a trailing `else` or `elif` branch; it's unspecified whether this will
+keep working in future versions.
 
 Because case statements are checked for exhaustiveness during semantic analysis,
 the value in every `of` branch must be a constant expression.
@@ -3054,15 +3060,15 @@ won't work:
   var foo = Foo(x: @[])
   foo.get_x().add("asd")
 
-This can be fixed by explicitly using `return`:
+This can be fixed by explicitly using `result` or `return`:
 
 .. code-block:: nim
   proc get_x(x: Foo): var seq[string] =
     case true
     of true:
-      return x.x
+      result = x.x
     else:
-      return x.x
+      result = x.x
 
 
 When statement

From 9559350e34a62614d1f7de0945095a4c4f4f5ece Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 13:16:33 -0700
Subject: [PATCH 0042/1654] add `os.getCacheDir` (#18126)

* add `os.getCacheDir`

* fixup

* address comments
---
 changelog.md         |  3 ++-
 lib/pure/os.nim      | 44 ++++++++++++++++++++++++++++++++++++++------
 tests/stdlib/tos.nim |  3 +++
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index 279ba38dd..d404b1965 100644
--- a/changelog.md
+++ b/changelog.md
@@ -175,10 +175,11 @@
 - `--gc:orc` is now 10% faster than previously for common workloads. If
   you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
 
-
 - `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
   determining preferred I/O block size for this file object.
 
+- Added `os.getCacheDir()` to return platform specific cache directory.
+
 - Added a simpler to use `io.readChars` overload.
 
 - Added `**` to jsffi.
diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 287fbe125..c48d0d84f 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -922,9 +922,7 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1",
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getTempDir proc <#getTempDir>`_
-  ## * `expandTilde proc <#expandTilde,string>`_
-  ## * `getCurrentDir proc <#getCurrentDir>`_
-  ## * `setCurrentDir proc <#setCurrentDir,string>`_
+  ## * `getCacheDir proc <#getCacheDir>`_
   runnableExamples:
     from std/strutils import endsWith
     # See `getHomeDir` for behavior regarding trailing DirSep.
@@ -935,6 +933,42 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1",
     result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config")
   result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
 
+proc getCacheDir*(): string =
+  ## Returns the cache directory of the current user for applications.
+  ## 
+  ## This makes use of the following environment variables:
+  ##
+  ## * On Windows: `getEnv("LOCALAPPDATA")`
+  ##
+  ## * On macOS: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")`
+  ##
+  ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")`
+  ##
+  ## **See also:**
+  ## * `getHomeDir proc <#getHomeDir>`_
+  ## * `getTempDir proc <#getTempDir>`_
+  ## * `getConfigDir proc <#getConfigDir>`_
+  # follows https://crates.io/crates/platform-dirs
+  when defined(windows):
+    result = getEnv("LOCALAPPDATA")
+  elif defined(osx):
+    result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / "Library/Caches")
+  else:
+    result = getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")
+  result.normalizePathEnd(false)
+
+proc getCacheDir*(app: string): string =
+  ## Returns the cache directory for an application `app`.
+  ##
+  ## * On windows, this uses: `getCacheDir() / app / "cache"`
+  ##
+  ## * On other platforms, this uses: `getCacheDir() / app`
+  when defined(windows):
+    getCacheDir() / app / "cache"
+  else:
+    getCacheDir() / app
+
+
 when defined(windows):
   type DWORD = uint32
 
@@ -972,9 +1006,7 @@ proc getTempDir*(): string {.rtl, extern: "nos$1",
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
-  ## * `expandTilde proc <#expandTilde,string>`_
-  ## * `getCurrentDir proc <#getCurrentDir>`_
-  ## * `setCurrentDir proc <#setCurrentDir,string>`_
+  ## * `getCacheDir proc <#getCacheDir>`_
   runnableExamples:
     from std/strutils import endsWith
     # See `getHomeDir` for behavior regarding trailing DirSep.
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 1fcee5a2e..b9769d05c 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -556,6 +556,9 @@ block getTempDir:
       else:
         doAssert getTempDir() == "/tmp"
 
+block: # getCacheDir
+  doAssert getCacheDir().dirExists
+
 block osenv:
   block delEnv:
     const dummyEnvVar = "DUMMY_ENV_VAR" # This env var wouldn't be likely to exist to begin with

From 369a7d12467831295c12f5f51aa8c69ec3b3596a Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 31 May 2021 13:17:52 -0700
Subject: [PATCH 0043/1654] jsonutils.toJson now serializes JsonNode as is by
 default (#18097)

* jsonutils.toJson now serializes JsonNode as is (without deep copy nor treating it as a regular ref object)

* JsonNodeMode
---
 changelog.md                |  4 +++-
 lib/std/jsonutils.nim       | 20 ++++++++++++++++----
 tests/stdlib/tjsonutils.nim | 22 ++++++++++++++++++++++
 3 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/changelog.md b/changelog.md
index d404b1965..9806a6011 100644
--- a/changelog.md
+++ b/changelog.md
@@ -69,7 +69,9 @@
   previous behavior for a transition time, see PR #17467.
 
 - `jsonutils` now serializes/deserializes holey enums as regular enums (via `ord`) instead of as strings.
-  Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period.
+  Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period. `toJson` now serializes `JsonNode`
+  as is via reference (without a deep copy) instead of treating `JsonNode` as a regular ref object,
+  this can be customized via `jsonNodeMode`.
 
 - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
   `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim
index 1e222e3a2..e33ae4401 100644
--- a/lib/std/jsonutils.nim
+++ b/lib/std/jsonutils.nim
@@ -48,13 +48,18 @@ type
     joptEnumOrd
     joptEnumSymbol
     joptEnumString
+  JsonNodeMode* = enum ## controls `toJson` for JsonNode types
+    joptJsonNodeAsRef ## returns the ref as is
+    joptJsonNodeAsCopy ## returns a deep copy of the JsonNode
+    joptJsonNodeAsObject ## treats JsonNode as a regular ref object
   ToJsonOptions* = object
     enumMode*: EnumMode
-    # xxx charMode
+    jsonNodeMode*: JsonNodeMode
+    # xxx charMode, etc
 
 proc initToJsonOptions*(): ToJsonOptions =
   ## initializes `ToJsonOptions` with sane options.
-  ToJsonOptions(enumMode: joptEnumOrd)
+  ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef)
 
 proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
 proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".}
@@ -286,8 +291,15 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode =
       result = newJArray()
       for v in a.fields: result.add toJson(v, opt)
   elif T is ref | ptr:
-    if system.`==`(a, nil): result = newJNull()
-    else: result = toJson(a[], opt)
+    template impl =
+      if system.`==`(a, nil): result = newJNull()
+      else: result = toJson(a[], opt)
+    when T is JsonNode:
+      case opt.jsonNodeMode
+      of joptJsonNodeAsRef: result = a
+      of joptJsonNodeAsCopy: result = copy(a)
+      of joptJsonNodeAsObject: impl()
+    else: impl()
   elif T is array | seq | set:
     result = newJArray()
     for ai in a: result.add toJson(ai, opt)
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 62486b896..88c05facf 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -91,6 +91,28 @@ template fn() =
     doAssert b2.ord == 1 # explains the `1`
     testRoundtrip(a): """[1,2,3]"""
 
+  block: # JsonNode
+    let a = ((1, 2.5, "abc").toJson, (3, 4.5, "foo"))
+    testRoundtripVal(a): """[[1,2.5,"abc"],[3,4.5,"foo"]]"""
+
+    block:
+      template toInt(a): untyped = cast[int](a)
+
+      let a = 3.toJson
+      let b = (a, a)
+
+      let c1 = b.toJson
+      doAssert c1[0].toInt == a.toInt
+      doAssert c1[1].toInt == a.toInt
+
+      let c2 = b.toJson(ToJsonOptions(jsonNodeMode: joptJsonNodeAsCopy))
+      doAssert c2[0].toInt != a.toInt
+      doAssert c2[1].toInt != c2[0].toInt
+      doAssert c2[1] == c2[0]
+
+      let c3 = b.toJson(ToJsonOptions(jsonNodeMode: joptJsonNodeAsObject))
+      doAssert $c3 == """[{"isUnquoted":false,"kind":2,"num":3},{"isUnquoted":false,"kind":2,"num":3}]"""
+
   block: # ToJsonOptions
     let a = (me1, me2)
     doAssert $a.toJson() == "[1,2]"

From 06d50bfd4cac6611c5fc775c8130cad87e4dcee1 Mon Sep 17 00:00:00 2001
From: Dylan Modesitt 
Date: Mon, 31 May 2021 16:51:32 -0400
Subject: [PATCH 0044/1654] Fixes #5034 illformed AST from getImpl with proc
 returning value (#17976)

* Fixes 5034

* address comments
---
 compiler/semstmts.nim          | 25 +++++++++++++++++++++----
 tests/macros/tmacrogetimpl.nim | 31 +++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 tests/macros/tmacrogetimpl.nim

diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index b4630294b..085146f7f 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1439,16 +1439,33 @@ proc semBorrow(c: PContext, n: PNode, s: PSym) =
   else:
     localError(c.config, n.info, errNoSymbolToBorrowFromFound)
 
+proc swapResult(n: PNode, sRes: PSym, dNode: PNode) =
+  ## Swap nodes that are (skResult) symbols to d(estination)Node.
+  for i in 0.. resultPos and n[resultPos] != nil:
-      if n[resultPos].sym.kind != skResult or n[resultPos].sym.owner != getCurrOwner(c):
+      if n[resultPos].sym.kind != skResult:
         localError(c.config, n.info, "incorrect result proc symbol")
+      if n[resultPos].sym.owner != getCurrOwner(c):
+        # re-write result with new ownership, and re-write the proc accordingly
+        let sResSym = n[resultPos].sym
+        genResSym(s)
+        n[resultPos] = newSymNode(s)
+        swapResult(n, sResSym, n[resultPos])
       c.p.resultSym = n[resultPos].sym
     else:
-      var s = newSym(skResult, getIdent(c.cache, "result"), nextSymId c.idgen, getCurrOwner(c), n.info)
-      s.typ = t
-      incl(s.flags, sfUsed)
+      genResSym(s)
       c.p.resultSym = s
       n.add newSymNode(c.p.resultSym)
     addParamOrResult(c, c.p.resultSym, owner)
diff --git a/tests/macros/tmacrogetimpl.nim b/tests/macros/tmacrogetimpl.nim
new file mode 100644
index 000000000..1d996ff29
--- /dev/null
+++ b/tests/macros/tmacrogetimpl.nim
@@ -0,0 +1,31 @@
+import macros
+
+# bug #5034
+
+macro copyImpl(srsProc: typed, toSym: untyped) =
+  result = copyNimTree(getImplTransformed(srsProc))
+  result[0] = ident $toSym.toStrLit()
+
+proc foo1(x: float, one: bool = true): float =
+  if one:
+    return 1'f
+  result = x
+
+proc bar1(what: string): string =
+  ## be a little more adversarial with `skResult`
+  proc buzz: string =
+    result = "lightyear"
+  if what == "buzz":
+    result = "buzz " & buzz()
+  else:
+    result = what
+  return result
+
+copyImpl(foo1, foo2)
+doAssert foo1(1'f) == 1.0
+doAssert foo2(10.0, false) == 10.0
+doAssert foo2(10.0) == 1.0
+
+copyImpl(bar1, bar2)
+doAssert bar1("buzz") == "buzz lightyear"
+doAssert bar1("macros") == "macros"

From c0e8199acc9f49caa36bae34d103600d112b66f3 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 1 Jun 2021 05:28:22 +0800
Subject: [PATCH 0045/1654] [std/re] fix findBounds and find procs (#18028)

* [std/re] make interface consistent

* tiny

* revert
---
 lib/impure/re.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 0c96876b9..40860cc88 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -171,7 +171,7 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string],
 
 proc findBounds*(buf: cstring, pattern: Regex,
                  matches: var openArray[tuple[first, last: int]],
-                 start = 0, bufSize = 0): tuple[first, last: int] =
+                 start = 0, bufSize: int): tuple[first, last: int] =
   ## returns the starting position and end position of `pattern` in `buf`
   ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated),
   ## and the captured substrings in the array `matches`.
@@ -290,7 +290,7 @@ proc match*(buf: cstring, pattern: Regex, matches: var openArray[string],
   result = matchLen(buf, pattern, matches, start, bufSize) != -1
 
 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string],
-           start = 0, bufSize = 0): int =
+           start = 0, bufSize: int): int =
   ## returns the starting position of `pattern` in `buf` and the captured
   ## substrings in the array `matches`. If it does not match, nothing
   ## is written into `matches` and `-1` is returned.

From c2e3dc0ed16a854981769b426e9e327663447df7 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 2 Jun 2021 00:16:25 +0800
Subject: [PATCH 0046/1654] close #18129 Add setCurrentException for JS backend
 (#18145)

* [std/re] make interface consistent

* tiny

* revert

* close #18129  add setCurrentException

* changelog entry
---
 changelog.md                       | 2 +-
 lib/system/jssys.nim               | 3 +++
 tests/exception/tsetexceptions.nim | 8 ++++++++
 3 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 tests/exception/tsetexceptions.nim

diff --git a/changelog.md b/changelog.md
index 9806a6011..900b23ad8 100644
--- a/changelog.md
+++ b/changelog.md
@@ -329,7 +329,7 @@
 
 - Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
-
+- Added setCurrentException for JS backend.
 
 ## Language changes
 
diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim
index ef06437e5..371cb7962 100644
--- a/lib/system/jssys.nim
+++ b/lib/system/jssys.nim
@@ -69,6 +69,9 @@ proc getCurrentExceptionMsg*(): string =
         return $msg
   return ""
 
+proc setCurrentException*(exc: ref Exception) =
+  lastJSError = cast[PJSError](exc)
+
 proc auxWriteStackTrace(f: PCallFrame): string =
   type
     TempFrame = tuple[procname: cstring, line: int, filename: cstring]
diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim
new file mode 100644
index 000000000..557fc1898
--- /dev/null
+++ b/tests/exception/tsetexceptions.nim
@@ -0,0 +1,8 @@
+discard """
+  targets: "c cpp js"
+"""
+
+let ex = newException(CatchableError, "test")
+setCurrentException(ex)
+doAssert getCurrentException().msg == ex.msg
+doAssert getCurrentExceptionMsg() == ex.msg

From ba3ec7b04954a9a60adb6e1a64405f7644f799ee Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Tue, 1 Jun 2021 21:47:23 +0300
Subject: [PATCH 0047/1654] docs: Latex generation improvements (#18141)

* docs: improve Latex generation

* make it work on Windows + fix ] escaping

* minor fixes with escapes and style
---
 compiler/docgen.nim              |   2 +-
 config/nimdoc.tex.cfg            | 202 ++++++++++++++++++++++---------
 lib/packages/docutils/rstast.nim |  18 ++-
 lib/packages/docutils/rstgen.nim | 140 +++++++++++----------
 tests/stdlib/trstgen.nim         |   4 +-
 tools/kochdocs.nim               |  14 +--
 6 files changed, 252 insertions(+), 128 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index e07db9188..18c568ba0 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -549,7 +549,7 @@ proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var string,
         var msg = "Example:"
         if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
         dispA(d.conf, dest, "\n
$1
\n",
-            "\n\\textbf{$1}\n", [msg])
+            "\n\n\\textbf{$1}\n", [msg])
         inc d.listingCounter
         let id = $d.listingCounter
         dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""])
diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg
index b19b25678..75382ce26 100644
--- a/config/nimdoc.tex.cfg
+++ b/config/nimdoc.tex.cfg
@@ -8,7 +8,8 @@ split.item.toc = "20"
 # after this number of characters
 
 doc.section = """
-\rsthA{$sectionTitle}\label{$sectionID}
+\rsthA[$sectionTitle]{$sectionTitle}\label{$sectionID}
+
 $content
 """
 
@@ -17,12 +18,16 @@ doc.section.toc = ""
 
 doc.item = """
 
+\vspace{1em}
 \phantomsection\addcontentsline{toc}{subsection}{$uniqueName}
 
-\begin{rstpre}
+\begin{rstdocitem}
 $header
-\end{rstpre}
+\end{rstdocitem}
+
+\begin{addmargin}[0.05\linewidth]{0pt}
 $desc
+\end{addmargin}
 """
 
 doc.item.toc = ""
@@ -42,46 +47,118 @@ $content
 
 # $1 - number of listing in document, $2 - language (e.g. langNim), $3 - anchor
 doc.listing_start = "\\begin{rstpre}\n"
-doc.listing_end = "\n\\end{rstpre}\n"
+doc.listing_end = "\n\\end{rstpre}\n\n"
 
 doc.file = """
 % This file was generated by Nim.
 % Generated: $date $time UTC
-\documentclass[a4paper]{article}
-\usepackage[left=2cm,right=3cm,top=3cm,bottom=3cm]{geometry}
-\usepackage[utf8]{inputenc}
-\usepackage[T1]{fontenc}
+%
+% Compile it by:   xelatex    (up to 3 times to get labels generated)
+%                  -------
+%
+\documentclass[a4paper,11pt]{article}
+\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry}
+
+%   for 2-sided printing with larger inner "binding" margin
+%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry}
+%   for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080)
+%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+%   for e-readers with 1.45:1 aspect ratio (e.g. 1200x825)
+%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+%   for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404)
+%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry}
+
+\usepackage{fontspec}
+% logic to select default font with some fall-back fonts.
+\IfFontExistsTF{Times New Roman}{%
+  \setmainfont{Times New Roman}  % the default font
+  \typeout{========================================= nim: using Times New Roman}
+}{
+  \IfFontExistsTF{FreeSerif}{%
+    \setmainfont{FreeSerif}  % fallback #1 - official GNU font, resembles Times
+    \typeout{========================================= nim: using FreeSerif}
+  }{
+    \IfFontExistsTF{DejaVuSerif}{%
+      \setmainfont{DejaVuSerif}  % fallback #2 - very widespread free font
+      \typeout{========================================= nim: using DejaVuSerif}
+    }{
+      \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!}
+    }
+  }
+}
+
+% default monospace font for code:
+\usepackage{GoMono}
+\usepackage{relsize}
+% make this monospace font 2 steps smaller to hold 80-character line
+\newcommand{\rstverbblockfont}{\smaller[2]}
+\newcommand{\rstverbinlinefont}{\smaller}
+
+\usepackage{parskip}  % paragraphs delimited by vertical space, no indent
 \usepackage{graphicx}
-\usepackage{lmodern}
-\usepackage{fancyvrb, courier}
-\usepackage{tabularx}
-\usepackage{hyperref}
-\usepackage{enumitem}  % for option list, enumList, and rstfootnote
+
+\usepackage{dingbat} % for \carriagereturn, etc
+\usepackage{fvextra}  % for code blocks (works better than original fancyvrb)
+\fvset{
+  breaklines,
+  breakafter={=}:|\_\{\}[](){,}.;+-*/'",
+  breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, 
+  breaksymbolright=\color{red}{\small\carriagereturn}
+}
+\fvinlineset{%
+   breaklines,
+   breakafter={=}:|\_\{\}[](){,}.;+-*/'",
+     % that does not work at all when we underline inline code by ulem :-(
+   commandchars=\\\{\}
+}
+
+\usepackage{scrextend}  % for the `addmargin` environment
 
 \usepackage{xcolor}
-\usepackage[tikz]{mdframed}
-\usetikzlibrary{shadows}
-\mdfsetup{%
-linewidth=3,
-topline=false,
-rightline=false,
-bottomline=false}
+\usepackage[urlbordercolor=blue,linkbordercolor=cyan,
+            pdfborderstyle={/S/U/W 1}]{hyperref}
+\usepackage{enumitem}  % for option list, enumList, and rstfootnote
 
-\begin{document}
-\title{$title $version $subtitle}
-\author{$author}
+\usepackage[most]{tcolorbox}  % boxes around admonitions, code blocks, doc.item
 
-\tolerance 1414 
-\hbadness 1414 
-\emergencystretch 1.5em 
-\hfuzz 0.3pt 
-\widowpenalty=10000 
-\vfuzz \hfuzz 
-\raggedbottom 
+\newtcolorbox{rstadmonition}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     before upper=\indent, parbox=false, #1}
+
+\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6}
+
+\newtcolorbox{rstprebox}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     borderline ={0.1em}{0pt}{rstframecolor},
+     before upper=\indent, parbox=false, #1}
+
+\newenvironment{rstpre}{%
+\VerbatimEnvironment\begingroup\begin{rstprebox}%
+\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}%
+{\end{Verbatim}\end{rstprebox}\endgroup}
+
+\newtcolorbox{rstdocitembox}[1][]{blanker, breakable,
+     left=3mm, right=3mm, top=1mm, bottom=1mm,
+     borderline ={1pt}{0pt}{cyan},
+     before upper=\indent, parbox=false, #1}
+
+% Inline code formatting: grey underline,
+% use \Verb from fvextras e.g. to display -- correctly as double -
+\usepackage[normalem]{ulem}
+\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon}
+
+\newcommand{\rstcode}[1]{%
+{\rstverbinlinefont\Verb{\rstuline{#1}}}%
+}
+
+\newcommand{\rstcodeitem}[1]{\Verb{#1}}
+
+\newenvironment{rstdocitem}{%
+\VerbatimEnvironment\begingroup\begin{rstdocitembox}%
+\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}%
+{\end{Verbatim}\end{rstdocitembox}\endgroup}
 
-\maketitle
 
-\newenvironment{rstpre}{\VerbatimEnvironment\begingroup\begin{Verbatim}[fontsize=\footnotesize , commandchars=\\\{\}]}{\end{Verbatim}\endgroup}
 \newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}}
 \ifdim\linewidth<30em
   \def\rstoptleftmargin{0.4\linewidth}
@@ -93,37 +170,41 @@ bottomline=false}
 \newenvironment{rstoptlist}{%
 \begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}}
 
-% to pack tabularx into a new environment, special syntax is needed :-(
-\newenvironment{rsttab}[1]{\tabularx{\linewidth}{#1}}{\endtabularx}
+\usepackage{tabulary}  % tables with adjustable cell width and no overflow
+% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088)
+\tymin=60pt
+\tymax=\maxdimen
+% to pack tabulary into a new environment, special syntax is needed :-(
+\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary}
 
 \newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}}
 \newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}}
 
-\newcommand{\rsthA}[1]{\section{#1}}
-\newcommand{\rsthB}[1]{\subsection{#1}}
-\newcommand{\rsthC}[1]{\subsubsection{#1}}
-\newcommand{\rsthD}[1]{\paragraph{#1}}
-\newcommand{\rsthE}[1]{\paragraph{#1}}
+\newcommand{\rsthA}[2][]{\section[#1]{#2}}
+\newcommand{\rsthB}[2][]{\subsection[#1]{#2}}
+\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}}
+\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}}
+\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}}
 
-\newcommand{\rstovA}[1]{\section*{#1}}
-\newcommand{\rstovB}[1]{\subsection*{#1}}
-\newcommand{\rstovC}[1]{\subsubsection*{#1}}
-\newcommand{\rstovD}[1]{\paragraph*{#1}}
-\newcommand{\rstovE}[1]{\paragraph*{#1}}
+\newcommand{\rstovA}[2][]{\section*[#1]{#2}}
+\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}}
+\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}}
+\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}}
+\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}}
 
 % Syntax highlighting:
-\newcommand{\spanDecNumber}[1]{#1}
-\newcommand{\spanBinNumber}[1]{#1}
-\newcommand{\spanHexNumber}[1]{#1}
-\newcommand{\spanOctNumber}[1]{#1}
-\newcommand{\spanFloatNumber}[1]{#1}
+\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}}
 \newcommand{\spanIdentifier}[1]{#1}
 \newcommand{\spanKeyword}[1]{\textbf{#1}}
-\newcommand{\spanStringLit}[1]{#1}
-\newcommand{\spanLongStringLit}[1]{#1}
+\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}}
+\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}}
 \newcommand{\spanCharLit}[1]{#1}
 \newcommand{\spanEscapeSequence}[1]{#1}
-\newcommand{\spanOperator}[1]{#1}
+\newcommand{\spanOperator}[1]{\textbf{#1}}
 \newcommand{\spanPunctuation}[1]{#1}
 \newcommand{\spanComment}[1]{\emph{#1}}
 \newcommand{\spanLongComment}[1]{\emph{#1}}
@@ -132,7 +213,7 @@ bottomline=false}
 \newcommand{\spanTagEnd}[1]{#1}
 \newcommand{\spanKey}[1]{#1}
 \newcommand{\spanValue}[1]{#1}
-\newcommand{\spanRawData}[1]{#1}
+\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}}
 \newcommand{\spanAssembler}[1]{#1}
 \newcommand{\spanPreprocessor}[1]{#1}
 \newcommand{\spanDirective}[1]{#1}
@@ -142,11 +223,20 @@ bottomline=false}
 \newcommand{\spanLabel}[1]{#1}
 \newcommand{\spanReference}[1]{#1}
 \newcommand{\spanOther}[1]{#1}
-\newcommand{\spantok}[1]{\frame{#1}}
+\newcommand{\spantok}[1]{\fbox{#1}}
 \newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}}
-\newcommand{\spanProgramOutput}[1]{\textcolor{gray}{\textbf{#1}}}
+\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}}
 \newcommand{\spanprogram}[1]{\textbf{\underline{#1}}}
-\newcommand{\spanoption}[1]{\textbf{#1}}
+\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}}
+
+\begin{document}
+\title{$title $version $subtitle}
+\author{$author}
+
+% Never allow text overflow to margin:
+\setlength\emergencystretch{\hsize}\hbadness=10000
+
+\maketitle
 
 $content
 \end{document}
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 81e3ba6d9..2489ce40c 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -349,8 +349,24 @@ proc renderRstToJson*(node: PRstNode): string =
   ##   }
   renderRstToJsonNode(node).pretty
 
+proc renderRstToText*(node: PRstNode): string =
+  ## minimal text representation of markup node
+  const code = {rnCodeFragment, rnInterpretedText, rnInlineLiteral, rnInlineCode}
+  if node == nil:
+    return ""
+  case node.kind
+  of rnLeaf, rnSmiley:
+    result.add node.text
+  else:
+    if node.kind in code: result.add "`"
+    for i in 0 ..< node.sons.len:
+      if node.kind in {rnInlineCode, rnCodeBlock} and i == 0:
+        continue  # omit language specifier
+      result.add renderRstToText(node.sons[i])
+    if node.kind in code: result.add "`"
+
 proc renderRstToStr*(node: PRstNode, indent=0): string =
-  ## Writes the parsed RST `node` into a compact string
+  ## Writes the parsed RST `node` into an AST tree with compact string
   ## representation in the format (one line per every sub-node):
   ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)``
   ## (suitable for debugging of RST parsing).
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index 5ab6f28ee..e3f889283 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -60,6 +60,10 @@ type
   MetaEnum* = enum
     metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion
 
+  EscapeMode = enum  # in Latex text inside options [] and URLs is
+                     # escaped slightly differently than in normal text
+    emText, emOption, emUrl  # emText is currently used for code also
+
   RstGenerator* = object of RootObj
     target*: OutputTarget
     config*: StringTableRef
@@ -84,6 +88,7 @@ type
     id*: int               ## A counter useful for generating IDs.
     onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int;
                           content: string)
+    escMode*: EscapeMode
 
   PDoc = var RstGenerator ## Alias to type less.
 
@@ -161,6 +166,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   g.findFile = findFile
   g.currentSection = ""
   g.id = 0
+  g.escMode = emText
   let fileParts = filename.splitFile
   if fileParts.ext == ".nim":
     g.currentSection = "Module " & fileParts.name
@@ -189,28 +195,34 @@ proc addHtmlChar(dest: var string, c: char) =
   of '\"': add(dest, """)
   else: add(dest, c)
 
-proc addRtfChar(dest: var string, c: char) =
+proc addTexChar(dest: var string, c: char, escMode: EscapeMode) =
+  ## Escapes 10 special Latex characters and sometimes ` and [, ].
+  ## TODO: @ is always a normal symbol (besides the header), am I wrong?
+  ## All escapes that need to work in text and code blocks (`emText` mode)
+  ## should start from \ (to be compatible with fancyvrb/fvextra).
   case c
-  of '{': add(dest, "\\{")
-  of '}': add(dest, "\\}")
-  of '\\': add(dest, "\\\\")
-  else: add(dest, c)
-
-proc addTexChar(dest: var string, c: char) =
-  # Escapes 10 special Latex characters. Note that [, ], and ` are not
-  # considered as such. TODO: neither is @, am I wrong?
-  case c
-  of '_', '{', '}', '$', '&', '#', '%': add(dest, "\\" & c)
+  of '_', '$', '&', '#', '%': add(dest, "\\" & c)
   # \~ and \^ have a special meaning unless they are followed by {}
   of '~', '^': add(dest, "\\" & c & "{}")
+  # Latex loves to substitute ` to opening quote, even in texttt mode!
+  of '`': add(dest, "\\textasciigrave{}")
   # add {} to avoid gobbling up space by \textbackslash
   of '\\': add(dest, "\\textbackslash{}")
+  # Using { and } in URL in Latex: https://tex.stackexchange.com/a/469175
+  of '{':
+    add(dest, if escMode == emUrl: "\\%7B" else: "\\{")
+  of '}':
+    add(dest, if escMode == emUrl: "\\%7D" else: "\\}")
+  of ']':
+    # escape ] inside an optional argument in e.g. \section[static[T]]{..
+    add(dest, if escMode == emOption: "\\text{]}" else: "]")
   else: add(dest, c)
 
-proc escChar*(target: OutputTarget, dest: var string, c: char) {.inline.} =
+proc escChar*(target: OutputTarget, dest: var string,
+              c: char, escMode: EscapeMode) {.inline.} =
   case target
   of outHtml:  addHtmlChar(dest, c)
-  of outLatex: addTexChar(dest, c)
+  of outLatex: addTexChar(dest, c, escMode)
 
 proc addSplitter(target: OutputTarget; dest: var string) {.inline.} =
   case target
@@ -229,7 +241,7 @@ proc nextSplitPoint*(s: string, start: int): int =
     inc(result)
   dec(result)                 # last valid index
 
-proc esc*(target: OutputTarget, s: string, splitAfter = -1): string =
+proc esc*(target: OutputTarget, s: string, splitAfter = -1, escMode = emText): string =
   ## Escapes the HTML.
   result = ""
   if splitAfter >= 0:
@@ -240,11 +252,11 @@ proc esc*(target: OutputTarget, s: string, splitAfter = -1): string =
       #if (splitter != " ") or (partLen + k - j + 1 > splitAfter):
       partLen = 0
       addSplitter(target, result)
-      for i in countup(j, k): escChar(target, result, s[i])
+      for i in countup(j, k): escChar(target, result, s[i], escMode)
       inc(partLen, k - j + 1)
       j = k + 1
   else:
-    for i in countup(0, len(s) - 1): escChar(target, result, s[i])
+    for i in countup(0, len(s) - 1): escChar(target, result, s[i], escMode)
 
 
 proc disp(target: OutputTarget, xml, tex: string): string =
@@ -760,6 +772,8 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
       sectionPrefix = rstnodeToRefname(n2) & "-"
       break
   var refname = sectionPrefix & rstnodeToRefname(n)
+  var tocName = esc(d.target, renderRstToText(n), escMode = emOption)
+    # for Latex: simple text without commands that may break TOC/hyperref
   if d.hasToc:
     var length = len(d.tocPart)
     setLen(d.tocPart, length + 1)
@@ -768,13 +782,14 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) =
     d.tocPart[length].header = tmp
 
     dispA(d.target, result, "\n$3", "\\rsth$4{$3}$2\n",
-      [$n.level, refname.idS, tmp, $chr(n.level - 1 + ord('A')), refname])
+      "$2 href=\"#$5\">$3", "\\rsth$4[$6]{$3}$2\n",
+      [$n.level, refname.idS, tmp,
+       $chr(n.level - 1 + ord('A')), refname, tocName])
   else:
     dispA(d.target, result, "\n$3",
-                            "\\rsth$4{$3}$2\n", [
+                            "\\rsth$4[$5]{$3}$2\n", [
         $n.level, refname.idS, tmp,
-        $chr(n.level - 1 + ord('A'))])
+        $chr(n.level - 1 + ord('A')), tocName])
 
   # Generate index entry using spaces to indicate TOC level for the output HTML.
   assert n.level >= 0
@@ -802,9 +817,10 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) =
     var tmp = ""
     for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp)
     d.currentSection = tmp
+    var tocName = esc(d.target, renderRstToText(n), escMode=emOption)
     dispA(d.target, result, "
$3
",
-                   "\\rstov$4{$3}$2\n", [$n.level,
-        rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A'))])
+                   "\\rstov$4[$5]{$3}$2\n", [$n.level,
+        rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A')), tocName])
 
 
 proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) =
@@ -870,7 +886,7 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) =
     htmlOut = ""
 
   # support for `:target:` links for images:
-  var target = esc(d.target, getFieldValue(n, "target").strip())
+  var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl)
   if target.len > 0:
     # `htmlOut` needs to be of the following format for link to work for images:
     # 
@@ -1031,16 +1047,16 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
       blockEnd = ""
   of outLatex:
     if n.kind == rnCodeBlock:
-      blockStart = "\n\n\\begin{rstpre}" & n.anchor.idS & "\n"
-      blockEnd = "\n\\end{rstpre}\n"
+      blockStart = "\n\n" & n.anchor.idS & "\\begin{rstpre}\n"
+      blockEnd = "\n\\end{rstpre}\n\n"
     else:  # rnInlineCode
-      blockStart = "\\texttt{"
+      blockStart = "\\rstcode{"
       blockEnd = "}"
   dispA(d.target, result, blockStart, blockStart, [])
   if params.lang == langNone:
     if len(params.langStr) > 0:
       d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
-    for letter in m.text: escChar(d.target, result, letter)
+    for letter in m.text: escChar(d.target, result, letter, emText)
   else:
     renderCodeLang(result, params.lang, m.text, d.target)
   dispA(d.target, result, blockEnd, blockEnd)
@@ -1055,9 +1071,8 @@ proc renderContainer(d: PDoc, n: PRstNode, result: var string) =
     dispA(d.target, result, "
$2
", "$2", [arg, tmp])
 
 proc texColumns(n: PRstNode): string =
-  result = ""
   let nColumns = if n.sons.len > 0: len(n.sons[0]) else: 1
-  for i in countup(1, nColumns): add(result, "|X")
+  result = "L".repeat(nColumns)
 
 proc renderField(d: PDoc, n: PRstNode, result: var string) =
   var b = false
@@ -1144,19 +1159,38 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) =
   renderAux(d, n,
       htmlHead & "" & txt &
         ":\n" & "$1\n",
-      "\n\n\\begin{mdframed}[linecolor=" & texColor & "]$2\n" &
+      "\n\n\\begin{rstadmonition}[borderline west={0.2em}{0pt}{" &
+        texColor & "}]$2\n" &
         "{" & texSz & "\\color{" & texColor & "}{\\textbf{" & txt & ":}}} " &
-        "$1\n\\end{mdframed}\n",
+        "$1\n\\end{rstadmonition}\n",
       result)
 
+proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external: bool) =
+  var linkStr = ""
+  block:
+    let mode = d.escMode
+    d.escMode = emUrl
+    renderRstToOut(d, link, linkStr)
+    d.escMode = mode
+  var textStr = ""
+  renderRstToOut(d, text, textStr)
+  if external:
+    dispA(d.target, result,
+      "$1",
+      "\\href{$2}{$1}", [textStr, linkStr])
+  else:
+    dispA(d.target, result,
+      "$1",
+      "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr])
+
 proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
   if n == nil: return
   case n.kind
   of rnInner: renderAux(d, n, result)
   of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, result)
   of rnOverline: renderOverline(d, n, result)
-  of rnTransition: renderAux(d, n, "\n", "\\hrule$2\n", result)
-  of rnParagraph: renderAux(d, n, "$1
\n", "$2\n$1\n\n", result)
+  of rnTransition: renderAux(d, n, "\n", "\n\n\\vspace{0.6em}\\hrule$2\n", result)
+  of rnParagraph: renderAux(d, n, "$1
\n", "\n\n$2\n$1\n\n", result)
   of rnBulletList:
     renderAux(d, n, "$1\n",
                     "\\begin{itemize}\n$2\n$1\\end{itemize}\n", result)
@@ -1202,7 +1236,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
     renderAux(d, n,
         "
" &
         "$1
",
-        "\\item[$1]", result)
+        "\\item[\\rstcodeitem{\\spanoption{$1}}]", result)
   of rnDescription:
     renderAux(d, n, "
$1
",
         " $1\n", result)
@@ -1210,7 +1244,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
     doAssert false, "renderRstToOut"
   of rnLiteralBlock:
     renderAux(d, n, "$1
\n", - "\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n", result) + "\n\n\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n\n", result) of rnQuotedLiteralBlock: doAssert false, "renderRstToOut" of rnLineBlock: @@ -1237,8 +1271,8 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTable, rnGridTable, rnMarkdownTable: renderAux(d, n, "$1", - "\\begin{table}\n$2\n\\begin{rsttab}{" & - texColumns(n) & "|}\n\\hline\n$1\\end{rsttab}\\end{table}", result) + "\n$2\n\\begin{rsttab}{" & + texColumns(n) & "}\n\\hline\n$1\\end{rsttab}", result) of rnTableRow: if len(n) >= 1: if d.target == outLatex: @@ -1275,21 +1309,13 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "\\item[\\textsuperscript{[$3]}]$2 $1\n", [body, n.anchor.idS, mark, n.anchor]) of rnRef: - var tmp = "" - renderAux(d, n, tmp) - dispA(d.target, result, - "$1", - "$1\\ref{$2}", [tmp, rstnodeToRefname(n)]) + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=false) of rnStandaloneHyperlink: - renderAux(d, n, - "$1", - "\\href{$1}{$1}", result) + renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) of rnInternalRef: - var tmp = "" - renderAux(d, n.sons[0], tmp) - dispA(d.target, result, - "$1", - "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [tmp, n.sons[1].text]) + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnHyperlink: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) of rnFootnoteRef: var tmp = "[" renderAux(d, n.sons[0], tmp) @@ -1299,14 +1325,6 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "$1", "\\textsuperscript{\\hyperlink{$2}{\\textbf{$1}}}", [tmp, n.sons[1].text]) - of rnHyperlink: - var tmp0 = "" - var tmp1 = "" - renderRstToOut(d, n.sons[0], tmp0) - renderRstToOut(d, n.sons[1], tmp1) - dispA(d.target, result, - "$1", - "\\href{$2}{$1}", [tmp0, tmp1]) of rnDirArg, rnRaw: renderAux(d, n, result) of rnRawHtml: if d.target != outLatex and not lastSon(n).isNil: @@ -1334,7 +1352,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = dispA(d.target, result, "" & "$1", - "\\texttt{\\span$2{$1}}", [tmp0, class]) + "\\rstcode{\\span$2{$1}}", [tmp0, class]) else: # rnUnknownRole, not necessarily code/monospace font dispA(d.target, result, "$1", "\\span$2{$1}", [tmp0, class]) @@ -1351,7 +1369,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnInlineLiteral, rnInterpretedText: renderAux(d, n, "$1", - "\\texttt{$1}", result) + "\\rstcode{$1}", result) of rnInlineTarget: var tmp = "" renderAux(d, n, tmp) @@ -1360,7 +1378,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = "\\label{$2}\\hypertarget{$2}{$1}", [tmp, rstnodeToRefname(n)]) of rnSmiley: renderSmiley(d, n, result) - of rnLeaf: result.add(esc(d.target, n.text)) + of rnLeaf: result.add(esc(d.target, n.text, escMode=d.escMode)) of rnContents: d.hasToc = true of rnDefaultRole: discard of rnTitle: diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 864728686..34b172935 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -259,7 +259,7 @@ A2 A3 A4 A5 ==== === """ let output1 = rstToLatex(input1, {}) - doAssert "{|X|X|}" in output1 # 2 columns + doAssert "{LL}" in output1 # 2 columns doAssert count(output1, "\\\\") == 4 # 4 rows for cell in ["H0", "H1", "A0", "A1", "A2", "A3", "A4", "A5"]: doAssert cell in output1 @@ -274,7 +274,7 @@ A0 A1 X Ax Y ==== === = """ let output2 = rstToLatex(input2, {}) - doAssert "{|X|X|X|}" in output2 # 3 columns + doAssert "{LLL}" in output2 # 3 columns doAssert count(output2, "\\\\") == 2 # 2 rows for cell in ["H0", "H1", "H", "A0", "A1", "X", "Ax", "Y"]: doAssert cell in output2 diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 621dc643f..f25564fad 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -284,22 +284,22 @@ proc buildDoc(nimArgs, destPath: string) = proc nim2pdf(src: string, dst: string, nimArgs: string) = # xxx expose as a `nim` command or in some other reusable way. - let outDir = "build" / "pdflatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex exec("$# rst2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) let texFile = outDir / src.lastPathPart.changeFileExt("tex") - for i in 0..<2: # call LaTeX twice to get cross references right: - let pdflatexLog = outDir / "pdflatex.log" + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" # `>` should work on windows, if not, we can use `execCmdEx` - let cmd = "pdflatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, pdflatexLog.quoteShell] - exec(cmd) # on error, user can inspect `pdflatexLog` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` moveFile(texFile.changeFileExt("pdf"), dst) proc buildPdfDoc*(nimArgs, destPath: string) = var pdfList: seq[string] createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - doAssert false, "pdflatex not found" # or, raise an exception + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception else: for src in items(rstPdfList): let dst = destPath / src.lastPathPart.changeFileExt("pdf") From 63db2b19bf0787db89ce89c8861e976d44e33cbd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 1 Jun 2021 22:29:53 +0200 Subject: [PATCH 0048/1654] use dragonbox algorithm; alternative to #18008 (#18139) * use dragonbox algorithm; alternative to #18008 * removed unsafe code --- changelog.md | 4 + compiler/condsyms.nim | 2 + lib/system/dragonbox.nim | 1347 ++++++++++++++++++++++++++++++++++++ lib/system/formatfloat.nim | 97 +-- tests/float/tfloat6.nim | 8 +- 5 files changed, 1408 insertions(+), 50 deletions(-) create mode 100644 lib/system/dragonbox.nim diff --git a/changelog.md b/changelog.md index 900b23ad8..a23aedbdf 100644 --- a/changelog.md +++ b/changelog.md @@ -76,6 +76,10 @@ - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json. +- `system.addFloat` now uses the "Dragonbox" algorithm, which ensures correct roundtrips of floating point + numbers, that the minimum length representation of a floating point number is used and correct rounding. + Use `-d:nimLegacyAddFloat` for a transition period. + - `strformat` is now part of `include std/prelude`. - Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 6a49584c8..ff0c603cb 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -134,3 +134,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasUnifiedTuple") defineSymbol("nimHasIterable") defineSymbol("nimHasTypeofVoid") + + defineSymbol("nimHasDragonBox") diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim new file mode 100644 index 000000000..db608c630 --- /dev/null +++ b/lib/system/dragonbox.nim @@ -0,0 +1,1347 @@ +## Copyright 2020 Junekey Jeon +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## char* output_end = Dtoa(buffer, value); +## +## Converts the given double-precision number into decimal form and stores the result in the given +## buffer. +## +## The buffer must be large enough, i.e. >= DtoaMinBufferLength. +## The output format is similar to printf("%g"). +## The output is _not_ null-terminted. +## +## The output is optimal, i.e. the output string +## 1. rounds back to the input number when read in (using round-to-nearest-even) +## 2. is as short as possible, +## 3. is as close to the input number as possible. +## +## Note: +## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + +const + dtoaMinBufferLength*: cint = 64 + +## -------------------------------------------------------------------------------------------------- +## This file contains an implementation of Junekey Jeon's Dragonbox algorithm. +## +## It is a simplified version of the reference implementation found here: +## https://github.com/jk-jeon/dragonbox +## +## The reference implementation also works with single-precision floating-point numbers and +## has options to configure the rounding mode. +## -------------------------------------------------------------------------------------------------- + +template dragonbox_Assert*(x: untyped): untyped = + assert(x) + +## ================================================================================================== +## +## ================================================================================================== + +type + ValueType* = float + BitsType* = uint64 + +type + Double* = object + bits*: BitsType + +const ## = p (includes the hidden bit) + significandSize*: int32 = 53 + +const ## static constexpr int32_t MaxExponent = 1024 - 1 - (SignificandSize - 1); + ## static constexpr int32_t MinExponent = std::numeric_limits::min_exponent - 1 - (SignificandSize - 1); + exponentBias*: int32 = 1024 - 1 + (significandSize - 1) + +const + maxIeeeExponent*: BitsType = BitsType(2 * 1024 - 1) + +const ## = 2^(p-1) + hiddenBit*: BitsType = BitsType(1) shl (significandSize - 1) + +const ## = 2^(p-1) - 1 + significandMask*: BitsType = hiddenBit - 1 + +const + exponentMask*: BitsType = maxIeeeExponent shl (significandSize - 1) + +const + signMask*: BitsType = not (not BitsType(0) shr 1) + +proc constructDouble*(bits: BitsType): Double {.constructor.} = + result.bits = bits + +proc constructDouble*(value: ValueType): Double {.constructor.} = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand*(this: Double): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent*(this: Double): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN*(this: Double): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero*(this: Double): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit*(this: Double): int {.noSideEffect.} = + return ord((this.bits and signMask) != 0) + +## namespace +## ================================================================================================== +## +## ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2*(x: int32; n: int32): int32 {.inline.} = + return x shr n + +proc floorLog2Pow10*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1233) + dragonbox_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc floorLog10Pow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611, 22) + +proc floorLog10ThreeQuartersPow2*(e: int32): int32 {.inline.} = + dragonbox_Assert(e >= -1500) + dragonbox_Assert(e <= 1500) + return floorDivPow2(e * 1262611 - 524031, 22) + +## ================================================================================================== +## +## ================================================================================================== + +type + uint64x2* {.bycopy.} = object + hi*: uint64 + lo*: uint64 + + +proc computePow10*(k: int32): uint64x2 {.inline.} = + const + kMin: int32 = -292 + const + kMax: int32 = 326 + const + pow10: array[kMax - kMin + 1, uint64x2] = [ + uint64x2(hi: 0xFF77B1FCBEBCDC4F'u, lo: 0x25E8E89C13BB0F7B'u), + uint64x2(hi: 0x9FAACF3DF73609B1'u, lo: 0x77B191618C54E9AD'u), + uint64x2(hi: 0xC795830D75038C1D'u, lo: 0xD59DF5B9EF6A2418'u), + uint64x2(hi: 0xF97AE3D0D2446F25'u, lo: 0x4B0573286B44AD1E'u), + uint64x2(hi: 0x9BECCE62836AC577'u, lo: 0x4EE367F9430AEC33'u), + uint64x2(hi: 0xC2E801FB244576D5'u, lo: 0x229C41F793CDA740'u), + uint64x2(hi: 0xF3A20279ED56D48A'u, lo: 0x6B43527578C11110'u), + uint64x2(hi: 0x9845418C345644D6'u, lo: 0x830A13896B78AAAA'u), + uint64x2(hi: 0xBE5691EF416BD60C'u, lo: 0x23CC986BC656D554'u), + uint64x2(hi: 0xEDEC366B11C6CB8F'u, lo: 0x2CBFBE86B7EC8AA9'u), + uint64x2(hi: 0x94B3A202EB1C3F39'u, lo: 0x7BF7D71432F3D6AA'u), + uint64x2(hi: 0xB9E08A83A5E34F07'u, lo: 0xDAF5CCD93FB0CC54'u), + uint64x2(hi: 0xE858AD248F5C22C9'u, lo: 0xD1B3400F8F9CFF69'u), + uint64x2(hi: 0x91376C36D99995BE'u, lo: 0x23100809B9C21FA2'u), + uint64x2(hi: 0xB58547448FFFFB2D'u, lo: 0xABD40A0C2832A78B'u), + uint64x2(hi: 0xE2E69915B3FFF9F9'u, lo: 0x16C90C8F323F516D'u), + uint64x2(hi: 0x8DD01FAD907FFC3B'u, lo: 0xAE3DA7D97F6792E4'u), + uint64x2(hi: 0xB1442798F49FFB4A'u, lo: 0x99CD11CFDF41779D'u), + uint64x2(hi: 0xDD95317F31C7FA1D'u, lo: 0x40405643D711D584'u), + uint64x2(hi: 0x8A7D3EEF7F1CFC52'u, lo: 0x482835EA666B2573'u), + uint64x2(hi: 0xAD1C8EAB5EE43B66'u, lo: 0xDA3243650005EED0'u), + uint64x2(hi: 0xD863B256369D4A40'u, lo: 0x90BED43E40076A83'u), + uint64x2(hi: 0x873E4F75E2224E68'u, lo: 0x5A7744A6E804A292'u), + uint64x2(hi: 0xA90DE3535AAAE202'u, lo: 0x711515D0A205CB37'u), + uint64x2(hi: 0xD3515C2831559A83'u, lo: 0x0D5A5B44CA873E04'u), + uint64x2(hi: 0x8412D9991ED58091'u, lo: 0xE858790AFE9486C3'u), + uint64x2(hi: 0xA5178FFF668AE0B6'u, lo: 0x626E974DBE39A873'u), + uint64x2(hi: 0xCE5D73FF402D98E3'u, lo: 0xFB0A3D212DC81290'u), + uint64x2(hi: 0x80FA687F881C7F8E'u, lo: 0x7CE66634BC9D0B9A'u), + uint64x2(hi: 0xA139029F6A239F72'u, lo: 0x1C1FFFC1EBC44E81'u), + uint64x2(hi: 0xC987434744AC874E'u, lo: 0xA327FFB266B56221'u), + uint64x2(hi: 0xFBE9141915D7A922'u, lo: 0x4BF1FF9F0062BAA9'u), + uint64x2(hi: 0x9D71AC8FADA6C9B5'u, lo: 0x6F773FC3603DB4AA'u), + uint64x2(hi: 0xC4CE17B399107C22'u, lo: 0xCB550FB4384D21D4'u), + uint64x2(hi: 0xF6019DA07F549B2B'u, lo: 0x7E2A53A146606A49'u), + uint64x2(hi: 0x99C102844F94E0FB'u, lo: 0x2EDA7444CBFC426E'u), + uint64x2(hi: 0xC0314325637A1939'u, lo: 0xFA911155FEFB5309'u), + uint64x2(hi: 0xF03D93EEBC589F88'u, lo: 0x793555AB7EBA27CB'u), + uint64x2(hi: 0x96267C7535B763B5'u, lo: 0x4BC1558B2F3458DF'u), + uint64x2(hi: 0xBBB01B9283253CA2'u, lo: 0x9EB1AAEDFB016F17'u), + uint64x2(hi: 0xEA9C227723EE8BCB'u, lo: 0x465E15A979C1CADD'u), + uint64x2(hi: 0x92A1958A7675175F'u, lo: 0x0BFACD89EC191ECA'u), + uint64x2(hi: 0xB749FAED14125D36'u, lo: 0xCEF980EC671F667C'u), + uint64x2(hi: 0xE51C79A85916F484'u, lo: 0x82B7E12780E7401B'u), + uint64x2(hi: 0x8F31CC0937AE58D2'u, lo: 0xD1B2ECB8B0908811'u), + uint64x2(hi: 0xB2FE3F0B8599EF07'u, lo: 0x861FA7E6DCB4AA16'u), + uint64x2(hi: 0xDFBDCECE67006AC9'u, lo: 0x67A791E093E1D49B'u), + uint64x2(hi: 0x8BD6A141006042BD'u, lo: 0xE0C8BB2C5C6D24E1'u), + uint64x2(hi: 0xAECC49914078536D'u, lo: 0x58FAE9F773886E19'u), + uint64x2(hi: 0xDA7F5BF590966848'u, lo: 0xAF39A475506A899F'u), + uint64x2(hi: 0x888F99797A5E012D'u, lo: 0x6D8406C952429604'u), + uint64x2(hi: 0xAAB37FD7D8F58178'u, lo: 0xC8E5087BA6D33B84'u), + uint64x2(hi: 0xD5605FCDCF32E1D6'u, lo: 0xFB1E4A9A90880A65'u), + uint64x2(hi: 0x855C3BE0A17FCD26'u, lo: 0x5CF2EEA09A550680'u), + uint64x2(hi: 0xA6B34AD8C9DFC06F'u, lo: 0xF42FAA48C0EA481F'u), + uint64x2(hi: 0xD0601D8EFC57B08B'u, lo: 0xF13B94DAF124DA27'u), + uint64x2(hi: 0x823C12795DB6CE57'u, lo: 0x76C53D08D6B70859'u), + uint64x2(hi: 0xA2CB1717B52481ED'u, lo: 0x54768C4B0C64CA6F'u), + uint64x2(hi: 0xCB7DDCDDA26DA268'u, lo: 0xA9942F5DCF7DFD0A'u), + uint64x2(hi: 0xFE5D54150B090B02'u, lo: 0xD3F93B35435D7C4D'u), + uint64x2(hi: 0x9EFA548D26E5A6E1'u, lo: 0xC47BC5014A1A6DB0'u), + uint64x2(hi: 0xC6B8E9B0709F109A'u, lo: 0x359AB6419CA1091C'u), + uint64x2(hi: 0xF867241C8CC6D4C0'u, lo: 0xC30163D203C94B63'u), + uint64x2(hi: 0x9B407691D7FC44F8'u, lo: 0x79E0DE63425DCF1E'u), + uint64x2(hi: 0xC21094364DFB5636'u, lo: 0x985915FC12F542E5'u), + uint64x2(hi: 0xF294B943E17A2BC4'u, lo: 0x3E6F5B7B17B2939E'u), + uint64x2(hi: 0x979CF3CA6CEC5B5A'u, lo: 0xA705992CEECF9C43'u), + uint64x2(hi: 0xBD8430BD08277231'u, lo: 0x50C6FF782A838354'u), + uint64x2(hi: 0xECE53CEC4A314EBD'u, lo: 0xA4F8BF5635246429'u), + uint64x2(hi: 0x940F4613AE5ED136'u, lo: 0x871B7795E136BE9A'u), + uint64x2(hi: 0xB913179899F68584'u, lo: 0x28E2557B59846E40'u), + uint64x2(hi: 0xE757DD7EC07426E5'u, lo: 0x331AEADA2FE589D0'u), + uint64x2(hi: 0x9096EA6F3848984F'u, lo: 0x3FF0D2C85DEF7622'u), + uint64x2(hi: 0xB4BCA50B065ABE63'u, lo: 0x0FED077A756B53AA'u), + uint64x2(hi: 0xE1EBCE4DC7F16DFB'u, lo: 0xD3E8495912C62895'u), + uint64x2(hi: 0x8D3360F09CF6E4BD'u, lo: 0x64712DD7ABBBD95D'u), + uint64x2(hi: 0xB080392CC4349DEC'u, lo: 0xBD8D794D96AACFB4'u), + uint64x2(hi: 0xDCA04777F541C567'u, lo: 0xECF0D7A0FC5583A1'u), + uint64x2(hi: 0x89E42CAAF9491B60'u, lo: 0xF41686C49DB57245'u), + uint64x2(hi: 0xAC5D37D5B79B6239'u, lo: 0x311C2875C522CED6'u), + uint64x2(hi: 0xD77485CB25823AC7'u, lo: 0x7D633293366B828C'u), + uint64x2(hi: 0x86A8D39EF77164BC'u, lo: 0xAE5DFF9C02033198'u), + uint64x2(hi: 0xA8530886B54DBDEB'u, lo: 0xD9F57F830283FDFD'u), + uint64x2(hi: 0xD267CAA862A12D66'u, lo: 0xD072DF63C324FD7C'u), + uint64x2(hi: 0x8380DEA93DA4BC60'u, lo: 0x4247CB9E59F71E6E'u), + uint64x2(hi: 0xA46116538D0DEB78'u, lo: 0x52D9BE85F074E609'u), + uint64x2(hi: 0xCD795BE870516656'u, lo: 0x67902E276C921F8C'u), + uint64x2(hi: 0x806BD9714632DFF6'u, lo: 0x00BA1CD8A3DB53B7'u), + uint64x2(hi: 0xA086CFCD97BF97F3'u, lo: 0x80E8A40ECCD228A5'u), + uint64x2(hi: 0xC8A883C0FDAF7DF0'u, lo: 0x6122CD128006B2CE'u), + uint64x2(hi: 0xFAD2A4B13D1B5D6C'u, lo: 0x796B805720085F82'u), + uint64x2(hi: 0x9CC3A6EEC6311A63'u, lo: 0xCBE3303674053BB1'u), + uint64x2(hi: 0xC3F490AA77BD60FC'u, lo: 0xBEDBFC4411068A9D'u), + uint64x2(hi: 0xF4F1B4D515ACB93B'u, lo: 0xEE92FB5515482D45'u), + uint64x2(hi: 0x991711052D8BF3C5'u, lo: 0x751BDD152D4D1C4B'u), + uint64x2(hi: 0xBF5CD54678EEF0B6'u, lo: 0xD262D45A78A0635E'u), + uint64x2(hi: 0xEF340A98172AACE4'u, lo: 0x86FB897116C87C35'u), + uint64x2(hi: 0x9580869F0E7AAC0E'u, lo: 0xD45D35E6AE3D4DA1'u), + uint64x2(hi: 0xBAE0A846D2195712'u, lo: 0x8974836059CCA10A'u), + uint64x2(hi: 0xE998D258869FACD7'u, lo: 0x2BD1A438703FC94C'u), + uint64x2(hi: 0x91FF83775423CC06'u, lo: 0x7B6306A34627DDD0'u), + uint64x2(hi: 0xB67F6455292CBF08'u, lo: 0x1A3BC84C17B1D543'u), + uint64x2(hi: 0xE41F3D6A7377EECA'u, lo: 0x20CABA5F1D9E4A94'u), + uint64x2(hi: 0x8E938662882AF53E'u, lo: 0x547EB47B7282EE9D'u), + uint64x2(hi: 0xB23867FB2A35B28D'u, lo: 0xE99E619A4F23AA44'u), + uint64x2(hi: 0xDEC681F9F4C31F31'u, lo: 0x6405FA00E2EC94D5'u), + uint64x2(hi: 0x8B3C113C38F9F37E'u, lo: 0xDE83BC408DD3DD05'u), + uint64x2(hi: 0xAE0B158B4738705E'u, lo: 0x9624AB50B148D446'u), + uint64x2(hi: 0xD98DDAEE19068C76'u, lo: 0x3BADD624DD9B0958'u), + uint64x2(hi: 0x87F8A8D4CFA417C9'u, lo: 0xE54CA5D70A80E5D7'u), + uint64x2(hi: 0xA9F6D30A038D1DBC'u, lo: 0x5E9FCF4CCD211F4D'u), + uint64x2(hi: 0xD47487CC8470652B'u, lo: 0x7647C32000696720'u), + uint64x2(hi: 0x84C8D4DFD2C63F3B'u, lo: 0x29ECD9F40041E074'u), + uint64x2(hi: 0xA5FB0A17C777CF09'u, lo: 0xF468107100525891'u), + uint64x2(hi: 0xCF79CC9DB955C2CC'u, lo: 0x7182148D4066EEB5'u), + uint64x2(hi: 0x81AC1FE293D599BF'u, lo: 0xC6F14CD848405531'u), + uint64x2(hi: 0xA21727DB38CB002F'u, lo: 0xB8ADA00E5A506A7D'u), + uint64x2(hi: 0xCA9CF1D206FDC03B'u, lo: 0xA6D90811F0E4851D'u), + uint64x2(hi: 0xFD442E4688BD304A'u, lo: 0x908F4A166D1DA664'u), + uint64x2(hi: 0x9E4A9CEC15763E2E'u, lo: 0x9A598E4E043287FF'u), + uint64x2(hi: 0xC5DD44271AD3CDBA'u, lo: 0x40EFF1E1853F29FE'u), + uint64x2(hi: 0xF7549530E188C128'u, lo: 0xD12BEE59E68EF47D'u), + uint64x2(hi: 0x9A94DD3E8CF578B9'u, lo: 0x82BB74F8301958CF'u), + uint64x2(hi: 0xC13A148E3032D6E7'u, lo: 0xE36A52363C1FAF02'u), + uint64x2(hi: 0xF18899B1BC3F8CA1'u, lo: 0xDC44E6C3CB279AC2'u), + uint64x2(hi: 0x96F5600F15A7B7E5'u, lo: 0x29AB103A5EF8C0BA'u), + uint64x2(hi: 0xBCB2B812DB11A5DE'u, lo: 0x7415D448F6B6F0E8'u), + uint64x2(hi: 0xEBDF661791D60F56'u, lo: 0x111B495B3464AD22'u), + uint64x2(hi: 0x936B9FCEBB25C995'u, lo: 0xCAB10DD900BEEC35'u), + uint64x2(hi: 0xB84687C269EF3BFB'u, lo: 0x3D5D514F40EEA743'u), + uint64x2(hi: 0xE65829B3046B0AFA'u, lo: 0x0CB4A5A3112A5113'u), + uint64x2(hi: 0x8FF71A0FE2C2E6DC'u, lo: 0x47F0E785EABA72AC'u), + uint64x2(hi: 0xB3F4E093DB73A093'u, lo: 0x59ED216765690F57'u), + uint64x2(hi: 0xE0F218B8D25088B8'u, lo: 0x306869C13EC3532D'u), + uint64x2(hi: 0x8C974F7383725573'u, lo: 0x1E414218C73A13FC'u), + uint64x2(hi: 0xAFBD2350644EEACF'u, lo: 0xE5D1929EF90898FB'u), + uint64x2(hi: 0xDBAC6C247D62A583'u, lo: 0xDF45F746B74ABF3A'u), + uint64x2(hi: 0x894BC396CE5DA772'u, lo: 0x6B8BBA8C328EB784'u), + uint64x2(hi: 0xAB9EB47C81F5114F'u, lo: 0x066EA92F3F326565'u), + uint64x2(hi: 0xD686619BA27255A2'u, lo: 0xC80A537B0EFEFEBE'u), + uint64x2(hi: 0x8613FD0145877585'u, lo: 0xBD06742CE95F5F37'u), + uint64x2(hi: 0xA798FC4196E952E7'u, lo: 0x2C48113823B73705'u), + uint64x2(hi: 0xD17F3B51FCA3A7A0'u, lo: 0xF75A15862CA504C6'u), + uint64x2(hi: 0x82EF85133DE648C4'u, lo: 0x9A984D73DBE722FC'u), + uint64x2(hi: 0xA3AB66580D5FDAF5'u, lo: 0xC13E60D0D2E0EBBB'u), + uint64x2(hi: 0xCC963FEE10B7D1B3'u, lo: 0x318DF905079926A9'u), + uint64x2(hi: 0xFFBBCFE994E5C61F'u, lo: 0xFDF17746497F7053'u), + uint64x2(hi: 0x9FD561F1FD0F9BD3'u, lo: 0xFEB6EA8BEDEFA634'u), + uint64x2(hi: 0xC7CABA6E7C5382C8'u, lo: 0xFE64A52EE96B8FC1'u), + uint64x2(hi: 0xF9BD690A1B68637B'u, lo: 0x3DFDCE7AA3C673B1'u), + uint64x2(hi: 0x9C1661A651213E2D'u, lo: 0x06BEA10CA65C084F'u), + uint64x2(hi: 0xC31BFA0FE5698DB8'u, lo: 0x486E494FCFF30A63'u), + uint64x2(hi: 0xF3E2F893DEC3F126'u, lo: 0x5A89DBA3C3EFCCFB'u), + uint64x2(hi: 0x986DDB5C6B3A76B7'u, lo: 0xF89629465A75E01D'u), + uint64x2(hi: 0xBE89523386091465'u, lo: 0xF6BBB397F1135824'u), + uint64x2(hi: 0xEE2BA6C0678B597F'u, lo: 0x746AA07DED582E2D'u), + uint64x2(hi: 0x94DB483840B717EF'u, lo: 0xA8C2A44EB4571CDD'u), + uint64x2(hi: 0xBA121A4650E4DDEB'u, lo: 0x92F34D62616CE414'u), + uint64x2(hi: 0xE896A0D7E51E1566'u, lo: 0x77B020BAF9C81D18'u), + uint64x2(hi: 0x915E2486EF32CD60'u, lo: 0x0ACE1474DC1D122F'u), + uint64x2(hi: 0xB5B5ADA8AAFF80B8'u, lo: 0x0D819992132456BB'u), + uint64x2(hi: 0xE3231912D5BF60E6'u, lo: 0x10E1FFF697ED6C6A'u), + uint64x2(hi: 0x8DF5EFABC5979C8F'u, lo: 0xCA8D3FFA1EF463C2'u), + uint64x2(hi: 0xB1736B96B6FD83B3'u, lo: 0xBD308FF8A6B17CB3'u), + uint64x2(hi: 0xDDD0467C64BCE4A0'u, lo: 0xAC7CB3F6D05DDBDF'u), + uint64x2(hi: 0x8AA22C0DBEF60EE4'u, lo: 0x6BCDF07A423AA96C'u), + uint64x2(hi: 0xAD4AB7112EB3929D'u, lo: 0x86C16C98D2C953C7'u), + uint64x2(hi: 0xD89D64D57A607744'u, lo: 0xE871C7BF077BA8B8'u), + uint64x2(hi: 0x87625F056C7C4A8B'u, lo: 0x11471CD764AD4973'u), + uint64x2(hi: 0xA93AF6C6C79B5D2D'u, lo: 0xD598E40D3DD89BD0'u), + uint64x2(hi: 0xD389B47879823479'u, lo: 0x4AFF1D108D4EC2C4'u), + uint64x2(hi: 0x843610CB4BF160CB'u, lo: 0xCEDF722A585139BB'u), + uint64x2(hi: 0xA54394FE1EEDB8FE'u, lo: 0xC2974EB4EE658829'u), + uint64x2(hi: 0xCE947A3DA6A9273E'u, lo: 0x733D226229FEEA33'u), + uint64x2(hi: 0x811CCC668829B887'u, lo: 0x0806357D5A3F5260'u), + uint64x2(hi: 0xA163FF802A3426A8'u, lo: 0xCA07C2DCB0CF26F8'u), + uint64x2(hi: 0xC9BCFF6034C13052'u, lo: 0xFC89B393DD02F0B6'u), + uint64x2(hi: 0xFC2C3F3841F17C67'u, lo: 0xBBAC2078D443ACE3'u), + uint64x2(hi: 0x9D9BA7832936EDC0'u, lo: 0xD54B944B84AA4C0E'u), + uint64x2(hi: 0xC5029163F384A931'u, lo: 0x0A9E795E65D4DF12'u), + uint64x2(hi: 0xF64335BCF065D37D'u, lo: 0x4D4617B5FF4A16D6'u), + uint64x2(hi: 0x99EA0196163FA42E'u, lo: 0x504BCED1BF8E4E46'u), + uint64x2(hi: 0xC06481FB9BCF8D39'u, lo: 0xE45EC2862F71E1D7'u), + uint64x2(hi: 0xF07DA27A82C37088'u, lo: 0x5D767327BB4E5A4D'u), + uint64x2(hi: 0x964E858C91BA2655'u, lo: 0x3A6A07F8D510F870'u), + uint64x2(hi: 0xBBE226EFB628AFEA'u, lo: 0x890489F70A55368C'u), + uint64x2(hi: 0xEADAB0ABA3B2DBE5'u, lo: 0x2B45AC74CCEA842F'u), + uint64x2(hi: 0x92C8AE6B464FC96F'u, lo: 0x3B0B8BC90012929E'u), + uint64x2(hi: 0xB77ADA0617E3BBCB'u, lo: 0x09CE6EBB40173745'u), + uint64x2(hi: 0xE55990879DDCAABD'u, lo: 0xCC420A6A101D0516'u), + uint64x2(hi: 0x8F57FA54C2A9EAB6'u, lo: 0x9FA946824A12232E'u), + uint64x2(hi: 0xB32DF8E9F3546564'u, lo: 0x47939822DC96ABFA'u), + uint64x2(hi: 0xDFF9772470297EBD'u, lo: 0x59787E2B93BC56F8'u), + uint64x2(hi: 0x8BFBEA76C619EF36'u, lo: 0x57EB4EDB3C55B65B'u), + uint64x2(hi: 0xAEFAE51477A06B03'u, lo: 0xEDE622920B6B23F2'u), + uint64x2(hi: 0xDAB99E59958885C4'u, lo: 0xE95FAB368E45ECEE'u), + uint64x2(hi: 0x88B402F7FD75539B'u, lo: 0x11DBCB0218EBB415'u), + uint64x2(hi: 0xAAE103B5FCD2A881'u, lo: 0xD652BDC29F26A11A'u), + uint64x2(hi: 0xD59944A37C0752A2'u, lo: 0x4BE76D3346F04960'u), + uint64x2(hi: 0x857FCAE62D8493A5'u, lo: 0x6F70A4400C562DDC'u), + uint64x2(hi: 0xA6DFBD9FB8E5B88E'u, lo: 0xCB4CCD500F6BB953'u), + uint64x2(hi: 0xD097AD07A71F26B2'u, lo: 0x7E2000A41346A7A8'u), + uint64x2(hi: 0x825ECC24C873782F'u, lo: 0x8ED400668C0C28C9'u), + uint64x2(hi: 0xA2F67F2DFA90563B'u, lo: 0x728900802F0F32FB'u), + uint64x2(hi: 0xCBB41EF979346BCA'u, lo: 0x4F2B40A03AD2FFBA'u), + uint64x2(hi: 0xFEA126B7D78186BC'u, lo: 0xE2F610C84987BFA9'u), + uint64x2(hi: 0x9F24B832E6B0F436'u, lo: 0x0DD9CA7D2DF4D7CA'u), + uint64x2(hi: 0xC6EDE63FA05D3143'u, lo: 0x91503D1C79720DBC'u), + uint64x2(hi: 0xF8A95FCF88747D94'u, lo: 0x75A44C6397CE912B'u), + uint64x2(hi: 0x9B69DBE1B548CE7C'u, lo: 0xC986AFBE3EE11ABB'u), + uint64x2(hi: 0xC24452DA229B021B'u, lo: 0xFBE85BADCE996169'u), + uint64x2(hi: 0xF2D56790AB41C2A2'u, lo: 0xFAE27299423FB9C4'u), + uint64x2(hi: 0x97C560BA6B0919A5'u, lo: 0xDCCD879FC967D41B'u), + uint64x2(hi: 0xBDB6B8E905CB600F'u, lo: 0x5400E987BBC1C921'u), + uint64x2(hi: 0xED246723473E3813'u, lo: 0x290123E9AAB23B69'u), + uint64x2(hi: 0x9436C0760C86E30B'u, lo: 0xF9A0B6720AAF6522'u), + uint64x2(hi: 0xB94470938FA89BCE'u, lo: 0xF808E40E8D5B3E6A'u), + uint64x2(hi: 0xE7958CB87392C2C2'u, lo: 0xB60B1D1230B20E05'u), + uint64x2(hi: 0x90BD77F3483BB9B9'u, lo: 0xB1C6F22B5E6F48C3'u), + uint64x2(hi: 0xB4ECD5F01A4AA828'u, lo: 0x1E38AEB6360B1AF4'u), + uint64x2(hi: 0xE2280B6C20DD5232'u, lo: 0x25C6DA63C38DE1B1'u), + uint64x2(hi: 0x8D590723948A535F'u, lo: 0x579C487E5A38AD0F'u), + uint64x2(hi: 0xB0AF48EC79ACE837'u, lo: 0x2D835A9DF0C6D852'u), + uint64x2(hi: 0xDCDB1B2798182244'u, lo: 0xF8E431456CF88E66'u), + uint64x2(hi: 0x8A08F0F8BF0F156B'u, lo: 0x1B8E9ECB641B5900'u), + uint64x2(hi: 0xAC8B2D36EED2DAC5'u, lo: 0xE272467E3D222F40'u), + uint64x2(hi: 0xD7ADF884AA879177'u, lo: 0x5B0ED81DCC6ABB10'u), + uint64x2(hi: 0x86CCBB52EA94BAEA'u, lo: 0x98E947129FC2B4EA'u), + uint64x2(hi: 0xA87FEA27A539E9A5'u, lo: 0x3F2398D747B36225'u), + uint64x2(hi: 0xD29FE4B18E88640E'u, lo: 0x8EEC7F0D19A03AAE'u), + uint64x2(hi: 0x83A3EEEEF9153E89'u, lo: 0x1953CF68300424AD'u), + uint64x2(hi: 0xA48CEAAAB75A8E2B'u, lo: 0x5FA8C3423C052DD8'u), + uint64x2(hi: 0xCDB02555653131B6'u, lo: 0x3792F412CB06794E'u), + uint64x2(hi: 0x808E17555F3EBF11'u, lo: 0xE2BBD88BBEE40BD1'u), + uint64x2(hi: 0xA0B19D2AB70E6ED6'u, lo: 0x5B6ACEAEAE9D0EC5'u), + uint64x2(hi: 0xC8DE047564D20A8B'u, lo: 0xF245825A5A445276'u), + uint64x2(hi: 0xFB158592BE068D2E'u, lo: 0xEED6E2F0F0D56713'u), + uint64x2(hi: 0x9CED737BB6C4183D'u, lo: 0x55464DD69685606C'u), + uint64x2(hi: 0xC428D05AA4751E4C'u, lo: 0xAA97E14C3C26B887'u), + uint64x2(hi: 0xF53304714D9265DF'u, lo: 0xD53DD99F4B3066A9'u), + uint64x2(hi: 0x993FE2C6D07B7FAB'u, lo: 0xE546A8038EFE402A'u), + uint64x2(hi: 0xBF8FDB78849A5F96'u, lo: 0xDE98520472BDD034'u), + uint64x2(hi: 0xEF73D256A5C0F77C'u, lo: 0x963E66858F6D4441'u), + uint64x2(hi: 0x95A8637627989AAD'u, lo: 0xDDE7001379A44AA9'u), + uint64x2(hi: 0xBB127C53B17EC159'u, lo: 0x5560C018580D5D53'u), + uint64x2(hi: 0xE9D71B689DDE71AF'u, lo: 0xAAB8F01E6E10B4A7'u), + uint64x2(hi: 0x9226712162AB070D'u, lo: 0xCAB3961304CA70E9'u), + uint64x2(hi: 0xB6B00D69BB55C8D1'u, lo: 0x3D607B97C5FD0D23'u), + uint64x2(hi: 0xE45C10C42A2B3B05'u, lo: 0x8CB89A7DB77C506B'u), + uint64x2(hi: 0x8EB98A7A9A5B04E3'u, lo: 0x77F3608E92ADB243'u), + uint64x2(hi: 0xB267ED1940F1C61C'u, lo: 0x55F038B237591ED4'u), + uint64x2(hi: 0xDF01E85F912E37A3'u, lo: 0x6B6C46DEC52F6689'u), + uint64x2(hi: 0x8B61313BBABCE2C6'u, lo: 0x2323AC4B3B3DA016'u), + uint64x2(hi: 0xAE397D8AA96C1B77'u, lo: 0xABEC975E0A0D081B'u), + uint64x2(hi: 0xD9C7DCED53C72255'u, lo: 0x96E7BD358C904A22'u), + uint64x2(hi: 0x881CEA14545C7575'u, lo: 0x7E50D64177DA2E55'u), + uint64x2(hi: 0xAA242499697392D2'u, lo: 0xDDE50BD1D5D0B9EA'u), + uint64x2(hi: 0xD4AD2DBFC3D07787'u, lo: 0x955E4EC64B44E865'u), + uint64x2(hi: 0x84EC3C97DA624AB4'u, lo: 0xBD5AF13BEF0B113F'u), + uint64x2(hi: 0xA6274BBDD0FADD61'u, lo: 0xECB1AD8AEACDD58F'u), + uint64x2(hi: 0xCFB11EAD453994BA'u, lo: 0x67DE18EDA5814AF3'u), + uint64x2(hi: 0x81CEB32C4B43FCF4'u, lo: 0x80EACF948770CED8'u), + uint64x2(hi: 0xA2425FF75E14FC31'u, lo: 0xA1258379A94D028E'u), + uint64x2(hi: 0xCAD2F7F5359A3B3E'u, lo: 0x096EE45813A04331'u), + uint64x2(hi: 0xFD87B5F28300CA0D'u, lo: 0x8BCA9D6E188853FD'u), + uint64x2(hi: 0x9E74D1B791E07E48'u, lo: 0x775EA264CF55347E'u), + uint64x2(hi: 0xC612062576589DDA'u, lo: 0x95364AFE032A819E'u), + uint64x2(hi: 0xF79687AED3EEC551'u, lo: 0x3A83DDBD83F52205'u), + uint64x2(hi: 0x9ABE14CD44753B52'u, lo: 0xC4926A9672793543'u), + uint64x2(hi: 0xC16D9A0095928A27'u, lo: 0x75B7053C0F178294'u), + uint64x2(hi: 0xF1C90080BAF72CB1'u, lo: 0x5324C68B12DD6339'u), + uint64x2(hi: 0x971DA05074DA7BEE'u, lo: 0xD3F6FC16EBCA5E04'u), + uint64x2(hi: 0xBCE5086492111AEA'u, lo: 0x88F4BB1CA6BCF585'u), + uint64x2(hi: 0xEC1E4A7DB69561A5'u, lo: 0x2B31E9E3D06C32E6'u), + uint64x2(hi: 0x9392EE8E921D5D07'u, lo: 0x3AFF322E62439FD0'u), + uint64x2(hi: 0xB877AA3236A4B449'u, lo: 0x09BEFEB9FAD487C3'u), + uint64x2(hi: 0xE69594BEC44DE15B'u, lo: 0x4C2EBE687989A9B4'u), + uint64x2(hi: 0x901D7CF73AB0ACD9'u, lo: 0x0F9D37014BF60A11'u), + uint64x2(hi: 0xB424DC35095CD80F'u, lo: 0x538484C19EF38C95'u), + uint64x2(hi: 0xE12E13424BB40E13'u, lo: 0x2865A5F206B06FBA'u), + uint64x2(hi: 0x8CBCCC096F5088CB'u, lo: 0xF93F87B7442E45D4'u), + uint64x2(hi: 0xAFEBFF0BCB24AAFE'u, lo: 0xF78F69A51539D749'u), + uint64x2(hi: 0xDBE6FECEBDEDD5BE'u, lo: 0xB573440E5A884D1C'u), + uint64x2(hi: 0x89705F4136B4A597'u, lo: 0x31680A88F8953031'u), + uint64x2(hi: 0xABCC77118461CEFC'u, lo: 0xFDC20D2B36BA7C3E'u), + uint64x2(hi: 0xD6BF94D5E57A42BC'u, lo: 0x3D32907604691B4D'u), + uint64x2(hi: 0x8637BD05AF6C69B5'u, lo: 0xA63F9A49C2C1B110'u), + uint64x2(hi: 0xA7C5AC471B478423'u, lo: 0x0FCF80DC33721D54'u), + uint64x2(hi: 0xD1B71758E219652B'u, lo: 0xD3C36113404EA4A9'u), + uint64x2(hi: 0x83126E978D4FDF3B'u, lo: 0x645A1CAC083126EA'u), + uint64x2(hi: 0xA3D70A3D70A3D70A'u, lo: 0x3D70A3D70A3D70A4'u), + uint64x2(hi: 0xCCCCCCCCCCCCCCCC'u, lo: 0xCCCCCCCCCCCCCCCD'u), + uint64x2(hi: 0x8000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA000000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC800000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xFA00000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9C40000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xC350000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xF424000000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9896800000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBEBC200000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xEE6B280000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9502F90000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xBA43B74000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE8D4A51000000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x9184E72A00000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB5E620F480000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xE35FA931A0000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8E1BC9BF04000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xB1A2BC2EC5000000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xDE0B6B3A76400000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x8AC7230489E80000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xAD78EBC5AC620000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD8D726B7177A8000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x878678326EAC9000'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA968163F0A57B400'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xD3C21BCECCEDA100'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x84595161401484A0'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xA56FA5B99019A5C8'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0xCECB8F27F4200F3A'u, lo: 0x0000000000000000'u), + uint64x2(hi: 0x813F3978F8940984'u, lo: 0x4000000000000000'u), + uint64x2(hi: 0xA18F07D736B90BE5'u, lo: 0x5000000000000000'u), + uint64x2(hi: 0xC9F2C9CD04674EDE'u, lo: 0xA400000000000000'u), + uint64x2(hi: 0xFC6F7C4045812296'u, lo: 0x4D00000000000000'u), + uint64x2(hi: 0x9DC5ADA82B70B59D'u, lo: 0xF020000000000000'u), + uint64x2(hi: 0xC5371912364CE305'u, lo: 0x6C28000000000000'u), + uint64x2(hi: 0xF684DF56C3E01BC6'u, lo: 0xC732000000000000'u), + uint64x2(hi: 0x9A130B963A6C115C'u, lo: 0x3C7F400000000000'u), + uint64x2(hi: 0xC097CE7BC90715B3'u, lo: 0x4B9F100000000000'u), + uint64x2(hi: 0xF0BDC21ABB48DB20'u, lo: 0x1E86D40000000000'u), + uint64x2(hi: 0x96769950B50D88F4'u, lo: 0x1314448000000000'u), + uint64x2(hi: 0xBC143FA4E250EB31'u, lo: 0x17D955A000000000'u), + uint64x2(hi: 0xEB194F8E1AE525FD'u, lo: 0x5DCFAB0800000000'u), + uint64x2(hi: 0x92EFD1B8D0CF37BE'u, lo: 0x5AA1CAE500000000'u), + uint64x2(hi: 0xB7ABC627050305AD'u, lo: 0xF14A3D9E40000000'u), + uint64x2(hi: 0xE596B7B0C643C719'u, lo: 0x6D9CCD05D0000000'u), + uint64x2(hi: 0x8F7E32CE7BEA5C6F'u, lo: 0xE4820023A2000000'u), + uint64x2(hi: 0xB35DBF821AE4F38B'u, lo: 0xDDA2802C8A800000'u), + uint64x2(hi: 0xE0352F62A19E306E'u, lo: 0xD50B2037AD200000'u), + uint64x2(hi: 0x8C213D9DA502DE45'u, lo: 0x4526F422CC340000'u), + uint64x2(hi: 0xAF298D050E4395D6'u, lo: 0x9670B12B7F410000'u), + uint64x2(hi: 0xDAF3F04651D47B4C'u, lo: 0x3C0CDD765F114000'u), + uint64x2(hi: 0x88D8762BF324CD0F'u, lo: 0xA5880A69FB6AC800'u), + uint64x2(hi: 0xAB0E93B6EFEE0053'u, lo: 0x8EEA0D047A457A00'u), + uint64x2(hi: 0xD5D238A4ABE98068'u, lo: 0x72A4904598D6D880'u), + uint64x2(hi: 0x85A36366EB71F041'u, lo: 0x47A6DA2B7F864750'u), + uint64x2(hi: 0xA70C3C40A64E6C51'u, lo: 0x999090B65F67D924'u), + uint64x2(hi: 0xD0CF4B50CFE20765'u, lo: 0xFFF4B4E3F741CF6D'u), + uint64x2(hi: 0x82818F1281ED449F'u, lo: 0xBFF8F10E7A8921A4'u), + uint64x2(hi: 0xA321F2D7226895C7'u, lo: 0xAFF72D52192B6A0D'u), + uint64x2(hi: 0xCBEA6F8CEB02BB39'u, lo: 0x9BF4F8A69F764490'u), + uint64x2(hi: 0xFEE50B7025C36A08'u, lo: 0x02F236D04753D5B4'u), + uint64x2(hi: 0x9F4F2726179A2245'u, lo: 0x01D762422C946590'u), + uint64x2(hi: 0xC722F0EF9D80AAD6'u, lo: 0x424D3AD2B7B97EF5'u), + uint64x2(hi: 0xF8EBAD2B84E0D58B'u, lo: 0xD2E0898765A7DEB2'u), + uint64x2(hi: 0x9B934C3B330C8577'u, lo: 0x63CC55F49F88EB2F'u), + uint64x2(hi: 0xC2781F49FFCFA6D5'u, lo: 0x3CBF6B71C76B25FB'u), + uint64x2(hi: 0xF316271C7FC3908A'u, lo: 0x8BEF464E3945EF7A'u), + uint64x2(hi: 0x97EDD871CFDA3A56'u, lo: 0x97758BF0E3CBB5AC'u), + uint64x2(hi: 0xBDE94E8E43D0C8EC'u, lo: 0x3D52EEED1CBEA317'u), + uint64x2(hi: 0xED63A231D4C4FB27'u, lo: 0x4CA7AAA863EE4BDD'u), + uint64x2(hi: 0x945E455F24FB1CF8'u, lo: 0x8FE8CAA93E74EF6A'u), + uint64x2(hi: 0xB975D6B6EE39E436'u, lo: 0xB3E2FD538E122B44'u), + uint64x2(hi: 0xE7D34C64A9C85D44'u, lo: 0x60DBBCA87196B616'u), + uint64x2(hi: 0x90E40FBEEA1D3A4A'u, lo: 0xBC8955E946FE31CD'u), + uint64x2(hi: 0xB51D13AEA4A488DD'u, lo: 0x6BABAB6398BDBE41'u), + uint64x2(hi: 0xE264589A4DCDAB14'u, lo: 0xC696963C7EED2DD1'u), + uint64x2(hi: 0x8D7EB76070A08AEC'u, lo: 0xFC1E1DE5CF543CA2'u), + uint64x2(hi: 0xB0DE65388CC8ADA8'u, lo: 0x3B25A55F43294BCB'u), + uint64x2(hi: 0xDD15FE86AFFAD912'u, lo: 0x49EF0EB713F39EBE'u), + uint64x2(hi: 0x8A2DBF142DFCC7AB'u, lo: 0x6E3569326C784337'u), + uint64x2(hi: 0xACB92ED9397BF996'u, lo: 0x49C2C37F07965404'u), + uint64x2(hi: 0xD7E77A8F87DAF7FB'u, lo: 0xDC33745EC97BE906'u), + uint64x2(hi: 0x86F0AC99B4E8DAFD'u, lo: 0x69A028BB3DED71A3'u), + uint64x2(hi: 0xA8ACD7C0222311BC'u, lo: 0xC40832EA0D68CE0C'u), + uint64x2(hi: 0xD2D80DB02AABD62B'u, lo: 0xF50A3FA490C30190'u), + uint64x2(hi: 0x83C7088E1AAB65DB'u, lo: 0x792667C6DA79E0FA'u), + uint64x2(hi: 0xA4B8CAB1A1563F52'u, lo: 0x577001B891185938'u), + uint64x2(hi: 0xCDE6FD5E09ABCF26'u, lo: 0xED4C0226B55E6F86'u), + uint64x2(hi: 0x80B05E5AC60B6178'u, lo: 0x544F8158315B05B4'u), + uint64x2(hi: 0xA0DC75F1778E39D6'u, lo: 0x696361AE3DB1C721'u), + uint64x2(hi: 0xC913936DD571C84C'u, lo: 0x03BC3A19CD1E38E9'u), + uint64x2(hi: 0xFB5878494ACE3A5F'u, lo: 0x04AB48A04065C723'u), + uint64x2(hi: 0x9D174B2DCEC0E47B'u, lo: 0x62EB0D64283F9C76'u), + uint64x2(hi: 0xC45D1DF942711D9A'u, lo: 0x3BA5D0BD324F8394'u), + uint64x2(hi: 0xF5746577930D6500'u, lo: 0xCA8F44EC7EE36479'u), + uint64x2(hi: 0x9968BF6ABBE85F20'u, lo: 0x7E998B13CF4E1ECB'u), + uint64x2(hi: 0xBFC2EF456AE276E8'u, lo: 0x9E3FEDD8C321A67E'u), + uint64x2(hi: 0xEFB3AB16C59B14A2'u, lo: 0xC5CFE94EF3EA101E'u), + uint64x2(hi: 0x95D04AEE3B80ECE5'u, lo: 0xBBA1F1D158724A12'u), + uint64x2(hi: 0xBB445DA9CA61281F'u, lo: 0x2A8A6E45AE8EDC97'u), + uint64x2(hi: 0xEA1575143CF97226'u, lo: 0xF52D09D71A3293BD'u), + uint64x2(hi: 0x924D692CA61BE758'u, lo: 0x593C2626705F9C56'u), + uint64x2(hi: 0xB6E0C377CFA2E12E'u, lo: 0x6F8B2FB00C77836C'u), + uint64x2(hi: 0xE498F455C38B997A'u, lo: 0x0B6DFB9C0F956447'u), + uint64x2(hi: 0x8EDF98B59A373FEC'u, lo: 0x4724BD4189BD5EAC'u), + uint64x2(hi: 0xB2977EE300C50FE7'u, lo: 0x58EDEC91EC2CB657'u), + uint64x2(hi: 0xDF3D5E9BC0F653E1'u, lo: 0x2F2967B66737E3ED'u), + uint64x2(hi: 0x8B865B215899F46C'u, lo: 0xBD79E0D20082EE74'u), + uint64x2(hi: 0xAE67F1E9AEC07187'u, lo: 0xECD8590680A3AA11'u), + uint64x2(hi: 0xDA01EE641A708DE9'u, lo: 0xE80E6F4820CC9495'u), + uint64x2(hi: 0x884134FE908658B2'u, lo: 0x3109058D147FDCDD'u), + uint64x2(hi: 0xAA51823E34A7EEDE'u, lo: 0xBD4B46F0599FD415'u), + uint64x2(hi: 0xD4E5E2CDC1D1EA96'u, lo: 0x6C9E18AC7007C91A'u), + uint64x2(hi: 0x850FADC09923329E'u, lo: 0x03E2CF6BC604DDB0'u), + uint64x2(hi: 0xA6539930BF6BFF45'u, lo: 0x84DB8346B786151C'u), + uint64x2(hi: 0xCFE87F7CEF46FF16'u, lo: 0xE612641865679A63'u), + uint64x2(hi: 0x81F14FAE158C5F6E'u, lo: 0x4FCB7E8F3F60C07E'u), + uint64x2(hi: 0xA26DA3999AEF7749'u, lo: 0xE3BE5E330F38F09D'u), + uint64x2(hi: 0xCB090C8001AB551C'u, lo: 0x5CADF5BFD3072CC5'u), + uint64x2(hi: 0xFDCB4FA002162A63'u, lo: 0x73D9732FC7C8F7F6'u), + uint64x2(hi: 0x9E9F11C4014DDA7E'u, lo: 0x2867E7FDDCDD9AFA'u), + uint64x2(hi: 0xC646D63501A1511D'u, lo: 0xB281E1FD541501B8'u), + uint64x2(hi: 0xF7D88BC24209A565'u, lo: 0x1F225A7CA91A4226'u), + uint64x2(hi: 0x9AE757596946075F'u, lo: 0x3375788DE9B06958'u), + uint64x2(hi: 0xC1A12D2FC3978937'u, lo: 0x0052D6B1641C83AE'u), + uint64x2(hi: 0xF209787BB47D6B84'u, lo: 0xC0678C5DBD23A49A'u), + uint64x2(hi: 0x9745EB4D50CE6332'u, lo: 0xF840B7BA963646E0'u), + uint64x2(hi: 0xBD176620A501FBFF'u, lo: 0xB650E5A93BC3D898'u), + uint64x2(hi: 0xEC5D3FA8CE427AFF'u, lo: 0xA3E51F138AB4CEBE'u), + uint64x2(hi: 0x93BA47C980E98CDF'u, lo: 0xC66F336C36B10137'u), + uint64x2(hi: 0xB8A8D9BBE123F017'u, lo: 0xB80B0047445D4184'u), + uint64x2(hi: 0xE6D3102AD96CEC1D'u, lo: 0xA60DC059157491E5'u), + uint64x2(hi: 0x9043EA1AC7E41392'u, lo: 0x87C89837AD68DB2F'u), + uint64x2(hi: 0xB454E4A179DD1877'u, lo: 0x29BABE4598C311FB'u), + uint64x2(hi: 0xE16A1DC9D8545E94'u, lo: 0xF4296DD6FEF3D67A'u), + uint64x2(hi: 0x8CE2529E2734BB1D'u, lo: 0x1899E4A65F58660C'u), + uint64x2(hi: 0xB01AE745B101E9E4'u, lo: 0x5EC05DCFF72E7F8F'u), + uint64x2(hi: 0xDC21A1171D42645D'u, lo: 0x76707543F4FA1F73'u), + uint64x2(hi: 0x899504AE72497EBA'u, lo: 0x6A06494A791C53A8'u), + uint64x2(hi: 0xABFA45DA0EDBDE69'u, lo: 0x0487DB9D17636892'u), + uint64x2(hi: 0xD6F8D7509292D603'u, lo: 0x45A9D2845D3C42B6'u), + uint64x2(hi: 0x865B86925B9BC5C2'u, lo: 0x0B8A2392BA45A9B2'u), + uint64x2(hi: 0xA7F26836F282B732'u, lo: 0x8E6CAC7768D7141E'u), + uint64x2(hi: 0xD1EF0244AF2364FF'u, lo: 0x3207D795430CD926'u), + uint64x2(hi: 0x8335616AED761F1F'u, lo: 0x7F44E6BD49E807B8'u), + uint64x2(hi: 0xA402B9C5A8D3A6E7'u, lo: 0x5F16206C9C6209A6'u), + uint64x2(hi: 0xCD036837130890A1'u, lo: 0x36DBA887C37A8C0F'u), + uint64x2(hi: 0x802221226BE55A64'u, lo: 0xC2494954DA2C9789'u), + uint64x2(hi: 0xA02AA96B06DEB0FD'u, lo: 0xF2DB9BAA10B7BD6C'u), + uint64x2(hi: 0xC83553C5C8965D3D'u, lo: 0x6F92829494E5ACC7'u), + uint64x2(hi: 0xFA42A8B73ABBF48C'u, lo: 0xCB772339BA1F17F9'u), + uint64x2(hi: 0x9C69A97284B578D7'u, lo: 0xFF2A760414536EFB'u), + uint64x2(hi: 0xC38413CF25E2D70D'u, lo: 0xFEF5138519684ABA'u), + uint64x2(hi: 0xF46518C2EF5B8CD1'u, lo: 0x7EB258665FC25D69'u), + uint64x2(hi: 0x98BF2F79D5993802'u, lo: 0xEF2F773FFBD97A61'u), + uint64x2(hi: 0xBEEEFB584AFF8603'u, lo: 0xAAFB550FFACFD8FA'u), + uint64x2(hi: 0xEEAABA2E5DBF6784'u, lo: 0x95BA2A53F983CF38'u), + uint64x2(hi: 0x952AB45CFA97A0B2'u, lo: 0xDD945A747BF26183'u), + uint64x2(hi: 0xBA756174393D88DF'u, lo: 0x94F971119AEEF9E4'u), + uint64x2(hi: 0xE912B9D1478CEB17'u, lo: 0x7A37CD5601AAB85D'u), + uint64x2(hi: 0x91ABB422CCB812EE'u, lo: 0xAC62E055C10AB33A'u), + uint64x2(hi: 0xB616A12B7FE617AA'u, lo: 0x577B986B314D6009'u), + uint64x2(hi: 0xE39C49765FDF9D94'u, lo: 0xED5A7E85FDA0B80B'u), + uint64x2(hi: 0x8E41ADE9FBEBC27D'u, lo: 0x14588F13BE847307'u), + uint64x2(hi: 0xB1D219647AE6B31C'u, lo: 0x596EB2D8AE258FC8'u), + uint64x2(hi: 0xDE469FBD99A05FE3'u, lo: 0x6FCA5F8ED9AEF3BB'u), + uint64x2(hi: 0x8AEC23D680043BEE'u, lo: 0x25DE7BB9480D5854'u), + uint64x2(hi: 0xADA72CCC20054AE9'u, lo: 0xAF561AA79A10AE6A'u), + uint64x2(hi: 0xD910F7FF28069DA4'u, lo: 0x1B2BA1518094DA04'u), + uint64x2(hi: 0x87AA9AFF79042286'u, lo: 0x90FB44D2F05D0842'u), + uint64x2(hi: 0xA99541BF57452B28'u, lo: 0x353A1607AC744A53'u), + uint64x2(hi: 0xD3FA922F2D1675F2'u, lo: 0x42889B8997915CE8'u), + uint64x2(hi: 0x847C9B5D7C2E09B7'u, lo: 0x69956135FEBADA11'u), + uint64x2(hi: 0xA59BC234DB398C25'u, lo: 0x43FAB9837E699095'u), + uint64x2(hi: 0xCF02B2C21207EF2E'u, lo: 0x94F967E45E03F4BB'u), + uint64x2(hi: 0x8161AFB94B44F57D'u, lo: 0x1D1BE0EEBAC278F5'u), + uint64x2(hi: 0xA1BA1BA79E1632DC'u, lo: 0x6462D92A69731732'u), + uint64x2(hi: 0xCA28A291859BBF93'u, lo: 0x7D7B8F7503CFDCFE'u), + uint64x2(hi: 0xFCB2CB35E702AF78'u, lo: 0x5CDA735244C3D43E'u), + uint64x2(hi: 0x9DEFBF01B061ADAB'u, lo: 0x3A0888136AFA64A7'u), + uint64x2(hi: 0xC56BAEC21C7A1916'u, lo: 0x088AAA1845B8FDD0'u), + uint64x2(hi: 0xF6C69A72A3989F5B'u, lo: 0x8AAD549E57273D45'u), + uint64x2(hi: 0x9A3C2087A63F6399'u, lo: 0x36AC54E2F678864B'u), + uint64x2(hi: 0xC0CB28A98FCF3C7F'u, lo: 0x84576A1BB416A7DD'u), + uint64x2(hi: 0xF0FDF2D3F3C30B9F'u, lo: 0x656D44A2A11C51D5'u), + uint64x2(hi: 0x969EB7C47859E743'u, lo: 0x9F644AE5A4B1B325'u), + uint64x2(hi: 0xBC4665B596706114'u, lo: 0x873D5D9F0DDE1FEE'u), + uint64x2(hi: 0xEB57FF22FC0C7959'u, lo: 0xA90CB506D155A7EA'u), + uint64x2(hi: 0x9316FF75DD87CBD8'u, lo: 0x09A7F12442D588F2'u), + uint64x2(hi: 0xB7DCBF5354E9BECE'u, lo: 0x0C11ED6D538AEB2F'u), + uint64x2(hi: 0xE5D3EF282A242E81'u, lo: 0x8F1668C8A86DA5FA'u), + uint64x2(hi: 0x8FA475791A569D10'u, lo: 0xF96E017D694487BC'u), + uint64x2(hi: 0xB38D92D760EC4455'u, lo: 0x37C981DCC395A9AC'u), + uint64x2(hi: 0xE070F78D3927556A'u, lo: 0x85BBE253F47B1417'u), + uint64x2(hi: 0x8C469AB843B89562'u, lo: 0x93956D7478CCEC8E'u), + uint64x2(hi: 0xAF58416654A6BABB'u, lo: 0x387AC8D1970027B2'u), + uint64x2(hi: 0xDB2E51BFE9D0696A'u, lo: 0x06997B05FCC0319E'u), + uint64x2(hi: 0x88FCF317F22241E2'u, lo: 0x441FECE3BDF81F03'u), + uint64x2(hi: 0xAB3C2FDDEEAAD25A'u, lo: 0xD527E81CAD7626C3'u), + uint64x2(hi: 0xD60B3BD56A5586F1'u, lo: 0x8A71E223D8D3B074'u), + uint64x2(hi: 0x85C7056562757456'u, lo: 0xF6872D5667844E49'u), + uint64x2(hi: 0xA738C6BEBB12D16C'u, lo: 0xB428F8AC016561DB'u), + uint64x2(hi: 0xD106F86E69D785C7'u, lo: 0xE13336D701BEBA52'u), + uint64x2(hi: 0x82A45B450226B39C'u, lo: 0xECC0024661173473'u), + uint64x2(hi: 0xA34D721642B06084'u, lo: 0x27F002D7F95D0190'u), + uint64x2(hi: 0xCC20CE9BD35C78A5'u, lo: 0x31EC038DF7B441F4'u), + uint64x2(hi: 0xFF290242C83396CE'u, lo: 0x7E67047175A15271'u), + uint64x2(hi: 0x9F79A169BD203E41'u, lo: 0x0F0062C6E984D386'u), + uint64x2(hi: 0xC75809C42C684DD1'u, lo: 0x52C07B78A3E60868'u), + uint64x2(hi: 0xF92E0C3537826145'u, lo: 0xA7709A56CCDF8A82'u), + uint64x2(hi: 0x9BBCC7A142B17CCB'u, lo: 0x88A66076400BB691'u), + uint64x2(hi: 0xC2ABF989935DDBFE'u, lo: 0x6ACFF893D00EA435'u), + uint64x2(hi: 0xF356F7EBF83552FE'u, lo: 0x0583F6B8C4124D43'u), + uint64x2(hi: 0x98165AF37B2153DE'u, lo: 0xC3727A337A8B704A'u), + uint64x2(hi: 0xBE1BF1B059E9A8D6'u, lo: 0x744F18C0592E4C5C'u), + uint64x2(hi: 0xEDA2EE1C7064130C'u, lo: 0x1162DEF06F79DF73'u), + uint64x2(hi: 0x9485D4D1C63E8BE7'u, lo: 0x8ADDCB5645AC2BA8'u), + uint64x2(hi: 0xB9A74A0637CE2EE1'u, lo: 0x6D953E2BD7173692'u), + uint64x2(hi: 0xE8111C87C5C1BA99'u, lo: 0xC8FA8DB6CCDD0437'u), + uint64x2(hi: 0x910AB1D4DB9914A0'u, lo: 0x1D9C9892400A22A2'u), + uint64x2(hi: 0xB54D5E4A127F59C8'u, lo: 0x2503BEB6D00CAB4B'u), + uint64x2(hi: 0xE2A0B5DC971F303A'u, lo: 0x2E44AE64840FD61D'u), + uint64x2(hi: 0x8DA471A9DE737E24'u, lo: 0x5CEAECFED289E5D2'u), + uint64x2(hi: 0xB10D8E1456105DAD'u, lo: 0x7425A83E872C5F47'u), + uint64x2(hi: 0xDD50F1996B947518'u, lo: 0xD12F124E28F77719'u), + uint64x2(hi: 0x8A5296FFE33CC92F'u, lo: 0x82BD6B70D99AAA6F'u), + uint64x2(hi: 0xACE73CBFDC0BFB7B'u, lo: 0x636CC64D1001550B'u), + uint64x2(hi: 0xD8210BEFD30EFA5A'u, lo: 0x3C47F7E05401AA4E'u), + uint64x2(hi: 0x8714A775E3E95C78'u, lo: 0x65ACFAEC34810A71'u), + uint64x2(hi: 0xA8D9D1535CE3B396'u, lo: 0x7F1839A741A14D0D'u), + uint64x2(hi: 0xD31045A8341CA07C'u, lo: 0x1EDE48111209A050'u), + uint64x2(hi: 0x83EA2B892091E44D'u, lo: 0x934AED0AAB460432'u), + uint64x2(hi: 0xA4E4B66B68B65D60'u, lo: 0xF81DA84D5617853F'u), + uint64x2(hi: 0xCE1DE40642E3F4B9'u, lo: 0x36251260AB9D668E'u), + uint64x2(hi: 0x80D2AE83E9CE78F3'u, lo: 0xC1D72B7C6B426019'u), + uint64x2(hi: 0xA1075A24E4421730'u, lo: 0xB24CF65B8612F81F'u), + uint64x2(hi: 0xC94930AE1D529CFC'u, lo: 0xDEE033F26797B627'u), + uint64x2(hi: 0xFB9B7CD9A4A7443C'u, lo: 0x169840EF017DA3B1'u), + uint64x2(hi: 0x9D412E0806E88AA5'u, lo: 0x8E1F289560EE864E'u), + uint64x2(hi: 0xC491798A08A2AD4E'u, lo: 0xF1A6F2BAB92A27E2'u), + uint64x2(hi: 0xF5B5D7EC8ACB58A2'u, lo: 0xAE10AF696774B1DB'u), + uint64x2(hi: 0x9991A6F3D6BF1765'u, lo: 0xACCA6DA1E0A8EF29'u), + uint64x2(hi: 0xBFF610B0CC6EDD3F'u, lo: 0x17FD090A58D32AF3'u), + uint64x2(hi: 0xEFF394DCFF8A948E'u, lo: 0xDDFC4B4CEF07F5B0'u), + uint64x2(hi: 0x95F83D0A1FB69CD9'u, lo: 0x4ABDAF101564F98E'u), + uint64x2(hi: 0xBB764C4CA7A4440F'u, lo: 0x9D6D1AD41ABE37F1'u), + uint64x2(hi: 0xEA53DF5FD18D5513'u, lo: 0x84C86189216DC5ED'u), + uint64x2(hi: 0x92746B9BE2F8552C'u, lo: 0x32FD3CF5B4E49BB4'u), + uint64x2(hi: 0xB7118682DBB66A77'u, lo: 0x3FBC8C33221DC2A1'u), + uint64x2(hi: 0xE4D5E82392A40515'u, lo: 0x0FABAF3FEAA5334A'u), + uint64x2(hi: 0x8F05B1163BA6832D'u, lo: 0x29CB4D87F2A7400E'u), + uint64x2(hi: 0xB2C71D5BCA9023F8'u, lo: 0x743E20E9EF511012'u), + uint64x2(hi: 0xDF78E4B2BD342CF6'u, lo: 0x914DA9246B255416'u), + uint64x2(hi: 0x8BAB8EEFB6409C1A'u, lo: 0x1AD089B6C2F7548E'u), + uint64x2(hi: 0xAE9672ABA3D0C320'u, lo: 0xA184AC2473B529B1'u), + uint64x2(hi: 0xDA3C0F568CC4F3E8'u, lo: 0xC9E5D72D90A2741E'u), + uint64x2(hi: 0x8865899617FB1871'u, lo: 0x7E2FA67C7A658892'u), + uint64x2(hi: 0xAA7EEBFB9DF9DE8D'u, lo: 0xDDBB901B98FEEAB7'u), + uint64x2(hi: 0xD51EA6FA85785631'u, lo: 0x552A74227F3EA565'u), + uint64x2(hi: 0x8533285C936B35DE'u, lo: 0xD53A88958F87275F'u), + uint64x2(hi: 0xA67FF273B8460356'u, lo: 0x8A892ABAF368F137'u), + uint64x2(hi: 0xD01FEF10A657842C'u, lo: 0x2D2B7569B0432D85'u), + uint64x2(hi: 0x8213F56A67F6B29B'u, lo: 0x9C3B29620E29FC73'u), + uint64x2(hi: 0xA298F2C501F45F42'u, lo: 0x8349F3BA91B47B8F'u), + uint64x2(hi: 0xCB3F2F7642717713'u, lo: 0x241C70A936219A73'u), + uint64x2(hi: 0xFE0EFB53D30DD4D7'u, lo: 0xED238CD383AA0110'u), + uint64x2(hi: 0x9EC95D1463E8A506'u, lo: 0xF4363804324A40AA'u), + uint64x2(hi: 0xC67BB4597CE2CE48'u, lo: 0xB143C6053EDCD0D5'u), + uint64x2(hi: 0xF81AA16FDC1B81DA'u, lo: 0xDD94B7868E94050A'u), + uint64x2(hi: 0x9B10A4E5E9913128'u, lo: 0xCA7CF2B4191C8326'u), + uint64x2(hi: 0xC1D4CE1F63F57D72'u, lo: 0xFD1C2F611F63A3F0'u), + uint64x2(hi: 0xF24A01A73CF2DCCF'u, lo: 0xBC633B39673C8CEC'u), + uint64x2(hi: 0x976E41088617CA01'u, lo: 0xD5BE0503E085D813'u), + uint64x2(hi: 0xBD49D14AA79DBC82'u, lo: 0x4B2D8644D8A74E18'u), + uint64x2(hi: 0xEC9C459D51852BA2'u, lo: 0xDDF8E7D60ED1219E'u), + uint64x2(hi: 0x93E1AB8252F33B45'u, lo: 0xCABB90E5C942B503'u), + uint64x2(hi: 0xB8DA1662E7B00A17'u, lo: 0x3D6A751F3B936243'u), + uint64x2(hi: 0xE7109BFBA19C0C9D'u, lo: 0x0CC512670A783AD4'u), + uint64x2(hi: 0x906A617D450187E2'u, lo: 0x27FB2B80668B24C5'u), + uint64x2(hi: 0xB484F9DC9641E9DA'u, lo: 0xB1F9F660802DEDF6'u), + uint64x2(hi: 0xE1A63853BBD26451'u, lo: 0x5E7873F8A0396973'u), + uint64x2(hi: 0x8D07E33455637EB2'u, lo: 0xDB0B487B6423E1E8'u), + uint64x2(hi: 0xB049DC016ABC5E5F'u, lo: 0x91CE1A9A3D2CDA62'u), + uint64x2(hi: 0xDC5C5301C56B75F7'u, lo: 0x7641A140CC7810FB'u), + uint64x2(hi: 0x89B9B3E11B6329BA'u, lo: 0xA9E904C87FCB0A9D'u), + uint64x2(hi: 0xAC2820D9623BF429'u, lo: 0x546345FA9FBDCD44'u), + uint64x2(hi: 0xD732290FBACAF133'u, lo: 0xA97C177947AD4095'u), + uint64x2(hi: 0x867F59A9D4BED6C0'u, lo: 0x49ED8EABCCCC485D'u), + uint64x2(hi: 0xA81F301449EE8C70'u, lo: 0x5C68F256BFFF5A74'u), + uint64x2(hi: 0xD226FC195C6A2F8C'u, lo: 0x73832EEC6FFF3111'u), + uint64x2(hi: 0x83585D8FD9C25DB7'u, lo: 0xC831FD53C5FF7EAB'u), + uint64x2(hi: 0xA42E74F3D032F525'u, lo: 0xBA3E7CA8B77F5E55'u), + uint64x2(hi: 0xCD3A1230C43FB26F'u, lo: 0x28CE1BD2E55F35EB'u), + uint64x2(hi: 0x80444B5E7AA7CF85'u, lo: 0x7980D163CF5B81B3'u), + uint64x2(hi: 0xA0555E361951C366'u, lo: 0xD7E105BCC332621F'u), + uint64x2(hi: 0xC86AB5C39FA63440'u, lo: 0x8DD9472BF3FEFAA7'u), + uint64x2(hi: 0xFA856334878FC150'u, lo: 0xB14F98F6F0FEB951'u), + uint64x2(hi: 0x9C935E00D4B9D8D2'u, lo: 0x6ED1BF9A569F33D3'u), + uint64x2(hi: 0xC3B8358109E84F07'u, lo: 0x0A862F80EC4700C8'u), + uint64x2(hi: 0xF4A642E14C6262C8'u, lo: 0xCD27BB612758C0FA'u), + uint64x2(hi: 0x98E7E9CCCFBD7DBD'u, lo: 0x8038D51CB897789C'u), + uint64x2(hi: 0xBF21E44003ACDD2C'u, lo: 0xE0470A63E6BD56C3'u), + uint64x2(hi: 0xEEEA5D5004981478'u, lo: 0x1858CCFCE06CAC74'u), + uint64x2(hi: 0x95527A5202DF0CCB'u, lo: 0x0F37801E0C43EBC8'u), + uint64x2(hi: 0xBAA718E68396CFFD'u, lo: 0xD30560258F54E6BA'u), + uint64x2(hi: 0xE950DF20247C83FD'u, lo: 0x47C6B82EF32A2069'u), + uint64x2(hi: 0x91D28B7416CDD27E'u, lo: 0x4CDC331D57FA5441'u), + uint64x2(hi: 0xB6472E511C81471D'u, lo: 0xE0133FE4ADF8E952'u), + uint64x2(hi: 0xE3D8F9E563A198E5'u, lo: 0x58180FDDD97723A6'u), + uint64x2(hi: 0x8E679C2F5E44FF8F'u, lo: 0x570F09EAA7EA7648'u), + uint64x2(hi: 0xB201833B35D63F73'u, lo: 0x2CD2CC6551E513DA'u), + uint64x2(hi: 0xDE81E40A034BCF4F'u, lo: 0xF8077F7EA65E58D1'u), + uint64x2(hi: 0x8B112E86420F6191'u, lo: 0xFB04AFAF27FAF782'u), + uint64x2(hi: 0xADD57A27D29339F6'u, lo: 0x79C5DB9AF1F9B563'u), + uint64x2(hi: 0xD94AD8B1C7380874'u, lo: 0x18375281AE7822BC'u), + uint64x2(hi: 0x87CEC76F1C830548'u, lo: 0x8F2293910D0B15B5'u), + uint64x2(hi: 0xA9C2794AE3A3C69A'u, lo: 0xB2EB3875504DDB22'u), + uint64x2(hi: 0xD433179D9C8CB841'u, lo: 0x5FA60692A46151EB'u), + uint64x2(hi: 0x849FEEC281D7F328'u, lo: 0xDBC7C41BA6BCD333'u), + uint64x2(hi: 0xA5C7EA73224DEFF3'u, lo: 0x12B9B522906C0800'u), + uint64x2(hi: 0xCF39E50FEAE16BEF'u, lo: 0xD768226B34870A00'u), + uint64x2(hi: 0x81842F29F2CCE375'u, lo: 0xE6A1158300D46640'u), + uint64x2(hi: 0xA1E53AF46F801C53'u, lo: 0x60495AE3C1097FD0'u), + uint64x2(hi: 0xCA5E89B18B602368'u, lo: 0x385BB19CB14BDFC4'u), + uint64x2(hi: 0xFCF62C1DEE382C42'u, lo: 0x46729E03DD9ED7B5'u), + uint64x2(hi: 0x9E19DB92B4E31BA9'u, lo: 0x6C07A2C26A8346D1'u), + uint64x2(hi: 0xC5A05277621BE293'u, lo: 0xC7098B7305241885'u), + uint64x2(hi: 0xF70867153AA2DB38'u, lo: 0xB8CBEE4FC66D1EA7'u)] + dragonbox_Assert(k >= kMin) + dragonbox_Assert(k <= kMax) + return pow10[k - kMin] + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2*(value: uint64; e2: int32): bool {.inline.} = + dragonbox_Assert(e2 >= 0) + return e2 < 64 and (value and ((uint64(1) shl e2) - 1)) == 0 + +## Returns whether value is divisible by 5^e5 + +proc multipleOfPow5*(value: uint64; e5: int32): bool {.inline.} = + type + MulCmp {.bycopy.} = object + mul: uint64 + cmp: uint64 + + const + mod5 = [MulCmp(mul: 0x0000000000000001'u, cmp: 0xFFFFFFFFFFFFFFFF'u), + MulCmp(mul: 0xCCCCCCCCCCCCCCCD'u, cmp: 0x3333333333333333'u), + MulCmp(mul: 0x8F5C28F5C28F5C29'u, cmp: 0x0A3D70A3D70A3D70'u), + MulCmp(mul: 0x1CAC083126E978D5'u, cmp: 0x020C49BA5E353F7C'u), + MulCmp(mul: 0xD288CE703AFB7E91'u, cmp: 0x0068DB8BAC710CB2'u), + MulCmp(mul: 0x5D4E8FB00BCBE61D'u, cmp: 0x0014F8B588E368F0'u), + MulCmp(mul: 0x790FB65668C26139'u, cmp: 0x000431BDE82D7B63'u), + MulCmp(mul: 0xE5032477AE8D46A5'u, cmp: 0x0000D6BF94D5E57A'u), + MulCmp(mul: 0xC767074B22E90E21'u, cmp: 0x00002AF31DC46118'u), + MulCmp(mul: 0x8E47CE423A2E9C6D'u, cmp: 0x0000089705F4136B'u), + MulCmp(mul: 0x4FA7F60D3ED61F49'u, cmp: 0x000001B7CDFD9D7B'u), + MulCmp(mul: 0x0FEE64690C913975'u, cmp: 0x00000057F5FF85E5'u), + MulCmp(mul: 0x3662E0E1CF503EB1'u, cmp: 0x000000119799812D'u), + MulCmp(mul: 0xA47A2CF9F6433FBD'u, cmp: 0x0000000384B84D09'u), + MulCmp(mul: 0x54186F653140A659'u, cmp: 0x00000000B424DC35'u), + MulCmp(mul: 0x7738164770402145'u, cmp: 0x0000000024075F3D'u), + MulCmp(mul: 0xE4A4D1417CD9A041'u, cmp: 0x000000000734ACA5'u), + MulCmp(mul: 0xC75429D9E5C5200D'u, cmp: 0x000000000170EF54'u), + MulCmp(mul: 0xC1773B91FAC10669'u, cmp: 0x000000000049C977'u), + MulCmp(mul: 0x26B172506559CE15'u, cmp: 0x00000000000EC1E4'u), + MulCmp(mul: 0xD489E3A9ADDEC2D1'u, cmp: 0x000000000002F394'u), + MulCmp(mul: 0x90E860BB892C8D5D'u, cmp: 0x000000000000971D'u), + MulCmp(mul: 0x502E79BF1B6F4F79'u, cmp: 0x0000000000001E39'u), + MulCmp(mul: 0xDCD618596BE30FE5'u, cmp: 0x000000000000060B'u), + MulCmp(mul: 0x2C2AD1AB7BFA3661'u, cmp: 0x0000000000000135'u)] + dragonbox_Assert(e5 >= 0) + dragonbox_Assert(e5 <= 24) + let m5: MulCmp = mod5[e5] + return value * m5.mul <= m5.cmp + +type + FloatingDecimal64* {.bycopy.} = object + significand*: uint64 + exponent*: int32 + + +proc toDecimal64AsymmetricInterval*(e2: int32): FloatingDecimal64 {.inline.} = + ## NB: + ## accept_lower_endpoint = true + ## accept_upper_endpoint = true + const + P: int32 = significandSize + ## Compute k and beta + let minusK: int32 = floorLog10ThreeQuartersPow2(e2) + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + ## Compute xi and zi + let pow10: uint64x2 = computePow10(-minusK) + let lowerEndpoint: uint64 = (pow10.hi - (pow10.hi shr (P + 1))) shr + (64 - P - betaMinus1) + let upperEndpoint: uint64 = (pow10.hi + (pow10.hi shr (P + 0))) shr + (64 - P - betaMinus1) + ## If we don't accept the left endpoint (but we do!) or + ## if the left endpoint is not an integer, increase it + let lowerEndpointIsInteger: bool = (2 <= e2 and e2 <= 3) + let xi: uint64 = lowerEndpoint + uint64(not lowerEndpointIsInteger) + let zi: uint64 = upperEndpoint + ## Try bigger divisor + var q: uint64 = zi div 10 + if q * 10 >= xi: + return FloatingDecimal64(significand: q, exponent: minusK + 1) + q = ((pow10.hi shr (64 - (P + 1) - betaMinus1)) + 1) div 2 + ## When tie occurs, choose one of them according to the rule + if e2 == -77: + dec(q, ord(q mod 2 != 0)) + ## Round to even. + else: + inc(q, ord(q < xi)) + return FloatingDecimal64(significand: q, exponent: minusK) + +proc computeDelta*(pow10: uint64x2; betaMinus1: int32): uint32 {.inline.} = + dragonbox_Assert(betaMinus1 >= 0) + dragonbox_Assert(betaMinus1 <= 63) + return cast[uint32](pow10.hi shr (64 - 1 - betaMinus1)) + +when defined(sizeof_Int128): + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + ## 1 mulx + type + uint128T = uint128 + let p: uint128T = uint128T(x) * y + let hi: uint64 = cast[uint64](p shr 64) + let lo: uint64 = cast[uint64](p) + return (hi, lo) + +elif defined(vcc) and defined(cpu64): + proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = + var hi: uint64 = 0 + var lo: uint64 = umul128(x, y, addr(hi)) + return (hi, lo) + +else: + proc lo32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + + proc hi32*(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + + proc mul128*(a: uint64; b: uint64): uint64x2 {.inline.} = + let b00: uint64 = uint64(lo32(a)) * lo32(b) + let b01: uint64 = uint64(lo32(a)) * hi32(b) + let b10: uint64 = uint64(hi32(a)) * lo32(b) + let b11: uint64 = uint64(hi32(a)) * hi32(b) + let mid1: uint64 = b10 + hi32(b00) + let mid2: uint64 = b01 + lo32(mid1) + let hi: uint64 = b11 + hi32(mid1) + hi32(mid2) + let lo: uint64 = lo32(b00) or uint64(lo32(mid2)) shl 32 + return uint64x2(hi: hi, lo: lo) + +## Returns (x * y) / 2^128 + +proc mulShift*(x: uint64; y: uint64x2): uint64 {.inline.} = + ## 2 mulx + var p1: uint64x2 = mul128(x, y.hi) + var p0: uint64x2 = mul128(x, y.lo) + p1.lo += p0.hi + inc(p1.hi, ord(p1.lo < p0.hi)) + return p1.hi + +proc mulParity*(twoF: uint64; pow10: uint64x2; betaMinus1: int32): bool {.inline.} = + ## 1 mulx, 1 mul + dragonbox_Assert(betaMinus1 >= 1) + dragonbox_Assert(betaMinus1 <= 63) + let p01: uint64 = twoF * pow10.hi + let p10: uint64 = mul128(twoF, pow10.lo).hi + let mid: uint64 = p01 + p10 + return (mid and (uint64(1) shl (64 - betaMinus1))) != 0 + +proc isIntegralEndpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -2: + return false + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc isIntegralMidpoint*(twoF: uint64; e2: int32; minusK: int32): bool {.inline.} = + if e2 < -4: + return multipleOfPow2(twoF, minusK - e2 + 1) + if e2 <= 9: + return true + if e2 <= 86: + return multipleOfPow5(twoF, minusK) + return false + +proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecimal64 {. + inline.} = + const + kappa: int32 = 2 + const + bigDivisor: uint32 = 1000 + ## 10^(kappa + 1) + const + smallDivisor: uint32 = 100 + ## 10^(kappa) + ## + ## Step 1: + ## integer promotion & Schubfach multiplier calculation. + ## + var m2: uint64 + var e2: int32 + if ieeeExponent != 0: + m2 = hiddenBit or ieeeSignificand + e2 = cast[int32](ieeeExponent) - exponentBias + if 0 <= -e2 and -e2 < significandSize and multipleOfPow2(m2, -e2): + ## Small integer. + return FloatingDecimal64(significand: m2 shr -e2, exponent: 0) + if ieeeSignificand == 0 and ieeeExponent > 1: + ## Shorter interval case; proceed like Schubfach. + return toDecimal64AsymmetricInterval(e2) + else: + ## Subnormal case; interval is always regular. + m2 = ieeeSignificand + e2 = 1 - exponentBias + let isEven: bool = (m2 mod 2 == 0) + let acceptLower: bool = isEven + let acceptUpper: bool = isEven + ## Compute k and beta. + let minusK: int32 = floorLog10Pow2(e2) - kappa + let betaMinus1: int32 = e2 + floorLog2Pow10(-minusK) + dragonbox_Assert(betaMinus1 >= 6) + dragonbox_Assert(betaMinus1 <= 9) + let pow10: uint64x2 = computePow10(-minusK) + ## Compute delta + ## 10^kappa <= delta < 10^(kappa + 1) + ## 100 <= delta < 1000 + let delta: uint32 = computeDelta(pow10, betaMinus1) + dragonbox_Assert(delta >= smallDivisor) + dragonbox_Assert(delta < bigDivisor) + let twoFl: uint64 = 2 * m2 - 1 + let twoFc: uint64 = 2 * m2 + let twoFr: uint64 = 2 * m2 + 1 + ## (54 bits) + ## Compute zi + ## (54 + 9 = 63 bits) + let zi: uint64 = mulShift(twoFr shl betaMinus1, pow10) + ## 2 mulx + ## + ## Step 2: + ## Try larger divisor. + ## + var q: uint64 = zi div bigDivisor + ## uint64_t q = Mul128(zi, 0x83126E978D4FDF3Cu).hi >> 9; // 1 mulx + var r: uint32 = cast[uint32](zi) - bigDivisor * cast[uint32](q) + ## r = zi % BigDivisor + ## 0 <= r < 1000 + if r < delta: ## likely ~50% ?! + ## (r > deltai) + ## Exclude the right endpoint if necessary + if r != 0 or acceptUpper or not isIntegralEndpoint(twoFr, e2, minusK): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + dragonbox_Assert(q != 0) + dec(q) + r = bigDivisor + elif r == delta: ## unlikely + ## Compare fractional parts. + ## Check conditions in the order different from the paper + ## to take advantage of short-circuiting + if (acceptLower and isIntegralEndpoint(twoFl, e2, minusK)) or + mulParity(twoFl, pow10, betaMinus1): + return FloatingDecimal64(significand: q, exponent: minusK + kappa + 1) + else: + discard + ## + ## Step 3: + ## Find the significand with the smaller divisor + ## + q = q * 10 + ## 1 hmul + ## 0 <= r <= 1000 + let dist: uint32 = r - (delta div 2) + (smallDivisor div 2) + let distQ: uint32 = dist div 100 + ## 1 mul + ## const uint32_t dist_r = dist % 100; + q += distQ + ## if /*likely*/ (dist_r == 0) + if dist == distQ * 100: + ## const bool approx_y_parity = ((dist ^ (SmallDivisor / 2)) & 1) != 0; + let approxYParity: bool = (dist and 1) != 0 + ## Check z^(f) >= epsilon^(f) + ## We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1, + ## where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f) + ## Since there are only 2 possibilities, we only need to care about the + ## parity. Also, zi and r should have the same parity since the divisor + ## is an even number + if mulParity(twoFc, pow10, betaMinus1) != approxYParity: + dec(q) + elif q mod 2 != 0 and isIntegralMidpoint(twoFc, e2, minusK): + dec(q) + return FloatingDecimal64(significand: q, exponent: minusK + kappa) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + const + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + dragonbox_Assert(digits <= 99) + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = + const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + dragonbox_Assert(digits <= 99) + return trailingZeros100[digits] + +when false: + template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) + + template dec(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) - uint(offset)) + template inc(x: cstring; offset=1) = x = cast[cstring](cast[uint](x) + uint(offset)) + + proc memset(x: cstring; ch: char; L: int) {.importc, nodecl.} + proc memmove(a, b: cstring; L: int) {.importc, nodecl.} + +proc utoa8DigitsSkipTrailingZeros*(buf: var openArray[char]; pos: int; digits: uint32): int32 {.inline.} = + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999'u32) + let q: uint32 = digits div 10000 + let r: uint32 = digits mod 10000 + let qH: uint32 = q div 100 + let qL: uint32 = q mod 100 + utoa2Digits(buf, pos, qH) + utoa2Digits(buf, pos + 2, qL) + if r == 0: + return trailingZeros2Digits(if qL == 0: qH else: qL) + (if qL == 0: 6 else: 4) + else: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos + 4, rH) + utoa2Digits(buf, pos + 6, rL) + return trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + +proc printDecimalDigitsBackwards*(buf: var openArray[char]; pos: int; output64: uint64): int32 {.inline.} = + var pos = pos + var output64 = output64 + var tz: int32 = 0 + ## number of trailing zeros removed. + var nd: int32 = 0 + ## number of decimal digits processed. + ## At most 17 digits remaining + if output64 >= 100000000'u64: + let q: uint64 = output64 div 100000000'u64 + let r: uint32 = cast[uint32](output64 mod 100000000'u64) + output64 = q + dec(pos, 8) + if r != 0: + tz = utoa8DigitsSkipTrailingZeros(buf, pos, r) + dragonbox_Assert(tz >= 0) + dragonbox_Assert(tz <= 7) + else: + tz = 8 + nd = 8 + dragonbox_Assert(output64 <= high(uint32)) + var output = cast[uint32](output64) + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + if tz == nd: + inc(tz, trailingZeros2Digits(if rL == 0: rH else: rL) + + (if rL == 0: 2 else: 0)) + else: + if tz == nd: + inc(tz, 4) + else: + for i in 0..3: buf[pos+i] = '0' + ## (actually not required...) + inc(nd, 4) + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + dragonbox_Assert(output >= 1) + dragonbox_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + dragonbox_Assert(q >= 1) + dragonbox_Assert(q <= 9) + dec(pos) + buf[pos] = chr(ord('0') + q) + return tz + +proc decimalLength*(v: uint64): int32 {.inline.} = + dragonbox_Assert(v >= 1) + dragonbox_Assert(v <= 99999999999999999'u64) + if cast[uint32](v shr 32) != 0: + if v >= 10000000000000000'u64: + return 17 + if v >= 1000000000000000'u64: + return 16 + if v >= 100000000000000'u64: + return 15 + if v >= 10000000000000'u64: + return 14 + if v >= 1000000000000'u64: + return 13 + if v >= 100000000000'u64: + return 12 + if v >= 10000000000'u64: + return 11 + return 10 + let v32: uint32 = cast[uint32](v) + if v32 >= 1000000000'u: + return 10 + if v32 >= 100000000'u: + return 9 + if v32 >= 10000000'u: + return 8 + if v32 >= 1000000'u: + return 7 + if v32 >= 100000'u: + return 6 + if v32 >= 10000'u: + return 5 + if v32 >= 1000'u: + return 4 + if v32 >= 100'u: + return 3 + if v32 >= 10'u: + return 2 + return 1 + +proc formatDigits*(buffer: var openArray[char]; pos: int; digits: uint64; decimalExponent: int32; + forceTrailingDotZero = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -6 + const + maxFixedDecimalPoint: int32 = 17 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 17, "internal error") + dragonbox_Assert(digits >= 1) + dragonbox_Assert(digits <= 99999999999999999'u64) + dragonbox_Assert(decimalExponent >= -999) + dragonbox_Assert(decimalExponent <= 999) + var numDigits: int32 = decimalLength(digits) + let decimalPoint: int32 = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int32 + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + int(decimalDigitsPosition + numDigits) + let tz: int32 = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + when true: #defined(vcc) and not defined(clang): + ## VC does not inline the memmove call below. (Even if compiled with /arch:AVX2.) + ## However, memcpy will be inlined. + var tmp: array[16, char] + for i in 0..<16: tmp[i] = buffer[i+pos+decimalPoint] + for i in 0..<16: buffer[i+pos+decimalPoint+1] = tmp[i] + else: + memmove(buffer +! (decimalPoint + 1), buffer +! decimalPoint, 16) + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + ## Copy the first digit one place to the left. + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int32 = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(ord('0') + k) + inc(pos) + elif k < 100: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + else: + let q: uint32 = k div 100 + let r: uint32 = k mod 100 + buffer[pos] = chr(ord('0') + q) + inc(pos) + utoa2Digits(buffer, pos, r) + inc(pos, 2) + return pos + +proc toChars*(buffer: var openArray[char]; v: float; forceTrailingDotZero = false): int {. + inline.} = + var pos = 0 + let significand: uint64 = physicalSignificand(constructDouble(v)) + let exponent: uint64 = physicalExponent(constructDouble(v)) + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec = toDecimal64(significand, exponent) + return formatDigits(buffer, pos, dec.significand, dec.exponent, + forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructDouble(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 + +when false: + proc toString*(value: float): string = + var buffer: array[dtoaMinBufferLength, char] + let last = toChars(addr buffer, value) + let L = cast[int](last) - cast[int](addr(buffer)) + result = newString(L) + copyMem(addr result[0], addr buffer[0], L) + diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim index cb46c8c36..41bc56adc 100644 --- a/lib/system/formatfloat.nim +++ b/lib/system/formatfloat.nim @@ -7,53 +7,58 @@ # distribution, for details about the copyright. # -proc c_sprintf(buf, frmt: cstring): cint {.header: "", - importc: "sprintf", varargs, noSideEffect.} +when not defined(nimLegacyAddFloat) and not defined(nimscript) and + not defined(js) and defined(nimHasDragonBox): + import dragonbox -proc writeToBuffer(buf: var array[65, char]; value: cstring) = - var i = 0 - while value[i] != '\0': - buf[i] = value[i] - inc i + proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + result = toChars(buf, value, forceTrailingDotZero=true) + buf[result] = '\0' -proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int = - ## This is the implementation to format floats in the Nim - ## programming language. The specific format for floating point - ## numbers is not specified in the Nim programming language and - ## might change slightly in the future, but at least wherever you - ## format a float, it should be consistent. - ## - ## returns the amount of bytes written to `buf` not counting the - ## terminating '\0' character. - ## - ## * `buf` - A buffer to write into. The buffer does not need to be - ## initialized and it will be overridden. - ## - var n: int = c_sprintf(addr buf, "%.16g", value) - var hasDot = false - for i in 0..n-1: - if buf[i] == ',': - buf[i] = '.' - hasDot = true - elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: - hasDot = true - if not hasDot: - buf[n] = '.' - buf[n+1] = '0' - buf[n+2] = '\0' - result = n + 2 - else: - result = n - # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' - # of '-1.#IND' are produced. - # We want to get rid of these here: - if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: - writeToBuffer(buf, "nan") - result = 3 - elif buf[n-1] == 'F': - if buf[0] == '-': - writeToBuffer(buf, "-inf") - result = 4 +else: + proc c_sprintf(buf, frmt: cstring): cint {.header: "", + importc: "sprintf", varargs, noSideEffect.} + + proc writeToBuffer(buf: var array[65, char]; value: cstring) = + var i = 0 + while value[i] != '\0': + buf[i] = value[i] + inc i + + proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int = + ## This is the implementation to format floats. + ## + ## returns the amount of bytes written to `buf` not counting the + ## terminating '\0' character. + var n: int = c_sprintf(addr buf, "%.16g", value) + var hasDot = false + for i in 0..n-1: + if buf[i] == ',': + buf[i] = '.' + hasDot = true + elif buf[i] in {'a'..'z', 'A'..'Z', '.'}: + hasDot = true + if not hasDot: + buf[n] = '.' + buf[n+1] = '0' + buf[n+2] = '\0' + result = n + 2 else: - writeToBuffer(buf, "inf") + result = n + # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)' + # of '-1.#IND' are produced. + # We want to get rid of these here: + if buf[n-1] in {'n', 'N', 'D', 'd', ')'}: + writeToBuffer(buf, "nan") result = 3 + elif buf[n-1] == 'F': + if buf[0] == '-': + writeToBuffer(buf, "-inf") + result = 4 + else: + writeToBuffer(buf, "inf") + result = 3 diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim index c4cd6e932..c0e6e419f 100644 --- a/tests/float/tfloat6.nim +++ b/tests/float/tfloat6.nim @@ -1,10 +1,10 @@ discard """ output: ''' -1e-06 : 1e-06 -1e-06 : 1e-06 +0.000001 : 0.000001 +0.000001 : 0.000001 0.001 : 0.001 -1e-06 : 1e-06 -1e-06 : 1e-06 +0.000001 : 0.000001 +0.000001 : 0.000001 10.000001 : 10.000001 100.000001 : 100.000001 ''' From f10eef29b5d77ad45b00b0d4eb7e87f930c2a0a4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 2 Jun 2021 13:13:23 +0200 Subject: [PATCH 0049/1654] fixes #18059 (#18140) * fixes #18059 --- compiler/semexprs.nim | 28 ++++++++++++++++++++++++++-- compiler/semstmts.nim | 20 +++++++++++++++++++- tests/metatype/tcps.nim | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 tests/metatype/tcps.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 8c4f25e12..c839d04d9 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -54,7 +54,8 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExpr(c, n, flags + {efOperand}) if result.typ != nil: # XXX tyGenericInst here? - if result.typ.kind == tyProc and tfUnresolved in result.typ.flags: + if result.typ.kind == tyProc and hasUnresolvedParams(result, {efOperand}): + #and tfUnresolved in result.typ.flags: localError(c.config, n.info, errProcHasNoConcreteType % n.renderTree) if result.typ.kind in {tyVar, tyLent}: result = newDeref(result) elif {efWantStmt, efAllowStmt} * flags != {}: @@ -2649,6 +2650,23 @@ proc shouldBeBracketExpr(n: PNode): bool = n[0] = be return true +proc asBracketExpr(c: PContext; n: PNode): PNode = + proc isGeneric(c: PContext; n: PNode): bool = + if n.kind in {nkIdent, nkAccQuoted}: + let s = qualifiedLookUp(c, n, {}) + result = s != nil and isGenericRoutineStrict(s) + + assert n.kind in nkCallKinds + if n.len > 1 and isGeneric(c, n[1]): + let b = n[0] + if b.kind in nkSymChoices: + for i in 0.. Date: Wed, 2 Jun 2021 13:14:10 +0200 Subject: [PATCH 0050/1654] refactoring: removed empty withBracketExpr template (#18152) --- compiler/semgnrc.nim | 6 ++---- compiler/semtempl.nim | 14 ++++---------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index ebd361db6..4f3c50dae 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -303,8 +303,7 @@ proc semGenericStmt(c: PContext, n: PNode, result = newNodeI(nkCall, n.info) result.add newIdentNode(getIdent(c.cache, "[]"), n.info) for i in 0.. Date: Wed, 2 Jun 2021 09:02:14 -0700 Subject: [PATCH 0052/1654] fix #16993, #18054, #17835 runnableExamples now works with templates and nested templates (#18082) --- compiler/evaltempl.nim | 12 ++-- lib/system/assertions.nim | 47 ++++++--------- .../expected/subdir/subdir_b/utils.html | 4 +- nimdoc/testproject/expected/testproject.html | 11 ++-- nimdoc/testproject/subdir/subdir_b/utils.nim | 10 +++- tests/js/tstdlib_various.nim | 2 +- tests/nimdoc/trunnableexamples.nim | 57 ++++++++++++++++++- tests/nimdoc/trunnableexamples2.nim | 11 ++++ 8 files changed, 108 insertions(+), 46 deletions(-) create mode 100644 tests/nimdoc/trunnableexamples2.nim diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index a85314ac2..8a6bf9287 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -83,10 +83,14 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = not c.isDeclarative: c.isDeclarative = true isDeclarative = true - var res = copyNode(c, templ, actual) - for i in 0..`_. + runnableExamples: assert 1 == 1 + runnableExamples("--assertions:off"): + assert 1 == 2 # no code generated, no failure here + runnableExamples("-d:danger"): assert 1 == 2 # ditto assertImpl(cond, msg, astToStr(cond), compileOption("assertions")) template doAssert*(cond: untyped, msg = "") = ## Similar to `assert <#assert.t,untyped,string>`_ but is always turned on regardless of `--assertions`. + runnableExamples: + doAssert 1 == 1 # generates code even when built with `-d:danger` or `--assertions:off` assertImpl(cond, msg, astToStr(cond), true) template onFailedAssert*(msg, code: untyped): untyped {.dirty.} = ## Sets an assertion failure handler that will intercept any assert ## statements following `onFailedAssert` in the current scope. + runnableExamples: + type MyError = object of CatchableError + lineinfo: tuple[filename: string, line: int, column: int] + # block-wide policy to change the failed assert exception type in order to + # include a lineinfo + onFailedAssert(msg): + raise (ref MyError)(msg: msg, lineinfo: instantiationInfo(-2)) + doAssertRaises(MyError): doAssert false template failedAssertImpl(msgIMPL: string): untyped {.dirty.} = let msg = msgIMPL code template doAssertRaises*(exception: typedesc, code: untyped) = ## Raises `AssertionDefect` if specified `code` does not raise `exception`. + runnableExamples: + doAssertRaises(ValueError): raise newException(ValueError, "Hello World") + doAssertRaises(CatchableError): raise newException(ValueError, "Hello World") + doAssertRaises(AssertionDefect): doAssert false var wrong = false const begin = "expected raising '" & astToStr(exception) & "', instead" const msgEnd = " by: " & astToStr(code) diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 794066cae..826cad02a 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -206,9 +206,9 @@ constructor. 
template fromUtilsGen(): untyped
-this should be shown in utils.html +should be shown in utils.html only

Example:

-
assert 3*2 == 6
ditto +
discard "should be in utils.html only, not in module that calls fromUtilsGen"
ditto
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index d2fe38021..513dd507b 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -431,10 +431,7 @@ window.addEventListener('DOMContentLoaded', main); discard "in top2"top2 after

Example:

import testproject
-discard "in top3"
top3 after -

Example:

-
import testproject
-assert 3*2 == 6

+discard "in top3"top3 after

Imports

@@ -574,7 +571,8 @@ My someFunc. Stuff in fromUtilsGen is called

Example:

-
discard 1
+
discard """should be shown as examples for fromUtils3
+       in module calling fromUtilsGen"""
@@ -957,6 +955,9 @@ cz18
ok3 +

Example:

+
discard """should be shown as examples for fromUtils2
+       in module calling fromUtilsGen"""
diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 128e8e481..4e3aa1f10 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -50,9 +50,9 @@ template bEnum*(): untyped = discard template fromUtilsGen*(): untyped = - ## this should be shown in utils.html + ## should be shown in utils.html only runnableExamples: - assert 3*2 == 6 + discard "should be in utils.html only, not in module that calls fromUtilsGen" ## ditto iterator fromUtils1*(): int = @@ -64,7 +64,11 @@ template fromUtilsGen*(): untyped = template fromUtils2*() = ## ok3 + runnableExamples: + discard """should be shown as examples for fromUtils2 + in module calling fromUtilsGen""" proc fromUtils3*() = ## came form utils but should be shown where `fromUtilsGen` is called - runnableExamples: discard 1 + runnableExamples: discard """should be shown as examples for fromUtils3 + in module calling fromUtilsGen""" diff --git a/tests/js/tstdlib_various.nim b/tests/js/tstdlib_various.nim index a1bb63d46..4b5ce1de8 100644 --- a/tests/js/tstdlib_various.nim +++ b/tests/js/tstdlib_various.nim @@ -153,7 +153,7 @@ block tsplit2: var errored = false try: discard "hello".split("") - except AssertionError: + except AssertionDefect: errored = true doAssert errored diff --git a/tests/nimdoc/trunnableexamples.nim b/tests/nimdoc/trunnableexamples.nim index bc5ea0e5e..ac7a0e26f 100644 --- a/tests/nimdoc/trunnableexamples.nim +++ b/tests/nimdoc/trunnableexamples.nim @@ -1,6 +1,7 @@ discard """ -cmd: "nim doc --doccmd:-d:testFooExternal --hints:off $file" +cmd: "nim doc --doccmd:--hints:off --hints:off $file" action: "compile" +nimoutFull: true nimout: ''' foo1 foo2 @@ -8,12 +9,23 @@ foo3 foo5 foo6 foo7 +in examplesInTemplate1 +doc in outer +doc in inner1 +doc in inner2 foo8 foo9 ''' joinable: false """ +#[ +pending bug #18077, use instead: +cmd: "nim doc --doccmd:'-d:testFooExternal --hints:off' --hints:off $file" +and merge trunnableexamples2 back here +]# +{.define(testFooExternal).} + proc fun*() = runnableExamples: block: # `defer` only allowed inside a block @@ -109,6 +121,47 @@ when true: # runnableExamples with rdoccmd # passing seq (to run with multiple compilation options) runnableExamples(@["-b:cpp", "-b:js"]): discard +when true: # bug #16993 + template examplesInTemplate1*(cond: untyped) = + ## in examplesInTemplate1 + runnableExamples: + echo "in examplesInTemplate1" + discard + examplesInTemplate1 true + examplesInTemplate1 true + examplesInTemplate1 true + +when true: # bug #18054 + template outer*(body: untyped) = + ## outer template doc string. + runnableExamples: + echo "doc in outer" + ## + template inner1*() = + ## inner1 template doc string. + runnableExamples: + echo "doc in inner1" + ## + + template inner2*() = + ## inner2 template doc string. + runnableExamples: + echo "doc in inner2" + body + outer: + inner1() + inner2() + +when true: # bug #17835 + template anyItFake*(s, pred: untyped): bool = + ## Foo + runnableExamples: discard + true + + proc anyItFakeMain*(n: seq[int]): bool = + result = anyItFake(n, it == 0) + # this was giving: Error: runnableExamples must appear before the first non-comment statement + runnableExamples: block: # bug #17279 when int.sizeof == 8: @@ -118,7 +171,7 @@ runnableExamples: # bug #13491 block: proc fun(): int = doAssert false - doAssertRaises(AssertionError, (discard fun())) + doAssertRaises(AssertionDefect, (discard fun())) block: template foo(body) = discard diff --git a/tests/nimdoc/trunnableexamples2.nim b/tests/nimdoc/trunnableexamples2.nim new file mode 100644 index 000000000..5a437744e --- /dev/null +++ b/tests/nimdoc/trunnableexamples2.nim @@ -0,0 +1,11 @@ +discard """ +cmd: "nim doc --doccmd:-d:testFooExternal --hints:off $file" +action: "compile" +joinable: false +""" + +# pending bug #18077, merge back inside trunnableexamples.nim +when true: # runnableExamples with rdoccmd + runnableExamples "-d:testFoo -d:testBar": + doAssert defined(testFoo) and defined(testBar) + doAssert defined(testFooExternal) From f27f3f65df0bab567cc416cd1332a502833bd12f Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 10:59:06 +0800 Subject: [PATCH 0053/1654] [std/hashcommon]improve docs a bit (#18153) * reduce duplicated deprecated messages --- lib/pure/collections/hashcommon.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index 030446176..a169418ce 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -34,10 +34,10 @@ proc slotsNeeded(count: Natural): int {.inline.} = result = nextPowerOfTwo(count * 3 div 2 + 4) proc rightSize*(count: Natural): int {.inline, deprecated: "Deprecated since 1.4.0".} = - ## **Deprecated since Nim v1.4.0**, it is not needed anymore - ## because picking the correct size is done internally. + ## It is not needed anymore because + ## picking the correct size is done internally. ## - ## Return the value of `initialSize` to support `count` items. + ## Returns the value of `initialSize` to support `count` items. ## ## If more items are expected to be added, simply add that ## expected extra amount to the parameter before calling this. From bbce3d2da919bef2900e4061b9b8a99cb84fdf57 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 12:09:04 +0800 Subject: [PATCH 0054/1654] [std/tables] remove unnecessary `do: ` (#18160) --- lib/pure/collections/tables.nim | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index b71bb0845..d617e06e4 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -594,17 +594,17 @@ template withValue*[A, B](t: var Table[A, B], key: A, value, body: untyped) = let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - t.withValue(2, value) do: + t.withValue(2, value): value.name = "No" value.uid = 521 - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + assert t[1].name == "Nim" + assert t[1].uid == 1314 mixin rawGet var hc: Hash @@ -629,16 +629,22 @@ template withValue*[A, B](t: var Table[A, B], key: A, let u = User(name: "Hello", uid: 99) t[1] = u - t.withValue(1, value) do: + t.withValue(1, value): # block is executed only if `key` in `t` value.name = "Nim" value.uid = 1314 - # do: - # # block is executed when `key` not in `t` - # raise newException(KeyError, "Key not found") - doAssert t[1].name == "Nim" - doAssert t[1].uid == 1314 + t.withValue(521, value): + doAssert false + do: + # block is executed when `key` not in `t` + t[1314] = User(name: "exist", uid: 521) + + assert t[1].name == "Nim" + assert t[1].uid == 1314 + assert t[1314].name == "exist" + assert t[1314].uid == 521 + mixin rawGet var hc: Hash var index = rawGet(t, key, hc) From 06960bb9cb6bb548c8f72882169ae65e14f29e89 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 13:35:24 +0800 Subject: [PATCH 0055/1654] Ref #17697 improve withValue docs (#18154) * Ref #17697 improve withValue docs * address comments --- lib/pure/collections/sharedtables.nim | 81 +++++++++++++++++---------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 4ac3befb6..2788ec785 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -60,17 +60,27 @@ template withLock(t, x: untyped) = template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + table.withValue("a", value): + assert value[] == "x" + + table.withValue("b", value): + value[] = "modified" + + table.withValue("b", value): + assert value[] == "modified" + + table.withValue("nonexistent", value): + assert false # not called acquire(t.lock) try: var hc: Hash @@ -84,20 +94,29 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at `t[key]`. + ## Retrieves the value at `t[key]`. ## `value` can be modified in the scope of the `withValue` call. - ## - ## .. code-block:: nim - ## - ## sharedTable.withValue(key, value) do: - ## # block is executed only if `key` in `t` - ## # value is threadsafe in block - ## value.name = "username" - ## value.uid = 1000 - ## do: - ## # block is executed when `key` not in `t` - ## raise newException(KeyError, "Key not found") - ## + runnableExamples: + var table: SharedTable[string, string] + init(table) + + table["a"] = "x" + table["b"] = "y" + table["c"] = "z" + + + table.withValue("a", value): + value[] = "m" + + table.withValue("d", value): + discard value + doAssert false + do: # if "d" notin table + table["d"] = "n" + + assert table.mget("a") == "m" + assert table.mget("d") == "n" + acquire(t.lock) try: var hc: Hash @@ -112,7 +131,7 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, release(t.lock) proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = - ## retrieves the value at `t[key]`. The value can be modified. + ## Retrieves the value at `t[key]`. The value can be modified. ## If `key` is not in `t`, the `KeyError` exception is raised. withLock t: var hc: Hash @@ -126,7 +145,7 @@ proc mget*[A, B](t: var SharedTable[A, B], key: A): var B = raise newException(KeyError, "key not found") proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = - ## retrieves value at `t[key]` or puts `val` if not present, either way + ## Retrieves value at `t[key]` or puts `val` if not present, either way ## returning a value which can be modified. **Note**: This is inherently ## unsafe in the context of multi-threading since it returns a pointer ## to `B`. @@ -134,7 +153,7 @@ proc mgetOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): var B = mgetOrPutImpl(enlarge) proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool = - ## returns true if `key` is in the table, otherwise inserts `value`. + ## Returns true if `key` is in the table, otherwise inserts `value`. withLock t: hasKeyOrPutImpl(enlarge) @@ -191,28 +210,28 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A, st_maybeRehashPutImpl(enlarge) proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a (key, value)-pair into `t`. + ## Puts a (key, value)-pair into `t`. withLock t: putImpl(enlarge) proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) = - ## puts a new (key, value)-pair into `t` even if `t[key]` already exists. + ## Puts a new (key, value)-pair into `t` even if `t[key]` already exists. ## This can introduce duplicate keys into the table! withLock t: addImpl(enlarge) proc del*[A, B](t: var SharedTable[A, B], key: A) = - ## deletes `key` from hash table `t`. + ## Deletes `key` from hash table `t`. withLock t: delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash) proc len*[A, B](t: var SharedTable[A, B]): int = - ## number of elements in `t` + ## Number of elements in `t`. withLock t: result = t.counter proc init*[A, B](t: var SharedTable[A, B], initialSize = 32) = - ## creates a new hash table that is empty. + ## Creates a new hash table that is empty. ## ## This proc must be called before any other usage of `t`. let initialSize = slotsNeeded(initialSize) From daaa40973c65c46f8d38dc6033f56fffcd7d8d3f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 12:09:40 +0200 Subject: [PATCH 0056/1654] added float32 schubfach algorithm; wip (#18155) * added float32 schubfach algorithm; wip * fixes #18418 --- compiler/ccgexprs.nim | 6 +- lib/system/dollars.nim | 4 + lib/system/schubfach.nim | 451 +++++++++++++++++++++++++++++++++++++++ lib/system/strmantle.nim | 13 ++ tests/float/tfloat6.nim | 5 + 5 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 lib/system/schubfach.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7cb403a94..7ed80145b 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2300,7 +2300,11 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)") of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)") of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)") - of mFloatToStr: genDollar(p, e, d, "#nimFloatToStr($1)") + of mFloatToStr: + if e[1].typ.skipTypes(abstractInst).kind == tyFloat32: + genDollar(p, e, d, "#nimFloat32ToStr($1)") + else: + genDollar(p, e, d, "#nimFloatToStr($1)") of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) of mIsolate: genCall(p, e, d) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index ce4e8e0ca..5a7a77e56 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -58,6 +58,10 @@ proc `$`*(x: float): string {.magic: "FloatToStr", noSideEffect.} ## The stringify operator for a float argument. Returns `x` ## converted to a decimal string. +proc `$`*(x: float32): string {.magic: "FloatToStr", noSideEffect.} + ## The stringify operator for a float32 argument. Returns `x` + ## converted to a decimal string. + proc `$`*(x: bool): string {.magic: "BoolToStr", noSideEffect.} ## The stringify operator for a boolean argument. Returns `x` ## converted to the string "false" or "true". diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim new file mode 100644 index 000000000..c344b7425 --- /dev/null +++ b/lib/system/schubfach.nim @@ -0,0 +1,451 @@ +## Copyright 2020 Alexander Bolz +## +## Distributed under the Boost Software License, Version 1.0. +## (See accompanying file LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt) + +## -------------------------------------------------------------------------------------------------- +## This file contains an implementation of the Schubfach algorithm as described in +## +## [1] Raffaello Giulietti, "The Schubfach way to render doubles", +## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN +## -------------------------------------------------------------------------------------------------- + +template sf_Assert(x: untyped): untyped = + assert(x) + +## ================================================================================================== +## +## ================================================================================================== + +type + ValueType = float32 + BitsType = uint32 + Single {.bycopy.} = object + bits: BitsType + +const + significandSize: int32 = 24 + MaxExponent = 128 + exponentBias: int32 = MaxExponent - 1 + (significandSize - 1) + maxIeeeExponent: BitsType = BitsType(2 * MaxExponent - 1) + hiddenBit: BitsType = BitsType(1) shl (significandSize - 1) + significandMask: BitsType = hiddenBit - 1 + exponentMask: BitsType = maxIeeeExponent shl (significandSize - 1) + signMask: BitsType = not (not BitsType(0) shr 1) + +proc constructSingle(bits: BitsType): Single {.constructor.} = + result.bits = bits + +proc constructSingle(value: ValueType): Single {.constructor.} = + result.bits = cast[typeof(result.bits)](value) + +proc physicalSignificand(this: Single): BitsType {.noSideEffect.} = + return this.bits and significandMask + +proc physicalExponent(this: Single): BitsType {.noSideEffect.} = + return (this.bits and exponentMask) shr (significandSize - 1) + +proc isFinite(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) != exponentMask + +proc isInf(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) == 0 + +proc isNaN(this: Single): bool {.noSideEffect.} = + return (this.bits and exponentMask) == exponentMask and + (this.bits and significandMask) != 0 + +proc isZero(this: Single): bool {.noSideEffect.} = + return (this.bits and not signMask) == 0 + +proc signBit(this: Single): int {.noSideEffect.} = + return int((this.bits and signMask) != 0) + +## ================================================================================================== +## Returns floor(x / 2^n). +## +## Technically, right-shift of negative integers is implementation defined... +## Should easily be optimized into SAR (or equivalent) instruction. + +proc floorDivPow2(x: int32; n: int32): int32 {.inline.} = + return x shr n + +## Returns floor(log_10(2^e)) +## static inline int32_t FloorLog10Pow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611, 22); +## } +## Returns floor(log_10(3/4 2^e)) +## static inline int32_t FloorLog10ThreeQuartersPow2(int32_t e) +## { +## SF_ASSERT(e >= -1500); +## SF_ASSERT(e <= 1500); +## return FloorDivPow2(e * 1262611 - 524031, 22); +## } +## Returns floor(log_2(10^e)) + +proc floorLog2Pow10(e: int32): int32 {.inline.} = + sf_Assert(e >= -1233) + sf_Assert(e <= 1233) + return floorDivPow2(e * 1741647, 19) + +proc computePow10Single(k: int32): uint64 {.inline.} = + ## There are unique beta and r such that 10^k = beta 2^r and + ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and + ## beta = 2^-r 10^k. + ## Let g = ceil(beta), so (g-1) 2^r < 10^k <= g 2^r, with the latter + ## value being a pretty good overestimate for 10^k. + ## NB: Since for all the required exponents k, we have g < 2^64, + ## all constants can be stored in 128-bit integers. + const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + sf_Assert(k >= kMin) + sf_Assert(k <= kMax) + return g[k - kMin] + +proc lo32(x: uint64): uint32 {.inline.} = + return cast[uint32](x) + +proc hi32(x: uint64): uint32 {.inline.} = + return cast[uint32](x shr 32) + +when defined(sizeof_Int128): + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let p: uint128 = uint128(g) * cp + let y1: uint32 = lo32(cast[uint64](p shr 64)) + let y0: uint32 = hi32(cast[uint64](p)) + return y1 or uint32(y0 > 1) + +elif defined(vcc) and defined(cpu64): + proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = + var p1: uint64 = 0 + var p0: uint64 = umul128(g, cpHi, addr(p1)) + let y1: uint32 = lo32(p1) + let y0: uint32 = hi32(p0) + return y1 or uint32(y0 > 1) + +else: + proc roundToOdd(g: uint64; cp: uint32): uint32 {.inline.} = + let b01: uint64 = uint64(lo32(g)) * cp + let b11: uint64 = uint64(hi32(g)) * cp + let hi: uint64 = b11 + hi32(b01) + let y1: uint32 = hi32(hi) + let y0: uint32 = lo32(hi) + return y1 or uint32(y0 > 1) + +## Returns whether value is divisible by 2^e2 + +proc multipleOfPow2(value: uint32; e2: int32): bool {.inline.} = + sf_Assert(e2 >= 0) + sf_Assert(e2 <= 31) + return (value and ((uint32(1) shl e2) - 1)) == 0 + +type + FloatingDecimal32 {.bycopy.} = object + digits: uint32 ## num_digits <= 9 + exponent: int32 + +proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal32 {. + inline.} = + var c: uint32 + var q: int32 + if ieeeExponent != 0: + c = hiddenBit or ieeeSignificand + q = cast[int32](ieeeExponent) - exponentBias + if 0 <= -q and -q < significandSize and multipleOfPow2(c, -q): + return FloatingDecimal32(digits: c shr -q, exponent: 0'i32) + else: + c = ieeeSignificand + q = 1 - exponentBias + let isEven: bool = (c mod 2 == 0) + let lowerBoundaryIsCloser: bool = (ieeeSignificand == 0 and ieeeExponent > 1) + ## const int32_t qb = q - 2; + let cbl: uint32 = 4 * c - 2 + uint32(lowerBoundaryIsCloser) + let cb: uint32 = 4 * c + let cbr: uint32 = 4 * c + 2 + ## (q * 1262611 ) >> 22 == floor(log_10( 2^q)) + ## (q * 1262611 - 524031) >> 22 == floor(log_10(3/4 2^q)) + sf_Assert(q >= -1500) + sf_Assert(q <= 1500) + let k: int32 = floorDivPow2(q * 1262611 - (if lowerBoundaryIsCloser: 524031 else: 0), 22) + let h: int32 = q + floorLog2Pow10(-k) + 1 + sf_Assert(h >= 1) + sf_Assert(h <= 4) + let pow10: uint64 = computePow10Single(-k) + let vbl: uint32 = roundToOdd(pow10, cbl shl h) + let vb: uint32 = roundToOdd(pow10, cb shl h) + let vbr: uint32 = roundToOdd(pow10, cbr shl h) + let lower: uint32 = vbl + uint32(not isEven) + let upper: uint32 = vbr - uint32(not isEven) + ## See Figure 4 in [1]. + ## And the modifications in Figure 6. + let s: uint32 = vb div 4 + ## NB: 4 * s == vb & ~3 == vb & -4 + if s >= 10: + let sp: uint32 = s div 10 + ## = vb / 40 + let upInside: bool = lower <= 40 * sp + let wpInside: bool = 40 * sp + 40 <= upper + ## if (up_inside || wp_inside) // NB: At most one of u' and w' is in R_v. + if upInside != wpInside: + return FloatingDecimal32(digits: sp + uint32(wpInside), exponent: k + 1) + let uInside: bool = lower <= 4 * s + let wInside: bool = 4 * s + 4 <= upper + if uInside != wInside: + return FloatingDecimal32(digits: s + uint32(wInside), exponent: k) + let mid: uint32 = 4 * s + 2 + ## = 2(s + t) + let roundUp: bool = vb > mid or (vb == mid and (s and 1) != 0) + return FloatingDecimal32(digits: s + uint32(roundUp), exponent: k) + +## ================================================================================================== +## ToChars +## ================================================================================================== + +proc utoa2Digits(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + const + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + sf_Assert(digits <= 99) + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + +proc trailingZeros2Digits(digits: uint32): int32 {.inline.} = + const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + sf_Assert(digits <= 99) + return trailingZeros100[digits] + +proc printDecimalDigitsBackwards(buf: var openArray[char]; pos: int; output: uint32): int32 {.inline.} = + var output = output + var pos = pos + var tz: int32 = 0 + ## number of trailing zeros removed. + var nd: int32 = 0 + ## number of decimal digits processed. + ## At most 9 digits remaining + if output >= 10000: + let q: uint32 = output div 10000 + let r: uint32 = output mod 10000 + output = q + dec(pos, 4) + if r != 0: + let rH: uint32 = r div 100 + let rL: uint32 = r mod 100 + utoa2Digits(buf, pos, rH) + utoa2Digits(buf, pos + 2, rL) + tz = trailingZeros2Digits(if rL == 0: rH else: rL) + (if rL == 0: 2 else: 0) + else: + tz = 4 + nd = 4 + if output >= 100: + let q: uint32 = output div 100 + let r: uint32 = output mod 100 + output = q + dec(pos, 2) + utoa2Digits(buf, pos, r) + if tz == nd: + inc(tz, trailingZeros2Digits(r)) + inc(nd, 2) + if output >= 100: + let q2: uint32 = output div 100 + let r2: uint32 = output mod 100 + output = q2 + dec(pos, 2) + utoa2Digits(buf, pos, r2) + if tz == nd: + inc(tz, trailingZeros2Digits(r2)) + inc(nd, 2) + sf_Assert(output >= 1) + sf_Assert(output <= 99) + if output >= 10: + let q: uint32 = output + dec(pos, 2) + utoa2Digits(buf, pos, q) + if tz == nd: + inc(tz, trailingZeros2Digits(q)) + else: + let q: uint32 = output + sf_Assert(q >= 1) + sf_Assert(q <= 9) + dec(pos) + buf[pos] = chr(uint32('0') + q) + return tz + +proc decimalLength(v: uint32): int32 {.inline.} = + sf_Assert(v >= 1) + sf_Assert(v <= 999999999'u) + if v >= 100000000'u: + return 9 + if v >= 10000000'u: + return 8 + if v >= 1000000'u: + return 7 + if v >= 100000'u: + return 6 + if v >= 10000'u: + return 5 + if v >= 1000'u: + return 4 + if v >= 100'u: + return 3 + if v >= 10'u: + return 2 + return 1 + +proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimalExponent: int32; + forceTrailingDotZero: bool = false): int {.inline.} = + const + minFixedDecimalPoint: int32 = -4 + maxFixedDecimalPoint: int32 = 9 + var pos = pos + assert(minFixedDecimalPoint <= -1, "internal error") + assert(maxFixedDecimalPoint >= 1, "internal error") + sf_Assert(digits >= 1) + sf_Assert(digits <= 999999999'u) + sf_Assert(decimalExponent >= -99) + sf_Assert(decimalExponent <= 99) + var numDigits: int32 = decimalLength(digits) + let decimalPoint: int32 = numDigits + decimalExponent + let useFixed: bool = minFixedDecimalPoint <= decimalPoint and + decimalPoint <= maxFixedDecimalPoint + ## Prepare the buffer. + ## Avoid calling memset/memcpy with variable arguments below... + for i in 0..<32: buffer[pos+i] = '0' + assert(minFixedDecimalPoint >= -30, "internal error") + assert(maxFixedDecimalPoint <= 32, "internal error") + var decimalDigitsPosition: int32 + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + decimalDigitsPosition = 2 - decimalPoint + else: + ## dig.its + ## digits[000] + decimalDigitsPosition = 0 + else: + ## dE+123 or d.igitsE+123 + decimalDigitsPosition = 1 + var digitsEnd = pos + decimalDigitsPosition + numDigits + let tz: int32 = printDecimalDigitsBackwards(buffer, digitsEnd, digits) + dec(digitsEnd, tz) + dec(numDigits, tz) + ## decimal_exponent += tz; // => decimal_point unchanged. + if useFixed: + if decimalPoint <= 0: + ## 0.[000]digits + buffer[pos+1] = '.' + pos = digitsEnd + elif decimalPoint < numDigits: + ## dig.its + for i in 0..<8: + buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] + buffer[pos+decimalPoint] = '.' + pos = digitsEnd + 1 + else: + ## digits[000] + inc(pos, decimalPoint) + if forceTrailingDotZero: + buffer[pos] = '.' + buffer[pos+1] = '0' + inc(pos, 2) + else: + buffer[pos] = buffer[pos+1] + if numDigits == 1: + ## dE+123 + inc(pos) + else: + ## d.igitsE+123 + buffer[pos+1] = '.' + pos = digitsEnd + let scientificExponent: int32 = decimalPoint - 1 + ## SF_ASSERT(scientific_exponent != 0); + buffer[pos] = 'e' + buffer[pos+1] = if scientificExponent < 0: '-' else: '+' + inc(pos, 2) + let k: uint32 = cast[uint32](if scientificExponent < 0: -scientificExponent else: scientificExponent) + if k < 10: + buffer[pos] = chr(uint32('0') + k) + inc pos + else: + utoa2Digits(buffer, pos, k) + inc(pos, 2) + return pos + +proc float32ToChars*(buffer: var openArray[char]; v: float32; forceTrailingDotZero = false): int {. + inline.} = + let significand: uint32 = physicalSignificand(constructSingle(v)) + let exponent: uint32 = physicalExponent(constructSingle(v)) + var pos = 0 + if exponent != maxIeeeExponent: + ## Finite + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + if exponent != 0 or significand != 0: + ## != 0 + let dec: auto = toDecimal32(significand, exponent) + return formatDigits(buffer, pos, dec.digits, dec.exponent, forceTrailingDotZero) + else: + buffer[pos] = '0' + buffer[pos+1] = '.' + buffer[pos+2] = '0' + buffer[pos+3] = ' ' + inc(pos, if forceTrailingDotZero: 3 else: 1) + return pos + if significand == 0: + buffer[pos] = '-' + inc(pos, signBit(constructSingle(v))) + buffer[pos] = 'i' + buffer[pos+1] = 'n' + buffer[pos+2] = 'f' + buffer[pos+3] = ' ' + return pos + 3 + else: + buffer[pos] = 'n' + buffer[pos+1] = 'a' + buffer[pos+2] = 'n' + buffer[pos+3] = ' ' + return pos + 3 diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 42ea9d226..1a32a4afe 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -164,6 +164,19 @@ proc nimFloatToStr(f: float): string {.compilerproc.} = result = newStringOfCap(8) result.addFloat f +when not defined(nimLegacyAddFloat) and not defined(nimscript) and + not defined(js) and defined(nimHasDragonBox): + import schubfach + +proc nimFloat32ToStr(f: float32): string {.compilerproc.} = + when declared(float32ToChars): + result = newString(65) + let L = float32ToChars(result, f, forceTrailingDotZero=true) + setLen(result, L) + else: + result = newStringOfCap(8) + result.addFloat f + proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "", noSideEffect.} diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim index c0e6e419f..fa8fc9da8 100644 --- a/tests/float/tfloat6.nim +++ b/tests/float/tfloat6.nim @@ -21,3 +21,8 @@ echo "0.00000_1".parseFloat(), " : ", 1E-6 echo "1_0.00_0001".parseFloat(), " : ", 10.000001 echo "1__00.00_0001".parseFloat(), " : ", 1_00.000001 + +# bug #18148 + +var a = 1.1'f32 +doAssert $a == "1.1", $a # fails From 282d61bafa4f1b06e1328cf8e517f556fb86d592 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 3 Jun 2021 14:00:53 +0200 Subject: [PATCH 0057/1654] added lib/deps.txt --- lib/deps.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lib/deps.txt diff --git a/lib/deps.txt b/lib/deps.txt new file mode 100644 index 000000000..2a44b5fe2 --- /dev/null +++ b/lib/deps.txt @@ -0,0 +1,14 @@ +# Dragonbox.nim and Schubfach.nim + +"Dragonbox" is Nim's "float64 to string" algorithm. +"Schubfach" is Nim's "float32 to string" algorithm. These are based on + +https://github.com/abolz/Drachennest/blob/master/src/dragonbox.cc +https://github.com/abolz/Drachennest/blob/master/src/schubfach_32.cc + +commit e6714a39ad331b4489d0b6aaf3968635bff4eb5e + +The `.cc` files were translated by c2nim via `--cpp --keepBodies --nep1` +and then modified to remove the unsafe code. + +We used c2nim as of commit f0469c909d9e2e28d59687e394bf5ac862f561b6. From 9df631a379998f554748697da7fe95ce4463cb83 Mon Sep 17 00:00:00 2001 From: n5m <72841454+n5m@users.noreply.github.com> Date: Thu, 3 Jun 2021 12:25:52 +0000 Subject: [PATCH 0058/1654] reuse algorithm.fill while building SkipTable (#18138) * reuse algorithm.fill while building SkipTable * Update lib/pure/strutils.nim Co-authored-by: Timothee Cour Co-authored-by: Timothee Cour --- lib/pure/strutils.nim | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 3fb67ead1..8e0a6158b 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -72,7 +72,7 @@ runnableExamples: import parseutils from math import pow, floor, log10 -from algorithm import reverse +from algorithm import fill, reverse import std/enumutils from unicode import toLower, toUpper @@ -1779,17 +1779,7 @@ func initSkipTable*(a: var SkipTable, sub: string) {.rtl, extern: "nsuInitSkipTable".} = ## Preprocess table `a` for `sub`. let m = len(sub) - var i = 0 - while i <= 0xff-7: - a[chr(i + 0)] = m - a[chr(i + 1)] = m - a[chr(i + 2)] = m - a[chr(i + 3)] = m - a[chr(i + 4)] = m - a[chr(i + 5)] = m - a[chr(i + 6)] = m - a[chr(i + 7)] = m - i += 8 + fill(a, m) for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i From 3e57c2f7800dbedd0df735a0236f09410f6db11a Mon Sep 17 00:00:00 2001 From: Artem Klevtsov Date: Thu, 3 Jun 2021 19:41:57 +0700 Subject: [PATCH 0059/1654] Improve db_postgres iterators (#18144) * Fix pqSetSingleRowMode case. Add links to the docs * Add missing PGContextVisibility enum * Remove unused PGContextVisibility enum * Improve db_postgres iterators * Fix instantRows with DbColumns. Cosmetics. * Reduce copy&paste in db_postgres * Move pqclear inside loop --- lib/impure/db_postgres.nim | 179 ++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 62 deletions(-) diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 4edeac2fc..36e035d3d 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -96,7 +96,6 @@ type ## converted to nil. InstantRow* = object ## a handle that can be res: PPGresult ## used to get a row's - line: int ## column text on demand SqlPrepared* = distinct string ## a identifier for the prepared queries proc dbError*(db: DbConn) {.noreturn.} = @@ -184,6 +183,22 @@ proc setupQuery(db: DbConn, stmtName: SqlPrepared, deallocCStringArray(arr) if pqResultStatus(result) != PGRES_TUPLES_OK: dbError(db) +proc setupSingeRowQuery(db: DbConn, query: SqlQuery, + args: varargs[string]) = + if pqsendquery(db, dbFormat(query, args)) != 1: + dbError(db) + if pqSetSingleRowMode(db) != 1: + dbError(db) + +proc setupSingeRowQuery(db: DbConn, stmtName: SqlPrepared, + args: varargs[string]) = + var arr = allocCStringArray(args) + if pqsendqueryprepared(db, stmtName.string, int32(args.len), arr, nil, nil, 0) != 1: + dbError(db) + if pqSetSingleRowMode(db) != 1: + dbError(db) + deallocCStringArray(arr) + proc prepare*(db: DbConn; stmtName: string, query: SqlQuery; nParams: int): SqlPrepared = ## Creates a new `SqlPrepared` statement. Parameter substitution is done @@ -203,49 +218,70 @@ proc setRow(res: PPGresult, r: var Row, line, cols: int32) = else: add(r[col], x) +template fetchRows(db: DbConn): untyped = + var res: PPGresult = nil + while true: + res = pqgetresult(db) + if res == nil: + break + let status = pqresultStatus(res) + if status == PGRES_TUPLES_OK: + discard + elif status != PGRES_SINGLE_TUPLE: + dbError(db) + else: + let L = pqNfields(res) + var result = newRow(L) + setRow(res, result, 0, L) + yield result + pqclear(res) + iterator fastRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the query and iterates over the result dataset. This is very ## fast, but potentially dangerous: If the for-loop-body executes another ## query, the results can be undefined. For Postgres it is safe though. - var res = setupQuery(db, query, args) - var L = pqnfields(res) - var result = newRow(L) - for i in 0'i32..pqntuples(res)-1: - setRow(res, result, i, L) - yield result - pqclear(res) + setupSingeRowQuery(db, query, args) + fetchRows(db) iterator fastRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - ## executes the prepared query and iterates over the result dataset. - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - var result = newRow(L) - for i in 0'i32..pqNtuples(res)-1: - setRow(res, result, i, L) - yield result - pqClear(res) + ## executes the query and iterates over the result dataset. This is very + ## fast, but potentially dangerous: If the for-loop-body executes another + ## query, the results can be undefined. For Postgres it is safe though. + setupSingeRowQuery(db, stmtName, args) + fetchRows(db) + +template fetchinstantRows(db: DbConn): untyped = + var res: PPGresult = nil + while true: + res = pqgetresult(db) + if res == nil: + break + let status = pqresultStatus(res) + if status == PGRES_TUPLES_OK: + discard + elif status != PGRES_SINGLE_TUPLE: + dbError(db) + else: + yield InstantRow(res: res) + pqclear(res) iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, query, args) - for i in 0'i32..pqNtuples(res)-1: - yield InstantRow(res: res, line: i) - pqClear(res) + setupSingeRowQuery(db, query, args) + fetchinstantRows(db) iterator instantRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): InstantRow {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within iterator body. - var res = setupQuery(db, stmtName, args) - for i in 0'i32..pqNtuples(res)-1: - yield InstantRow(res: res, line: i) - pqClear(res) + setupSingeRowQuery(db, stmtName, args) + fetchinstantRows(db) proc getColumnType(res: PPGresult, col: int) : DbType = ## returns DbType for given column in the row @@ -382,7 +418,7 @@ proc getColumnType(res: PPGresult, col: int) : DbType = of 705: return DbType(kind: DbTypeKind.dbUnknown, name: "unknown") else: return DbType(kind: DbTypeKind.dbUnknown, name: $oid) ## Query the system table pg_type to determine exactly which type is referenced. -proc setColumnInfo(columns: var DbColumns; res: PPGresult; L: int32) = +proc setColumnInfo(columns: var DbColumns; res: PPGresult, L: int32) = setLen(columns, L) for i in 0'i32.. 0: setRow(res, result, 0, L) pqclear(res) +proc getRow*(db: DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = + ## retrieves a single row. If the query doesn't return any rows, this proc + ## will return a Row with empty strings for each column. + let res = setupQuery(db, query, args) + getRow(res) + proc getRow*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = - var res = setupQuery(db, stmtName, args) - var L = pqNfields(res) - result = newRow(L) - if pqntuples(res) > 0: - setRow(res, result, 0, L) - pqClear(res) + let res = setupQuery(db, stmtName, args) + getRow(res) + +proc getAllRows(res: PPGresult): seq[Row] = + let N = pqntuples(res) + let L = pqnfields(res) + result = newSeqOfCap[Row](N) + var row = newRow(L) + for i in 0'i32..N-1: + setRow(res, row, i, L) + result.add(row) + pqclear(res) proc getAllRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): seq[Row] {. tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. - result = @[] - for r in fastRows(db, query, args): - result.add(r) + let res = setupQuery(db, query, args) + getAllRows(res) proc getAllRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = ## executes the prepared query and returns the whole result dataset. - result = @[] - for r in fastRows(db, stmtName, args): - result.add(r) + let res = setupQuery(db, stmtName, args) + getAllRows(res) iterator rows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = @@ -459,18 +515,21 @@ iterator rows*(db: DbConn, stmtName: SqlPrepared, ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, stmtName, args)): yield r +proc getValue(res: PPGresult): string = + if pqntuples(res) > 0: + var x = pqgetvalue(res, 0, 0) + result = if isNil(x): "" else: $x + else: + result = "" + proc getValue*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): string {. tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. - var res = setupQuery(db, query, args) - if pqntuples(res) > 0: - var x = pqgetvalue(res, 0, 0) - result = if isNil(x): "" else: $x - else: - result = "" + let res = setupQuery(db, query, args) + getValue(res) proc getValue*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): string {. @@ -478,12 +537,8 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared, ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. - var res = setupQuery(db, stmtName, args) - if pqntuples(res) > 0: - var x = pqgetvalue(res, 0, 0) - result = if isNil(x): "" else: $x - else: - result = "" + let res = setupQuery(db, stmtName, args) + getValue(res) proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. From 7ef364a402d3d827f10c893280f8dc7b9ef056f5 Mon Sep 17 00:00:00 2001 From: narimiran Date: Thu, 3 Jun 2021 15:20:42 +0200 Subject: [PATCH 0060/1654] add missing import to asynchttpserver's example --- lib/pure/asynchttpserver.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index fd0cad5ca..8cea7f162 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -18,7 +18,7 @@ runnableExamples("-r:off"): # This example will create an HTTP server on an automatically chosen port. # It will respond to all requests with a `200 OK` response code and "Hello World" # as the response body. - import std/asyncdispatch + import std/asyncdispatch, asynchttpserver proc main {.async.} = var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = From 0aa8b793a5f5ef58afe5e8aa96b7646a8afde765 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 16:27:34 +0200 Subject: [PATCH 0061/1654] clarify what a 'monotonic' timestamp is (#18163) --- lib/std/monotimes.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index 8f6aa5b66..3bb4fa141 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -10,7 +10,7 @@ ##[ The `std/monotimes` module implements monotonic timestamps. A monotonic timestamp represents the time that has passed since some system defined -point in time. The monotonic timestamps are guaranteed to always increase, +point in time. The monotonic timestamps are guaranteed not to decrease, meaning that that the following is guaranteed to work: ]## @@ -18,9 +18,8 @@ runnableExamples: import std/os let a = getMonoTime() - sleep(10) let b = getMonoTime() - assert a < b + assert a <= b ##[ This is not guaranteed for the `times.Time` type! This means that the From d31cbfd167c0bf2fcf2b1e6d401a20a16acaaebb Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Jun 2021 22:44:11 +0800 Subject: [PATCH 0062/1654] Revert "add missing import to asynchttpserver's example" (#18164) This reverts commit 7ef364a402d3d827f10c893280f8dc7b9ef056f5. --- lib/pure/asynchttpserver.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 8cea7f162..fd0cad5ca 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -18,7 +18,7 @@ runnableExamples("-r:off"): # This example will create an HTTP server on an automatically chosen port. # It will respond to all requests with a `200 OK` response code and "Hello World" # as the response body. - import std/asyncdispatch, asynchttpserver + import std/asyncdispatch proc main {.async.} = var server = newAsyncHttpServer() proc cb(req: Request) {.async.} = From 06232b7f2e230a46beea982f172087e9da51814a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 17:12:45 +0200 Subject: [PATCH 0063/1654] fixes #18058 (#18162) --- compiler/commands.nim | 2 ++ compiler/msgs.nim | 12 ++++-------- compiler/options.nim | 1 + doc/advopt.txt | 2 ++ tests/misc/trunner.nim | 4 ++-- tests/osproc/treadlines.nim | 4 ++-- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 605e2930c..994b224f1 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -919,6 +919,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; excl(conf.notes, hintProcessing) excl(conf.mainPackageNotes, hintProcessing) else: localError(conf, info, "expected: dots|filenames|off, got: $1" % arg) + of "unitsep": + conf.unitSep = if switchOn(arg): "\31" else: "" of "listfullpaths": # xxx in future work, use `warningDeprecated` conf.filenameOption = if switchOn(arg): foAbs else: foCanonical diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 6a8c45db5..053b5c928 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -289,10 +289,6 @@ proc `??`* (conf: ConfigRef; info: TLineInfo, filename: string): bool = # only for debugging purposes result = filename in toFilename(conf, info) -const - UnitSep = "\31" - # this needs care to avoid issues similar to https://github.com/nim-lang/Nim/issues/17853 - type MsgFlag* = enum ## flags altering msgWriteln behavior msgStdout, ## force writing to stdout, even stderr is default @@ -309,7 +305,7 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) = ## This is used for 'nim dump' etc. where we don't have nimsuggest ## support. #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return - let sep = if msgNoUnitSep notin flags: UnitSep else: "" + let sep = if msgNoUnitSep notin flags: conf.unitSep else: "" if not isNil(conf.writelnHook) and msgSkipHook notin flags: conf.writelnHook(s & sep) elif optStdout in conf.globalOptions or msgStdout in flags: @@ -409,7 +405,7 @@ proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} = styledMsgWriteln(fgRed, """ No stack traceback available To create a stacktrace, rerun compilation with './koch temp $1 ', see $2 for details""" % - [conf.command, "intern.html#debugging-the-compiler".createDocLink], UnitSep) + [conf.command, "intern.html#debugging-the-compiler".createDocLink], conf.unitSep) quit 1 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) = @@ -550,13 +546,13 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, msgWrite(conf, ".") else: styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg, - resetStyle, conf.getSurroundingSrc(info), UnitSep) + resetStyle, conf.getSurroundingSrc(info), conf.unitSep) if hintMsgOrigin in conf.mainPackageNotes: # xxx needs a bit of refactoring to honor `conf.filenameOption` styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle, " compiler msg initiated here", KindColor, KindFormat % $hintMsgOrigin, - resetStyle, UnitSep) + resetStyle, conf.unitSep) handleError(conf, msg, eh, s) template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = diff --git a/compiler/options.nim b/compiler/options.nim index 9cbb747c9..e6c667d92 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -283,6 +283,7 @@ type arcToExpand*: StringTableRef m*: MsgConfig filenameOption*: FilenameOption # how to render paths in compiler messages + unitSep*: string evalTemplateCounter*: int evalMacroCounter*: int exitcode*: int8 diff --git a/doc/advopt.txt b/doc/advopt.txt index ea3370880..e627d4768 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -41,6 +41,8 @@ Advanced options: defaults to `abs` (absolute) --processing:dots|filenames|off show files as they're being processed by nim compiler + --unitsep:on|off use the ASCII unit separator (31) between error + messages, useful for IDE-like tooling --declaredLocs:on|off show declaration locations in messages --spellSuggest|:num show at most `num >= 0` spelling suggestions on typos. if `num` is not specified (or `auto`), return diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 3297b3a24..b6b302836 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -249,7 +249,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines var expected = "" for a in files: let b = dir / a - expected.add &"Hint: used config file '{b}' [Conf]\31\n" + expected.add &"Hint: used config file '{b}' [Conf]\n" doAssert outp.endsWith expected, outp & "\n" & expected block: # mfoo2.customext @@ -257,7 +257,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines let cmd = fmt"{nim} e --hint:conf {filename}" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) doAssert exitCode == 0 - var expected = &"Hint: used config file '{filename}' [Conf]\31\n" + var expected = &"Hint: used config file '{filename}' [Conf]\n" doAssert outp.endsWith "123" & "\n" & expected diff --git a/tests/osproc/treadlines.nim b/tests/osproc/treadlines.nim index 200a7c299..bb6a7f129 100644 --- a/tests/osproc/treadlines.nim +++ b/tests/osproc/treadlines.nim @@ -1,7 +1,7 @@ discard """ output: ''' -Error: cannot open 'a.nim'\31 -Error: cannot open 'b.nim'\31 +Error: cannot open 'a.nim' +Error: cannot open 'b.nim' ''' targets: "c" """ From 28f2abe1a20a7f7f758b42c233deb7daa78406b1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 3 Jun 2021 20:55:41 +0200 Subject: [PATCH 0064/1654] fixes #18112 (#18165) --- compiler/docgen.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 18c568ba0..dc2a63f48 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -20,7 +20,7 @@ import from uri import encodeUrl from std/private/globs import nativeToUnixPath - +from nodejs import findNodeJs const exportSection = skField @@ -430,7 +430,9 @@ proc runAllExamples(d: PDoc) = "rdoccmd", group.rdoccmd, "docCmd", group.docCmd, ] - if os.execShellCmd(cmd) != 0: + if d.conf.backend == backendJs and findNodeJs() == "": + discard "ignore JS runnableExample" + elif os.execShellCmd(cmd) != 0: d.conf.quitOrRaise "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, group[].prettyString, cmd] else: # keep generated source file `outp` to allow inspection. From d91d78f8aed2b1b59effa3b98765aaf675d7e4c3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 3 Jun 2021 13:25:36 -0700 Subject: [PATCH 0065/1654] changelog for --unitsep (#18167) --- changelog.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index a23aedbdf..06251abfc 100644 --- a/changelog.md +++ b/changelog.md @@ -61,9 +61,6 @@ - Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. -- Nim compiler now adds ASCII unit separator `\31` before a newline for every generated - message (potentially multiline), so tooling can tell when messages start and end. - - `random.initRand(seed)` now produces non-skewed values for the 1st call to `rand()` after initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition time, see PR #17467. @@ -395,6 +392,9 @@ - Added `--processing:dots|filenames|off` which customizes `hintProcessing` +- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` before a newline + for every generated message (potentially multiline), so tooling can tell when messages start and end. + - Source+Edit links now appear on top of every docgen'd page when `nim doc --git.url:url ...` is given. From 654a20166eabb614cbf0379c67115981f0532079 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 3 Jun 2021 13:29:45 -0700 Subject: [PATCH 0066/1654] simplify extccomp.nim json logic via jsonutils; fix #18084 (#18100) * simplify extccomp.nim json logic via jsonutils * fix #18084 * simplify further * workaround for bootstrap that can be removed after updating csources_v1 >= 1.2 --- compiler/extccomp.nim | 256 +++++++++++++----------------------------- compiler/nimconf.nim | 11 +- compiler/options.nim | 4 +- lib/std/jsonutils.nim | 18 ++- 4 files changed, 99 insertions(+), 190 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 1cea9edeb..ea2b09866 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils -import os, strutils, osproc, std/sha1, streams, sequtils, times, strtabs, json +import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] type TInfoCCProp* = enum # properties of the C compiler: @@ -942,195 +942,91 @@ proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = # works out of the box with `hashMainCompilationParams`. result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") +const cacheVersion = "D20210525T193831" # update when `BuildCache` spec changes +type BuildCache = object + cacheVersion: string + outputFile: string + compile: seq[(string, string)] + link: seq[string] + linkcmd: string + extraCmds: seq[string] + configFiles: seq[string] # the hash shouldn't be needed + stdinInput: bool + projectIsCmd: bool + cmdInput: string + currentDir: string + cmdline: string + depfiles: seq[(string, string)] + nimexe: string + proc writeJsonBuildInstructions*(conf: ConfigRef) = - # xxx use std/json instead, will result in simpler, more maintainable code. - template lit(x: string) = f.write x - template str(x: string) = - buf.setLen 0 - escapeJson(x, buf) - f.write buf - - proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) = - var comma = false - for i, it in clist: - if CfileFlag.Cached in it.flags: continue - let compileCmd = getCompileCFileCmd(conf, it) - if comma: lit ",\L" else: comma = true - lit "[" - str it.cname.string - lit ", " - str compileCmd - lit "]" - - proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList; - llist: seq[string]) = - var pastStart = false - template impl(path) = - let path2 = quoteShell(path) - objfiles.add(' ') - objfiles.add(path2) - if pastStart: lit ",\L" - str path2 - pastStart = true - for it in llist: - let objfile = if noAbsolutePaths(conf): it.extractFilename else: it - impl(addFileExt(objfile, CC[conf.cCompiler].objExt)) - for it in clist: - impl(it.obj) - lit "\L" - - proc depfiles(conf: ConfigRef; f: File; buf: var string) = - var i = 0 - for it in conf.m.fileInfos: + var linkFiles = collect(for it in conf.externalToLink: + var it = it + if conf.noAbsolutePaths: it = it.extractFilename + it.addFileExt(CC[conf.cCompiler].objExt)) + for it in conf.toCompile: linkFiles.add it.obj.string + var bcache = BuildCache( + cacheVersion: cacheVersion, + outputFile: conf.absOutFile.string, + compile: collect(for i, it in conf.toCompile: + if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), + link: linkFiles, + linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand), + extraCmds: getExtraCmds(conf, conf.absOutFile), + stdinInput: conf.projectIsStdin, + projectIsCmd: conf.projectIsCmd, + cmdInput: conf.cmdInput, + configFiles: conf.configFiles.mapIt(it.string), + currentDir: getCurrentDir()) + if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): + bcache.cmdline = conf.commandLine + bcache.depfiles = collect(for it in conf.m.fileInfos: let path = it.fullPath.string if isAbsolute(path): # TODO: else? - if i > 0: lit "],\L" - lit "[" - str path - lit ", " - str $secureHashFile(path) - inc i - lit "]\L" - - - var buf = newStringOfCap(50) - let jsonFile = conf.jsonBuildInstructionsFile - conf.jsonBuildFile = jsonFile - let output = conf.absOutFile - - var f: File - if open(f, jsonFile.string, fmWrite): - lit "{\L" - lit "\"outputFile\": " - str $output - - lit ",\L\"compile\":[\L" - cfiles(conf, f, buf, conf.toCompile, false) - lit "],\L\"link\":[\L" - var objfiles = "" - # XXX add every file here that is to link - linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) - - lit "],\L\"linkcmd\": " - str getLinkCmd(conf, output, objfiles) - - lit ",\L\"extraCmds\": " - lit $(%* getExtraCmds(conf, conf.absOutFile)) - - lit ",\L\"stdinInput\": " - lit $(%* conf.projectIsStdin) - lit ",\L\"projectIsCmd\": " - lit $(%* conf.projectIsCmd) - lit ",\L\"cmdInput\": " - lit $(%* conf.cmdInput) - lit ",\L\"currentDir\": " - lit $(%* getCurrentDir()) - - if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): - lit ",\L\"cmdline\": " - str conf.commandLine - lit ",\L\"depfiles\":[\L" - depfiles(conf, f, buf) - lit "],\L\"nimexe\": \L" - str hashNimExe() - lit "\L" - - lit "\L}\L" - close(f) + (path, $secureHashFile(path))) + bcache.nimexe = hashNimExe() + conf.jsonBuildFile = conf.jsonBuildInstructionsFile + conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = - if not fileExists(jsonFile): return true - if not fileExists(conf.absOutFile): return true - result = false - try: - let data = json.parseFile(jsonFile.string) - for key in "depfiles cmdline stdinInput currentDir".split: - if not data.hasKey(key): return true - if getCurrentDir() != data["currentDir"].getStr: - # fixes bug #16271 - # Note that simply comparing `expandFilename(projectFile)` would - # not be sufficient in case other flags depend implicitly on `getCurrentDir`, - # and would require much more care. Simply re-compiling is safer for now. - # A better strategy for future work would be to cache (with an LRU cache) - # the N most recent unique build instructions, as done with `rdmd`, - # which is both robust and avoids recompilation when switching back and forth - # between projects, see https://github.com/timotheecour/Nim/issues/199 - return true - let oldCmdLine = data["cmdline"].getStr - if conf.commandLine != oldCmdLine: - return true - if hashNimExe() != data["nimexe"].getStr: - return true - let stdinInput = data["stdinInput"].getBool - let projectIsCmd = data["projectIsCmd"].getBool - if conf.projectIsStdin or stdinInput: - # could optimize by returning false if stdin input was the same, - # but I'm not sure how to get full stding input - return true - - if conf.projectIsCmd or projectIsCmd: - if not (conf.projectIsCmd and projectIsCmd): return true - if not data.hasKey("cmdInput"): return true - let cmdInput = data["cmdInput"].getStr - if cmdInput != conf.cmdInput: return true - - let depfilesPairs = data["depfiles"] - doAssert depfilesPairs.kind == JArray - for p in depfilesPairs: - doAssert p.kind == JArray - # >= 2 for forwards compatibility with potential later .json files: - doAssert p.len >= 2 - let depFilename = p[0].getStr - let oldHashValue = p[1].getStr - let newHashValue = $secureHashFile(depFilename) - if oldHashValue != newHashValue: - return true + if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true + var bcache: BuildCache + try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - echo "Warning: JSON processing failed: ", getCurrentExceptionMsg() - result = true + stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string, getCurrentExceptionMsg()] + return true + if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.configFiles != conf.configFiles.mapIt(it.string) or + bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or + bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or + bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true + if bcache.stdinInput or conf.projectIsStdin: return true + # xxx optimize by returning false if stdin input was the same + for (file, hash) in bcache.depfiles: + if $secureHashFile(file) != hash: return true proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = - try: - let data = json.parseFile(jsonFile.string) - let output = data["outputFile"].getStr - createDir output.parentDir - let outputCurrent = $conf.absOutFile - if output != outputCurrent: - # previously, any specified output file would be silently ignored; - # simply copying won't work in some cases, for example with `extraCmds`, - # so we just make it an error, user should use same command for jsonscript - # as was used with --compileOnly. - globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string]) - - let toCompile = data["compile"] - doAssert toCompile.kind == JArray - var cmds: TStringSeq - var prettyCmds: TStringSeq - let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) - for c in toCompile: - doAssert c.kind == JArray - doAssert c.len >= 2 - - cmds.add(c[1].getStr) - prettyCmds.add displayProgressCC(conf, c[0].getStr, c[1].getStr) - - execCmdsInParallel(conf, cmds, prettyCb) - - let linkCmd = data["linkcmd"] - doAssert linkCmd.kind == JString - execLinkCmd(conf, linkCmd.getStr) - if data.hasKey("extraCmds"): - let extraCmds = data["extraCmds"] - doAssert extraCmds.kind == JArray - for cmd in extraCmds: - doAssert cmd.kind == JString, $cmd.kind - let cmd2 = cmd.getStr - execExternalProgram(conf, cmd2, hintExecuting) - + var bcache: BuildCache + try: bcache.fromJson(jsonFile.string.parseFile) except: let e = getCurrentException() - conf.quitOrRaise "\ncaught exception:\n" & e.msg & "\nstacktrace:\n" & e.getStackTrace() & - "error evaluating JSON file: " & jsonFile.string + conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % + [e.msg, e.getStackTrace(), jsonFile.string] + let output = bcache.outputFile + createDir output.parentDir + let outputCurrent = $conf.absOutFile + if output != outputCurrent or bcache.cacheVersion != cacheVersion: + globalError(conf, gCmdLineInfo, + "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % + [outputCurrent, output, jsonFile.string]) + var cmds, prettyCmds: TStringSeq + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + for (name, cmd) in bcache.compile: + cmds.add cmd + prettyCmds.add displayProgressCC(conf, name, cmd) + execCmdsInParallel(conf, cmds, prettyCb) + execLinkCmd(conf, bcache.linkcmd) + for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = for it in list: diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index b63e5a0ad..94c93a283 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -240,13 +240,10 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) = setDefaultLibpath(conf) - - var configFiles = newSeq[AbsoluteFile]() - template readConfigFile(path) = let configPath = path if readConfigFile(configPath, cache, conf): - configFiles.add(configPath) + conf.configFiles.add(configPath) template runNimScriptIfExists(path: AbsoluteFile, isMain = false) = let p = path # eval once @@ -256,7 +253,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: elif conf.projectIsCmd: s = llStreamOpen(conf.cmdInput) if s == nil and fileExists(p): s = llStreamOpen(p, fmRead) if s != nil: - configFiles.add(p) + conf.configFiles.add(p) runNimScript(cache, p, idgen, freshDefines = false, conf, s) if optSkipSystemConfigFile notin conf.globalOptions: @@ -295,12 +292,12 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: let scriptFile = conf.projectFull.changeFileExt("nims") let scriptIsProj = scriptFile == conf.projectFull template showHintConf = - for filename in configFiles: + for filename in conf.configFiles: # delayed to here so that `hintConf` is honored rawMessage(conf, hintConf, filename.string) if conf.cmd == cmdNimscript: showHintConf() - configFiles.setLen 0 + conf.configFiles.setLen 0 if conf.cmd != cmdIdeTools: if conf.cmd == cmdNimscript: runNimScriptIfExists(conf.projectFull, isMain = true) diff --git a/compiler/options.nim b/compiler/options.nim index e6c667d92..afe8b0fc7 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -308,7 +308,7 @@ type ## should be run ideCmd*: IdeCmd oldNewlines*: bool - cCompiler*: TSystemCC + cCompiler*: TSystemCC # the used compiler modifiedyNotes*: TNoteKinds # notes that have been set/unset from either cmdline/configs cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline foreignPackageNotes*: TNoteKinds @@ -353,7 +353,7 @@ type docRoot*: string ## see nim --fullhelp for --docRoot docCmd*: string ## see nim --fullhelp for --docCmd - # the used compiler + configFiles*: seq[AbsoluteFile] # config files (cfg,nims) cIncludes*: seq[AbsoluteDir] # directories to search for included files cLibs*: seq[AbsoluteDir] # directories to search for lib files cLinkedLibs*: seq[string] # libraries to link diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index e33ae4401..60d78fea5 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -34,6 +34,23 @@ import macros from enumutils import symbolName from typetraits import OrdinalEnum +when not defined(nimFixedForwardGeneric): + # xxx remove pending csources_v1 update >= 1.2.0 + proc to[T](node: JsonNode, t: typedesc[T]): T = + when T is string: node.getStr + elif T is bool: node.getBool + else: static: doAssert false, $T # support as needed (only needed during bootstrap) + proc isNamedTuple(T: typedesc): bool = # old implementation + when T isnot tuple: result = false + else: + var t: T + for name, _ in t.fieldPairs: + when name == "Field0": return compiles(t.Field0) + else: return true + return false +else: + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + type Joptions* = object # xxx rename FromJsonOptions ## Options controlling the behavior of `fromJson`. @@ -61,7 +78,6 @@ proc initToJsonOptions*(): ToJsonOptions = ## initializes `ToJsonOptions` with sane options. ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) -proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a) From 7e21218a07f919d56cd0974c657aefb11574da26 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 4 Jun 2021 17:16:51 +0300 Subject: [PATCH 0067/1654] follow-up #18013 - inline syntax highlighting (#18166) --- doc/destructors.rst | 2 +- doc/docgen.rst | 2 +- doc/drnim.rst | 4 +- doc/estp.rst | 2 +- doc/hcr.rst | 31 +++++++++------ doc/intern.rst | 2 +- doc/koch.rst | 27 +++++++------ doc/lib.rst | 12 +++--- doc/manual.rst | 48 ++++++++++++------------ doc/manual_experimental.rst | 20 +++++----- doc/manual_experimental_strictnotnil.rst | 26 ++++++++----- doc/nep1.rst | 11 +++--- doc/nimgrep.rst | 31 ++++++++------- doc/niminst.rst | 24 ++++++------ doc/nims.rst | 45 +++++++++++----------- doc/nimsuggest.rst | 37 +++++++++--------- doc/packaging.rst | 2 +- doc/regexprs.txt | 4 +- doc/testament.rst | 2 +- doc/tools.rst | 15 ++++---- 20 files changed, 183 insertions(+), 164 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 3cd4cd925..c99606b95 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -5,8 +5,8 @@ Nim Destructors and Move Semantics :Authors: Andreas Rumpf :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: nim .. contents:: diff --git a/doc/docgen.rst b/doc/docgen.rst index 86352a989..e9ff7f1e3 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -5,8 +5,8 @@ :Author: Erik O'Leary :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim .. contents:: diff --git a/doc/drnim.rst b/doc/drnim.rst index 08de7cd6d..070cd1787 100644 --- a/doc/drnim.rst +++ b/doc/drnim.rst @@ -5,8 +5,8 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim .. contents:: @@ -43,7 +43,7 @@ your code. Installation ============ -Run `koch drnim`:option:, the executable will afterwards be +Run `koch drnim`:cmd:, the executable will afterwards be in ``$nim/bin/drnim``. diff --git a/doc/estp.rst b/doc/estp.rst index 43128bef7..550194d2d 100644 --- a/doc/estp.rst +++ b/doc/estp.rst @@ -2,8 +2,8 @@ Embedded Stack Trace Profiler (ESTP) User Guide =================================================== +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim :Author: Andreas Rumpf :Version: |nimversion| diff --git a/doc/hcr.rst b/doc/hcr.rst index 574c4f39d..dd25e39b3 100644 --- a/doc/hcr.rst +++ b/doc/hcr.rst @@ -1,9 +1,10 @@ -.. default-role:: code - =================================== Hot code reloading =================================== +.. default-role:: code +.. include:: rstcommon.rst + The `hotCodeReloading`:idx: option enables special compilation mode where changes in the code can be applied automatically to a running program. The code reloading happens at the granularity of an individual module. @@ -22,7 +23,7 @@ during development resides. In this example, we use SDL2 to create a window and we reload the logic code when `F9` is pressed. The important lines are marked with `#***`. -To install SDL2 you can use `nimble install sdl2`. +To install SDL2 you can use `nimble install sdl2`:cmd:. .. code-block:: nim @@ -98,34 +99,40 @@ To install SDL2 you can use `nimble install sdl2`. main() -Compile this example via:: +Compile this example via: +```cmd nim c --hotcodereloading:on mymain.nim +``` Now start the program and KEEP it running! -:: - +.. code:: cmd # Unix: mymain & # or Windows (click on the .exe) mymain.exe # edit -For example, change the line:: +For example, change the line: +```nim discard renderer.setDrawColor(255, 128, 128, 0) +``` -into:: +into: +```nim discard renderer.setDrawColor(255, 255, 128, 0) +``` (This will change the color of the rectangle.) Then recompile the project, but do not restart or quit the mymain.exe program! -:: +```cmd nim c --hotcodereloading:on mymain.nim +``` Now give the `mymain` SDL window the focus, press F9, and watch the updated version of the program. @@ -188,15 +195,15 @@ Native code targets =================== Native projects using the hot code reloading option will be implicitly -compiled with the `-d:useNimRtl` option and they will depend on both +compiled with the `-d:useNimRtl`:option: option and they will depend on both the `nimrtl` library and the `nimhcr` library which implements the hot code reloading run-time. Both libraries can be found in the `lib` folder of Nim and can be compiled into dynamic libraries to satisfy runtime demands of the example code above. An example of compiling -`nimhcr.nim` and `nimrtl.nim` when the source dir of Nim is installed +``nimhcr.nim`` and ``nimrtl.nim`` when the source dir of Nim is installed with choosenim follows. -:: +.. code:: console # Unix/MacOS # Make sure you are in the directory containing your .nim files diff --git a/doc/intern.rst b/doc/intern.rst index 2f83e0d06..ab74c4a3a 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -6,8 +6,8 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: Nim .. contents:: "Abstraction is layering ignorance on top of reality." -- Richard Gabriel diff --git a/doc/koch.rst b/doc/koch.rst index 1eb02d785..b8d85dff4 100644 --- a/doc/koch.rst +++ b/doc/koch.rst @@ -1,11 +1,11 @@ -.. default-role:: code - =============================== Nim maintenance script =============================== :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: .. raw:: html @@ -19,7 +19,7 @@ Introduction The `koch`:idx: program is Nim's maintenance script. It is a replacement for make and shell scripting with the advantage that it is much more portable. -The word *koch* means *cook* in German. `koch` is used mainly to build the +The word *koch* means *cook* in German. `koch`:cmd: is used mainly to build the Nim compiler, but it can also be used for other tasks. This document describes the supported commands and their options. @@ -33,12 +33,11 @@ boot command The `boot`:idx: command bootstraps the compiler, and it accepts different options: --d:release - By default a debug version is created, passing this option will +-d:release By default a debug version is created, passing this option will force a release build, which is much faster and should be preferred unless you are debugging the compiler. --d:nimUseLinenoise - Use the linenoise library for interactive mode (not needed on Windows). +-d:nimUseLinenoise Use the linenoise library for interactive mode + (not needed on Windows). After compilation is finished you will hopefully end up with the nim compiler in the `bin` directory. You can add Nim's `bin` directory to @@ -56,16 +55,16 @@ temp command ------------ The temp command builds the Nim compiler but with a different final name -(`nim_temp`), so it doesn't overwrite your normal compiler. You can use +(`nim_temp`:cmd:), so it doesn't overwrite your normal compiler. You can use this command to test different options, the same you would issue for the `boot command <#commands-boot-command>`_. test command ------------ -The `test`:idx: command can also be invoked with the alias `tests`. This -command will compile and run `testament/tester.nim`, which is the main -driver of Nim's test suite. You can pass options to the `test` command, +The `test`:idx: command can also be invoked with the alias `tests`:option:. This +command will compile and run ``testament/tester.nim``, which is the main +driver of Nim's test suite. You can pass options to the `test`:option: command, they will be forwarded to the tester. See its source code for available options. @@ -74,13 +73,13 @@ web command The `web`:idx: command converts the documentation in the `doc` directory from rst to HTML. It also repeats the same operation but places the result in -the `web/upload` which can be used to update the website at +the ``web/upload`` which can be used to update the website at https://nim-lang.org. By default, the documentation will be built in parallel using the number of available CPU cores. If any documentation build sub-commands fail, they will be rerun in serial fashion so that meaningful error output can be gathered for -inspection. The `--parallelBuild:n` switch or configuration option can be +inspection. The `--parallelBuild:n`:option: switch or configuration option can be used to force a specific number of parallel jobs or run everything serially from the start (`n == 1`). @@ -89,4 +88,4 @@ pdf command The `pdf`:idx: command builds PDF versions of Nim documentation: Manual, Tutorial and a few other documents. To run it one needs to -`install Latex/pdflatex `_ first. +`install Latex/xelatex `_ first. diff --git a/doc/lib.rst b/doc/lib.rst index f09fe16b2..988fbf6f0 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ==================== Nim Standard Library ==================== @@ -7,11 +5,13 @@ Nim Standard Library :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Nim's library is divided into *pure libraries*, *impure libraries*, and *wrappers*. -Pure libraries do not depend on any external `*.dll` or `lib*.so` binary +Pure libraries do not depend on any external ``*.dll`` or ``lib*.so`` binary while impure libraries do. A wrapper is an impure library that is a very low-level interface to a C library. @@ -39,11 +39,11 @@ Automatic imports * `threads `_ Basic Nim thread support. **Note:** This is part of the system module. Do not - import it explicitly. Enabled with `--threads:on`. + import it explicitly. Enabled with `--threads:on`:option:. * `channels `_ Nim message passing support for threads. **Note:** This is part of the - system module. Do not import it explicitly. Enabled with `--threads:on`. + system module. Do not import it explicitly. Enabled with `--threads:on`:option:. Core @@ -366,7 +366,7 @@ Parsers * `parsecfg `_ The `parsecfg` module implements a high-performance configuration file - parser. The configuration file's syntax is similar to the Windows `.ini` + parser. The configuration file's syntax is similar to the Windows ``.ini`` format, but much more powerful, as it is not a line based parser. String literals, raw string literals, and triple quote string literals are supported as in the Nim programming language. diff --git a/doc/manual.rst b/doc/manual.rst index ff34b62ca..867db245b 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -572,7 +572,8 @@ In the following examples, `-1` is a single token: "abc";-1 -In the following examples, `-1` is parsed as two separate tokens (as `- 1`): +In the following examples, `-1` is parsed as two separate tokens +(as `-`:tok: `1`:tok:): .. code-block:: nim @@ -2509,7 +2510,7 @@ of the argument. range. 3. Generic match: `f` is a generic type and `a` matches, for instance `a` is `int` and `f` is a generic (constrained) parameter - type (like in `[T]` or `[T: int|char]`. + type (like in `[T]` or `[T: int|char]`). 4. Subrange or subtype match: `a` is a `range[T]` and `T` matches `f` exactly. Or: `a` is a subtype of `f`. 5. Integral conversion match: `a` is convertible to `f` and `f` and `a` @@ -3840,7 +3841,7 @@ Type bound operators currently include: (some of which are still implementation defined and not yet documented). For more details on some of those procs, see -`lifetimeminustracking-hooks `_. +`Lifetime-tracking hooks `_. Nonoverloadable builtins ------------------------ @@ -5283,7 +5284,7 @@ instantiations cross multiple different modules: In module B has an `init` proc from module C in its scope that is not taken into account when `genericB` is instantiated which leads to the -instantiation of `genericA`. The solution is to `forward`:idx these +instantiation of `genericA`. The solution is to `forward`:idx: these symbols by a `bind` statement inside `genericB`. @@ -7305,19 +7306,19 @@ Produces: supplied pattern to denote the concrete type parameters of the generic type. See the usage of the apostrophe operator in proc patterns for more details. -.. code-block:: nim + .. code-block:: nim - type - VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object + type + VectorIterator {.importcpp: "std::vector<'0>::iterator".} [T] = object - var x: VectorIterator[cint] + var x: VectorIterator[cint] -Produces: + Produces: -.. code-block:: C + .. code-block:: C - std::vector::iterator x; + std::vector::iterator x; ImportJs pragma @@ -7444,7 +7445,7 @@ compile-time define pragmas --------------------------- The pragmas listed here can be used to optionally accept values from -the -d/--define option at compile time. +the `-d/--define`:option: option at compile time. The implementation currently provides the following possible options (various others may be added later). @@ -7461,7 +7462,7 @@ pragma description const FooBar {.intdefine.}: int = 5 echo FooBar -:: +.. code:: cmd nim c -d:FooBar=42 foobar.nim In the above example, providing the `-d`:option: flag causes the symbol @@ -7480,7 +7481,8 @@ pragma pragma ------------- The `pragma` pragma can be used to declare user-defined pragmas. This is -useful because Nim's templates and macros do not affect pragmas. User-defined pragmas are in a different module-wide scope than all other symbols. +useful because Nim's templates and macros do not affect pragmas. +User-defined pragmas are in a different module-wide scope than all other symbols. They cannot be imported from a module. Example: @@ -7553,18 +7555,18 @@ More examples with custom pragmas: - Better serialization/deserialization control: -.. code-block:: nim - type MyObj = object - a {.dontSerialize.}: int - b {.defaultDeserialize: 5.}: int - c {.serializationKey: "_c".}: string + .. code-block:: nim + type MyObj = object + a {.dontSerialize.}: int + b {.defaultDeserialize: 5.}: int + c {.serializationKey: "_c".}: string - Adopting type for gui inspector in a game engine: -.. code-block:: nim - type MyComponent = object - position {.editable, animatable.}: Vector3 - alpha {.editRange: [0.0..1.0], animatable.}: float32 + .. code-block:: nim + type MyComponent = object + position {.editable, animatable.}: Vector3 + alpha {.editRange: [0.0..1.0], animatable.}: float32 Macro pragmas diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 51037b509..2c1043ecb 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ========================= Nim Experimental Features ========================= @@ -7,6 +5,8 @@ Nim Experimental Features :Authors: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: @@ -15,7 +15,7 @@ About this document This document describes features of Nim that are to be considered experimental. Some of these are not covered by the `.experimental` pragma or -`--experimental` switch because they are already behind a special syntax and +`--experimental`:option: switch because they are already behind a special syntax and one may want to use Nim libraries using these features without using them oneself. @@ -1109,7 +1109,7 @@ The `parameter constraint`:idx: expression can use the operators `|` (or), Predicate Meaning =================== ===================================================== `atom` The matching node has no children. -`lit` The matching node is a literal like "abc", 12. +`lit` The matching node is a literal like `"abc"`, `12`. `sym` The matching node must be a symbol (a bound identifier). `ident` The matching node must be an identifier (an unbound @@ -1185,7 +1185,7 @@ constant folding, so the following does not work: The reason is that the compiler already transformed the 1 into "1" for the `echo` statement. However, a term rewriting macro should not change the -semantics anyway. In fact they can be deactivated with the `--patterns:off` +semantics anyway. In fact they can be deactivated with the `--patterns:off`:option: command line option or temporarily with the `patterns` pragma. @@ -1779,7 +1779,7 @@ Noalias annotation ================== Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables -and parameters. It is mapped directly to C/C++'s `restrict` keyword and means that +and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that the underlying pointer is pointing to a unique location in memory, no other aliases to this location exist. It is *unchecked* that this alias restriction is followed, if the restriction is violated, the backend optimizer is free to miscompile the code. @@ -1832,8 +1832,8 @@ the `view types section <#view-types-algorithm>`_. View types ========== -**Note**: `--experimental:views` is more effective -with `--experimental:strictFuncs`. +**Note**: `--experimental:views`:option: is more effective +with `--experimental:strictFuncs`:option:. A view type is a type that is or contains one of the following types: @@ -1981,8 +1981,8 @@ The scope of the view does not matter: The analysis requires as much precision about mutations as is reasonably obtainable, so it is more effective with the experimental `strict funcs <#strict-funcs>`_ -feature. In other words `--experimental:views` works better -with `--experimental:strictFuncs`. +feature. In other words `--experimental:views`:option: works better +with `--experimental:strictFuncs`:option:. The analysis is currently control flow insensitive: diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst index cb3bfeca9..b14e5f9f3 100644 --- a/doc/manual_experimental_strictnotnil.rst +++ b/doc/manual_experimental_strictnotnil.rst @@ -1,8 +1,9 @@ -.. default-role:: code - Strict not nil checking ========================= +.. default-role:: code +.. include:: rstcommon.rst + **Note:** This feature is experimental, you need to enable it with .. code-block:: nim @@ -10,7 +11,7 @@ Strict not nil checking or -.. code-block:: bash +.. code-block:: cmd nim c --experimental:strictNotNil In the second case it would check builtin and imported modules as well. @@ -60,14 +61,16 @@ You can annotate a type where nil isn't a valid value with `not nil`. If a type can include `nil` as a valid value, dereferencing values of the type -is checked by the compiler: if a value which might be nil is derefenced, this produces a warning by default, you can turn this into an error using the compiler options `--warningAsError:strictNotNil` +is checked by the compiler: if a value which might be nil is derefenced, this +produces a warning by default, you can turn this into an error using +the compiler options `--warningAsError:strictNotNil`:option:. If a type is nilable, you should dereference its values only after a `isNil` or equivalent check. local turn on/off --------------------- -You can still turn off nil checking on function/module level by using a `{.strictNotNil: off}.` pragma. +You can still turn off nil checking on function/module level by using a `{.strictNotNil: off.}` pragma. Note: test that/TODO for code/manual. nilability state @@ -75,11 +78,14 @@ nilability state Currently a nilable value can be `Safe`, `MaybeNil` or `Nil` : we use internally `Parent` and `Unreachable` but this is an implementation detail(a parent layer has the actual nilability). -`Safe` means it shouldn't be nil at that point: e.g. after assignment to a non-nil value or `not a.isNil` check -`MaybeNil` means it might be nil, but it might not be nil: e.g. an argument, a call argument or a value after an `if` and `else`. -`Nil` means it should be nil at that point; e.g. after an assignment to `nil` or a `.isNil` check. - -`Unreachable` means it shouldn't be possible to access this in this branch: so we do generate a warning as well. +- `Safe` means it shouldn't be nil at that point: e.g. after assignment to + a non-nil value or `not a.isNil` check +- `MaybeNil` means it might be nil, but it might not be nil: e.g. an argument, + a call argument or a value after an `if` and `else`. +- `Nil` means it should be nil at that point; e.g. after an assignment to + `nil` or a `.isNil` check. +- `Unreachable` means it shouldn't be possible to access this in this branch: + so we do generate a warning as well. We show an error for each dereference (`[]`, `.field`, `[index]` `()` etc) which is of a tracked expression which is in `MaybeNil` or `Nil` state. diff --git a/doc/nep1.rst b/doc/nep1.rst index 8687725bb..fc69bab69 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -1,11 +1,12 @@ -.. default-role:: code - ========================================================== Nim Enhancement Proposal #1 - Standard Library Style Guide ========================================================== + :Author: Clay Sweetser, Dominik Picheta :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: @@ -139,7 +140,7 @@ Naming Conventions - rotate and rotated - When the 'returns transformed copy' version already exists like `strutils.replace` - an in-place version should get an `-In` suffix (`replaceIn` for this example). + an in-place version should get an ``-In`` suffix (`replaceIn` for this example). - Use `subjectVerb`, not `verbSubject`, e.g.: `fileExists`, not `existsFile`. @@ -225,8 +226,8 @@ indentation indent Coding Conventions ------------------ -- The 'return' statement should ideally be used when its control-flow properties - are required. Use a procedure's implicit 'result' variable whenever possible. +- The `return` statement should ideally be used when its control-flow properties + are required. Use a procedure's implicit `result` variable whenever possible. This improves readability. .. code-block:: nim diff --git a/doc/nimgrep.rst b/doc/nimgrep.rst index 52fc4d62a..6088a4c45 100644 --- a/doc/nimgrep.rst +++ b/doc/nimgrep.rst @@ -1,14 +1,11 @@ -.. default-role:: code - ========================= nimgrep User's manual ========================= -.. default-role:: literal - :Author: Andreas Rumpf :Version: 1.6.0 +.. default-role:: option .. contents:: Nimgrep is a command line tool for search and replace tasks. It can search for @@ -23,11 +20,12 @@ Apart from that it is a generic text manipulation tool. Installation ============ -Compile nimgrep with the command:: +Compile nimgrep with the command: +.. code:: cmd nim c -d:release tools/nimgrep.nim -And copy the executable somewhere in your `$PATH`. +And copy the executable somewhere in your ``$PATH``. Command line switches @@ -40,31 +38,32 @@ Examples All examples below use default PCRE Regex patterns: -+ To search recursively in Nim files using style-insensitive identifiers:: ++ To search recursively in Nim files using style-insensitive identifiers: - --recursive --ext:'nim|nims' --ignoreStyle + .. code:: cmd + nimgrep --recursive --ext:'nim|nims' --ignoreStyle # short: -r --ext:'nim|nims' -y .. Note:: we used `'` quotes to avoid special treatment of `|` symbol for shells like Bash + To exclude version control directories (Git, Mercurial=hg, Subversion=svn) - from the search:: + from the search: - --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' + .. code:: cmd + nimgrep --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' # short: --ed:'^\.git$' --ed:'^\.hg$' --ed:'^\.svn$' + To search only in paths containing the `tests` sub-directory recursively:: - --recursive --includeDir:'(^|/)tests($|/)' + .. code:: cmd + nimgrep --recursive --includeDir:'(^|/)tests($|/)' # short: -r --id:'(^|/)tests($|/)' - .. Attention:: note the subtle difference between `--excludeDir` and - `--includeDir`: the former is applied to relative directory entries + .. Attention:: note the subtle difference between `--excludeDir`:option: and + `--includeDir`:option:\: the former is applied to relative directory entries and the latter is applied to the whole paths + Nimgrep can search multi-line, e.g. to find files containing `import` - and then `strutils` use:: - - 'import(.|\n)*?strutils' + and then `strutils` use pattern `'import(.|\n)*?strutils'`:option:. diff --git a/doc/niminst.rst b/doc/niminst.rst index 3ccb47cc8..8fff1f0e8 100644 --- a/doc/niminst.rst +++ b/doc/niminst.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ========================= niminst User's manual ========================= @@ -7,6 +5,8 @@ :Author: Andreas Rumpf :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Introduction @@ -34,7 +34,7 @@ configuration file. Here's an example of how the syntax looks like: The value of a key-value pair can reference user-defined variables via the `$variable` notation: They can be defined in the command line with the -`--var:name=value` switch. This is useful to not hard-coding the +`--var:name=value`:option: switch. This is useful to not hard-coding the program's version number into the configuration file, for instance. It follows a description of each possible section and how it affects the @@ -91,10 +91,10 @@ Documentation section The `documentation` section supports the `files` key. Listed files will be installed into the OS's native documentation directory -(which might be `$appdir/doc`). +(which might be ``$appdir/doc``). There is a `start` key which determines whether the Windows installer -generates a link to e.g. the `index.html` of your documentation. +generates a link to e.g. the ``index.html`` of your documentation. Other section @@ -125,9 +125,9 @@ Other possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`BinPath` paths to add to the Windows `%PATH%` environment +`BinPath` paths to add to the Windows `%PATH%` environment variable. Example: ``BinPath: r"bin;dist\mingw\bin"`` -`InnoSetup` boolean flag whether an Inno Setup installer should be +`InnoSetup` boolean flag whether an Inno Setup installer should be generated for Windows. Example: `InnoSetup: "Yes"` ==================== ======================================================= @@ -137,7 +137,7 @@ UnixBin section The `UnixBin` section currently only supports the `files` key. Listed files will be installed into the OS's native bin directory -(e.g. `/usr/local/bin`). The exact location depends on the +(e.g. ``/usr/local/bin``). The exact location depends on the installation path the user specifies when running the `install.sh` script. @@ -165,9 +165,9 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`path` Path to Inno Setup. +`path` Path to Inno Setup. Example: ``path = r"c:\inno setup 5\iscc.exe"`` -`flags` Flags to pass to Inno Setup. +`flags` Flags to pass to Inno Setup. Example: `flags = "/Q"` ==================== ======================================================= @@ -180,8 +180,8 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`path` Path to the C compiler. -`flags` Flags to pass to the C Compiler. +`path` Path to the C compiler. +`flags` Flags to pass to the C Compiler. Example: `flags = "-w"` ==================== ======================================================= diff --git a/doc/nims.rst b/doc/nims.rst index f81637d73..5ae1d7baa 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -1,32 +1,33 @@ -.. default-role:: code - ================================ NimScript ================================ +.. default-role:: code +.. include:: rstcommon.rst + Strictly speaking, `NimScript` is the subset of Nim that can be evaluated by Nim's builtin virtual machine (VM). This VM is used for Nim's compiletime function evaluation features. -The `nim` executable processes the `.nims` configuration files in +The `nim`:cmd: executable processes the ``.nims`` configuration files in the following directories (in this order; later files overwrite previous settings): 1) If environment variable `XDG_CONFIG_HOME` is defined, - `$XDG_CONFIG_HOME/nim/config.nims` or - `~/.config/nim/config.nims` (POSIX) or - `%APPDATA%/nim/config.nims` (Windows). This file can be skipped - with the `--skipUserCfg` command line option. -2) `$parentDir/config.nims` where `$parentDir` stands for any + ``$XDG_CONFIG_HOME/nim/config.nims`` or + ``~/.config/nim/config.nims`` (POSIX) or + ``%APPDATA%/nim/config.nims`` (Windows). This file can be skipped + with the `--skipUserCfg`:option: command line option. +2) ``$parentDir/config.nims`` where ``$parentDir`` stands for any parent directory of the project file's path. These files can be - skipped with the `--skipParentCfg` command line option. -3) `$projectDir/config.nims` where `$projectDir` stands for the - project's path. This file can be skipped with the `--skipProjCfg` + skipped with the `--skipParentCfg`:option: command line option. +3) ``$projectDir/config.nims`` where ``$projectDir`` stands for the + project's path. This file can be skipped with the `--skipProjCfg`:option: command line option. 4) A project can also have a project specific configuration file named - `$project.nims` that resides in the same directory as - `$project.nim`. This file can be skipped with the same - `--skipProjCfg` command line option. + ``$project.nims`` that resides in the same directory as + ``$project.nim``. This file can be skipped with the same + `--skipProjCfg`:option: command line option. For available procs and implementation details see `nimscript `_. @@ -111,8 +112,8 @@ See also: NimScript as a configuration file ================================= -A command-line switch `--FOO` is written as `switch("FOO")` in -NimScript. Similarly, command-line `--FOO:VAL` translates to +A command-line switch `--FOO`:option: is written as `switch("FOO")` in +NimScript. Similarly, command-line `--FOO:VAL`:option: translates to `switch("FOO", "VAL")`. Here are few examples of using the `switch` proc: @@ -125,7 +126,7 @@ Here are few examples of using the `switch` proc: # command-line: --forceBuild switch("forceBuild") -NimScripts also support `--` templates for convenience, which look +NimScripts also support `--`:option: templates for convenience, which look like command-line switches written as-is in the NimScript file. So the above example can be rewritten as: @@ -136,7 +137,7 @@ above example can be rewritten as: **Note**: In general, the *define* switches can also be set in NimScripts using `switch` or `--`, as shown in above -examples. Only the `release` define (`-d:release`) cannot be set +examples. Only the `release` define (`-d:release`:option:) cannot be set in NimScripts. @@ -145,7 +146,7 @@ NimScript as a build tool The `task` template that the `system` module defines allows a NimScript file to be used as a build tool. The following example defines a -task `build` that is an alias for the `c` command: +task `build` that is an alias for the `c`:option: command: .. code-block:: nim task build, "builds an example": @@ -159,7 +160,7 @@ Task Description ========= =================================================== `help` List all the available NimScript tasks along with their docstrings. `build` Build the project with the required - backend (`c`, `cpp` or `js`). + backend (`c`:option:, `cpp`:option: or `js`:option:). `tests` Runs the tests belonging to the project. `bench` Runs benchmarks belonging to the project. ========= =================================================== @@ -180,7 +181,7 @@ Standalone NimScript ==================== NimScript can also be used directly as a portable replacement for Bash and -Batch files. Use `nim myscript.nims` to run `myscript.nims`. For example, +Batch files. Use `nim myscript.nims`:cmd: to run ``myscript.nims``. For example, installation of Nimble could be accomplished with this simple script: .. code-block:: nim @@ -199,7 +200,7 @@ installation of Nimble could be accomplished with this simple script: mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe On Unix, you can also use the shebang `#!/usr/bin/env nim`, as long as your filename -ends with `.nims`: +ends with ``.nims``: .. code-block:: nim diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.rst index 509f72e8a..82693da0e 100644 --- a/doc/nimsuggest.rst +++ b/doc/nimsuggest.rst @@ -1,5 +1,3 @@ -.. default-role:: code - ================================ Nim IDE Integration Guide ================================ @@ -7,14 +5,16 @@ :Author: Unknown :Version: |nimversion| +.. default-role:: code +.. include:: rstcommon.rst .. contents:: Nim differs from many other compilers in that it is really fast, and being so fast makes it suited to provide external queries for text editors about the source code being written. Through the -`nimsuggest` tool, any IDE -can query a `.nim` source file and obtain useful information like +`nimsuggest`:cmd: tool, any IDE +can query a ``.nim`` source file and obtain useful information like definition of symbols or suggestions for completion. This document will guide you through the available options. If you @@ -27,29 +27,30 @@ already available. Installation ============ -Nimsuggest is part of Nim's core. Build it via:: +Nimsuggest is part of Nim's core. Build it via: +.. code:: cmd koch nimsuggest Nimsuggest invocation ===================== -Run it via `nimsuggest --stdin --debug myproject.nim`. Nimsuggest is a +Run it via `nimsuggest --stdin --debug myproject.nim`:cmd:. Nimsuggest is a server that takes queries that are related to `myproject`. There is some -support so that you can throw random `.nim` files which are not part +support so that you can throw random ``.nim`` files which are not part of `myproject` at Nimsuggest too, but usually the query refer to modules/files that are part of `myproject`. -`--stdin` means that Nimsuggest reads the query from `stdin`. This is great +`--stdin`:option: means that Nimsuggest reads the query from `stdin`. This is great for testing things out and playing with it but for an editor communication via sockets is more reasonable so that is the default. It listens to port 6000 by default. -Nimsuggest is basically a frontend for the nim compiler so `--path` flags and +Nimsuggest is basically a frontend for the nim compiler so `--path`:option: flags and `config files `_ can be used to specify additional dependencies like -`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`. +`nimsuggest --stdin --debug --path:"dependencies" myproject.nim`:cmd:. Specifying the location of the query @@ -60,25 +61,25 @@ cryptic 3 letter "command" `def` or `con` or `sug` or `use` followed by a location. A query location consists of: -`file.nim` +``file.nim`` This is the name of the module or include file the query refers to. -`dirtyfile.nim` +``dirtyfile.nim`` This is optional. The `file` parameter is enough for static analysis, but IDEs tend to have *unsaved buffers* where the user may still be in the middle of typing a line. In such situations the IDE can save the current contents to a temporary file and then use the - `dirtyfile.nim` option to tell Nimsuggest that `foobar.nim` should - be taken from `temporary/foobar.nim`. + ``dirtyfile.nim`` option to tell Nimsuggest that ``foobar.nim`` should + be taken from ``temporary/foobar.nim``. -`line` +``line`` An integer with the line you are going to query. For the compiler lines start at **1**. -`col` +``col`` An integer with the column you are going to query. For the compiler columns start at **0**. @@ -149,9 +150,9 @@ tab characters (``\t``). The values of each column are: 1. Three characters indicating the type of returned answer (e.g. `def` for definition, `sug` for suggestion, etc). 2. Type of the symbol. This can be `skProc`, `skLet`, and just - about any of the enums defined in the module `compiler/ast.nim`. + about any of the enums defined in the module ``compiler/ast.nim``. 3. Fully qualified path of the symbol. If you are querying a symbol - defined in the `proj.nim` file, this would have the form + defined in the ``proj.nim`` file, this would have the form `proj.symbolName`. 4. Type/signature. For variables and enums this will contain the type of the symbol, for procs, methods and templates this will diff --git a/doc/packaging.rst b/doc/packaging.rst index ecde73f61..ecb17e482 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -45,7 +45,7 @@ The Debian package ships bash and ksh completion and manpages that can be reused Hints on the build process: -:: +.. code:: cmd # build from C sources and then using koch ./build.sh --os $os_type --cpu $cpu_arch diff --git a/doc/regexprs.txt b/doc/regexprs.txt index 9ec08b810..f5544cc28 100644 --- a/doc/regexprs.txt +++ b/doc/regexprs.txt @@ -47,9 +47,11 @@ full documentation of Perl's regular expressions. Because the backslash ``\`` is a meta character both in the Nim programming language and in regular expressions, it is strongly recommended that one uses the *raw* strings of Nim, so that -backslashes are interpreted by the regular expression engine:: +backslashes are interpreted by the regular expression engine: +```nim r"\S" # matches any character that is not whitespace +``` A regular expression is a pattern that is matched against a subject string from left to right. Most characters stand for themselves in a pattern, and diff --git a/doc/testament.rst b/doc/testament.rst index ed1bd63a1..5590cc6d7 100644 --- a/doc/testament.rst +++ b/doc/testament.rst @@ -2,8 +2,8 @@ Testament =========== +.. default-role:: code .. include:: rstcommon.rst -.. default-role:: nim .. contents:: Testament is an advanced automatic unittests runner for Nim tests, is used for the development of Nim itself, diff --git a/doc/tools.rst b/doc/tools.rst index e0044e1ca..0de4ac914 100644 --- a/doc/tools.rst +++ b/doc/tools.rst @@ -1,9 +1,10 @@ -.. default-role:: code - ======================== Tools available with Nim ======================== +.. default-role:: code +.. include:: rstcommon.rst + The standard distribution ships with the following tools: - | `Hot code reloading `_ @@ -11,11 +12,11 @@ The standard distribution ships with the following tools: document explaining how it works. - | `Documentation generator `_ - | The builtin document generator `nim doc` generates HTML documentation - from `.nim` source files. + | The builtin document generator `nim doc`:cmd: generates HTML documentation + from ``.nim`` source files. - | `Nimsuggest for IDE support `_ - | Through the `nimsuggest` tool, any IDE can query a `.nim` source file + | Through the `nimsuggest`:cmd: tool, any IDE can query a ``.nim`` source file and obtain useful information like the definition of symbols or suggestions for completion. @@ -29,11 +30,11 @@ The standard distribution ships with the following tools: | Nim search and replace utility. - | nimpretty - | `nimpretty` is a Nim source code beautifier, + | `nimpretty`:cmd: is a Nim source code beautifier, to format code according to the official style guide. - | `testament `_ - | `testament` is an advanced automatic *unittests runner* for Nim tests, + | `testament`:cmd: is an advanced automatic *unittests runner* for Nim tests, is used for the development of Nim itself, offers process isolation for your tests, it can generate statistics about test cases, supports multiple targets (C, JS, etc), `simulated Dry-Runs `_, From 5423915e5ade14fb98c2aba28f269311506960e5 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 4 Jun 2021 16:20:47 +0200 Subject: [PATCH 0068/1654] real bugfix for #17170 (#18171) * real bugfix for #17170 * better fix --- compiler/closureiters.nim | 10 ++++++---- compiler/injectdestructors.nim | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index adc5a4873..a8da7485e 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -414,8 +414,11 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] = proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode = - result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v) - result.info = v.info + if isEmptyType(v.typ): + result = v + else: + result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v) + result.info = v.info proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) = if input.kind == nkStmtListExpr: @@ -427,8 +430,7 @@ proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) = proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode = result = newNodeI(nkStmtList, exprBody.info) - if exprBody.typ != nil: - ctx.addExprAssgn(result, exprBody, res) + ctx.addExprAssgn(result, exprBody, res) proc newNotCall(g: ModuleGraph; e: PNode): PNode = result = newTree(nkCall, newSymNode(g.getSysMagic(e.info, "not", mNot), e.info), e) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 48a852d26..8ef08adeb 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -364,6 +364,7 @@ proc genMarkCyclic(c: var Con; result, dest: PNode) = proc genCopyNoCheck(c: var Con; dest, ri: PNode): PNode = let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink}) result = c.genOp(t, attachedAsgn, dest, ri) + assert ri.typ != nil proc genCopy(c: var Con; dest, ri: PNode): PNode = let t = dest.typ @@ -371,6 +372,7 @@ proc genCopy(c: var Con; dest, ri: PNode): PNode = # try to improve the error message here: c.checkForErrorPragma(t, ri, "=copy") result = c.genCopyNoCheck(dest, ri) + assert ri.typ != nil proc genDiscriminantAsgn(c: var Con; s: var Scope; n: PNode): PNode = # discriminator is ordinal value that doesn't need sink destroy From a77360da5bad40d22e942cdd5cb9614c3548bf34 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 5 Jun 2021 03:41:13 +0800 Subject: [PATCH 0069/1654] workaround bug in sharedtables.withValue to unblock frozen CI on OSX (#18172) --- lib/pure/collections/sharedtables.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 2788ec785..a85674df9 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -108,10 +108,14 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, table.withValue("a", value): value[] = "m" + var flag = false table.withValue("d", value): discard value doAssert false do: # if "d" notin table + flag = true + + if flag: table["d"] = "n" assert table.mget("a") == "m" From cc7ec5a6a42c2aead2c411228f023cbeff4220de Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 4 Jun 2021 15:32:46 -0700 Subject: [PATCH 0070/1654] fix #7295: use move(result) inside template to avoid copy with --gc:refc (#18168) --- lib/pure/collections/sequtils.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index e937b8394..6facfcc32 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -1014,27 +1014,27 @@ template applyIt*(varSeq, op: untyped) = template newSeqWith*(len: int, init: untyped): untyped = - ## Creates a new sequence of length `len`, calling `init` to initialize - ## each value of the sequence. - ## - ## Useful for creating "2D" sequences - sequences containing other sequences - ## or to populate fields of the created sequence. + ## Creates a new `seq` of length `len`, calling `init` to initialize + ## each value of the seq. ## + ## Useful for creating "2D" seqs - seqs containing other seqs + ## or to populate fields of the created seq. runnableExamples: - ## Creates a sequence containing 5 bool sequences, each of length of 3. + ## Creates a seq containing 5 bool seqs, each of length of 3. var seq2D = newSeqWith(5, newSeq[bool](3)) assert seq2D.len == 5 assert seq2D[0].len == 3 assert seq2D[4][2] == false - ## Creates a sequence of 20 random numbers from 1 to 10 - import random - var seqRand = newSeqWith(20, rand(10)) + ## Creates a seq with random numbers + import std/random + var seqRand = newSeqWith(20, rand(1.0)) + assert seqRand[0] != seqRand[1] var result = newSeq[typeof(init)](len) for i in 0 ..< len: result[i] = init - result + move(result) # refs bug #7295 func mapLitsImpl(constructor: NimNode; op: NimNode; nested: bool; filter = nnkLiterals): NimNode = From a2b60812568b5b90af9bb15c50873a726dd49b94 Mon Sep 17 00:00:00 2001 From: Euan Date: Fri, 4 Jun 2021 23:46:57 +0100 Subject: [PATCH 0071/1654] Ref #18177 - lbgc.so.5.0 on OpenBSD 6.9 (#18180) --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index 942d29fe8..cae0ec3e2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1146,7 +1146,7 @@ when defined(boehmgc): elif defined(macosx): const boehmLib = "libgc.dylib" elif defined(openbsd): - const boehmLib = "libgc.so.4.0" + const boehmLib = "libgc.so.(4|5).0" elif defined(freebsd): const boehmLib = "libgc-threaded.so.1" else: From 3cc547f2dfc3d2a981ccde38adb1f3deeb9715c2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 4 Jun 2021 21:58:26 -0700 Subject: [PATCH 0072/1654] macros.treeRepr + friends: collapse SymChoice (#18072) * macros.treeRepr + friends: collapse SymChoice * make repr+friends work with invalid symchoice nodes * address comment --- changelog.md | 3 + lib/core/macros.nim | 63 +++++++++++++------- tests/macros/tdumpast.nim | 122 +++++++++++++++++++++++++++++++------- tests/stdlib/tmacros.nim | 6 ++ 4 files changed, 151 insertions(+), 43 deletions(-) diff --git a/changelog.md b/changelog.md index 06251abfc..1050e4916 100644 --- a/changelog.md +++ b/changelog.md @@ -81,6 +81,9 @@ - Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, + use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. + - The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. The downside is that these defines now have custom logic that doesn't apply for other defines. diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 89f90a23b..37f855d94 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -856,6 +856,29 @@ proc nestList*(op: NimNode; pack: NimNode; init: NimNode): NimNode = for i in countdown(pack.len - 1, 0): result = newCall(op, pack[i], result) +proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. + +proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` can be an identifier or a + ## symbol. `a` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `b` can be an identifier or a + ## symbol. `b` may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} + ## Style insensitive comparison. `a` and `b` can be an + ## identifier or a symbol. Both may be wrapped in an export marker + ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), + ## these nodes will be unwrapped. + +const collapseSymChoice = not defined(nimLegacyMacrosCollapseSymChoice) + proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indented = false) {.benign.} = if level > 0: if indented: @@ -883,8 +906,21 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent res.add(" " & $n.strVal.newLit.repr) of nnkNone: assert false + elif n.kind in {nnkOpenSymChoice, nnkClosedSymChoice} and collapseSymChoice: + res.add(" " & $n.len) + if n.len > 0: + var allSameSymName = true + for i in 0.. 0: + res.add(" " & n[0].strVal.newLit.repr) else: res.add(".newTree(") for j in 0..`_. return node.copyNimTree() -proc eqIdent*(a: string; b: string): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. - -proc eqIdent*(a: NimNode; b: string): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. `a` can be an identifier or a - ## symbol. `a` may be wrapped in an export marker - ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), - ## these nodes will be unwrapped. - -proc eqIdent*(a: string; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. `b` can be an identifier or a - ## symbol. `b` may be wrapped in an export marker - ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), - ## these nodes will be unwrapped. - -proc eqIdent*(a: NimNode; b: NimNode): bool {.magic: "EqIdent", noSideEffect.} - ## Style insensitive comparison. `a` and `b` can be an - ## identifier or a symbol. Both may be wrapped in an export marker - ## (`nnkPostfix`) or quoted with backticks (`nnkAccQuoted`), - ## these nodes will be unwrapped. - proc expectIdent*(n: NimNode, name: string) {.since: (1,1).} = ## Check that `eqIdent(n,name)` holds true. If this is not the ## case, compilation aborts with an error message. This is useful diff --git a/tests/macros/tdumpast.nim b/tests/macros/tdumpast.nim index b6bcbe8f0..484b3c2f3 100644 --- a/tests/macros/tdumpast.nim +++ b/tests/macros/tdumpast.nim @@ -2,34 +2,69 @@ import macros -template plus(a, b: untyped): untyped {.dirty} = - a + b +block: + template plus(a, b: untyped): untyped {.dirty} = + a + b -macro call(e: untyped): untyped = - result = newCall("foo", newStrLitNode("bar")) + macro call(e: untyped): untyped = + result = newCall("foo", newStrLitNode("bar")) -macro dumpAST(n: untyped): untyped = - # dump AST as a side-effect and return the inner node - let n = callsite() - echo n.lispRepr - echo n.treeRepr + macro dumpAST(n: untyped): string = + var msg = "" + msg.add "lispRepr:\n" & n.lispRepr & "\n" + msg.add "treeRepr:\n" & n.treeRepr & "\n" - var plusAst = getAst(plus(1, 2)) - echo plusAst.lispRepr + var plusAst = getAst(plus(1, 2)) + msg.add "lispRepr:\n" & n.lispRepr & "\n" - var callAst = getAst(call(4)) - echo callAst.lispRepr + var callAst = getAst(call(4)) + msg.add "callAst.lispRepr:\n" & callAst.lispRepr & "\n" - var e = parseExpr("foo(bar + baz)") - echo e.lispRepr + var e = parseExpr("foo(bar + baz)") + msg.add "e.lispRepr:\n" & e.lispRepr & "\n" + result = msg.newLit - result = n[1] + let a = dumpAST: + proc add(x, y: int): int = + return x + y + const foo = 3 -dumpAST: - proc add(x, y: int): int = - return x + y - - proc sub(x, y: int): int = return x - y + doAssert a == """ +lispRepr: +(StmtList (ProcDef (Ident "add") (Empty) (Empty) (FormalParams (Ident "int") (IdentDefs (Ident "x") (Ident "y") (Ident "int") (Empty))) (Empty) (Empty) (StmtList (ReturnStmt (Infix (Ident "+") (Ident "x") (Ident "y"))))) (ConstSection (ConstDef (Ident "foo") (Empty) (IntLit 3)))) +treeRepr: +StmtList + ProcDef + Ident "add" + Empty + Empty + FormalParams + Ident "int" + IdentDefs + Ident "x" + Ident "y" + Ident "int" + Empty + Empty + Empty + StmtList + ReturnStmt + Infix + Ident "+" + Ident "x" + Ident "y" + ConstSection + ConstDef + Ident "foo" + Empty + IntLit 3 +lispRepr: +(StmtList (ProcDef (Ident "add") (Empty) (Empty) (FormalParams (Ident "int") (IdentDefs (Ident "x") (Ident "y") (Ident "int") (Empty))) (Empty) (Empty) (StmtList (ReturnStmt (Infix (Ident "+") (Ident "x") (Ident "y"))))) (ConstSection (ConstDef (Ident "foo") (Empty) (IntLit 3)))) +callAst.lispRepr: +(Call (Ident "foo") (StrLit "bar")) +e.lispRepr: +(Call (Ident "foo") (Infix (Ident "+") (Ident "bar") (Ident "baz"))) +""" macro fun() = let n = quote do: @@ -79,3 +114,48 @@ block: # repr doAssert repr2((1, 2, 3.0)) == "(1, 2, 3.0)" doAssert repr2((1)) == "(1)" doAssert repr2((1+2)) == "(1 + 2)" + +block: # treeRepr + macro treeRepr2(a: untyped): string = newLit a.treeRepr + macro treeRepr3(a: typed): string = newLit a.treeRepr + + doAssert treeRepr2(1+1 == 2) == """ +Infix + Ident "==" + Infix + Ident "+" + IntLit 1 + IntLit 1 + IntLit 2""" + + proc baz() = discard + proc baz(a: int) = discard + proc baz(a: float) = discard + + doAssert treeRepr3(baz()) == """ +Call + Sym "baz"""" + + let a = treeRepr3(block: + proc bar(a: auto) = baz()) + doAssert a == """ +BlockStmt + Empty + ProcDef + Sym "bar" + Empty + GenericParams + Sym "a:type" + FormalParams + Empty + IdentDefs + Sym "a" + Sym "auto" + Empty + Empty + Bracket + Empty + Empty + StmtList + Call + OpenSymChoice 3 "baz"""" diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 52ea1e898..18cfdc75c 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -1,3 +1,9 @@ +#[ +xxx macros tests need to be reorganized to makes sure each API is tested once +See also: + tests/macros/tdumpast.nim for treeRepr + friends +]# + import std/macros block: # hasArgOfName From 295429f42595ce5c707456e657e4de6b2734df42 Mon Sep 17 00:00:00 2001 From: Rainbow Asteroids Date: Sat, 5 Jun 2021 03:07:26 -0400 Subject: [PATCH 0073/1654] add `dom.scrollIntoView` with options; refs #18093 (#18181) --- changelog.md | 2 ++ lib/js/dom.nim | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/changelog.md b/changelog.md index 1050e4916..7619ee6a3 100644 --- a/changelog.md +++ b/changelog.md @@ -335,6 +335,8 @@ - Added setCurrentException for JS backend. +- Added `dom.scrollIntoView` proc with options + ## Language changes - `nimscript` now handles `except Exception as e`. diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 16fc96b68..2b2332020 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1317,6 +1317,11 @@ type ready*: FontFaceSetReady onloadingdone*: proc(event: Event) + ScrollIntoViewOptions* = object + behavior*: cstring + `block`*: cstring + inline*: cstring + since (1, 3): type DomParser* = ref object @@ -1538,6 +1543,7 @@ proc removeAttribute*(n: Node, attr: cstring) proc removeAttributeNode*(n, attr: Node) proc replaceData*(n: Node, start, len: int, text: cstring) proc scrollIntoView*(n: Node) +proc scrollIntoView*(n: Node, options: ScrollIntoViewOptions) proc setAttribute*(n: Node, name, value: cstring) proc setAttributeNode*(n: Node, attr: Node) proc querySelector*(n: Node, selectors: cstring): Element From 9c0666e0bbb7af69e2d87e7e81302ed3d3f06412 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 5 Jun 2021 10:12:37 +0300 Subject: [PATCH 0074/1654] rst: remove `roSkipPounds` option (#18175) --- lib/packages/docutils/rst.nim | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index dae692fb7..336caa2df 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -201,8 +201,6 @@ from highlite import SourceLanguage, getSourceLanguage type RstParseOption* = enum ## options for the RST parser - roSkipPounds, ## skip ``#`` at line beginning (documentation - ## embedded in Nim comments) roSupportSmilies, ## make the RST parser support smilies like ``:)`` roSupportRawDirective, ## support the ``raw`` directive (don't support ## it for sandboxing) @@ -307,7 +305,6 @@ type buf*: cstring bufpos*: int line*, col*, baseIndent*: int - skipPounds*: bool adornmentLine*: bool escapeNext*: bool @@ -380,9 +377,6 @@ proc getIndentAux(L: var Lexer, start: int): int = else: inc pos elif L.buf[pos] == '\n': inc pos - if L.skipPounds: - if L.buf[pos] == '#': inc pos - if L.buf[pos] == '#': inc pos while true: case L.buf[pos] of ' ', '\v', '\f': @@ -447,26 +441,13 @@ proc rawGetTok(L: var Lexer, tok: var Token) = inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, skipPounds: bool, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq): int = var L: Lexer var length = tokens.len L.buf = cstring(buffer) L.line = 0 # skip UTF-8 BOM if L.buf[0] == '\xEF' and L.buf[1] == '\xBB' and L.buf[2] == '\xBF': inc L.bufpos, 3 - L.skipPounds = skipPounds - if skipPounds: - if L.buf[L.bufpos] == '#': - inc L.bufpos - inc result - if L.buf[L.bufpos] == '#': - inc L.bufpos - inc result - L.baseIndent = 0 - while L.buf[L.bufpos] == ' ': - inc L.bufpos - inc L.baseIndent - inc result while true: inc length setLen(tokens, length) @@ -2123,7 +2104,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode = q.col = cols[j] q.line = line - 1 q.filename = p.filename - q.col += getTokens(row[j], false, q.tok) + q.col += getTokens(row[j], q.tok) b = newRstNode(rnTableDataCell) b.add(parseDoc(q)) a.add(b) @@ -2176,7 +2157,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = q.col = p.col q.line = currentTok(p).line - 1 q.filename = p.filename - q.col += getTokens(getColContents(p, row[j]), false, q.tok) + q.col += getTokens(getColContents(p, row[j]), q.tok) b.add(parseDoc(q)) a.add(b) result.add(a) @@ -2584,7 +2565,6 @@ proc dirInclude(p: var RstParser): PRstNode = q.filename = path q.col += getTokens( inputString[startPosition..endPosition].strip(), - false, q.tok) # workaround a GCC bug; more like the interior pointer bug? #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0: @@ -2948,7 +2928,7 @@ proc rstParse*(text, filename: string, initParser(p, newSharedState(options, findFile, msgHandler)) p.filename = filename p.line = line - p.col = column + getTokens(text, roSkipPounds in options, p.tok) + p.col = column + getTokens(text, p.tok) let unresolved = parseDoc(p) countTitles(p, unresolved) orderFootnotes(p) From 9c6259e55717acbc954140ef2bb0ca363971c8c7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 5 Jun 2021 00:45:37 -0700 Subject: [PATCH 0075/1654] up to 20x faster jsonutils deserialization (#18183) * up to 20x faster jsonutils deserialization * noinline --- lib/std/jsonutils.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 60d78fea5..c141985d3 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -128,14 +128,14 @@ macro initCaseObject(T: typedesc, fun: untyped): untyped = `fun`(`key2`, typedesc[`typ`]) result.add newTree(nnkExprColonExpr, key, val) -proc checkJsonImpl(cond: bool, condStr: string, msg = "") = - if not cond: - # just pick 1 exception type for simplicity; other choices would be: - # JsonError, JsonParser, JsonKindError - raise newException(ValueError, msg) +proc raiseJsonException(condStr: string, msg: string) {.noinline.} = + # just pick 1 exception type for simplicity; other choices would be: + # JsonError, JsonParser, JsonKindError + raise newException(ValueError, condStr & " failed: " & msg) template checkJson(cond: untyped, msg = "") = - checkJsonImpl(cond, astToStr(cond), msg) + if not cond: + raiseJsonException(astToStr(cond), msg) proc hasField[T](obj: T, field: string): bool = for k, _ in fieldPairs(obj): From b9843b156dfc16f1f99cbee8fc8e55b7abd6c21b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 5 Jun 2021 00:47:28 -0700 Subject: [PATCH 0076/1654] fix #18178, set a CI timeout for github action pipelines instead of the 6 hour default (#18179) --- .github/workflows/ci_docs.yml | 1 + .github/workflows/ci_packages.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml index ddef94fbe..6ae228d2d 100644 --- a/.github/workflows/ci_docs.yml +++ b/.github/workflows/ci_docs.yml @@ -44,6 +44,7 @@ jobs: name: ${{ matrix.target }} runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 steps: - name: 'Checkout' diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index aeb29e147..529e6e8b6 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -11,6 +11,7 @@ jobs: batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num` name: '${{ matrix.os }} (batch: ${{ matrix.batch }})' runs-on: ${{ matrix.os }} + timeout-minutes: 60 # refs bug #18178 env: NIM_TEST_PACKAGES: "1" NIM_TESTAMENT_BATCH: ${{ matrix.batch }} From 9106ab7db19e22782c544c6362741dff4142820e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 6 Jun 2021 00:53:22 -0700 Subject: [PATCH 0077/1654] make changeDetectedViaJsonBuildInstructions warning less verbose when json file invalid (eg due to spec change) (#18197) --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index ea2b09866..831acae68 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -993,7 +993,7 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute var bcache: BuildCache try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - stderr.write "Warning: JSON processing failed for $#: $#\n" % [jsonFile.string, getCurrentExceptionMsg()] + stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string return true if bcache.currentDir != getCurrentDir() or # fixes bug #16271 bcache.configFiles != conf.configFiles.mapIt(it.string) or From e7816a3e77ab0f821d66667dfc448f1898199f59 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 6 Jun 2021 10:55:20 +0300 Subject: [PATCH 0078/1654] update docgen documentation (#18192) * update docgen documentation * --fullhelp * Update doc/docgen.rst Co-authored-by: Timothee Cour * [skip ci] fix typos Co-authored-by: Timothee Cour --- changelog.md | 7 +++ doc/advopt.txt | 3 +- doc/docgen.rst | 123 ++++++++++++++++++++++++++++++------------------- 3 files changed, 84 insertions(+), 49 deletions(-) diff --git a/changelog.md b/changelog.md index 7619ee6a3..f141ea95a 100644 --- a/changelog.md +++ b/changelog.md @@ -449,6 +449,13 @@ ## Tool changes +- Latex doc generation is revised: output `.tex` files should be compiled + by `xelatex` (not by `pdflatex` as before). Now default Latex settings + provide support for Unicode and do better job for avoiding margin overflows. + +- Implemented `doc2tex` compiler command which converts documentation in + `.nim` files to Latex. + - The rst parser now supports markdown table syntax. Known limitations: - cell alignment is not supported, i.e. alignment annotations in a delimiter diff --git a/doc/advopt.txt b/doc/advopt.txt index e627d4768..e5b4c9568 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -6,7 +6,8 @@ Advanced commands: //e run a Nimscript file //rst2html convert a reStructuredText file to HTML use `--docCmd:skip` to skip compiling snippets - //rst2tex convert a reStructuredText file to TeX + //rst2tex convert a reStructuredText file to LaTeX + //doc2tex extract the documentation to a LaTeX file //jsondoc extract the documentation to a json file //ctags create a tags file //buildIndex build an index for the whole documentation diff --git a/doc/docgen.rst b/doc/docgen.rst index e9ff7f1e3..2074ee06f 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -14,12 +14,22 @@ Introduction ============ This document describes the `documentation generation tools`:idx: built into -the `Nim compiler `_, which can generate HTML and JSON output -from input .nim files and projects, as well as HTML and LaTeX from input RST +the `Nim compiler `_, which can generate HTML, Latex and JSON output +from input ``.nim`` files and projects, as well as HTML and LaTeX from input RST (reStructuredText) files. The output documentation will include the module dependencies (`import`), any top-level documentation comments (`##`), and exported symbols (`*`), including procedures, types, and variables. +=================== ====================== ============ ============== +command runs on... input format output format +=================== ====================== ============ ============== +`nim doc`:cmd: documentation comments ``.nim`` ``.html`` HTML +`nim doc2tex`:cmd: ″ ″ ``.tex`` LaTeX +`nim jsondoc`:cmd: ″ ″ ``.json`` JSON +`nim rst2html`:cmd: standalone rst files ``.rst`` ``.html`` HTML +`nim rst2tex`:cmd: ″ ″ ``.tex`` LaTeX +=================== ====================== ============ ============== + Quick start ----------- @@ -53,7 +63,8 @@ Any comments which are preceded by a double-hash (`##`), are interpreted as documentation. Comments are parsed as RST (see `reference `_), providing Nim module authors the ability to easily generate richly formatted -documentation with only their well-documented code. +documentation with only their well-documented code! +Basic Markdown syntax is also supported inside the doc comments. Example: @@ -99,32 +110,22 @@ won't influence RST formatting. ## ## Paragraph. -Nim file input ------------------ - -The following examples will generate documentation for the below contrived -*Nim* module, aptly named 'sample.nim' - -sample.nim: - -.. code-block:: nim - ## This module is a sample. - - import std/strutils - - proc helloWorld*(times: int) = - ## Takes an integer and outputs - ## as many "hello world!"s - - for i in 0 .. times-1: - echo "hello world!" - - helloWorld(5) - Document Types ============== +Example of Nim file input +------------------------- + +The following examples will generate documentation for this sample +*Nim* module, aptly named ``doc/docgen_sample.nim``: + +.. code:: nim + :file: docgen_sample.nim + +All the below commands save their output to ``htmldocs`` directory relative to +the directory of file; +hence the output for this sample will be in ``doc/htmldocs``. HTML ---- @@ -138,7 +139,7 @@ The `doc`:option: command: .. code:: cmd - nim doc sample + nim doc docgen_sample.nim Partial Output:: ... @@ -149,6 +150,27 @@ The full output can be seen here: `docgen_sample.html `_. It runs after semantic checking and includes pragmas attached implicitly by the compiler. +LaTeX +----- + +LaTeX files are intended to be converted to PDF, especially for offline +reading or making hard copies. (LaTeX output is oftentimes better than +HTML -> PDF conversion). + +The `doc2tex`:option: command: + +.. code:: cmd + + nim doc2tex docgen_sample.nim + cd htmldocs + xelatex docgen_sample.tex + xelatex docgen_sample.tex + # It is usually necessary to run `xelatex` 2 times (or even 3 times for + # large documents) to get all labels generated. + # That depends on this warning in the end of `xelatex` output: + # LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. + +The output is ``docgen_sample.pdf``. JSON ---- @@ -163,7 +185,7 @@ The `jsondoc`:option: command: .. code:: cmd - nim jsondoc sample + nim jsondoc docgen_sample.nim Output:: { @@ -189,7 +211,7 @@ The `jsondoc0`:option: command: .. code:: cmd - nim jsondoc0 sample + nim jsondoc0 docgen_sample.nim Output:: [ @@ -204,7 +226,7 @@ Output:: } ] -Note that the `jsondoc`:option: command outputs it's JSON without pretty-printing it, +Note that the `jsondoc`:option: command outputs its JSON without pretty-printing it, while `jsondoc0`:option: outputs pretty-printed JSON. Related Options @@ -217,9 +239,10 @@ Project switch nim doc --project filename.nim -This will recursively generate documentation of all nim modules imported +This will recursively generate documentation of all Nim modules imported into the input module that belong to the Nimble package that ``filename.nim`` -belongs to. +belongs to. The index files and the corresponding ``theindex.html`` will +also be generated. Index switch @@ -252,12 +275,13 @@ documented item in your source code pointing to the implementation of that item on a GitHub repository. You can click the link to see the implementation of the item. -The `git.commit`:option: switch overrides the hardcoded `devel` branch in config/nimdoc.cfg. +The `git.commit`:option: switch overrides the hardcoded `devel` branch in +``config/nimdoc.cfg``. This is useful to link to a different branch e.g. `--git.commit:master`:option:, or to a tag e.g. `--git.commit:1.2.3`:option: or a commit. Source URLs are generated as ``href="${url}/tree/${commit}/${path}#L${line}"`` -by default and this compatible with GitHub but not with GitLab. +by default and thus compatible with GitHub but not with GitLab. Similarly, `git.devel`:option: switch overrides the hardcoded `devel` branch for the `Edit` link which is also useful if you have a different working @@ -282,9 +306,10 @@ Other Input Formats The *Nim compiler* also has support for RST (reStructuredText) files with the `rst2html`:option: and `rst2tex`:option: commands. Documents like this one are -initially written in a dialect of RST which adds support for nim source code +initially written in a dialect of RST which adds support for Nim source code highlighting with the ``.. code-block:: nim`` prefix. ``code-block`` also -supports highlighting of C++ and some other c-like languages. +supports highlighting of a few other languages supported by the +`packages/docutils/highlite module `_. Usage: @@ -333,16 +358,16 @@ if they don't have any parameters. If there are parameters, they are represented by their types and will be comma-separated. To the plain symbol a suffix may be added depending on the type of the callable: -------------- -------------- -Callable type Suffix -------------- -------------- -proc *empty string* -macro ``.m`` -method ``.e`` -iterator ``.i`` -template ``.t`` -converter ``.c`` -------------- -------------- +============== ============== +Callable type Suffix +============== ============== +`proc`, `func` *empty string* +`macro` ``.m`` +`method` ``.e`` +`iterator` ``.i`` +`template` ``.t`` +`converter` ``.c`` +============== ============== The relationship of type to suffix is made by the proc `complexName` in the ``compiler/docgen.nim`` file. Here are some examples of complex names for @@ -422,10 +447,12 @@ final index, and TOC entries found in ``.nim`` files are discarded. Additional resources ==================== -`Nim Compiler User Guide `_ +* `Nim Compiler User Guide `_ -`RST Quick Reference -`_ +* Documentation for `rst module `_ -- Nim RST/Markdown parser. + +* `RST Quick Reference + `_ The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and ``config/nimdoc.tex.cfg`` configuration files. You can add and modify these From e7aa41ff3f6820d8ada3fd650d523c7f7d584fda Mon Sep 17 00:00:00 2001 From: slangmgh <37659406+slangmgh@users.noreply.github.com> Date: Mon, 7 Jun 2021 17:42:32 +0800 Subject: [PATCH 0079/1654] fix 18186 with adding importc _umul128 (#18200) --- lib/system/dragonbox.nim | 3 ++- lib/system/schubfach.nim | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim index db608c630..eec8749da 100644 --- a/lib/system/dragonbox.nim +++ b/lib/system/dragonbox.nim @@ -867,10 +867,11 @@ when defined(sizeof_Int128): return (hi, lo) elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "".} proc mul128*(x: uint64; y: uint64): uint64x2 {.inline.} = var hi: uint64 = 0 var lo: uint64 = umul128(x, y, addr(hi)) - return (hi, lo) + return uint64x2(hi: hi, lo: lo) else: proc lo32*(x: uint64): uint32 {.inline.} = diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index c344b7425..94d3d3c25 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -147,6 +147,7 @@ when defined(sizeof_Int128): return y1 or uint32(y0 > 1) elif defined(vcc) and defined(cpu64): + proc umul128(x, y: uint64, z: ptr uint64): uint64 {.importc: "_umul128", header: "".} proc roundToOdd(g: uint64; cpHi: uint32): uint32 {.inline.} = var p1: uint64 = 0 var p0: uint64 = umul128(g, cpHi, addr(p1)) From 2ec52faae5ba5c233f977a8a4b997734c4ad83e7 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 7 Jun 2021 10:44:43 +0100 Subject: [PATCH 0080/1654] koch: print help if no command is given (#18199) --- koch.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 3d7276aee..665b5e042 100644 --- a/koch.nim +++ b/koch.nim @@ -708,4 +708,5 @@ when isMainModule: of "branchdone": branchDone() else: showHelp(success = false) break - of cmdEnd: break + of cmdEnd: + showHelp(success = false) From 429b1286325e7b6cf86ed4e29cea3e11a8c0e2df Mon Sep 17 00:00:00 2001 From: Bung Date: Mon, 7 Jun 2021 19:57:42 +0800 Subject: [PATCH 0081/1654] change mimedb stroe stringtable to orderedtable (#18065) * change mimedb stroe stringtable to orderedtable * Update lib/pure/mimetypes.nim Co-authored-by: Andreas Rumpf --- lib/pure/mimetypes.nim | 7 ++++--- tests/stdlib/tmimetypes.nim | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index c10739fec..6ae63a0ae 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -26,12 +26,12 @@ runnableExamples: doAssert m.getMimetype("fakext") == "text/fakelang" doAssert m.getMimetype("FaKeXT") == "text/fakelang" -import strtabs +import tables from strutils import startsWith, toLowerAscii, strip type MimeDB* = object - mimes: StringTableRef + mimes: OrderedTableRef[string, string] const mimes* = { "123": "application/vnd.lotus-1-2-3", @@ -1903,7 +1903,8 @@ const mimes* = { func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. - result.mimes = mimes.newStringTable() + {.cast(noSideEffect).}: + result.mimes = mimes.newOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` diff --git a/tests/stdlib/tmimetypes.nim b/tests/stdlib/tmimetypes.nim index cd41e614b..93c20d4a3 100644 --- a/tests/stdlib/tmimetypes.nim +++ b/tests/stdlib/tmimetypes.nim @@ -6,6 +6,7 @@ import std/mimetypes template main() = var m = newMimetypes() doAssert m.getMimetype("mp4") == "video/mp4" + doAssert m.getExt("application/json") == "json" # see also `runnableExamples`. # xxx we should have a way to avoid duplicating code between runnableExamples and tests From 5bad022d58e21f0da455fbd7d421cd26a2b5c812 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 7 Jun 2021 21:32:37 +0800 Subject: [PATCH 0082/1654] alternative to #18185 (#18206) --- lib/core/rlocks.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 5f155262a..a1ee0a6ec 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -11,7 +11,10 @@ when not compileOption("threads") and not defined(nimdoc): - {.error: "Rlocks requires --threads:on option.".} + when false: + # make rlocks modlue consistent with locks module, + # so they can replace each other seamlessly. + {.error: "Rlocks requires --threads:on option.".} const insideRLocksModule = true include "system/syslocks" From 21f3b8539a2e440f7cb69502721b6c56070e122f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 7 Jun 2021 17:22:35 +0200 Subject: [PATCH 0083/1654] fixes #18088 (#18209) --- lib/pure/nativesockets.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 605f62321..124afce56 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -438,7 +438,7 @@ proc getHostname*(): string {.tags: [ReadIOEffect].} = ## Returns the local hostname (not the FQDN) # https://tools.ietf.org/html/rfc1035#section-2.3.1 # https://tools.ietf.org/html/rfc2181#section-11 - const size = 64 + const size = 256 result = newString(size) when useWinVersion: let success = winlean.gethostname(result, size) From 51ab7ccec1ffa21cef67f1a75dea889653751cfc Mon Sep 17 00:00:00 2001 From: drtheuns Date: Mon, 7 Jun 2021 18:40:09 +0200 Subject: [PATCH 0084/1654] Fix JS error on index page and detect dark mode (#18191) * Fix JS error on index page and detect dark mode The theindex.html page doesn't have a dark mode switch so the main function will error because `toggleSwitch` is not defined. Checks have been added to prevent this from happening. Also add automatic detection of system settings for dark-mode. This could also be done with pure css, but then the dark mode variable declarations would have to be duplicated to work with the switch so I went with this approach. * Fix nimdoc tests * Fix rst2html tests --- config/nimdoc.cfg | 13 +++++++++---- nimdoc/rst2html/expected/rst_examples.html | 13 +++++++++---- nimdoc/test_out_index_dot_html/expected/index.html | 13 +++++++++---- .../test_out_index_dot_html/expected/theindex.html | 13 +++++++++---- .../testproject/expected/subdir/subdir_b/utils.html | 13 +++++++++---- nimdoc/testproject/expected/testproject.html | 13 +++++++++---- nimdoc/testproject/expected/theindex.html | 13 +++++++++---- 7 files changed, 63 insertions(+), 28 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index ed1b346a2..cdf49197d 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -252,7 +252,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -263,13 +262,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 5c434193e..68b796be7 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index bac39fd33..fac758c9b 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index aa9ca2645..34ddf8f6a 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 826cad02a..6aa71e8c2 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 513dd507b..65d9f83f5 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index e6efee986..55f7f33c9 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -34,7 +34,6 @@ function main() { } } - const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); function switchTheme(e) { if (e.target.checked) { document.documentElement.setAttribute('data-theme', 'dark'); @@ -45,13 +44,19 @@ function main() { } } - toggleSwitch.addEventListener('change', switchTheme, false); + const toggleSwitch = document.querySelector('.theme-switch input[type="checkbox"]'); + if (toggleSwitch !== null) { + toggleSwitch.addEventListener('change', switchTheme, false); + } - const currentTheme = localStorage.getItem('theme') ? localStorage.getItem('theme') : null; + var currentTheme = localStorage.getItem('theme'); + if (!currentTheme && window.matchMedia('(prefers-color-scheme: dark)').matches) { + currentTheme = 'dark'; + } if (currentTheme) { document.documentElement.setAttribute('data-theme', currentTheme); - if (currentTheme === 'dark') { + if (currentTheme === 'dark' && toggleSwitch !== null) { toggleSwitch.checked = true; } } From 47acc80f4e05053d2653c7218434bc7fad880741 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 9 Jun 2021 17:33:19 +0200 Subject: [PATCH 0085/1654] make strict funcs analysis smarter (#18219) * make strict funcs analysis smarter: varParam[i] = v is different from varParam[i][] = v * added a test case * Update compiler/varpartitions.nim Co-authored-by: Clyybber --- compiler/varpartitions.nim | 87 +++++++++++++++---------- tests/effects/tfuncs_cannot_mutate2.nim | 27 ++++++++ tests/effects/tstrict_funcs.nim | 17 +++++ 3 files changed, 95 insertions(+), 36 deletions(-) create mode 100644 tests/effects/tfuncs_cannot_mutate2.nim diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 810ce8586..9cf1e38ad 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -175,12 +175,18 @@ proc root(v: var Partitions; start: int): int = v.s[it].con = Connection(kind: dependsOn, parent: result) it = next -proc potentialMutation(v: var Partitions; s: PSym; info: TLineInfo) = +proc potentialMutation(v: var Partitions; s: PSym; level: int; info: TLineInfo) = let id = variableId(v, s) if id >= 0: let r = root(v, id) - let flags = if s.kind == skParam and isConstParam(s): - {isMutated, isMutatedDirectly} + let flags = if s.kind == skParam: + if isConstParam(s): + {isMutated, isMutatedDirectly} + elif s.typ.kind == tyVar and level <= 1: + # varParam[i] = v is different from varParam[i][] = v + {} + else: + {isMutated} else: {isMutated} @@ -339,36 +345,44 @@ proc pathExpr(node: PNode; owner: PSym): PNode = if result == nil and borrowFromConstExpr(n): result = n -proc allRoots(n: PNode; result: var seq[PSym]; followDotExpr = true) = +const + RootEscapes = 1000 # in 'p(r)' we don't know what p does to our poor root. + # so we assume a high level of indirections + +proc allRoots(n: PNode; result: var seq[(PSym, int)]; level: int) = case n.kind of nkSym: if n.sym.kind in {skParam, skVar, skTemp, skLet, skResult, skForVar}: - result.add(n.sym) + result.add((n.sym, level)) - of nkDotExpr, nkDerefExpr, nkBracketExpr, nkHiddenDeref, - nkCheckedFieldExpr, nkAddr, nkHiddenAddr: - if followDotExpr: - allRoots(n[0], result, followDotExpr) + of nkDerefExpr, nkHiddenDeref: + allRoots(n[0], result, level+1) + of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr, nkAddr, nkHiddenAddr: + allRoots(n[0], result, level) of nkExprEqExpr, nkExprColonExpr, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkCast, nkObjUpConv, nkObjDownConv: if n.len > 0: - allRoots(n.lastSon, result, followDotExpr) + allRoots(n.lastSon, result, level) of nkCaseStmt, nkObjConstr: for i in 1.. 1: - allRoots(n[1], result, followDotExpr) + # XXX We really need the unwritten RFC here and distinguish between + # proc `[]`(x: var Container): var T # resizes the container + # and + # proc `[]`(x: Container): var T # only allows for slot mutation + allRoots(n[1], result, RootEscapes) else: let m = getMagic(n) case m @@ -387,12 +401,12 @@ proc allRoots(n: PNode; result: var seq[PSym]; followDotExpr = true) = let paramType = typ.n[i].typ if not paramType.isCompileTimeOnly and not typ.sons[0].isEmptyType and canAlias(paramType, typ.sons[0]): - allRoots(it, result, followDotExpr) + allRoots(it, result, RootEscapes) else: - allRoots(it, result, followDotExpr) + allRoots(it, result, RootEscapes) of mSlice: - allRoots(n[1], result, followDotExpr) + allRoots(n[1], result, level+1) else: discard "harmless operation" else: @@ -466,10 +480,10 @@ proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = dest.flags.incl ownsData elif n.typ.kind in {tyLent, tyVar}: # we know the result is derived from the first argument: - var roots: seq[PSym] - allRoots(n[1], roots) + var roots: seq[(PSym, int)] + allRoots(n[1], roots, RootEscapes) for r in roots: - connect(c, dest.sym, r, n[1].info) + connect(c, dest.sym, r[0], n[1].info) else: let magic = if n[0].kind == nkSym: n[0].sym.magic else: mNone @@ -582,19 +596,19 @@ proc deps(c: var Partitions; dest, src: PNode) = if borrowChecking in c.goals: borrowingAsgn(c, dest, src) - var targets, sources: seq[PSym] - allRoots(dest, targets) - allRoots(src, sources) + var targets, sources: seq[(PSym, int)] + allRoots(dest, targets, 0) + allRoots(src, sources, 0) let destIsComplex = containsPointer(dest.typ) for t in targets: if dest.kind != nkSym and c.inNoSideEffectSection == 0: - potentialMutation(c, t, dest.info) + potentialMutation(c, t[0], t[1], dest.info) if destIsComplex: for s in sources: - connect(c, t, s, dest.info) + connect(c, t[0], s[0], dest.info) if cursorInference in c.goals and src.kind != nkEmpty: let d = pathExpr(dest, c.owner) @@ -602,7 +616,8 @@ proc deps(c: var Partitions; dest, src: PNode) = let vid = variableId(c, d.sym) if vid >= 0: destMightOwn(c, c.s[vid], src) - for s in sources: + for source in sources: + let s = source[0] if s == d.sym: discard "assignments like: it = it.next are fine" elif {sfGlobal, sfThread} * s.flags != {} or hasDisabledAsgn(c.g, s.typ): @@ -638,9 +653,9 @@ proc potentialMutationViaArg(c: var Partitions; n: PNode; callee: PType) = if constParameters in c.goals and tfNoSideEffect in callee.flags: discard "we know there are no hidden mutations through an immutable parameter" elif c.inNoSideEffectSection == 0 and containsPointer(n.typ): - var roots: seq[PSym] - allRoots(n, roots) - for r in roots: potentialMutation(c, r, n.info) + var roots: seq[(PSym, int)] + allRoots(n, roots, RootEscapes) + for r in roots: potentialMutation(c, r[0], r[1], n.info) proc traverse(c: var Partitions; n: PNode) = inc c.abstractTime @@ -682,12 +697,12 @@ proc traverse(c: var Partitions; n: PNode) = if i < L: let paramType = parameters[i].skipTypes({tyGenericInst, tyAlias}) if not paramType.isCompileTimeOnly and paramType.kind in {tyVar, tySink, tyOwned}: - var roots: seq[PSym] - allRoots(it, roots) + var roots: seq[(PSym, int)] + allRoots(it, roots, RootEscapes) if paramType.kind == tyVar: if c.inNoSideEffectSection == 0: - for r in roots: potentialMutation(c, r, it.info) - for r in roots: noCursor(c, r) + for r in roots: potentialMutation(c, r[0], r[1], it.info) + for r in roots: noCursor(c, r[0]) if borrowChecking in c.goals: # a call like 'result.add toOpenArray()' can also be a borrow @@ -703,10 +718,10 @@ proc traverse(c: var Partitions; n: PNode) = when false: # XXX investigate if this is required, it doesn't look # like it is! - var roots: seq[PSym] - allRoots(n[0], roots) + var roots: seq[(PSym, int)] + allRoots(n[0], roots, RootEscapes) for r in roots: - potentialMutation(c, r, it.info) + potentialMutation(c, r[0], r[1], it.info) of nkTupleConstr, nkBracket: for child in n: traverse(c, child) diff --git a/tests/effects/tfuncs_cannot_mutate2.nim b/tests/effects/tfuncs_cannot_mutate2.nim new file mode 100644 index 000000000..d5082e57b --- /dev/null +++ b/tests/effects/tfuncs_cannot_mutate2.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "'copy' can have side effects" + nimout: '''an object reachable from 'y' is potentially mutated +tfuncs_cannot_mutate2.nim(15, 7) the mutation is here +tfuncs_cannot_mutate2.nim(13, 10) is the statement that connected the mutation to the parameter +''' +""" + +{.experimental: "strictFuncs".} + +func copy[T](x: var openArray[T]; y: openArray[T]) = + for i in 0..high(x): + x[i] = y[i] + + x[0].a = nil + +type + R = ref object + a, b: R + data: string + +proc main = + var a, b: array[3, R] + b = [R(data: "a"), R(data: "b"), R(data: "c")] + copy a, b + +main() diff --git a/tests/effects/tstrict_funcs.nim b/tests/effects/tstrict_funcs.nim index 044bc7ee1..9d20f5d7e 100644 --- a/tests/effects/tstrict_funcs.nim +++ b/tests/effects/tstrict_funcs.nim @@ -27,3 +27,20 @@ block: var x = @[0, 1] let z = x &&& 2 + + +func copy[T](x: var openArray[T]; y: openArray[T]) = + for i in 0..high(x): + x[i] = y[i] + +type + R = ref object + a, b: R + data: string + +proc main = + var a, b: array[3, R] + b = [R(data: "a"), R(data: "b"), R(data: "c")] + copy a, b + +main() From 19918ceb2b099440cd3e43b6d7fff313de6a84a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 9 Jun 2021 14:46:00 -0700 Subject: [PATCH 0086/1654] update .github/ISSUE_TEMPLATE/bug_report.md, mention PRs, contributing, devel branch (#18224) --- .github/ISSUE_TEMPLATE/bug_report.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 9571896d8..807e50d35 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,6 +7,8 @@ assignees: '' --- +(Consider writing a PR targetting devel branch after filing this, see [contributing.html](https://nim-lang.github.io/Nim/contributing.html)). + Function `echo` outputs the wrong string. ### Example From 79ded694d7f913755dc7261c7468f04fe421e0b3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 9 Jun 2021 22:00:34 -0700 Subject: [PATCH 0087/1654] avoid re-exporting options from std/wrapnils (#18222) --- lib/std/wrapnils.nim | 2 +- tests/stdlib/twrapnils.nim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index 708faf4cf..ed0a79d79 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -25,7 +25,6 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early from options import Option, isSome, get, option, unsafeGet, UnpackDefect -export options.get, options.isSome, options.isNone template fakeDot*(a: Option, b): untyped = ## See top-level example. @@ -90,6 +89,7 @@ macro `?.`*(a: untyped): auto = macro `??.`*(a: untyped): Option = ## Same as `?.` but returns an `Option`. runnableExamples: + import std/options type Foo = ref object x1: ref int x2: int diff --git a/tests/stdlib/twrapnils.nim b/tests/stdlib/twrapnils.nim index af0978762..956204355 100644 --- a/tests/stdlib/twrapnils.nim +++ b/tests/stdlib/twrapnils.nim @@ -1,4 +1,5 @@ import std/wrapnils +from std/options import get, isSome proc checkNotZero(x: float): float = doAssert x != 0 From 0a4858dc59b1464d4057822d60e888fa4403223e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 10 Jun 2021 09:25:51 +0200 Subject: [PATCH 0088/1654] fixes #18220 (#18227) --- lib/pure/hashes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 46b4fc948..1c49b0317 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -509,7 +509,7 @@ proc hashIgnoreCase*(sBuf: string, sPos, ePos: int): Hash = h = h !& ord(c) result = !$h -proc hash*[T: tuple | object | proc](x: T): Hash {.inline.} = +proc hash*[T: tuple | object | proc](x: T): Hash = ## Efficient `hash` overload. runnableExamples: # for `tuple|object`, `hash` must be defined for each component of `x`. From 8a9b20579e4afde6ccd515b9840ab2c68d0fe9e6 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jun 2021 01:27:12 -0700 Subject: [PATCH 0089/1654] update changelog for wrapnils (#18228) --- changelog.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/changelog.md b/changelog.md index f141ea95a..cc6f826ee 100644 --- a/changelog.md +++ b/changelog.md @@ -285,9 +285,6 @@ - Added `std/strbasics` for high performance string operations. Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. - -- Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`. - - `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` and `$none(int)` to `"none(int)"` instead of `"None[int]"`. From 7bf0404dd86ff094ebd3d5b4a39bc4c52b442593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=B6hlich=20A?= Date: Thu, 10 Jun 2021 14:28:00 +0200 Subject: [PATCH 0090/1654] #18216 make moveDir work across partitions on windows (#18223) * return false if AccessDeniedError in tryMoveFSObject - fixes #18216 * add moveDir & moveFile tests * rename `isMoveDir` parameter to `isDir` --- lib/pure/os.nim | 44 ++++++++++++++++++++++++------------------- tests/stdlib/tos.nim | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index c48d0d84f..4dd5c8d8b 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2009,28 +2009,32 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n if not tryRemoveFile(file): raiseOSError(osLastError(), file) -proc tryMoveFSObject(source, dest: string): bool {.noWeirdTarget.} = - ## Moves a file or directory from `source` to `dest`. +proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = + ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. ## - ## Returns false in case of `EXDEV` error. + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on windows (if `isDir` is true). ## In case of other errors `OSError` is raised. ## Returns true in case of success. when defined(windows): when useWinUnicode: let s = newWideCString(source) let d = newWideCString(dest) - if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 else: - if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest)) + result = moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 else: - if c_rename(source, dest) != 0'i32: - let err = osLastError() - if err == EXDEV.OSErrorCode: - return false + result = c_rename(source, dest) == 0'i32 + + if not result: + let err = osLastError() + let isAccessDeniedError = + when defined(windows): + const AccessDeniedError = OSErrorCode(5) + isDir and err == AccessDeniedError else: - # see whether `strerror(errno)` is redundant with what raiseOSError already shows - raiseOSError(err, $(source, dest, strerror(errno))) - return true + err == EXDEV.OSErrorCode + if not isAccessDeniedError: + raiseOSError(err, $(source, dest)) proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = @@ -2051,8 +2055,10 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", ## * `removeFile proc <#removeFile,string>`_ ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): + if not tryMoveFSObject(source, dest, isDir = false): + when defined(windows): + doAssert false + else: # Fallback to copy & del copyFile(source, dest, {cfSymlinkAsIs}) try: @@ -2060,6 +2066,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", except: discard tryRemoveFile(dest) raise + proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. @@ -2614,11 +2621,10 @@ proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWei ## * `removeDir proc <#removeDir,string>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ ## * `createDir proc <#createDir,string>`_ - if not tryMoveFSObject(source, dest): - when not defined(windows): - # Fallback to copy & del - copyDir(source, dest) - removeDir(source) + if not tryMoveFSObject(source, dest, isDir = true): + # Fallback to copy & del + copyDir(source, dest) + removeDir(source) proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## Create a hard link at `dest` which points to the item specified diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index b9769d05c..a4f030dde 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -261,6 +261,51 @@ block fileOperations: removeDir(dname) +block: # moveFile + let tempDir = getTempDir() / "D20210609T151608" + createDir(tempDir) + defer: removeDir(tempDir) + + writeFile(tempDir / "a.txt", "") + moveFile(tempDir / "a.txt", tempDir / "b.txt") + doAssert not fileExists(tempDir / "a.txt") + doAssert fileExists(tempDir / "b.txt") + removeFile(tempDir / "b.txt") + + createDir(tempDir / "moveFile_test") + writeFile(tempDir / "moveFile_test/a.txt", "") + moveFile(tempDir / "moveFile_test/a.txt", tempDir / "moveFile_test/b.txt") + doAssert not fileExists(tempDir / "moveFile_test/a.txt") + doAssert fileExists(tempDir / "moveFile_test/b.txt") + removeDir(tempDir / "moveFile_test") + + createDir(tempDir / "moveFile_test") + writeFile(tempDir / "a.txt", "") + moveFile(tempDir / "a.txt", tempDir / "moveFile_test/b.txt") + doAssert not fileExists(tempDir / "a.txt") + doAssert fileExists(tempDir / "moveFile_test/b.txt") + removeDir(tempDir / "moveFile_test") + +block: # moveDir + let tempDir = getTempDir() / "D20210609T161443" + createDir(tempDir) + defer: removeDir(tempDir) + + createDir(tempDir / "moveDir_test") + moveDir(tempDir / "moveDir_test/", tempDir / "moveDir_test_dest") + doAssert not dirExists(tempDir / "moveDir_test") + doAssert dirExists(tempDir / "moveDir_test_dest") + removeDir(tempDir / "moveDir_test_dest") + + createDir(tempDir / "moveDir_test") + writeFile(tempDir / "moveDir_test/a.txt", "") + moveDir(tempDir / "moveDir_test", tempDir / "moveDir_test_dest") + doAssert not dirExists(tempDir / "moveDir_test") + doAssert not fileExists(tempDir / "moveDir_test/a.txt") + doAssert dirExists(tempDir / "moveDir_test_dest") + doAssert fileExists(tempDir / "moveDir_test_dest/a.txt") + removeDir(tempDir / "moveDir_test_dest") + import times block modificationTime: # Test get/set modification times From 3481ff6172a2e11c990cc67a7fd74a2f0a2445d2 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Thu, 10 Jun 2021 08:29:03 -0400 Subject: [PATCH 0091/1654] Update nims.rst to reflect that -d:release is now allowed in config.nims (#18221) `-d:release` and `-d:danger` started working in `config.nims` after https://github.com/nim-lang/Nim/commit/df429fa28772e077faa30dd6e3a701abf48c7669 . --- doc/nims.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/nims.rst b/doc/nims.rst index 5ae1d7baa..4141c3bdc 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -121,8 +121,8 @@ Here are few examples of using the `switch` proc: .. code-block:: nim # command-line: --opt:size switch("opt", "size") - # command-line: --define:foo or -d:foo - switch("define", "foo") + # command-line: --define:release or -d:release + switch("define", "release") # command-line: --forceBuild switch("forceBuild") @@ -132,13 +132,13 @@ above example can be rewritten as: .. code-block:: nim --opt:size - --define:foo + --define:release --forceBuild **Note**: In general, the *define* switches can also be set in -NimScripts using `switch` or `--`, as shown in above -examples. Only the `release` define (`-d:release`:option:) cannot be set -in NimScripts. +NimScripts using `switch` or `--`, as shown in above examples. Few +`define` switches such as `-d:strip`:option:, `-d:lto`:option: and +`-d:lto_incremental`:option: cannot be set in NimScripts. NimScript as a build tool From 2ea7287217ffe3ad6c3790ff1233845de2aedcf4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 10 Jun 2021 16:49:17 +0200 Subject: [PATCH 0092/1654] view types: spec changes (#18226) * view types: spec changes * Update doc/manual_experimental.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/manual_experimental.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/varpartitions.nim | 5 +++-- doc/manual_experimental.rst | 31 ++++++++++++++----------------- tests/views/tdont_mutate.nim | 9 ++++----- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 9cf1e38ad..04c7c629d 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -572,7 +572,7 @@ proc borrowingAsgn(c: var Partitions; dest, src: PNode) = borrowFrom(c, dest.sym, src) elif dest.kind in {nkHiddenDeref, nkDerefExpr, nkBracketExpr}: case directViewType(dest[0].typ) - of mutableView: + of mutableView, immutableView: # we do not borrow, but we use the view to mutate the borrowed # location: let viewOrigin = pathExpr(dest, c.owner) @@ -580,12 +580,13 @@ proc borrowingAsgn(c: var Partitions; dest, src: PNode) = let vid = variableId(c, viewOrigin.sym) if vid >= 0: c.s[vid].flags.incl viewDoesMutate - of immutableView: + #[of immutableView: if dest.kind == nkBracketExpr and dest[0].kind == nkHiddenDeref and mutableParameter(dest[0][0]): discard "remains a mutable location anyhow" else: localError(c.g.config, dest.info, "attempt to mutate a borrowed location from an immutable view") + ]# of noView: discard "nothing to do" proc containsPointer(t: PType): bool = diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 2c1043ecb..3157708da 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -475,8 +475,8 @@ Not nil annotation **Note:** This is an experimental feature. It can be enabled with `{.experimental: "notnil"}`. -All types for which `nil` is a valid value can be annotated with the `not -nil` annotation to exclude `nil` as a valid value: +All types for which `nil` is a valid value can be annotated with the +`not nil` annotation to exclude `nil` as a valid value: .. code-block:: nim {.experimental: "notnil"} @@ -1837,8 +1837,7 @@ with `--experimental:strictFuncs`:option:. A view type is a type that is or contains one of the following types: -- `var T` (mutable view into `T`) -- `lent T` (immutable view into `T`) +- `lent T` (view into `T`) - `openArray[T]` (pair of (pointer to array of `T`, size)) For example: @@ -1846,10 +1845,9 @@ For example: .. code-block:: nim type - View1 = var int - View2 = openArray[byte] - View3 = lent string - View4 = Table[openArray[char], int] + View1 = openArray[byte] + View2 = lent string + View3 = Table[openArray[char], int] Exceptions to this rule are types constructed via `ptr` or `proc`. @@ -1860,11 +1858,11 @@ For example, the following types are **not** view types: type NotView1 = proc (x: openArray[int]) NotView2 = ptr openArray[char] - NotView3 = ptr array[4, var int] + NotView3 = ptr array[4, lent int] -A *mutable* view type is a type that is or contains a `var T` type. -An *immutable* view type is a view type that is not a mutable view type. +The mutability aspect of a view type is not part of the type but part +of the locations it's derived from. More on this later. A *view* is a symbol (a let, var, const, etc.) that has a view type. @@ -1948,11 +1946,14 @@ details about how this is done for `var T`. A mutable view can borrow from a mutable location, an immutable view can borrow from both a mutable or an immutable location. +If a view borrows from a mutable location, the view can be used to update the +location. Otherwise it cannot be used for mutations. + The *duration* of a borrow is the span of commands beginning from the assignment to the view and ending with the last usage of the view. For the duration of the borrow operation, no mutations to the borrowed locations -may be performed except via the potentially mutable view that borrowed from the +may be performed except via the view that borrowed from the location. The borrowed location is said to be *sealed* during the borrow. .. code-block:: nim @@ -2064,11 +2065,7 @@ and `b` the location that is borrowed from. - The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of a parameter is the complete proc body. -- If `v` is a mutable view and `v` is used to actually mutate the - borrowed location, then `b` has to be a mutable location. - Note: If it is not actually used for mutation, borrowing a mutable view from an - immutable location is allowed! This allows for many important idioms and will be - justified in an upcoming RFC. +- If `v` is used for a mutation, `b` must be a mutable location too. - During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if `v` is a mutable view). - If `v` is `result` then `b` has to be a location derived from the first diff --git a/tests/views/tdont_mutate.nim b/tests/views/tdont_mutate.nim index d243c7c7a..43acaaf71 100644 --- a/tests/views/tdont_mutate.nim +++ b/tests/views/tdont_mutate.nim @@ -9,8 +9,9 @@ import tables const Whitespace = {' ', '\t', '\n', '\r'} -proc split*(s: string, seps: set[char] = Whitespace, - maxsplit: int = -1): Table[int, openArray[char]] = +proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): Table[int, openArray[char]] #[tt.Error + 'result' borrows from the immutable location 's' and attempts to mutate it + ]# = var last = 0 var splits = maxsplit result = initTable[int, openArray[char]]() @@ -22,9 +23,7 @@ proc split*(s: string, seps: set[char] = Whitespace, if splits == 0: last = len(s) result[first] = toOpenArray(s, first, last-1) - result[first][0] = 'c' #[tt.Error - attempt to mutate a borrowed location from an immutable view - ]# + result[first][0] = 'c' if splits == 0: break dec(splits) From f65f760dee2d4e19be826c256be50d34f8873d48 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 10 Jun 2021 18:19:20 +0200 Subject: [PATCH 0093/1654] fixes #15884 (#18230) * fixes #15884 * micro optimization --- compiler/renderer.nim | 2 ++ compiler/sem.nim | 9 +++++++++ compiler/semexprs.nim | 2 +- compiler/semstmts.nim | 2 +- compiler/varpartitions.nim | 6 +++++- 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 8bc3c3306..ac3f47948 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1238,6 +1238,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = putWithSpace(g, tkBind, "bind") gsub(g, n, 0) of nkCheckedFieldExpr, nkHiddenAddr, nkHiddenDeref, nkStringToCString, nkCStringToString: + if renderIds in g.flags: + putWithSpace(g, tkAddr, $n.kind) gsub(g, n, 0) of nkLambda: putWithSpace(g, tkProc, "proc") diff --git a/compiler/sem.nim b/compiler/sem.nim index 72b8dfe10..879f6efc9 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -100,6 +100,15 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = else: result = fitNodePostMatch(c, formal, result) +proc fitNodeForLocalVar(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = + let a = fitNode(c, formal, arg, info) + if formal.kind in {tyVar, tyLent}: + #classifyViewType(formal) != noView: + result = newNodeIT(nkHiddenAddr, a.info, formal) + result.add a + else: + result = a + proc inferWithMetatype(c: PContext, formal: PType, arg: PNode, coerceDistincts = false): PNode diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c839d04d9..ba8b93ae6 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1643,7 +1643,7 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if le.kind == nkHiddenDeref: var x = le[0] - if (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView) and x.kind == nkSym and x.sym.kind == skResult: + if x.kind == nkSym and x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): n[0] = x # 'result[]' --> 'result' n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) x.typ.flags.incl tfVarIsPtr diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index a2401b376..8f87ce83c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -526,7 +526,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = else: # BUGFIX: ``fitNode`` is needed here! # check type compatibility between def.typ and typ - def = fitNode(c, typ, def, def.info) + def = fitNodeForLocalVar(c, typ, def, def.info) #changeType(def.skipConv, typ, check=true) else: typ = def.typ.skipTypes({tyStatic, tySink}).skipIntLit(c.idgen) diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 04c7c629d..697dd2b1d 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -519,13 +519,17 @@ const proc isConstSym(s: PSym): bool = result = s.kind in {skConst, skLet} or isConstParam(s) +proc toString(n: PNode): string = + if n.kind == nkEmpty: result = "" + else: result = $n + proc borrowFrom(c: var Partitions; dest: PSym; src: PNode) = const url = "see https://nim-lang.github.io/Nim/manual_experimental.html#view-types-algorithm-path-expressions for details" let s = pathExpr(src, c.owner) if s == nil: - localError(c.g.config, src.info, "cannot borrow from " & $src & ", it is not a path expression; " & url) + localError(c.g.config, src.info, "cannot borrow from " & src.toString & ", it is not a path expression; " & url) elif s.kind == nkSym: if dest.kind == skResult: if s.sym.kind != skParam or s.sym.position != 0: From 6b97889f44d06f6649b6547a4ffdb18b94d6e578 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 11 Jun 2021 00:26:58 +0800 Subject: [PATCH 0094/1654] fix #9437(fix `re.replace` wrong behaviour) (#17546) * fix nim js cmp fails at CT * fix --- lib/impure/re.nim | 20 ++++++++++++++++++-- tests/stdlib/tre.nim | 9 ++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 40860cc88..78ea7b002 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -199,6 +199,18 @@ proc findBounds*(s: string, pattern: Regex, ## `(-1,0)` is returned. result = findBounds(cstring(s), pattern, matches, start, s.len) +proc findBoundsImpl(buf: cstring, pattern: Regex, + start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = + var rtarray = initRtArray[cint](3) + let rawMatches = rtarray.getRawData + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32, + cast[ptr cint](rawMatches), 3) + + if res < 0'i32: + result = (-1, 0) + else: + result = (int(rawMatches[0]), int(rawMatches[1]-1)) + proc findBounds*(buf: cstring, pattern: Regex, start = 0, bufSize: int): tuple[first, last: int] = ## returns the `first` and `last` position of `pattern` in `buf`, @@ -433,12 +445,16 @@ proc replace*(s: string, sub: Regex, by = ""): string = doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?" result = "" var prev = 0 + var flags = int32(0) while prev < s.len: - var match = findBounds(s, sub, prev) + var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags) + flags = 0 if match.first < 0: break add(result, substr(s, prev, match.first-1)) add(result, by) - if match.last + 1 == prev: break + if match.first > match.last: + # 0-len match + flags = pcre.NOTEMPTY_ATSTART prev = match.last + 1 add(result, substr(s, prev)) diff --git a/tests/stdlib/tre.nim b/tests/stdlib/tre.nim index ea1b5af32..2857c6c9e 100644 --- a/tests/stdlib/tre.nim +++ b/tests/stdlib/tre.nim @@ -99,10 +99,13 @@ proc testAll() = accum.add($x) doAssert(accum == @["a","b","c"]) - block: - # bug #9306 + block: # bug #9306 doAssert replace("bar", re"^", "foo") == "foobar" - doAssert replace("foo", re"", "-") == "-foo" doAssert replace("foo", re"$", "bar") == "foobar" + + block: # bug #9437 + doAssert replace("foo", re"", "-") == "-f-o-o-" + doAssert replace("ooo", re"o", "-") == "---" + testAll() From c64d9176190b3b691391f1645873dd4ca5b09933 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 10 Jun 2021 23:36:06 -0700 Subject: [PATCH 0095/1654] handle tyUserTypeClassInst in addDeclaredLoc (#18236) --- compiler/astmsgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index 5be8dc38f..924f0ddea 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -13,7 +13,7 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = # xxx figure out how to resolve `tyGenericParam`, e.g. for # proc fn[T](a: T, b: T) = discard # fn(1.1, "a") - let typ = typ.skipTypes(abstractInst + {tyStatic} - {tyRange}) + let typ = typ.skipTypes(abstractInst + {tyStatic, tyUserTypeClassInst} - {tyRange}) result.add " [$1" % typ.kind.toHumanStr if typ.sym != nil: result.add " declared in " & toFileLineCol(conf, typ.sym.info) From 57609902c4438da2bcc156296128591a9906e922 Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 11 Jun 2021 10:07:23 +0200 Subject: [PATCH 0096/1654] use more meaningful name than "workaround14447" (#18237) --- compiler/lambdalifting.nim | 2 +- lib/system/iterators.nim | 2 +- testament/important_packages.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 6f43649a1..c555cedfe 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -292,7 +292,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) = if illegalCapture(s): localError(g.config, n.info, ("'$1' is of type <$2> which cannot be captured as it would violate memory" & - " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") % + " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases") % [s.name.s, typeToString(s.typ), g.config$s.info]) elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags): localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" % diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index 0f79970b8..f23f7aa11 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -1,4 +1,4 @@ -when defined(nimHasLentIterators) and not defined(nimWorkaround14447): +when defined(nimHasLentIterators) and not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: template lent2(T): untyped = T diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 975dd63ad..3d3c340eb 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -156,5 +156,5 @@ pkg "winim", allowFailure = true pkg "with" pkg "ws" pkg "yaml", "nim build" -pkg "zero_functional", "nim c -r -d:nimWorkaround14447 test.nim" +pkg "zero_functional", "nim c -r -d:nimNoLentIterators test.nim" pkg "zippy" From 2ca6169ba10dca704712b59339a24208e30e79ac Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jun 2021 11:09:00 +0200 Subject: [PATCH 0097/1654] added a test case ensuring exception inference continues to work --- tests/exception/texception_inference.nim | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/exception/texception_inference.nim diff --git a/tests/exception/texception_inference.nim b/tests/exception/texception_inference.nim new file mode 100644 index 000000000..7dd01cca1 --- /dev/null +++ b/tests/exception/texception_inference.nim @@ -0,0 +1,32 @@ +discard """ + output: '''good''' + cmd: "nim c --gc:orc -d:release $file" +""" + +type + Raising[T, E] = object + +proc foo[T, Errors](x: proc (x: Raising[T, Errors])) {.raises: Errors.} = + discard + +proc callback(x: Raising[int, ValueError]) = + echo "callback" + +proc xy() {.raises: [ValueError].} = + foo callback + +proc x[E]() {.raises: [E, IOError].} = + raise newException(E, "text here") + +try: + x[ValueError]() +except ValueError: + echo "good" + +proc callback2(x: Raising[int, IOError]) = + discard + +proc foo2[T, OtherErrors](x: proc(x: Raising[T, OtherErrors])) {.raises: [ValueError, OtherErrors].} = + discard + +foo2 callback2 From 897e50d5fe274850d4772612a90b325e33cca8a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 12 Jun 2021 12:35:12 -0700 Subject: [PATCH 0098/1654] getType now works with tyInferred (arising from concepts); refs #18220 (#18241) * getType now works with tyInferred (concepts); refs #18220 * avoid cast * add more docs --- compiler/vmdeps.nim | 2 +- tests/macros/tgettype.nim | 100 +++++++++++++++++++++++++++++--------- 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index b67e2e48a..1afda14b0 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -289,7 +289,7 @@ proc mapTypeToAstX(cache: IdentCache; t: PType; info: TLineInfo; of tyNot: result = mapTypeToBracket("not", mNot, t, info) of tyIterable: result = mapTypeToBracket("iterable", mIterableType, t, info) of tyAnything: result = atomicType("anything", mNone) - of tyInferred: assert false + of tyInferred: result = mapTypeToAstX(cache, t.lastSon, info, idgen, inst, allowRecursion) of tyStatic, tyFromExpr: if inst: if t.n != nil: result = t.n.copyTree diff --git a/tests/macros/tgettype.nim b/tests/macros/tgettype.nim index bd70a1c30..d91efe1fe 100644 --- a/tests/macros/tgettype.nim +++ b/tests/macros/tgettype.nim @@ -1,29 +1,85 @@ -discard """ -output: ''' -(ObjectTy (Empty) (Sym "Model") (RecList (Sym "name") (Sym "password"))) -(BracketExpr (Sym "typeDesc") (Sym "User")) -''' -""" -import macros +import std/macros +import stdtest/testutils -type - Model = object of RootObj - User = object of Model - name : string - password : string +# getType -macro testUser: string = - result = newLit(User.getType.lispRepr) +block: + type + Model = object of RootObj + User = object of Model + name : string + password : string -macro testGeneric(T: typedesc[Model]): string= - result = newLit(T.getType.lispRepr) + macro testUser: string = + result = newLit(User.getType.lispRepr) -echo testUser -echo User.testGeneric + macro testGeneric(T: typedesc[Model]): string= + result = newLit(T.getType.lispRepr) -macro assertVoid(e: typed): untyped = - assert(getTypeInst(e).typeKind == ntyVoid) + doAssert testUser == """(ObjectTy (Empty) (Sym "Model") (RecList (Sym "name") (Sym "password")))""" + doAssert User.testGeneric == """(BracketExpr (Sym "typeDesc") (Sym "User"))""" -proc voidProc() = discard + macro assertVoid(e: typed): untyped = + assert(getTypeInst(e).typeKind == ntyVoid) -assertVoid voidProc() + proc voidProc() = discard + + assertVoid voidProc() + +block: + # refs #18220; not an actual solution (yet) but at least shows what's currently + # possible + + type Callable1[R, T, U] = concept fn + fn(default(T)) is R + fn is U + + # note that typetraits.arity doesn't work + macro arity(a: typed): int = + # number of params + # this is not production code! + let a2 = a.getType[1] # this used to crash nim, with: `vmdeps.nim(292, 25) `false`` + newLit a2.len - 1 + + type Callable2[R, T, U] = concept fn + fn(default(T)) is R + fn is U + arity(U) == 2 + + proc map1[T, R, U](a: T, fn: Callable1[R, T, U]): R = + let fn = U(fn) + # `cast[U](fn)` would also work; + # this is currently needed otherwise, sigmatch errors with: + # Error: attempting to call routine: 'fn' + # found 'fn' [param declared in tgettype.nim(53, 28)] + # this can be fixed in future work + fn(a) + + proc map2[T, R, U](a: T, fn: Callable2[R, T, U]): R = + let fn = U(fn) + fn(a) + + proc fn1(a: int, a2 = 'x'): string = $(a, a2, "fn1") + proc fn2(a: int, a2 = "zoo"): string = $(a, a2, "fn2") + proc fn3(a: int, a2 = "zoo2"): string = $(a, a2, "fn3") + proc fn4(a: int): string {.inline.} = $(a, "fn4") + proc fn5(a: int): string = $(a, "fn5") + + assertAll: + # Callable1 + 1.map1(fn1) == """(1, 'x', "fn1")""" + 1.map1(fn2) == """(1, "zoo", "fn2")""" + 1.map1(fn3) == """(1, "zoo", "fn3")""" + # fn3's optional param is not honored, because fn3 and fn2 yield same + # generic instantiation; this is a caveat with this approach + # There are several possible ways to improve things in future work. + 1.map1(fn4) == """(1, "fn4")""" + 1.map1(fn5) == """(1, "fn5")""" + + # Callable2; prevents passing procs with optional params to avoid above + # mentioned caveat, but more restrictive + not compiles(1.map2(fn1)) + not compiles(1.map2(fn2)) + not compiles(1.map2(fn3)) + 1.map2(fn4) == """(1, "fn4")""" + 1.map2(fn5) == """(1, "fn5")""" From c871e22da2ad8f9caf82fdba43fccb7230d726e1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 12 Jun 2021 22:32:47 -0700 Subject: [PATCH 0099/1654] fix #7717 roundtrip float to string; fix `parseFloat` for js (#18248) * refs #7717 roundtrip float to string * make parseFloat more correct * improve float tests * improve float tests * cleanup --- lib/system/jssys.nim | 99 +++++++++++++++++++---------------------- tests/float/tfloat6.nim | 28 ------------ tests/float/tfloats.nim | 84 ++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 81 deletions(-) delete mode 100644 tests/float/tfloat6.nim create mode 100644 tests/float/tfloats.nim diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 371cb7962..7cb5652c5 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -716,19 +716,23 @@ proc tenToThePowerOf(b: int): BiggestFloat = const IdentChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'} -# XXX use JS's native way here -proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int {. - compilerproc.} = - var - esign = 1.0 - sign = 1.0 - i = start - exponent: int - flags: int - number = 0.0 + +proc parseFloatNative(a: string): float = + let a2 = a.cstring + asm """ + `result` = Number(`a2`); + """ + +#[ +xxx how come code like this doesn't give IndexDefect ? +let z = s[10000] == 'a' +]# +proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start: int): int {.compilerproc.} = + var sign: bool + var i = start if s[i] == '+': inc(i) elif s[i] == '-': - sign = -1.0 + sign = true inc(i) if s[i] == 'N' or s[i] == 'n': if s[i+1] == 'A' or s[i+1] == 'a': @@ -741,51 +745,40 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int { if s[i+1] == 'N' or s[i+1] == 'n': if s[i+2] == 'F' or s[i+2] == 'f': if s[i+3] notin IdentChars: - number = Inf*sign + number = if sign: -Inf else: Inf return i+3 - start return 0 - while s[i] in {'0'..'9'}: - # Read integer part - flags = flags or 1 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - inc(i) - while s[i] == '_': inc(i) - # Decimal? - if s[i] == '.': - var hd = 1.0 - inc(i) - while s[i] in {'0'..'9'}: - # Read fractional part - flags = flags or 2 - number = number * 10.0 + toFloat(ord(s[i]) - ord('0')) - hd = hd * 10.0 - inc(i) - while s[i] == '_': inc(i) - number = number / hd # this complicated way preserves precision - # Again, read integer and fractional part - if flags == 0: return 0 - # Exponent? - if s[i] in {'e', 'E'}: - inc(i) - if s[i] == '+': - inc(i) - elif s[i] == '-': - esign = -1.0 - inc(i) - if s[i] notin {'0'..'9'}: - return 0 - while s[i] in {'0'..'9'}: - exponent = exponent * 10 + ord(s[i]) - ord('0') - inc(i) - while s[i] == '_': inc(i) - # Calculate Exponent - let hd = tenToThePowerOf(exponent) - if esign > 0.0: number = number * hd - else: number = number / hd - # evaluate sign - number = number * sign - result = i - start + var buf: string + # we could also use an `array[char, N]` buffer to avoid reallocs, or + # use a 2-pass algorithm that first computes the length. + if sign: buf.add '-' + template addInc = + buf.add s[i] + inc(i) + template eatUnderscores = + while s[i] == '_': inc(i) + while s[i] in {'0'..'9'}: # Read integer part + buf.add s[i] + inc(i) + eatUnderscores() + if s[i] == '.': # Decimal? + addInc() + while s[i] in {'0'..'9'}: # Read fractional part + addInc() + eatUnderscores() + # Again, read integer and fractional part + if buf.len == ord(sign): return 0 + if s[i] in {'e', 'E'}: # Exponent? + addInc() + if s[i] == '+': inc(i) + elif s[i] == '-': addInc() + if s[i] notin {'0'..'9'}: return 0 + while s[i] in {'0'..'9'}: + addInc() + eatUnderscores() + number = parseFloatNative(buf) + result = i - start # Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce # 'Math.trunc' for Nim's ``div`` and ``mod`` operators: diff --git a/tests/float/tfloat6.nim b/tests/float/tfloat6.nim deleted file mode 100644 index fa8fc9da8..000000000 --- a/tests/float/tfloat6.nim +++ /dev/null @@ -1,28 +0,0 @@ -discard """ - output: ''' -0.000001 : 0.000001 -0.000001 : 0.000001 -0.001 : 0.001 -0.000001 : 0.000001 -0.000001 : 0.000001 -10.000001 : 10.000001 -100.000001 : 100.000001 -''' - disabled: "windows" -""" - -import strutils - -echo "0.00_0001".parseFloat(), " : ", 1E-6 -echo "0.00__00_01".parseFloat(), " : ", 1E-6 -echo "0.0_01".parseFloat(), " : ", 0.001 -echo "0.00_000_1".parseFloat(), " : ", 1E-6 -echo "0.00000_1".parseFloat(), " : ", 1E-6 - -echo "1_0.00_0001".parseFloat(), " : ", 10.000001 -echo "1__00.00_0001".parseFloat(), " : ", 1_00.000001 - -# bug #18148 - -var a = 1.1'f32 -doAssert $a == "1.1", $a # fails diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim new file mode 100644 index 000000000..406ddb6d9 --- /dev/null +++ b/tests/float/tfloats.nim @@ -0,0 +1,84 @@ +discard """ + targets: "c cpp js" +""" +# disabled: "windows" + +#[ +xxx merge all or most float tests into this file +]# + +import std/[fenv, math, strutils] + +proc equalsOrNaNs(a, b: float): bool = + if isNaN(a): isNaN(b) + elif a == 0: + b == 0 and signbit(a) == signbit(b) + else: + a == b + +template reject(a) = + doAssertRaises(ValueError): discard parseFloat(a) + +template main = + block: + proc test(a: string, b: float) = + let a2 = a.parseFloat + doAssert equalsOrNaNs(a2, b), $(a, a2, b) + test "0.00_0001", 1E-6 + test "0.00__00_01", 1E-6 + test "0.0_01", 0.001 + test "0.00_000_1", 1E-6 + test "0.00000_1", 1E-6 + test "1_0.00_0001", 10.000001 + test "1__00.00_0001", 1_00.000001 + test "inf", Inf + test "-inf", -Inf + test "-Inf", -Inf + test "-INF", -Inf + test "NaN", NaN + test "-nan", NaN + test ".1", 0.1 + test "-.1", -0.1 + test "-0", -0.0 + when false: # pending bug #18246 + test "-0", -0.0 + test ".1e-1", 0.1e-1 + test "0_1_2_3.0_1_2_3E+0_1_2", 123.0123e12 + test "0_1_2.e-0", 12e0 + test "0_1_2e-0", 12e0 + test "-0e0", -0.0 + test "-0e-0", -0.0 + + reject "a" + reject "" + reject "e1" + reject "infa" + reject "infe1" + reject "_" + reject "1e" + + when false: # gray area; these numbers should probably be invalid + reject "1_" + reject "1_.0" + reject "1.0_" + + block: # bug #18148 + var a = 1.1'f32 + doAssert $a == "1.1", $a # was failing + + block: # bug #7717 + proc test(f: float) = + let f2 = $f + let f3 = parseFloat(f2) + doAssert equalsOrNaNs(f, f3), $(f, f2, f3) + + test 1.0 + epsilon(float64) + test 1000000.0000000123 + test log2(100000.0) + test maximumPositiveValue(float32) + test maximumPositiveValue(float64) + test minimumPositiveValue(float32) + test minimumPositiveValue(float64) + +static: main() +main() From a266c549212d1d6e09dbfa01344edbca8b2f6222 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 13 Jun 2021 16:50:04 +0100 Subject: [PATCH 0100/1654] Improve httpClient docs on SSL cert verification (#15201) * Improve httpClient docs on SSL cert verification Cert verification is enabled by default after CVE-2021-29495 * Update httpclient.nim Co-authored-by: Dominik Picheta --- lib/pure/httpclient.nim | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 0dbf8a045..2f9f1913f 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -115,7 +115,7 @@ ## ## SSL/TLS support ## =============== -## This requires the OpenSSL library, fortunately it's widely used and installed +## This requires the OpenSSL library. Fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the `https` schema, for example: ## `https://github.com/`. @@ -123,12 +123,25 @@ ## You will also have to compile with `ssl` defined like so: ## `nim c -d:ssl ...`. ## -## Certificate validation is NOT performed by default. -## This will change in the future. +## Certificate validation is performed by default. ## ## A set of directories and files from the `ssl_certs `_ ## module are scanned to locate CA certificates. ## +## Example of setting SSL verification parameters in a new client: +## +## .. code-block:: Nim +## import httpclient +## var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeer)) +## +## There are three options for verify mode: +## +## * ``CVerifyNone``: certificates are not verified; +## * ``CVerifyPeer``: certificates are verified; +## * ``CVerifyPeerUseEnvVars``: certificates are verified and the optional +## environment variables SSL_CERT_FILE and SSL_CERT_DIR are also used to +## locate certificates +## ## See `newContext `_ to tweak or disable certificate validation. ## ## Timeouts From 065243dc5988fbfcbed8236e19a588b98d63b199 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 13 Jun 2021 23:21:18 -0700 Subject: [PATCH 0101/1654] followup #17777: declaredloc field error msgs now work with generics (#18259) * followup #17777: declaredloc field error msgs now work with generics * fix tests * cleanup --- compiler/astmsgs.nim | 5 ++++ compiler/semcall.nim | 5 ++-- tests/errmsgs/tundeclared_field.nim | 40 ++++++++++++++++++----------- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index 924f0ddea..90bdce7cf 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -2,6 +2,11 @@ import std/strutils import options, ast, msgs +proc typSym*(t: PType): PSym = + result = t.sym + if result == nil and t.kind == tyGenericInst: # this might need to be refined + result = t[0].sym + proc addDeclaredLoc*(result: var string, conf: ConfigRef; sym: PSym) = result.add " [$1 declared in $2]" % [sym.kind.toHumanStr, toFileLineCol(conf, sym.info)] diff --git a/compiler/semcall.nim b/compiler/semcall.nim index facaa89f7..b56a1dbac 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -326,7 +326,7 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string = let ident = considerQuotedIdent(c, f, n).s if {nfDotField, nfExplicitCall} * n.flags == {nfDotField}: - let sym = n[1].typ.sym + let sym = n[1].typ.typSym var typeHint = "" if sym == nil: # Perhaps we're in a `compiles(foo.bar)` expression, or @@ -337,7 +337,8 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string = discard else: typeHint = " for type " & getProcHeader(c.config, sym) - result = errUndeclaredField % ident & typeHint & " " & result + let suffix = if result.len > 0: " " & result else: "" + result = errUndeclaredField % ident & typeHint & suffix else: if result.len == 0: result = errUndeclaredRoutine % ident else: result = errBadRoutine % [ident, result] diff --git a/tests/errmsgs/tundeclared_field.nim b/tests/errmsgs/tundeclared_field.nim index 2b274ae58..5668050e0 100644 --- a/tests/errmsgs/tundeclared_field.nim +++ b/tests/errmsgs/tundeclared_field.nim @@ -2,38 +2,48 @@ discard """ cmd: '''nim check --hints:off $file''' action: reject nimout: ''' -tundeclared_field.nim(25, 12) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(22, 8)] -tundeclared_field.nim(30, 16) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(28, 8)] -tundeclared_field.nim(36, 4) Error: undeclared field: 'bad' for type tundeclared_field.A [type declared in tundeclared_field.nim(33, 8)] -tundeclared_field.nim(40, 13) Error: cannot instantiate Foo [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(25, 12) Error: undeclared field: 'bad1' for type tundeclared_field.A [type declared in tundeclared_field.nim(22, 8)] +tundeclared_field.nim(30, 17) Error: undeclared field: 'bad2' for type tundeclared_field.A [type declared in tundeclared_field.nim(28, 8)] +tundeclared_field.nim(36, 4) Error: undeclared field: 'bad3' for type tundeclared_field.A [type declared in tundeclared_field.nim(33, 8)] +tundeclared_field.nim(42, 12) Error: undeclared field: 'bad4' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(43, 4) Error: undeclared field: 'bad5' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(44, 23) Error: undeclared field: 'bad6' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(46, 19) Error: undeclared field: 'bad7' for type tundeclared_field.B [type declared in tundeclared_field.nim(39, 8)] +tundeclared_field.nim(50, 13) Error: cannot instantiate Foo [type declared in tundeclared_field.nim(49, 8)] ''' """ - - - - - - - - +#[ +xxx in future work, generic instantiations (e.g. `B[int]`) should be shown with their instantiation instead of `tundeclared_field.B`, +maybe using TPreferedDesc.preferResolved or preferMixed +]# # line 20 block: type A = object a0: int var a: A - discard a.bad + discard a.bad1 block: type A = object a0: int - var a = A(bad: 0) + var a = A(bad2: 0) block: type A = object a0: int var a: A - a.bad = 0 + a.bad3 = 0 + +block: + type B[T] = object + b0: int + var b: B[int] + discard b.bad4 + b.bad5 = 0 + var b2 = B[int](bad6: 0) + type Bi = B[int] + var b3 = Bi(bad7: 0) block: type Foo[T: SomeInteger] = object From e1e8af535ec195e5749dffa728add95770e5cbd7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 13 Jun 2021 23:51:40 -0700 Subject: [PATCH 0102/1654] merge BuildMode into SuccessX, remove code duplication w drnim, add useful info to successx, add gc to compilesettings (#18252) * merge BuildMode into SuccessX, add more info * refactor duplicated with drnim * fixup * address comment --- compiler/lineinfos.nim | 7 +++--- compiler/main.nim | 40 ++++++---------------------------- compiler/msgs.nim | 43 +++++++++++++++++++++++++++++++++---- compiler/options.nim | 13 +++++++++-- compiler/vmops.nim | 1 + doc/nimc.rst | 1 - drnim/drnim.nim | 19 +--------------- lib/std/compilesettings.nim | 1 + testament/testament.nim | 3 +-- tests/misc/trunner.nim | 2 +- 10 files changed, 64 insertions(+), 66 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index bf81aed02..da0a8d623 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -65,7 +65,7 @@ type warnFileChanged = "FileChanged", warnUser = "User", - hintSuccess = "Success", hintSuccessX = "SuccessX", hintBuildMode = "BuildMode", + hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", @@ -145,8 +145,7 @@ const warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` - hintSuccessX: "$loc lines; ${sec}s; $mem; proj: $project; out: $output", - hintBuildMode: "$1", + hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output", hintCC: "CC: $1", hintLineTooLong: "line too long", hintXDeclaredButNotUsed: "'$1' is declared but not used", @@ -200,7 +199,7 @@ proc computeNotesVerbosity(): array[0..3, TNoteKinds] = result[1] = result[2] - {warnProveField, warnProveIndex, warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd, hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance} - result[0] = result[1] - {hintSuccessX, hintBuildMode, hintSuccess, hintConf, + result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf, hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC} const diff --git a/compiler/main.nim b/compiler/main.nim index 71e549ab8..7824e8cb2 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -13,13 +13,14 @@ when not defined(nimcore): {.error: "nimcore MUST be defined for Nim's core tooling".} import - llstream, strutils, os, ast, lexer, syntaxes, options, msgs, - condsyms, times, + std/[strutils, os, times, tables, sha1, with, json], + llstream, ast, lexer, syntaxes, options, msgs, + condsyms, sem, idents, passes, extccomp, - cgen, json, nversion, + cgen, nversion, platform, nimconf, passaux, depends, vm, modules, - modulegraphs, tables, lineinfos, pathutils, vmprofiler, std/[sha1, with] + modulegraphs, lineinfos, pathutils, vmprofiler import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer @@ -387,38 +388,9 @@ proc mainCommand*(graph: ModuleGraph) = rawMessage(conf, errGenerated, "invalid command: " & conf.command) if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}: - # D20210419T170230:here - let mem = - when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" - else: formatSize(getTotalMem()) & " totmem" - let loc = $conf.linesCompiled - let build = if isDefined(conf, "danger"): "Dangerous Release build" - elif isDefined(conf, "release"): "Release build" - else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster." - let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) - let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName - # xxx honor conf.filenameOption more accurately - var output: string - if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript: - output = $conf.jsonBuildFile - elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends: - # for some cmd we expect a valid absOutFile - output = "unknownOutput" - else: - output = $conf.absOutFile - if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename - # xxx honor filenameOption more accurately if optProfileVM in conf.globalOptions: echo conf.dump(conf.vmProfileData) - rawMessage(conf, hintSuccessX, [ - "loc", loc, - "sec", sec, - "mem", mem, - "project", project, - "output", output, - ]) - if conf.cmd in cmdBackends: - rawMessage(conf, hintBuildMode, build) + genSuccessX(conf) when PrintRopeCacheStats: echo "rope cache stats: " diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 053b5c928..6b005be3d 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -8,10 +8,9 @@ # import - options, strutils, os, tables, ropes, terminal, macros, - lineinfos, pathutils -import std/private/miscdollars -import strutils2 + std/[strutils, os, tables, terminal, macros, times], + std/private/miscdollars, + options, ropes, lineinfos, pathutils, strutils2 type InstantiationInfo* = typeof(instantiationInfo()) template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true) @@ -656,3 +655,39 @@ proc uniqueModuleName*(conf: ConfigRef; fid: FileIndex): string = # We mangle upper letters and digits too so that there cannot # be clashes with our special meanings of 'Z' and 'O' result.addInt ord(c) + +proc genSuccessX*(conf: ConfigRef) = + let mem = + when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" + else: formatSize(getTotalMem()) & " totmem" + let loc = $conf.linesCompiled + var build = "" + if conf.cmd in cmdBackends: + build.add "gc: $#; " % $conf.selectedGC + if optThreads in conf.globalOptions: build.add "threads: on; " + build.add "opt: " + if optOptimizeSpeed in conf.options: build.add "speed" + elif optOptimizeSize in conf.options: build.add "size" + else: build.add "none (DEBUG BUILD, `-d:release` generates faster code)" + # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html + let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) + let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName + # xxx honor conf.filenameOption more accurately + var output: string + if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript: + output = $conf.jsonBuildFile + elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends: + # for some cmd we expect a valid absOutFile + output = "unknownOutput" + else: + output = $conf.absOutFile + if conf.filenameOption != foAbs: output = output.AbsoluteFile.extractFilename + # xxx honor filenameOption more accurately + rawMessage(conf, hintSuccessX, [ + "build", build, + "loc", loc, + "sec", sec, + "mem", mem, + "project", project, + "output", output, + ]) diff --git a/compiler/options.nim b/compiler/options.nim index afe8b0fc7..1851770d9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -167,8 +167,17 @@ const type TStringSeq* = seq[string] TGCMode* = enum # the selected GC - gcUnselected, gcNone, gcBoehm, gcRegions, gcArc, gcOrc, - gcMarkAndSweep, gcHooks, gcRefc, gcV2, gcGo + gcUnselected = "unselected" + gcNone = "none" + gcBoehm = "boehm" + gcRegions = "regions" + gcArc = "arc" + gcOrc = "orc" + gcMarkAndSweep = "markAndSweep" + gcHooks = "hooks" + gcRefc = "refc" + gcV2 = "v2" + gcGo = "go" # gcRefc and the GCs that follow it use a write barrier, # as far as usesWriteBarrier() is concerned diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 04356fc76..85729fd59 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -136,6 +136,7 @@ when defined(nimHasInvariant): of ccompilerPath: result = conf.cCompilerPath of backend: result = $conf.backend of libPath: result = conf.libpath.string + of gc: result = $conf.selectedGC proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = template copySeq(field: untyped): untyped = diff --git a/doc/nimc.rst b/doc/nimc.rst index e467a33af..2b9f33cb5 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -108,7 +108,6 @@ Source The source line that triggered a diagnostic message. StackTrace Success, SuccessX Successful compilation of a library or a binary. -BuildMode Kind of build: debug, release, danger User UserRaw XDeclaredButNotUsed Unused symbols in the code. diff --git a/drnim/drnim.nim b/drnim/drnim.nim index 1eded533d..eb0d89aa2 100644 --- a/drnim/drnim.nim +++ b/drnim/drnim.nim @@ -1205,24 +1205,7 @@ proc mainCommand(graph: ModuleGraph) = registerPass graph, semPass compileProject(graph) if conf.errorCounter == 0: - # xxx deduplicate with D20210419T170230 - let mem = - when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem" - else: formatSize(getTotalMem()) & " totmem" - let loc = $conf.linesCompiled - let build = if isDefined(conf, "danger"): "Dangerous Release build" - elif isDefined(conf, "release"): "Release build" - else: "***SLOW, DEBUG BUILD***; -d:release makes code run faster." - let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) - let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName - rawMessage(conf, hintSuccessX, [ - "loc", loc, - "sec", sec, - "mem", mem, - "project", project, - "output", "" - ]) - rawMessage(conf, hintBuildMode, build) + genSuccessX(graph.config) proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = var p = parseopt.initOptParser(cmd) diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 273cfdb57..e1ffc954a 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -32,6 +32,7 @@ type backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` ## and `nim js` would imply backend=js libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 + gc ## gc selected MultipleValueSetting* {.pure.} = enum ## \ ## settings resulting in a seq of string values diff --git a/testament/testament.nim b/testament/testament.nim index 789c987c0..ab6620c84 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -108,8 +108,7 @@ proc isSuccess(input: string): bool = # that may appear in user config (eg: `--filenames`). # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory - input.startsWith("Hint: ") and - (input.endsWith("[SuccessX]") or input.endsWith("[BuildMode]")) + input.startsWith("Hint: ") and input.endsWith("[SuccessX]") proc getFileDir(filename: string): string = result = filename.splitFile().dir diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index b6b302836..f2ea1d040 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -23,7 +23,7 @@ proc isDots(a: string): bool = a.startsWith(".") and a.strip(chars = {'.'}) == "" const - defaultHintsOff = "--hint:successx:off --hint:buildmode:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" + defaultHintsOff = "--hint:successx:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" # useful when you want to turn only some hints on, and some common ones off. # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() From 488acd9d077e8179d35d665ac0591c456bfa93aa Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Mon, 14 Jun 2021 00:21:33 -0700 Subject: [PATCH 0103/1654] fixes #18235 - proc annotation type macro sym leak (#18249) * fixes #18235 - proc annotation type macro sym leak - also fixed a typo - proc annotations guard symbol exports with shadow scopes - symbol handling is shadow scope aware * test for exporting an existing unexported sym this one is for my homie alaviss. * Special handling not needed in semProcAnnotation * Testcasing * [skip ci] clean-up and add some more comments * [skip ci] rm trailing whitespace Co-authored-by: Clyybber --- compiler/lookups.nim | 27 +++++++++++++++++++++----- compiler/semstmts.nim | 8 ++++---- tests/macros/m18235.nim | 42 +++++++++++++++++++++++++++++++++++++++++ tests/macros/t18235.nim | 18 ++++++++++++++++++ 4 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 tests/macros/m18235.nim create mode 100644 tests/macros/t18235.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 64476d816..a21e35ddc 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -314,6 +314,7 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = from ic / ic import addHidden proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = + ## adds symbol to the module for either private or public access. if sfExported in sym.flags or forceExport: # add to interface: if c.module != nil: exportSym(c, sym) @@ -324,10 +325,15 @@ proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = addHidden(c.encoder, c.packedRepr, sym) proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = + ## adds a symbol on the scope and the interface if appropriate addDeclAt(c, scope, sym) - addInterfaceDeclAux(c, sym) + if not scope.isShadowScope: + # adding into a non-shadow scope, we need to handle exports, etc + addInterfaceDeclAux(c, sym) proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = + ## adds an symbol to the given scope, will check for and raise errors if it's + ## a redefinition as opposed to an overload. if fn.kind notin OverloadableSyms: internalError(c.config, fn.info, "addOverloadableSymAt") return @@ -338,24 +344,35 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = scope.addSym(fn) proc addInterfaceDecl*(c: PContext, sym: PSym) = - # it adds the symbol to the interface if appropriate + ## adds a decl and the interface if appropriate addDecl(c, sym) - addInterfaceDeclAux(c, sym) + if not c.currentScope.isShadowScope: + addInterfaceDeclAux(c, sym) proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) = - # it adds the symbol to the interface if appropriate + ## adds an overloadable symbol on the scope and the interface if appropriate addOverloadableSymAt(c, scope, sym) - addInterfaceDeclAux(c, sym) + if not scope.isShadowScope: + addInterfaceDeclAux(c, sym) proc openShadowScope*(c: PContext) = + ## opens a shadow scope, just like any other scope except the depth is the + ## same as the parent -- see `isShadowScope`. c.currentScope = PScope(parent: c.currentScope, symbols: newStrTable(), depthLevel: c.scopeDepth) proc closeShadowScope*(c: PContext) = + ## closes the shadow scope, but doesn't merge any of the symbols c.closeScope proc mergeShadowScope*(c: PContext) = + ## close the existing scope and merge in all defined symbols, this will also + ## trigger any export related code if this is into a non-shadow scope. + ## + ## Merges: + ## shadow -> shadow: add symbols to the parent but check for redefinitions etc + ## shadow -> non-shadow: the above, but also handle exports and all that let shadowScope = c.currentScope c.rawCloseScope for sym in shadowScope.symbols: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 8f87ce83c..d6ad7ab54 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1844,7 +1844,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, # before compiling the proc params & body, set as current the scope # where the proc was declared - let delcarationScope = c.currentScope + let declarationScope = c.currentScope pushOwner(c, s) openScope(c) @@ -1889,7 +1889,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, var (proto, comesFromShadowScope) = if isAnon: (nil, false) - else: searchForProc(c, delcarationScope, s) + else: searchForProc(c, declarationScope, s) if proto == nil and sfForward in s.flags: ## In cases such as a macro generating a proc with a gensymmed name we ## know `searchForProc` will not find it and sfForward will be set. In @@ -1916,9 +1916,9 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not hasProto and sfGenSym notin s.flags: #and not isAnon: if s.kind in OverloadableSyms: - addInterfaceOverloadableSymAt(c, delcarationScope, s) + addInterfaceOverloadableSymAt(c, declarationScope, s) else: - addInterfaceDeclAt(c, delcarationScope, s) + addInterfaceDeclAt(c, declarationScope, s) pragmaCallable(c, s, n, validPragmas) if not hasProto: diff --git a/tests/macros/m18235.nim b/tests/macros/m18235.nim new file mode 100644 index 000000000..6bb4547e0 --- /dev/null +++ b/tests/macros/m18235.nim @@ -0,0 +1,42 @@ +import macros + +# Necessary code to update the AST on a symbol across module boundaries when +# processed by a type macro. Used by a test of a corresponding name of this +# file. + +macro eexport(n: typed): untyped = + result = copyNimTree(n) + # turn exported nnkSym -> nnkPostfix(*, nnkIdent), forcing re-sem + result[0] = nnkPostfix.newTree(ident"*").add: + n.name.strVal.ident + +macro unexport(n: typed): untyped = + result = copyNimTree(n) + # turn nnkSym -> nnkIdent, forcing re-sem and dropping any exported-ness + # that might be present + result[0] = n.name.strVal.ident + +proc foo*() {.unexport.} = discard +proc bar() {.eexport.} = discard + +proc foooof*() {.unexport, eexport, unexport.} = discard +proc barrab() {.eexport, unexport, eexport.} = discard + +macro eexportMulti(n: typed): untyped = + # use the call version of `eexport` macro for one or more decls + result = copyNimTree(n) + for i in 0.. Date: Mon, 14 Jun 2021 08:30:04 +0100 Subject: [PATCH 0104/1654] Enable parallel build in build.sh (#18195) Add help --- tools/niminst/buildsh.nimf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index e13221781..e7afd402e 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -21,10 +21,23 @@ do optosname=$2 shift 2 ;; + --parallel) + parallel=$2 + shift 2 + ;; --extraBuildArgs) extraBuildArgs=" $2" shift 2 ;; + -h | --help) + echo "Options:" + echo " --os " + echo " --cpu " + echo " --osname Additional OS specification (used for Android)" + echo " --extraBuildArgs Additional arguments passed to the compiler" + echo " --parallel Multiprocess build. Requires GNU parallel" + exit 0 + ;; --) # End of all options shift break; @@ -39,7 +52,15 @@ do esac done +parallel="${parallel:-0}" CC="${CC:-gcc}" +if [ "$parallel" -gt 1 ]; then + if ! command -v sem > /dev/null; then + echo "Error: GNU parallel is required to use --parallel" + exit 1 + fi + CC="sem -j $parallel --id $$ ${CC}" +fi COMP_FLAGS="${CPPFLAGS:-} ${CFLAGS:-} ?{c.ccompiler.flags}$extraBuildArgs" LINK_FLAGS="${LDFLAGS:-} ?{c.linker.flags}" PS4="" @@ -222,6 +243,9 @@ case $myos in $CC $COMP_FLAGS -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} # add(linkCmd, " \\\n" & changeFileExt(f, "o")) # end for + if [ "$parallel" -gt 0 ]; then + sem --wait --id $$ + fi $CC -o ?{"$binDir/" & toLowerAscii(c.name)} ?linkCmd $LINK_FLAGS ;; # end for From 0adb47aa15e242983c8251d85367c0fe45fc5f12 Mon Sep 17 00:00:00 2001 From: alaviss Date: Mon, 14 Jun 2021 11:26:12 -0500 Subject: [PATCH 0105/1654] system/excpt: check if the exception is not nil before pop (#18247) In CPS we would consume an exception in the except branch by stashing it into a local then remove the exception from Nim environment so as not to leak it to other code that would be running before the continuation continues However since popCurrentException() assumes that the exception always exist, removing the exception from an except branch will cause a SIGSEGV to happen. This commit fixes that. --- lib/system/excpt.nim | 5 +++-- tests/exception/tsetexceptions.nim | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 5b7d4d49f..72225cf96 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -135,8 +135,9 @@ proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = #showErrorMessage2 "A" proc popCurrentException {.compilerRtl, inl.} = - currException = currException.up - #showErrorMessage2 "B" + if currException != nil: + currException = currException.up + #showErrorMessage2 "B" proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = discard "only for bootstrapping compatbility" diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim index 557fc1898..386a6ae4c 100644 --- a/tests/exception/tsetexceptions.nim +++ b/tests/exception/tsetexceptions.nim @@ -6,3 +6,10 @@ let ex = newException(CatchableError, "test") setCurrentException(ex) doAssert getCurrentException().msg == ex.msg doAssert getCurrentExceptionMsg() == ex.msg +setCurrentException(nil) + +try: + raise newException(CatchableError, "test2") +except: + setCurrentException(nil) +doAssert getCurrentException() == nil From 8c42f5be0274d8d16910401d7d5a45aa35a63afd Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 14 Jun 2021 19:19:58 +0200 Subject: [PATCH 0106/1654] Small scope refactoring (#18263) * Small scope refactoring * Add test for #10251 * Add inline where appropriate --- compiler/astalgo.nim | 3 ++- compiler/lookups.nim | 36 +++++++++++++++++------------------- compiler/semtypes.nim | 6 ++---- tests/errmsgs/t10251.nim | 12 ++++++++++++ 4 files changed, 33 insertions(+), 24 deletions(-) create mode 100644 tests/errmsgs/t10251.nim diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 300089d81..5af8caad8 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -777,9 +777,10 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym; replaceSlot = h h = nextTry(h, high(t.data)) if replaceSlot >= 0: + result = t.data[replaceSlot] # found it if not onConflictKeepOld: t.data[replaceSlot] = n # overwrite it with newer definition! - return t.data[replaceSlot] # found it + return result elif mustRehash(t.data.len, t.counter): strTableEnlarge(t) strTableRawInsert(t.data, n) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index a21e35ddc..9dac106e4 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -293,29 +293,28 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) -proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) = - let conflict = c.currentScope.addUniqueSym(sym) +proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = + let conflict = scope.addUniqueSym(sym) if conflict != nil: wrongRedefinition(c, info, sym.name.s, conflict.info) -proc addDecl*(c: PContext, sym: PSym) = - let conflict = strTableInclReportConflict(c.currentScope.symbols, sym, true) - if conflict != nil: - wrongRedefinition(c, sym.info, sym.name.s, conflict.info) +proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} = + addDeclAt(c, scope, sym, sym.info) + +proc addDecl*(c: PContext, sym: PSym, info: TLineInfo) {.inline.} = + addDeclAt(c, c.currentScope, sym, info) + +proc addDecl*(c: PContext, sym: PSym) {.inline.} = + addDeclAt(c, c.currentScope, sym) proc addPrelimDecl*(c: PContext, sym: PSym) = discard c.currentScope.addUniqueSym(sym) -proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) = - let conflict = scope.addUniqueSym(sym) - if conflict != nil: - wrongRedefinition(c, sym.info, sym.name.s, conflict.info) - from ic / ic import addHidden -proc addInterfaceDeclAux*(c: PContext, sym: PSym, forceExport = false) = +proc addInterfaceDeclAux(c: PContext, sym: PSym) = ## adds symbol to the module for either private or public access. - if sfExported in sym.flags or forceExport: + if sfExported in sym.flags: # add to interface: if c.module != nil: exportSym(c, sym) else: internalError(c.config, sym.info, "addInterfaceDeclAux") @@ -331,6 +330,10 @@ proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) = # adding into a non-shadow scope, we need to handle exports, etc addInterfaceDeclAux(c, sym) +proc addInterfaceDecl*(c: PContext, sym: PSym) {.inline.} = + ## adds a decl and the interface if appropriate + addInterfaceDeclAt(c, c.currentScope, sym) + proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = ## adds an symbol to the given scope, will check for and raise errors if it's ## a redefinition as opposed to an overload. @@ -343,16 +346,11 @@ proc addOverloadableSymAt*(c: PContext; scope: PScope, fn: PSym) = else: scope.addSym(fn) -proc addInterfaceDecl*(c: PContext, sym: PSym) = - ## adds a decl and the interface if appropriate - addDecl(c, sym) - if not c.currentScope.isShadowScope: - addInterfaceDeclAux(c, sym) - proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) = ## adds an overloadable symbol on the scope and the interface if appropriate addOverloadableSymAt(c, scope, sym) if not scope.isShadowScope: + # adding into a non-shadow scope, we need to handle exports, etc addInterfaceDeclAux(c, sym) proc openShadowScope*(c: PContext) = diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 84556ca1a..38307a9ac 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -138,15 +138,13 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = identToReplace[] = symNode if e.position == 0: hasNull = true if result.sym != nil and sfExported in result.sym.flags: - incl(e.flags, {sfUsed, sfExported}) - if result.sym != nil and not isPure: - addInterfaceDeclAux(c, e, forceExport = sfExported in result.sym.flags) + e.flags.incl {sfUsed, sfExported} result.n.add symNode styleCheckDef(c.config, e) onDef(e.info, e) if sfGenSym notin e.flags: - if not isPure: addDecl(c, e) + if not isPure: addInterfaceDecl(c, e) else: declarePureEnumField(c, e) if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil): wrongRedefinition(c, e.info, e.name.s, conflict.info) diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim new file mode 100644 index 000000000..5ad373ba3 --- /dev/null +++ b/tests/errmsgs/t10251.nim @@ -0,0 +1,12 @@ +discard """ +errormsg: "redefinition of 'foo'; previous declaration here: t10251.nim(9, 9)" +line: 11 +column: 9 +""" + +type + Enum1 = enum + foo, bar, baz + Enum2 = enum + foo, bar, baz + From e80d7ff0f2716fd1f892a936a78986531b107fb6 Mon Sep 17 00:00:00 2001 From: Mark Pointing Date: Tue, 15 Jun 2021 03:29:23 +1000 Subject: [PATCH 0107/1654] httpclient.nim Fixes #14794 and an issue where content-header is not set on postContent (#18208) * Fixed missing newline after bound marker in mulipart post (#14794) and a problem where calling postContent with multipart data does not set content-length header. * Update lib/pure/httpclient.nim * Added comment outlining the reason for changes to httpclient.nim and added tests to ensure that multipart post has a newline at the end of the body, and that the content-length header is present. * Fixed typo in comments. * Removed redundant blank lines in thttpclient_standalone.nim. Co-authored-by: Mark Pointing Co-authored-by: Andreas Rumpf --- lib/pure/httpclient.nim | 8 ++++-- tests/stdlib/thttpclient_standalone.nim | 33 ++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 2f9f1913f..33da6ef6c 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -975,7 +975,7 @@ proc format(client: HttpClient | AsyncHttpClient, if entry.isFile: length += entry.fileSize + httpNewLine.len - result.add "--" & bound & "--" + result.add "--" & bound & "--" & httpNewLine for s in result: length += s.len client.headers["Content-Length"] = $length @@ -1010,12 +1010,16 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: Uri, await newConnection(client, url) - let newHeaders = client.headers.override(headers) + var newHeaders: HttpHeaders var data: seq[string] if multipart != nil and multipart.content.len > 0: + # `format` modifies `client.headers`, see + # https://github.com/nim-lang/Nim/pull/18208#discussion_r647036979 data = await client.format(multipart) + newHeaders = client.headers.override(headers) else: + newHeaders = client.headers.override(headers) # Only change headers if they have not been specified already if not newHeaders.hasKey("Content-Length"): if body.len != 0: diff --git a/tests/stdlib/thttpclient_standalone.nim b/tests/stdlib/thttpclient_standalone.nim index 44a88e91e..5fa1ea41a 100644 --- a/tests/stdlib/thttpclient_standalone.nim +++ b/tests/stdlib/thttpclient_standalone.nim @@ -2,7 +2,7 @@ discard """ cmd: "nim c --threads:on $file" """ -import asynchttpserver, httpclient, asyncdispatch +import asynchttpserver, httpclient, asyncdispatch, strutils block: # bug #16436 proc startServer() {.async.} = @@ -21,3 +21,34 @@ block: # bug #16436 asyncCheck startServer() doAssertRaises(ProtocolError): waitFor runClient() + +block: # bug #14794 (And test for presence of content-length header when using postContent) + proc startServer() {.async.} = + var killServer = false + proc cb(req: Request) {.async.} = + doAssert(req.body.endsWith(httpNewLine), "Multipart body does not end with a newline.") + # this next line is probably not required because asynchttpserver does not call + # the callback when there is no content-length header. It instead errors with + # Error: unhandled exception: 411 Length Required + # Added for good measure in case the server becomes more permissive. + doAssert(req.headers.hasKey("content-length"), "Content-Length header is not present.") + killServer = true + asyncCheck req.respond(Http200, "OK") + + var server = newAsyncHttpServer() + server.listen(Port(5556)) + while not killServer: + if server.shouldAcceptRequest(): + await server.acceptRequest(cb) + else: + poll() + + proc runClient() {.async.} = + let c = newAsyncHttpClient() + var data = newMultipartData() + data.add("file.txt", "This is intended to be an example text file.\r\nThis would be the second line.\r\n") + let r = await c.postContent("http://127.0.0.1:5556", multipart = data) + c.close() + + asyncCheck startServer() + waitFor runClient() From 8ef6073543b14bd48665ae075df66c2c36f564dd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 14 Jun 2021 23:56:10 -0700 Subject: [PATCH 0108/1654] add test case for pure enum redefinition error within enum (fixed in recent PR) (#18266) * add test case for pure enum redefinition error within enum (fixed in recent PR) * remove code duplication * Revert "remove code duplication" (would require bootstrap >= 1.4) This reverts commit 3f793874c231f847ef015e37a5fd6851f85d9675. * fixup --- compiler/astalgo.nim | 8 ++++---- compiler/lookups.nim | 2 ++ tests/errmsgs/t10251.nim | 21 ++++++++++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 5af8caad8..3c97665f0 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -759,9 +759,9 @@ proc strTableAdd*(t: var TStrTable, n: PSym) = proc strTableInclReportConflict*(t: var TStrTable, n: PSym; onConflictKeepOld = false): PSym = - # returns true if n is already in the string table: - # It is essential that `n` is written nevertheless! - # This way the newest redefinition is picked by the semantic analyses! + # if `t` has a conflicting symbol (same identifier as `n`), return it + # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true` + # and a conflict was found. assert n.name != nil var h: Hash = n.name.h and high(t.data) var replaceSlot = -1 @@ -780,7 +780,7 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym; result = t.data[replaceSlot] # found it if not onConflictKeepOld: t.data[replaceSlot] = n # overwrite it with newer definition! - return result + return result # but return the old one elif mustRehash(t.data.len, t.counter): strTableEnlarge(t) strTableRawInsert(t.data, n) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 9dac106e4..f72b58ac6 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -293,6 +293,8 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) +# xxx pending bootstrap >= 1.4, replace all those overloads with a single one: +# proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = let conflict = scope.addUniqueSym(sym) if conflict != nil: diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim index 5ad373ba3..0c7fe0b3d 100644 --- a/tests/errmsgs/t10251.nim +++ b/tests/errmsgs/t10251.nim @@ -1,12 +1,19 @@ discard """ -errormsg: "redefinition of 'foo'; previous declaration here: t10251.nim(9, 9)" -line: 11 -column: 9 + action:reject + cmd: "nim check $options $file" + nimout: ''' +t10251.nim(15, 5) Error: redefinition of 'foo'; previous declaration here: t10251.nim(13, 5) +t10251.nim(19, 23) Error: redefinition of 'goo1'; previous declaration here: t10251.nim(19, 11) +''' """ +# line 10 type - Enum1 = enum - foo, bar, baz - Enum2 = enum - foo, bar, baz + Enum1 = enum + foo, bar, baz + Enum2 = enum + foo, bar, baz +type + Enum3 {.pure.} = enum # fixed (by accident?) in https://github.com/nim-lang/Nim/pull/18263 + goo0, goo1, goo2, goo1 From d3b27eb63e18fbe36a7579a3322db316115a3d81 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 15 Jun 2021 14:56:49 -0700 Subject: [PATCH 0109/1654] link stable and devel docs in nim docs (#18272) [backport:1.2] --- config/nimdoc.cfg | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index cdf49197d..15f887023 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -127,21 +127,12 @@ doc.body_toc_group = """
From c5cf21c0c4f314e4128658d35000339d67e4824b Mon Sep 17 00:00:00 2001 From: Clyybber Date: Wed, 16 Jun 2021 02:45:05 +0200 Subject: [PATCH 0110/1654] Don't report unused hints for consumed AST (#18270) * Fix #18203 * Add testcase * Fix testcase * Fix test --- compiler/lookups.nim | 4 +++- tests/macros/t18203.nim | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/macros/t18203.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index f72b58ac6..4f413f472 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -364,7 +364,9 @@ proc openShadowScope*(c: PContext) = proc closeShadowScope*(c: PContext) = ## closes the shadow scope, but doesn't merge any of the symbols - c.closeScope + ## Does not check for unused symbols or missing forward decls since a macro + ## or template consumes this AST + rawCloseScope(c) proc mergeShadowScope*(c: PContext) = ## close the existing scope and merge in all defined symbols, this will also diff --git a/tests/macros/t18203.nim b/tests/macros/t18203.nim new file mode 100644 index 000000000..aae0a2690 --- /dev/null +++ b/tests/macros/t18203.nim @@ -0,0 +1,15 @@ +discard """ + matrix: "--hint:SuccessX:off --hint:Link:off --hint:Conf:off --hint:CC:off --hint:XDeclaredButNotUsed:on" + nimout: ''' +''' +nimoutFull: true +action: compile +""" + +# bug #18203 +import std/macros + +macro foo(x: typed) = newProc ident"bar" +proc bar() {.foo.} = raise +bar() + From c51680e7012bff32156623fad0996d62f7918222 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 16 Jun 2021 17:31:20 +0800 Subject: [PATCH 0111/1654] fixes #17696 (#18276) --- lib/pure/collections/sharedlist.nim | 7 ++-- tests/stdlib/tsharedlist.nim | 53 +++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index c72477675..79f2391f6 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -35,8 +35,10 @@ template withLock(t, x: untyped) = release(t.lock) proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = - ## iterates over the list. If 'action' returns true, the + ## Iterates over the list. If 'action' returns true, the ## current item is removed from the list. + ## + ## .. warning:: It may not preserve the element order after some modifications. withLock(x): var n = x.head while n != nil: @@ -47,8 +49,9 @@ proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = if action(n.d[i]): acquire(x.lock) let t = x.tail + dec t.dataLen # TODO considering t.dataLen == 0, + # probably the module should be refactored using doubly linked lists n.d[i] = t.d[t.dataLen] - dec t.dataLen else: acquire(x.lock) inc i diff --git a/tests/stdlib/tsharedlist.nim b/tests/stdlib/tsharedlist.nim index a795be0f3..693f1018e 100644 --- a/tests/stdlib/tsharedlist.nim +++ b/tests/stdlib/tsharedlist.nim @@ -1,17 +1,48 @@ -import sharedlist +discard """ + matrix: "--threads:on" +""" -var - list: SharedList[int] - count: int +import std/sharedlist -init(list) +block: + var + list: SharedList[int] + count: int -for i in 1 .. 250: - list.add i + init(list) -for i in list: - inc count + for i in 1 .. 250: + list.add i -doAssert count == 250 + for i in list: + inc count -deinitSharedList(list) + doAssert count == 250 + + deinitSharedList(list) + + +block: # bug #17696 + var keysList = SharedList[string]() + init(keysList) + + keysList.add("a") + keysList.add("b") + keysList.add("c") + keysList.add("d") + keysList.add("e") + keysList.add("f") + + + # Remove element "b" and "d" from the list. + keysList.iterAndMutate(proc (key: string): bool = + if key == "b" or key == "d": # remove only "b" and "d" + return true + return false + ) + + var results: seq[string] + for key in keysList.items: + results.add key + + doAssert results == @["a", "f", "c", "e"] From 13b94c0297806911cf317ff6169dba8595a0333d Mon Sep 17 00:00:00 2001 From: Clyybber Date: Wed, 16 Jun 2021 16:40:22 +0200 Subject: [PATCH 0112/1654] Fix doubly typed forward declarations (#18279) * Add testcase * Fix testcase * Fix doubly typed forward decls * Better fix --- compiler/semstmts.nim | 7 ++++++- tests/macros/tmacros_issues.nim | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d6ad7ab54..f29dc24ef 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1890,11 +1890,16 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, var (proto, comesFromShadowScope) = if isAnon: (nil, false) else: searchForProc(c, declarationScope, s) - if proto == nil and sfForward in s.flags: + if proto == nil and sfForward in s.flags and n[bodyPos].kind != nkEmpty: ## In cases such as a macro generating a proc with a gensymmed name we ## know `searchForProc` will not find it and sfForward will be set. In ## such scenarios the sym is shared between forward declaration and we ## can treat the `s` as the proto. + ## To differentiate between that happening and a macro just returning a + ## forward declaration that has been typed before we check if the body + ## is not empty. This has the sideeffect of allowing multiple forward + ## declarations if they share the same sym. + ## See the "doubly-typed forward decls" case in tmacros_issues.nim proto = s let hasProto = proto != nil diff --git a/tests/macros/tmacros_issues.nim b/tests/macros/tmacros_issues.nim index a964f46ba..a81c51658 100644 --- a/tests/macros/tmacros_issues.nim +++ b/tests/macros/tmacros_issues.nim @@ -484,6 +484,26 @@ func expMin: float {.aadMin.} = 1 echo expMin() +# doubly-typed forward decls +macro noop(x: typed) = x +noop: + proc cally() = discard + +cally() + +noop: + proc barry() + +proc barry() = discard + +# some more: +proc barry2() {.noop.} +proc barry2() = discard + +proc barry3() {.noop.} +proc barry3() {.noop.} = discard + + # issue #15389 block double_sem_for_procs: @@ -498,4 +518,4 @@ block double_sem_for_procs: return x1 + 1.0 result = 10.0 - discard exp(5.0) \ No newline at end of file + discard exp(5.0) From 3c854e8c24b394900dc65df680b7679f09cee97c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jun 2021 23:04:12 -0700 Subject: [PATCH 0113/1654] followup #16400; use use -d:nimCompilerStackraceHints in more places (#18282) --- compiler/vmgen.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index c6fd75259..bb203e0ab 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -35,6 +35,9 @@ import from modulegraphs import getBody +when defined(nimCompilerStackraceHints): + import std/stackframes + const debugEchoCode* = defined(nimVMDebug) @@ -2000,6 +2003,8 @@ proc procIsCallback(c: PCtx; s: PSym): bool = dec i proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = + when defined(nimCompilerStackraceHints): + setFrameMsg c.config$n.info & " " & $n.kind & " " & $flags case n.kind of nkSym: let s = n.sym From 49e945ed082bacd5cd5ecff305ec47ff3a00f1c5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jun 2021 23:07:40 -0700 Subject: [PATCH 0114/1654] increase backoffDuration to avoid timeouts (#18281) --- testament/categories.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/categories.nim b/testament/categories.nim index 7f05797ff..bf7855b2c 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -429,7 +429,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = let buildPath = packagesDir / pkg.name template tryCommand(cmd: string, workingDir2 = buildPath, reFailed = reInstallFailed, maxRetries = 1): string = var outp: string - let ok = retryCall(maxRetry = maxRetries, backoffDuration = 1.0): + let ok = retryCall(maxRetry = maxRetries, backoffDuration = 10.0): var status: int (outp, status) = execCmdEx(cmd, workingDir = workingDir2) status == QuitSuccess From 969cb97c0a9f045a2fb31866fc804cdf1b2698dd Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 17 Jun 2021 09:19:52 +0300 Subject: [PATCH 0115/1654] PCRE, nimgrep: add limit for buffer size (#18280) --- lib/impure/re.nim | 13 ++++++++++--- tools/nimgrep.nim | 5 ++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 78ea7b002..504d8f22e 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -141,6 +141,10 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string], else: matches[i-1] = "" return rawMatches[1] - rawMatches[0] +const MaxReBufSize* = high(cint) + ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even + ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1. + proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): tuple[first, last: int] = ## returns the starting position and end position of `pattern` in `buf` @@ -167,7 +171,8 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], ## and the captured substrings in the array `matches`. ## If it does not match, nothing ## is written into `matches` and `(-1,0)` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[tuple[first, last: int]], @@ -197,7 +202,8 @@ proc findBounds*(s: string, pattern: Regex, ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and ## `(-1,0)` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc findBoundsImpl(buf: cstring, pattern: Regex, start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = @@ -232,7 +238,8 @@ proc findBounds*(s: string, pattern: Regex, ## Note: there is a speed improvement if the matches do not need to be captured. runnableExamples: assert findBounds("01234abc89", re"abc") == (5,7) - result = findBounds(cstring(s), pattern, start, s.len) + result = findBounds(cstring(s), pattern, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint = var diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index a368c40ef..a589cfb14 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -649,7 +649,7 @@ template updateCounters(output: Output) = proc printInfo(filename:string, output: Output) = case output.kind of openError: - printError("can not open path " & filename & " " & output.msg) + printError("cannot open path '" & filename & "': " & output.msg) of rejected: if optVerbose in options: echo "(rejected: ", output.reason, ")" @@ -719,6 +719,9 @@ iterator searchFile(pattern: Pattern; buffer: string): Output = pre: pre, match: move(curMi)) i = t.last+1 + when typeof(pattern) is Regex: + if buffer.len > MaxReBufSize: + yield Output(kind: openError, msg: "PCRE size limit is " & $MaxReBufSize) func detectBin(buffer: string): bool = for i in 0 ..< min(1024, buffer.len): From fd8b79707c1b936ed1ba42892f5e06f846e2816b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 17 Jun 2021 08:20:10 +0200 Subject: [PATCH 0116/1654] Revert "system/excpt: check if the exception is not nil before pop (#18247)" (#18265) This reverts commit 0adb47aa15e242983c8251d85367c0fe45fc5f12. --- lib/system/excpt.nim | 5 ++--- tests/exception/tsetexceptions.nim | 7 ------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 72225cf96..5b7d4d49f 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -135,9 +135,8 @@ proc pushCurrentException(e: sink(ref Exception)) {.compilerRtl, inl.} = #showErrorMessage2 "A" proc popCurrentException {.compilerRtl, inl.} = - if currException != nil: - currException = currException.up - #showErrorMessage2 "B" + currException = currException.up + #showErrorMessage2 "B" proc popCurrentExceptionEx(id: uint) {.compilerRtl.} = discard "only for bootstrapping compatbility" diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim index 386a6ae4c..557fc1898 100644 --- a/tests/exception/tsetexceptions.nim +++ b/tests/exception/tsetexceptions.nim @@ -6,10 +6,3 @@ let ex = newException(CatchableError, "test") setCurrentException(ex) doAssert getCurrentException().msg == ex.msg doAssert getCurrentExceptionMsg() == ex.msg -setCurrentException(nil) - -try: - raise newException(CatchableError, "test2") -except: - setCurrentException(nil) -doAssert getCurrentException() == nil From 99411674a6da72f608bfa870f51b3a296cd85eee Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 16 Jun 2021 23:20:33 -0700 Subject: [PATCH 0117/1654] tests/exception/tsetexceptions.nim not joinable (#18264) --- tests/exception/tsetexceptions.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/exception/tsetexceptions.nim b/tests/exception/tsetexceptions.nim index 557fc1898..0e353f43e 100644 --- a/tests/exception/tsetexceptions.nim +++ b/tests/exception/tsetexceptions.nim @@ -1,7 +1,10 @@ discard """ targets: "c cpp js" + joinable: false """ +# refs https://github.com/nim-lang/Nim/pull/18247#issuecomment-860877161 + let ex = newException(CatchableError, "test") setCurrentException(ex) doAssert getCurrentException().msg == ex.msg From c83ac16671170e513fa1ca40419892ccc618ec85 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 17 Jun 2021 00:17:46 -0700 Subject: [PATCH 0118/1654] Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. (#18283) --- changelog.md | 1 + compiler/ccgexprs.nim | 4 ++-- compiler/semexprs.nim | 4 ++-- compiler/vmgen.nim | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index cc6f826ee..26048eedd 100644 --- a/changelog.md +++ b/changelog.md @@ -88,6 +88,7 @@ The downside is that these defines now have custom logic that doesn't apply for other defines. +- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. ## Standard library additions and changes diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7ed80145b..c07ddf1db 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -9,7 +9,7 @@ # included from cgen.nim -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode, @@ -2694,7 +2694,7 @@ proc genConstStmt(p: BProc, n: PNode) = genConstDefinition(m, p, sym) proc expr(p: BProc, n: PNode, d: var TLoc) = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg p.config$n.info & " " & $n.kind p.currLineInfo = n.info diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ba8b93ae6..53a5f149e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -10,7 +10,7 @@ # this module does the semantic checking for expressions # included from sem.nim -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes const @@ -2714,7 +2714,7 @@ proc getNilType(c: PContext): PType = c.nilTypeCache = result proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind result = n if c.config.cmd == cmdIdeTools: suggestExpr(c, n) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bb203e0ab..14f9e0b30 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -35,7 +35,7 @@ import from modulegraphs import getBody -when defined(nimCompilerStackraceHints): +when defined(nimCompilerStacktraceHints): import std/stackframes const @@ -2003,7 +2003,7 @@ proc procIsCallback(c: PCtx; s: PSym): bool = dec i proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = - when defined(nimCompilerStackraceHints): + when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind & " " & $flags case n.kind of nkSym: From 5a3456d220d0c2a4022969b98ab33baaf1fd4d00 Mon Sep 17 00:00:00 2001 From: Adrian Veith Date: Thu, 17 Jun 2021 11:23:38 +0200 Subject: [PATCH 0119/1654] fix for #18284 int32 should be int (#18285) the var exp was typed as int32 - it should be int since frep expects an int --- lib/pure/math.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index cff946581..410660d4a 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -997,7 +997,7 @@ when not defined(js): # taken from Go-lang Math.Log2 const ln2 = 0.693147180559945309417232121458176568075500134360255254120680009 template log2Impl[T](x: T): T = - var exp: int32 + var exp: int var frac = frexp(x, exp) # Make sure exact powers of two give an exact answer. # Don't depend on Log(0.5)*(1/Ln2)+exp being exactly exp-1. From ef121f3b99c4d020d3df7cb369f5060c76c73d57 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 17 Jun 2021 16:19:16 -0700 Subject: [PATCH 0120/1654] followup #17876: remove annoying enum name clashes in tests/enum/tenum.nim (#18291) --- tests/enum/tenum.nim | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index 97fd4c68c..37383890c 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -128,22 +128,21 @@ block tnamedfields: doAssert $x == "abc", $x -block toptions: +block tfakeOptions: type - # please make sure we have under 32 options (improves code efficiency!) - TOption = enum - optNone, optForceFullMake, optBoehmGC, optRefcGC, optRangeCheck, - optBoundsCheck, optOverflowCheck, optNilCheck, optAssert, optLineDir, - optWarns, optHints, optListCmd, optCompileOnly, - optSafeCode, # only allow safe code - optStyleCheck, optOptimizeSpeed, optOptimizeSize, optGenDynLib, - optGenGuiApp, optStackTrace + TFakeOption = enum + fakeNone, fakeForceFullMake, fakeBoehmGC, fakeRefcGC, fakeRangeCheck, + fakeBoundsCheck, fakeOverflowCheck, fakeNilCheck, fakeAssert, fakeLineDir, + fakeWarns, fakeHints, fakeListCmd, fakeCompileOnly, + fakeSafeCode, # only allow safe code + fakeStyleCheck, fakeOptimizeSpeed, fakeOptimizeSize, fakeGenDynLib, + fakeGenGuiApp, fakeStackTrace - TOptionset = set[TOption] + TFakeOptionset = set[TFakeOption] var - gOptions: TOptionset = {optRefcGC, optRangeCheck, optBoundsCheck, - optOverflowCheck, optAssert, optWarns, optHints, optLineDir, optStackTrace} + gFakeOptions: TFakeOptionset = {fakeRefcGC, fakeRangeCheck, fakeBoundsCheck, + fakeOverflowCheck, fakeAssert, fakeWarns, fakeHints, fakeLineDir, fakeStackTrace} compilerArgs: int gExitcode: int8 From fc76565574b86566e4a642b9aac541e025cf5de3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 17 Jun 2021 16:38:15 -0700 Subject: [PATCH 0121/1654] disable pkg manu (#18292) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 3d3c340eb..efee361da 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -82,7 +82,7 @@ pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" pkg "loopfusion" pkg "macroutils" -pkg "manu" +pkg "manu", allowFailure=true # pending https://github.com/planetis-m/manu/issues/5 pkg "markdown" pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" From a250481dcde2167d568cc1b61624658419c18e69 Mon Sep 17 00:00:00 2001 From: j-james <35242550+j-james@users.noreply.github.com> Date: Fri, 18 Jun 2021 03:10:45 -0700 Subject: [PATCH 0122/1654] Make the existence of high(type) clearer, especially near subranges (#18286) * Make the existence of high(type) clearer, especially around subranges * Use a better example and dry up the wording --- doc/manual.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 867db245b..46a40a690 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -989,10 +989,12 @@ Ordinal types have the following characteristics: - Ordinal types are countable and ordered. This property allows the operation of functions such as `inc`, `ord`, and `dec` on ordinal types to be defined. -- Ordinal values have the smallest possible value. Trying to count further - down than the smallest value produces a panic or a static error. -- Ordinal values have the largest possible value. Trying to count further - than the largest value produces a panic or a static error. +- Ordinal types have a smallest possible value, accessible with `low(type)`. + Trying to count further down than the smallest value produces a panic or + a static error. +- Ordinal types have a largest possible value, accessible with `high(type)`. + Trying to count further up than the largest value produces a panic or + a static error. Integers, bool, characters, and enumeration types (and subranges of these types) belong to ordinal types. @@ -1095,6 +1097,7 @@ lowest and highest value of the type. For example: type Subrange = range[0..5] PositiveFloat = range[0.0..Inf] + Positive* = range[1..high(int)] # as defined in `system` `Subrange` is a subrange of an integer which can only hold the values 0 From 87cd9b24a3df66dcde96bbdec2bd7620015ddeba Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jun 2021 06:52:08 -0700 Subject: [PATCH 0123/1654] add more APIs to compiler/debugutils; re-export it (#18243) --- compiler/debugutils.nim | 37 +++++++++++++++++++++++++++++++++ compiler/options.nim | 3 +++ compiler/semexprs.nim | 7 +++++++ tests/compiler/tdebugutils.nim | 38 ++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+) create mode 100644 tests/compiler/tdebugutils.nim diff --git a/compiler/debugutils.nim b/compiler/debugutils.nim index 235c100b3..d109d2121 100644 --- a/compiler/debugutils.nim +++ b/compiler/debugutils.nim @@ -4,7 +4,26 @@ Utilities to help with debugging nim compiler. Experimental API, subject to change. ]## +#[ +## example +useful debugging flags: +--stacktrace -d:debug -d:nimDebugUtils + nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim + +## future work +* expose and improve astalgo.debug, replacing it by std/prettyprints, + refs https://github.com/nim-lang/RFCs/issues/385 +]# + import options +import std/wrapnils +export wrapnils + # allows using things like: `?.n.sym.typ.len` + +import std/stackframes +export stackframes + # allows using things like: `setFrameMsg c.config$n.info & " " & $n.kind` + # which doesn't log, but augments stacktrace with side channel information var conf0: ConfigRef @@ -17,3 +36,21 @@ proc onNewConfigRef*(conf: ConfigRef) {.inline.} = proc getConfigRef*(): ConfigRef = ## nil, if -d:nimDebugUtils wasn't specified result = conf0 + +proc isCompilerDebug*(): bool = + ##[ + Provides a simple way for user code to enable/disable logging in the compiler + in a granular way. This can then be used in the compiler as follows: + ```nim + if isCompilerDebug(): + echo ?.n.sym.typ.len + ``` + ]## + runnableExamples: + proc main = + echo 2 + {.define(nimCompilerDebug).} + echo 3.5 # code section in which `isCompilerDebug` will be true + {.undef(nimCompilerDebug).} + echo 'x' + conf0.isDefined("nimCompilerDebug") diff --git a/compiler/options.nim b/compiler/options.nim index 1851770d9..05d8ee5b7 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -472,7 +472,10 @@ const foreignPackageNotesDefault* = { proc isDefined*(conf: ConfigRef; symbol: string): bool when defined(nimDebugUtils): + # this allows inserting debugging utilties in all modules that import `options` + # with a single switch, which is useful when debugging compiler. import debugutils + export debugutils proc initConfigRefCommon(conf: ConfigRef) = conf.selectedGC = gcRefc diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 53a5f149e..c885f4963 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2716,6 +2716,13 @@ proc getNilType(c: PContext): PType = proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind + when false: # see `tdebugutils` + if isCompilerDebug(): + echo (">", c.config$n.info, n, flags, n.kind) + defer: + if isCompilerDebug(): + echo ("<", c.config$n.info, n, ?.result.typ) + result = n if c.config.cmd == cmdIdeTools: suggestExpr(c, n) if nfSem in n.flags: return diff --git a/tests/compiler/tdebugutils.nim b/tests/compiler/tdebugutils.nim new file mode 100644 index 000000000..50b15cb78 --- /dev/null +++ b/tests/compiler/tdebugutils.nim @@ -0,0 +1,38 @@ +discard """ + joinable: false + disabled: true +""" + +#[ +This test illustrates some features of `debugutils` to debug the compiler. + +## example +this shows how to enable compiler logging just for a section of user code, +without generating tons of unrelated log messages for code you're not interested +in debugging. + +```sh +# enable some debugging code, e.g. the `when false:` block in `semExpr` +nim c -o:bin/nim_temp --stacktrace -d:debug -d:nimDebugUtils compiler/nim +bin/nim_temp c tests/compiler/tdebugutils.nim +``` + +(use --filenames:abs for abs files) + +## result +("<", "tdebugutils.nim(16, 3)", {.define(nimCompilerDebug).}, nil) +(">", "tdebugutils.nim(17, 3)", let a = 2.5 * 3, {}, nkLetSection) +(">", "tdebugutils.nim(17, 15)", 2.5 * 3, {efAllowDestructor, efWantValue}, nkInfix) +(">", "tdebugutils.nim(17, 11)", 2.5, {efAllowStmt, efDetermineType, efOperand}, nkFloatLit) +("<", "tdebugutils.nim(17, 11)", 2.5, float64) +(">", "tdebugutils.nim(17, 17)", 3, {efAllowStmt, efDetermineType, efOperand}, nkIntLit) +("<", "tdebugutils.nim(17, 17)", 3, int literal(3)) +("<", "tdebugutils.nim(17, 15)", 2.5 * 3, float) +("<", "tdebugutils.nim(17, 3)", let a = 2.5 * 3, nil) +(">", "tdebugutils.nim(18, 3)", {.undef(nimCompilerDebug).}, {}, nkPragma) +]# + +proc main = + {.define(nimCompilerDebug).} + let a = 2.5 * 3 + {.undef(nimCompilerDebug).} From 5600a622297a9b06b2f47039e0397cded1bd425c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jun 2021 08:57:51 -0700 Subject: [PATCH 0124/1654] strformat.fmt now supports non-literal const strings (#18274) Co-authored-by: Andreas Rumpf --- changelog.md | 4 ++- lib/core/macros.nim | 3 +- lib/pure/strformat.nim | 63 +++++++++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/changelog.md b/changelog.md index 26048eedd..4fd1ef355 100644 --- a/changelog.md +++ b/changelog.md @@ -93,7 +93,9 @@ ## Standard library additions and changes -- Added support for parenthesized expressions in `strformat` +- `strformat`: + added support for parenthesized expressions. + added support for const string's instead of just string literals - Fixed buffer overflow bugs in `net` diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 37f855d94..c09fae6b3 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -476,8 +476,9 @@ proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. ## needs to occur in a declaration context. proc callsite*(): NimNode {.magic: "NCallSite", benign, deprecated: - "Deprecated since v0.18.1; use varargs[untyped] in the macro prototype instead".} + "Deprecated since v0.18.1; use `varargs[untyped]` in the macro prototype instead".} ## Returns the AST of the invocation expression that invoked this macro. + # see https://github.com/nim-lang/RFCs/issues/387 as candidate replacement. proc toStrLit*(n: NimNode): NimNode = ## Converts the AST `n` to the concrete Nim code and wraps that diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 52731e970..872141383 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -573,15 +573,12 @@ template formatValue(result: var string; value: char; specifier: string) = template formatValue(result: var string; value: cstring; specifier: string) = result.add value -proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = - if pattern.kind notin {nnkStrLit..nnkTripleStrLit}: - error "string formatting (fmt(), &) only works with string literals", pattern +proc strformatImpl(f: string; openChar, closeChar: char): NimNode = if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" - let f = pattern.strVal var i = 0 let res = genSym(nskVar, "fmtRes") - result = newNimNode(nnkStmtListExpr, lineInfoFrom = pattern) + result = newNimNode(nnkStmtListExpr) # XXX: https://github.com/nim-lang/Nim/issues/8405 # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils # module are not accessible at compile-time: @@ -633,12 +630,8 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = var x: NimNode try: x = parseExpr(subexpr) - except ValueError: - when declared(getCurrentExceptionMsg): - let msg = getCurrentExceptionMsg() - error("could not parse `" & subexpr & "`.\n" & msg, pattern) - else: - error("could not parse `" & subexpr & "`.\n", pattern) + except ValueError as e: + error("could not parse `$#` in `$#`.\n$#" % [subexpr, f, e.msg]) let formatSym = bindSym("formatValue", brOpen) var options = "" if f[i] == ':': @@ -667,19 +660,39 @@ proc strformatImpl(pattern: NimNode; openChar, closeChar: char): NimNode = when defined(debugFmtDsl): echo repr result -macro `&`*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## For a specification of the `&` macro, see the module level documentation. - -macro fmt*(pattern: string): untyped = strformatImpl(pattern, '{', '}') - ## An alias for `& <#&.m,string>`_. - -macro fmt*(pattern: string; openChar, closeChar: char): untyped = - ## The same as `fmt <#fmt.m,string>`_, but uses `openChar` instead of `'{'` - ## and `closeChar` instead of `'}'`. +macro fmt*(pattern: static string; openChar: static char, closeChar: static char): string = + ## Interpolates `pattern` using symbols in scope. runnableExamples: - let testInt = 123 - assert "".fmt('<', '>') == "123" - assert """(()"foo" & "bar"())""".fmt(')', '(') == "(foobar)" - assert """ ""{"123+123"}"" """.fmt('"', '"') == " \"{246}\" " + let x = 7 + assert "var is {x * 2}".fmt == "var is 14" + assert "var is {{x}}".fmt == "var is {x}" # escape via doubling + const s = "foo: {x}" + assert s.fmt == "foo: 7" # also works with const strings - strformatImpl(pattern, openChar.intVal.char, closeChar.intVal.char) + assert fmt"\n" == r"\n" # raw string literal + assert "\n".fmt == "\n" # regular literal (likewise with `fmt("\n")` or `fmt "\n"`) + runnableExamples: + # custom `openChar`, `closeChar` + let x = 7 + assert "".fmt('<', '>') == "7" + assert "<<>>".fmt('<', '>') == "<7>" + assert "`x`".fmt('`', '`') == "7" + strformatImpl(pattern, openChar, closeChar) + +template fmt*(pattern: static string): untyped = + ## Alias for `fmt(pattern, '{', '}')`. + fmt(pattern, '{', '}') + +macro `&`*(pattern: string{lit}): string = + ## `&pattern` is the same as `pattern.fmt`. + ## For a specification of the `&` macro, see the module level documentation. + # pending bug #18275, bug #18278, use `pattern: static string` + # consider deprecating this, it's redundant with `fmt` and `fmt` is strictly + # more flexible, readable (no confusion with the binary `&`), self-documenting, + # not to mention #18275, bug #18278. + runnableExamples: + let x = 7 + assert &"{x}\n" == "7\n" # regular string literal + assert &"{x}\n" == "7\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"7\n" # see `fmt` docs, this would use a raw string literal + strformatImpl(pattern.strVal, '{', '}') From 5d15bd7b618cda06e3ae88883914467f89d38112 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 18 Jun 2021 10:23:27 -0700 Subject: [PATCH 0125/1654] refs #18278: do not gag fatal msgs (#18290) --- compiler/lineinfos.nim | 13 ++++++++----- compiler/modules.nim | 3 +-- compiler/msgs.nim | 15 +++++++++++---- compiler/pragmas.nim | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index da0a8d623..052dae272 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -27,7 +27,10 @@ proc createDocLink*(urlSuffix: string): string = type TMsgKind* = enum - errUnknown, errInternal, errIllFormedAstX, errCannotOpenFile, + # fatal errors + errUnknown, errFatal, errInternal, + # non-fatal errors + errIllFormedAstX, errCannotOpenFile, errXExpected, errGridTableNotImplemented, errMarkdownIllformedTable, @@ -38,7 +41,7 @@ type errProveInit, # deadcode errGenerated, errUser, - + # warnings warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape", warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit", warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", @@ -64,7 +67,7 @@ type warnCannotOpen = "CannotOpen", warnFileChanged = "FileChanged", warnUser = "User", - + # hints hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", @@ -83,6 +86,7 @@ type const MsgKindToStr*: array[TMsgKind, string] = [ errUnknown: "unknown error", + errFatal: "fatal error: $1", errInternal: "internal error: $1", errIllFormedAstX: "illformed AST: $1", errCannotOpenFile: "cannot open '$1'", @@ -180,8 +184,7 @@ const ] const - fatalMin* = errUnknown - fatalMax* = errInternal + fatalMsgs* = {errUnknown..errInternal} errMin* = errUnknown errMax* = errUser warnMin* = warnCannotOpenFile diff --git a/compiler/modules.nim b/compiler/modules.nim index 27b93e8df..dcfcfd508 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -153,8 +153,7 @@ proc compileSystemModule*(graph: ModuleGraph) = proc wantMainModule*(conf: ConfigRef) = if conf.projectFull.isEmpty: - fatal(conf, newLineInfo(conf, AbsoluteFile(commandLineDesc), 1, 1), errGenerated, - "command expects a filename") + fatal(conf, gCmdLineInfo, "command expects a filename") conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt)) proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 6b005be3d..8b4365ec4 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -408,7 +408,7 @@ To create a stacktrace, rerun compilation with './koch temp $1 ', see $2 f quit 1 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) = - if msg >= fatalMin and msg <= fatalMax: + if msg in fatalMsgs: if conf.cmd == cmdIdeTools: log(s) quit(conf, msg) if msg >= errMin and msg <= errMax or @@ -498,6 +498,12 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, color: ForegroundColor ignoreMsg = false sev: Severity + let errorOutputsOld = conf.m.errorOutputs + if msg in fatalMsgs: + # don't gag, refs bug #7080, bug #18278; this can happen with `{.fatal.}` + # or inside a `tryConstExpr`. + conf.m.errorOutputs = {eStdOut, eStdErr} + let kind = if msg in warnMin..hintMax and msg != hintUserRaw: $msg else: "" # xxx not sure why hintUserRaw is special case msg of errMin..errMax: @@ -553,6 +559,9 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, KindFormat % $hintMsgOrigin, resetStyle, conf.unitSep) handleError(conf, msg, eh, s) + if msg in fatalMsgs: + # most likely would have died here but just in case, we restore state + conf.m.errorOutputs = errorOutputsOld template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = let arg = msgKindToString(msg) % args @@ -561,9 +570,7 @@ template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) = template rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) = liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc()) -template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") = - # this fixes bug #7080 so that it is at least obvious 'fatal' was executed. - conf.m.errorOutputs = {eStdOut, eStdErr} +template fatal*(conf: ConfigRef; info: TLineInfo, arg = "", msg = errFatal) = liMessage(conf, info, msg, arg, doAbort, instLoc()) template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") = diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 3a2d9cede..695e595c3 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1047,7 +1047,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, let s = expectStrLit(c, it) recordPragma(c, it, "error", s) localError(c.config, it.info, errUser, s) - of wFatal: fatal(c.config, it.info, errUser, expectStrLit(c, it)) + of wFatal: fatal(c.config, it.info, expectStrLit(c, it)) of wDefine: processDefine(c, it) of wUndef: processUndef(c, it) of wCompile: processCompile(c, it) From 7714ab468a665e772c6d37d39038974cbcea75dc Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 19 Jun 2021 11:24:46 -0700 Subject: [PATCH 0126/1654] make privateAccess work with generic types and generic instantiations; fix a SIGSEGV (#18260) Co-authored-by: Andreas Rumpf --- compiler/ast.nim | 20 +++++++++++++ compiler/semmagic.nim | 10 ++++--- lib/std/importutils.nim | 32 +++++++++++++------- tests/stdlib/mimportutils.nim | 15 ++++++++++ tests/stdlib/timportutils.nim | 56 +++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 96adb8c1f..8bb8de1d6 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1887,6 +1887,26 @@ proc toObject*(typ: PType): PType = if t.kind == tyRef: t.lastSon else: typ +proc toObjectFromRefPtrGeneric*(typ: PType): PType = + #[ + See also `toObject`. + Finds the underlying `object`, even in cases like these: + type + B[T] = object f0: int + A1[T] = ref B[T] + A2[T] = ref object f1: int + A3 = ref object f2: int + A4 = object f3: int + ]# + result = typ + while true: + case result.kind + of tyGenericBody: result = result.lastSon + of tyRef, tyPtr, tyGenericInst, tyGenericInvocation, tyAlias: result = result[0] + # automatic dereferencing is deep, refs #18298. + else: break + assert result.sym != nil + proc isImportedException*(t: PType; conf: ConfigRef): bool = assert t != nil diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index e4e007678..d3f26e630 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -457,6 +457,11 @@ proc semOld(c: PContext; n: PNode): PNode = localError(c.config, n[1].info, n[1].sym.name.s & " does not belong to " & getCurrOwner(c).name.s) result = n +proc semPrivateAccess(c: PContext, n: PNode): PNode = + let t = n[1].typ[0].toObjectFromRefPtrGeneric + c.currentScope.allowPrivateAccess.add t.sym + result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) + proc magicsAfterOverloadResolution(c: PContext, n: PNode, flags: TExprFlags): PNode = ## This is the preferred code point to implement magics. @@ -574,9 +579,6 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, n[0].sym.magic = mSubU result = n of mPrivateAccess: - var t = n[1].typ[0] - if t.kind in {tyRef, tyPtr}: t = t[0] - c.currentScope.allowPrivateAccess.add t.sym - result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid)) + result = semPrivateAccess(c, n) else: result = n diff --git a/lib/std/importutils.nim b/lib/std/importutils.nim index 0c0f546b9..d2da76ea8 100644 --- a/lib/std/importutils.nim +++ b/lib/std/importutils.nim @@ -13,22 +13,32 @@ Possible future APIs: ]# when defined(nimImportutilsExample): - type Foo = object - x1: int # private + type + Foo = object + f0: int # private + Goo*[T] = object + g0: int # private proc initFoo*(): auto = Foo() -proc privateAccess*(t: typedesc[object|(ref object)|(ptr object)]) {.magic: "PrivateAccess".} = +proc privateAccess*(t: typedesc) {.magic: "PrivateAccess".} = ## Enables access to private fields of `t` in current scope. runnableExamples("-d:nimImportutilsExample"): # here we're importing a module containing: - # type Foo = object - # x1: int # private + # type + # Foo = object + # f0: int # private + # Goo*[T] = object + # g0: int # private # proc initFoo*(): auto = Foo() - var a = initFoo() + var f = initFoo() block: - assert not compiles(a.x1) - privateAccess(a.type) - a.x1 = 1 # accessible in this scope + assert not compiles(f.f0) + privateAccess(f.type) + f.f0 = 1 # accessible in this scope block: - assert a.x1 == 1 # still in scope - assert not compiles(a.x1) + assert f.f0 == 1 # still in scope + assert not compiles(f.f0) + + # this also works with generics + privateAccess(Goo) + assert Goo[float](g0: 1).g0 == 1 diff --git a/tests/stdlib/mimportutils.nim b/tests/stdlib/mimportutils.nim index d2b185cd3..e89d58d27 100644 --- a/tests/stdlib/mimportutils.nim +++ b/tests/stdlib/mimportutils.nim @@ -13,5 +13,20 @@ type hd1: float PA* = ref A PtA* = ptr A + E*[T] = object + he1: int + FSub[T1, T2] = object + h3: T1 + h4: T2 + F*[T1, T2] = ref FSub[T1, T2] + G*[T] = ref E[T] + H3*[T] = object + h5: T + H2*[T] = H3[T] + H1*[T] = ref H2[T] + H*[T] = H1[T] + +type BAalias* = typeof(B.default) + # typeof is not a transparent abstraction, creates a `tyAlias` proc initB*(): B = B() diff --git a/tests/stdlib/timportutils.nim b/tests/stdlib/timportutils.nim index 37e2b7102..be912e702 100644 --- a/tests/stdlib/timportutils.nim +++ b/tests/stdlib/timportutils.nim @@ -36,6 +36,55 @@ template main = let c = C(c0: 1, hc1: 2) c.hc1 == 2 + block: + assertAll: + not compiles(E[int](he1: 1)) + privateAccess E[int] + var e = E[int](he1: 1) + e.he1 == 1 + e.he1 = 2 + e.he1 == 2 + e.he1 += 3 + e.he1 == 5 + # xxx caveat: this currently compiles but in future, we may want + # to make `privateAccess E[int]` only affect a specific instantiation; + # note that `privateAccess E` does work to cover all instantiations. + var e2 = E[float](he1: 1) + + block: + assertAll: + not compiles(E[int](he1: 1)) + privateAccess E + var e = E[int](he1: 1) + e.he1 == 1 + + block: + assertAll: + not compiles(F[int, int](h3: 1)) + privateAccess F[int, int] + var e = F[int, int](h3: 1) + e.h3 == 1 + + block: + assertAll: + not compiles(F[int, int](h3: 1)) + privateAccess F[int, int].default[].typeof + var e = F[int, int](h3: 1) + e.h3 == 1 + + block: + assertAll: + var a = G[int]() + var b = a.addr + privateAccess b.type + discard b.he1 + discard b[][].he1 + + block: + assertAll: + privateAccess H[int] + var a = H[int](h5: 2) + block: assertAll: privateAccess PA @@ -44,6 +93,13 @@ template main = pa.ha1 = 3 pa.ha1 == 3 + block: + assertAll: + var b = BAalias() + not compiles(b.hb1) + privateAccess BAalias + discard b.hb1 + block: assertAll: var a = A(a0: 1) From ad5063aed101f0102fb867f0b4e518ede7fa188b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 19 Jun 2021 14:41:43 -0700 Subject: [PATCH 0127/1654] testutils.assertAll: show correct location on assert failure (#18299) --- testament/lib/stdtest/testutils.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 6d8620dc7..a5476b8d7 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -107,10 +107,12 @@ macro assertAll*(body) = # remove this once these support VM, pending #10129 (closed but not yet fixed) result = newStmtList() for a in body: - result.add genAst(a) do: + result.add genAst(a, a2 = a.repr, info = lineInfo(a)) do: # D20210421T014713:here # xxx pending https://github.com/nim-lang/Nim/issues/12030, # `typeof` should introduce its own scope, so that this # is sufficient: `typeof(a)` instead of `typeof(block: a)` when typeof(block: a) is void: a - else: doAssert a + else: + if not a: + raise newException(AssertionDefect, info & " " & a2) From 128d21be1cdf28d6aac6a63910fabf02f186e548 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 00:49:18 -0700 Subject: [PATCH 0128/1654] add `typetraits.pointerBase` to return `T` in `ref T|ptr T` (#18293) * add typetraits.deref to return T in ref T|ptr T * deref => refBase * refBase=>pointerBase * [skip ci] address comment --- changelog.md | 15 ++++++--------- lib/pure/typetraits.nim | 10 ++++++++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index 4fd1ef355..80e38f420 100644 --- a/changelog.md +++ b/changelog.md @@ -113,8 +113,12 @@ - Make `{.requiresInit.}` pragma to work for `distinct` types. -- Added a macros `enumLen` for returning the number of items in an enum to the - `typetraits.nim` module. +- `typetraits`: + `distinctBase` now is identity instead of error for non distinct types. + Added `enumLen` to return the number of elements in an enum. + Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. + Added `hasClosure`. + Added `pointerBase` to return `T` for `ref T | ptr T`. - `prelude` now works with the JavaScript target. Added `sequtils` import to `prelude`. @@ -162,8 +166,6 @@ Added `symbolName` to return the enum symbol name ignoring the human readable name. Added `symbolRank` to return the index in which an enum member is listed in an enum. -- Added `typetraits.HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. - - Removed deprecated `iup` module from stdlib, it has already moved to [nimble](https://github.com/nim-lang/iup). @@ -293,7 +295,6 @@ - Added `algorithm.merge`. - - Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target. - Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target. @@ -322,8 +323,6 @@ - Added `hasDataBuffered` to `asyncnet`. -- Added `hasClosure` to `std/typetraits`. - - Added `std/tempfiles`. - Added `genasts.genAst` that avoids the problems inherent with `quote do` and can @@ -345,8 +344,6 @@ - nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. -- `typetraits.distinctBase` now is identity instead of error for non distinct types. - - `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; use `-d:nimLegacyCopyFile` for OSX < 10.5. diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 3e215a0ea..e63585f63 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -100,6 +100,16 @@ proc isNamedTuple*(T: typedesc): bool {.magic: "TypeTrait".} = doAssert not isNamedTuple((string, int)) doAssert isNamedTuple(tuple[name: string, age: int]) +template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = + ## Returns `T` for `ref T | ptr T`. + runnableExamples: + assert (ref int).pointerBase is int + type A = ptr seq[float] + assert A.pointerBase is seq[float] + assert (ref A).pointerBase is A # not seq[float] + assert (var s = "abc"; s[0].addr).typeof.pointerBase is char + T + proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. ## From 590d457631290beacf97d945c5d3a625cc671645 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 20 Jun 2021 10:50:03 +0300 Subject: [PATCH 0129/1654] docgen: move to shared RST state (fix #16990) (#18256) * docgen: move to shared RST state (fix #16990) * Update lib/packages/docutils/rst.nim Co-authored-by: Andreas Rumpf * Update lib/packages/docutils/rst.nim Co-authored-by: Andreas Rumpf * Update lib/packages/docutils/rst.nim Co-authored-by: Andreas Rumpf * Update compiler/docgen.nim Co-authored-by: Timothee Cour * Update compiler/docgen.nim Co-authored-by: Timothee Cour * Update compiler/docgen.nim Co-authored-by: Timothee Cour * Update lib/packages/docutils/rst.nim Co-authored-by: Timothee Cour * rename `cmdDoc2` to `cmdDoc` * fix (P)RstSharedState convention * new style of initialization * misc suggestions * 1 more rename * fix a regression Co-authored-by: Andreas Rumpf Co-authored-by: Timothee Cour --- compiler/commands.nim | 2 +- compiler/docgen.nim | 241 ++++++++++++------ compiler/docgen2.nim | 1 + compiler/main.nim | 2 +- compiler/options.nim | 6 +- lib/packages/docutils/rst.nim | 215 +++++++++------- .../expected/subdir/subdir_b/utils.idx | 10 +- tests/stdlib/trstgen.nim | 7 +- 8 files changed, 292 insertions(+), 192 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 994b224f1..8f8df71a7 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -443,7 +443,7 @@ proc parseCommand*(command: string): Command = of "check": cmdCheck of "e": cmdNimscript of "doc0": cmdDoc0 - of "doc2", "doc": cmdDoc2 + of "doc2", "doc": cmdDoc of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html of "rst2tex": cmdRst2tex diff --git a/compiler/docgen.nim b/compiler/docgen.nim index dc2a63f48..2c99adba3 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -28,6 +28,24 @@ const DocColOffset = "## ".len # assuming that a space was added after ## type + ItemFragment = object ## A fragment from each item will be eventually + ## constructed by converting `rst` fields to strings. + case isRst: bool + of true: + rst: PRstNode + of false: ## contains ready markup e.g. from runnableExamples + str: string + ItemPre = seq[ItemFragment] ## A pre-processed item. + Item = object ## Any item in documentation, e.g. symbol + ## entry. Configuration variable ``doc.item`` + ## is used for its HTML rendering. + descRst: ItemPre ## Description of the item (may contain + ## runnableExamples). + substitutions: seq[string] ## Variable names in `doc.item`... + ModSection = object ## Section like Procs, Types, etc. + secItems: seq[Item] ## Pre-processed items. + finalMarkup: string ## The items, after RST pass 2 and rendering. + ModSections = array[TSymKind, ModSection] TSections = array[TSymKind, string] ExampleGroup = ref object ## a group of runnableExamples with same rdoccmd @@ -35,17 +53,25 @@ type docCmd: string ## from user config, e.g. --doccmd:-d:foo code: string ## contains imports; each import contains `body` index: int ## group index + JsonItem = object # pre-processed item: `rst` should be finalized + json: JsonNode + rst: PRstNode + rstField: string TDocumentor = object of rstgen.RstGenerator - modDesc: string # module description + modDescPre: ItemPre # module description, not finalized + modDescFinal: string # module description, after RST pass 2 and rendering module: PSym modDeprecationMsg: string - toc, toc2, section: TSections + section: ModSections # entries of ``.nim`` file (for `proc`s, etc) + toc, toc2: TSections # toc2 - grouped TOC tocTable: array[TSymKind, Table[string, string]] indexValFilename: string analytics: string # Google Analytics javascript, "" if doesn't exist seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. - jArray: JsonNode + jEntriesPre: seq[JsonItem] # pre-processed RST + JSON content + jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable + sharedState: PRstSharedState isPureRst: bool conf*: ConfigRef cache*: IdentCache @@ -58,6 +84,9 @@ type PDoc* = ref TDocumentor ## Alias to type less. +proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) +proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) + proc prettyString(a: object): string = # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906 for k, v in fieldPairs(a): @@ -151,12 +180,12 @@ template declareClosures = if not fileExists(result): result = "" proc parseRst(text, filename: string, - line, column: int, hasToc: var bool, + line, column: int, rstOptions: RstParseOptions; - conf: ConfigRef): PRstNode = + conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() - result = rstParse(text, filename, line, column, hasToc, rstOptions, - docgenFindFile, compilerMsgHandler) + result = rstParsePass1(text, filename, line, column, rstOptions, + sharedState) proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, guessTarget: bool): AbsoluteFile = @@ -179,10 +208,13 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.conf = conf result.cache = cache result.outDir = conf.outDir.string + const options = {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown, roNimFile} + result.sharedState = newRstSharedState( + options, filename.string, + docgenFindFile, compilerMsgHandler) initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml), - conf.configVars, filename.string, - {roSupportRawDirective, roSupportMarkdown, - roPreferMarkdown, roNimFile}, + conf.configVars, filename.string, options, docgenFindFile, compilerMsgHandler) if conf.configVars.hasKey("doc.googleAnalytics"): @@ -203,7 +235,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.seenSymbols = newStringTable(modeCaseInsensitive) result.id = 100 - result.jArray = newJArray() + result.jEntriesFinal = newJArray() initStrTable result.types result.onTestSnippet = proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) = @@ -261,37 +293,31 @@ proc getVarIdx(varnames: openArray[string], id: string): int = return i result = -1 -proc genComment(d: PDoc, n: PNode): string = - result = "" +proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - let comment = n.comment - when false: - # RFC: to preseve newlines in comments, this would work: - comment = comment.replace("\n", "\n\n") - renderRstToOut(d[], - parseRst(comment, toFullPath(d.conf, n.info), - toLinenumber(n.info), - toColumn(n.info) + DocColOffset, - (var dummy: bool; dummy), d.options, d.conf), - result) + result = parseRst(n.comment, toFullPath(d.conf, n.info), + toLinenumber(n.info), + toColumn(n.info) + DocColOffset, + d.options, d.conf, + d.sharedState) -proc genRecCommentAux(d: PDoc, n: PNode): string = - if n == nil: return "" +proc genRecCommentAux(d: PDoc, n: PNode): PRstNode = + if n == nil: return nil result = genComment(d, n) - if result == "": + if result == nil: if n.kind in {nkStmtList, nkStmtListExpr, nkTypeDef, nkConstDef, nkObjectTy, nkRefTy, nkPtrTy, nkAsgn, nkFastAsgn, nkHiddenStdConv}: # notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: for i in 0.. 0: msg.add " cmd: " & rdoccmd - dispA(d.conf, dest, "\n

$1

\n", + var s: string + dispA(d.conf, s, "\n

$1

\n", "\n\n\\textbf{$1}\n", [msg]) + dest.add s inc d.listingCounter let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim", ""]) @@ -597,7 +625,7 @@ proc getRoutineBody(n: PNode): PNode = doAssert result.len == 2 result = result[1] -proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var string) = +proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var ItemPre) = var n = n var state = rsStart template fn(n2, topLevel) = @@ -706,7 +734,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string = ## types will be added with a preceding dash. Return types won't be added. ## ## If you modify the output of this proc, please update the anchor generation - ## section of ``doc/docgen.txt``. + ## section of ``doc/docgen.rst``. result = baseName case k of skProc, skFunc: discard @@ -804,7 +832,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = var result = "" var literal, plainName = "" var kind = tkEof - var comm = "" + var comm: ItemPre if n.kind in routineDefs: getAllRunnableExamples(d, n, comm) else: @@ -840,9 +868,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) - d.section[k].add(getConfigVar(d.conf, "doc.item") % - ["name", name, "uniqueName", uniqueName, - "header", result, "desc", comm, "itemID", $d.id, + d.section[k].secItems.add Item(descRst: comm, substitutions: @[ + "name", name, "uniqueName", uniqueName, + "header", result, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, @@ -886,25 +914,26 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if k == skType and nameNode.kind == nkSym: d.types.strTableAdd nameNode.sym -proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = +proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem = if not isVisible(d, nameNode): return var name = getName(d, nameNode) comm = genRecComment(d, n) r: TSrcGen initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments}) - result = %{ "name": %name, "type": %($k), "line": %n.info.line.int, - "col": %n.info.col} - if comm.len > 0: - result["description"] = %comm + result.json = %{ "name": %name, "type": %($k), "line": %n.info.line.int, + "col": %n.info.col} + if comm != nil: + result.rst = comm + result.rstField = "description" if r.buf.len > 0: - result["code"] = %r.buf + result.json["code"] = %r.buf if k in routineKinds: - result["signature"] = newJObject() + result.json["signature"] = newJObject() if n[paramsPos][0].kind != nkEmpty: - result["signature"]["return"] = %($n[paramsPos][0]) + result.json["signature"]["return"] = %($n[paramsPos][0]) if n[paramsPos].len > 1: - result["signature"]["arguments"] = newJArray() + result.json["signature"]["arguments"] = newJArray() for paramIdx in 1 ..< n[paramsPos].len: for identIdx in 0 ..< n[paramsPos][paramIdx].len - 2: let @@ -912,22 +941,22 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = paramType = $n[paramsPos][paramIdx][^2] if n[paramsPos][paramIdx][^1].kind != nkEmpty: let paramDefault = $n[paramsPos][paramIdx][^1] - result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} + result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType, "default": %paramDefault} else: - result["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} + result.json["signature"]["arguments"].add %{"name": %paramName, "type": %paramType} if n[pragmasPos].kind != nkEmpty: - result["signature"]["pragmas"] = newJArray() + result.json["signature"]["pragmas"] = newJArray() for pragma in n[pragmasPos]: - result["signature"]["pragmas"].add %($pragma) + result.json["signature"]["pragmas"].add %($pragma) if n[genericParamsPos].kind != nkEmpty: - result["signature"]["genericParams"] = newJArray() + result.json["signature"]["genericParams"] = newJArray() for genericParam in n[genericParamsPos]: var param = %{"name": %($genericParam)} if genericParam.sym.typ.sons.len > 0: param["types"] = newJArray() for kind in genericParam.sym.typ.sons: param["types"].add %($kind) - result["signature"]["genericParams"].add param + result.json["signature"]["genericParams"].add param proc checkForFalse(n: PNode): bool = result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 @@ -946,8 +975,8 @@ proc traceDeps(d: PDoc, it: PNode) = traceDeps(d, a) elif it.kind == nkSym and belongsToPackage(d.conf, it.sym): let external = externalDep(d, it.sym) - if d.section[k] != "": d.section[k].add(", ") - dispA(d.conf, d.section[k], + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, external.prettyLink), changeFileExt(external, "html")]) @@ -956,8 +985,8 @@ proc exportSym(d: PDoc; s: PSym) = const k = exportSection if s.kind == skModule and belongsToPackage(d.conf, s): let external = externalDep(d, s) - if d.section[k] != "": d.section[k].add(", ") - dispA(d.conf, d.section[k], + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, external.prettyLink), changeFileExt(external, "html")]) @@ -968,9 +997,9 @@ proc exportSym(d: PDoc; s: PSym) = complexSymbol = complexName(s.kind, s.ast, s.name.s) symbolOrId = d.newUniquePlainSymbol(complexSymbol) external = externalDep(d, module) - if d.section[k] != "": d.section[k].add(", ") + if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") # XXX proper anchor generation here - dispA(d.conf, d.section[k], + dispA(d.conf, d.section[k].finalMarkup, "$1", "$1", [esc(d.target, s.name.s), changeFileExt(external, "html"), @@ -1034,13 +1063,16 @@ proc documentRaises*(cache: IdentCache; n: PNode) = if p5 != nil: n[pragmasPos].add p5 proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = + ## Goes through nim nodes recursively and collects doc comments. + ## Main function for `doc`:option: command, + ## which is implemented in ``docgen2.nim``. template genItemAux(skind) = genItem(d, n, n[namePos], skind, docFlags) case n.kind of nkPragma: let pragmaNode = findPragma(n, wDeprecated) d.modDeprecationMsg.add(genDeprecationMsg(d, pragmaNode)) - of nkCommentStmt: d.modDesc.add(genComment(d, n)) + of nkCommentStmt: d.modDescPre.add(genComment(d, n)) of nkProcDef, nkFuncDef: when useEffectSystem: documentRaises(d.cache, n) genItemAux(skProc) @@ -1079,21 +1111,63 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept" of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0]) of nkCallKinds: - var comm = "" + var comm: ItemPre getAllRunnableExamples(d, n, comm) - if comm != "": d.modDesc.add(comm) + if comm.len != 0: d.modDescPre.add(comm) else: discard -proc add(d: PDoc; j: JsonNode) = - if j != nil: d.jArray.add j +proc finishGenerateDoc*(d: var PDoc) = + ## Perform 2nd RST pass for resolution of links/footnotes/headings... + # Main title/subtitle are allowed only in the first RST fragment of document + var firstRst = PRstNode(nil) + for fragment in d.modDescPre: + if fragment.isRst: + firstRst = fragment.rst + break + preparePass2(d.sharedState, firstRst) + + # Finalize fragments of ``.nim`` or ``.rst`` file + proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = + for f in fragments: + case f.isRst: + of true: + var resolved = resolveSubs(d.sharedState, f.rst) + renderRstToOut(d[], resolved, result) + of false: result &= f.str + for k in TSymKind: + for item in d.section[k].secItems: + var itemDesc: string + renderItemPre(d, item.descRst, itemDesc) + d.section[k].finalMarkup.add( + getConfigVar(d.conf, "doc.item") % ( + item.substitutions & @["desc", itemDesc])) + itemDesc = "" + d.section[k].secItems.setLen 0 + renderItemPre(d, d.modDescPre, d.modDescFinal) + d.modDescPre.setLen 0 + d.hasToc = d.hasToc or d.sharedState.hasToc + + # Finalize fragments of ``.json`` file + for i, entry in d.jEntriesPre: + if entry.rst != nil: + let resolved = resolveSubs(d.sharedState, entry.rst) + var str: string + renderRstToOut(d[], resolved, str) + entry.json[entry.rstField] = %str + d.jEntriesFinal.add entry.json + d.jEntriesPre[i].rst = nil + +proc add(d: PDoc; j: JsonItem) = + if j.json != nil or j.rst != nil: d.jEntriesPre.add j proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) = case n.kind of nkCommentStmt: if includeComments: - d.add %*{"comment": genComment(d, n)} + d.add JsonItem(rst: genComment(d, n), rstField: "comment", + json: %Table[string, string]()) else: - d.modDesc.add(genComment(d, n)) + d.modDescPre.add(genComment(d, n)) of nkProcDef, nkFuncDef: when useEffectSystem: documentRaises(d.cache, n) d.add genJsonItem(d, n, n[namePos], skProc) @@ -1173,11 +1247,11 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) = "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs", "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports" ] - if d.section[kind] == "": return + if d.section[kind].finalMarkup == "": return var title = sectionNames[kind] - d.section[kind] = getConfigVar(d.conf, "doc.section") % [ + d.section[kind].finalMarkup = getConfigVar(d.conf, "doc.section") % [ "sectionid", $ord(kind), "sectionTitle", title, - "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind]] + "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup] var tocSource = d.toc if groupedToc: @@ -1209,7 +1283,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = if toc != "" or d.target == outLatex: # for Latex $doc.toc will automatically generate TOC if `d.hasToc` is set toc = getConfigVar(d.conf, "doc.toc") % ["content", toc] - for i in TSymKind: code.add(d.section[i]) + for i in TSymKind: code.add(d.section[i].finalMarkup) # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: @@ -1234,7 +1308,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = let seeSrc = genSeeSrc(d, d.filename, 1) content = getConfigVar(d.conf, bodyname) % [ "title", title, "subtitle", subtitle, - "tableofcontents", toc, "moduledesc", d.modDesc, "date", getDateStr(), + "tableofcontents", toc, "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(), "content", code, "deprecationMsg", d.modDeprecationMsg, "theindexhref", relLink(d.conf.outDir, d.destFile.AbsoluteFile, @@ -1248,7 +1322,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = "dochackjs", relLink(d.conf.outDir, d.destFile.AbsoluteFile, docHackJsFname.RelativeFile), "title", title, "subtitle", subtitle, "tableofcontents", toc, - "moduledesc", d.modDesc, "date", getDateStr(), "time", getClockStr(), + "moduledesc", d.modDescFinal, "date", getDateStr(), "time", getClockStr(), "content", content, "author", d.meta[metaAuthor], "version", esc(d.target, d.meta[metaVersion]), "analytics", d.analytics, "deprecationMsg", d.modDeprecationMsg] @@ -1297,12 +1371,12 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = proc writeOutputJson*(d: PDoc, useWarning = false) = runAllExamples(d) var modDesc: string - for desc in d.modDesc: + for desc in d.modDescFinal: modDesc &= desc let content = %*{"orig": d.filename, "nimble": getPackageName(d.conf, d.filename), "moduleDescription": modDesc, - "entries": d.jArray} + "entries": d.jEntriesFinal} if optStdout in d.conf.globalOptions: write(stdout, $content) else: @@ -1324,12 +1398,14 @@ proc handleDocOutputOptions*(conf: ConfigRef) = conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) proc commandDoc*(cache: IdentCache, conf: ConfigRef) = + ## implementation of deprecated ``doc0`` command (without semantic checking) handleDocOutputOptions conf var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) d.hasToc = true generateDoc(d, ast, ast) + finishGenerateDoc(d) writeOutput(d) generateIndex(d) @@ -1339,14 +1415,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; var d = newDocumentor(filen, cache, conf, outExt) d.isPureRst = true - var rst = parseRst(readFile(filen.string), filen.string, + let rst = parseRst(readFile(filen.string), filen.string, line=LineRstInit, column=ColRstInit, - d.hasToc, - {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}, - conf) - var modDesc = newStringOfCap(30_000) - renderRstToOut(d[], rst, modDesc) - d.modDesc = modDesc + {roSupportRawDirective, roSupportMarkdown, + roPreferMarkdown}, conf, + d.sharedState) + d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] + finishGenerateDoc(d) writeOutput(d) generateIndex(d) @@ -1357,6 +1432,7 @@ proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) = commandRstAux(cache, conf, conf.projectFull, TexExt) proc commandJson*(cache: IdentCache, conf: ConfigRef) = + ## implementation of a deprecated jsondoc0 command var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) @@ -1366,7 +1442,8 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) = warnUser, "the ':test:' attribute is not supported by this backend") d.hasToc = true generateJson(d, ast) - let json = d.jArray + finishGenerateDoc(d) + let json = d.jEntriesFinal let content = pretty(json) if optStdout in d.conf.globalOptions: diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index d077ddc67..bfdb4568c 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -31,6 +31,7 @@ template closeImpl(body: untyped) {.dirty.} = let useWarning = sfMainModule notin g.module.flags let groupedToc = true if shouldProcess(g): + finishGenerateDoc(g.doc) body try: generateIndex(g.doc) diff --git a/compiler/main.nim b/compiler/main.nim index 7824e8cb2..1c8034c6c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -282,7 +282,7 @@ proc mainCommand*(graph: ModuleGraph) = else: rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc") of cmdDoc0: docLikeCmd commandDoc(cache, conf) - of cmdDoc2: + of cmdDoc: docLikeCmd(): conf.setNoteDefaults(warnLockLevel, false) # issue #13218 conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218 diff --git a/compiler/options.nim b/compiler/options.nim index 05d8ee5b7..ffb0f4501 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -143,8 +143,8 @@ type cmdIdeTools # ide tools (e.g. nimsuggest) cmdNimscript # evaluate nimscript cmdDoc0 - cmdDoc2 - cmdDoc2tex + cmdDoc # convert .nim doc comments to HTML + cmdDoc2tex # convert .nim doc comments to LaTeX cmdRst2html # convert a reStructuredText file to HTML cmdRst2tex # convert a reStructuredText file to TeX cmdJsondoc0 @@ -161,7 +161,7 @@ type const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToJS, cmdCrun} - cmdDocLike* = {cmdDoc0, cmdDoc2, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, + cmdDocLike* = {cmdDoc0, cmdDoc, cmdDoc2tex, cmdJsondoc0, cmdJsondoc, cmdCtags, cmdBuildindex} type diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 336caa2df..031a86d98 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -441,7 +441,7 @@ proc rawGetTok(L: var Lexer, tok: var Token) = inc L.col tok.col = max(tok.col - L.baseIndent, 0) -proc getTokens(buffer: string, tokens: var TokenSeq): int = +proc getTokens(buffer: string, tokens: var TokenSeq) = var L: Lexer var length = tokens.len L.buf = cstring(buffer) @@ -489,7 +489,7 @@ type autoSymIdx: int # order of occurence: fnAutoSymbol label: string # valid for fnAutoNumberLabel - SharedState = object + RstSharedState = object options: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles hTitleCnt: int # =0 if no title, =1 if only main title, @@ -498,8 +498,8 @@ type currRole: string # current interpreted text role currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions - refs: seq[Substitution] # references - anchors: seq[AnchorSubst] # internal target substitutions + refs*: seq[Substitution] # references + anchors*: seq[AnchorSubst] # internal target substitutions lineFootnoteNum: seq[int] # footnote line, auto numbers .. [#] lineFootnoteNumRef: seq[int] # footnote line, their reference [#]_ lineFootnoteSym: seq[int] # footnote line, auto symbols .. [*] @@ -508,19 +508,19 @@ type # number, order of occurrence msgHandler: MsgHandler # How to handle errors. findFile: FindFileHandler # How to find files. + filename: string + hasToc*: bool - PSharedState = ref SharedState + PRstSharedState* = ref RstSharedState RstParser = object of RootObj idx*: int tok*: TokenSeq - s*: PSharedState + s*: PRstSharedState indentStack*: seq[int] - filename*: string line*, col*: int ## initial line/column of whole text or ## documenation fragment that will be added ## in case of error/warning reporting to ## (relative) line/column of the token. - hasToc*: bool curAnchor*: string # variable to track latest anchor in s.anchors EParseError* = object of ValueError @@ -578,35 +578,42 @@ proc whichRoleAux(sym: string): RstNodeKind = else: # unknown role result = rnUnknownRole -proc newSharedState(options: RstParseOptions, - findFile: FindFileHandler, - msgHandler: MsgHandler): PSharedState = - new(result) - result.currRole = defaultRole(options) - result.currRoleKind = whichRoleAux(result.currRole) - result.subs = @[] - result.refs = @[] - result.options = options - result.msgHandler = if not isNil(msgHandler): msgHandler else: defaultMsgHandler - result.findFile = if not isNil(findFile): findFile else: defaultFindFile +proc newRstSharedState*(options: RstParseOptions, + filename: string, + findFile: FindFileHandler, + msgHandler: MsgHandler): PRstSharedState = + let r = defaultRole(options) + result = PRstSharedState( + currRole: r, + currRoleKind: whichRoleAux(r), + options: options, + msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, + filename: filename, + findFile: if not isNil(findFile): findFile else: defaultFindFile + ) proc curLine(p: RstParser): int = p.line + currentTok(p).line proc findRelativeFile(p: RstParser; filename: string): string = - result = p.filename.splitFile.dir / filename + result = p.s.filename.splitFile.dir / filename if not fileExists(result): result = p.s.findFile(filename) proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = - p.s.msgHandler(p.filename, curLine(p), + p.s.msgHandler(p.s.filename, curLine(p), p.col + currentTok(p).col, msgKind, arg) +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = + ## Print warnings for footnotes/substitutions. + ## TODO: their line/column info is not known, to fix it. + s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg) + proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) = - p.s.msgHandler(p.filename, p.line + line, + p.s.msgHandler(p.s.filename, p.line + line, p.col + col, msgKind, arg) proc rstMessage(p: RstParser, msgKind: MsgKind) = - p.s.msgHandler(p.filename, curLine(p), + p.s.msgHandler(p.s.filename, curLine(p), p.col + currentTok(p).col, msgKind, currentTok(p).symbol) @@ -680,12 +687,10 @@ proc popInd(p: var RstParser) = # whether it should continue its processing or not, and decided not to, # then this B.E. handler should step back (e.g. do `dec p.idx`). -proc initParser(p: var RstParser, sharedState: PSharedState) = +proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.indentStack = @[0] p.tok = @[] p.idx = 0 - p.filename = "" - p.hasToc = false p.col = ColRstInit p.line = LineRstInit p.s = sharedState @@ -754,14 +759,14 @@ proc rstnodeToRefname(n: PRstNode): string = var b = false rstnodeToRefnameAux(n, result, b) -proc findSub(p: var RstParser, n: PRstNode): int = +proc findSub(s: PRstSharedState, n: PRstNode): int = var key = addNodes(n) # the spec says: if no exact match, try one without case distinction: - for i in countup(0, high(p.s.subs)): - if key == p.s.subs[i].key: + for i in countup(0, high(s.subs)): + if key == s.subs[i].key: return i - for i in countup(0, high(p.s.subs)): - if cmpIgnoreStyle(key, p.s.subs[i].key) == 0: + for i in countup(0, high(s.subs)): + if cmpIgnoreStyle(key, s.subs[i].key) == 0: return i result = -1 @@ -783,10 +788,10 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) = return p.s.refs.add(Substitution(key: key, value: value)) -proc findRef(p: var RstParser, key: string): PRstNode = - for i in countup(0, high(p.s.refs)): - if key == p.s.refs[i].key: - return p.s.refs[i].value +proc findRef(s: PRstSharedState, key: string): PRstNode = + for i in countup(0, high(s.refs)): + if key == s.refs[i].key: + return s.refs[i].value proc addAnchor(p: var RstParser, refn: string, reset: bool) = ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` @@ -800,8 +805,8 @@ proc addAnchor(p: var RstParser, refn: string, reset: bool) = else: p.curAnchor = refn -proc findMainAnchor(p: RstParser, refn: string): string = - for subst in p.s.anchors: +proc findMainAnchor(s: PRstSharedState, refn: string): string = + for subst in s.anchors: if subst.mainAnchor == refn: # no need to rename result = subst.mainAnchor break @@ -838,28 +843,28 @@ proc addFootnoteSymAuto(p: var RstParser) = p.s.lineFootnoteSym.add curLine(p) p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, "")) -proc orderFootnotes(p: var RstParser) = +proc orderFootnotes(s: PRstSharedState) = ## numerate auto-numbered footnotes taking into account that all ## manually numbered ones always have preference. - ## Save the result back to p.s.footnotes. + ## Save the result back to `s.footnotes`. # Report an error if found any mismatch in number of automatic footnotes proc listFootnotes(lines: seq[int]): string = result.add $lines.len & " (lines " & join(lines, ", ") & ")" - if p.s.lineFootnoteNum.len != p.s.lineFootnoteNumRef.len: - rstMessage(p, meFootnoteMismatch, - "$1 != $2" % [listFootnotes(p.s.lineFootnoteNum), - listFootnotes(p.s.lineFootnoteNumRef)] & + if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteNum), + listFootnotes(s.lineFootnoteNumRef)] & " for auto-numbered footnotes") - if p.s.lineFootnoteSym.len != p.s.lineFootnoteSymRef.len: - rstMessage(p, meFootnoteMismatch, - "$1 != $2" % [listFootnotes(p.s.lineFootnoteSym), - listFootnotes(p.s.lineFootnoteSymRef)] & + if s.lineFootnoteSym.len != s.lineFootnoteSymRef.len: + rstMessage(s, meFootnoteMismatch, + "$1 != $2" % [listFootnotes(s.lineFootnoteSym), + listFootnotes(s.lineFootnoteSymRef)] & " for auto-symbol footnotes") var result: seq[FootnoteSubst] var manuallyN, autoN, autoSymbol: seq[FootnoteSubst] - for fs in p.s.footnotes: + for fs in s.footnotes: if fs.kind == fnManualNumber: manuallyN.add fs elif fs.kind in {fnAutoNumber, fnAutoNumberLabel}: autoN.add fs else: autoSymbol.add fs @@ -906,26 +911,26 @@ proc orderFootnotes(p: var RstParser) = let label = footnoteAutoSymbols[symbolNum].repeat(nSymbols) result.add((fs.kind, -1, -1, fs.autoSymIdx, label)) - p.s.footnotes = result + s.footnotes = result -proc getFootnoteNum(p: var RstParser, label: string): int = +proc getFootnoteNum(s: PRstSharedState, label: string): int = ## get number from label. Must be called after `orderFootnotes`. result = -1 - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.label == label: return fnote.number -proc getFootnoteNum(p: var RstParser, order: int): int = +proc getFootnoteNum(s: PRstSharedState, order: int): int = ## get number from occurrence. Must be called after `orderFootnotes`. result = -1 - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.autoNumIdx == order: return fnote.number -proc getAutoSymbol(p: var RstParser, order: int): string = +proc getAutoSymbol(s: PRstSharedState, order: int): string = ## get symbol from occurrence of auto-symbol footnote. result = "???" - for fnote in p.s.footnotes: + for fnote in s.footnotes: if fnote.autoSymIdx == order: return fnote.label @@ -1765,17 +1770,18 @@ proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = line: curLine(p), hasPeers: false) result = p.s.hLevels.len - 1 -proc countTitles(p: var RstParser, n: PRstNode) = - ## Fill `p.s.hTitleCnt` +proc countTitles(s: PRstSharedState, n: PRstNode) = + ## Fill `s.hTitleCnt` + if n == nil: return for node in n.sons: if node != nil: if node.kind notin {rnOverline, rnSubstitutionDef, rnDefaultRole}: break if node.kind == rnOverline: - if p.s.hLevels[p.s.hTitleCnt].hasPeers: + if s.hLevels[s.hTitleCnt].hasPeers: break - inc p.s.hTitleCnt - if p.s.hTitleCnt >= 2: + inc s.hTitleCnt + if s.hTitleCnt >= 2: break proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool = @@ -2103,8 +2109,7 @@ proc parseSimpleTable(p: var RstParser): PRstNode = initParser(q, p.s) q.col = cols[j] q.line = line - 1 - q.filename = p.filename - q.col += getTokens(row[j], q.tok) + getTokens(row[j], q.tok) b = newRstNode(rnTableDataCell) b.add(parseDoc(q)) a.add(b) @@ -2156,8 +2161,7 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = initParser(q, p.s) q.col = p.col q.line = currentTok(p).line - 1 - q.filename = p.filename - q.col += getTokens(getColContents(p, row[j]), q.tok) + getTokens(getColContents(p, row[j]), q.tok) b.add(parseDoc(q)) a.add(b) result.add(a) @@ -2562,8 +2566,8 @@ proc dirInclude(p: var RstParser): PRstNode = var q: RstParser initParser(q, p.s) - q.filename = path - q.col += getTokens( + q.s.filename = path + getTokens( inputString[startPosition..endPosition].strip(), q.tok) # workaround a GCC bug; more like the interior pointer bug? @@ -2806,7 +2810,27 @@ proc parseDotDot(p: var RstParser): PRstNode = else: result = parseComment(p) -proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = +proc rstParsePass1*(fragment, filename: string, + line, column: int, + options: RstParseOptions, + sharedState: PRstSharedState): PRstNode = + ## Parses an RST `fragment`. + ## The result should be further processed by + ## `preparePass2` and `resolveSubs` (which is pass 2). + var p: RstParser + initParser(p, sharedState) + p.line = line + p.col = column + getTokens(fragment, p.tok) + result = parseDoc(p) + +proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = + ## Records titles in node `mainNode` and orders footnotes. + countTitles(s, mainNode) + orderFootnotes(s) + +proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = + ## Makes pass 2 of RST parsing. ## Resolves substitutions and anchor aliases, groups footnotes. ## Takes input node `n` and returns the same node with recursive ## substitutions in `n.sons` to `result`. @@ -2814,32 +2838,32 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = if n == nil: return case n.kind of rnSubstitutionReferences: - var x = findSub(p, n) + var x = findSub(s, n) if x >= 0: - result = p.s.subs[x].value + result = s.subs[x].value else: var key = addNodes(n) var e = getEnv(key) if e != "": result = newLeaf(e) - else: rstMessage(p, mwUnknownSubstitution, key) + else: rstMessage(s, mwUnknownSubstitution, key) of rnHeadline, rnOverline: # fix up section levels depending on presence of a title and subtitle - if p.s.hTitleCnt == 2: + if s.hTitleCnt == 2: if n.level == 1: # it's the subtitle n.level = 0 elif n.level >= 2: # normal sections n.level -= 1 - elif p.s.hTitleCnt == 0: + elif s.hTitleCnt == 0: n.level += 1 of rnRef: let refn = rstnodeToRefname(n) - var y = findRef(p, refn) + var y = findRef(s, refn) if y != nil: result = newRstNode(rnHyperlink) let text = newRstNode(rnInner, n.sons) result.sons = @[text, y] else: - let s = findMainAnchor(p, refn) + let s = findMainAnchor(s, refn) if s != "": result = newRstNode(rnInternalRef) let text = newRstNode(rnInner, n.sons) @@ -2853,16 +2877,16 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = of fnAutoNumberLabel, fnAutoNumber: if fnType == fnAutoNumberLabel: let labelR = rstnodeToRefname(n.sons[0]) - num = getFootnoteNum(p, labelR) + num = getFootnoteNum(s, labelR) else: - num = getFootnoteNum(p, n.order) + num = getFootnoteNum(s, n.order) var nn = newRstNode(rnInner) nn.add newLeaf($num) result.sons[0] = nn of fnAutoSymbol: - let sym = getAutoSymbol(p, n.order) + let sym = getAutoSymbol(s, n.order) n.sons[0].sons[0].text = sym - n.sons[1] = resolveSubs(p, n.sons[1]) + n.sons[1] = resolveSubs(s, n.sons[1]) of rnFootnoteRef: var (fnType, num) = getFootnoteType(n.sons[0]) template addLabel(number: int | string) = @@ -2877,31 +2901,31 @@ proc resolveSubs(p: var RstParser, n: PRstNode): PRstNode = addLabel num refn.add $num of fnAutoNumber: - addLabel getFootnoteNum(p, n.order) + addLabel getFootnoteNum(s, n.order) refn.add $n.order of fnAutoNumberLabel: - addLabel getFootnoteNum(p, rstnodeToRefname(n)) + addLabel getFootnoteNum(s, rstnodeToRefname(n)) refn.add rstnodeToRefname(n) of fnAutoSymbol: - addLabel getAutoSymbol(p, n.order) + addLabel getAutoSymbol(s, n.order) refn.add $n.order of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) - let s = findMainAnchor(p, refn) - if s != "": - result.add newLeaf(s) # add link + let anch = findMainAnchor(s, refn) + if anch != "": + result.add newLeaf(anch) # add link else: - rstMessage(p, mwUnknownSubstitution, refn) + rstMessage(s, mwUnknownSubstitution, refn) result.add newLeaf(refn) # add link of rnLeaf: discard of rnContents: - p.hasToc = true + s.hasToc = true else: var regroup = false for i in 0 ..< n.len: - n.sons[i] = resolveSubs(p, n.sons[i]) + n.sons[i] = resolveSubs(s, n.sons[i]) if n.sons[i] != nil and n.sons[i].kind == rnFootnote: regroup = true if regroup: # group footnotes together into rnFootnoteGroup @@ -2924,13 +2948,10 @@ proc rstParse*(text, filename: string, options: RstParseOptions, findFile: FindFileHandler = nil, msgHandler: MsgHandler = nil): PRstNode = - var p: RstParser - initParser(p, newSharedState(options, findFile, msgHandler)) - p.filename = filename - p.line = line - p.col = column + getTokens(text, p.tok) - let unresolved = parseDoc(p) - countTitles(p, unresolved) - orderFootnotes(p) - result = resolveSubs(p, unresolved) - hasToc = p.hasToc + ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`. + var sharedState = newRstSharedState(options, filename, findFile, msgHandler) + let unresolved = rstParsePass1(text, filename, line, column, + options, sharedState) + preparePass2(sharedState, unresolved) + result = resolveSubs(sharedState, unresolved) + hasToc = sharedState.hasToc diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index b49a777c8..6dc3953af 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,8 +1,3 @@ -This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header -Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header -And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on -More headers subdir/subdir_b/utils.html#more-headers More headers -Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC @@ -11,3 +6,8 @@ someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header +Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header +And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on +More headers subdir/subdir_b/utils.html#more-headers More headers +Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 34b172935..300229648 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -997,7 +997,7 @@ Test1 """ var error5 = new string let output5 = input5.toHtml(error=error5) - check(error5[] == "input(6, 1) Error: mismatch in number of footnotes " & + check(error5[] == "input(1, 1) Error: mismatch in number of footnotes " & "and their refs: 1 (lines 2) != 0 (lines ) for auto-numbered " & "footnotes") @@ -1011,7 +1011,7 @@ Test1 """ var error6 = new string let output6 = input6.toHtml(error=error6) - check(error6[] == "input(6, 1) Error: mismatch in number of footnotes " & + check(error6[] == "input(1, 1) Error: mismatch in number of footnotes " & "and their refs: 1 (lines 3) != 2 (lines 2, 6) for auto-symbol " & "footnotes") @@ -1034,7 +1034,8 @@ Test1 """ var warnings8 = new seq[string] let output8 = input8.toHtml(warnings=warnings8) - check(warnings8[] == @["input(4, 1) Warning: unknown substitution " & + # TODO: the line 1 is arbitrary because reference lines are not preserved + check(warnings8[] == @["input(1, 1) Warning: unknown substitution " & "\'citation-som\'"]) # check that footnote group does not break parsing of other directives: From 6030e139b5c630b5508dbe12d7b58c56264212b1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 00:51:07 -0700 Subject: [PATCH 0130/1654] move {.injectStmt.} to experimental; add a test (#18300) * move {.injectStmt.} to experimental; add a test * undocument and deprecat `.injectStmt` but keep its implementation until we have a replacement --- compiler/pragmas.nim | 5 ++-- doc/manual.rst | 11 -------- doc/manual_experimental.rst | 1 + lib/pure/asyncdispatch.nim | 2 -- lib/system/gc_ms.nim | 1 + tests/pragmas/tinjectstmt.nim | 48 +++++++++++++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 tests/pragmas/tinjectstmt.nim diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 695e595c3..8ae931d84 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -1183,6 +1183,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, if sym == nil: invalidPragma(c, it) else: magicsys.registerNimScriptSymbol(c.graph, sym) of wInjectStmt: + warningDeprecated(c.config, it.info, "'.injectStmt' pragma is deprecated") if it.kind notin nkPragmaCallKinds or it.len != 2: localError(c.config, it.info, "expression expected") else: @@ -1194,10 +1195,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wThis: if it.kind in nkPragmaCallKinds and it.len == 2: c.selfName = considerQuotedIdent(c, it[1]) - message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated") + message(c.config, n.info, warnDeprecated, "'.this' pragma is deprecated") elif it.kind == nkIdent or it.len == 1: c.selfName = getIdent(c.cache, "self") - message(c.config, n.info, warnDeprecated, "the '.this' pragma is deprecated") + message(c.config, n.info, warnDeprecated, "'.this' pragma is deprecated") else: localError(c.config, it.info, "'this' pragma is allowed to have zero or one arguments") of wNoRewrite: diff --git a/doc/manual.rst b/doc/manual.rst index 46a40a690..e8eec9a57 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7432,17 +7432,6 @@ work properly (in particular regarding constructor and destructor) for proc main()= var a {.threadvar.}: Foo -InjectStmt pragma ------------------ - -The `injectStmt` pragma can be used to inject a statement before every -other statement in the current module. It is only supposed to be used for -debugging: - -.. code-block:: nim - {.injectStmt: gcInvariants().} - - # ... complex code here that produces crashes ... compile-time define pragmas --------------------------- diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 3157708da..d10ef8d1b 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -380,6 +380,7 @@ pass multiple blocks to a macro: # code to undo it + Special Operators ================= diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 3866ebd24..b88d5ede8 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -176,8 +176,6 @@ export Port, SocketFlag export asyncfutures except callSoon export asyncstreams -#{.injectStmt: newGcInvariant().} - # TODO: Check if yielded future is nil and throw a more meaningful exception type diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index c20e93699..9e306c497 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -429,6 +429,7 @@ proc sweep(gch: var GcHeap) = else: freeCyclicCell(gch, c) when false: + # meant to be used with the now-deprected `.injectStmt`: {.injectStmt: newGcInvariant().} proc newGcInvariant*() = for x in allObjects(gch.region): if isCell(x): diff --git a/tests/pragmas/tinjectstmt.nim b/tests/pragmas/tinjectstmt.nim new file mode 100644 index 000000000..bca041e46 --- /dev/null +++ b/tests/pragmas/tinjectstmt.nim @@ -0,0 +1,48 @@ +discard """ + joinable: false + output:''' +onInject: 1 +onInject: 2 +ok0 +ok1 +onInject: 3 +onInject: 4 +0 +onInject: 5 +onInject: 6 +1 +onInject: 7 +onInject: 8 +2 +ok2 +onInject: 9 +''' +""" + +# test {.injectStmt.} + +#[ +{.injectStmt.} pragma can be used to inject a statement before every +other statement in the current module. It's now undocumented and may be removed +in the future and replaced with something more general and without its limitations. +e.g. (e.g. doesn't work in VM or js backends). +]# + +from system/ansi_c import c_printf + +var count = 0 +proc onInject*() = + count.inc + # echo count # xxx would fail, probably infinite recursion + c_printf("onInject: %d\n", cast[int](count)) + +{.injectStmt: onInject().} +echo "ok0" +proc main()= + echo "ok1" + for a in 0..<3: + echo a + echo "ok2" + +static: main() # xxx injectStmt not honred in VM +main() From e45b858f0402bbae428bdfa86f72613b69c2671a Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 20 Jun 2021 15:57:59 +0800 Subject: [PATCH 0131/1654] [std/terminal] improve docs a bit (#18296) * Revert "add missing import to asynchttpserver's example" This reverts commit 7ef364a402d3d827f10c893280f8dc7b9ef056f5. * alternative to #18185 * add std/mutexes * cvlose #17696 * Revert "add std/mutexes" This reverts commit 69abc8b64954206da6ffe5fc40a1142b39777762. * tiny * test * improve terminal docs * follow advice --- lib/pure/terminal.nim | 115 +++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 70 deletions(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index cada72196..283652bb3 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -16,6 +16,34 @@ ## Similarly, if you hide the cursor, make sure to unhide it with ## `showCursor` before quitting. +##[ +## Playing with colorful and styled text +]## + +## Procs like `styledWriteLine`, `styledEcho` etc. have a temporary effect on +## text parameters. Style parameters only affect the text parameter right after them. +## After being called, these procs will reset the default style of the terminal. +## While `setBackGroundColor`, `setForeGroundColor` etc. have a lasting +## influence on the terminal, you can use `resetAttributes` to +## reset the default style of the terminal. +runnableExamples("-r:off"): + stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") + stdout.styledWriteLine(fgRed, "red text ") + stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") + stdout.styledWriteLine(" ordinary text without style ") + + stdout.setBackGroundColor(bgCyan, true) + stdout.setForeGroundColor(fgBlue) + stdout.write("blue text in cyan background") + stdout.resetAttributes() + + # You can specify multiple text parameters. Style parameters + # only affect the text parameter right after them. + styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" + + stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") + + import macros import strformat from strutils import toLowerAscii, `%` @@ -418,6 +446,9 @@ else: proc eraseLine*(f: File) = ## Erases the entire current line. + runnableExamples("-r:off"): + write(stdout, "never mind") + stdout.eraseLine() # nothing will be printed on the screen when defined(windows): let h = conHandle(f) var scrbuf: CONSOLE_SCREEN_BUFFER_INFO @@ -480,7 +511,7 @@ proc resetAttributes*(f: File) = gBG = 0 type - Style* = enum ## different styles for text output + Style* = enum ## Different styles for text output. styleBright = 1, ## bright text styleDim, ## dim text styleItalic, ## italic (or reverse on terminals not supporting) @@ -534,7 +565,7 @@ proc writeStyled*(txt: string, style: set[Style] = {styleBright}) = stdout.write(ansiStyleCode(gBG)) type - ForegroundColor* = enum ## terminal's foreground colors + ForegroundColor* = enum ## Terminal's foreground colors. fgBlack = 30, ## black fgRed, ## red fgGreen, ## green @@ -546,7 +577,7 @@ type fg8Bit, ## 256-color (not supported, see `enableTrueColors` instead.) fgDefault ## default terminal foreground color - BackgroundColor* = enum ## terminal's background colors + BackgroundColor* = enum ## Terminal's background colors. bgBlack = 40, ## black bgRed, ## red bgGreen, ## green @@ -701,14 +732,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = ## When some argument is `Style`, `set[Style]`, `ForegroundColor`, ## `BackgroundColor` or `TerminalCmd` then it is not sent directly to ## `f`, but instead corresponding terminal style proc is called. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## stdout.styledWrite(fgRed, "red text ") - ## stdout.styledWrite(fgGreen, "green text") - ## + runnableExamples("-r:off"): + stdout.styledWrite(fgRed, "red text ") + stdout.styledWrite(fgGreen, "green text") + var reset = false result = newNimNode(nnkStmtList) @@ -731,14 +758,10 @@ macro styledWrite*(f: File, m: varargs[typed]): untyped = template styledWriteLine*(f: File, args: varargs[untyped]) = ## Calls `styledWrite` and appends a newline at the end. - ## - ## Example: - ## - ## .. code-block:: nim - ## - ## proc error(msg: string) = - ## styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) - ## + runnableExamples: + proc error(msg: string) = + styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg) + styledWrite(f, args) write(f, "\n") @@ -747,7 +770,7 @@ template styledEcho*(args: varargs[untyped]) = stdout.styledWriteLine(args) proc getch*(): char = - ## Read a single character from the terminal, blocking until it is entered. + ## Reads a single character from the terminal, blocking until it is entered. ## The character is not printed to the terminal. when defined(windows): let fd = getStdHandle(STD_INPUT_HANDLE) @@ -852,7 +875,7 @@ when defined(windows): import os proc enableTrueColors*() = - ## Enable true color. + ## Enables true color. var term = getTerminal() when defined(windows): var @@ -885,7 +908,7 @@ proc enableTrueColors*() = term.trueColorIsEnabled = term.trueColorIsSupported proc disableTrueColors*() = - ## Disable true color. + ## Disables true color. var term = getTerminal() when defined(windows): if term.trueColorIsSupported: @@ -902,51 +925,3 @@ proc newTerminal(): owned(PTerminal) = new result when defined(windows): initTerminal(result) - -when not defined(testing) and isMainModule: - assert ansiStyleCode(styleBright) == "\e[1m" - assert ansiStyleCode(styleStrikethrough) == "\e[9m" - # exitprocs.addExitProc(resetAttributes) - write(stdout, "never mind") - stdout.eraseLine() - stdout.styledWriteLine({styleBright, styleBlink, styleUnderscore}, "styled text ") - stdout.styledWriteLine("italic text ", {styleItalic}) - stdout.setBackGroundColor(bgCyan, true) - stdout.setForeGroundColor(fgBlue) - stdout.write("blue text in cyan background") - stdout.resetAttributes() - echo "" - stdout.writeLine("ordinary text") - echo "more ordinary text" - styledEcho styleBright, fgGreen, "[PASS]", resetStyle, fgGreen, " Yay!" - echo "ordinary text again" - styledEcho styleBright, fgRed, "[FAIL]", resetStyle, fgRed, " Nay :(" - echo "ordinary text again" - setForeGroundColor(fgGreen) - echo "green text" - echo "more green text" - setForeGroundColor(fgBlue) - echo "blue text" - resetAttributes() - echo "ordinary text" - - stdout.styledWriteLine(fgRed, "red text ") - stdout.styledWriteLine(fgWhite, bgRed, "white text in red background") - stdout.styledWriteLine(" ordinary text ") - stdout.styledWriteLine(fgGreen, "green text") - - writeStyled("underscored text", {styleUnderscore}) - stdout.styledWrite(fgRed, " red text ") - writeStyled("bright text ", {styleBright}) - echo "ordinary text" - - stdout.styledWrite(fgRed, "red text ") - stdout.styledWrite(fgWhite, bgRed, "white text in red background") - stdout.styledWrite(" ordinary text ") - stdout.styledWrite(fgGreen, "green text") - echo "" - echo "ordinary text" - stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - stdout.styledWriteLine(bgYellow, "text in yellow bg", styleBright, - " bold text in yellow bg", bgDefault, " bold text") - echo "ordinary text" From eb0b323f452f5f06d74a4f747c50ae5115e44eae Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Sun, 20 Jun 2021 15:51:13 +0300 Subject: [PATCH 0132/1654] Revert "disable pkg manu (#18292)" (#18307) This reverts commit fc76565574b86566e4a642b9aac541e025cf5de3. --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index efee361da..3d3c340eb 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -82,7 +82,7 @@ pkg "karax", "nim c -r tests/tester.nim" pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree" pkg "loopfusion" pkg "macroutils" -pkg "manu", allowFailure=true # pending https://github.com/planetis-m/manu/issues/5 +pkg "manu" pkg "markdown" pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" From 92cb765714325703d1a8e1b0409bfcd1b17af103 Mon Sep 17 00:00:00 2001 From: Heiko Nickerl Date: Sun, 20 Jun 2021 17:56:33 +0200 Subject: [PATCH 0133/1654] Raise IndexDefect when deleting element at out of bounds index (#17821) Co-authored-by: Timothee Cour Co-authored-by: Heiko Nickerl Co-authored-by: Heiko Nickerl --- changelog.md | 3 ++ lib/system.nim | 58 ++++++++++++++++++++--------------- tests/gc/gcleak3.nim | 8 ++--- tests/stdlib/tsystem.nim | 32 ++++++++++++++++++- tests/system/tsystem_misc.nim | 5 --- 5 files changed, 70 insertions(+), 36 deletions(-) diff --git a/changelog.md b/changelog.md index 80e38f420..5af195e59 100644 --- a/changelog.md +++ b/changelog.md @@ -28,6 +28,9 @@ - `math.round` now is rounded "away from zero" in JS backend which is consistent with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. +- Instead of deleting the element at the last index, + `system.delete()` now raises `IndexDefect` when given index is out of bounds. + - Changed the behavior of `uri.decodeQuery` when there are unencoded `=` characters in the decoded values. Prior versions would raise an error. This is no longer the case to comply with the HTML spec and other languages diff --git a/lib/system.nim b/lib/system.nim index cae0ec3e2..a4608997a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1325,30 +1325,6 @@ proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = movingCopy(x[i], x[xl]) setLen(x, xl) -proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = - ## Deletes the item at index `i` by moving all `x[i+1..]` items by one position. - ## - ## This is an `O(n)` operation. - ## - ## See also: - ## * `del <#del,seq[T],Natural>`_ for O(1) operation - ## - ## .. code-block:: Nim - ## var i = @[1, 2, 3, 4, 5] - ## i.delete(2) # => @[1, 2, 4, 5] - template defaultImpl = - let xl = x.len - for j in i.int..xl-2: movingCopy(x[j], x[j+1]) - setLen(x, xl-1) - - when nimvm: - defaultImpl() - else: - when defined(js): - {.emit: "`x`.splice(`i`, 1);".} - else: - defaultImpl() - proc insert*[T](x: var seq[T], item: sink T, i = 0.Natural) {.noSideEffect.} = ## Inserts `item` into `x` at position `i`. ## @@ -2154,6 +2130,40 @@ const import system/dollars export dollars +proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = + ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position. + ## + ## This is an `O(n)` operation. + ## + ## See also: + ## * `del <#del,seq[T],Natural>`_ for O(1) operation + ## + runnableExamples: + var s = @[1, 2, 3, 4, 5] + s.delete(2) + doAssert s == @[1, 2, 4, 5] + + doAssertRaises(IndexDefect): + s.delete(4) + + if i > high(x): + # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring + raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed") + + template defaultImpl = + let xl = x.len + for j in i.int..xl-2: movingCopy(x[j], x[j+1]) + setLen(x, xl-1) + + when nimvm: + defaultImpl() + else: + when defined(js): + {.emit: "`x`.splice(`i`, 1);".} + else: + defaultImpl() + + const NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch ## is the version of Nim as a string. diff --git a/tests/gc/gcleak3.nim b/tests/gc/gcleak3.nim index 588e238e9..5e146d69f 100644 --- a/tests/gc/gcleak3.nim +++ b/tests/gc/gcleak3.nim @@ -17,14 +17,10 @@ for i in 0..1024: s.add(obj) proc limit*[t](a: var seq[t]) = - var loop = s.len() - 512 - for i in 0..loop: - #echo i - #GC_fullCollect() + while s.len > 0: if getOccupiedMem() > 3000_000: quit("still a leak!") - s.delete(i) + s.delete(0) s.limit() - echo "no leak: ", getOccupiedMem() diff --git a/tests/stdlib/tsystem.nim b/tests/stdlib/tsystem.nim index f8b172b44..b48034043 100644 --- a/tests/stdlib/tsystem.nim +++ b/tests/stdlib/tsystem.nim @@ -2,9 +2,10 @@ discard """ targets: "c cpp js" """ +import stdtest/testutils + # TODO: in future work move existing `system` tests here, where they belong -import stdtest/testutils template main = block: # closure @@ -42,5 +43,34 @@ template main = # doAssert rawEnv(inner3) != rawEnv(inner1) # because `a` vs `b` # this doesn't hold outer() + block: # system.delete + block: + var s = @[1] + s.delete(0) + doAssert s == @[] + + block: + var s = @["foo", "bar"] + s.delete(1) + doAssert s == @["foo"] + + block: + var s: seq[string] + doAssertRaises(IndexDefect): + s.delete(0) + + block: + doAssert not compiles(@["foo"].delete(-1)) + + block: # bug #6710 + var s = @["foo"] + s.delete(0) + doAssert s == @[] + + block: # bug #16544: deleting out of bounds index should raise + var s = @["foo"] + doAssertRaises(IndexDefect): + s.delete(1) + static: main() main() diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index cb879d3b3..13b8c18a9 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -59,11 +59,6 @@ doAssert high(float) > low(float) doAssert high(float32) > low(float32) doAssert high(float64) > low(float64) -# bug #6710 -var s = @[1] -s.delete(0) - - proc foo(a: openArray[int]) = for x in a: echo x From 7d5e6b0169fb83c7682bfad5d675c21db785c324 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 10:52:22 -0700 Subject: [PATCH 0134/1654] support `--hint:all:off --hint:x` (ditto with `--warnings` + friends) (#17852) * select all hints via `--hint:all:on|off`, etc * simplify code with setutils * address comment --- changelog.md | 4 +++ compiler/commands.nim | 59 +++++++++++++++++++++--------------------- compiler/condsyms.nim | 2 +- doc/advopt.txt | 15 ++++++----- tests/misc/trunner.nim | 9 +++---- 5 files changed, 45 insertions(+), 44 deletions(-) diff --git a/changelog.md b/changelog.md index 5af195e59..570aff542 100644 --- a/changelog.md +++ b/changelog.md @@ -429,6 +429,10 @@ - `--hint:CC` now goes to stderr (like all other hints) instead of stdout. +- `--hint:all:on|off` is now supported to select or deselect all hints; it + differs from `--hints:on|off` which acts as a (reversible) gate. + Likewise with `--warning:all:on|off`. + - json build instructions are now generated in `$nimcache/outFileBasename.json` instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project compiled with different options if the output file differs. diff --git a/compiler/commands.nim b/compiler/commands.nim index 8f8df71a7..4c1db8ead 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -25,10 +25,10 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep") bootSwitch(usedGoGC, defined(gogc), "--gc:go") bootSwitch(usedNoGC, defined(nogc), "--gc:none") +import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs] import - os, msgs, options, nversion, condsyms, strutils, extccomp, platform, - wordrecg, parseutils, nimblecmd, parseopt, sequtils, lineinfos, - pathutils, strtabs, pathnorm + msgs, options, nversion, condsyms, extccomp, platform, + wordrecg, nimblecmd, lineinfos, pathutils, pathnorm from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect @@ -182,7 +182,7 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, info: TLineInfo; orig: string; conf: ConfigRef) = var id = "" # arg = key or [key] or key:val or [key]:val; with val=on|off var i = 0 - var n = hintMin + var notes: set[TMsgKind] var isBracket = false if i < arg.len and arg[i] == '[': isBracket = true @@ -197,37 +197,36 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, if i == arg.len: discard elif i < arg.len and (arg[i] in {':', '='}): inc(i) else: invalidCmdLineOption(conf, pass, orig, info) - # unfortunately, hintUser and warningUser clash - if state in {wHint, wHintAsError}: - let x = findStr(hintMin, hintMax, id, errUnknown) - if x != errUnknown: n = TNoteKind(x) - else: localError(conf, info, "unknown hint: " & id) - else: - let x = findStr(warnMin, warnMax, id, errUnknown) - if x != errUnknown: n = TNoteKind(x) - else: localError(conf, info, "unknown warning: " & id) + let isSomeHint = state in {wHint, wHintAsError} + template findNote(noteMin, noteMax, name) = + # unfortunately, hintUser and warningUser clash, otherwise implementation would simplify a bit + let x = findStr(noteMin, noteMax, id, errUnknown) + if x != errUnknown: notes = {TNoteKind(x)} + else: localError(conf, info, "unknown $#: $#" % [name, id]) + case id.normalize + of "all": # other note groups would be easy to support via additional cases + notes = if isSomeHint: {hintMin..hintMax} else: {warnMin..warnMax} + elif isSomeHint: findNote(hintMin, hintMax, "hint") + else: findNote(warnMin, warnMax, "warning") var val = substr(arg, i).normalize if val == "": val = "on" if val notin ["on", "off"]: + # xxx in future work we should also allow users to have control over `foreignPackageNotes` + # so that they can enable `hints|warnings|warningAsErrors` for all the code they depend on. localError(conf, info, errOnOrOffExpectedButXFound % arg) - elif n notin conf.cmdlineNotes or pass == passCmd1: - if pass == passCmd1: incl(conf.cmdlineNotes, n) - incl(conf.modifiedyNotes, n) - case val - of "on": - if state in {wWarningAsError, wHintAsError}: - incl(conf.warningAsErrors, n) # xxx rename warningAsErrors to noteAsErrors - else: - incl(conf.notes, n) - incl(conf.mainPackageNotes, n) - of "off": - if state in {wWarningAsError, wHintAsError}: - excl(conf.warningAsErrors, n) - else: - excl(conf.notes, n) - excl(conf.mainPackageNotes, n) - excl(conf.foreignPackageNotes, n) + else: + let isOn = val == "on" + for n in notes: + if n notin conf.cmdlineNotes or pass == passCmd1: + if pass == passCmd1: incl(conf.cmdlineNotes, n) + incl(conf.modifiedyNotes, n) + if state in {wWarningAsError, wHintAsError}: + conf.warningAsErrors[n] = isOn # xxx rename warningAsErrors to noteAsErrors + else: + conf.notes[n] = isOn + conf.mainPackageNotes[n] = isOn + if not isOn: excl(conf.foreignPackageNotes, n) proc processCompile(conf: ConfigRef; filename: string) = var found = findFile(conf, filename) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index ff0c603cb..b3f8465f0 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -134,5 +134,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasUnifiedTuple") defineSymbol("nimHasIterable") defineSymbol("nimHasTypeofVoid") - defineSymbol("nimHasDragonBox") + defineSymbol("nimHasHintAll") diff --git a/doc/advopt.txt b/doc/advopt.txt index e5b4c9568..b56649a86 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -48,14 +48,15 @@ Advanced options: --spellSuggest|:num show at most `num >= 0` spelling suggestions on typos. if `num` is not specified (or `auto`), return an implementation defined set of suggestions. - -w:on|off|list, --warnings:on|off|list - turn all warnings on|off or list all available - --warning:X:on|off turn specific warning X on|off. - `warning:X` means `warning:X:on`, as with similar flags. - --hints:on|off|list turn all hints on|off or list all available - --hint:X:on|off turn specific hint X on|off. - --warningAsError:X:on|off turn specific warning X into an error on|off + --hints:on|off|list. `on|off` enables or disables hints. + `list` reports which hints are selected. + --hint:X:on|off turn specific hint X on|off. `hint:X` means `hint:X:on`, + as with similar flags. `all` can be used for "all hints". --hintAsError:X:on|off turn specific hint X into an error on|off + -w:on|off|list, --warnings:on|off|list + same as `--hints` but for warnings. + --warning:X:on|off ditto + --warningAsError:X:on|off ditto --styleCheck:off|hint|error produce hints or errors for Nim identifiers that do not adhere to Nim's official style guide diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index f2ea1d040..77ae10f3d 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -23,9 +23,6 @@ proc isDots(a: string): bool = a.startsWith(".") and a.strip(chars = {'.'}) == "" const - defaultHintsOff = "--hint:successx:off --hint:exec:off --hint:link:off --hint:cc:off --hint:conf:off --hint:processing:off --hint:QuitCalled:off" - # useful when you want to turn only some hints on, and some common ones off. - # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() mode = querySetting(backend) nimcache = buildDir / "nimcacheTrunner" @@ -236,7 +233,7 @@ sub/mmain.idx""", context check execCmdEx(cmd) == ("witness\n", 0) block: # config.nims, nim.cfg, hintConf, bug #16557 - let cmd = fmt"{nim} r {defaultHintsOff} --hint:conf tests/newconfig/bar/mfoo.nim" + let cmd = fmt"{nim} r --hint:all:off --hint:conf tests/newconfig/bar/mfoo.nim" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) doAssert exitCode == 0 let dir = getCurrentDir() @@ -268,7 +265,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines check fmt"""{nim} r -b:js {opt} --eval:"echo defined(js)"""".execCmdEx == ("true\n", 0) block: # `hintProcessing` dots should not interfere with `static: echo` + friends - let cmd = fmt"""{nim} r {defaultHintsOff} --hint:processing -f --eval:"static: echo 1+1"""" + let cmd = fmt"""{nim} r --hint:all:off --hint:processing -f --eval:"static: echo 1+1"""" let (outp, exitCode) = execCmdEx(cmd, options = {poStdErrToStdOut}) template check3(cond) = doAssert cond, $(outp,) doAssert exitCode == 0 @@ -282,7 +279,7 @@ tests/newconfig/bar/mfoo.nims""".splitLines check3 "2" in outp block: # nim secret - let opt = fmt"{defaultHintsOff} --hint:processing" + let opt = "--hint:all:off --hint:processing" template check3(cond) = doAssert cond, $(outp,) for extra in ["", "--stdout"]: let cmd = fmt"""{nim} secret {opt} {extra}""" From 1b3c0f142dd5abc3b3509feb194f54e6acf046c0 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 20 Jun 2021 21:00:42 +0300 Subject: [PATCH 0135/1654] validate rst field for :number-lines:, :status: (#18304) --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 2 ++ lib/packages/docutils/rst.nim | 1 + lib/packages/docutils/rstgen.nim | 30 ++++++++++++++++++++++-------- tests/stdlib/trstgen.nim | 12 ++++++++++++ 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2c99adba3..08311eade 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -164,6 +164,7 @@ template declareClosures = of meNewSectionExpected: k = errNewSectionExpected of meGeneralParseError: k = errGeneralParseError of meInvalidDirective: k = errInvalidDirectiveX + of meInvalidRstField: k = errInvalidRstField of meFootnoteMismatch: k = errFootnoteMismatch of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel of mwUnknownSubstitution: k = warnUnknownSubstitutionX diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 052dae272..abe6bce7c 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -37,6 +37,7 @@ type errGeneralParseError, errNewSectionExpected, errInvalidDirectiveX, + errInvalidRstField, errFootnoteMismatch, errProveInit, # deadcode errGenerated, @@ -96,6 +97,7 @@ const errGeneralParseError: "general parse error", errNewSectionExpected: "new section expected $1", errInvalidDirectiveX: "invalid directive: '$1'", + errInvalidRstField: "invalid field: $1", errFootnoteMismatch: "number of footnotes and their references don't match: $1", errProveInit: "Cannot prove that '$1' is initialized.", # deadcode errGenerated: "$1", diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 031a86d98..64910aa19 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -225,6 +225,7 @@ type meNewSectionExpected = "new section expected $1", meGeneralParseError = "general parse error", meInvalidDirective = "invalid directive: '$1'", + meInvalidRstField = "invalid field: $1", meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index e3f889283..829177263 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -907,6 +907,25 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) = "\\includegraphics{$1}", [d.config.getOrDefault"doc.smiley_format" % n.text]) +proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = + # TODO: proper column/line info + template err(msg: string) = + d.msgHandler(d.filename, 1, 0, meInvalidRstField, msg) + let value = n.getFieldValue + var number: int + let nChars = parseInt(value, number) + if nChars == 0: + if value.len == 0: + err("field $1 requires an argument" % [fieldName]) + else: + err("field $1 requires an integer, but '$2' was given" % + [fieldName, value]) + elif nChars < value.len: + err("extra arguments were given to $1: '$2'" % + [fieldName, value[nChars..^1]]) + else: + result = number + proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = ## Parses useful fields which can appear before a code block. ## @@ -916,9 +935,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = of "number-lines": params.numberLines = true # See if the field has a parameter specifying a different line than 1. - var number: int - if parseInt(n.getFieldValue, number) > 0: - params.startLine = number + params.startLine = getField1Int(d, n, "number-lines") of "file", "filename": # The ``file`` option is a Nim extension to the official spec, it acts # like it would for other directives like ``raw`` or ``cvs-table``. This @@ -936,9 +953,7 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) = # consider whether `$docCmd` should be appended here too params.testCmd = unescape(params.testCmd) of "status", "exitcode": - var status: int - if parseInt(n.getFieldValue, status) > 0: - params.status = status + params.status = getField1Int(d, n, n.getArgument) of "default-language": params.langStr = n.getFieldValue.strip params.lang = params.langStr.getSourceLanguage @@ -954,7 +969,6 @@ proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams = if n.isNil: return assert n.kind in {rnCodeBlock, rnInlineCode} - assert(not n.sons[2].isNil) # Parse the field list for rendering parameters if there are any. if not n.sons[1].isNil: @@ -1028,8 +1042,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) = ## option to differentiate between a plain code block and Nim's code block ## extension. assert n.kind in {rnCodeBlock, rnInlineCode} - if n.sons[2] == nil: return var params = d.parseCodeBlockParams(n) + if n.sons[2] == nil: return var m = n.sons[2].sons[0] assert m.kind == rnLeaf diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 300229648..d5d902e1d 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1136,6 +1136,18 @@ Test1 check "
55\n
" in output check "x" in output + test "Nim code-block indentation": + let input = dedent """ + .. code-block:: nim + :number-lines: 55 + let a = 1 + """ + var error = new string + let output = input.toHtml(error=error) + check(error[] == "input(1, 1) Error: invalid field: " & + "extra arguments were given to number-lines: ' let a = 1'") + check "" == output + test "RST admonitions": # check that all admonitions are implemented let input0 = dedent """ From 40ec8184ad51ccdaa6042132dc9b37b6f242c362 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 20 Jun 2021 13:49:14 -0700 Subject: [PATCH 0136/1654] followup #17852, disallow all:on for now (#18311) --- compiler/commands.nim | 2 ++ doc/advopt.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 4c1db8ead..1c3de29ba 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -217,6 +217,8 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, localError(conf, info, errOnOrOffExpectedButXFound % arg) else: let isOn = val == "on" + if isOn and id.normalize == "all": + localError(conf, info, "only 'all:off' is supported") for n in notes: if n notin conf.cmdlineNotes or pass == passCmd1: if pass == passCmd1: incl(conf.cmdlineNotes, n) diff --git a/doc/advopt.txt b/doc/advopt.txt index b56649a86..063d018d1 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -51,7 +51,8 @@ Advanced options: --hints:on|off|list. `on|off` enables or disables hints. `list` reports which hints are selected. --hint:X:on|off turn specific hint X on|off. `hint:X` means `hint:X:on`, - as with similar flags. `all` can be used for "all hints". + as with similar flags. `all` is the set of all hints + (only `all:off` is supported). --hintAsError:X:on|off turn specific hint X into an error on|off -w:on|off|list, --warnings:on|off|list same as `--hints` but for warnings. From abd21ef213b7368253050a18e5c4907d22992c8c Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 21 Jun 2021 00:45:26 -0700 Subject: [PATCH 0137/1654] close #17403; improve docs for tuple (#18312) --- doc/manual.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index e8eec9a57..f80b03369 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1616,7 +1616,7 @@ heterogeneous storage types with few abstractions. The `()` syntax can be used to construct tuples. The order of the fields in the constructor must match the order of the tuple's definition. Different tuple-types are *equivalent* if they specify the same fields of the same type in the same -order. The *names* of the fields also have to be identical. +order. The *names* of the fields also have to be the same. The assignment operator for tuples copies each component. The default assignment operator for objects copies each component. Overloading @@ -1627,15 +1627,16 @@ of the assignment operator is described `here type Person = tuple[name: string, age: int] # type representing a person: - # a person consists of a name - # and an age - var - person: Person + # it consists of a name and an age. + var person: Person person = (name: "Peter", age: 30) - echo person.name + assert person.name == "Peter" # the same, but less readable: person = ("Peter", 30) - echo person[0] + assert person[0] == "Peter" + assert Person is (string, int) + assert (string, int) is Person + assert Person isnot tuple[other: string, age: int] # `other` is a different identifier A tuple with one unnamed field can be constructed with the parentheses and a trailing comma: From ad70a65e0e3eda39a3ca074af9feadb68f10598f Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 21 Jun 2021 00:46:31 -0700 Subject: [PATCH 0138/1654] fix #18310 system.== in vm for NimNode (#18313) * fix #18310 == in vm * fixup * fixup --- compiler/vm.nim | 11 ++++++----- tests/vm/tvmmisc.nim | 11 +++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index b540ac071..837a1c366 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -10,14 +10,15 @@ ## This file implements the new evaluation engine for Nim code. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import ast except getstr import - strutils, msgs, vmdef, vmgen, nimsets, types, passes, - parser, vmdeps, idents, trees, renderer, options, transf, parseutils, - vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl, + std/[strutils, tables, parseutils], + msgs, vmdef, vmgen, nimsets, types, passes, + parser, vmdeps, idents, trees, renderer, options, transf, + vmmarshal, gorgeimpl, lineinfos, btrees, macrocacheimpl, modulegraphs, sighashes, int128, vmprofiler +import ast except getstr from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate from magicsys import getSysType @@ -1026,7 +1027,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let nc = regs[rc].node if nb.kind != nc.kind: discard elif (nb == nc) or (nb.kind == nkNilLit): ret = true # intentional - elif (nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ.kind == tyProc) and sameConstant(nb, nc): + elif nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ != nil and nb.typ.kind == tyProc and sameConstant(nb, nc): ret = true # this also takes care of procvar's, represented as nkTupleConstr, e.g. (nil, nil) elif nb.kind == nkIntLit and nc.kind == nkIntLit and nb.intVal == nc.intVal: # TODO: nkPtrLit diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index f0d2c2dbf..a4f6391cc 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -256,6 +256,17 @@ block: doAssert f == @[] +block: # bug #18310 + macro t() : untyped = + let + x = nnkTupleConstr.newTree(newLit(1)) + y = nnkTupleConstr.newTree(newLit(2)) + doAssert not (x == y) # not using != intentionally + doAssert not(cast[int](x) == cast[int](y)) + doAssert not(system.`==`(x, y)) + doAssert system.`==`(x, x) + t() + block: # bug #10815 type Opcode = enum From 2deb7011f5388559a70617c398360639102724c4 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 21 Jun 2021 18:50:47 +0800 Subject: [PATCH 0139/1654] fixes #17768 [backport:1.4] (#18317) * fixes #17768 [backport:1.4] * tiny --- lib/pure/unicode.nim | 10 ++++------ tests/stdlib/tunicode.nim | 9 ++++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 903f01fb4..dda631633 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -333,7 +333,7 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = ## from the end (starting with 1) and the total ## number of runes in the string. ## - ## Returns a negative value for offset if there are to few runes in + ## Returns a negative value for offset if there are too few runes in ## the string to satisfy the request. ## ## **Beware:** This can lead to unoptimized code and slow execution! @@ -346,16 +346,14 @@ proc runeReverseOffset*(s: string, rev: Positive): (int, int) = a = rev.int o = 0 x = 0 + let times = 2*rev.int-s.runeLen # transformed from rev.int - a < s.runeLen - rev.int while o < s.len: let r = runeLenAt(s, o) o += r - if a < 0: + if a > times: x += r dec a - - if a > 0: - return (-a, rev.int-a) - return (x, -a+rev.int) + result = if a > 0: (-a, rev.int-a) else: (x, -a+rev.int) proc runeAtPos*(s: string, pos: int): Rune = ## Returns the rune at position ``pos``. diff --git a/tests/stdlib/tunicode.nim b/tests/stdlib/tunicode.nim index a346106f9..33405cf38 100644 --- a/tests/stdlib/tunicode.nim +++ b/tests/stdlib/tunicode.nim @@ -1,4 +1,4 @@ -import unicode +import std/unicode proc asRune(s: static[string]): Rune = @@ -213,3 +213,10 @@ block differentSizes: doAssert swapCase("ⱥbCd") == "ȺBcD" doAssert swapCase("XyꟆaB") == "xYᶎAb" doAssert swapCase("aᵹcᲈd") == "AꝽCꙊD" + +block: # bug #17768 + let s1 = "abcdef" + let s2 = "abcdéf" + + doAssert s1.runeSubstr(0, -1) == "abcde" + doAssert s2.runeSubstr(0, -1) == "abcdé" From d398c558a44ad346c2cfadf266a8cf808585de6d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 21 Jun 2021 03:51:00 -0700 Subject: [PATCH 0140/1654] close #13196 now that we have dragonbox (#18316) --- tests/stdlib/tjson.nim | 11 +++++++++++ tests/stdlib/tjsonutils.nim | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim index e757e6c7e..289ef9d05 100644 --- a/tests/stdlib/tjson.nim +++ b/tests/stdlib/tjson.nim @@ -11,6 +11,8 @@ import std/[json,parsejson,strutils] from std/math import isNaN when not defined(js): import std/streams +import stdtest/testutils +from std/fenv import epsilon proc testRoundtrip[T](t: T, expected: string) = # checks that `T => json => T2 => json2` is such that json2 = json @@ -325,6 +327,15 @@ block: # bug #18007 let a = parseJson($(%NaN)).to(float) doAssert a.isNaN + whenRuntimeJs: discard # refs bug #18009 + do: + testRoundtripVal(0.0): "0.0" + testRoundtripVal(-0.0): "-0.0" + +block: # bug #15397, bug #13196 + testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002" + testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568" + block: let a = "18446744073709551615" let b = a.parseJson diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 88c05facf..31ec4316f 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -5,6 +5,7 @@ discard """ import std/jsonutils import std/json from std/math import isNaN, signbit +from std/fenv import epsilon from stdtest/testutils import whenRuntimeJs proc testRoundtrip[T](t: T, expected: string) = @@ -160,6 +161,16 @@ template fn() = doAssert b[2].signbit doAssert not b[3].signbit + block: # bug #15397, bug #13196 + let a = 0.1 + let x = 0.12345678901234567890123456789 + let b = (a + 0.2, 0.3, x) + testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]" + + testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568" + testRoundtripVal(epsilon(float64)): "2.220446049250313e-16" + testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002" + block: # case object type Foo = object x0: float From 16461a88101a398fd7349eb56392b27cdb285c17 Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Mon, 21 Jun 2021 18:53:35 -0700 Subject: [PATCH 0141/1654] add multi type exception catching to manual (#18258) (#18323) --- doc/manual.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index f80b03369..a99c4b5af 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -4462,10 +4462,8 @@ Example: echo "sum: " & $(parseInt(a) + parseInt(b)) except OverflowDefect: echo "overflow!" - except ValueError: - echo "could not convert string to integer" - except IOError: - echo "IO error!" + except ValueError, IOError: + echo "catch multiple exceptions!" except: echo "Unknown exception!" finally: From 9d3a813fb96950631c69858e8e26b6503abd02fe Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Mon, 21 Jun 2021 18:56:16 -0700 Subject: [PATCH 0142/1654] convert code-blocks to runnableExamples in io (#18315) Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/system/io.nim | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/system/io.nim b/lib/system/io.nim index 594c78209..62a9d1e18 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -912,14 +912,14 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = ## If the file does not exist `IOError` is raised. The trailing newline ## character(s) are removed from the iterated lines. Example: ## - ## .. code-block:: nim - ## import std/strutils - ## - ## proc transformLetters(filename: string) = - ## var buffer = "" - ## for line in filename.lines: - ## buffer.add(line.replace("a", "0") & '\n') - ## writeFile(filename, buffer) + runnableExamples: + import std/strutils + + proc transformLetters(filename: string) = + var buffer = "" + for line in filename.lines: + buffer.add(line.replace("a", "0") & '\n') + writeFile(filename, buffer) var f = open(filename, bufSize=8000) try: var res = newStringOfCap(80) @@ -931,14 +931,13 @@ iterator lines*(f: File): string {.tags: [ReadIOEffect].} = ## Iterate over any line in the file `f`. ## ## The trailing newline character(s) are removed from the iterated lines. - ## Example: ## - ## .. code-block:: nim - ## proc countZeros(filename: File): tuple[lines, zeros: int] = - ## for line in filename.lines: - ## for letter in line: - ## if letter == '0': - ## result.zeros += 1 - ## result.lines += 1 + runnableExamples: + proc countZeros(filename: File): tuple[lines, zeros: int] = + for line in filename.lines: + for letter in line: + if letter == '0': + result.zeros += 1 + result.lines += 1 var res = newStringOfCap(80) while f.readLine(res): yield res From d8488e41e85c84a6b5fe687940b7fee10314d1f1 Mon Sep 17 00:00:00 2001 From: mantielero Date: Tue, 22 Jun 2021 06:19:11 +0200 Subject: [PATCH 0143/1654] readAsText supports both Blob and File (fixes #18187) (#18189) --- lib/js/dom.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 2b2332020..ca325fdd2 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1767,5 +1767,5 @@ since (1, 3): ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsBinaryString proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL - proc readAsText*(f: FileReader, b: Blob, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} - ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText + proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText \ No newline at end of file From 037715285c088af46c94939f36abef63c95406e8 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 22 Jun 2021 18:44:56 +0800 Subject: [PATCH 0144/1654] fix #18327 (#18328) --- compiler/semtypes.nim | 2 +- tests/errmsgs/t18327.nim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tests/errmsgs/t18327.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 38307a9ac..484997e3a 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1295,7 +1295,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, let param = strTableGet(c.signatures, arg.name) if param != nil: typ = param.typ else: - localError(c.config, a.info, "parameter '$1' requires a type" % param.name.s) + localError(c.config, a.info, "parameter '$1' requires a type" % arg.name.s) typ = errorType(c) let lifted = liftParamType(c, kind, genericParams, typ, arg.name.s, arg.info) diff --git a/tests/errmsgs/t18327.nim b/tests/errmsgs/t18327.nim new file mode 100644 index 000000000..686a1bd0c --- /dev/null +++ b/tests/errmsgs/t18327.nim @@ -0,0 +1,5 @@ +discard """ + errormsg: "parameter 'n' requires a type" +""" + +proc fn3(n) = discard \ No newline at end of file From 9a81e91fa5d01cd6d60a408d0888dce4a9f49898 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 22 Jun 2021 23:02:32 +0800 Subject: [PATCH 0145/1654] merge similar procs regarding digits (#18318) --- lib/std/private/digitsutils.nim | 88 +++++++++++++++++++++++++++++++++ lib/system/dollars.nim | 23 ++++----- lib/system/dragonbox.nim | 32 ++---------- lib/system/schubfach.nim | 30 ++--------- lib/system/strmantle.nim | 65 +----------------------- 5 files changed, 105 insertions(+), 133 deletions(-) create mode 100644 lib/std/private/digitsutils.nim diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim new file mode 100644 index 000000000..cc985c7bd --- /dev/null +++ b/lib/std/private/digitsutils.nim @@ -0,0 +1,88 @@ +const + trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0] + + digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', + '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', + '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', + '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', + '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', + '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', + '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', + '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', + '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', + '9', '6', '9', '7', '9', '8', '9', '9'] + +# Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c +# Generates: +# .. code-block:: nim +# var res = "" +# for i in 0 .. 99: +# if i < 10: +# res.add "0" & $i +# else: +# res.add $i +# doAssert res == digits100 + +proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = + assert(digits <= 99) + buf[pos] = digits100[2 * digits] + buf[pos+1] = digits100[2 * digits + 1] + #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) + +proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = + assert(digits <= 99) + return trailingZeros100[digits] + +func digits10*(num: uint64): int {.noinline.} = + if num < 10'u64: + result = 1 + elif num < 100'u64: + result = 2 + elif num < 1_000'u64: + result = 3 + elif num < 10_000'u64: + result = 4 + elif num < 100_000'u64: + result = 5 + elif num < 1_000_000'u64: + result = 6 + elif num < 10_000_000'u64: + result = 7 + elif num < 100_000_000'u64: + result = 8 + elif num < 1_000_000_000'u64: + result = 9 + elif num < 10_000_000_000'u64: + result = 10 + elif num < 100_000_000_000'u64: + result = 11 + elif num < 1_000_000_000_000'u64: + result = 12 + else: + result = 12 + digits10(num div 1_000_000_000_000'u64) + +template numToString*(result: var string, origin: uint64, length: int) = + var num = origin + var next = length - 1 + const nbatch = 100 + + while num >= nbatch: + let originNum = num + num = num div nbatch + let index = (originNum - num * nbatch) shl 1 + result[next] = digits100[index + 1] + result[next - 1] = digits100[index] + dec(next, 2) + + # process last 1-2 digits + if num < 10: + result[next] = chr(ord('0') + num) + else: + let index = num * 2 + result[next] = digits100[index + 1] + result[next - 1] = digits100[index] diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 5a7a77e56..75390b0e1 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -1,3 +1,6 @@ +import std/private/digitsutils + + proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} ## The stringify operator for an integer argument. Returns `x` ## converted to a decimal string. `$` is Nim's general way of @@ -6,22 +9,14 @@ proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} template dollarImpl(x: uint | uint64, result: var string) = type destTyp = typeof(x) if x == 0: - result = "0" - else: - result = newString(60) - var i = 0 - var n = x - while n != 0: - let nn = n div destTyp(10) - result[i] = char(n - destTyp(10) * nn + ord('0')) - inc i - n = nn - result.setLen i + return "0" + elif x == 1: + return "1" - let half = i div 2 - # Reverse - for t in 0 .. half-1: swap(result[t], result[i-t-1]) + let length = digits10(x) + setLen(result, length) + numToString(result, x, length) when defined(js): import std/private/since diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim index eec8749da..336c982c1 100644 --- a/lib/system/dragonbox.nim +++ b/lib/system/dragonbox.nim @@ -21,6 +21,10 @@ ## Note: ## This function may temporarily write up to DtoaMinBufferLength characters into the buffer. + +import std/private/digitsutils + + const dtoaMinBufferLength*: cint = 64 @@ -1038,34 +1042,6 @@ proc toDecimal64*(ieeeSignificand: uint64; ieeeExponent: uint64): FloatingDecima ## ToChars ## ================================================================================================== -proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = - const - digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', - '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', - '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', - '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', - '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', - '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', - '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', - '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', - '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', - '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', - '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', - '9', '6', '9', '7', '9', '8', '9', '9'] - dragonbox_Assert(digits <= 99) - buf[pos] = digits100[2 * digits] - buf[pos+1] = digits100[2 * digits + 1] - #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) - -proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = - const - trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0] - dragonbox_Assert(digits <= 99) - return trailingZeros100[digits] - when false: template `+!`(x: cstring; offset: int): cstring = cast[cstring](cast[uint](x) + uint(offset)) diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index 94d3d3c25..5f974a79e 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -10,6 +10,9 @@ ## https://drive.google.com/open?id=1luHhyQF9zKlM8yJ1nebU0OgVYhfC6CBN ## -------------------------------------------------------------------------------------------------- +import std/private/digitsutils + + template sf_Assert(x: untyped): untyped = assert(x) @@ -233,33 +236,6 @@ proc toDecimal32(ieeeSignificand: uint32; ieeeExponent: uint32): FloatingDecimal ## ToChars ## ================================================================================================== -proc utoa2Digits(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = - const - digits100: array[200, char] = ['0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', - '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', - '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', '2', '0', '2', '1', '2', '2', '2', '3', - '2', '4', '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', '3', '0', '3', '1', '3', '2', - '3', '3', '3', '4', '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', '4', '0', '4', '1', - '4', '2', '4', '3', '4', '4', '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', '5', '0', - '5', '1', '5', '2', '5', '3', '5', '4', '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', - '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', '6', '5', '6', '6', '6', '7', '6', '8', - '6', '9', '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', '7', '5', '7', '6', '7', '7', - '7', '8', '7', '9', '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', '8', '5', '8', '6', - '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', - '9', '6', '9', '7', '9', '8', '9', '9'] - sf_Assert(digits <= 99) - buf[pos] = digits100[2 * digits] - buf[pos+1] = digits100[2 * digits + 1] - -proc trailingZeros2Digits(digits: uint32): int32 {.inline.} = - const - trailingZeros100: array[100, int8] = [2'i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0] - sf_Assert(digits <= 99) - return trailingZeros100[digits] - proc printDecimalDigitsBackwards(buf: var openArray[char]; pos: int; output: uint32): int32 {.inline.} = var output = output var pos = pos diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 1a32a4afe..b60bc4b80 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -9,72 +9,9 @@ # Compilerprocs for strings that do not depend on the string implementation. -const digitsTable = "0001020304050607080910111213141516171819" & - "2021222324252627282930313233343536373839" & - "4041424344454647484950515253545556575859" & - "6061626364656667686970717273747576777879" & - "8081828384858687888990919293949596979899" - # Inspired by https://engineering.fb.com/2013/03/15/developer-tools/three-optimization-tips-for-c - # Generates: - # .. code-block:: nim - # var res = "" - # for i in 0 .. 99: - # if i < 10: - # res.add "0" & $i - # else: - # res.add $i - # doAssert res == digitsTable +import std/private/digitsutils -func digits10(num: uint64): int {.noinline.} = - if num < 10'u64: - result = 1 - elif num < 100'u64: - result = 2 - elif num < 1_000'u64: - result = 3 - elif num < 10_000'u64: - result = 4 - elif num < 100_000'u64: - result = 5 - elif num < 1_000_000'u64: - result = 6 - elif num < 10_000_000'u64: - result = 7 - elif num < 100_000_000'u64: - result = 8 - elif num < 1_000_000_000'u64: - result = 9 - elif num < 10_000_000_000'u64: - result = 10 - elif num < 100_000_000_000'u64: - result = 11 - elif num < 1_000_000_000_000'u64: - result = 12 - else: - result = 12 + digits10(num div 1_000_000_000_000'u64) - -template numToString(result: var string, origin: uint64, length: int) = - var num = origin - var next = length - 1 - const nbatch = 100 - - while num >= nbatch: - let originNum = num - num = num div nbatch - let index = (originNum - num * nbatch) shl 1 - result[next] = digitsTable[index + 1] - result[next - 1] = digitsTable[index] - dec(next, 2) - - # process last 1-2 digits - if num < 10: - result[next] = chr(ord('0') + num) - else: - let index = num * 2 - result[next] = digitsTable[index + 1] - result[next - 1] = digitsTable[index] - proc cmpStrings(a, b: string): int {.inline, compilerproc.} = let alen = a.len let blen = b.len From 5badeea17055c85ea8177baeee62c8d1d06ac13e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 22 Jun 2021 21:42:39 -0700 Subject: [PATCH 0146/1654] followup #18318: simplify `dollarImpl` and add a test (#18330) --- lib/system/dollars.nim | 7 ------- tests/stdlib/tdigitsutils.nim | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 tests/stdlib/tdigitsutils.nim diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 75390b0e1..3d824a965 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -7,15 +7,8 @@ proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} ## spelling `toString`:idx:. template dollarImpl(x: uint | uint64, result: var string) = - type destTyp = typeof(x) - if x == 0: - return "0" - elif x == 1: - return "1" - let length = digits10(x) setLen(result, length) - numToString(result, x, length) when defined(js): diff --git a/tests/stdlib/tdigitsutils.nim b/tests/stdlib/tdigitsutils.nim new file mode 100644 index 000000000..7cf63411f --- /dev/null +++ b/tests/stdlib/tdigitsutils.nim @@ -0,0 +1,23 @@ +import std/private/digitsutils + +template main = + block: # digits10 + doAssert digits10(0'u64) == 1 + # checks correctness on all powers of 10 + [0,-1,1] + var x = 1'u64 + var num = 1 + while true: + # echo (x, num) + doAssert digits10(x) == num + doAssert digits10(x+1) == num + if x > 1: + doAssert digits10(x-1) == num - 1 + num += 1 + let xOld = x + x *= 10 + if x < xOld: + # wrap-around + break + +static: main() +main() From 9c43f05099aaddae236df8b86da0cee342a785b8 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 23 Jun 2021 08:50:05 +0300 Subject: [PATCH 0147/1654] Markdown: allow to end URL with balanced parenthesis (#18321) * Markdown: allow to end URL with balanced parenthesis * Update lib/packages/docutils/rst.nim Co-authored-by: Timothee Cour * apply suggestion * remove unnecessary if Co-authored-by: Timothee Cour --- lib/packages/docutils/rst.nim | 36 ++++++++++++++++++++++++++++-- tests/stdlib/trst.nim | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 64910aa19..7833e6e9a 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1246,15 +1246,43 @@ proc isUrl(p: RstParser, i: int): bool = p.tok[i+3].kind == tkWord and p.tok[i].symbol in ["http", "https", "ftp", "telnet", "file"] +proc checkParen(token: Token, parensStack: var seq[char]): bool {.inline.} = + ## Returns `true` iff `token` is a closing parenthesis for some + ## previous opening parenthesis saved in `parensStack`. + ## This is according Markdown balanced parentheses rule + ## (https://spec.commonmark.org/0.29/#link-destination) + ## to allow links like + ## https://en.wikipedia.org/wiki/APL_(programming_language), + ## we use it for RST also. + result = false + if token.kind == tkPunct: + let c = token.symbol[0] + if c in {'(', '[', '{'}: # push + parensStack.add c + elif c in {')', ']', '}'}: # try pop + # a case like ([) inside a link is allowed and [ is also `pop`ed: + for i in countdown(parensStack.len - 1, 0): + if (parensStack[i] == '(' and c == ')' or + parensStack[i] == '[' and c == ']' or + parensStack[i] == '{' and c == '}'): + parensStack.setLen i + result = true + break + proc parseUrl(p: var RstParser): PRstNode = ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#standalone-hyperlinks result = newRstNode(rnStandaloneHyperlink) var lastIdx = p.idx + var closedParenIdx = p.idx - 1 # for balanced parens rule + var parensStack: seq[char] while p.tok[lastIdx].kind in {tkWord, tkPunct, tkOther}: + let isClosing = checkParen(p.tok[lastIdx], parensStack) + if isClosing: + closedParenIdx = lastIdx inc lastIdx dec lastIdx # standalone URL can not end with punctuation in RST - while lastIdx >= p.idx and p.tok[lastIdx].kind == tkPunct and + while lastIdx > closedParenIdx and p.tok[lastIdx].kind == tkPunct and p.tok[lastIdx].symbol != "/": dec lastIdx var s = "" @@ -1393,11 +1421,15 @@ proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = var desc, link = "" var i = p.idx + var parensStack: seq[char] template parse(endToken, dest) = + parensStack.setLen 0 inc i # skip begin token while true: if p.tok[i].kind in {tkEof, tkIndent}: return false - if p.tok[i].symbol == endToken: break + let isClosing = checkParen(p.tok[i], parensStack) + if p.tok[i].symbol == endToken and not isClosing: + break dest.add p.tok[i].symbol inc i inc i # skip end token diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index ec34edc91..c69450cd9 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -580,3 +580,44 @@ suite "RST inline markup": rnLeaf ' ' rnLeaf 'end' """) + + test "URL with balanced parentheses (Markdown rule)": + # 2 balanced parens, 1 unbalanced: + check(dedent""" + https://en.wikipedia.org/wiki/APL_((programming_language)))""".toAst == + dedent""" + rnInner + rnStandaloneHyperlink + rnLeaf 'https://en.wikipedia.org/wiki/APL_((programming_language))' + rnLeaf ')' + """) + + # the same for Markdown-style link: + check(dedent""" + [foo [bar]](https://en.wikipedia.org/wiki/APL_((programming_language))))""".toAst == + dedent""" + rnInner + rnHyperlink + rnLeaf 'foo [bar]' + rnLeaf 'https://en.wikipedia.org/wiki/APL_((programming_language))' + rnLeaf ')' + """) + + # unbalanced (here behavior is more RST-like actually): + check(dedent""" + https://en.wikipedia.org/wiki/APL_(programming_language(""".toAst == + dedent""" + rnInner + rnStandaloneHyperlink + rnLeaf 'https://en.wikipedia.org/wiki/APL_(programming_language' + rnLeaf '(' + """) + + # unbalanced [, but still acceptable: + check(dedent""" + [my {link example](http://example.com/bracket_(symbol_[))""".toAst == + dedent""" + rnHyperlink + rnLeaf 'my {link example' + rnLeaf 'http://example.com/bracket_(symbol_[)' + """) From 0eee7f572796d3626db1d94d3eb344b399780db1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 23 Jun 2021 14:39:10 +0200 Subject: [PATCH 0148/1654] OS.nim: style changes (#18331) --- lib/pure/os.nim | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 4dd5c8d8b..213da2aed 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -320,17 +320,17 @@ else: template `!=?`(a, b: char): bool = toLowerAscii(a) != toLowerAscii(b) when doslikeFileSystem: - proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: []} = + proc isAbsFromCurrentDrive(path: string): bool {.noSideEffect, raises: [].} = ## An absolute path from the root of the current drive (e.g. "\foo") path.len > 0 and (path[0] == AltSep or (path[0] == DirSep and (path.len == 1 or path[1] notin {DirSep, AltSep, ':'}))) - proc isUNCPrefix(path: string): bool {.noSideEffect, raises: []} = + proc isUNCPrefix(path: string): bool {.noSideEffect, raises: [].} = path[0] == DirSep and path[1] == DirSep - proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: []} = + proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = ## Return true if path1 and path2 have a same root. ## ## Detail of windows path formats: @@ -935,7 +935,7 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", proc getCacheDir*(): string = ## Returns the cache directory of the current user for applications. - ## + ## ## This makes use of the following environment variables: ## ## * On Windows: `getEnv("LOCALAPPDATA")` @@ -1038,7 +1038,7 @@ proc expandTilde*(path: string): string {. ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). ## - ## Windows: this is still supported despite Windows platform not having this + ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. ## ## .. warning:: `~bob` and `~bob/` are not yet handled correctly. @@ -1101,18 +1101,14 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. - ## Based on Python's `pipes.quote`. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', '0'..'9', 'A'..'Z', 'a'..'z'} if s.len == 0: - return "''" - - let safe = s.allCharsInSet(safeUnixChars) - - if safe: - return s + result = "''" + elif s.allCharsInSet(safeUnixChars): + result = s else: - return "'" & s.replace("'", "'\"'\"'") & "'" + result = "'" & s.replace("'", "'\"'\"'") & "'" when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = @@ -1122,9 +1118,9 @@ when defined(windows) or defined(posix) or defined(nintendoswitch): ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc ## <#quoteShellPosix,string>`_. when defined(windows): - return quoteShellWindows(s) + result = quoteShellWindows(s) else: - return quoteShellPosix(s) + result = quoteShellPosix(s) proc quoteShellCommand*(args: openArray[string]): string = ## Concatenates and quotes shell arguments `args`. @@ -1217,7 +1213,7 @@ proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 else: var res: Stat - return stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) + result = stat(dir, res) >= 0'i32 and S_ISDIR(res.st_mode) proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect], @@ -1239,7 +1235,7 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", result = (a and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32 else: var res: Stat - return lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) + result = lstat(link, res) >= 0'i32 and S_ISLNK(res.st_mode) when not defined(windows): @@ -1752,17 +1748,18 @@ proc isAdmin*: bool {.noWeirdTarget.} = addr administratorsGroup)): raiseOSError(osLastError(), "could not get SID for Administrators group") - defer: + try: + var b: WINBOOL + if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): + raiseOSError(osLastError(), "could not check access token membership") + + result = isSuccess(b) + finally: if freeSid(administratorsGroup) != nil: raiseOSError(osLastError(), "failed to free SID for Administrators group") - var b: WINBOOL - if not isSuccess(checkTokenMembership(0, administratorsGroup, addr b)): - raiseOSError(osLastError(), "could not check access token membership") - - return isSuccess(b) else: - return geteuid() == 0 + result = geteuid() == 0 proc createSymlink*(src, dest: string) {.noWeirdTarget.} = ## Create a symbolic link at `dest` which points to the item specified @@ -1828,10 +1825,11 @@ when hasCCopyfile: COPYFILE_XATTR {.nodecl.}: copyfile_flags_t {.pop.} -type CopyFlag* = enum ## Copy options. - cfSymlinkAsIs, ## Copy symlinks as symlinks - cfSymlinkFollow, ## Copy the files symlinks point to - cfSymlinkIgnore ## Ignore symlinks +type + CopyFlag* = enum ## Copy options. + cfSymlinkAsIs, ## Copy symlinks as symlinks + cfSymlinkFollow, ## Copy the files symlinks point to + cfSymlinkIgnore ## Ignore symlinks const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore} @@ -2027,7 +2025,7 @@ proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} if not result: let err = osLastError() - let isAccessDeniedError = + let isAccessDeniedError = when defined(windows): const AccessDeniedError = OSErrorCode(5) isDir and err == AccessDeniedError @@ -2066,7 +2064,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", except: discard tryRemoveFile(dest) raise - + proc exitStatusLikeShell*(status: cint): cint = ## Converts exit code from `c_system` into a shell exit code. From 63456c6d7f84f29da38703fb3e59acf00acec1b8 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Wed, 23 Jun 2021 18:56:20 +0300 Subject: [PATCH 0149/1654] Add some tests (#18333) --- tests/destructor/tv2_cast.nim | 72 ++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index ef0b3a936..5502cb38d 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -1,10 +1,78 @@ discard """ - cmd: '''nim c --gc:arc $file''' output: '''@[1] @[116, 101, 115, 116] @[1953719668, 875770417]''' + cmd: '''nim c --gc:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' + nimout: '''--expandArc: main + +var + data + :tmpD +`=copy`(data, cast[string](encode(cast[seq[byte]]( + :tmpD = newString(100) + :tmpD)))) +`=destroy`(:tmpD) +`=destroy`(data) +-- end of expandArc ------------------------ +--expandArc: main1 + +var + s + data +s = newString(100) +`=copy`(data, cast[string](encode(toOpenArrayByte(s, 0, len(s) - 1)))) +`=destroy`(data) +`=destroy`(s) +-- end of expandArc ------------------------ +--expandArc: main2 + +var + s + data +s = newSeq(100) +`=copy`(data, cast[string](encode(s))) +`=destroy`(data) +`=destroy_1`(s) +-- end of expandArc ------------------------ +--expandArc: main3 + +var + data + :tmpD +`=copy`(data, cast[string](encode do: + :tmpD = newSeq(100) + :tmpD)) +`=destroy`(:tmpD) +`=destroy_1`(data) +-- end of expandArc ------------------------''' """ +func encode*(src: openArray[byte]): seq[byte] = + result = newSeq[byte](src.len) + +template compress*(src: string): string = + cast[string](encode(cast[seq[byte]](src))) + +proc main = + let data = compress(newString(100)) +main() + +proc main1 = + var + s = newString(100) + let data = cast[string](encode(s.toOpenArrayByte(0, s.len-1))) +main1() + +proc main2 = + var + s = newSeq[byte](100) + let data = cast[string](encode(s)) +main2() + +proc main3 = + let data = cast[string](encode(newSeq[byte](100))) +main3() + # bug #11018 discard cast[seq[uint8]](@[1]) discard cast[seq[uint8]]("test") @@ -20,4 +88,4 @@ echo a #issue 11204 var ac {.compileTime.} = @["a", "b"] -const bc = ac.len \ No newline at end of file +const bc = ac.len From 496bd790e1b641a7c025bde2c6a1c6dca433ed5d Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Jun 2021 03:33:19 +0800 Subject: [PATCH 0150/1654] [std/times]getTime now uses high resolution API on windows (#17901) --- lib/pure/times.nim | 5 +++-- lib/windows/winlean.nim | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index a92eccc8c..7bb393aa0 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -880,6 +880,7 @@ since((1, 1)): export fromUnixFloat export toUnixFloat + proc fromWinTime*(win: int64): Time = ## Convert a Windows file time (100-nanosecond intervals since ## `1601-01-01T00:00:00Z`) to a `Time`. @@ -912,7 +913,7 @@ proc getTime*(): Time {.tags: [TimeEffect], benign.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) elif defined(windows): var f {.noinit.}: FILETIME - getSystemTimeAsFileTime(f) + getSystemTimePreciseAsFileTime(f) result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = @@ -2595,7 +2596,7 @@ proc epochTime*(): float {.tags: [TimeEffect].} = toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 elif defined(windows): var f {.noinit.}: winlean.FILETIME - getSystemTimeAsFileTime(f) + getSystemTimePreciseAsFileTime(f) var i64 = rdFileTime(f) - epochDiff var secs = i64 div rateDiff var subsecs = i64 mod rateDiff diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index d2d7d26cc..a71dd97d8 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -928,6 +928,9 @@ proc getProcessTimes*(hProcess: Handle; lpCreationTime, lpExitTime, lpKernelTime, lpUserTime: var FILETIME): WINBOOL {.stdcall, dynlib: "kernel32", importc: "GetProcessTimes".} +proc getSystemTimePreciseAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. + importc: "GetSystemTimePreciseAsFileTime", dynlib: "kernel32", stdcall, sideEffect.} + type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.gcsafe, stdcall, tags: [].} From 0f91b67f5c15328330f74a8769aed9961940aab2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 23 Jun 2021 23:31:17 +0200 Subject: [PATCH 0151/1654] fixes #18326 (#18335) * fixes #18326 * make tests green again --- compiler/isolation_check.nim | 4 ++-- tests/isolate/tisolate.nim | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 43db5d59b..777e7f6ce 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -64,7 +64,7 @@ proc canAlias(arg, ret: PType; marker: var IntSet): bool = else: result = false -proc isValueOnlyType(t: PType): bool = +proc isValueOnlyType(t: PType): bool = # t doesn't contain pointers and references proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent} result = not types.searchTypeFor(t, wrap) @@ -88,7 +88,7 @@ proc checkIsolate*(n: PNode): bool = # XXX: as long as we don't update the analysis while examining arguments # we can do an early check of the return type, otherwise this is a # bug and needs to be moved below - if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}: + if tfNoSideEffect notin n[0].typ.flags: return false for i in 1.. Date: Wed, 23 Jun 2021 23:31:55 -0700 Subject: [PATCH 0152/1654] fix #18332: XDeclaredButNotUsed hints now in deterministic order (#18336) --- compiler/lookups.nim | 10 ++++++---- tests/errmsgs/treportunused.nim | 24 +++++++++++++----------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 4f413f472..ee79673d0 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -8,11 +8,10 @@ # # This module implements lookup helpers. - +import std/[algorithm, strutils] import intsets, ast, astalgo, idents, semdata, types, msgs, options, - renderer, nimfix/prettybase, lineinfos, strutils, - modulegraphs, astmsgs + renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) @@ -268,6 +267,7 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) = var it: TTabIter var s = initTabIter(it, scope.symbols) var missingImpls = 0 + var unusedSyms: seq[tuple[sym: PSym, key: string]] while s != nil: if sfForward in s.flags and s.kind notin {skType, skModule}: # too many 'implementation of X' errors are annoying @@ -282,8 +282,10 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) = # maybe they can be made skGenericParam as well. if s.typ != nil and tfImplicitTypeParam notin s.typ.flags and s.typ.kind != tyGenericParam: - message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s) + unusedSyms.add (s, toFileLineCol(c.config, s.info)) s = nextIter(it, scope.symbols) + for (s, _) in sortedByIt(unusedSyms, it.key): + message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s) proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; conflictsWith: TLineInfo) = diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim index b339e06bf..3e945013e 100644 --- a/tests/errmsgs/treportunused.nim +++ b/tests/errmsgs/treportunused.nim @@ -1,16 +1,18 @@ discard """ + matrix: "--hint:all:off --hint:XDeclaredButNotUsed" + nimoutFull: true nimout: ''' -treportunused.nim(19, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(26, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(22, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(25, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(24, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(23, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(20, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(29, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(27, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(21, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(28, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(21, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(22, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(23, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(24, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(25, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(26, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(27, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(28, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(29, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(30, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(31, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] ''' action: compile """ From 55c1953f636ff2d2e2b9e9599194fa6c3fb49050 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 24 Jun 2021 00:58:11 -0700 Subject: [PATCH 0153/1654] fix #18334: handle path with spaces on windows during bootstrap (#18337) --- compiler/nim.nim | 27 +++++++++++++++------------ koch.nim | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/compiler/nim.nim b/compiler/nim.nim index f18238f60..91b1bc2db 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -7,21 +7,24 @@ # distribution, for details about the copyright. # -when defined(gcc) and defined(windows): - when defined(x86): - {.link: "../icons/nim.res".} - else: - {.link: "../icons/nim_icon.o".} +import std/[os, strutils, parseopt] +when defined(windows) and not defined(nimKochBootstrap): + # remove workaround pending bootstrap >= 1.5.1 + # refs https://github.com/nim-lang/Nim/issues/18334#issuecomment-867114536 + # alternative would be to prepend `currentSourcePath.parentDir.quoteShell` + when defined(gcc): + when defined(x86): + {.link: "../icons/nim.res".} + else: + {.link: "../icons/nim_icon.o".} -when defined(amd64) and defined(windows) and defined(vcc): - {.link: "../icons/nim-amd64-windows-vcc.res".} -when defined(i386) and defined(windows) and defined(vcc): - {.link: "../icons/nim-i386-windows-vcc.res".} + when defined(amd64) and defined(vcc): + {.link: "../icons/nim-amd64-windows-vcc.res".} + when defined(i386) and defined(vcc): + {.link: "../icons/nim-i386-windows-vcc.res".} import - commands, options, msgs, - extccomp, strutils, os, main, parseopt, - idents, lineinfos, cmdlinehelper, + commands, options, msgs, extccomp, main, idents, lineinfos, cmdlinehelper, pathutils, modulegraphs from browsers import openDefaultBrowser diff --git a/koch.nim b/koch.nim index 665b5e042..51be53cc7 100644 --- a/koch.nim +++ b/koch.nim @@ -308,7 +308,7 @@ proc boot(args: string) = var nimi = i.thVersion if i == 0: nimi = nimStart - extraOption.add " --skipUserCfg --skipParentCfg" + extraOption.add " --skipUserCfg --skipParentCfg -d:nimKochBootstrap" # The configs are skipped for bootstrap # (1st iteration) to prevent newer flags from breaking bootstrap phase. let ret = execCmdEx(nimStart & " --version") From 0c8d3ae9852721147ddf878e204c14e949ed676a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 24 Jun 2021 11:28:28 +0300 Subject: [PATCH 0154/1654] rst: allow comment to continue on second line (#18338) --- lib/packages/docutils/rst.nim | 28 ++++++--------- tests/stdlib/trst.nim | 66 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 7833e6e9a..8ffbbd4d3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1615,24 +1615,16 @@ proc getDirective(p: var RstParser): string = "too many colons for a directive (should be ::)", p.tok[afterIdx].line, p.tok[afterIdx].col) -proc parseComment(p: var RstParser): PRstNode = - case currentTok(p).kind - of tkIndent, tkEof: - if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: - inc p.idx # empty comment - else: - var indent = currentTok(p).ival - while true: - case currentTok(p).kind - of tkEof: - break - of tkIndent: - if currentTok(p).ival < indent: break - else: - discard - inc p.idx +proc parseComment(p: var RstParser, col: int): PRstNode = + if currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: + inc p.idx # empty comment else: - while currentTok(p).kind notin {tkIndent, tkEof}: inc p.idx + while currentTok(p).kind != tkEof: + if currentTok(p).kind == tkIndent and currentTok(p).ival > col or + currentTok(p).kind != tkIndent and currentTok(p).col > col: + inc p.idx + else: + break result = nil proc parseLine(p: var RstParser, father: PRstNode) = @@ -2841,7 +2833,7 @@ proc parseDotDot(p: var RstParser): PRstNode = (n = parseFootnote(p); n != nil): result = n else: - result = parseComment(p) + result = parseComment(p, col) proc rstParsePass1*(fragment, filename: string, line, column: int, diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index c69450cd9..d53092d3b 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -83,6 +83,72 @@ suite "RST parsing": rnLeaf 'set' """) + test "RST comment": + check(dedent""" + .. comment1 + comment2 + someParagraph""".toAst == + dedent""" + rnLeaf 'someParagraph' + """) + + check(dedent""" + .. + comment1 + comment2 + someParagraph""".toAst == + dedent""" + rnLeaf 'someParagraph' + """) + + test "check that additional line right after .. ends comment": + check(dedent""" + .. + + notAcomment1 + notAcomment2 + someParagraph""".toAst == + dedent""" + rnInner + rnBlockQuote + rnInner + rnLeaf 'notAcomment1' + rnLeaf ' ' + rnLeaf 'notAcomment2' + rnParagraph + rnLeaf 'someParagraph' + """) + + test "but blank lines after 2nd non-empty line don't end the comment": + check(dedent""" + .. + comment1 + + + comment2 + someParagraph""".toAst == + dedent""" + rnLeaf 'someParagraph' + """) + + test "using .. as separator b/w directives and block quotes": + check(dedent""" + .. note:: someNote + + .. + + someBlockQuote""".toAst == + dedent""" + rnInner + rnAdmonition adType=note + [nil] + [nil] + rnLeaf 'someNote' + rnBlockQuote + rnInner + rnLeaf 'someBlockQuote' + """) + suite "RST indentation": test "nested bullet lists": let input = dedent """ From 565e07a9936657fa356777eec5f0f52b2d5d7e02 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 24 Jun 2021 02:55:31 -0700 Subject: [PATCH 0155/1654] enable VM tracing in user code via `{.define(nimVmTrace).}` (#18244) * enable VM tracing in user code via `{.define(nimVmTrace).}` * add vmutils.vmTrace * add vmTrace --- changelog.md | 2 ++ compiler/options.nim | 1 + compiler/vm.nim | 7 +++++-- compiler/vmops.nim | 3 +++ lib/std/vmutils.nim | 11 +++++++++++ tests/stdlib/tvmutils.nim | 31 +++++++++++++++++++++++++++++++ 6 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 lib/std/vmutils.nim create mode 100644 tests/stdlib/tvmutils.nim diff --git a/changelog.md b/changelog.md index 570aff542..6672f7538 100644 --- a/changelog.md +++ b/changelog.md @@ -387,6 +387,8 @@ - Added `--declaredlocs` to show symbol declaration location in messages. +- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. + - Deprecated `TaintedString` and `--taintmode`. - Deprecated `--nilseqs` which is now a noop. diff --git a/compiler/options.nim b/compiler/options.nim index ffb0f4501..135ecf1de 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -330,6 +330,7 @@ type warnCounter*: int errorMax*: int maxLoopIterationsVM*: int ## VM: max iterations of all loops + isVmTrace*: bool configVars*: StringTableRef symbols*: StringTableRef ## We need to use a StringTableRef here as defined ## symbols are always guaranteed to be style diff --git a/compiler/vm.nim b/compiler/vm.nim index 837a1c366..9541af701 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -548,9 +548,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = "pc", $pc, "opcode", alignLeft($c.code[pc].opcode, 15), "ra", regDescr("ra", ra), "rb", regDescr("rb", instr.regB), "rc", regDescr("rc", instr.regC)] - + if c.config.isVmTrace: + # unlike nimVMDebug, this doesn't require re-compiling nim and is controlled by user code + let info = c.debug[pc] + # other useful variables: c.loopIterations + echo "$# [$#] $#" % [c.config$info, $instr.opcode, c.config.sourceLine(info)] c.profiler.enter(c, tos) - case instr.opcode of opcEof: return regs[ra] of opcRet: diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 85729fd59..67a196b8f 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -249,6 +249,9 @@ proc registerAdditionalOps*(c: PCtx) = "isExported() requires a symbol. '" & $n & "' is of kind '" & $n.kind & "'", n.info) setResult(a, sfExported in n.sym.flags) + registerCallback c, "stdlib.vmutils.vmTrace", proc (a: VmArgs) = + c.config.isVmTrace = getBool(a, 0) + proc hashVmImpl(a: VmArgs) = var res = hashes.hash(a.getString(0), a.getInt(1).int, a.getInt(2).int) if c.config.backend == backendJs: diff --git a/lib/std/vmutils.nim b/lib/std/vmutils.nim new file mode 100644 index 000000000..e16912a3c --- /dev/null +++ b/lib/std/vmutils.nim @@ -0,0 +1,11 @@ +##[ +Experimental API, subject to change. +]## + +proc vmTrace*(on: bool) {.compileTime.} = + runnableExamples: + static: vmTrace(true) + proc fn = + var a = 1 + vmTrace(false) + static: fn() diff --git a/tests/stdlib/tvmutils.nim b/tests/stdlib/tvmutils.nim new file mode 100644 index 000000000..f43557ad8 --- /dev/null +++ b/tests/stdlib/tvmutils.nim @@ -0,0 +1,31 @@ +discard """ + joinable: false + nimout: ''' +0 +1 +2 +tvmutils.nim(28, 13) [opcLdImmInt] if i == 4: +tvmutils.nim(28, 10) [opcEqInt] if i == 4: +tvmutils.nim(28, 10) [opcFJmp] if i == 4: +tvmutils.nim(28, 13) [opcLdImmInt] if i == 4: +tvmutils.nim(28, 10) [opcEqInt] if i == 4: +tvmutils.nim(28, 10) [opcFJmp] if i == 4: +tvmutils.nim(29, 7) [opcLdConst] vmTrace(false) +tvmutils.nim(29, 15) [opcLdImmInt] vmTrace(false) +tvmutils.nim(29, 14) [opcIndCall] vmTrace(false) +5 +6 +''' +""" +# line 20 (only showing a subset of nimout to avoid making the test rigid) +import std/vmutils + +proc main() = + for i in 0..<7: + echo i + if i == 2: + vmTrace(true) + if i == 4: + vmTrace(false) + +static: main() From 16038d44f6b246c5f39d1a8609a8c45f2cd005a4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 24 Jun 2021 18:37:04 +0200 Subject: [PATCH 0156/1654] fixes #18320 (#18343) * TSymFlag has 47 flags already * fixes #18320 --- compiler/ast.nim | 2 +- compiler/varpartitions.nim | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 8bb8de1d6..56043692f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -229,7 +229,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 46 flags! + TSymFlag* = enum # 47 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 697dd2b1d..213c3b80e 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -51,6 +51,7 @@ type SubgraphFlag = enum isMutated, # graph might be mutated isMutatedDirectly, # graph is mutated directly by a non-var parameter. + isMutatedByVarParam, # graph is mutated by a var parameter. connectsConstParam # graph is connected to a non-var parameter. VarFlag = enum @@ -184,7 +185,7 @@ proc potentialMutation(v: var Partitions; s: PSym; level: int; info: TLineInfo) {isMutated, isMutatedDirectly} elif s.typ.kind == tyVar and level <= 1: # varParam[i] = v is different from varParam[i][] = v - {} + {isMutatedByVarParam} else: {isMutated} else: @@ -878,7 +879,7 @@ proc computeGraphPartitions*(s: PSym; n: PNode; g: ModuleGraph; goals: set[Goal] proc dangerousMutation(g: MutationInfo; v: VarIndex): bool = #echo "range ", v.aliveStart, " .. ", v.aliveEnd, " ", v.sym - if isMutated in g.flags: + if {isMutated, isMutatedByVarParam} * g.flags != {}: for m in g.mutations: #echo "mutation ", m if m in v.aliveStart..v.aliveEnd: @@ -941,4 +942,5 @@ proc computeCursors*(s: PSym; n: PNode; g: ModuleGraph) = discard "cannot cursor into a graph that is mutated" else: v.sym.flags.incl sfCursor - #echo "this is now a cursor ", v.sym, " ", par.s[rid].flags, " ", g.config $ v.sym.info + when false: + echo "this is now a cursor ", v.sym, " ", par.s[rid].flags, " ", g.config $ v.sym.info From f128f7971f6eda5dc9b21af3dbaeccc5dc71da5a Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Thu, 24 Jun 2021 13:19:50 -0400 Subject: [PATCH 0157/1654] float parsing: Add test for a fixed issue (#18232) Fixes https://github.com/nim-lang/Nim/issues/14407 . This issue was fixed by https://github.com/nim-lang/Nim/pull/18139. --- tests/errmsgs/treportunused.nim | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim index 3e945013e..65e00163b 100644 --- a/tests/errmsgs/treportunused.nim +++ b/tests/errmsgs/treportunused.nim @@ -2,17 +2,19 @@ discard """ matrix: "--hint:all:off --hint:XDeclaredButNotUsed" nimoutFull: true nimout: ''' -treportunused.nim(21, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(22, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(23, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(24, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(25, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(26, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(27, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(28, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(29, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(30, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(31, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(23, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(24, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(25, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(26, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(27, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(28, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(29, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed] ''' action: compile """ @@ -29,3 +31,8 @@ let s8 = 0 var s9: int type s10 = object type s11 = type(1.2) + +# https://github.com/nim-lang/Nim/issues/14407 +let + `v0.99` = "0.99" + `v0.99.99` = "0.99.99" From 6be8a66833ee25ad183fa3e4a667e3c0948413d8 Mon Sep 17 00:00:00 2001 From: Smooth Operator Date: Thu, 24 Jun 2021 13:48:57 -0400 Subject: [PATCH 0158/1654] couple tiny typo fixes (#18344) --- lib/pure/times.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 7bb393aa0..fb48a199c 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -359,7 +359,7 @@ type Timezone* = ref object ## \ ## Timezone interface for supporting `DateTime <#DateTime>`_\s of arbitrary ## timezones. The `times` module only supplies implementations for the - ## systems local time and UTC. + ## system's local time and UTC. zonedTimeFromTimeImpl: proc (x: Time): ZonedTime {.tags: [], raises: [], benign.} zonedTimeFromAdjTimeImpl: proc (x: Time): ZonedTime @@ -1145,7 +1145,7 @@ proc name*(zone: Timezone): string = ## If the timezone doesn't exist in the tz database, or if the timezone ## name is unknown, then any string that describes the timezone ## unambiguously might be used. For example, the string "LOCAL" is used - ## for the systems local timezone. + ## for the system's local timezone. ## ## See also: https://en.wikipedia.org/wiki/Tz_database zone.name From 0d194cdbf90953f28450c4bf1db744d2c1332996 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 25 Jun 2021 06:22:52 +0200 Subject: [PATCH 0159/1654] fixes #18287 (#18346) --- compiler/varpartitions.nim | 48 ++++++++++++++++----------------- tests/arc/topt_no_cursor.nim | 51 +++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 25 deletions(-) diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 213c3b80e..8f422face 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -416,7 +416,6 @@ proc allRoots(n: PNode; result: var seq[(PSym, int)]; level: int) = proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = ## Analyse if 'n' is an expression that owns the data, if so mark 'dest' ## with 'ownsData'. - if n.typ == nil: return case n.kind of nkEmpty, nkCharLit..nkNilLit: # primitive literals including the empty are harmless: @@ -475,30 +474,31 @@ proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = destMightOwn(c, dest, n[0]) of nkCallKinds: - if hasDestructor(n.typ): - # calls do construct, what we construct must be destroyed, - # so dest cannot be a cursor: - dest.flags.incl ownsData - elif n.typ.kind in {tyLent, tyVar}: - # we know the result is derived from the first argument: - var roots: seq[(PSym, int)] - allRoots(n[1], roots, RootEscapes) - for r in roots: - connect(c, dest.sym, r[0], n[1].info) + if n.typ != nil: + if hasDestructor(n.typ): + # calls do construct, what we construct must be destroyed, + # so dest cannot be a cursor: + dest.flags.incl ownsData + elif n.typ.kind in {tyLent, tyVar}: + # we know the result is derived from the first argument: + var roots: seq[(PSym, int)] + allRoots(n[1], roots, RootEscapes) + for r in roots: + connect(c, dest.sym, r[0], n[1].info) - else: - let magic = if n[0].kind == nkSym: n[0].sym.magic else: mNone - # this list is subtle, we try to answer the question if after 'dest = f(src)' - # there is a connection betwen 'src' and 'dest' so that mutations to 'src' - # also reflect 'dest': - if magic in {mNone, mMove, mSlice, mAppendStrCh, mAppendStrStr, mAppendSeqElem, mArrToSeq}: - for i in 1.. Date: Fri, 25 Jun 2021 14:12:23 +0200 Subject: [PATCH 0160/1654] fixes #18240 (#18354) * ORC: track escaping parameters properly * fixes #18240 --- compiler/sempass2.nim | 37 +++++++++++++++++++++++++------------ tests/arc/torcmisc.nim | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index e66c9596a..e124baf26 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -77,6 +77,7 @@ type config: ConfigRef graph: ModuleGraph c: PContext + escapingParams: IntSet PEffects = var TEffects proc `<`(a, b: TLockLevel): bool {.borrow.} @@ -900,6 +901,25 @@ proc castBlock(tracked: PEffects, pragma: PNode, bc: var PragmaBlockContext) = localError(tracked.config, pragma.info, "invalid pragma block: " & $pragma) +proc trackInnerProc(tracked: PEffects, n: PNode) = + case n.kind + of nkSym: + let s = n.sym + if s.kind == skParam and s.owner == tracked.owner: + tracked.escapingParams.incl s.id + of nkNone..pred(nkSym), succ(nkSym)..nkNilLit: + discard + of nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, nkLambda, nkFuncDef, nkDo: + if n[0].kind == nkSym and n[0].sym.ast != nil: + trackInnerProc(tracked, getBody(tracked.graph, n[0].sym)) + of nkTypeSection, nkMacroDef, nkTemplateDef, nkError, + nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, + nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, + nkTypeOfExpr, nkMixinStmt, nkBindStmt: + discard + else: + for ch in n: trackInnerProc(tracked, ch) + proc track(tracked: PEffects, n: PNode) = case n.kind of nkSym: @@ -1095,8 +1115,10 @@ proc track(tracked: PEffects, n: PNode) = track(tracked, n.lastSon) unapplyBlockContext(tracked, bc) - of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, - nkMacroDef, nkTemplateDef, nkLambda, nkDo, nkFuncDef: + of nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef, nkLambda, nkFuncDef, nkDo: + if n[0].kind == nkSym and n[0].sym.ast != nil: + trackInnerProc(tracked, getBody(tracked.graph, n[0].sym)) + of nkTypeSection, nkMacroDef, nkTemplateDef: discard of nkCast: if n.len == 2: @@ -1259,15 +1281,6 @@ proc hasRealBody(s: PSym): bool = ## which is not a real implementation, refs #14314 result = {sfForward, sfImportc} * s.flags == {} -proc maybeWrappedInClosure(tracked: PEffects; t: PType): bool {.inline.} = - ## The spec does say when to produce destructors. However, the spec - ## was written in mind with the idea that "lambda lifting" already - ## happened. Not true in our implementation, so we need to workaround - ## here: - result = tracked.isInnerProc and - sfSystemModule notin tracked.c.module.flags and - tfCheckedForDestructor notin t.flags and containsGarbageCollectedRef(t) - proc trackProc*(c: PContext; s: PSym, body: PNode) = let g = c.graph var effects = s.typ.n[0] @@ -1287,7 +1300,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let typ = param.typ if isSinkTypeForParam(typ) or (t.config.selectedGC in {gcArc, gcOrc} and - (isClosure(typ.skipTypes(abstractInst)) or maybeWrappedInClosure(t, typ))): + (isClosure(typ.skipTypes(abstractInst)) or param.id in t.escapingParams)): createTypeBoundOps(t, typ, param.info) when false: if typ.kind == tyOut and param.id notin t.init: diff --git a/tests/arc/torcmisc.nim b/tests/arc/torcmisc.nim index 20dd18fb3..e41ad7c77 100644 --- a/tests/arc/torcmisc.nim +++ b/tests/arc/torcmisc.nim @@ -32,3 +32,35 @@ when true: waitFor hello(Flags()) echo "success" +# bug #18240 +import tables + +type + TopicHandler* = proc(topic: string, + data: seq[byte]): Future[void] {.gcsafe, raises: [Defect].} + + PeerID* = object + data*: seq[byte] + + PeerInfo* = ref object of RootObj + peerId*: PeerID + + Connection* = ref object of RootObj + peerInfo*: PeerInfo + + PubSubPeer* = ref object of RootObj + codec*: string + + PubSub* = ref object of RootObj + topics*: Table[string, seq[TopicHandler]] + peers*: Table[PeerID, PubSubPeer] + +proc getOrCreatePeer*(myParam: PubSub, peerId: PeerID, protos: seq[string]): PubSubPeer = + myParam.peers.withValue(peerId, peer): + return peer[] + +method handleConn*(myParam: PubSub, + conn: Connection, + proto: string) {.base, async.} = + myParam.peers.withValue(conn.peerInfo.peerId, peer): + let peerB = peer[] From 8535b26a754b2daea46627019e3145d7c12d826e Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 25 Jun 2021 07:47:10 -0700 Subject: [PATCH 0161/1654] docs now show nimExperimentalX APIs (#18345) * docs now show nimExperimentalX APIs * fix for windows --- lib/std/jsfetch.nim | 2 +- lib/wrappers/linenoise/linenoise.nim | 12 ++++++------ tests/config.nims | 3 +++ tools/kochdocs.nim | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 36bc772d5..1e7bb4d29 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -3,7 +3,7 @@ when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -when defined(nimExperimentalJsfetch) or defined(nimdoc): +when defined(nimExperimentalJsfetch): import std/[asyncjs, jsheaders, jsformdata] from std/httpcore import HttpMethod from std/jsffi import JsObject diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index 5b1fa7190..186b3b252 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -32,17 +32,17 @@ proc printKeyCodes*() {.importc: "linenoisePrintKeyCodes".} proc free*(s: cstring) {.importc: "free", header: "".} -when defined nimExperimentalLinenoiseExtra: +when defined(nimExperimentalLinenoiseExtra) and not defined(windows): # C interface - type linenoiseStatus = enum + type LinenoiseStatus = enum linenoiseStatus_ctrl_unknown linenoiseStatus_ctrl_C linenoiseStatus_ctrl_D - type linenoiseData* = object - status: linenoiseStatus + type LinenoiseData* = object + status: LinenoiseStatus - proc linenoiseExtra(prompt: cstring, data: ptr linenoiseData): cstring {.importc.} + proc linenoiseExtra(prompt: cstring, data: ptr LinenoiseData): cstring {.importc.} # stable nim interface type Status* = enum @@ -65,7 +65,7 @@ when defined nimExperimentalLinenoiseExtra: if ret.line.len > 0: echo ret.line if ret.status == lnCtrlD: break echo "exiting" - var data: linenoiseData + var data: LinenoiseData let buf = linenoiseExtra(prompt, data.addr) result.line = $buf free(buf) diff --git a/tests/config.nims b/tests/config.nims index 578b69e76..539de5e8d 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -30,4 +30,7 @@ hint("Processing", off) # switch("hint", "ConvFromXtoItselfNotNeeded") # experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 +# sync with `kochdocs.docDefines` or refactor. switch("define", "nimExperimentalAsyncjsThen") +switch("define", "nimExperimentalJsfetch") +switch("define", "nimExperimentalLinenoiseExtra") diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index f25564fad..469059348 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -8,7 +8,8 @@ const gaCode* = " --doc.googleAnalytics:UA-48159761-1" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$#" % system.NimVersion + docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalJsfetch -d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" From f6bea08eac7938e460ca9088693f0cf3f274e75b Mon Sep 17 00:00:00 2001 From: tomc1998 Date: Fri, 25 Jun 2021 21:06:24 +0100 Subject: [PATCH 0162/1654] Add sink annotation to option some() (#18358) --- lib/pure/options.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/options.nim b/lib/pure/options.nim index bd5aa389e..850bfa555 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -92,7 +92,7 @@ type UnpackDefect* = object of Defect UnpackError* {.deprecated: "See corresponding Defect".} = UnpackDefect -proc option*[T](val: T): Option[T] {.inline.} = +proc option*[T](val: sink T): Option[T] {.inline.} = ## Can be used to convert a pointer type (`ptr`, `pointer`, `ref` or `proc`) to an option type. ## It converts `nil` to `none(T)`. When `T` is no pointer type, this is equivalent to `some(val)`. ## @@ -112,7 +112,7 @@ proc option*[T](val: T): Option[T] {.inline.} = when T isnot SomePointer: result.has = true -proc some*[T](val: T): Option[T] {.inline.} = +proc some*[T](val: sink T): Option[T] {.inline.} = ## Returns an `Option` that has the value `val`. ## ## **See also:** From 39fbf3c84bd83613407e22b3de215d9a221b9422 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Jun 2021 03:39:52 +0200 Subject: [PATCH 0163/1654] ensure 'koch boot --gc:orc' stays green (#18353) * ensure 'koch boot --gc:orc' stays green * disable for C++ code generator for now --- koch.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/koch.nim b/koch.nim index 51be53cc7..6a0b7a0d3 100644 --- a/koch.nim +++ b/koch.nim @@ -582,6 +582,11 @@ proc runCI(cmd: string) = when defined(posix): execFold("Run nimsuggest tests", "nim r nimsuggest/tester") + when not defined(bsd): + if not doUseCpp: + # the BSDs are overwhelmed already, so only run this test on the other machines: + kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") + proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) xz(false, cmdLineRest) From b8f761b7e2cb3f28abd6486d28ea19228887cdf5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 26 Jun 2021 06:21:46 -0700 Subject: [PATCH 0164/1654] even lighter version of #17938: fix most issues with UnusedImport, XDeclaredButNotUsed, etc; fix #17511, #17510, #14246 (without realModule) (#18362) * {.used: symbol} * add tests * fix tests with --import * --import works without giving spurious unused warnings * new warning warnDuplicateModuleImport for `import foo; import foo` * fix test, add resolveModuleAlias, use proper line info for module aliases * fix spurious warnings * fix deprecation msg for deprecated modules even with `import foo as bar` * disable a test for i386 pending sorting XDeclaredButNotUsed errors * UnusedImport now works with re-exported symbols * fix typo [skip ci] * ic support * add genPNode to allow writing PNode-based compiler code similarly to `genAst` * fix DuplicateModuleImport warning * adjust test * fixup * fixup * fixup * fix after rebase * fix for IC * keep the proc inline, move the const out * [skip ci] fix changelog * experiment: remove calls to resolveModuleAlias * followup * fixup * fix tests/modules/tselfimport.nim * workaround tests/deprecated/tmodule1.nim * fix properly * simplify --- compiler/importer.nim | 34 +++++++++-------- compiler/lineinfos.nim | 2 + compiler/lookups.nim | 18 +++++---- compiler/passaux.nim | 2 +- lib/system/schubfach.nim | 59 +++++++++++++++--------------- testament/categories.nim | 1 + tests/deprecated/tmodule1.nim | 22 ++++++++--- tests/modules/tselfimport.nim | 2 +- tests/pragmas/mused2a.nim | 23 ++++++++++++ tests/pragmas/mused2b.nim | 3 ++ tests/pragmas/mused2c.nim | 1 + tests/pragmas/tused2.nim | 46 +++++++++++++++++++++++ tests/statictypes/tstatictypes.nim | 2 +- 13 files changed, 155 insertions(+), 60 deletions(-) create mode 100644 tests/pragmas/mused2a.nim create mode 100644 tests/pragmas/mused2b.nim create mode 100644 tests/pragmas/mused2c.nim create mode 100644 tests/pragmas/tused2.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index 692cdb604..7a5c4de4a 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -13,6 +13,7 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, sets, modulegraphs, wordrecg +from strutils import `%` proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} @@ -224,19 +225,20 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym; im proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool): PSym = result = realModule - c.unusedImports.add((realModule, n.info)) template createModuleAliasImpl(ident): untyped = - createModuleAlias(realModule, nextSymId c.idgen, ident, realModule.info, c.config.options) + createModuleAlias(realModule, nextSymId c.idgen, ident, n.info, c.config.options) if n.kind != nkImportAs: discard elif n.len != 2 or n[1].kind != nkIdent: localError(c.config, n.info, "module alias must be an identifier") elif n[1].ident.id != realModule.name.id: # some misguided guy will write 'import abc.foo as foo' ... result = createModuleAliasImpl(n[1].ident) + if result == realModule: + # avoids modifying `realModule`, see D20201209T194412 for `import {.all.}` + result = createModuleAliasImpl(realModule.name) if importHidden: - if result == realModule: # avoids modifying `realModule`, see D20201209T194412. - result = createModuleAliasImpl(realModule.name) result.options.incl optImportHidden + c.unusedImports.add((result, n.info)) proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] = var ret: typeof(result) @@ -274,23 +276,22 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = toFullPath(c.config, c.graph.importStack[i+1]) c.recursiveDep = err + var realModule: PSym discard pushOptionEntry(c) - result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f), transf.importHidden) + realModule = c.graph.importModuleCallback(c.graph, c.module, f) + result = importModuleAs(c, n, realModule, transf.importHidden) popOptionEntry(c) #echo "set back to ", L c.graph.importStack.setLen(L) # we cannot perform this check reliably because of - # test: modules/import_in_config) - when true: - if result.info.fileIndex == c.module.info.fileIndex and - result.info.fileIndex == n.info.fileIndex: - localError(c.config, n.info, "A module cannot import itself") - if sfDeprecated in result.flags: - if result.constraint != nil: - message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated") - else: - message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated") + # test: modules/import_in_config) # xxx is that still true? + if realModule == c.module: + localError(c.config, n.info, "module '$1' cannot import itself" % realModule.name.s) + if sfDeprecated in realModule.flags: + var prefix = "" + if realModule.constraint != nil: prefix = realModule.constraint.strVal & "; " + message(c.config, n.info, warnDeprecated, prefix & realModule.name.s & " is deprecated") suggestSym(c.graph, n.info, result, c.graph.usageSym, false) importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) @@ -303,6 +304,9 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = addDecl(c, m, it.info) # add symbol to symbol table of module importAllSymbols(c, m) #importForwarded(c, m.ast, emptySet, m) + for s in allSyms(c.graph, m): # fixes bug #17510, for re-exported symbols + if s.owner != m: + c.exportIndirections.incl((m.id, s.id)) proc evalImport*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index abe6bce7c..11fa45480 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -67,6 +67,7 @@ type warnResultUsed = "ResultUsed", warnCannotOpen = "CannotOpen", warnFileChanged = "FileChanged", + warnDuplicateModuleImport = "DuplicateModuleImport", warnUser = "User", # hints hintSuccess = "Success", hintSuccessX = "SuccessX", @@ -148,6 +149,7 @@ const warnResultUsed: "used 'result' variable", warnCannotOpen: "cannot open: $1", warnFileChanged: "file changed: $1", + warnDuplicateModuleImport: "$1", warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` diff --git a/compiler/lookups.nim b/compiler/lookups.nim index ee79673d0..0e057a79a 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -58,8 +58,8 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent = template addSym*(scope: PScope, s: PSym) = strTableAdd(scope.symbols, s) -proc addUniqueSym*(scope: PScope, s: PSym): PSym = - result = strTableInclReportConflict(scope.symbols, s) +proc addUniqueSym*(scope: PScope, s: PSym, onConflictKeepOld: bool): PSym = + result = strTableInclReportConflict(scope.symbols, s, onConflictKeepOld) proc openScope*(c: PContext): PScope {.discardable.} = result = PScope(parent: c.currentScope, @@ -288,19 +288,23 @@ proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) = message(c.config, s.info, hintXDeclaredButNotUsed, s.name.s) proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; - conflictsWith: TLineInfo) = + conflictsWith: TLineInfo, note = errGenerated) = ## Emit a redefinition error if in non-interactive mode if c.config.cmd != cmdInteractive: - localError(c.config, info, + localError(c.config, info, note, "redefinition of '$1'; previous declaration here: $2" % [s, c.config $ conflictsWith]) # xxx pending bootstrap >= 1.4, replace all those overloads with a single one: # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = - let conflict = scope.addUniqueSym(sym) + let conflict = scope.addUniqueSym(sym, onConflictKeepOld = true) if conflict != nil: - wrongRedefinition(c, info, sym.name.s, conflict.info) + var note = errGenerated + if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: + # import foo; import foo + note = warnDuplicateModuleImport + wrongRedefinition(c, info, sym.name.s, conflict.info, note) proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} = addDeclAt(c, scope, sym, sym.info) @@ -312,7 +316,7 @@ proc addDecl*(c: PContext, sym: PSym) {.inline.} = addDeclAt(c, c.currentScope, sym) proc addPrelimDecl*(c: PContext, sym: PSym) = - discard c.currentScope.addUniqueSym(sym) + discard c.currentScope.addUniqueSym(sym, onConflictKeepOld = false) from ic / ic import addHidden diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 9abc97be4..87cb3a730 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -10,7 +10,7 @@ ## implements some little helper passes import - ast, passes, idents, msgs, options, lineinfos + ast, passes, msgs, options, lineinfos from modulegraphs import ModuleGraph, PPassContext diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index 5f974a79e..d072cde99 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -95,6 +95,36 @@ proc floorLog2Pow10(e: int32): int32 {.inline.} = sf_Assert(e <= 1233) return floorDivPow2(e * 1741647, 19) +const + kMin: int32 = -31 + kMax: int32 = 45 + g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, + 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, + 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, + 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, + 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, + 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, + 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, + 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, + 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, + 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, + 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, + 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, + 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, + 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, + 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, + 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, + 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, + 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, + 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, + 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, + 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, + 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, + 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, + 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, + 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, + 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] + proc computePow10Single(k: int32): uint64 {.inline.} = ## There are unique beta and r such that 10^k = beta 2^r and ## 2^63 <= beta < 2^64, namely r = floor(log_2 10^k) - 63 and @@ -103,35 +133,6 @@ proc computePow10Single(k: int32): uint64 {.inline.} = ## value being a pretty good overestimate for 10^k. ## NB: Since for all the required exponents k, we have g < 2^64, ## all constants can be stored in 128-bit integers. - const - kMin: int32 = -31 - kMax: int32 = 45 - g: array[kMax - kMin + 1, uint64] = [0x81CEB32C4B43FCF5'u64, 0xA2425FF75E14FC32'u64, - 0xCAD2F7F5359A3B3F'u64, 0xFD87B5F28300CA0E'u64, 0x9E74D1B791E07E49'u64, - 0xC612062576589DDB'u64, 0xF79687AED3EEC552'u64, 0x9ABE14CD44753B53'u64, - 0xC16D9A0095928A28'u64, 0xF1C90080BAF72CB2'u64, 0x971DA05074DA7BEF'u64, - 0xBCE5086492111AEB'u64, 0xEC1E4A7DB69561A6'u64, 0x9392EE8E921D5D08'u64, - 0xB877AA3236A4B44A'u64, 0xE69594BEC44DE15C'u64, 0x901D7CF73AB0ACDA'u64, - 0xB424DC35095CD810'u64, 0xE12E13424BB40E14'u64, 0x8CBCCC096F5088CC'u64, - 0xAFEBFF0BCB24AAFF'u64, 0xDBE6FECEBDEDD5BF'u64, 0x89705F4136B4A598'u64, - 0xABCC77118461CEFD'u64, 0xD6BF94D5E57A42BD'u64, 0x8637BD05AF6C69B6'u64, - 0xA7C5AC471B478424'u64, 0xD1B71758E219652C'u64, 0x83126E978D4FDF3C'u64, - 0xA3D70A3D70A3D70B'u64, 0xCCCCCCCCCCCCCCCD'u64, 0x8000000000000000'u64, - 0xA000000000000000'u64, 0xC800000000000000'u64, 0xFA00000000000000'u64, - 0x9C40000000000000'u64, 0xC350000000000000'u64, 0xF424000000000000'u64, - 0x9896800000000000'u64, 0xBEBC200000000000'u64, 0xEE6B280000000000'u64, - 0x9502F90000000000'u64, 0xBA43B74000000000'u64, 0xE8D4A51000000000'u64, - 0x9184E72A00000000'u64, 0xB5E620F480000000'u64, 0xE35FA931A0000000'u64, - 0x8E1BC9BF04000000'u64, 0xB1A2BC2EC5000000'u64, 0xDE0B6B3A76400000'u64, - 0x8AC7230489E80000'u64, 0xAD78EBC5AC620000'u64, 0xD8D726B7177A8000'u64, - 0x878678326EAC9000'u64, 0xA968163F0A57B400'u64, 0xD3C21BCECCEDA100'u64, - 0x84595161401484A0'u64, 0xA56FA5B99019A5C8'u64, 0xCECB8F27F4200F3A'u64, - 0x813F3978F8940985'u64, 0xA18F07D736B90BE6'u64, 0xC9F2C9CD04674EDF'u64, - 0xFC6F7C4045812297'u64, 0x9DC5ADA82B70B59E'u64, 0xC5371912364CE306'u64, - 0xF684DF56C3E01BC7'u64, 0x9A130B963A6C115D'u64, 0xC097CE7BC90715B4'u64, - 0xF0BDC21ABB48DB21'u64, 0x96769950B50D88F5'u64, 0xBC143FA4E250EB32'u64, - 0xEB194F8E1AE525FE'u64, 0x92EFD1B8D0CF37BF'u64, 0xB7ABC627050305AE'u64, - 0xE596B7B0C643C71A'u64, 0x8F7E32CE7BEA5C70'u64, 0xB35DBF821AE4F38C'u64] sf_Assert(k >= kMin) sf_Assert(k <= kMax) return g[k - kMin] diff --git a/testament/categories.nim b/testament/categories.nim index bf7855b2c..b5e8a5454 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -508,6 +508,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; const tempExt = "_temp.nim" for it in walkDirRec(testsDir): + # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test if isTestFile(it) and not it.endsWith(tempExt): let nimcache = nimcacheDir(it, options, getTestSpecTarget()) removeDir(nimcache) diff --git a/tests/deprecated/tmodule1.nim b/tests/deprecated/tmodule1.nim index 954836889..f26e4ce3f 100644 --- a/tests/deprecated/tmodule1.nim +++ b/tests/deprecated/tmodule1.nim @@ -1,13 +1,23 @@ discard """ - nimout: '''tmodule1.nim(11, 8) Warning: goodbye; importme is deprecated [Deprecated] -tmodule1.nim(14, 10) Warning: Ty is deprecated [Deprecated] -tmodule1.nim(17, 10) Warning: hello; Ty1 is deprecated [Deprecated] -tmodule1.nim(20, 8) Warning: aVar is deprecated [Deprecated] -tmodule1.nim(22, 3) Warning: aProc is deprecated [Deprecated] -tmodule1.nim(23, 3) Warning: hello; aProc1 is deprecated [Deprecated] + matrix: "--hint:all:off" + nimoutFull: true + nimout: ''' +tmodule1.nim(21, 8) Warning: goodbye; importme is deprecated [Deprecated] +tmodule1.nim(24, 10) Warning: Ty is deprecated [Deprecated] +tmodule1.nim(27, 10) Warning: hello; Ty1 is deprecated [Deprecated] +tmodule1.nim(30, 8) Warning: aVar is deprecated [Deprecated] +tmodule1.nim(32, 3) Warning: aProc is deprecated [Deprecated] +tmodule1.nim(33, 3) Warning: hello; aProc1 is deprecated [Deprecated] ''' """ + + + + + + +# line 20 import importme block: diff --git a/tests/modules/tselfimport.nim b/tests/modules/tselfimport.nim index 7e50bef7c..ba5d9b4cf 100644 --- a/tests/modules/tselfimport.nim +++ b/tests/modules/tselfimport.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "A module cannot import itself" + errormsg: "module 'tselfimport' cannot import itself" file: "tselfimport.nim" line: 7 """ diff --git a/tests/pragmas/mused2a.nim b/tests/pragmas/mused2a.nim new file mode 100644 index 000000000..d9b2bb9bf --- /dev/null +++ b/tests/pragmas/mused2a.nim @@ -0,0 +1,23 @@ +import std/strutils + +from std/os import fileExists + +import std/typetraits as typetraits2 +from std/setutils import complement + + + + + +proc fn1() = discard +proc fn2*() = discard + + +let fn4 = 0 +let fn5* = 0 + + +const fn7 = 0 +const fn8* = 0 + +type T1 = object diff --git a/tests/pragmas/mused2b.nim b/tests/pragmas/mused2b.nim new file mode 100644 index 000000000..39c92b964 --- /dev/null +++ b/tests/pragmas/mused2b.nim @@ -0,0 +1,3 @@ +import mused2c +export mused2c + diff --git a/tests/pragmas/mused2c.nim b/tests/pragmas/mused2c.nim new file mode 100644 index 000000000..a374e634e --- /dev/null +++ b/tests/pragmas/mused2c.nim @@ -0,0 +1 @@ +proc baz*() = discard \ No newline at end of file diff --git a/tests/pragmas/tused2.nim b/tests/pragmas/tused2.nim new file mode 100644 index 000000000..f80c198d8 --- /dev/null +++ b/tests/pragmas/tused2.nim @@ -0,0 +1,46 @@ +discard """ + matrix: "--hint:all:off --hint:XDeclaredButNotUsed --path:." + joinable: false + nimoutFull: true + nimout: ''' +mused2a.nim(12, 6) Hint: 'fn1' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(16, 5) Hint: 'fn4' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(20, 7) Hint: 'fn7' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(23, 6) Hint: 'T1' is declared but not used [XDeclaredButNotUsed] +mused2a.nim(1, 11) Warning: imported and not used: 'strutils' [UnusedImport] +mused2a.nim(3, 9) Warning: imported and not used: 'os' [UnusedImport] +mused2a.nim(5, 23) Warning: imported and not used: 'typetraits2' [UnusedImport] +mused2a.nim(6, 9) Warning: imported and not used: 'setutils' [UnusedImport] +tused2.nim(42, 8) Warning: imported and not used: 'mused2a' [UnusedImport] +tused2.nim(45, 11) Warning: imported and not used: 'strutils' [UnusedImport] +''' +""" + + + + + + + + + + + + + + + + + + + + + + +# line 40 + +import mused2a +import mused2b + +import std/strutils +baz() diff --git a/tests/statictypes/tstatictypes.nim b/tests/statictypes/tstatictypes.nim index 16cb55c28..5df3f35fd 100644 --- a/tests/statictypes/tstatictypes.nim +++ b/tests/statictypes/tstatictypes.nim @@ -271,7 +271,7 @@ block: fails(foo) -import macros, tables +import tables var foo{.compileTime.} = [ "Foo", From 1b9b8060075efab82912dc33ba64e671d102b999 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 26 Jun 2021 19:10:46 +0300 Subject: [PATCH 0165/1654] rst: fix bug 20 from #17340 (#18360) and a leftover bug: priority of option list inside definition list --- lib/packages/docutils/rst.nim | 4 ++- tests/stdlib/trst.nim | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 8ffbbd4d3..419cf50e7 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2245,7 +2245,7 @@ proc parseOptionList(p: var RstParser): PRstNode = popInd(p) else: parseLine(p, b) - if currentTok(p).kind == tkIndent: inc p.idx + while currentTok(p).kind == tkIndent: inc p.idx c.add(a) c.add(b) c.order = order; inc order @@ -2262,6 +2262,8 @@ proc parseDefinitionList(p: var RstParser): PRstNode = var col = currentTok(p).col result = newRstNodeA(p, rnDefList) while true: + if isOptionList(p): + break # option list has priority over def.list j = p.idx var a = newRstNode(rnDefName) parseLine(p, a) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index d53092d3b..fb9552479 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -83,6 +83,53 @@ suite "RST parsing": rnLeaf 'set' """) + test "items of 1 option list can be separated by blank lines": + check(dedent""" + -a desc1 + + -b desc2 + """.toAst == + dedent""" + rnOptionList + rnOptionListItem order=1 + rnOptionGroup + rnLeaf '-' + rnLeaf 'a' + rnDescription + rnLeaf 'desc1' + rnOptionListItem order=2 + rnOptionGroup + rnLeaf '-' + rnLeaf 'b' + rnDescription + rnLeaf 'desc2' + """) + + test "option list has priority over definition list": + check(dedent""" + defName + defBody + + -b desc2 + """.toAst == + dedent""" + rnInner + rnDefList + rnDefItem + rnDefName + rnLeaf 'defName' + rnDefBody + rnInner + rnLeaf 'defBody' + rnOptionList + rnOptionListItem order=1 + rnOptionGroup + rnLeaf '-' + rnLeaf 'b' + rnDescription + rnLeaf 'desc2' + """) + test "RST comment": check(dedent""" .. comment1 From 0b7361e938134305d6893ee2876b5fc8f9ba419b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 27 Jun 2021 11:39:16 -0700 Subject: [PATCH 0166/1654] followup #18362: make `UnusedImport` work robustly (#18366) * warnDuplicateModuleImport => hintDuplicateModuleImport * improve DuplicateModuleImport msg, add test --- compiler/cgen.nim | 2 - compiler/dfa.nim | 2 +- compiler/docgen.nim | 2 +- compiler/importer.nim | 16 +++++-- compiler/injectdestructors.nim | 6 +-- compiler/jsgen.nim | 2 - compiler/lineinfos.nim | 6 +-- compiler/lookups.nim | 12 ++++-- compiler/semdata.nim | 3 +- compiler/suggest.nim | 5 ++- tests/misc/trunner.nim | 26 +++++++++++- tests/pragmas/mused3.nim | 76 ++++++++++++++++++++++++++++++++++ tests/pragmas/mused3a.nim | 41 ++++++++++++++++++ tests/pragmas/mused3b.nim | 12 ++++++ 14 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 tests/pragmas/mused3.nim create mode 100644 tests/pragmas/mused3a.nim create mode 100644 tests/pragmas/mused3b.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5d6de2a03..3b004d399 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -24,8 +24,6 @@ import strutils except `%` # collides with ropes.`%` from ic / ic import ModuleBackendFlag from modulegraphs import ModuleGraph, PPassContext -from lineinfos import - warnGcMem, errXMustBeCompileTime, hintDependency, errGenerated, errCannotOpenFile import dynlib when not declared(dynlib.libCandidates): diff --git a/compiler/dfa.nim b/compiler/dfa.nim index ea60dbc59..402945042 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -29,7 +29,7 @@ ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen. ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf -import ast, types, intsets, lineinfos, renderer +import ast, intsets, lineinfos, renderer import std/private/asciitables type diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 08311eade..e6bb7cef8 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -16,7 +16,7 @@ import packages/docutils/rst, packages/docutils/rstgen, json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, trees, tables, nimpaths, renderverbatim, osproc + pathutils, tables, nimpaths, renderverbatim, osproc from uri import encodeUrl from std/private/globs import nativeToUnixPath diff --git a/compiler/importer.nim b/compiler/importer.nim index 7a5c4de4a..af392f849 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -12,7 +12,7 @@ import intsets, ast, astalgo, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, sets, - modulegraphs, wordrecg + modulegraphs, wordrecg, tables from strutils import `%` proc readExceptSet*(c: PContext, n: PNode): IntSet = @@ -239,6 +239,7 @@ proc importModuleAs(c: PContext; n: PNode, realModule: PSym, importHidden: bool) if importHidden: result.options.incl optImportHidden c.unusedImports.add((result, n.info)) + c.importModuleMap[result.id] = realModule.id proc transformImportAs(c: PContext; n: PNode): tuple[node: PNode, importHidden: bool] = var ret: typeof(result) @@ -296,6 +297,13 @@ proc myImportModule(c: PContext, n: var PNode, importStmtResult: PNode): PSym = importStmtResult.add newSymNode(result, n.info) #newStrNode(toFullPath(c.config, f), n.info) +proc afterImport(c: PContext, m: PSym) = + # fixes bug #17510, for re-exported symbols + let realModuleId = c.importModuleMap[m.id] + for s in allSyms(c.graph, m): + if s.owner.id != realModuleId: + c.exportIndirections.incl((m.id, s.id)) + proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = var it = it let m = myImportModule(c, it, importStmtResult) @@ -304,9 +312,7 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) = addDecl(c, m, it.info) # add symbol to symbol table of module importAllSymbols(c, m) #importForwarded(c, m.ast, emptySet, m) - for s in allSyms(c.graph, m): # fixes bug #17510, for re-exported symbols - if s.owner != m: - c.exportIndirections.incl((m.id, s.id)) + afterImport(c, m) proc evalImport*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) @@ -345,6 +351,7 @@ proc evalFrom*(c: PContext, n: PNode): PNode = if n[i].kind != nkNilLit: importSymbol(c, n[i], m, im.imported) c.addImport im + afterImport(c, m) proc evalImportExcept*(c: PContext, n: PNode): PNode = result = newNodeI(nkImportStmt, n.info) @@ -355,3 +362,4 @@ proc evalImportExcept*(c: PContext, n: PNode): PNode = addDecl(c, m, n.info) # add symbol to symbol table of module importAllSymbolsExcept(c, m, readExceptSet(c, n)) #importForwarded(c, m.ast, exceptSet, m) + afterImport(c, m) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 8ef08adeb..6447557c5 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -15,7 +15,7 @@ import intsets, strtabs, ast, astalgo, msgs, renderer, magicsys, types, idents, - strutils, options, dfa, lowerings, tables, modulegraphs, msgs, + strutils, options, dfa, lowerings, tables, modulegraphs, lineinfos, parampatterns, sighashes, liftdestructors, optimizer, varpartitions @@ -73,7 +73,7 @@ proc nestedScope(parent: var Scope): Scope = proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope; isDecl = false): PNode -import sets, hashes, tables +import sets, hashes proc hash(n: PNode): Hash = hash(cast[pointer](n)) @@ -245,8 +245,6 @@ template isUnpackedTuple(n: PNode): bool = ## hence unpacked tuples themselves don't need to be destroyed (n.kind == nkSym and n.sym.kind == skTemp and n.sym.typ.kind == tyTuple) -from strutils import parseInt - proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) = var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">" if (opname == "=" or opname == "=copy") and ri != nil: diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 3b183b9d6..393ac83c2 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -37,8 +37,6 @@ import import json, sets, math, tables, intsets, strutils -from modulegraphs import ModuleGraph, PPassContext - type TJSGen = object of PPassContext module: PSym diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 11fa45480..131fe480b 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -67,12 +67,12 @@ type warnResultUsed = "ResultUsed", warnCannotOpen = "CannotOpen", warnFileChanged = "FileChanged", - warnDuplicateModuleImport = "DuplicateModuleImport", warnUser = "User", # hints hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC", - hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed", + hintLineTooLong = "LineTooLong", + hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX", hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin", @@ -149,7 +149,6 @@ const warnResultUsed: "used 'result' variable", warnCannotOpen: "cannot open: $1", warnFileChanged: "file changed: $1", - warnDuplicateModuleImport: "$1", warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` @@ -157,6 +156,7 @@ const hintCC: "CC: $1", hintLineTooLong: "line too long", hintXDeclaredButNotUsed: "'$1' is declared but not used", + hintDuplicateModuleImport: "$1", hintXCannotRaiseY: "$1", hintConvToBaseNotNeeded: "conversion to base object is not needed", hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless", diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 0e057a79a..9b878dd4b 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -300,11 +300,15 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = let conflict = scope.addUniqueSym(sym, onConflictKeepOld = true) if conflict != nil: - var note = errGenerated if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: - # import foo; import foo - note = warnDuplicateModuleImport - wrongRedefinition(c, info, sym.name.s, conflict.info, note) + # e.g.: import foo; import foo + # xxx we could refine this by issuing a different hint for the case + # where a duplicate import happens inside an include. + localError(c.config, info, hintDuplicateModuleImport, + "duplicate import of '$1'; previous import here: $2" % + [sym.name.s, c.config $ conflict.info]) + else: + wrongRedefinition(c, info, sym.name.s, conflict.info, errGenerated) proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) {.inline.} = addDeclAt(c, scope, sym, sym.info) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index f763a00e4..bd863505c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -156,7 +156,8 @@ type features*: set[Feature] inTypeContext*, inConceptDecl*: int unusedImports*: seq[(PSym, TLineInfo)] - exportIndirections*: HashSet[(int, int)] + exportIndirections*: HashSet[(int, int)] # (module.id, symbol.id) + importModuleMap*: Table[int, int] # (module.id, module.id) lastTLineInfo*: TLineInfo template config*(c: PContext): ConfigRef = c.graph.config diff --git a/compiler/suggest.nim b/compiler/suggest.nim index a5b4ac87d..eaa30040a 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -32,7 +32,7 @@ # included from sigmatch.nim -import algorithm, sets, prefixmatches, lineinfos, parseutils, linter +import algorithm, sets, prefixmatches, lineinfos, parseutils, linter, tables from wordrecg import wDeprecated, wError, wAddr, wYield when defined(nimsuggest): @@ -572,7 +572,8 @@ proc markOwnerModuleAsUsed(c: PContext; s: PSym) = var i = 0 while i <= high(c.unusedImports): let candidate = c.unusedImports[i][0] - if candidate == module or c.exportIndirections.contains((candidate.id, s.id)): + if candidate == module or c.importModuleMap.getOrDefault(candidate.id, int.low) == module.id or + c.exportIndirections.contains((candidate.id, s.id)): # mark it as used: c.unusedImports.del(i) else: diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 77ae10f3d..8426e9aee 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -31,7 +31,7 @@ const proc runNimCmd(file, options = "", rtarg = ""): auto = let fileabs = testsDir / file.unixToNativePath # doAssert fileabs.fileExists, fileabs # disabled because this allows passing `nim r --eval:code fakefile` - let cmd = fmt"{nim} {mode} {options} --hints:off {fileabs} {rtarg}" + let cmd = fmt"{nim} {mode} --hint:all:off {options} {fileabs} {rtarg}" result = execCmdEx(cmd) when false: # for debugging echo cmd @@ -352,5 +352,29 @@ running: v2 doAssert outp2 == "12345\n", outp2 doAssert status2 == 0 + block: # UnusedImport + proc fn(opt: string, expected: string) = + let output = runNimCmdChk("pragmas/mused3.nim", fmt"--warning:all:off --warning:UnusedImport --hint:DuplicateModuleImport {opt}") + doAssert output == expected, opt & "\noutput:\n" & output & "expected:\n" & expected + fn("-d:case1"): """ +mused3.nim(13, 8) Warning: imported and not used: 'mused3b' [UnusedImport] +""" + fn("-d:case2"): "" + fn("-d:case3"): "" + fn("-d:case4"): "" + fn("-d:case5"): "" + fn("-d:case6"): "" + fn("-d:case7"): "" + fn("-d:case8"): "" + fn("-d:case9"): "" + fn("-d:case10"): "" + when false: + fn("-d:case11"): """ + Warning: imported and not used: 'm2' [UnusedImport] + """ + fn("-d:case12"): """ +mused3.nim(75, 10) Hint: duplicate import of 'mused3a'; previous import here: mused3.nim(74, 10) [DuplicateModuleImport] +""" + else: discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled diff --git a/tests/pragmas/mused3.nim b/tests/pragmas/mused3.nim new file mode 100644 index 000000000..0beec1d44 --- /dev/null +++ b/tests/pragmas/mused3.nim @@ -0,0 +1,76 @@ +#[ +ran from trunner +]# + + + + + + +# line 10 +when defined case1: + from mused3a import nil + from mused3b import nil + mused3a.fn1() + +when defined case2: + from mused3a as m1 import nil + m1.fn1() + +when defined case3: + from mused3a import fn1 + fn1() + +when defined case4: + from mused3a as m1 import fn1 + m1.fn1() + +when defined case5: + import mused3a as m1 + fn1() + +when defined case6: + import mused3a except nonexistent + fn1() + +when defined case7: + import mused3a + mused3a.fn1() + +when defined case8: + # re-export test + import mused3a except nonexistent + gn1() + +when defined case9: + # re-export test + import mused3a + gn1() + +when defined case10: + #[ + edge case which happens a lot in compiler code: + don't report UnusedImport for mused3b here even though it works without `import mused3b`, + because `a.b0.f0` is accessible from both mused3a and mused3b (fields are given implicit access) + ]# + import mused3a + import mused3b + var a: Bar + discard a.b0.f0 + +when false: + when defined case11: + #[ + xxx minor bug: this should give: + Warning: imported and not used: 'm2' [UnusedImport] + but doesn't, because currently implementation in `markOwnerModuleAsUsed` + only looks at `fn1`, not fully qualified call `m1.fn1() + ]# + from mused3a as m1 import nil + from mused3a as m2 import nil + m1.fn1() + +when defined case12: + import mused3a + import mused3a + fn1() diff --git a/tests/pragmas/mused3a.nim b/tests/pragmas/mused3a.nim new file mode 100644 index 000000000..c33d1ecb3 --- /dev/null +++ b/tests/pragmas/mused3a.nim @@ -0,0 +1,41 @@ +when defined case1: + proc fn1*() = discard + +when defined case2: + proc fn1*() = discard + +when defined case3: + proc fn1*() = discard + proc fn2*() = discard + +when defined case4: + proc fn1*() = discard + proc fn2*() = discard + +when defined case5: + proc fn1*() = discard + +when defined case6: + proc fn1*() = discard + +when defined case7: + proc fn1*() = discard + +when defined case8: + import mused3b + export mused3b + +when defined case9: + import mused3b + export mused3b + +when defined case10: + import mused3b + type Bar* = object + b0*: Foo + +when defined case11: + proc fn1*() = discard + +when defined case12: + proc fn1*() = discard diff --git a/tests/pragmas/mused3b.nim b/tests/pragmas/mused3b.nim new file mode 100644 index 000000000..de288bb08 --- /dev/null +++ b/tests/pragmas/mused3b.nim @@ -0,0 +1,12 @@ +when defined case1: + proc gn1*()=discard + +when defined case8: + proc gn1*()=discard + +when defined case9: + proc gn1*()=discard + +when defined case10: + type Foo* = object + f0*: int From 0804b4b37d5fcc43d975f56a4c2c88dd58a0b882 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 27 Jun 2021 19:51:28 +0100 Subject: [PATCH 0167/1654] Switch packaging.rst to use makefile Minor cleanup --- doc/packaging.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/packaging.rst b/doc/packaging.rst index ecb17e482..fcdc4ae04 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -48,14 +48,15 @@ Hints on the build process: .. code:: cmd # build from C sources and then using koch - ./build.sh --os $os_type --cpu $cpu_arch - ./bin/nim c koch + make -j # supports parallel build + # alternatively: ./build.sh --os $os_type --cpu $cpu_arch + ./bin/nim c -d:release koch ./koch boot -d:release # optionally generate docs into doc/html ./koch docs - ./koch tools -d:release + ./koch tools # extract files to be really installed ./install.sh From 908b2cc2e44bededdcb943778f44b908a4f3c1df Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Sun, 27 Jun 2021 21:12:12 +0100 Subject: [PATCH 0168/1654] Switch IRC links to Libera Chat (#18370) --- readme.md | 4 ++-- tools/website.nimf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 37babf711..7c6e28be3 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( [![Follow @nim_lang on Twitter][badge-nim-twitter]][nim-twitter] * The [forum][nim-forum] - the best place to ask questions and to discuss Nim. -* [#nim IRC Channel (Freenode)][nim-irc] - a place to discuss Nim in real-time. +* [#nim IRC Channel (Libera Chat)][nim-irc] - a place to discuss Nim in real-time. Also where most development decisions get made. * [Discord][nim-discord] - an additional place to discuss Nim in real-time. Most channels there are bridged to IRC. @@ -206,7 +206,7 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nim-forum]: https://forum.nim-lang.org [nim-issues]: https://github.com/nim-lang/Nim/issues [nim-issues-easy]: https://github.com/nim-lang/Nim/labels/Easy -[nim-irc]: https://webchat.freenode.net/?channels=nim +[nim-irc]: https://web.libera.chat/#nim [nim-twitter]: https://twitter.com/nim_lang [nim-stackoverflow]: https://stackoverflow.com/questions/tagged/nim-lang [nim-stackoverflow-newest]: https://stackoverflow.com/questions/tagged/nim-lang?sort=newest&pageSize=15 diff --git a/tools/website.nimf b/tools/website.nimf index bde84b32f..cea30f74f 100644 --- a/tools/website.nimf +++ b/tools/website.nimf @@ -188,7 +188,7 @@ runForever()
From e720bbdd76303e2cc0a3a169e870859470a1da84 Mon Sep 17 00:00:00 2001 From: gemath <33119724+gemath@users.noreply.github.com> Date: Mon, 28 Jun 2021 12:33:20 +0200 Subject: [PATCH 0169/1654] Peg captures get stack-like behavior (#18369) * Implements reverse capture indexing. * Now works for modified backrefs too. * Changed reverse indexing syntax prefix for back-references to '$^'. --- doc/pegdocs.txt | 10 +++-- lib/pure/pegs.nim | 90 +++++++++++++++++++++++++++--------------- tests/stdlib/tpegs.nim | 28 +++++++++++++ 3 files changed, 94 insertions(+), 34 deletions(-) diff --git a/doc/pegdocs.txt b/doc/pegdocs.txt index 4c557aed8..0363d4874 100644 --- a/doc/pegdocs.txt +++ b/doc/pegdocs.txt @@ -27,7 +27,10 @@ notation meaning ``{E}`` Capture: Apply expression `E` and store the substring that matched `E` into a *capture* that can be accessed after the matching process. -``$i`` Back reference to the ``i``th capture. ``i`` counts from 1. +``{}`` Empty capture: Delete the last capture. No character + is consumed. +``$i`` Back reference to the ``i``th capture. ``i`` counts forwards + from 1 or backwards (last capture to first) from ^1. ``$`` Anchor: Matches at the end of the input. No character is consumed. Same as ``!.``. ``^`` Anchor: Matches at the start of the input. No character @@ -149,14 +152,15 @@ The PEG parser implements this grammar (written in PEG syntax):: rule <- identifier \s* "<-" expr ig identNoArrow <- identifier !(\s* "<-") prefixOpr <- ig '&' / ig '!' / ig '@' / ig '{@}' / ig '@@' - literal <- ig identifier? '$' [0-9]+ / '$' / '^' / + literal <- ig identifier? '$' '^'? [0-9]+ / '$' / '^' / ig identNoArrow / ig charset / ig stringlit / ig builtin / ig '.' / ig '_' / - (ig "(" expr ig ")") + (ig "(" expr ig ")") / + (ig "{" expr? ig "}") postfixOpr <- ig '?' / ig '*' / ig '+' primary <- prefixOpr* (literal postfixOpr*) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index b5d9b1d07..b6b05cdc1 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -83,7 +83,7 @@ type of pkChar, pkGreedyRepChar: ch: char of pkCharChoice, pkGreedyRepSet: charChoice: ref set[char] of pkNonTerminal: nt: NonTerminal - of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] + of pkBackRef..pkBackRefIgnoreStyle: index: range[-MaxSubpatterns..MaxSubpatterns-1] else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj @@ -106,7 +106,7 @@ proc nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[0..MaxSubpatterns] = p.index +proc index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -304,34 +304,37 @@ proc endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +proc capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns]): Peg {. +proc backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. noSideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. - result = Peg(kind: pkBackRef, index: index-1) + ## from 1. `reverse` specifies wether indexing starts from the end of the + ## capture list. + result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. +proc backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. noSideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores case for matching. - result = Peg(kind: pkBackRefIgnoreCase, index: index-1) + ## from 1. `reverse` specifies wether indexing starts from the end of the + ## capture list. Ignores case for matching. + result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. +proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. noSideEffect, rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting - ## from 1. Ignores style for matching. - result = Peg(kind: pkBackRefIgnoreStyle, index: index-1) + ## from 1. `reverse` specifies wether indexing starts from the end of the + ## capture list. Ignores style for matching. + result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) proc spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, - pkAny..pkWhitespace, pkGreedyAny: + pkAny..pkWhitespace, pkGreedyAny, pkBackRef..pkBackRefIgnoreStyle: result = 1 of pkNonTerminal: # we cannot inline a rule with a non-terminal @@ -561,8 +564,10 @@ template matchOrParse(mopProc: untyped) = # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for # *pkBackRef..pkBackRefIgnoreStyle* here. - if p.index >= c.ml: return -1 - var (a, b) = c.matches[p.index] + var index = p.index + if index < 0: index.inc(c.ml) + if index < 0 or index >= c.ml: return -1 + var (a, b) = c.matches[index] var n: Peg case p.kind of pkBackRef: @@ -822,15 +827,19 @@ template matchOrParse(mopProc: untyped) = leave(pkNotPredicate, s, p, start, result) of pkCapture: enter(pkCapture, s, p, start) - var idx = c.ml # reserve a slot for the subpattern - inc(c.ml) - result = mopProc(s, p.sons[0], start, c) - if result >= 0: - if idx < MaxSubpatterns: - c.matches[idx] = (start, start+result-1) - #else: silently ignore the capture + if p.sons.len == 0 or p.sons[0].kind == pkEmpty: + # empty capture removes last match + dec(c.ml) + c.matches[c.ml] = (0, 0) + result = 0 # match of length 0 else: - c.ml = idx + var idx = c.ml # reserve a slot for the subpattern + result = mopProc(s, p.sons[0], start, c) + if result >= 0: + inc(c.ml) + if idx < MaxSubpatterns: + c.matches[idx] = (start, start+result-1) + #else: silently ignore the capture leave(pkCapture, s, p, start, result) of pkBackRef: enter(pkBackRef, s, p, start) @@ -1395,6 +1404,7 @@ type tkCurlyLe, ## '{' tkCurlyRi, ## '}' tkCurlyAt, ## '{@}' + tkEmptyCurl, ## '{}' tkArrow, ## '<-' tkBar, ## '/' tkStar, ## '*' @@ -1427,7 +1437,7 @@ type const tokKindToStr: array[TokKind, string] = [ "invalid", "[EOF]", ".", "_", "identifier", "string literal", - "character set", "(", ")", "{", "}", "{@}", + "character set", "(", ")", "{", "}", "{@}", "{}", "<-", "/", "*", "+", "&", "!", "?", "@", "built-in", "escaped", "$", "$", "^" ] @@ -1564,13 +1574,21 @@ proc getString(c: var PegLexer, tok: var Token) = proc getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 + var neg = false + if pos < c.buf.len and c.buf[pos] == '^': + neg = true + inc(pos) if pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.kind = tkBackref tok.index = 0 while pos < c.buf.len and c.buf[pos] in {'0'..'9'}: tok.index = tok.index * 10 + ord(c.buf[pos]) - ord('0') inc(pos) + if neg: + tok.index = -tok.index else: + if neg: + dec(pos) tok.kind = tkDollar c.bufpos = pos @@ -1670,6 +1688,10 @@ proc getTok(c: var PegLexer, tok: var Token) = tok.kind = tkCurlyAt inc(c.bufpos, 2) add(tok.literal, "{@}") + elif c.buf[c.bufpos] == '}' and c.bufpos < c.buf.len: + tok.kind = tkEmptyCurl + inc(c.bufpos) + add(tok.literal, "{}") else: tok.kind = tkCurlyLe add(tok.literal, '{') @@ -1705,7 +1727,7 @@ proc getTok(c: var PegLexer, tok: var Token) = return if c.buf[c.bufpos] in {'\'', '"'} or c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and - c.buf[c.bufpos+1] in {'0'..'9'}: + c.buf[c.bufpos+1] in {'^', '0'..'9'}: case tok.literal of "i": tok.modifier = modIgnoreCase of "y": tok.modifier = modIgnoreStyle @@ -1819,10 +1841,13 @@ proc modifiedTerm(s: string, m: Modifier): Peg = of modIgnoreStyle: result = termIgnoreStyle(s) proc modifiedBackref(s: int, m: Modifier): Peg = + var + reverse = s < 0 + index = if reverse: -s else: s case m - of modNone, modVerbatim: result = backref(s) - of modIgnoreCase: result = backrefIgnoreCase(s) - of modIgnoreStyle: result = backrefIgnoreStyle(s) + of modNone, modVerbatim: result = backref(index, reverse) + of modIgnoreCase: result = backrefIgnoreCase(index, reverse) + of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) proc builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous @@ -1896,6 +1921,9 @@ proc primary(p: var PegParser): Peg = result = capture(parseExpr(p)).token(p) eat(p, tkCurlyRi) inc(p.captures) + of tkEmptyCurl: + result = capture() + getTok(p) of tkAny: result = any().token(p) getTok(p) @@ -1915,11 +1943,11 @@ proc primary(p: var PegParser): Peg = result = startAnchor() getTok(p) of tkBackref: + if abs(p.tok.index) > p.captures or p.tok.index == 0: + pegError(p, "invalid back reference index: " & $p.tok.index) var m = p.tok.modifier if m == modNone: m = p.modifier result = modifiedBackref(p.tok.index, m).token(p) - if p.tok.index < 0 or p.tok.index > p.captures: - pegError(p, "invalid back reference index: " & $p.tok.index) getTok(p) else: pegError(p, "expression expected, but found: " & p.tok.literal) @@ -1943,7 +1971,7 @@ proc seqExpr(p: var PegParser): Peg = case p.tok.kind of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, - tkHat, tkCurlyAt: + tkHat, tkCurlyAt, tkEmptyCurl: result = sequence(result, primary(p)) of tkIdentifier: if not arrowIsNextTok(p): diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index f2c93c2e5..1261d55b8 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -293,6 +293,34 @@ block: doAssert "test1".match(peg"""{@}$""") doAssert "test2".match(peg"""{(!$ .)*} $""") + + doAssert "abbb".match(peg"{a} {b} $2 $^1") + doAssert "abBA".match(peg"{a} {b} i$2 i$^2") + + doAssert "abba".match(peg"{a} {b} $^1 {} $^1") + + block: + let grammar = peg""" +program <- {''} stmt* $ +stmt <- call / block +call <- 'call()' EOL +EOL <- \n / $ +block <- 'block:' \n indBody +indBody <- {$^1 ' '+} stmt ($^1 stmt)* {} +""" + let program = """ +call() +block: + block: + call() + call() + call() +call() +""" + var c: Captures + doAssert program.len == program.rawMatch(grammar, 0, c) + doAssert c.ml == 1 + pegsTest() static: pegsTest() From 6e74fecb4275272e5224e2a0ba45df5ca2c40b82 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 15:00:14 +0200 Subject: [PATCH 0170/1654] Simplify addInt, remove digits10 (#18356) * Simplify addInt, remove digits10 Co-authored-by: Charles Blake * Fix bootstrapping * Add noInit to tmp array * noInit -> noinit Co-authored-by: Charles Blake --- lib/std/private/digitsutils.nim | 58 +++++++++++++-------------------- lib/system/dollars.nim | 4 +-- lib/system/strmantle.nim | 10 ++---- tests/stdlib/tdigitsutils.nim | 23 ------------- 4 files changed, 27 insertions(+), 68 deletions(-) delete mode 100644 tests/stdlib/tdigitsutils.nim diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index cc985c7bd..7aefc36bc 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -38,51 +38,39 @@ proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = assert(digits <= 99) return trailingZeros100[digits] -func digits10*(num: uint64): int {.noinline.} = - if num < 10'u64: - result = 1 - elif num < 100'u64: - result = 2 - elif num < 1_000'u64: - result = 3 - elif num < 10_000'u64: - result = 4 - elif num < 100_000'u64: - result = 5 - elif num < 1_000_000'u64: - result = 6 - elif num < 10_000_000'u64: - result = 7 - elif num < 100_000_000'u64: - result = 8 - elif num < 1_000_000_000'u64: - result = 9 - elif num < 10_000_000_000'u64: - result = 10 - elif num < 100_000_000_000'u64: - result = 11 - elif num < 1_000_000_000_000'u64: - result = 12 - else: - result = 12 + digits10(num div 1_000_000_000_000'u64) - -template numToString*(result: var string, origin: uint64, length: int) = +func addIntImpl*(result: var string, origin: uint64) = + var tmp {.noinit.}: array[24, char] var num = origin - var next = length - 1 + var next = tmp.len - 1 const nbatch = 100 while num >= nbatch: let originNum = num num = num div nbatch let index = (originNum - num * nbatch) shl 1 - result[next] = digits100[index + 1] - result[next - 1] = digits100[index] + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] dec(next, 2) # process last 1-2 digits if num < 10: - result[next] = chr(ord('0') + num) + tmp[next] = chr(ord('0') + num) else: let index = num * 2 - result[next] = digits100[index + 1] - result[next - 1] = digits100[index] + tmp[next] = digits100[index + 1] + tmp[next - 1] = digits100[index] + dec next + let n = result.len + let length = tmp.len - next + result.setLen n + length + when nimvm: + for i in 0.. 1: - doAssert digits10(x-1) == num - 1 - num += 1 - let xOld = x - x *= 10 - if x < xOld: - # wrap-around - break - -static: main() -main() From 0be17f5d9c6b66a8f8dda83c9a3c77c61be3844d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Jun 2021 17:08:45 +0200 Subject: [PATCH 0171/1654] fixes #18319 (#18375) --- compiler/jsgen.nim | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 393ac83c2..a9339c7fd 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -29,7 +29,7 @@ implements the required case distinction. import - ast, trees, magicsys, options, + ast, astalgo, trees, magicsys, options, nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, @@ -2234,6 +2234,16 @@ proc genTupleConstr(p: PProc, n: PNode, r: var TCompRes) = r.res.addf("Field$#: $#", [i.rope, a.res]) r.res.add("}") +proc lookupFieldAgain(ty: PType; field: PSym): PSym = + var ty = ty + while ty != nil: + ty = ty.skipTypes(skipPtrs) + assert(ty.kind in {tyTuple, tyObject}) + result = lookupInRecord(ty.n, field.name) + if result != nil: break + ty = ty[0] + if result == nil: result = field + proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) = var a: TCompRes r.kind = resExpr @@ -2247,7 +2257,7 @@ proc genObjConstr(p: PProc, n: PNode, r: var TCompRes) = gen(p, val, a) var f = it[0].sym if f.loc.r == nil: f.loc.r = mangleName(p.module, f) - fieldIDs.incl(f.id) + fieldIDs.incl(lookupFieldAgain(n.typ, f).id) let typ = val.typ.skipTypes(abstractInst) if a.typ == etyBaseIndex: From cb9496693a718cdb9ad8b764f82f524efe943bd2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 28 Jun 2021 08:15:20 -0700 Subject: [PATCH 0172/1654] `./koch tools` now builds bin/nim_dbg, a debug version of nim (#18289) --- koch.nim | 10 ++++++++-- tools/kochdocs.nim | 5 +++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 6a0b7a0d3..d3957069b 100644 --- a/koch.nim +++ b/koch.nim @@ -215,8 +215,14 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile testament", "testament/testament.nim", - options = "-d:release " & args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release" & args) + + # pre-packages a debug version of nim which can help in many cases investigate issuses + # withouth having to rebuild compiler. + # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler + # `-d:debug` should be changed to a flag that doesn't require re-compiling nim + # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints" & args, outputName = "nim_dbg") proc nsis(latest: bool; args: string) = bundleNimbleExe(latest, args) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 469059348..8e8085f73 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -98,8 +98,9 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) From 72d6a9c885504a1752ee0639aa110d4bf42d95c8 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 17:41:37 +0200 Subject: [PATCH 0173/1654] Fix #16426 (#18377) --- compiler/parser.nim | 13 ++++++++----- tests/parser/tstmtlists.nim | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 9cddf8e4f..b6c01e9c6 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -537,13 +537,16 @@ proc parseGStrLit(p: var Parser, a: PNode): PNode = proc complexOrSimpleStmt(p: var Parser): PNode proc simpleExpr(p: var Parser, mode = pmNormal): PNode -proc parseIfExpr(p: var Parser, kind: TNodeKind): PNode +proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode proc semiStmtList(p: var Parser, result: PNode) = inc p.inSemiStmtList withInd(p): # Be lenient with the first stmt/expr - let a = if p.tok.tokType == tkIf: parseIfExpr(p, nkIfStmt) else: complexOrSimpleStmt(p) + let a = case p.tok.tokType + of tkIf: parseIfOrWhenExpr(p, nkIfStmt) + of tkWhen: parseIfOrWhenExpr(p, nkWhenStmt) + else: complexOrSimpleStmt(p) result.add a while p.tok.tokType != tkEof: @@ -1203,13 +1206,13 @@ proc parseExpr(p: var Parser): PNode = result = parseBlock(p) of tkIf: nimprettyDontTouch: - result = parseIfExpr(p, nkIfExpr) + result = parseIfOrWhenExpr(p, nkIfExpr) of tkFor: nimprettyDontTouch: result = parseFor(p) of tkWhen: nimprettyDontTouch: - result = parseIfExpr(p, nkWhenExpr) + result = parseIfOrWhenExpr(p, nkWhenExpr) of tkCase: # Currently we think nimpretty is good enough with case expressions, # so it is allowed to touch them: @@ -1559,7 +1562,7 @@ proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = branch.add(parseStmt(p)) result.add(branch) -proc parseIfExpr(p: var Parser, kind: TNodeKind): PNode = +proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = #| condExpr = expr colcom expr optInd #| ('elif' expr colcom expr optInd)* #| 'else' colcom expr diff --git a/tests/parser/tstmtlists.nim b/tests/parser/tstmtlists.nim index b6c7e4a94..fa47ba364 100644 --- a/tests/parser/tstmtlists.nim +++ b/tests/parser/tstmtlists.nim @@ -50,6 +50,8 @@ hello 9 hello 10 +lucky +lucky ''' """ @@ -163,3 +165,16 @@ template dim: int = else: int.high) +# lenient indentation: + +echo (if 0 == 1: + "0 == 1" +else: + "lucky") + +# bug #16426 +echo (when 0 == 1: + "0 == 1" +else: + "lucky") + From dd2cbe3164e476dafda4b59d6ed518d79f69f195 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 19:25:30 +0200 Subject: [PATCH 0174/1654] Cleanup lookups.nim again.. (#18379) --- compiler/lookups.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 9b878dd4b..44a30eefa 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -58,8 +58,8 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent = template addSym*(scope: PScope, s: PSym) = strTableAdd(scope.symbols, s) -proc addUniqueSym*(scope: PScope, s: PSym, onConflictKeepOld: bool): PSym = - result = strTableInclReportConflict(scope.symbols, s, onConflictKeepOld) +proc addUniqueSym*(scope: PScope, s: PSym): PSym = + result = strTableInclReportConflict(scope.symbols, s) proc openScope*(c: PContext): PScope {.discardable.} = result = PScope(parent: c.currentScope, @@ -298,7 +298,7 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; # xxx pending bootstrap >= 1.4, replace all those overloads with a single one: # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = - let conflict = scope.addUniqueSym(sym, onConflictKeepOld = true) + let conflict = scope.addUniqueSym(sym) if conflict != nil: if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: # e.g.: import foo; import foo @@ -320,7 +320,7 @@ proc addDecl*(c: PContext, sym: PSym) {.inline.} = addDeclAt(c, c.currentScope, sym) proc addPrelimDecl*(c: PContext, sym: PSym) = - discard c.currentScope.addUniqueSym(sym, onConflictKeepOld = false) + discard c.currentScope.addUniqueSym(sym) from ic / ic import addHidden From 808db3b2845dab9822bc82ae80073a68396db999 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Mon, 28 Jun 2021 22:01:06 +0200 Subject: [PATCH 0175/1654] Fix unused warning for $ on empty object (#18381) --- lib/system/dollars.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index c23ea347d..baae8f090 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -113,7 +113,7 @@ proc `$`*[T: tuple|object](x: T): string = ## $() == "()" result = "(" const isNamed = T is object or isNamedTuple(T) - var count = 0 + var count {.used.} = 0 for name, value in fieldPairs(x): if count > 0: result.add(", ") when isNamed: From 6387e289631c22b8fa81e8e478c97cc500b90e26 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 28 Jun 2021 13:09:14 -0700 Subject: [PATCH 0176/1654] simplify rdstdin (#18382) --- lib/impure/rdstdin.nim | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index c580b89d1..32048b131 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -52,16 +52,6 @@ elif defined(genode): else: import linenoise - proc readLineFromStdin*(prompt: string): string {. - tags: [ReadIOEffect, WriteIOEffect].} = - var buffer = linenoise.readLine(prompt) - if isNil(buffer): - raise newException(IOError, "Linenoise returned nil") - result = $buffer - if result.len > 0: - historyAdd(buffer) - linenoise.free(buffer) - proc readLineFromStdin*(prompt: string, line: var string): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) @@ -73,3 +63,7 @@ else: historyAdd(buffer) linenoise.free(buffer) result = true + + proc readLineFromStdin*(prompt: string): string {.inline.} = + if not readLineFromStdin(prompt, result): + raise newException(IOError, "Linenoise returned nil") From 97fc95012d2725b625c492b6b72336a89c501076 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Jun 2021 15:34:39 +0200 Subject: [PATCH 0177/1654] fixes #16270 (#18388) --- compiler/ast.nim | 3 ++- compiler/semexprs.nim | 1 + compiler/sigmatch.nim | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 56043692f..2a2a84e76 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -499,7 +499,7 @@ type nfFirstWrite# this node is a first write TNodeFlags* = set[TNodeFlag] - TTypeFlag* = enum # keep below 32 for efficiency reasons (now: ~40) + TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 43) tfVarargs, # procedure has C styled varargs # tyArray type represeting a varargs list tfNoSideEffect, # procedure type does not allow side effects @@ -566,6 +566,7 @@ type # (for importc types); type is fully specified, allowing to compute # sizeof, alignof, offsetof at CT tfExplicitCallConv + tfIsConstructor TTypeFlags* = set[TTypeFlag] diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c885f4963..acf408db0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2412,6 +2412,7 @@ proc semWhen(c: PContext, n: PNode, semCheck = true): PNode = proc semSetConstr(c: PContext, n: PNode): PNode = result = newNodeI(nkCurly, n.info) result.typ = newTypeS(tySet, c) + result.typ.flags.incl tfIsConstructor if n.len == 0: rawAddSon(result.typ, newTypeS(tyEmpty, c)) else: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index b2003c0a8..071685057 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1341,8 +1341,13 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, result = isSubtype else: result = typeRel(c, f[0], a[0], flags) - if result <= isConvertible: - result = isNone # BUGFIX! + if result < isGeneric: + if result <= isConvertible: + result = isNone + elif tfIsConstructor notin a.flags: + # set constructors are a bit special... + result = isNone + of tyPtr, tyRef: skipOwned(a) if a.kind == f.kind: From 36d24846b6685a44ff2185a823cdcb0f3bc236d3 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 29 Jun 2021 18:08:50 -0700 Subject: [PATCH 0178/1654] typo: enmRange => enumElementsAsSet (#18394) --- lib/std/setutils.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index aba5f348a..84cc93623 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -36,7 +36,7 @@ template toSet*(iter: untyped): untyped = incl(result, x) result -macro enmRange(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) +macro enumElementsAsSet(enm: typed): untyped = result = newNimNode(nnkCurly).add(enm.getType[1][1..^1]) # func fullSet*(T: typedesc): set[T] {.inline.} = # xxx would give: Error: ordinal type expected func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = @@ -49,7 +49,7 @@ func fullSet*[T](U: typedesc[T]): set[T] {.inline.} = when T is Ordinal: {T.low..T.high} else: # Hole filled enum - enmRange(T) + enumElementsAsSet(T) func complement*[T](s: set[T]): set[T] {.inline.} = ## Returns the set complement of `a`. From afe5eb569b36ed310f6f93a33ae52274a6dce02e Mon Sep 17 00:00:00 2001 From: Elliot Waite Date: Tue, 29 Jun 2021 19:21:30 -0700 Subject: [PATCH 0179/1654] Add commas to docs for clarity (#18398) --- doc/tut2.rst | 2 +- lib/pure/asyncdispatch.nim | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/tut2.rst b/doc/tut2.rst index 2624b2690..725f68dd5 100644 --- a/doc/tut2.rst +++ b/doc/tut2.rst @@ -79,7 +79,7 @@ to introduce new object roots apart from `system.RootObj`. (This is used in the GTK wrapper for instance.) Ref objects should be used whenever inheritance is used. It isn't strictly -necessary, but with non-ref objects assignments such as `let person: Person = +necessary, but with non-ref objects, assignments such as `let person: Person = Student(id: 123)` will truncate subclass fields. **Note**: Composition (*has-a* relation) is often preferable to inheritance diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b88d5ede8..17fd5f824 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -61,8 +61,8 @@ ## callback on this future which will be called once the future completes. ## All the callback does is write the data stored in the future to `stdout`. ## The `read` function is used for this and it checks whether the future -## completes with an error for you (if it did it will simply raise the -## error), if there is no error however it returns the value of the future. +## completes with an error for you (if it did, it will simply raise the +## error), if there is no error, however, it returns the value of the future. ## ## Asynchronous procedures ## ======================= From 0483a5ffaf3284b6bfc51e2e08bbc1d5a57cd939 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 29 Jun 2021 22:16:31 -0700 Subject: [PATCH 0180/1654] improvements to hint:processing: show import stack, distinguish nims, show includes etc (#18372) * improvements to hint:processing * fix tests; do not show hintProcessing for nimscript unless given -d:nimHintProcessingNims * fix trunner and avoid need for -d:nimHintProcessingNims * fix some tests --- compiler/lineinfos.nim | 5 +++-- compiler/modulegraphs.nim | 12 ++++++++++++ compiler/modules.nim | 11 ++++++----- compiler/passaux.nim | 13 ++++--------- compiler/semstmts.nim | 1 + nimsuggest/tests/tchk1.nim | 2 +- nimsuggest/tests/tchk_compiles.nim | 2 +- nimsuggest/tests/ttempl_inst.nim | 2 +- 8 files changed, 29 insertions(+), 19 deletions(-) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 131fe480b..801c20d25 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -75,7 +75,7 @@ type hintXDeclaredButNotUsed = "XDeclaredButNotUsed", hintDuplicateModuleImport = "DuplicateModuleImport", hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded", hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX", - hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin", + hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintProcessingStmt = "ProcessingStmt", hintCodeBegin = "CodeBegin", hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path", hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name", hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency", @@ -163,6 +163,7 @@ const hintExprAlwaysX: "expression evaluates always to '$1'", hintQuitCalled: "quit() called", hintProcessing: "$1", + hintProcessingStmt: "$1", hintCodeBegin: "generated code listing:", hintCodeEnd: "end of listing", hintConf: "used config file '$1'", @@ -202,7 +203,7 @@ type proc computeNotesVerbosity(): array[0..3, TNoteKinds] = result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed} - result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc} + result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt} result[1] = result[2] - {warnProveField, warnProveIndex, warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd, hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin, hintPerformance} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 0d8d4f867..90b130e92 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -582,3 +582,15 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; proc configComplete*(g: ModuleGraph) = rememberStartupConfig(g.startupPackedConfig, g.config) + +from std/strutils import repeat, `%` + +proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) = + let conf = graph.config + let isNimscript = conf.isDefined("nimscript") + if (not isNimscript) or hintProcessing in conf.cmdlineNotes: + let path = toFilenameOption(conf, fileIdx, conf.filenameOption) + let indent = ">".repeat(graph.importStack.len) + let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)" + let mode = if isNimscript: "(nims) " else: "" + rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path]) diff --git a/compiler/modules.nim b/compiler/modules.nim index dcfcfd508..6fba606b2 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -84,12 +84,13 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = partialInitModule(result, graph, fileIdx, filename) graph.registerModule(result) -proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym = +proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = var flags = flags if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule result = graph.getModule(fileIdx) - template processModuleAux = + template processModuleAux(moduleStatus) = + onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) var s: PLLStream if sfMainModule in flags: if graph.config.projectIsStdin: s = stdin.llStreamOpen @@ -103,7 +104,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P result = newModule(graph, fileIdx) result.flags.incl flags registerModule(graph, result) - processModuleAux() + processModuleAux("import") else: if sfSystemModule in flags: graph.systemModule = result @@ -117,13 +118,13 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P # reset module fields: initStrTables(graph, result) result.ast = nil - processModuleAux() + processModuleAux("import(dirty)") graph.markClientsDirty(fileIdx) proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = # this is called by the semantic checking phase assert graph.config != nil - result = compileModule(graph, fileIdx, {}) + result = compileModule(graph, fileIdx, {}, s) graph.addDep(s, fileIdx) # keep track of import relationships if graph.config.hcrOn: diff --git a/compiler/passaux.nim b/compiler/passaux.nim index 87cb3a730..68b783248 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -19,18 +19,13 @@ type config: ConfigRef proc verboseOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = - let conf = graph.config - result = VerboseRef(config: conf, idgen: idgen) - let path = toFilenameOption(conf, s.position.FileIndex, conf.filenameOption) - rawMessage(conf, hintProcessing, path) + # xxx consider either removing this or keeping for documentation for how to add a pass + result = VerboseRef(config: graph.config, idgen: idgen) proc verboseProcess(context: PPassContext, n: PNode): PNode = + # called from `process` in `processTopLevelStmt`. result = n let v = VerboseRef(context) - if v.config.verbosity == 3: - # system.nim deactivates all hints, for verbosity:3 we want the processing - # messages nonetheless, so we activate them again (but honor cmdlineNotes) - v.config.setNote(hintProcessing) - message(v.config, n.info, hintProcessing, $v.idgen[]) + message(v.config, n.info, hintProcessingStmt, $v.idgen[]) const verbosePass* = makePass(open = verboseOpen, process = verboseProcess) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index f29dc24ef..2907d5172 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2158,6 +2158,7 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult: PNode) = var f = checkModuleName(c.config, it) if f != InvalidFileIdx: addIncludeFileDep(c, f) + onProcessing(c.graph, f, "include", c.module) if containsOrIncl(c.includedFiles, f.int): localError(c.config, n.info, errRecursiveDependencyX % toMsgFilename(c.config, f)) else: diff --git a/nimsuggest/tests/tchk1.nim b/nimsuggest/tests/tchk1.nim index 794392c1f..f8e2989c5 100644 --- a/nimsuggest/tests/tchk1.nim +++ b/nimsuggest/tests/tchk1.nim @@ -17,7 +17,7 @@ proc main = discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/tchk1.nim [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tchk1.nim [Processing]";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but got \'keyword template\'";;0 chk;;skUnknown;;;;Error;;$file;;14;;0;;"nestable statement requires indentation";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 diff --git a/nimsuggest/tests/tchk_compiles.nim b/nimsuggest/tests/tchk_compiles.nim index 2afa06899..c8a3daac4 100644 --- a/nimsuggest/tests/tchk_compiles.nim +++ b/nimsuggest/tests/tchk_compiles.nim @@ -4,5 +4,5 @@ discard compiles(2 + "hello") discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/tchk_compiles.nim [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/tchk_compiles.nim [Processing]";;0 """ diff --git a/nimsuggest/tests/ttempl_inst.nim b/nimsuggest/tests/ttempl_inst.nim index da413d99a..5f5b10fe9 100644 --- a/nimsuggest/tests/ttempl_inst.nim +++ b/nimsuggest/tests/ttempl_inst.nim @@ -7,7 +7,7 @@ foo() discard """ $nimsuggest --tester $file >chk $1 -chk;;skUnknown;;;;Hint;;???;;0;;-1;;"tests/ttempl_inst.nim [Processing]";;0 +chk;;skUnknown;;;;Hint;;???;;0;;-1;;">> (toplevel): import(dirty): tests/ttempl_inst.nim [Processing]";;0 chk;;skUnknown;;;;Hint;;$file;;4;;3;;"template/generic instantiation from here";;0 chk;;skUnknown;;;;Warning;;$file;;2;;11;;"foo [User]";;0 """ From 16f0b556643a8a9cc73781de6d71d0041a194640 Mon Sep 17 00:00:00 2001 From: Bung Date: Wed, 30 Jun 2021 13:28:41 +0800 Subject: [PATCH 0181/1654] remove isKeywordIgnoreCase from highlite, not used , can't even compile with -d:useNimRtl (#18399) --- lib/packages/docutils/highlite.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 014303837..d36b2c877 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -188,9 +188,6 @@ const proc isKeyword(x: openArray[string], y: string): int = binarySearch(x, y) -proc isKeywordIgnoreCase(x: openArray[string], y: string): int = - binarySearch(x, y, cmpIgnoreCase) - proc nimNextToken(g: var GeneralTokenizer, keywords: openArray[string] = @[]) = const hexChars = {'0'..'9', 'A'..'F', 'a'..'f', '_'} From eaa1c3a37754725f91f54f23080980b82c2f2356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Wed, 30 Jun 2021 07:29:10 +0200 Subject: [PATCH 0182/1654] genDepend: fix for module names that are reserved DOT keywords (#18392) (like "node" and "edge") --- compiler/depends.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/depends.nim b/compiler/depends.nim index 7225b6b47..30fc961c5 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -25,7 +25,7 @@ type dotGraph: Rope proc addDependencyAux(b: Backend; importing, imported: string) = - b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)]) + b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)]) # s1 -> s2_4[label="[0-9]"]; proc addDotDependency(c: PPassContext, n: PNode): PNode = From 19263f27756e56026fe8d1ac05966b84010ae8d1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 30 Jun 2021 12:36:24 +0200 Subject: [PATCH 0183/1654] fixes #18400 (#18402) --- lib/system/schubfach.nim | 2 +- tests/float/tfloat4.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index d072cde99..06813f632 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -359,7 +359,7 @@ proc formatDigits(buffer: var openArray[char]; pos: int; digits: uint32; decimal pos = digitsEnd elif decimalPoint < numDigits: ## dig.its - for i in 0..<8: + for i in countdown(7, 0): buffer[i + decimalPoint + 1] = buffer[i + decimalPoint] buffer[pos+decimalPoint] = '.' pos = digitsEnd + 1 diff --git a/tests/float/tfloat4.nim b/tests/float/tfloat4.nim index 6a87cbe66..5bedca371 100644 --- a/tests/float/tfloat4.nim +++ b/tests/float/tfloat4.nim @@ -54,4 +54,10 @@ doAssert 9999999999999999.0 == "9999999999999999.0".parseFloat doAssert 0.999999999999999 == ".999999999999999".parseFloat doAssert 0.9999999999999999 == ".9999999999999999".parseFloat +# bug #18400 +var s = [-13.888888'f32] +assert $s[0] == "-13.888888" +var x = 1.23456789012345'f32 +assert $x == "1.2345679" + echo("passed all tests.") From 28a4814019ea5d985d323def3766fe7f44f2e3e1 Mon Sep 17 00:00:00 2001 From: Endeg Date: Wed, 30 Jun 2021 23:19:22 +0600 Subject: [PATCH 0184/1654] Missing spaces when compiling testament and nim_dbg. (#18403) --- koch.nim | 4 ++-- lib/pure/times.nim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index d3957069b..4ba0d0eb5 100644 --- a/koch.nim +++ b/koch.nim @@ -215,14 +215,14 @@ proc buildTools(args: string = "") = options = "-d:release " & args) when defined(windows): buildVccTool(args) bundleNimpretty(args) - nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release" & args) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release " & args) # pre-packages a debug version of nim which can help in many cases investigate issuses # withouth having to rebuild compiler. # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler # `-d:debug` should be changed to a flag that doesn't require re-compiling nim # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces - nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints" & args, outputName = "nim_dbg") + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, outputName = "nim_dbg") proc nsis(latest: bool; args: string) = bundleNimbleExe(latest, args) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index fb48a199c..fcdd4ec8d 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -913,7 +913,7 @@ proc getTime*(): Time {.tags: [TimeEffect], benign.} = result = initTime(ts.tv_sec.int64, ts.tv_nsec.int) elif defined(windows): var f {.noinit.}: FILETIME - getSystemTimePreciseAsFileTime(f) + getSystemTimeAsFileTime(f) result = fromWinTime(rdFileTime(f)) proc `-`*(a, b: Time): Duration {.operator, extern: "ntDiffTime".} = @@ -2596,7 +2596,7 @@ proc epochTime*(): float {.tags: [TimeEffect].} = toBiggestFloat(ts.tv_nsec.int64) / 1_000_000_000 elif defined(windows): var f {.noinit.}: winlean.FILETIME - getSystemTimePreciseAsFileTime(f) + getSystemTimeAsFileTime(f) var i64 = rdFileTime(f) - epochDiff var secs = i64 div rateDiff var subsecs = i64 mod rateDiff From 17911a116d8677228480d74ceb2b9593b992253e Mon Sep 17 00:00:00 2001 From: Mark Pointing Date: Thu, 1 Jul 2021 03:22:38 +1000 Subject: [PATCH 0185/1654] Removed hard coded ports from thttpclient_standalone.nim. (#18374) Co-authored-by: Dominik Picheta Co-authored-by: Mark Pointing Co-authored-by: Andreas Rumpf Co-authored-by: Timothee Cour --- tests/stdlib/thttpclient_standalone.nim | 49 +++++++++++++------------ 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/stdlib/thttpclient_standalone.nim b/tests/stdlib/thttpclient_standalone.nim index 5fa1ea41a..362b1cb86 100644 --- a/tests/stdlib/thttpclient_standalone.nim +++ b/tests/stdlib/thttpclient_standalone.nim @@ -2,29 +2,37 @@ discard """ cmd: "nim c --threads:on $file" """ -import asynchttpserver, httpclient, asyncdispatch, strutils +import asynchttpserver, httpclient, asyncdispatch, strutils, net block: # bug #16436 - proc startServer() {.async.} = + proc startServer(): AsyncHttpServer = + result = newAsyncHttpServer() + result.listen(Port(0)) + + proc processRequest(server: AsyncHttpServer) {.async.} = proc cb(req: Request) {.async.} = let headers = { "Content-length": "15"} # Provide invalid content-length await req.respond(Http200, "Hello World", headers.newHttpHeaders()) - var server = newAsyncHttpServer() - await server.serve(Port(5555), cb) + await server.acceptRequest(cb) - proc runClient() {.async.} = + proc runClient(port: Port) {.async.} = let c = newAsyncHttpClient(headers = {"Connection": "close"}.newHttpHeaders) - let r = await c.getContent("http://127.0.0.1:5555") + discard await c.getContent("http://127.0.0.1:" & $port) doAssert false, "should fail earlier" - asyncCheck startServer() + let server = startServer() + asyncCheck processRequest(server) + let port = server.getPort() doAssertRaises(ProtocolError): - waitFor runClient() + waitFor runClient(port) block: # bug #14794 (And test for presence of content-length header when using postContent) - proc startServer() {.async.} = - var killServer = false + proc startServer(): AsyncHttpServer = + result = newAsyncHttpServer() + result.listen(Port(0)) + + proc runServer(server: AsyncHttpServer) {.async.} = proc cb(req: Request) {.async.} = doAssert(req.body.endsWith(httpNewLine), "Multipart body does not end with a newline.") # this next line is probably not required because asynchttpserver does not call @@ -32,23 +40,18 @@ block: # bug #14794 (And test for presence of content-length header when using p # Error: unhandled exception: 411 Length Required # Added for good measure in case the server becomes more permissive. doAssert(req.headers.hasKey("content-length"), "Content-Length header is not present.") - killServer = true asyncCheck req.respond(Http200, "OK") - var server = newAsyncHttpServer() - server.listen(Port(5556)) - while not killServer: - if server.shouldAcceptRequest(): - await server.acceptRequest(cb) - else: - poll() + await server.acceptRequest(cb) - proc runClient() {.async.} = + proc runClient(port: Port) {.async.} = let c = newAsyncHttpClient() - var data = newMultipartData() + let data = newMultipartData() data.add("file.txt", "This is intended to be an example text file.\r\nThis would be the second line.\r\n") - let r = await c.postContent("http://127.0.0.1:5556", multipart = data) + discard await c.postContent("http://127.0.0.1:" & $port, multipart = data) c.close() - asyncCheck startServer() - waitFor runClient() + let server = startServer() + let port = server.getPort() + asyncCheck runServer(server) + waitFor runClient(port) From 41c29cb3a1674a8b85844888e831b79050d91293 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 1 Jul 2021 06:51:08 +0200 Subject: [PATCH 0186/1654] fixes #18130 (#18407) --- compiler/injectdestructors.nim | 2 +- tests/destructor/tv2_cast.nim | 87 +++++++++++++++++++++++++++------- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 6447557c5..6e4eaa817 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -1057,7 +1057,7 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNod result = c.genCopy(dest, ri) result.add p(ri, c, s, consumed) c.finishCopy(result, dest, isFromSink = false) - of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv: + of nkHiddenSubConv, nkHiddenStdConv, nkConv, nkObjDownConv, nkObjUpConv, nkCast: result = c.genSink(dest, p(ri, c, s, sinkArg), isDecl) of nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt: template process(child, s): untyped = moveOrCopy(dest, child, c, s, isDecl) diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index 5502cb38d..917cf0eb3 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -1,48 +1,81 @@ discard """ output: '''@[1] @[116, 101, 115, 116] -@[1953719668, 875770417]''' +@[1953719668, 875770417] +destroying O1''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' nimout: '''--expandArc: main var data :tmpD -`=copy`(data, cast[string](encode(cast[seq[byte]]( - :tmpD = newString(100) - :tmpD)))) -`=destroy`(:tmpD) -`=destroy`(data) + :tmpD_1 + :tmpD_2 +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_2 = encode(cast[seq[byte]]( + :tmpD_1 = newString(100) + :tmpD_1)) + :tmpD_2)) + :tmpD +`=destroy`(:tmpD_2) +`=destroy_1`(:tmpD_1) +`=destroy_1`(data) -- end of expandArc ------------------------ --expandArc: main1 var s data + :tmpD + :tmpD_1 s = newString(100) -`=copy`(data, cast[string](encode(toOpenArrayByte(s, 0, len(s) - 1)))) -`=destroy`(data) -`=destroy`(s) +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_1 = encode(toOpenArrayByte(s, 0, len(s) - 1)) + :tmpD_1)) + :tmpD +`=destroy`(:tmpD_1) +`=destroy_1`(data) +`=destroy_1`(s) -- end of expandArc ------------------------ --expandArc: main2 var s data + :tmpD + :tmpD_1 s = newSeq(100) -`=copy`(data, cast[string](encode(s))) -`=destroy`(data) -`=destroy_1`(s) +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_1 = encode(s) + :tmpD_1)) + :tmpD +`=destroy`(:tmpD_1) +`=destroy_1`(data) +`=destroy`(s) -- end of expandArc ------------------------ --expandArc: main3 var data :tmpD -`=copy`(data, cast[string](encode do: - :tmpD = newSeq(100) - :tmpD)) -`=destroy`(:tmpD) + :tmpD_1 + :tmpD_2 +data = + wasMoved(:tmpD) + `=copy`(:tmpD, cast[string]( + :tmpD_2 = encode do: + :tmpD_1 = newSeq(100) + :tmpD_1 + :tmpD_2)) + :tmpD +`=destroy`(:tmpD_2) +`=destroy`(:tmpD_1) `=destroy_1`(data) -- end of expandArc ------------------------''' """ @@ -89,3 +122,25 @@ echo a #issue 11204 var ac {.compileTime.} = @["a", "b"] const bc = ac.len + + +type + O = object of RootRef + i: int + + O1 = object of O + O2 = object of O + +proc `=destroy`(o: var O) = + echo "destroying O" + +proc `=destroy`(o: var O1) = + echo "destroying O1" + +proc `=destroy`(o: var O2) = + echo "destroying O2" + +proc test = + let o3 = cast[ref O2]((ref O1)()) + +test() From 779b4e307bdeb018bb2884920144a8c0d1f3b377 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 1 Jul 2021 02:14:01 -0700 Subject: [PATCH 0187/1654] don't use `{.rtl.}` for generics, otherwise `-d:useNimRtl` gives `ambiguous identifier` nimrtl error (#18406) --- lib/pure/strutils.nim | 2 +- lib/system/inclrtl.nim | 13 +++++++------ tests/dll/client.nim | 8 +++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 8e0a6158b..9e0837942 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1760,7 +1760,7 @@ func join*(a: openArray[string], sep: string = ""): string {.rtl, else: result = "" -func join*[T: not string](a: openArray[T], sep: string = ""): string {.rtl.} = +func join*[T: not string](a: openArray[T], sep: string = ""): string = ## Converts all elements in the container `a` to strings using `$`, ## and concatenates them with `sep`. runnableExamples: diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 0bc51d693..ca41f39c6 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -30,12 +30,13 @@ when defined(createNimRtl): {.pragma: inl.} {.pragma: compilerRtl, compilerproc, exportc: "nimrtl_$1", dynlib.} elif defined(useNimRtl): - when defined(windows): - const nimrtl* = "nimrtl.dll" - elif defined(macosx): - const nimrtl* = "libnimrtl.dylib" - else: - const nimrtl* = "libnimrtl.so" + #[ + `{.rtl.}` should only be used for non-generic procs. + ]# + const nimrtl* = + when defined(windows): "nimrtl.dll" + elif defined(macosx): "libnimrtl.dylib" + else: "libnimrtl.so" {.pragma: rtl, importc: "nimrtl_$1", dynlib: nimrtl, gcsafe.} {.pragma: inl.} {.pragma: compilerRtl, compilerproc, importc: "nimrtl_$1", dynlib: nimrtl.} diff --git a/tests/dll/client.nim b/tests/dll/client.nim index 150af3a17..62697569f 100644 --- a/tests/dll/client.nim +++ b/tests/dll/client.nim @@ -1,5 +1,4 @@ discard """ - output: "Done" cmd: "nim $target --debuginfo --hints:on --define:useNimRtl $options $file" """ @@ -37,5 +36,8 @@ proc eval(n: PNode): int = for i in 0..100_000: discard eval(buildTree(2)) -echo "Done" - +# bug https://forum.nim-lang.org/t/8176; Error: ambiguous identifier: 'nimrtl' +import std/strutils +doAssert join(@[1, 2]) == "12" +doAssert join(@[1.5, 2.5]) == "1.52.5" +doAssert join(@["a", "bc"]) == "abc" From 3ceaf5c1309ac8d4d68e6f39c13b021bcc1b15f4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 1 Jul 2021 17:35:04 +0200 Subject: [PATCH 0188/1654] fixes #18030 (#18415) --- compiler/seminst.nim | 2 +- tests/arc/tarcmisc.nim | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 68ab2a310..e1fad236c 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -146,8 +146,8 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = freshGenSyms(c, b, result, orig, symMap) b = semProcBody(c, b) result.ast[bodyPos] = hloBody(c, b) - trackProc(c, result, result.ast[bodyPos]) excl(result.flags, sfForward) + trackProc(c, result, result.ast[bodyPos]) dec c.inGenericInst proc fixupInstantiatedSymbols(c: PContext, s: PSym) = diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index a52538085..5d5d8e914 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -28,6 +28,8 @@ aaaaa hello ok true +copying +123 closed destroying variable: 20 destroying variable: 10 @@ -433,3 +435,31 @@ proc t17712 = echo el != nil t17712() + +# bug #18030 + +type + Foo = object + n: int + +proc `=copy`(dst: var Foo, src: Foo) = + echo "copying" + dst.n = src.n + +proc `=sink`(dst: var Foo, src: Foo) = + echo "sinking" + dst.n = src.n + +var a: Foo + +proc putValue[T](n: T) + +proc useForward = + putValue(123) + +proc putValue[T](n: T) = + var b = Foo(n:n) + a = b + echo b.n + +useForward() From 60eac79f7f0478565a24ff423aa52ab53527d153 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 2 Jul 2021 20:38:01 +0800 Subject: [PATCH 0189/1654] [minor]remove unnecessary stringify (#18419) --- compiler/lexer.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 59b983060..c7a9aa47f 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -536,9 +536,9 @@ proc getNumber(L: var Lexer, result: var Token) = try: len = parseBiggestUInt(result.literal, iNumber) except ValueError: - raise newException(OverflowDefect, "number out of range: " & $result.literal) + raise newException(OverflowDefect, "number out of range: " & result.literal) if len != result.literal.len: - raise newException(ValueError, "invalid integer: " & $result.literal) + raise newException(ValueError, "invalid integer: " & result.literal) result.iNumber = cast[int64](iNumber) else: var iNumber: int64 @@ -546,9 +546,9 @@ proc getNumber(L: var Lexer, result: var Token) = try: len = parseBiggestInt(result.literal, iNumber) except ValueError: - raise newException(OverflowDefect, "number out of range: " & $result.literal) + raise newException(OverflowDefect, "number out of range: " & result.literal) if len != result.literal.len: - raise newException(ValueError, "invalid integer: " & $result.literal) + raise newException(ValueError, "invalid integer: " & result.literal) result.iNumber = iNumber # Explicit bounds checks. From d1d2498c7b933b6364af450df5088aaa89b4cf4b Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Sat, 3 Jul 2021 00:20:59 +0300 Subject: [PATCH 0190/1654] Add lent annotation to avoid extra copies in sample (#18404) * Add lent annotation to avoid extra copies in sample * Remove lent * Update random.nim --- lib/pure/random.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 6a5f35bde..07db0365c 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -410,7 +410,7 @@ proc sample*[T](r: var Rand; a: openArray[T]): T = result = a[r.rand(a.low..a.high)] -proc sample*[T](a: openArray[T]): T = +proc sample*[T](a: openArray[T]): lent T = ## Returns a random element from `a`. ## ## If `randomize <#randomize>`_ has not been called, the order of outcomes From ffa638ed5023834a7a5040a54c6cfde550d68054 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 5 Jul 2021 05:26:40 -0700 Subject: [PATCH 0191/1654] std/net: show addr+port on address already in use error; code-block => runnableExamples (#18428) * std/net: show addr+port on address already in use error; code-block => runnableExamples * var=>let --- lib/pure/net.nim | 80 ++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ff179fb2c..ec3e4e74d 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -44,46 +44,44 @@ ## After you create a socket with the `newSocket` procedure, you can easily ## connect it to a server running at a known hostname (or IP address) and port. ## To do so over TCP, use the example below. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.connect("google.com", Port(80)) -## -## For SSL, use the following example (and make sure to compile with `-d:ssl`): -## -## .. code-block:: Nim -## var socket = newSocket() -## var ctx = newContext() -## wrapSocket(ctx, socket) -## socket.connect("google.com", Port(443)) -## + +runnableExamples("-r:off"): + let socket = newSocket() + socket.connect("google.com", Port(80)) + +## For SSL, use the following example: + +runnableExamples("-r:off -d:ssl"): + let socket = newSocket() + let ctx = newContext() + wrapSocket(ctx, socket) + socket.connect("google.com", Port(443)) + ## UDP is a connectionless protocol, so UDP sockets don't have to explicitly ## call the `connect `_ procedure. They can ## simply start sending data immediately. -## -## .. code-block:: Nim -## var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) -## socket.sendTo("192.168.0.1", Port(27960), "status\n") -## + +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + socket.sendTo("192.168.0.1", Port(27960), "status\n") + ## Creating a server ## ----------------- ## ## After you create a socket with the `newSocket` procedure, you can create a ## TCP server by calling the `bindAddr` and `listen` procedures. -## -## .. code-block:: Nim -## var socket = newSocket() -## socket.bindAddr(Port(1234)) -## socket.listen() -## -## You can then begin accepting connections using the `accept` procedure. -## -## .. code-block:: Nim -## var client: Socket -## var address = "" -## while true: -## socket.acceptAddr(client, address) -## echo("Client connected from: ", address) + +runnableExamples("-r:off"): + let socket = newSocket() + socket.bindAddr(Port(1234)) + socket.listen() + + # You can then begin accepting connections using the `accept` procedure. + var client: Socket + var address = "" + while true: + socket.acceptAddr(client, address) + echo "Client connected from: ", address import std/private/since @@ -960,7 +958,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeaddrinfo(aiList) - raiseOSError(osLastError()) + raiseOSError(osLastError(), "address: $# port: $#" % [address, $port]) freeaddrinfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, @@ -1233,12 +1231,10 @@ proc getPeerAddr*(socket: Socket): (string, Port) = proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = ## Sets option `opt` to a boolean value specified by `value`. - ## - ## .. code-block:: Nim - ## var socket = newSocket() - ## socket.setSockOpt(OptReusePort, true) - ## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt) - ## + runnableExamples("-r:off"): + let socket = newSocket() + socket.setSockOpt(OptReusePort, true) + socket.setSockOpt(OptNoDelay, true, level = IPPROTO_TCP.cint) var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) @@ -2025,10 +2021,8 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress = ## ## Supports IPv4 and v6. ## Raises OSError if external networking is not set up. - ## - ## .. code-block:: Nim - ## echo $getPrimaryIPAddr() # "192.168.1.2" - + runnableExamples("-r:off"): + echo getPrimaryIPAddr() # "192.168.1.2" let socket = if dest.family == IpAddressFamily.IPv4: newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) From 1bed7773146d6acf704b0ae5cfe62455f7a127a1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 5 Jul 2021 18:04:07 +0200 Subject: [PATCH 0192/1654] fixes #18411 (#18432) [backport:1.4] --- lib/system/excpt.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 5b7d4d49f..9f65db2fe 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -446,9 +446,7 @@ proc raiseExceptionAux(e: sink(ref Exception)) {.nodestroy.} = pushCurrentException(e) {.emit: "throw `e`;".} elif defined(nimQuirky) or gotoBasedExceptions: - # XXX This check should likely also be done in the setjmp case below. - if e != currException: - pushCurrentException(e) + pushCurrentException(e) when gotoBasedExceptions: inc nimInErrorMode else: From 927a832dde9403492d4a2838780bf59bd1a48177 Mon Sep 17 00:00:00 2001 From: pyautogui <79544976+pyautogui@users.noreply.github.com> Date: Mon, 5 Jul 2021 10:12:58 -0700 Subject: [PATCH 0193/1654] deprecate std/mersenne (#18395) Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Timothee Cour Co-authored-by: Andreas Rumpf --- changelog.md | 2 ++ lib/pure/mersenne.nim | 12 ++++-------- tests/stdlib/tmersenne.nim | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index 6672f7538..a4c5b4e62 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,8 @@ ## Changes affecting backward compatibility +- Deprecated `std/mersenne` + - `cuchar` now aliases `uint8` instead of `char` - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent, diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index 6778e2d62..6ec0d66b4 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -2,24 +2,20 @@ # # Nim's Runtime Library # (c) Copyright 2015 Nim Contributors -# # See the file "copying.txt", included in this # distribution, for details about the copyright. -# - ## The [Mersenne Twister](https://en.wikipedia.org/wiki/Mersenne_Twister) ## random number generator. -## -## **Note:** The procs in this module work at compile-time. +## .. note:: The procs in this module work at compile-time. + +{.deprecated: "use `std/random` instead.".} runnableExamples: var rand = newMersenneTwister(uint32.high) ## must be "var" doAssert rand.getNum() != rand.getNum() ## pseudorandom number - ## See also ## ======== ## * `random module`_ for Nim's standard random number generator - type MersenneTwister* = object ## The Mersenne Twister. @@ -35,6 +31,7 @@ proc newMersenneTwister*(seed: uint32): MersenneTwister = (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = + for i in 0..623: var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) @@ -48,7 +45,6 @@ proc getNum*(m: var MersenneTwister): uint32 = generateNumbers(m) result = m.mt[m.index] m.index = (m.index + 1) mod m.mt.len - result = result xor (result shr 11'u32) result = result xor ((result shl 7'u32) and 0x9d2c5680'u32) result = result xor ((result shl 15'u32) and 0xefc60000'u32) diff --git a/tests/stdlib/tmersenne.nim b/tests/stdlib/tmersenne.nim index 2707aa2f2..54eb7b216 100644 --- a/tests/stdlib/tmersenne.nim +++ b/tests/stdlib/tmersenne.nim @@ -7,5 +7,6 @@ template main() = doAssert mt.getNum == 1071751096'u32 doAssert mt.getNum == 3805347140'u32 + static: main() main() From 36654fd577f46c49191c5e6f7e499ea31193018c Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Mon, 5 Jul 2021 21:55:46 +0300 Subject: [PATCH 0194/1654] Cleanup interval and timeout procs (#18431) clear/setTimeout procs were using ref ref Timeout. Added more interval overloads. Corrections. --- lib/js/dom.nim | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index ca325fdd2..3f0b9f630 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1301,7 +1301,7 @@ type width*: int TimeOut* {.importc.} = ref object of RootObj - Interval* {.importc.} = object of RootObj + Interval* {.importc.} = ref object of RootObj AddEventListenerOptions* = object capture*: bool @@ -1481,6 +1481,8 @@ else: proc setTimeout*(action: proc(); ms: int): TimeOut {.importc, nodecl.} proc clearTimeout*(t: TimeOut) {.importc, nodecl.} +proc setInterval*(action: proc(); ms: int): Interval {.importc, nodecl.} +proc clearInterval*(i: Interval) {.importc, nodecl.} {.push importcpp.} @@ -1494,8 +1496,8 @@ proc removeEventListener*(et: EventTarget; ev: cstring; cb: proc(ev: Event)) proc alert*(w: Window, msg: cstring) proc back*(w: Window) proc blur*(w: Window) -proc clearInterval*(w: Window, interval: ref Interval) -proc clearTimeout*(w: Window, timeout: ref TimeOut) +proc clearInterval*(w: Window, interval: Interval) +proc clearTimeout*(w: Window, timeout: TimeOut) proc close*(w: Window) proc confirm*(w: Window, msg: cstring): bool proc disableExternalCapture*(w: Window) @@ -1520,10 +1522,10 @@ proc resizeTo*(w: Window, x, y: int) proc routeEvent*(w: Window, event: Event) proc scrollBy*(w: Window, x, y: int) proc scrollTo*(w: Window, x, y: int) -proc setInterval*(w: Window, code: cstring, pause: int): ref Interval -proc setInterval*(w: Window, function: proc (), pause: int): ref Interval -proc setTimeout*(w: Window, code: cstring, pause: int): ref TimeOut -proc setTimeout*(w: Window, function: proc (), pause: int): ref Interval +proc setInterval*(w: Window, code: cstring, pause: int): Interval +proc setInterval*(w: Window, function: proc (), pause: int): Interval +proc setTimeout*(w: Window, code: cstring, pause: int): TimeOut +proc setTimeout*(w: Window, function: proc (), pause: int): Interval proc stop*(w: Window) proc requestAnimationFrame*(w: Window, function: proc (time: float)): int proc cancelAnimationFrame*(w: Window, id: int) @@ -1768,4 +1770,4 @@ since (1, 3): proc readAsDataURL*(f: FileReader, b: Blob) {.importcpp: "#.readAsDataURL(#)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} - ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText \ No newline at end of file + ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText From c522f7f33ca08e6a441ebbb08ea9d2d79a3c500c Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Tue, 6 Jul 2021 08:21:50 +0300 Subject: [PATCH 0195/1654] Add entry (#18434) --- changelog.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index a4c5b4e62..028743bab 100644 --- a/changelog.md +++ b/changelog.md @@ -86,8 +86,8 @@ - Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. -- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, - use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, + use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. - The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. The downside is that these defines now have custom logic that doesn't apply for @@ -95,6 +95,8 @@ - Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. +- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, + `clearTimeout`, `setInterval`, `clearInterval` were updated. ## Standard library additions and changes @@ -341,6 +343,8 @@ - Added `dom.scrollIntoView` proc with options +- Added `dom.setInterval`, `dom.clearInterval` overloads. + ## Language changes - `nimscript` now handles `except Exception as e`. From 252eea8cae33e486b179398e193aea9459954338 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 5 Jul 2021 23:28:38 -0600 Subject: [PATCH 0196/1654] Make procedure mismatch more informative with pragma/call convention mismatches (#18384) * Added more concise calling convention/pragma mismatch messages * Now only adds callConvMsg/lock message when sensible * Fixed message formatting * Added tests, and fixed some bugs * Tests joined, and always indenting * More tests and more bug fixes * Fixed first test in tprocmismatch * Using var param for writting mismatches * Better logic for handling proc type rel and conv/pragma mismatch * Refactored getProcConvMismatch * Fixed callConv message formatting * Fixed test for proper message * Cleanup to address issues * getProcConvMismatch now returns tuple, and reformatted code --- compiler/semcall.nim | 8 ++- compiler/sigmatch.nim | 33 +----------- compiler/types.nim | 87 ++++++++++++++++++++++++++++++++ tests/errmsgs/tproc_mismatch.nim | 74 +++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 32 deletions(-) create mode 100644 tests/errmsgs/tproc_mismatch.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index b56a1dbac..cec656fc2 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -247,7 +247,13 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): candidates.add typeToString(got) candidates.addDeclaredLocMaybe(c.config, got) doAssert wanted != nil - if got != nil: effectProblem(wanted, got, candidates, c) + if got != nil: + if got.kind == tyProc and wanted.kind == tyProc: + # These are proc mismatches so, + # add the extra explict detail of the mismatch + candidates.addPragmaAndCallConvMismatch(wanted, got, c.config) + effectProblem(wanted, got, candidates, c) + of kUnknown: discard "do not break 'nim check'" candidates.add "\n" if err.firstMismatch.arg == 1 and nArg.kind == nkTupleConstr and diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 071685057..73ab5e9a3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -86,21 +86,6 @@ type TTypeRelFlags* = set[TTypeRelFlag] - TTypeRelation* = enum # order is important! - isNone, isConvertible, - isIntConv, - isSubtype, - isSubrange, # subrange of the wanted type; no type conversion - # but apart from that counts as ``isSubtype`` - isBothMetaConvertible # generic proc parameter was matched against - # generic type, e.g., map(mySeq, x=>x+1), - # maybe recoverable by rerun if the parameter is - # the proc's return value - isInferred, # generic proc was matched against a concrete type - isInferredConvertible, # same as above, but requiring proc CC conversion - isGeneric, - isFromIntLit, # conversion *from* int literal; proven safe - isEqual const isNilConversion = isConvertible # maybe 'isIntConv' fits better? @@ -640,23 +625,9 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = return isNone elif a[0] != nil: return isNone + + result = getProcConvMismatch(c.c.config, f, a, result)[1] - if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags: - return isNone - elif tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {} and - optThreadAnalysis in c.c.config.globalOptions: - # noSideEffect implies ``tfThread``! - return isNone - elif f.flags * {tfIterator} != a.flags * {tfIterator}: - return isNone - elif f.callConv != a.callConv: - # valid to pass a 'nimcall' thingie to 'closure': - if f.callConv == ccClosure and a.callConv == ccNimCall: - result = if result == isInferred: isInferredConvertible - elif result == isBothMetaConvertible: isBothMetaConvertible - else: isConvertible - else: - return isNone when useEffectSystem: if compatibleEffects(f, a) != efCompat: return isNone when defined(drnim): diff --git a/compiler/types.nim b/compiler/types.nim index bc970a6d2..bdf08ffe9 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -26,6 +26,29 @@ type # most useful, shows: symbol + resolved symbols if it differs, e.g.: # tuple[a: MyInt{int}, b: float] + TTypeRelation* = enum # order is important! + isNone, isConvertible, + isIntConv, + isSubtype, + isSubrange, # subrange of the wanted type; no type conversion + # but apart from that counts as ``isSubtype`` + isBothMetaConvertible # generic proc parameter was matched against + # generic type, e.g., map(mySeq, x=>x+1), + # maybe recoverable by rerun if the parameter is + # the proc's return value + isInferred, # generic proc was matched against a concrete type + isInferredConvertible, # same as above, but requiring proc CC conversion + isGeneric, + isFromIntLit, # conversion *from* int literal; proven safe + isEqual + + ProcConvMismatch* = enum + pcmNoSideEffect + pcmNotGcSafe + pcmLockDifference + pcmNotIterator + pcmDifferentCallConv + proc typeToString*(typ: PType; prefer: TPreferedDesc = preferName): string template `$`*(typ: PType): string = typeToString(typ) @@ -1471,6 +1494,68 @@ proc skipHiddenSubConv*(n: PNode; g: ModuleGraph; idgen: IdGenerator): PNode = else: result = n +proc getProcConvMismatch*(c: ConfigRef, f, a: PType, rel = isNone): (set[ProcConvMismatch], TTypeRelation) = + ## Returns a set of the reason of mismatch, and the relation for conversion. + result[1] = rel + if tfNoSideEffect in f.flags and tfNoSideEffect notin a.flags: + # Formal is pure, but actual is not + result[0].incl pcmNoSideEffect + result[1] = isNone + + if tfThread in f.flags and a.flags * {tfThread, tfNoSideEffect} == {} and + optThreadAnalysis in c.globalOptions: + # noSideEffect implies ``tfThread``! + result[0].incl pcmNotGcSafe + result[1] = isNone + + if f.flags * {tfIterator} != a.flags * {tfIterator}: + # One of them is an iterator so not convertible + result[0].incl pcmNotIterator + result[1] = isNone + + if f.callConv != a.callConv: + # valid to pass a 'nimcall' thingie to 'closure': + if f.callConv == ccClosure and a.callConv == ccNimCall: + case result[1] + of isInferred: result[1] = isInferredConvertible + of isBothMetaConvertible: result[1] = isBothMetaConvertible + elif result[1] != isNone: result[1] = isConvertible + else: + result[1] = isNone + result[0].incl pcmDifferentCallConv + + if f.lockLevel.ord != UnspecifiedLockLevel.ord and + a.lockLevel.ord != UnspecifiedLockLevel.ord: + # proctypeRel has more logic to catch this difference, + # so dont need to do `rel = isNone` + # but it's a pragma mismatch reason which is why it's here + result[0].incl pcmLockDifference + +proc addPragmaAndCallConvMismatch*(message: var string, formal, actual: PType, conf: ConfigRef) = + assert formal.kind == tyProc and actual.kind == tyProc + let (convMismatch, _) = getProcConvMismatch(conf, formal, actual) + var + gotPragmas = "" + expectedPragmas = "" + for reason in convMismatch: + case reason + of pcmDifferentCallConv: + message.add "\n Calling convention mismatch: got '{.$1.}', but expected '{.$2.}'." % [$actual.callConv, $formal.callConv] + of pcmNoSideEffect: + expectedPragmas.add "noSideEffect, " + of pcmNotGcSafe: + expectedPragmas.add "gcsafe, " + of pcmLockDifference: + gotPragmas.add("locks: " & $actual.lockLevel & ", ") + expectedPragmas.add("locks: " & $formal.lockLevel & ", ") + of pcmNotIterator: discard + + if expectedPragmas.len > 0: + gotPragmas.setLen(max(0, gotPragmas.len - 2)) # Remove ", " + expectedPragmas.setLen(max(0, expectedPragmas.len - 2)) # Remove ", " + message.add "\n Pragma mismatch: got '{.$1.}', but expected '{.$2.}'." % [gotPragmas, expectedPragmas] + + proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: PNode) = if formal.kind != tyError and actual.kind != tyError: let actualStr = typeToString(actual) @@ -1491,6 +1576,7 @@ proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: P if verbose: msg.addDeclaredLoc(conf, formal) if formal.kind == tyProc and actual.kind == tyProc: + msg.addPragmaAndCallConvMismatch(formal, actual, conf) case compatibleEffects(formal, actual) of efCompat: discard of efRaisesDiffer: @@ -1503,6 +1589,7 @@ proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: P msg.add "\n.tag effect is 'any tag allowed'" of efLockLevelsDiffer: msg.add "\nlock levels differ" + if formal.kind == tyEnum and actual.kind == tyEnum: msg.add "\nmaybe use `-d:nimLegacyConvEnumEnum` for a transition period" localError(conf, info, msg) diff --git a/tests/errmsgs/tproc_mismatch.nim b/tests/errmsgs/tproc_mismatch.nim new file mode 100644 index 000000000..400c3d441 --- /dev/null +++ b/tests/errmsgs/tproc_mismatch.nim @@ -0,0 +1,74 @@ +discard """ + action: reject + cmd: '''nim check --hints:off $options $file''' + nimoutFull: true + nimout: ''' +tproc_mismatch.nim(35, 52) Error: type mismatch: got but expected 'proc (a: int, c: float){.closure, noSideEffect.}' + Calling convention mismatch: got '{.cdecl.}', but expected '{.closure.}'. +tproc_mismatch.nim(39, 6) Error: type mismatch: got +but expected one of: +proc bar(a: proc ()) + first type mismatch at position: 1 + required type for a: proc (){.closure.} + but expression 'fn1' is of type: proc (){.inline, noSideEffect, gcsafe, locks: 0.} + Calling convention mismatch: got '{.inline.}', but expected '{.closure.}'. + +expression: bar(fn1) +tproc_mismatch.nim(43, 8) Error: type mismatch: got but expected 'proc (){.closure.}' + Calling convention mismatch: got '{.inline.}', but expected '{.closure.}'. +tproc_mismatch.nim(48, 8) Error: type mismatch: got but expected 'proc (){.closure, noSideEffect.}' + Pragma mismatch: got '{..}', but expected '{.noSideEffect.}'. +tproc_mismatch.nim(52, 8) Error: type mismatch: got but expected 'proc (a: float){.closure.}' +tproc_mismatch.nim(61, 9) Error: type mismatch: got but expected 'proc (a: int){.closure, gcsafe.}' + Pragma mismatch: got '{..}', but expected '{.gcsafe.}'. +tproc_mismatch.nim(69, 9) Error: type mismatch: got but expected 'proc (a: int): int{.cdecl.}' + Calling convention mismatch: got '{.nimcall.}', but expected '{.cdecl.}'. +tproc_mismatch.nim(70, 9) Error: type mismatch: got but expected 'proc (a: int): int{.nimcall.}' + Calling convention mismatch: got '{.cdecl.}', but expected '{.nimcall.}'. +tproc_mismatch.nim(74, 9) Error: type mismatch: got but expected 'proc (a: int){.closure, locks: 1.}' + Pragma mismatch: got '{.locks: 3.}', but expected '{.locks: 1.}'. +lock levels differ +''' +""" +block: # CallConv mismatch + func a(a: int, c: float) {.cdecl.} = discard + var b: proc(a: int, c: float) {.noSideEffect.} = a +block: # Parameter CallConv mismatch + proc fn1() {.inline.} = discard + proc bar(a: proc()) = discard + bar(fn1) +block: # CallConv mismatch + proc fn1() {.inline.} = discard + var fn: proc() + fn = fn1 +block: # Pragma mismatch + var a = "" + proc fn1() = a.add "b" + var fn: proc() {.noSideEffect.} + fn = fn1 +block: # Fail match not do to Pragma or CallConv + proc fn1(a: int) = discard + var fn: proc(a: float) + fn = fn1 +block: # Infered noSideEffect assign + type Foo = ref object + x0: int + var g0 = Foo(x0: 1) + proc fn1(a: int) = g0.x0 = a + var fn2: proc(a: int) + var fn3: proc(a: int) {.gcsafe.} + fn2 = fn1 + fn3 = fn1 +block: # Indrection through pragmas + {.pragma: inl1, inline.} + {.pragma: inl2, inline.} + {.pragma: p1, nimcall.} + {.pragma: p2, cdecl.} + var fn1: proc(a: int): int {.inl1, p1.} + var fn2: proc(a: int): int {.inl2, p2.} + fn2 = fn1 + fn1 = fn2 +block: # Lock levels differ + var fn1: proc(a: int){.locks: 3.} + var fn2: proc(a: int){.locks: 1.} + fn2 = fn1 From b88496ac67bea92c40cf1d0f3dc55b0fa91550a4 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 6 Jul 2021 21:09:56 +0300 Subject: [PATCH 0197/1654] fix compilation on Debian 7 (no git -C) (#18427) * fix compilation on Debian 7 (no git -C) * address review * allow specify branch for testing unmerged csources * temporarily change csources checkout parameters for testing * Update tools/deps.nim Co-authored-by: Timothee Cour * fix failure * Update config/build_config.txt * set proper git branch/hash Co-authored-by: Timothee Cour Co-authored-by: Andreas Rumpf --- build_all.bat | 2 +- ci/build_autogen.bat | 2 +- ci/funs.sh | 8 ++++++-- config/build_config.txt | 3 ++- tools/ci_generate.nim | 2 +- tools/deps.nim | 9 +++++++-- tools/niminst/makefile.nimf | 3 ++- 7 files changed, 20 insertions(+), 9 deletions(-) diff --git a/build_all.bat b/build_all.bat index 1bff761b3..c8ba05e19 100644 --- a/build_all.bat +++ b/build_all.bat @@ -10,7 +10,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe echo "building from csources: %nim_csources%" if not exist %nim_csourcesDir% ( - git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) if not exist %nim_csources% ( diff --git a/ci/build_autogen.bat b/ci/build_autogen.bat index 8b1113175..0b6ca566e 100644 --- a/ci/build_autogen.bat +++ b/ci/build_autogen.bat @@ -10,7 +10,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe echo "building from csources: %nim_csources%" if not exist %nim_csourcesDir% ( - git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) if not exist %nim_csources% ( diff --git a/ci/funs.sh b/ci/funs.sh index 4a8887658..e25922c8c 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -120,8 +120,12 @@ nimBuildCsourcesIfNeeded(){ else # Note: using git tags would allow fetching just what's needed, unlike git hashes, e.g. # via `git clone -q --depth 1 --branch $tag $nim_csourcesUrl`. - echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir" - echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash + echo_run git clone -q --depth 1 -b $nim_csourcesBranch \ + $nim_csourcesUrl "$nim_csourcesDir" + # old `git` versions don't support -C option, using `cd` explicitly: + echo_run cd "$nim_csourcesDir" + echo_run git checkout $nim_csourcesHash + echo_run cd "$OLDPWD" # if needed we could also add: `git reset --hard $nim_csourcesHash` fi _nimBuildCsourcesIfNeeded "$@" diff --git a/config/build_config.txt b/config/build_config.txt index 2484d7a52..75dbb1da3 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -1,4 +1,5 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v1 nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git -nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff +nim_csourcesBranch=master +nim_csourcesHash=9a7f751d23c49c75a0b6f63a234c575dc0df7231 diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index 52b84f0d8..ecf0023d3 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -61,7 +61,7 @@ SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe echo "building from csources: %nim_csources%" if not exist %nim_csourcesDir% ( - git clone -q --depth 1 %nim_csourcesUrl% %nim_csourcesDir% + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% ) if not exist %nim_csources% ( diff --git a/tools/deps.nim b/tools/deps.nim index 8ed95b59a..95b2916a1 100644 --- a/tools/deps.nim +++ b/tools/deps.nim @@ -28,8 +28,13 @@ proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, # from failing execRetry fmt"git clone -q {url} {destDir2}" if isGitRepo(destDir): - execRetry fmt"git -C {destDir2} fetch -q" - exec fmt"git -C {destDir2} checkout -q {commit}" + let oldDir = getCurrentDir() + setCurrentDir(destDir2) + try: + execRetry "git fetch -q" + exec fmt"git checkout -q {commit}" + finally: + setCurrentDir(oldDir) elif allowBundled: discard "this dependency was bundled with Nim, don't do anything" else: diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index aff0c5da0..8c40cf50f 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -25,7 +25,8 @@ endif ifeq ($(uos),linux) myos = linux - LDFLAGS += -ldl -lm + # add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17 + LDFLAGS += -ldl -lm -lrt endif ifeq ($(uos),dragonfly) myos = freebsd From b72ecaf639ca7b2edf5a762cfea3a11f7cc5da9a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 6 Jul 2021 21:00:59 -0700 Subject: [PATCH 0198/1654] fix shebangs #! /xxx => #!/xxx (#18444) --- build_all.sh | 2 +- lib/pure/unidecode/gen.py | 2 +- tools/ci_generate.nim | 2 +- tools/niminst/buildsh.nimf | 2 +- tools/niminst/deinstall.nimf | 2 +- tools/niminst/install.nimf | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_all.sh b/build_all.sh index 37374aa7c..fb3d15d45 100755 --- a/build_all.sh +++ b/build_all.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh # DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim` # build development version of the compiler; can be rerun safely. diff --git a/lib/pure/unidecode/gen.py b/lib/pure/unidecode/gen.py index 0b180a381..2fb69f7b2 100644 --- a/lib/pure/unidecode/gen.py +++ b/lib/pure/unidecode/gen.py @@ -1,4 +1,4 @@ -#! usr/bin/env python3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Generates the unidecode.dat module diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index ecf0023d3..a8a80e026 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -82,7 +82,7 @@ if not exist %nim_csources% ( proc genPosixScript(): string = result = fmt""" -#! /bin/sh +#!/bin/sh # {doNotEdit} # build development version of the compiler; can be rerun safely. diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index e7afd402e..f74413c09 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateBuildShellScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated from niminst\n" & +# result = "#!/bin/sh\n# Generated from niminst\n" & # "# Template is in tools/niminst/buildsh.nimf\n" & # "# To regenerate run ``niminst csource`` or ``koch csource``\n" diff --git a/tools/niminst/deinstall.nimf b/tools/niminst/deinstall.nimf index 8b4477369..0dcfda75e 100644 --- a/tools/niminst/deinstall.nimf +++ b/tools/niminst/deinstall.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateDeinstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii if [ $# -eq 1 ] ; then diff --git a/tools/niminst/install.nimf b/tools/niminst/install.nimf index a78914ecd..c2fc96e94 100644 --- a/tools/niminst/install.nimf +++ b/tools/niminst/install.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar = '?') | standard #proc generateInstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii set -e From d1447fe25d40e35d4746d570701d23333ff480a0 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 6 Jul 2021 21:04:36 -0700 Subject: [PATCH 0199/1654] major improvements to `std/wrapnils`: optimal codegen, case objects, lvalue semantics (#18435) * wrapnils now generates optimal code; also handles case objects * changelog * unsafeAddr => addr --- changelog.md | 4 + lib/std/wrapnils.nim | 185 +++++++++++++++++++-------- tests/stdlib/twrapnils.nim | 249 ++++++++++++++++++++++++++++--------- 3 files changed, 329 insertions(+), 109 deletions(-) diff --git a/changelog.md b/changelog.md index 028743bab..1deb97161 100644 --- a/changelog.md +++ b/changelog.md @@ -279,6 +279,10 @@ issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages) for modules importing `std/wrapnils`. Added `??.` macro which returns an `Option`. + `std/wrapnils` can now be used to protect against `FieldDefect` errors in + case objects, generates optimal code (no overhead compared to manual + if-else branches), and preserves lvalue semantics which allows modifying + an expression. - Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. diff --git a/lib/std/wrapnils.nim b/lib/std/wrapnils.nim index ed0a79d79..facba85fa 100644 --- a/lib/std/wrapnils.nim +++ b/lib/std/wrapnils.nim @@ -1,9 +1,20 @@ -## This module allows chains of field-access and indexing where the LHS can be nil. -## This simplifies code by reducing need for if-else branches around intermediate values -## that may be nil. +## This module allows evaluating expressions safely against the following conditions: +## * nil dereferences +## * field accesses with incorrect discriminant in case objects +## +## `default(T)` is returned in those cases when evaluating an expression of type `T`. +## This simplifies code by reducing need for if-else branches. ## ## Note: experimental module, unstable API. +#[ +TODO: +consider handling indexing operations, eg: +doAssert ?.default(seq[int])[3] == default(int) +]# + +import macros + runnableExamples: type Foo = ref object x1: string @@ -24,8 +35,124 @@ runnableExamples: assert (?.f2.x2.x2).x3 == nil # this terminates ?. early +runnableExamples: + # ?. also allows case object + type B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + var b = B(cond: false, b0: 3) + doAssertRaises(FieldDefect): discard b.b1 # wrong discriminant + doAssert ?.b.b1 == 0.0 # safe + b = B(cond: true, b1: 4.5) + doAssert ?.b.b1 == 4.5 + + # lvalue semantics are preserved: + if (let p = ?.b.b1.addr; p != nil): p[] = 4.7 + doAssert b.b1 == 4.7 + +proc finalize(n: NimNode, lhs: NimNode, level: int): NimNode = + if level == 0: + result = quote: `lhs` = `n` + else: + result = quote: (let `lhs` = `n`) + +proc process(n: NimNode, lhs: NimNode, level: int): NimNode = + var n = n.copyNimTree + var it = n + let addr2 = bindSym"addr" + var old: tuple[n: NimNode, index: int] + while true: + if it.len == 0: + result = finalize(n, lhs, level) + break + elif it.kind == nnkCheckedFieldExpr: + let dot = it[0] + let obj = dot[0] + let objRef = quote do: `addr2`(`obj`) + # avoids a copy and preserves lvalue semantics, see tests + let check = it[1] + let okSet = check[1] + let kind1 = check[2] + let tmp = genSym(nskLet, "tmpCase") + let body = process(objRef, tmp, level + 1) + let tmp3 = nnkDerefExpr.newTree(tmp) + it[0][0] = tmp3 + let dot2 = nnkDotExpr.newTree(@[tmp, dot[1]]) + if old.n != nil: old.n[old.index] = dot2 + else: n = dot2 + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp3`.`kind1` notin `okSet`: break + `assgn` + break + elif it.kind in {nnkHiddenDeref, nnkDerefExpr}: + let tmp = genSym(nskLet, "tmp") + let body = process(it[0], tmp, level + 1) + it[0] = tmp + let assgn = finalize(n, lhs, level) + result = quote do: + `body` + if `tmp` == nil: break + `assgn` + break + elif it.kind == nnkCall: # consider extending to `nnkCallKinds` + # `copyNimTree` needed to avoid `typ = nil` issues + old = (it, 1) + it = it[1].copyNimTree + else: + old = (it, 0) + it = it[0] + +macro `?.`*(a: typed): auto = + ## Transforms `a` into an expression that can be safely evaluated even in + ## presence of intermediate nil pointers/references, in which case a default + ## value is produced. + let lhs = genSym(nskVar, "lhs") + let body = process(a, lhs, 0) + result = quote do: + var `lhs`: type(`a`) + block: + `body` + `lhs` + +# the code below is not needed for `?.` from options import Option, isSome, get, option, unsafeGet, UnpackDefect +macro `??.`*(a: typed): Option = + ## Same as `?.` but returns an `Option`. + runnableExamples: + import std/options + type Foo = ref object + x1: ref int + x2: int + # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: + var f1 = Foo(x1: int.new, x2: 2) + doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. + doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain + doAssert (??.f1.x2).get == 2 + + var f2: Foo + doAssert not (??.f2.x1[]).isSome # f2 was nil + + doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get + doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) + + let lhs = genSym(nskVar, "lhs") + let lhs2 = genSym(nskVar, "lhs") + let body = process(a, lhs2, 0) + result = quote do: + var `lhs`: Option[type(`a`)] + block: + var `lhs2`: type(`a`) + `body` + `lhs` = option(`lhs2`) + `lhs` + template fakeDot*(a: Option, b): untyped = ## See top-level example. let a1 = a # to avoid double evaluations @@ -58,51 +185,7 @@ func `[]`*[U](a: Option[U]): auto {.inline.} = if a2 != nil: result = option(a2[]) -import macros - -func replace(n: NimNode): NimNode = - if n.kind == nnkDotExpr: - result = newCall(bindSym"fakeDot", replace(n[0]), n[1]) - elif n.kind == nnkPar: - doAssert n.len == 1 - result = newCall(bindSym"option", n[0]) - elif n.kind in {nnkCall, nnkObjConstr}: - result = newCall(bindSym"option", n) - elif n.len == 0: - result = newCall(bindSym"option", n) - else: - n[0] = replace(n[0]) - result = n - -proc safeGet[T](a: Option[T]): T {.inline.} = - get(a, default(T)) - -macro `?.`*(a: untyped): auto = - ## Transforms `a` into an expression that can be safely evaluated even in - ## presence of intermediate nil pointers/references, in which case a default - ## value is produced. - result = replace(a) - result = quote do: - # `result`.val # TODO: expose a way to do this directly in std/options, e.g.: `getAsIs` - safeGet(`result`) - -macro `??.`*(a: untyped): Option = - ## Same as `?.` but returns an `Option`. - runnableExamples: - import std/options - type Foo = ref object - x1: ref int - x2: int - # `?.` can't distinguish between a valid vs invalid default value, but `??.` can: - var f1 = Foo(x1: int.new, x2: 2) - doAssert (??.f1.x1[]).get == 0 # not enough to tell when the chain was valid. - doAssert (??.f1.x1[]).isSome # a nil didn't occur in the chain - doAssert (??.f1.x2).get == 2 - - var f2: Foo - doAssert not (??.f2.x1[]).isSome # f2 was nil - from std/options import UnpackDefect - doAssertRaises(UnpackDefect): discard (??.f2.x1[]).get - doAssert ?.f2.x1[] == 0 # in contrast, this returns default(int) - - result = replace(a) +when false: + # xxx: expose a way to do this directly in std/options, e.g.: `getAsIs` + proc safeGet[T](a: Option[T]): T {.inline.} = + get(a, default(T)) diff --git a/tests/stdlib/twrapnils.nim b/tests/stdlib/twrapnils.nim index 956204355..1a66f4cd9 100644 --- a/tests/stdlib/twrapnils.nim +++ b/tests/stdlib/twrapnils.nim @@ -7,80 +7,213 @@ proc checkNotZero(x: float): float = proc main() = var witness = 0 - type Bar = object - b1: int - b2: ptr string + block: + type Bar = object + b1: int + b2: ptr string - type Foo = ref object - x1: float - x2: Foo - x3: string - x4: Bar - x5: seq[int] - x6: ptr Bar - x7: array[2, string] - x8: seq[int] - x9: ref Bar + type Foo = ref object + x1: float + x2: Foo + x3: string + x4: Bar + x5: seq[int] + x6: ptr Bar + x7: array[2, string] + x8: seq[int] + x9: ref Bar - type Gook = ref object - foo: Foo + type Gook = ref object + foo: Foo - proc fun(a: Bar): auto = a.b2 + proc fun(a: Bar): auto = a.b2 - var a: Foo + var a: Foo - var x6: ptr Bar - when nimvm: discard # pending https://github.com/timotheecour/Nim/issues/568 - else: - x6 = create(Bar) - x6.b1 = 42 - var a2 = Foo(x1: 1.0, x5: @[10, 11], x6: x6) - var a3 = Foo(x1: 1.2, x3: "abc") - a3.x2 = a3 + var x6: ptr Bar + when nimvm: discard # pending https://github.com/timotheecour/Nim/issues/568 + else: + x6 = create(Bar) + x6.b1 = 42 + var a2 = Foo(x1: 1.0, x5: @[10, 11], x6: x6) + var a3 = Foo(x1: 1.2, x3: "abc") + a3.x2 = a3 - var gook = Gook(foo: a) + var gook = Gook(foo: a) - proc initFoo(x1: float): auto = - witness.inc - result = Foo(x1: x1) + proc initFoo(x1: float): auto = + witness.inc + result = Foo(x1: x1) - doAssert ?.a.x2.x2.x1 == 0.0 - doAssert ?.a3.x2.x2.x1 == 1.2 - doAssert ?.a3.x2.x2.x3[1] == 'b' + doAssert ?.a.x2.x2.x1 == 0.0 + doAssert ?.a3.x2.x2.x1 == 1.2 + doAssert ?.a3.x2.x2.x3[1] == 'b' - doAssert ?.a3.x2.x2.x5.len == 0 - doAssert a3.x2.x2.x3.len == 3 + doAssert ?.a3.x2.x2.x5.len == 0 + doAssert a3.x2.x2.x3.len == 3 - doAssert ?.a.x2.x2.x3[1] == default(char) - # here we only apply wrapnil around gook.foo, not gook (and assume gook is not nil) - doAssert ?.(gook.foo).x2.x2.x1 == 0.0 + doAssert ?.a.x2.x2.x3[1] == default(char) + # here we only apply wrapnil around gook.foo, not gook (and assume gook is not nil) + doAssert ?.(gook.foo).x2.x2.x1 == 0.0 - when nimvm: discard - else: - doAssert ?.a2.x6[] == Bar(b1: 42) # deref for ptr Bar + when nimvm: discard + else: + doAssert ?.a2.x6[] == Bar(b1: 42) # deref for ptr Bar - doAssert ?.a2.x1.checkNotZero == 1.0 - doAssert a == nil - # shows that checkNotZero won't be called if a nil is found earlier in chain - doAssert ?.a.x1.checkNotZero == 0.0 + doAssert ?.a2.x1.checkNotZero == 1.0 + doAssert a == nil + # shows that checkNotZero won't be called if a nil is found earlier in chain + doAssert ?.a.x1.checkNotZero == 0.0 - when nimvm: discard - else: - # checks that a chain without nil but with an empty seq still raises - doAssertRaises(IndexDefect): discard ?.a2.x8[3] + when nimvm: discard + else: + # checks that a chain without nil but with an empty seq still raises + doAssertRaises(IndexDefect): discard ?.a2.x8[3] - # make sure no double evaluation bug - doAssert witness == 0 - doAssert ?.initFoo(1.3).x1 == 1.3 - doAssert witness == 1 + # make sure no double evaluation bug + doAssert witness == 0 + doAssert ?.initFoo(1.3).x1 == 1.3 + doAssert witness == 1 - # here, it's used twice, to deref `ref Bar` and then `ptr string` - doAssert ?.a.x9[].fun[] == "" + # here, it's used twice, to deref `ref Bar` and then `ptr string` + doAssert ?.a.x9[].fun[] == "" - block: # `??.` - doAssert (??.a3.x2.x2.x3.len).get == 3 - doAssert (??.a2.x4).isSome - doAssert not (??.a.x4).isSome + block: # `??.` + doAssert (??.a3.x2.x2.x3.len).get == 3 + doAssert (??.a2.x4).isSome + doAssert not (??.a.x4).isSome + + block: + type + A = object + b: B + B = object + c: C + C = object + d: D + D = ref object + e: E + e2: array[2, E] + e3: seq[E] + d3: D + i4: int + E = object + f: int + d2: D + proc identity[T](a: T): T = a + proc identity2[T](a: T, ignore: int): T = a + var a: A + doAssert ?.a.b.c.d.e.f == 0 + doAssert ?.a.b.c.d.e.d2.d3[].d3.e.d2.e.f == 0 + doAssert ?.a.b.c.d.d3[].e.f == 0 + doAssert ?.a.b.c.d.e2[0].d2.e3[0].f == 0 + doAssert ?.a == A.default + doAssert ?.a.b.c.d.e == E.default + doAssert ?.a.b.c.d.e.d2 == nil + + doAssert ?.a.identity.b.c.identity2(12).d.d3.e.f == 0 + doAssert ?.a.b.c.d.d3.e2[0].f == 0 + a.b.c.d = D() + a.b.c.d.d3 = a.b.c.d + a.b.c.d.e2[0].f = 5 + doAssert ?.a.b.c.d.d3.e2[0].f == 5 + + var d: D = nil + doAssert ?.d.identity.i4 == 0 + doAssert ?.d.i4.identity == 0 + + block: # case objects + type + Kind = enum k0, k1, k2 + V = object + case kind: Kind + of k0: + x0: int + of k1: + x1: int + of k2: + x2: int + A = object + v0: V + + block: + var a = V(kind: k0, x0: 3) + doAssert ?.a.x0 == 3 + doAssert ?.a.x1 == 0 + a = V(kind: k1, x1: 5) + doAssert ?.a.x0 == 0 + doAssert ?.a.x1 == 5 + + block: + var a = A(v0: V(kind: k0, x0: 10)) + doAssert ?.a.v0.x0 == 10 + doAssert ?.a.v0.x1 == 0 + a.v0 = V(kind: k2, x2: 8) + doAssert ?.a.v0.x0 == 0 + doAssert ?.a.v0.x1 == 0 + doAssert ?.a.v0.x2 == 8 + + block: # `nnkCall` + type + A = object + a0: int + d: D + D = ref object + i4: int + + proc identity[T](a: T): T = a + var d: D = nil + doAssert ?.d.i4.identity == 0 + doAssert ?.identity(?.d.i4) == 0 + doAssert ?.identity(d.i4) == 0 + doAssert ?.identity(d) == nil + doAssert ?.identity(d[]) == default(typeof(d[])) + doAssert ?.identity(d[]).i4 == 0 + var a: A + doAssert ?.identity(a) == default(A) + doAssert ?.identity(a.a0) == 0 + doAssert ?.identity(a.d) == nil + doAssert ?.identity(a.d.i4) == 0 + + block: # lvalue semantic propagation + type + A = ref object + a0: A + a1: seq[A] + a2: int + + B = object + b0: int + case cond: bool + of false: discard + of true: + b1: float + + block: + var a: A + doAssert ?.a.a0.a1[0].a2.addr == nil + a = A(a2: 3) + doAssert ?.a.a0.a1[0].a2.addr == nil + a.a0 = a + a.a1 = @[a] + let p = ?.a.a0.a1[0].a2.addr + doAssert p != nil + p[] = 5 + doAssert a.a2 == 5 + + block: + var b = B(cond: false, b0: 3) + let p = ?.b.b1.addr + doAssert p == nil + b = B(cond: true, b1: 4.5) + let p2 = ?.b.b1.addr + doAssert p2 != nil + p2[] = 4.6 + doAssert b.b1 == 4.6 + # useful pattern, impossible with Options + if (let p3 = ?.b.b1.addr; p3 != nil): + p3[] = 4.7 + doAssert b.b1 == 4.7 main() static: main() From 3eb3e6b9a38aed1f26014bd5dece216b0c625b1a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Jul 2021 09:39:01 +0200 Subject: [PATCH 0200/1654] ORC: use =destroy instead of =dispose (#18440) * ORC refactoring in preparation for further changes (=dispose must die) * ORC: embrace =destroy, avoid =dispose * ORC: no need for =dispose * closes #18421 --- compiler/ast.nim | 3 +- compiler/ccgtypes.nim | 5 +- compiler/liftdestructors.nim | 49 ++--------- compiler/semstmts.nim | 2 - doc/manual.rst | 2 +- lib/system.nim | 1 - lib/system/cellseqs_v2.nim | 26 +++--- lib/system/cyclebreaker.nim | 3 +- lib/system/orc.nim | 40 +++++---- tests/arc/thamming_orc.nim | 155 +++++++++++++++++++++++++++++++++++ 10 files changed, 205 insertions(+), 81 deletions(-) create mode 100644 tests/arc/thamming_orc.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 2a2a84e76..26050426a 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -911,7 +911,6 @@ type attachedAsgn, attachedSink, attachedTrace, - attachedDispose, attachedDeepCopy TType* {.acyclic.} = object of TIdObj # \ @@ -1408,7 +1407,7 @@ const MaxLockLevel* = 1000'i16 UnknownLockLevel* = TLockLevel(1001'i16) AttachedOpToStr*: array[TTypeAttachedOp, string] = [ - "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"] + "=destroy", "=copy", "=sink", "=trace", "=deepcopy"] proc `$`*(x: TLockLevel): string = if x.ord == UnspecifiedLockLevel.ord: result = "" diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 6e5ffeaa4..67e701804 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1328,14 +1328,13 @@ proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineIn m.s[cfsData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name]) let destroyImpl = genHook(m, t, info, attachedDestructor) let traceImpl = genHook(m, t, info, attachedTrace) - let disposeImpl = genHook(m, t, info, attachedDispose) var flags = 0 if not canFormAcycle(t): flags = flags or 1 - addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.disposeImpl = (void*)$6; $1.flags = $7;", [ + addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.flags = $6;", [ name, destroyImpl, getTypeDesc(m, t), typeName, - traceImpl, disposeImpl, rope(flags)]) + traceImpl, rope(flags)]) if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions: discard genTypeInfoV1(m, t, info) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 43f87e179..1824557e6 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -421,11 +421,6 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = result = considerAsgnOrSink(c, t, body, x, y, op) if op != nil: setAttachedOp(c.g, c.idgen.module, t, c.kind, op) - of attachedDispose: - var op = getAttachedOp(c.g, t, c.kind) - result = considerAsgnOrSink(c, t, body, x, nil, op) - if op != nil: - setAttachedOp(c.g, c.idgen.module, t, c.kind, op) of attachedDeepCopy: let op = getAttachedOp(c.g, t, attachedDeepCopy) if op != nil: @@ -516,9 +511,6 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if canFormAcycle(t.elemType): # follow all elements: forallElements(c, t, body, x, y) - of attachedDispose: - forallElements(c, t, body, x, y) - body.add genBuiltin(c, mDestroy, "destroy", x) proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, t, body.info, c.idgen) @@ -556,11 +548,6 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if op == nil: return # protect from recursion body.add newHookCall(c, op, x, y) - of attachedDispose: - let op = getAttachedOp(c.g, t, c.kind) - if op == nil: - return # protect from recursion - body.add newHookCall(c, op, x, nil) proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -572,7 +559,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = doAssert t.destructor != nil moveCall.add destructorCall(c, t.destructor, x) body.add moveCall - of attachedDestructor, attachedDispose: + of attachedDestructor: body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: discard "strings are atomic and have no inner elements that are to trace" @@ -667,16 +654,6 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # If the ref is polymorphic we have to account for this body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y) #echo "can follow ", elemType, " static ", isFinal(elemType) - of attachedDispose: - # this is crucial! dispose is like =destroy but we don't follow refs - # as that is dealt within the cycle collector. - if not isCyclic: - body.add genIf(c, cond, actions) - when false: - let cond = copyTree(x) - cond.typ = getSysType(c.g, x.info, tyBool) - actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x) - body.add genIf(c, cond, actions) proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = ## Closures are really like refs except they always use a virtual destructor @@ -725,14 +702,6 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y) - of attachedDispose: - # this is crucial! dispose is like =destroy but we don't follow refs - # as that is dealt within the cycle collector. - when false: - let cond = copyTree(xenv) - cond.typ = getSysType(c.g, xenv.info, tyBool) - actions.add callCodegenProc(c.g, "nimRawDispose", c.info, xenv) - body.add genIf(c, cond, actions) proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -756,7 +725,7 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = else: body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = var actions = newNodeI(nkStmtList, c.info) @@ -781,7 +750,7 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDestructor: body.add genIf(c, x, actions) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if c.kind == attachedDeepCopy: @@ -815,7 +784,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = else: body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) @@ -830,7 +799,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDestructor: body.add genIf(c, xx, actions) of attachedDeepCopy: assert(false, "cannot happen") - of attachedTrace, attachedDispose: discard + of attachedTrace: discard proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind @@ -946,7 +915,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp result.typ = newProcType(info, nextTypeId(idgen), owner) result.typ.addParam dest - if kind notin {attachedDestructor, attachedDispose}: + if kind != attachedDestructor: result.typ.addParam src if kind == attachedAsgn and g.config.selectedGC == gcOrc and @@ -980,7 +949,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; let dest = result.typ.n[1].sym let d = newDeref(newSymNode(dest)) - let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) + let src = if kind == attachedDestructor: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) else: newSymNode(result.typ.n[2].sym) # register this operation already: @@ -1098,13 +1067,13 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf # we do not generate '=trace' nor '=dispose' procs if we # have the cycle detection disabled, saves code size. - let lastAttached = if g.config.selectedGC == gcOrc: attachedDispose + let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace else: attachedSink # bug #15122: We need to produce all prototypes before entering the # mind boggling recursion. Hacks like these imply we should rewrite # this module. - var generics: array[attachedDestructor..attachedDispose, bool] + var generics: array[attachedDestructor..attachedTrace, bool] for k in attachedDestructor..lastAttached: generics[k] = getAttachedOp(g, canon, k) != nil if not generics[k]: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2907d5172..c3402aee3 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1753,8 +1753,6 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = "signature for '" & s.name.s & "' must be proc[T: object](x: var T; y: T)") of "=trace": bindTypeHook(c, s, n, attachedTrace) - of "=dispose": - bindTypeHook(c, s, n, attachedDispose) else: if sfOverriden in s.flags: localError(c.config, n.info, errGenerated, diff --git a/doc/manual.rst b/doc/manual.rst index a99c4b5af..9ddf02009 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3841,7 +3841,7 @@ the operator is in scope (including if it is private). doAssert witness == 3 Type bound operators currently include: -`=destroy`, `=copy`, `=sink`, `=trace`, `=dispose`, `=deepcopy` +`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy` (some of which are still implementation defined and not yet documented). For more details on some of those procs, see diff --git a/lib/system.nim b/lib/system.nim index a4608997a..090ab309e 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1779,7 +1779,6 @@ when not defined(js) and defined(nimV2): align: int name: cstring traceImpl: pointer - disposeImpl: pointer typeInfoV1: pointer # for backwards compat, usually nil flags: int PNimTypeV2 = ptr TNimTypeV2 diff --git a/lib/system/cellseqs_v2.nim b/lib/system/cellseqs_v2.nim index 59cf67dcd..27be48d78 100644 --- a/lib/system/cellseqs_v2.nim +++ b/lib/system/cellseqs_v2.nim @@ -10,20 +10,20 @@ # Cell seqs for cyclebreaker and cyclicrefs_v2. type - CellTuple = (PT, PNimTypeV2) - CellArray = ptr UncheckedArray[CellTuple] - CellSeq = object + CellTuple[T] = (T, PNimTypeV2) + CellArray[T] = ptr UncheckedArray[CellTuple[T]] + CellSeq[T] = object len, cap: int - d: CellArray + d: CellArray[T] -proc add(s: var CellSeq, c: PT; t: PNimTypeV2) {.inline.} = +proc add[T](s: var CellSeq[T], c: T; t: PNimTypeV2) {.inline.} = if s.len >= s.cap: s.cap = s.cap * 3 div 2 when compileOption("threads"): - var d = cast[CellArray](allocShared(uint(s.cap * sizeof(CellTuple)))) + var d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) else: - var d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) - copyMem(d, s.d, s.len * sizeof(CellTuple)) + var d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) + copyMem(d, s.d, s.len * sizeof(CellTuple[T])) when compileOption("threads"): deallocShared(s.d) else: @@ -33,15 +33,15 @@ proc add(s: var CellSeq, c: PT; t: PNimTypeV2) {.inline.} = s.d[s.len] = (c, t) inc(s.len) -proc init(s: var CellSeq, cap: int = 1024) = +proc init[T](s: var CellSeq[T], cap: int = 1024) = s.len = 0 s.cap = cap when compileOption("threads"): - s.d = cast[CellArray](allocShared(uint(s.cap * sizeof(CellTuple)))) + s.d = cast[CellArray[T]](allocShared(uint(s.cap * sizeof(CellTuple[T])))) else: - s.d = cast[CellArray](alloc(s.cap * sizeof(CellTuple))) + s.d = cast[CellArray[T]](alloc(s.cap * sizeof(CellTuple[T]))) -proc deinit(s: var CellSeq) = +proc deinit[T](s: var CellSeq[T]) = if s.d != nil: when compileOption("threads"): deallocShared(s.d) @@ -51,6 +51,6 @@ proc deinit(s: var CellSeq) = s.len = 0 s.cap = 0 -proc pop(s: var CellSeq): (PT, PNimTypeV2) = +proc pop[T](s: var CellSeq[T]): (T, PNimTypeV2) = result = s.d[s.len-1] dec s.len diff --git a/lib/system/cyclebreaker.nim b/lib/system/cyclebreaker.nim index a3159f95c..6ec0f01f7 100644 --- a/lib/system/cyclebreaker.nim +++ b/lib/system/cyclebreaker.nim @@ -53,7 +53,6 @@ depth-first traversal suffices. ]# -type PT = ptr pointer include cellseqs_v2 const @@ -78,7 +77,7 @@ proc nimMarkCyclic(p: pointer) {.compilerRtl, inl.} = discard type GcEnv = object - traceStack: CellSeq + traceStack: CellSeq[ptr pointer] proc trace(p: pointer; desc: PNimTypeV2; j: var GcEnv) {.inline.} = when false: diff --git a/lib/system/orc.nim b/lib/system/orc.nim index df81afcf1..6273e1a21 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -14,7 +14,6 @@ # R.D. Lins / Information Processing Letters 109 (2008) 71–78 # -type PT = Cell include cellseqs_v2 const @@ -68,10 +67,10 @@ const type GcEnv = object - traceStack: CellSeq + traceStack: CellSeq[ptr pointer] when useJumpStack: - jumpStack: CellSeq # Lins' jump stack in order to speed up traversals - toFree: CellSeq + jumpStack: CellSeq[ptr pointer] # Lins' jump stack in order to speed up traversals + toFree: CellSeq[Cell] freed, touched, edges, rcSum: int keepThreshold: bool @@ -92,13 +91,13 @@ proc free(s: Cell; desc: PNimTypeV2) {.inline.} = when logOrc: writeCell("free", s, desc) - if desc.disposeImpl != nil: - cast[DisposeProc](desc.disposeImpl)(p) + if desc.destructor != nil: + cast[DestructorProc](desc.destructor)(p) when false: cstderr.rawWrite desc.name cstderr.rawWrite " " - if desc.disposeImpl == nil: + if desc.destructor == nil: cstderr.rawWrite "lacks dispose" if desc.traceImpl != nil: cstderr.rawWrite ", but has trace\n" @@ -125,16 +124,16 @@ proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inli orcAssert strstr(desc.name, "TType") == nil, "following a TType but it's acyclic!" var j = cast[ptr GcEnv](env) - j.traceStack.add(head p[], desc) + j.traceStack.add(p, desc) proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inline.} = let p = cast[ptr pointer](q) if p[] != nil: var j = cast[ptr GcEnv](env) - j.traceStack.add(head p[], cast[ptr PNimTypeV2](p[])[]) + j.traceStack.add(p, cast[ptr PNimTypeV2](p[])[]) var - roots {.threadvar.}: CellSeq + roots {.threadvar.}: CellSeq[Cell] proc unregisterCycle(s: Cell) = # swap with the last element. O(1) @@ -162,7 +161,8 @@ proc scanBlack(s: Cell; desc: PNimTypeV2; j: var GcEnv) = trace(s, desc, j) when logOrc: writeCell("root still alive", s, desc) while j.traceStack.len > until: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] inc t.rc, rcIncrement if t.color != colBlack: t.setColor colBlack @@ -187,7 +187,8 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = orcAssert(j.traceStack.len == 0, "markGray: trace stack not empty") trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] dec t.rc, rcIncrement inc j.edges when useJumpStack: @@ -195,7 +196,7 @@ proc markGray(s: Cell; desc: PNimTypeV2; j: var GcEnv) = t.rc = t.rc or jumpStackFlag when traceCollector: cprintf("[Now in jumpstack] %p %ld color %ld in jumpstack %ld\n", t, t.rc shr rcShift, t.color, t.rc and jumpStackFlag) - j.jumpStack.add(t, desc) + j.jumpStack.add(entry, desc) if t.color != colGray: t.setColor colGray inc j.touched @@ -225,7 +226,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = # that are still alive; we also need to mark what they # refer to as alive: while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop + let (entry, desc) = j.jumpStack.pop + let t = head entry[] # not in jump stack anymore! t.rc = t.rc and not jumpStackFlag if t.color == colGray and (t.rc shr rcShift) >= 0: @@ -239,7 +241,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = s.setColor(colWhite) trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] if t.color == colGray: if (t.rc shr rcShift) >= 0: scanBlack(t, desc, j) @@ -249,7 +252,8 @@ proc scan(s: Cell; desc: PNimTypeV2; j: var GcEnv) = # that are still alive; we also need to mark what they # refer to as alive: while j.jumpStack.len > 0: - let (t, desc) = j.jumpStack.pop + let (entry, desc) = j.jumpStack.pop + let t = head entry[] # not in jump stack anymore! t.rc = t.rc and not jumpStackFlag if t.color == colGray and (t.rc shr rcShift) >= 0: @@ -285,7 +289,9 @@ proc collectColor(s: Cell; desc: PNimTypeV2; col: int; j: var GcEnv) = j.toFree.add(s, desc) trace(s, desc, j) while j.traceStack.len > 0: - let (t, desc) = j.traceStack.pop() + let (entry, desc) = j.traceStack.pop() + let t = head entry[] + entry[] = nil # ensure that the destructor does touch moribund objects! if t.color == col and t.rootIdx == 0: j.toFree.add(t, desc) t.setColor(colBlack) diff --git a/tests/arc/thamming_orc.nim b/tests/arc/thamming_orc.nim new file mode 100644 index 000000000..463f7117e --- /dev/null +++ b/tests/arc/thamming_orc.nim @@ -0,0 +1,155 @@ +discard """ + output: '''(allocCount: 1114, deallocCount: 1112) +created 439 destroyed 402''' + cmd: "nim c --gc:orc -d:nimAllocStats $file" +""" + +# bug #18421 + +# test Nim Hamming Number Lazy List algo with reference counts and not... +# compile with "-d:release -d:danger" and test with various +# memory managment GC's, allocators, threading, etc. +# it should be guaranteed to work with zero memory leaks with `--gc:orc`... + +# compile with `-d:trace20` to trace creation and destruction of first 20 values. + +from math import log2 + +# implement our own basic BigInt so the bigints library isn't necessary... +type + BigInt = object + digits: seq[uint32] +let zeroBigInt = BigInt(digits: @[ 0'u32 ]) +let oneBigInt = BigInt(digits: @[ 1'u32 ]) + +proc shladd(bi: var BigInt; n: int; a: BigInt) = + # assume that both `bi` and `a` are sized correctly with + # msuint32 for both not containing a zero + let alen = a.digits.len + let mx = max(bi.digits.len, a.digits.len) + for i in bi.digits.len ..< mx: bi.digits.add 0'u32 + var cry = 0'u64 + for i in 0 ..< alen: + cry += (bi.digits[i].uint64 shl n) + a.digits[i].uint64 + bi.digits[i] = cry.uint32; cry = cry shr 32 + for i in alen ..< mx: + cry += bi.digits[i].uint64 shl n + bi.digits[i] = cry.uint32; cry = cry shr 32 + if cry > 0'u64: + bi.digits.add cry.uint32 + +proc `$`(x: BigInt): string = + if x.digits.len == 0 or (x.digits.len == 1 and x.digits[0] == 0'u32): + return "0" + result = ""; var n = x; var msd = n.digits.high + while msd >= 0: + if n.digits[msd] == 0'u32: msd.dec; continue + var brw = 0.uint64 + for i in countdown(msd, 0): + let dvdnd = n.digits[i].uint64 + (brw shl 32) + let q = dvdnd div 10'u64; brw = dvdnd - q * 10'u64 + n.digits[i] = q.uint32 + result &= $brw + for i in 0 .. result.high shr 1: # reverse result string in place + let tmp = result[^(i + 1)] + result[^(i + 1)] = result[i] + result[i] = tmp + +type TriVal = (uint32, uint32, uint32) +type LogRep = (float64, TriVal) +type LogRepf = proc(x: LogRep): LogRep +const one: LogRep = (0.0'f64, (0'u32, 0'u32, 0'u32)) +proc `<`(me: LogRep, othr: LogRep): bool = me[0] < othr[0] + +proc convertTriVal2BigInt(tpl: TriVal): BigInt = + result = oneBigInt + let (x2, x3, x5) = tpl + for _ in 1 .. x2: result.shladd 1, zeroBigInt + for _ in 1 .. x3: result.shladd 1, result + for _ in 1 .. x5: result.shladd 2, result + +const lb2 = 1.0'f64 +const lb3 = 3.0'f64.log2 +const lb5 = 5.0'f64.log2 + +proc mul2(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb2, (x2 + 1, x3, x5)) + +proc mul3(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb3, (x2, x3 + 1, x5)) + +proc mul5(me: LogRep): LogRep = + let (lr, tpl) = me; let (x2, x3, x5) = tpl + (lr + lb5, (x2, x3, x5 + 1)) + +type + LazyListObj = object + hd: LogRep + tlf: proc(): LazyList {.closure.} + tl: LazyList + LazyList = ref LazyListObj + +var destroyed = 0 + +proc `=destroy`(ll: var LazyListObj) = + if ll.tlf == nil and ll.tl == nil: return + destroyed += 1 + + when defined(trace20): + echo "destroying: ", (destroyed, ll.hd[1].convertTriVal2BigInt) + if ll.tlf != nil: ll.tlf.`=destroy` + if ll.tl != nil: ll.tl.`=destroy` + #wasMoved(ll) + +proc rest(ll: LazyList): LazyList = # not thread-safe; needs lock on thunk + if ll.tlf != nil: ll.tl = ll.tlf(); ll.tlf = nil + ll.tl + +var created = 0 +iterator hammings(until: int): TriVal = + proc merge(x, y: LazyList): LazyList = + let xh = x.hd; let yh = y.hd; created += 1 + when defined(trace20): + echo "merge create: ", (created - 1, (if xh < yh: xh else: yh)[1].convertTriVal2BigInt) + if xh < yh: LazyList(hd: xh, tlf: proc(): auto = merge x.rest, y) + else: LazyList(hd: yh, tlf: proc(): auto = merge x, y.rest) + proc smult(mltf: LogRepf; s: LazyList): LazyList = + proc smults(ss: LazyList): LazyList = + when defined(trace20): + echo "mult create: ", (created, ss.hd.mltf[1].convertTriVal2BigInt) + created += 1; LazyList(hd: ss.hd.mltf, tlf: proc(): auto = ss.rest.smults) + s.smults + proc unnsm(s: LazyList, mltf: LogRepf): LazyList = + var r: LazyList = nil + when defined(trace20): + echo "first create: ", (created, one[1].convertTriVal2BigInt) + let frst = LazyList(hd: one, tlf: proc(): LazyList = r); created += 1 + r = if s == nil: smult(mltf, frst) else: s.merge smult(mltf, frst) + r + yield one[1] + var hmpll: LazyList = ((nil.unnsm mul5).unnsm mul3).unnsm mul2 + for _ in 2 .. until: + yield hmpll.hd[1]; hmpll = hmpll.rest # almost forever + +proc main = + var s = "" + for h in hammings(20): s &= $h.convertTrival2BigInt & " " + doAssert s == "1 2 3 4 5 6 8 9 10 12 15 16 18 20 24 25 27 30 32 36 ", + "Algorithmic error finding first 20 Hamming numbers!!!" + + when not defined(trace20): + var lsth: TriVal; created = 0; destroyed = 0 + for h in hammings(200): lsth = h + doAssert $lsth.convertTriVal2BigInt == "16200", + "Algorithmic error finding 200th Hamming number!!!" + +let mem = getOccupiedMem() +main() +GC_FullCollect() +let mb = getOccupiedMem() - mem +#doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!" + +echo getAllocStats() +echo "created ", created, " destroyed ", destroyed From ffce6de84c94348dd5a615c9855a14ed64807449 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 7 Jul 2021 00:43:16 -0700 Subject: [PATCH 0201/1654] improve rendering of newOSError.additionalInfo (#18443) * improve rendering of newOSError.additionalInfo * fixup --- lib/pure/includes/oserr.nim | 5 +++-- lib/pure/net.nim | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index b0740cbe6..e42771d52 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -80,8 +80,9 @@ proc newOSError*( e.msg = osErrorMsg(errorCode) if additionalInfo.len > 0: if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.addQuoted additionalInfo + e.msg.add "Additional info: " + e.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. if e.msg == "": e.msg = "unknown OS error" return e diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ec3e4e74d..9c3827751 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -958,7 +958,9 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeaddrinfo(aiList) - raiseOSError(osLastError(), "address: $# port: $#" % [address, $port]) + var address2: string + address2.addQuoted address + raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) freeaddrinfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, From 2d758795f05d8edf567011eeb49f0cdd38b6dc13 Mon Sep 17 00:00:00 2001 From: Aditya Siram Date: Wed, 7 Jul 2021 21:21:30 -0500 Subject: [PATCH 0202/1654] Rename test variable due to unfortunate connotations (#18453) --- tests/stdlib/twrapnils.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/stdlib/twrapnils.nim b/tests/stdlib/twrapnils.nim index 1a66f4cd9..a0549c1bc 100644 --- a/tests/stdlib/twrapnils.nim +++ b/tests/stdlib/twrapnils.nim @@ -23,7 +23,7 @@ proc main() = x8: seq[int] x9: ref Bar - type Gook = ref object + type Goo = ref object foo: Foo proc fun(a: Bar): auto = a.b2 @@ -39,7 +39,7 @@ proc main() = var a3 = Foo(x1: 1.2, x3: "abc") a3.x2 = a3 - var gook = Gook(foo: a) + var goo = Goo(foo: a) proc initFoo(x1: float): auto = witness.inc @@ -53,8 +53,8 @@ proc main() = doAssert a3.x2.x2.x3.len == 3 doAssert ?.a.x2.x2.x3[1] == default(char) - # here we only apply wrapnil around gook.foo, not gook (and assume gook is not nil) - doAssert ?.(gook.foo).x2.x2.x1 == 0.0 + # here we only apply wrapnil around goo.foo, not goo (and assume goo is not nil) + doAssert ?.(goo.foo).x2.x2.x1 == 0.0 when nimvm: discard else: From 5a42f4a53e22051493232652cda7fb3d32c16da8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 8 Jul 2021 03:26:57 -0700 Subject: [PATCH 0203/1654] followup #18252: show special flags -d:danger, -d:release when given (#18451) --- compiler/msgs.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 8b4365ec4..242b06587 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -677,6 +677,10 @@ proc genSuccessX*(conf: ConfigRef) = elif optOptimizeSize in conf.options: build.add "size" else: build.add "none (DEBUG BUILD, `-d:release` generates faster code)" # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html + var flags = "" + if isDefined(conf, "danger"): flags.add " -d:danger" + elif isDefined(conf, "release"): flags.add " -d:release" + if flags.len > 0: build.add "; options:" & flags let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName # xxx honor conf.filenameOption more accurately From 836b061ae30bb471dc7deef41bb454e1d047bab7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 8 Jul 2021 03:28:06 -0700 Subject: [PATCH 0204/1654] improve `--declaredLocs` to help disambiguate types (generics, aliases etc) (#18389) * improve --declaredlocs to help disambiguate types (generics, aliases etc) * avoid a cyclic deps * fix test after rebase --- changelog.md | 2 +- compiler/astmsgs.nim | 2 +- compiler/semcall.nim | 12 ++--- compiler/semexprs.nim | 2 +- compiler/types.nim | 8 +++ tests/errmsgs/tdeclaredlocs.nim | 92 ++++++++++++++++++++++++++++++++ tests/errmsgs/tproc_mismatch.nim | 2 +- tests/errmsgs/tsigmatch.nim | 6 +-- 8 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 tests/errmsgs/tdeclaredlocs.nim diff --git a/changelog.md b/changelog.md index 1deb97161..f4c31fbc9 100644 --- a/changelog.md +++ b/changelog.md @@ -395,7 +395,7 @@ ## Compiler changes -- Added `--declaredlocs` to show symbol declaration location in messages. +- Added `--declaredLocs` to show symbol declaration location in messages. - You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index 90bdce7cf..d9105b761 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -18,7 +18,7 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = # xxx figure out how to resolve `tyGenericParam`, e.g. for # proc fn[T](a: T, b: T) = discard # fn(1.1, "a") - let typ = typ.skipTypes(abstractInst + {tyStatic, tyUserTypeClassInst} - {tyRange}) + let typ = typ.skipTypes(abstractInst + {tyStatic, tySequence, tyArray, tySet, tyUserTypeClassInst, tyVar, tyRef, tyPtr} - {tyRange}) result.add " [$1" % typ.kind.toHumanStr if typ.sym != nil: result.add " declared in " & toFileLineCol(conf, typ.sym.info) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index cec656fc2..d8dba3e6c 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -231,11 +231,10 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): of kMissingParam: candidates.add("\n missing parameter: " & nameParam) of kTypeMismatch, kVarNeeded: doAssert nArg != nil - var wanted = err.firstMismatch.formal.typ + let wanted = err.firstMismatch.formal.typ doAssert err.firstMismatch.formal != nil candidates.add("\n required type for " & nameParam & ": ") - candidates.add typeToString(wanted) - candidates.addDeclaredLocMaybe(c.config, wanted) + candidates.addTypeDeclVerboseMaybe(c.config, wanted) candidates.add "\n but expression '" if err.firstMismatch.kind == kVarNeeded: candidates.add renderNotLValue(nArg) @@ -243,9 +242,8 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): else: candidates.add renderTree(nArg) candidates.add "' is of type: " - var got = nArg.typ - candidates.add typeToString(got) - candidates.addDeclaredLocMaybe(c.config, got) + let got = nArg.typ + candidates.addTypeDeclVerboseMaybe(c.config, got) doAssert wanted != nil if got != nil: if got.kind == tyProc and wanted.kind == tyProc: @@ -274,7 +272,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): const errTypeMismatch = "type mismatch: got <" - errButExpected = "but expected one of: " + errButExpected = "but expected one of:" errUndeclaredField = "undeclared field: '$1'" errUndeclaredRoutine = "attempting to call undeclared routine: '$1'" errBadRoutine = "attempting to call routine: '$1'$2" diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index acf408db0..f46d44bf0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -988,7 +988,7 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = break if not hasErrorType: let typ = n[0].typ - msg.add(">\nbut expected one of: \n" & + msg.add(">\nbut expected one of:\n" & typeToString(typ)) # prefer notin preferToResolveSymbols # t.sym != nil diff --git a/compiler/types.nim b/compiler/types.nim index bdf08ffe9..49ebb1dae 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -50,6 +50,14 @@ type pcmDifferentCallConv proc typeToString*(typ: PType; prefer: TPreferedDesc = preferName): string + +proc addTypeDeclVerboseMaybe*(result: var string, conf: ConfigRef; typ: PType) = + if optDeclaredLocs in conf.globalOptions: + result.add typeToString(typ, preferMixed) + result.addDeclaredLoc(conf, typ) + else: + result.add typeToString(typ) + template `$`*(typ: PType): string = typeToString(typ) proc base*(t: PType): PType = diff --git a/tests/errmsgs/tdeclaredlocs.nim b/tests/errmsgs/tdeclaredlocs.nim new file mode 100644 index 000000000..926ebf217 --- /dev/null +++ b/tests/errmsgs/tdeclaredlocs.nim @@ -0,0 +1,92 @@ +discard """ + action: reject + matrix: "--declaredLocs --hints:off" + nimoutFull: true + nimout: ''' +tdeclaredlocs.nim(92, 3) Error: type mismatch: got +but expected one of: +proc fn(a: Bam) [proc declared in tdeclaredlocs.nim(86, 6)] + first type mismatch at position: 1 + required type for a: Bam [object declared in tdeclaredlocs.nim(78, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: Goo[MyInt2]) [proc declared in tdeclaredlocs.nim(89, 6)] + first type mismatch at position: 1 + required type for a: Goo[MyInt2{char}] [object declared in tdeclaredlocs.nim(79, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: Goo[cint]) [proc declared in tdeclaredlocs.nim(88, 6)] + first type mismatch at position: 1 + required type for a: Goo[cint{int32}] [object declared in tdeclaredlocs.nim(79, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: array[3, Bar]) [proc declared in tdeclaredlocs.nim(82, 6)] + first type mismatch at position: 1 + required type for a: array[0..2, Bar] [object declared in tdeclaredlocs.nim(74, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: seq[Bar]) [proc declared in tdeclaredlocs.nim(81, 6)] + first type mismatch at position: 1 + required type for a: seq[Bar] [object declared in tdeclaredlocs.nim(74, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: seq[MyInt1]) [proc declared in tdeclaredlocs.nim(80, 6)] + first type mismatch at position: 1 + required type for a: seq[MyInt1{int}] [int declared in tdeclaredlocs.nim(72, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: set[Baz]) [proc declared in tdeclaredlocs.nim(84, 6)] + first type mismatch at position: 1 + required type for a: set[Baz{enum}] [enum declared in tdeclaredlocs.nim(75, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: set[MyInt2]) [proc declared in tdeclaredlocs.nim(83, 6)] + first type mismatch at position: 1 + required type for a: set[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: var SetBaz) [proc declared in tdeclaredlocs.nim(85, 6)] + first type mismatch at position: 1 + required type for a: var SetBaz [enum declared in tdeclaredlocs.nim(75, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] +proc fn(a: var ref ptr Bam) [proc declared in tdeclaredlocs.nim(87, 6)] + first type mismatch at position: 1 + required type for a: var ref ptr Bam [object declared in tdeclaredlocs.nim(78, 3)] + but expression 'a' is of type: seq[MyInt2{char}] [char declared in tdeclaredlocs.nim(73, 3)] + +expression: fn(a) +''' +""" + +#[ +see also: tests/errmsgs/tsigmatch.nim +]# + + + + + + + + + + + + + + +# line 70 +type + MyInt1 = int + MyInt2 = char + Bar = object + Baz = enum k0, k1 + Baz2 = Baz + SetBaz = set[Baz2] + Bam = ref object + Goo[T] = object +proc fn(a: seq[MyInt1]) = discard +proc fn(a: seq[Bar]) = discard +proc fn(a: array[3, Bar]) = discard +proc fn(a: set[MyInt2]) = discard +proc fn(a: set[Baz]) = discard +proc fn(a: var SetBaz) = discard +proc fn(a: Bam) = discard +proc fn(a: var ref ptr Bam) = discard +proc fn(a: Goo[cint]) = discard +proc fn(a: Goo[MyInt2]) = discard + +var a: seq[MyInt2] +fn(a) diff --git a/tests/errmsgs/tproc_mismatch.nim b/tests/errmsgs/tproc_mismatch.nim index 400c3d441..4ddc7635e 100644 --- a/tests/errmsgs/tproc_mismatch.nim +++ b/tests/errmsgs/tproc_mismatch.nim @@ -6,7 +6,7 @@ discard """ tproc_mismatch.nim(35, 52) Error: type mismatch: got but expected 'proc (a: int, c: float){.closure, noSideEffect.}' Calling convention mismatch: got '{.cdecl.}', but expected '{.closure.}'. tproc_mismatch.nim(39, 6) Error: type mismatch: got -but expected one of: +but expected one of: proc bar(a: proc ()) first type mismatch at position: 1 required type for a: proc (){.closure.} diff --git a/tests/errmsgs/tsigmatch.nim b/tests/errmsgs/tsigmatch.nim index 4f40be2d4..023b7d518 100644 --- a/tests/errmsgs/tsigmatch.nim +++ b/tests/errmsgs/tsigmatch.nim @@ -89,9 +89,9 @@ expression: fun1(default(Mystring), "asdf") - - - +#[ +see also: tests/errmsgs/tdeclaredlocs.nim +]# From 0d74f60742ecff2ac96d97e9ad421aa1b2e8fc85 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 8 Jul 2021 16:09:56 +0200 Subject: [PATCH 0205/1654] Revert "Make 'echo' raise IOErrors when appropriate (#16367)" (#18460) This reverts commit 23d23ecb081be6702d74024be8f96d92d9f88a59. --- changelog.md | 3 --- lib/system/io.nim | 24 ++++++++---------------- tests/exception/t16366.nim | 11 ----------- 3 files changed, 8 insertions(+), 30 deletions(-) delete mode 100644 tests/exception/t16366.nim diff --git a/changelog.md b/changelog.md index f4c31fbc9..849640e06 100644 --- a/changelog.md +++ b/changelog.md @@ -24,9 +24,6 @@ - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous behavior. -- `echo` and `debugEcho` will now raise `IOError` if writing to stdout fails. Previous behavior - silently ignored errors. See #16366. Use `-d:nimLegacyEchoNoRaise` for previous behavior. - - `math.round` now is rounded "away from zero" in JS backend which is consistent with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. diff --git a/lib/system/io.nim b/lib/system/io.nim index 62a9d1e18..591ea3e92 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -227,9 +227,6 @@ when defined(windows): # But we cannot call printf directly as the string might contain \0. # So we have to loop over all the sections separated by potential \0s. var i = c_fprintf(f, "%s", s) - if i < 0: - if doRaise: raiseEIO("cannot write string to file") - return while i < s.len: if s[i] == '\0': let w = c_fputc('\0', f) @@ -787,13 +784,6 @@ when declared(stdout): not defined(nintendoswitch) and not defined(freertos) and hostOS != "any" - const echoDoRaise = not defined(nimLegacyEchoNoRaise) and not defined(guiapp) # see PR #16366 - - template checkErrMaybe(succeeded: bool): untyped = - if not succeeded: - when echoDoRaise: - checkErr(stdout) - proc echoBinSafe(args: openArray[string]) {.compilerproc.} = when defined(androidNDK): var s = "" @@ -806,18 +796,20 @@ when declared(stdout): proc flockfile(f: File) {.importc, nodecl.} proc funlockfile(f: File) {.importc, nodecl.} flockfile(stdout) - defer: funlockfile(stdout) when defined(windows) and compileOption("threads"): acquireSys echoLock - defer: releaseSys echoLock for s in args: when defined(windows): - writeWindows(stdout, s, doRaise = echoDoRaise) + writeWindows(stdout, s) else: - checkErrMaybe(c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) == s.len) + discard c_fwrite(s.cstring, cast[csize_t](s.len), 1, stdout) const linefeed = "\n" - checkErrMaybe(c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) == linefeed.len) - checkErrMaybe(c_fflush(stdout) == 0) + discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout) + discard c_fflush(stdout) + when stdOutLock: + funlockfile(stdout) + when defined(windows) and compileOption("threads"): + releaseSys echoLock when defined(windows) and not defined(nimscript) and not defined(js): diff --git a/tests/exception/t16366.nim b/tests/exception/t16366.nim deleted file mode 100644 index 4bcad79ed..000000000 --- a/tests/exception/t16366.nim +++ /dev/null @@ -1,11 +0,0 @@ -discard """ - action: run - exitcode: 0 - targets: "c cpp" - disabled: openbsd -""" - -echo "foo1" -close stdout -doAssertRaises(IOError): - echo "foo" From 0e44d137f97a8206af1f2747391615e2f237f000 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 9 Jul 2021 03:24:57 +0300 Subject: [PATCH 0206/1654] rm redundant blank lines before literal blocks (#18465) --- lib/packages/docutils/rst.nim | 2 +- lib/packages/docutils/rstgen.nim | 2 +- nimdoc/rst2html/expected/rst_examples.html | 15 +++++---------- tests/stdlib/trst.nim | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 419cf50e7..9abdf3fc3 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1762,7 +1762,7 @@ proc parseLiteralBlock(p: var RstParser): PRstNode = var n = newLeaf("") if currentTok(p).kind == tkIndent: var indent = currentTok(p).ival - inc p.idx + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines while true: case currentTok(p).kind of tkEof: diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 829177263..b9c8f688b 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1258,7 +1258,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = doAssert false, "renderRstToOut" of rnLiteralBlock: renderAux(d, n, "$1\n", - "\n\n\\begin{rstpre}\n$2\n$1\n\\end{rstpre}\n\n", result) + "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) of rnQuotedLiteralBlock: doAssert false, "renderRstToOut" of rnLineBlock: diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 68b796be7..1df91ff1f 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -133,8 +133,7 @@ window.addEventListener('DOMContentLoaded', main);

Note: Assignments, moves, and destruction are specified in the destructors document.

The language constructs are explained using an extended BNF, in which (a)* means 0 or more a's, a+ means 1 or more a's, and (a)? means an optional a. Parentheses may be used to group elements.

& is the lookahead operator; &a means that an a is expected but not consumed. It will be consumed in the following rule.

-

Non-terminals start with a lowercase letter, abstract terminal symbols are in UPPERCASE. Verbatim terminal symbols (including keywords) are quoted with '. An example:

-ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?
+

Non-terminals start with a lowercase letter, abstract terminal symbols are in UPPERCASE. Verbatim terminal symbols (including keywords) are quoted with '. An example:

ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?

In a typical Nim program, most of the code is compiled into the executable. However, some of the code may be executed at compile-time. This can include constant expressions, macro definitions, and Nim procedures used by macro definitions. Most of the Nim language is supported at compile-time, but there are some restrictions -- see Restrictions on Compile-Time Execution for details. We use the term runtime to cover both compile-time execution and code execution in the executable.

var a: array[0..1, char]
@@ -146,8 +145,7 @@ ifStmt = 'if' expr ':' stmts ('elif' expr ':' stmts)* ('else' stmts)?

Encoding

All Nim source files are in the UTF-8 encoding (or its ASCII subset). Other encodings are not supported. Any of the standard platform line termination sequences can be used - the Unix form using ASCII LF (linefeed), the Windows form using the ASCII sequence CR LF (return followed by linefeed), or the old Macintosh form using the ASCII CR (return) character. All of these forms can be used equally, regardless of the platform.

Indentation

Nim's standard grammar describes an indentation sensitive language. This means that all the control structures are recognized by indentation. Indentation consists only of spaces; tabulators are not allowed.

-

With this notation we can now easily define the core of the grammar: A block of statements (simplified example):

-ifStmt = 'if' expr ':' stmt
+
With this notation we can now easily define the core of the grammar: A block of statements (simplified example):
ifStmt = 'if' expr ':' stmt
          (IND{=} 'elif' expr ':' stmt)*
          (IND{=} 'else' ':' stmt)?
 
@@ -174,18 +172,15 @@ stmt = IND{>} stmt ^+ IND{=} DED  # list of statements
 \x HHcharacter with hex value HH; exactly two hex digits are allowed
 \u HHHHunicode codepoint with hex value HHHH; exactly four hex digits are allowed
 \u {H+}unicode codepoint; all hex digits enclosed in {} are used for the codepoint
-
""""long string within quotes""""
Produces:
-"long string within quotes"

+
""""long string within quotes""""
Produces:
"long string within quotes"

 

 
-
Operators
Nim allows user defined operators. An operator is any combination of the following characters:
-=     +     -     *     /     <     >
+
Operators
Nim allows user defined operators. An operator is any combination of the following characters:
=     +     -     *     /     <     >
 @     $     ~     &     %     |
 !     ?     ^     .     :     \

 

 
(The grammar uses the terminal OPR to refer to operator symbols as defined here.)

-
The following strings denote other tokens:
-`   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:

+
The following strings denote other tokens:
`   (    )     {    }     [    ]    ,  ;   [.    .]  {.   .}  (.  .)  [:

 

 
Otherwise, precedence is determined by the first character.

 
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index fb9552479..fe99d6cfa 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -196,6 +196,22 @@ suite "RST parsing":
               rnLeaf  'someBlockQuote'
         """)
 
+  test "no redundant blank lines in literal blocks":
+    check(dedent"""
+      Check::
+
+
+        code
+
+      """.toAst ==
+      dedent"""
+        rnInner
+          rnLeaf  'Check'
+          rnLeaf  ':'
+          rnLiteralBlock
+            rnLeaf  'code'
+      """)
+
 suite "RST indentation":
   test "nested bullet lists":
     let input = dedent """

From 86f5a56fcdbdbd4748c979e6f517d8dfc64857f7 Mon Sep 17 00:00:00 2001
From: Ivan Bobev 
Date: Fri, 9 Jul 2021 03:26:49 +0300
Subject: [PATCH 0207/1654] Fix a bug with starting of asynchronous processes
 (#18464)

The asynchronous process completion handler callback should be called
only once. This is achieved by passing `WT_EXECUTEONLYONCE` flag to the
`registerWaitForSingleObject` Windows API procedure.

Related to cheatfate/asynctools#35
---
 lib/pure/asyncdispatch.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index 17fd5f824..d98ca77df 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -1022,7 +1022,7 @@ when defined(windows) or defined(nimdoc):
       raiseOSError(osLastError())
 
     var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData)))
-    var flags = WT_EXECUTEINWAITTHREAD.DWORD
+    var flags = WT_EXECUTEINWAITTHREAD.DWORD or WT_EXECUTEONLYONCE.DWORD
 
     proc proccb(fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) =
       closeWaitable(hProcess)

From 6869157cd17229dbadb8b1b43c218d5fb28e8eb2 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Fri, 9 Jul 2021 13:44:38 +0800
Subject: [PATCH 0208/1654] [docs] improve std/encodings (#18458)

* [docs] improve std/encodings

* tiny

* shrink

* Apply suggestions from code review

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Apply suggestions from code review

* Update lib/pure/encodings.nim

Co-authored-by: Varriount 

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Co-authored-by: Varriount 
---
 lib/pure/encodings.nim | 53 ++++++++++++++++++++++++++++++++----------
 1 file changed, 41 insertions(+), 12 deletions(-)

diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim
index 89db0a278..dcdc474ae 100644
--- a/lib/pure/encodings.nim
+++ b/lib/pure/encodings.nim
@@ -7,15 +7,42 @@
 #    distribution, for details about the copyright.
 #
 
-## Converts between different character encodings. On UNIX, this uses
+## Routines for converting between different character encodings. On UNIX, this uses
 ## the `iconv`:idx: library, on Windows the Windows API.
+##
+## The following example shows how to change character encodings.
+runnableExamples:
+  let
+    orig = "öäüß"
+    # convert `orig` from "UTF-8" to "CP1252"
+    cp1252 = convert(orig, "CP1252", "UTF-8")
+    # convert `cp1252` from "CP1252" to "ibm850"
+    ibm850 = convert(cp1252, "ibm850", "CP1252")
+    current = getCurrentEncoding()
+  assert orig == "\195\182\195\164\195\188\195\159"
+  assert ibm850 == "\148\132\129\225"
+  assert convert(ibm850, current, "ibm850") == orig
+
+## The example below uses a reuseable `EncodingConverter` object which is
+## created by `open` with `destEncoding` and `srcEncoding` specified. You can use
+## `convert` on this object multiple times.
+runnableExamples:
+  var fromGB2312 = open("utf-8", "gb2312")
+  let first = "\203\173\197\194\163\191\210\187" &
+      "\203\242\209\204\211\234\200\206\198\189\201\250"
+  assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生"
+
+  let second = "\211\208\176\215\205\183\200\231" &
+      "\208\194\163\172\199\227\184\199\200\231\185\202"
+  assert fromGB2312.convert(second) == "有白头如新，倾盖如故"
+
 
 import os
 
 when not defined(windows):
   type
     ConverterObj = object
-    EncodingConverter* = ptr ConverterObj ## can convert between two character sets
+    EncodingConverter* = ptr ConverterObj ## Can convert between two character sets.
 
 else:
   type
@@ -24,8 +51,8 @@ else:
       dest, src: CodePage
 
 type
-  EncodingError* = object of ValueError ## exception that is raised
-                                        ## for encoding errors
+  EncodingError* = object of ValueError ## Exception that is raised
+                                        ## for encoding errors.
 
 when defined(windows):
   import parseutils, strutils
@@ -298,7 +325,7 @@ else:
     importc: "iconv", importIconv.}
 
 proc getCurrentEncoding*(uiApp = false): string =
-  ## retrieves the current encoding. On Unix, always "UTF-8" is returned.
+  ## Retrieves the current encoding. On Unix, "UTF-8" is always returned.
   ## The `uiApp` parameter is Windows specific. If true, the UI's code-page
   ## is returned, if false, the Console's code-page is returned.
   when defined(windows):
@@ -307,7 +334,7 @@ proc getCurrentEncoding*(uiApp = false): string =
     result = "UTF-8"
 
 proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter =
-  ## opens a converter that can convert from `srcEncoding` to `destEncoding`.
+  ## Opens a converter that can convert from `srcEncoding` to `destEncoding`.
   ## Raises `IOError` if it cannot fulfill the request.
   when not defined(windows):
     result = iconvOpen(destEncoding, srcEncoding)
@@ -326,7 +353,7 @@ proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter =
         "cannot find encoding " & srcEncoding)
 
 proc close*(c: EncodingConverter) =
-  ## frees the resources the converter `c` holds.
+  ## Frees the resources the converter `c` holds.
   when not defined(windows):
     iconvClose(c)
 
@@ -421,12 +448,13 @@ when defined(windows):
            else: convertFromWideString(codePageTo, wideString)
 
   proc convert*(c: EncodingConverter, s: string): string =
-    ## converts `s` to `destEncoding` that was given to the converter `c`. It
-    ## assumed that `s` is in `srcEncoding`.
-    ## utf-16BE, utf-32 conversions not supported on windows
     result = convertWin(c.src, c.dest, s)
 else:
   proc convert*(c: EncodingConverter, s: string): string =
+    ## Converts `s` to `destEncoding` that was given to the converter `c`. It
+    ## assumes that `s` is in `srcEncoding`.
+    ##
+    ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows.
     result = newString(s.len)
     var inLen = csize_t len(s)
     var outLen = csize_t len(result)
@@ -467,10 +495,11 @@ else:
 
 proc convert*(s: string, destEncoding = "UTF-8",
                          srcEncoding = "CP1252"): string =
-  ## converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`.
+  ## Converts `s` to `destEncoding`. It assumed that `s` is in `srcEncoding`.
   ## This opens a converter, uses it and closes it again and is thus more
   ## convenient but also likely less efficient than re-using a converter.
-  ## utf-16BE, utf-32 conversions not supported on windows
+  ##
+  ## .. warning:: UTF-16BE and UTF-32 conversions are not supported on Windows.
   var c = open(destEncoding, srcEncoding)
   try:
     result = convert(c, s)

From ae7e7756fea146126ffc5200b2e66bfe2dab4cd4 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 9 Jul 2021 02:41:28 -0700
Subject: [PATCH 0209/1654] runnableExamples now show originating location in
 stacktraces on failure (#18457)

* runnableExamples now show originating location in stacktraces on failure

* PRTEMP

* fix indentation inside multiline strings
---
 compiler/docgen.nim         | 25 +++++++++++++++++++------
 compiler/renderverbatim.nim | 11 ++++++-----
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index e6bb7cef8..1ae00e22e 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -468,6 +468,10 @@ proc runAllExamples(d: PDoc) =
 
 proc quoted(a: string): string = result.addQuoted(a)
 
+proc toInstantiationInfo(conf: ConfigRef, info: TLineInfo): auto =
+  # xxx expose in compiler/lineinfos.nim
+  (conf.toMsgFilename(info), info.line.int, info.col.int + ColOffset)
+
 proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, code: string] =
   ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
@@ -481,6 +485,7 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
   let useRenderModule = false
   let loc = d.conf.toFileLineCol(n.info)
   let code = extractRunnableExamplesSource(d.conf, n)
+  let codeIndent = extractRunnableExamplesSource(d.conf, n, indent = 2)
 
   if d.conf.errorCounter > 0:
     return (rdoccmd, code)
@@ -489,7 +494,7 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
   let outputDir = d.exampleOutputDir
   createDir(outputDir)
   inc d.exampleCounter
-  let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" & ("_examples$1.nim" % $d.exampleCounter)))
+  let outp = outputDir / RelativeFile("$#_examples_$#.nim" % [d.filename.extractFilename.changeFileExt"", $d.exampleCounter])
 
   if useRenderModule:
     var docComment = newTree(nkCommentStmt)
@@ -507,13 +512,21 @@ proc prepareExample(d: PDoc; n: PNode, topLevel: bool): tuple[rdoccmd: string, c
     renderModule(runnableExamples, outp.string, conf = d.conf)
 
   else:
-    let code2 = """
+    var code2 = code
+    if code.len > 0 and "codeReordering" notin code:
+      # hacky but simplest solution, until we devise a way to make `{.line.}`
+      # work without introducing a scope
+      code2 = """
+{.line: $#.}:
+$#
+""" % [$toInstantiationInfo(d.conf, n.info), codeIndent]
+    code2 = """
 #[
-$1
+$#
 ]#
-import $2
-$3
-""" % [comment, d.filename.quoted, code]
+import $#
+$#
+""" % [comment, d.filename.quoted, code2]
     writeFile(outp.string, code2)
 
   if rdoccmd notin d.exampleGroups:
diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim
index a20c8873d..fc74f2322 100644
--- a/compiler/renderverbatim.nim
+++ b/compiler/renderverbatim.nim
@@ -83,7 +83,7 @@ proc startOfLineInsideTriple(ldata: LineData, line: int): bool =
   if index >= ldata.lines.len: false
   else: ldata.lines[index]
 
-proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode, indent = 0): string =
   ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
   ## we'd need to check performance impact to enable it for nimdoc.
   var first = n.lastSon.info
@@ -102,7 +102,7 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
 
   let last = n.lastNodeRec.info
   var info = first
-  var indent = info.col
+  var indent2 = info.col
   let numLines = numLines(conf, info.fileIndex).uint16
   var lastNonemptyPos = 0
 
@@ -118,14 +118,15 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
     info.line = line
     let src = sourceLine(conf, info)
     let special = startOfLineInsideTriple(ldata, line.int)
-    if line > last.line and not special and not isInIndentationBlock(src, indent):
+    if line > last.line and not special and not isInIndentationBlock(src, indent2):
       break
     if line > first.line: result.add "\n"
     if special:
       result.add src
       lastNonemptyPos = result.len
-    elif src.len > indent:
-      result.add src[indent..^1]
+    elif src.len > indent2:
+      for i in 0..
Date: Fri, 9 Jul 2021 15:15:49 +0200
Subject: [PATCH 0210/1654] ORC: support for custom =trace procs (#18459)

* ORC: support custom =trace procs (WIP)
* Update tests/arc/tcustomtrace.nim

Co-authored-by: Clyybber 

* =trace is now documented and seems to work
* make test green

Co-authored-by: Clyybber 
---
 changelog.md                 |   2 +
 compiler/ast.nim             |   2 +-
 compiler/ccgexprs.nim        |   1 +
 compiler/condsyms.nim        |   1 +
 compiler/jsgen.nim           |   2 +-
 compiler/liftdestructors.nim |  16 ++-
 compiler/semmagic.nim        |   6 +
 compiler/semstmts.nim        |  13 +-
 compiler/vmgen.nim           |   2 +-
 doc/destructors.rst          |  37 ++++++
 doc/manual.rst               |   5 +-
 lib/system.nim               |   5 +
 tests/arc/tcustomtrace.nim   | 240 +++++++++++++++++++++++++++++++++++
 13 files changed, 321 insertions(+), 11 deletions(-)
 create mode 100644 tests/arc/tcustomtrace.nim

diff --git a/changelog.md b/changelog.md
index 849640e06..bc64e9579 100644
--- a/changelog.md
+++ b/changelog.md
@@ -388,6 +388,8 @@
 
 - `typeof(voidStmt)` now works and returns `void`.
 
+- The `gc:orc` algorithm was refined so that custom container types can participate in the
+  cycle collection process.
 
 
 ## Compiler changes
diff --git a/compiler/ast.nim b/compiler/ast.nim
index 26050426a..f96e464af 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -667,7 +667,7 @@ type
     mIsPartOf, mAstToStr, mParallel,
     mSwap, mIsNil, mArrToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
-    mMove, mWasMoved, mDestroy,
+    mMove, mWasMoved, mDestroy, mTrace,
     mDefault, mUnown, mIsolate, mAccessEnv, mReset,
     mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index c07ddf1db..09326ceeb 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -2422,6 +2422,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mDestroy: genDestroy(p, e)
   of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0")
   of mSlice: genSlice(p, e, d)
+  of mTrace: discard "no code to generate"
   else:
     when defined(debugMagics):
       echo p.prc.name.s, " ", p.prc.id, " ", p.prc.flags, " ", p.prc.ast[genericParamsPos].kind
diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim
index b3f8465f0..33cd1818b 100644
--- a/compiler/condsyms.nim
+++ b/compiler/condsyms.nim
@@ -136,3 +136,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasTypeofVoid")
   defineSymbol("nimHasDragonBox")
   defineSymbol("nimHasHintAll")
+  defineSymbol("nimHasTrace")
diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim
index a9339c7fd..82fba02a6 100644
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -2079,7 +2079,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
       gen(p, n[1], x)
       useMagic(p, "nimCopy")
       r.res = "nimCopy(null, $1, $2)" % [x.rdLoc, genTypeInfo(p, n.typ)]
-  of mDestroy: discard "ignore calls to the default destructor"
+  of mDestroy, mTrace: discard "ignore calls to the default destructor"
   of mOrd: genOrd(p, n, r)
   of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray:
     var x: TCompRes
diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim
index 1824557e6..b2b70edee 100644
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -416,11 +416,23 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
-  of attachedAsgn, attachedSink, attachedTrace:
+  of attachedAsgn, attachedSink:
     var op = getAttachedOp(c.g, t, c.kind)
     result = considerAsgnOrSink(c, t, body, x, y, op)
     if op != nil:
       setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+  of attachedTrace:
+    var op = getAttachedOp(c.g, t, c.kind)
+    if op != nil and sfOverriden in op.flags:
+      if op.ast.isGenericRoutine:
+        # patch generic =trace:
+        op = instantiateGeneric(c, op, t, t.typeInst)
+        setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
+    result = considerAsgnOrSink(c, t, body, x, y, op)
+    if op != nil:
+      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
+
   of attachedDeepCopy:
     let op = getAttachedOp(c.g, t, attachedDeepCopy)
     if op != nil:
@@ -1065,7 +1077,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   # 4. We have a custom destructor.
   # 5. We have a (custom) generic destructor.
 
-  # we do not generate '=trace' nor '=dispose' procs if we
+  # we do not generate '=trace' procs if we
   # have the cycle detection disabled, saves code size.
   let lastAttached = if g.config.selectedGC == gcOrc: attachedTrace
                      else: attachedSink
diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim
index d3f26e630..04f4c4729 100644
--- a/compiler/semmagic.nim
+++ b/compiler/semmagic.nim
@@ -551,6 +551,12 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     let op = getAttachedOp(c.graph, t, attachedDestructor)
     if op != nil:
       result[0] = newSymNode(op)
+  of mTrace:
+    result = n
+    let t = n[1].typ.skipTypes(abstractVar)
+    let op = getAttachedOp(c.graph, t, attachedTrace)
+    if op != nil:
+      result[0] = newSymNode(op)
   of mUnown:
     result = semUnown(c, n)
   of mExists, mForall:
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index c3402aee3..fd2f8a1d9 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1649,6 +1649,8 @@ proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) =
   var noError = false
   let cond = if op == attachedDestructor:
                t.len == 2 and t[0] == nil and t[1].kind == tyVar
+             elif op == attachedTrace:
+               t.len == 3 and t[0] == nil and t[1].kind == tyVar and t[2].kind == tyPointer
              else:
                t.len >= 2 and t[0] == nil
 
@@ -1673,8 +1675,12 @@ proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) =
         localError(c.config, n.info, errGenerated,
           "type bound operation `" & s.name.s & "` can be defined only in the same module with its type (" & obj.typeToString() & ")")
   if not noError and sfSystemModule notin s.owner.flags:
-    localError(c.config, n.info, errGenerated,
-      "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
+    if op == attachedTrace:
+      localError(c.config, n.info, errGenerated,
+        "signature for '=trace' must be proc[T: object](x: var T; env: pointer)")
+    else:
+      localError(c.config, n.info, errGenerated,
+        "signature for '" & s.name.s & "' must be proc[T: object](x: var T)")
   incl(s.flags, sfUsed)
   incl(s.flags, sfOverriden)
 
@@ -1752,7 +1758,8 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
       localError(c.config, n.info, errGenerated,
                 "signature for '" & s.name.s & "' must be proc[T: object](x: var T; y: T)")
   of "=trace":
-    bindTypeHook(c, s, n, attachedTrace)
+    if s.magic != mTrace:
+      bindTypeHook(c, s, n, attachedTrace)
   else:
     if sfOverriden in s.flags:
       localError(c.config, n.info, errGenerated,
diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index 14f9e0b30..e5e4a854e 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -1362,7 +1362,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     globalError(c.config, n.info, sizeOfLikeMsg("offsetof"))
   of mRunnableExamples:
     discard "just ignore any call to runnableExamples"
-  of mDestroy: discard "ignore calls to the default destructor"
+  of mDestroy, mTrace: discard "ignore calls to the default destructor"
   of mMove:
     let arg = n[1]
     let a = c.genx(arg)
diff --git a/doc/destructors.rst b/doc/destructors.rst
index c99606b95..a94ccb9e7 100644
--- a/doc/destructors.rst
+++ b/doc/destructors.rst
@@ -42,6 +42,12 @@ written as:
       for i in 0..`_.
diff --git a/lib/system.nim b/lib/system.nim
index 090ab309e..8aa9bd718 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -488,6 +488,11 @@ proc `=sink`*[T](x: var T; y: T) {.inline, magic: "Asgn".} =
   ## Generic `sink`:idx: implementation that can be overridden.
   shallowCopy(x, y)
 
+when defined(nimHasTrace):
+  proc `=trace`*[T](x: var T; env: pointer) {.inline, magic: "Trace".} =
+    ## Generic `trace`:idx: implementation that can be overridden.
+    discard
+
 type
   HSlice*[T, U] = object   ## "Heterogeneous" slice type.
     a*: T                  ## The lower bound (inclusive).
diff --git a/tests/arc/tcustomtrace.nim b/tests/arc/tcustomtrace.nim
new file mode 100644
index 000000000..3977194d9
--- /dev/null
+++ b/tests/arc/tcustomtrace.nim
@@ -0,0 +1,240 @@
+discard """
+  outputsub: '''1
+2
+3
+4
+5
+6
+89
+90
+90
+0 0 1
+0 1 2
+0 2 3
+1 0 4
+1 1 5
+1 2 6
+1 3 7
+after 6 6
+MEM 0'''
+joinable: false
+  cmd: "nim c --gc:orc -d:useMalloc $file"
+  valgrind: "true"
+"""
+
+import typetraits
+
+type
+  myseq*[T] = object
+    len, cap: int
+    data: ptr UncheckedArray[T]
+
+# XXX make code memory safe for overflows in '*'
+var
+  allocCount, deallocCount: int
+
+proc `=destroy`*[T](x: var myseq[T]) =
+  if x.data != nil:
+    when not supportsCopyMem(T):
+      for i in 0..= x.cap: resize(x)
+  result = addr(x.data[x.len])
+  inc x.len
+
+template add*[T](x: var myseq[T]; y: T) =
+  reserveSlot(x)[] = y
+
+proc shrink*[T](x: var myseq[T]; newLen: int) =
+  assert newLen <= x.len
+  assert newLen >= 0
+  when not supportsCopyMem(T):
+    for i in countdown(x.len - 1, newLen - 1):
+      `=destroy`(x.data[i])
+  x.len = newLen
+
+proc grow*[T](x: var myseq[T]; newLen: int; value: T) =
+  if newLen <= x.len: return
+  assert newLen >= 0
+  let oldCap = x.cap
+  if oldCap == 0: x.cap = newLen
+  else: x.cap = max(newLen, (oldCap * 3) shr 1)
+  if x.data == nil: inc allocCount
+  x.data = cast[type(x.data)](realloc0(x.data, oldCap * sizeof(T), x.cap * sizeof(T)))
+  for i in x.len..
Date: Sat, 10 Jul 2021 09:41:07 +0300
Subject: [PATCH 0211/1654] sync with the same template from locks module
 (#18414)

---
 lib/core/rlocks.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim
index a1ee0a6ec..0444b9a83 100644
--- a/lib/core/rlocks.nim
+++ b/lib/core/rlocks.nim
@@ -48,7 +48,7 @@ proc release*(lock: var RLock) {.inline.} =
   ## Releases the given lock.
   releaseSys(lock)
 
-template withRLock*(lock: var RLock, code: untyped): untyped =
+template withRLock*(lock: RLock, code: untyped) =
   ## Acquires the given lock and then executes the code.
   acquire(lock)
   {.locks: [lock].}:

From 3645f5fc7ae885e22cb52b9f865edf062cfb8e59 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Sat, 10 Jul 2021 15:55:01 +0800
Subject: [PATCH 0212/1654] more important packages (#18472)

---
 testament/important_packages.nim | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index 3d3c340eb..afceffb2e 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -37,6 +37,7 @@ pkg "alea", allowFailure = true
 pkg "argparse"
 pkg "arraymancer", "nim c tests/tests_cpu.nim", allowFailure = true
 pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true
+pkg "asyncthreadpool"
 pkg "awk"
 pkg "bigints", url = "https://github.com/Araq/nim-bigints"
 pkg "binaryheap", "nim c -r binaryheap.nim"
@@ -56,6 +57,7 @@ pkg "combparser", "nimble test --gc:orc"
 pkg "compactdict"
 pkg "comprehension", "nimble test", "https://github.com/alehander42/comprehension"
 pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed)
+pkg "datamancer"
 pkg "dashing", "nim c tests/functional.nim"
 pkg "delaunay"
 pkg "docopt"
@@ -81,6 +83,7 @@ pkg "jstin"
 pkg "karax", "nim c -r tests/tester.nim"
 pkg "kdtree", "nimble test -d:nimLegacyRandomInitRand", "https://github.com/jblindsay/kdtree"
 pkg "loopfusion"
+pkg "lockfreequeues"
 pkg "macroutils"
 pkg "manu"
 pkg "markdown"
@@ -89,6 +92,7 @@ pkg "msgpack4nim", "nim c -r tests/test_spec.nim"
 pkg "nake", "nim c nakefile.nim"
 pkg "neo", "nim c -d:blas=openblas tests/all.nim"
 pkg "nesm", "nimble tests", allowFailure = true # notice plural 'tests'
+pkg "netty"
 pkg "nico", allowFailure = true
 pkg "nicy", "nim c -r src/nicy.nim"
 pkg "nigui", "nim c -o:niguii -r src/nigui.nim"
@@ -134,10 +138,12 @@ pkg "rosencrantz", "nim c -o:rsncntz -r rosencrantz.nim"
 pkg "sdl1", "nim c -r src/sdl.nim"
 pkg "sdl2_nim", "nim c -r sdl2/sdl.nim"
 pkg "sigv4", "nim c --gc:arc -r sigv4.nim", "https://github.com/disruptek/sigv4"
+pkg "sim"
 pkg "snip", "nimble test", "https://github.com/genotrance/snip"
 pkg "stint", "nim r stint.nim"
 pkg "strslice"
 pkg "strunicode", "nim c -r src/strunicode.nim"
+pkg "supersnappy"
 pkg "synthesis"
 pkg "telebot", "nim c -o:tbot -r src/telebot.nim", allowFailure = true # pending https://github.com/ba0f3/telebot.nim/issues/59
 pkg "tempdir"

From 980a9ed5239ef03e73946ffcc84671a6eaa35616 Mon Sep 17 00:00:00 2001
From: sivchari 
Date: Sun, 11 Jul 2021 15:19:20 +0900
Subject: [PATCH 0213/1654] fix comment (#18473)

---
 lib/impure/re.nim | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 504d8f22e..d6922f4f1 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -41,12 +41,12 @@ const
 
 type
   RegexFlag* = enum     ## options for regular expressions
-    reIgnoreCase = 0,    ## do caseless matching
-    reMultiLine = 1,     ## `^` and `$` match newlines within data
-    reDotAll = 2,        ## `.` matches anything including NL
-    reExtended = 3,      ## ignore whitespace and `#` comments
-    reStudy = 4          ## study the expression (may be omitted if the
-                         ## expression will be used only once)
+    reIgnoreCase = 0,   ## do caseless matching
+    reMultiLine = 1,    ## `^` and `$` match newlines within data
+    reDotAll = 2,       ## `.` matches anything including NL
+    reExtended = 3,     ## ignore whitespace and `#` comments
+    reStudy = 4         ## study the expression (may be omitted if the
+                        ## expression will be used only once)
 
   RegexDesc = object
     h: ptr Pcre

From 9ffc70851b586dbdc579d8c6af30a6a3604d7313 Mon Sep 17 00:00:00 2001
From: GordonBGood 
Date: Mon, 12 Jul 2021 16:55:40 +0700
Subject: [PATCH 0214/1654] thamming_orc test created/destroyed counts match
 (#18471)

The thamming_orc.nim code now counts all created objects being tested, not just the ones following the "first 20" test, and the position of the `destroyed += 1` counter has been adjusted so it counts all the calls that are as a result of `=trace` tracing and not just the original destruction calls.
---
 tests/arc/thamming_orc.nim | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/arc/thamming_orc.nim b/tests/arc/thamming_orc.nim
index 463f7117e..777efb38e 100644
--- a/tests/arc/thamming_orc.nim
+++ b/tests/arc/thamming_orc.nim
@@ -1,6 +1,6 @@
 discard """
   output: '''(allocCount: 1114, deallocCount: 1112)
-created 439 destroyed 402'''
+created 491 destroyed 491'''
   cmd: "nim c --gc:orc -d:nimAllocStats $file"
 """
 
@@ -94,8 +94,8 @@ type
 var destroyed = 0
 
 proc `=destroy`(ll: var LazyListObj) =
-  if ll.tlf == nil and ll.tl == nil: return
   destroyed += 1
+  if ll.tlf == nil and ll.tl == nil: return
 
   when defined(trace20):
     echo "destroying:  ", (destroyed, ll.hd[1].convertTriVal2BigInt)
@@ -140,7 +140,7 @@ proc main =
            "Algorithmic error finding first 20 Hamming numbers!!!"
 
   when not defined(trace20):
-    var lsth: TriVal; created = 0; destroyed = 0
+    var lsth: TriVal
     for h in hammings(200): lsth = h
     doAssert $lsth.convertTriVal2BigInt == "16200",
              "Algorithmic error finding 200th Hamming number!!!"
@@ -149,7 +149,7 @@ let mem = getOccupiedMem()
 main()
 GC_FullCollect()
 let mb = getOccupiedMem() - mem
-#doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!"
+doAssert mb == 0, "Found memory leak of " & $mb & " bytes!!!"
 
 echo getAllocStats()
 echo "created ", created, " destroyed ", destroyed

From 195300a938c7c8dd4ec7db3ca7cc76dddeab953e Mon Sep 17 00:00:00 2001
From: flywind 
Date: Mon, 12 Jul 2021 20:26:46 +0800
Subject: [PATCH 0215/1654] [source code filter]fix Nightlies bug (#18475)

* workaround Nightlies bug

* Update tools/niminst/makefile.nimf
---
 tools/niminst/makefile.nimf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf
index 8c40cf50f..d14960ac5 100644
--- a/tools/niminst/makefile.nimf
+++ b/tools/niminst/makefile.nimf
@@ -25,7 +25,7 @@ endif
 
 ifeq ($(uos),linux)
   myos = linux
-  # add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17
+  ## add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17
   LDFLAGS += -ldl -lm -lrt
 endif
 ifeq ($(uos),dragonfly)

From 1b128ac04533421547298ecca0d9b8e7de477280 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 13 Jul 2021 06:18:07 +0800
Subject: [PATCH 0216/1654] close #17986  add testcase (#18477)

---
 tests/stdlib/tjsonutils.nim | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 31ec4316f..b0c444cbf 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -311,6 +311,18 @@ template fn() =
       doAssert guide == AboutLifeUniverseAndEverythingElse(
         question: "6*9=?", answer: 42)
 
+      block refObject: #bug 17986
+        type A = ref object
+          case is_a: bool
+          of true:
+            a: int
+          else:
+            b: int
+
+        var a = A()
+        fromJson(a, """{"is_a": true, "a":1, "extra_key": 1}""".parse_json, Joptions(allowExtraKeys: true))
+        doAssert $a[] == "(is_a: true, a: 1)"
+
     block testAllowMissingKeys:
       var guide = AboutLifeUniverseAndEverythingElse(
         question: "6*9=?", answer: 54)

From 15fdcd0c4d4d5b44a08766120d88b085eb358ae1 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 13 Jul 2021 15:22:33 +0800
Subject: [PATCH 0217/1654] workaround #18481 (#18482)

---
 lib/pure/encodings.nim | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim
index dcdc474ae..36e891dd3 100644
--- a/lib/pure/encodings.nim
+++ b/lib/pure/encodings.nim
@@ -12,29 +12,31 @@
 ##
 ## The following example shows how to change character encodings.
 runnableExamples:
-  let
-    orig = "öäüß"
-    # convert `orig` from "UTF-8" to "CP1252"
-    cp1252 = convert(orig, "CP1252", "UTF-8")
-    # convert `cp1252` from "CP1252" to "ibm850"
-    ibm850 = convert(cp1252, "ibm850", "CP1252")
-    current = getCurrentEncoding()
-  assert orig == "\195\182\195\164\195\188\195\159"
-  assert ibm850 == "\148\132\129\225"
-  assert convert(ibm850, current, "ibm850") == orig
+  when defined(windows):
+    let
+      orig = "öäüß"
+      # convert `orig` from "UTF-8" to "CP1252"
+      cp1252 = convert(orig, "CP1252", "UTF-8")
+      # convert `cp1252` from "CP1252" to "ibm850"
+      ibm850 = convert(cp1252, "ibm850", "CP1252")
+      current = getCurrentEncoding()
+    assert orig == "\195\182\195\164\195\188\195\159"
+    assert ibm850 == "\148\132\129\225"
+    assert convert(ibm850, current, "ibm850") == orig
 
 ## The example below uses a reuseable `EncodingConverter` object which is
 ## created by `open` with `destEncoding` and `srcEncoding` specified. You can use
 ## `convert` on this object multiple times.
 runnableExamples:
-  var fromGB2312 = open("utf-8", "gb2312")
-  let first = "\203\173\197\194\163\191\210\187" &
-      "\203\242\209\204\211\234\200\206\198\189\201\250"
-  assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生"
+  when defined(windows):
+    var fromGB2312 = open("utf-8", "gb2312")
+    let first = "\203\173\197\194\163\191\210\187" &
+        "\203\242\209\204\211\234\200\206\198\189\201\250"
+    assert fromGB2312.convert(first) == "谁怕？一蓑烟雨任平生"
 
-  let second = "\211\208\176\215\205\183\200\231" &
-      "\208\194\163\172\199\227\184\199\200\231\185\202"
-  assert fromGB2312.convert(second) == "有白头如新，倾盖如故"
+    let second = "\211\208\176\215\205\183\200\231" &
+        "\208\194\163\172\199\227\184\199\200\231\185\202"
+    assert fromGB2312.convert(second) == "有白头如新，倾盖如故"
 
 
 import os

From b3aca78e228feb93d5dcd6f00cdd81e4178c24cc Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 13 Jul 2021 12:44:29 +0200
Subject: [PATCH 0218/1654] closes #18433 (#18484)

* beneficial refactoring; use system.Endianness

* closes #18433
---
 compiler/options.nim   |  4 ++--
 compiler/platform.nim  |  6 ++----
 tests/arc/tfuncobj.nim | 38 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 6 deletions(-)
 create mode 100644 tests/arc/tfuncobj.nim

diff --git a/compiler/options.nim b/compiler/options.nim
index 135ecf1de..45ca14e76 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -592,8 +592,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
       result = conf.target.targetOS == osNintendoSwitch
     of "freertos", "lwip":
       result = conf.target.targetOS == osFreeRTOS
-    of "littleendian": result = CPU[conf.target.targetCPU].endian == platform.littleEndian
-    of "bigendian": result = CPU[conf.target.targetCPU].endian == platform.bigEndian
+    of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian
+    of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian
     of "cpu8": result = CPU[conf.target.targetCPU].bit == 8
     of "cpu16": result = CPU[conf.target.targetCPU].bit == 16
     of "cpu32": result = CPU[conf.target.targetCPU].bit == 32
diff --git a/compiler/platform.nim b/compiler/platform.nim
index 413a91304..7bc77d809 100644
--- a/compiler/platform.nim
+++ b/compiler/platform.nim
@@ -196,13 +196,11 @@ type
     cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32
 
 type
-  TEndian* = enum
-    littleEndian, bigEndian
-  TInfoCPU* = tuple[name: string, intSize: int, endian: TEndian,
+  TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness,
                     floatSize, bit: int]
 
 const
-  EndianToStr*: array[TEndian, string] = ["littleEndian", "bigEndian"]
+  EndianToStr*: array[Endianness, string] = ["littleEndian", "bigEndian"]
   CPU*: array[succ(low(TSystemCPU))..high(TSystemCPU), TInfoCPU] = [
     (name: "i386", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "m68k", intSize: 32, endian: bigEndian, floatSize: 64, bit: 32),
diff --git a/tests/arc/tfuncobj.nim b/tests/arc/tfuncobj.nim
new file mode 100644
index 000000000..50cd9425e
--- /dev/null
+++ b/tests/arc/tfuncobj.nim
@@ -0,0 +1,38 @@
+discard """
+  outputsub: '''f1
+f2
+f3'''
+  cmd: "nim c --gc:orc $file"
+  valgrind: true
+"""
+
+type
+  FuncObj = object
+    fn: proc (env: pointer) {.cdecl.}
+    env: pointer
+
+proc `=destroy`(x: var FuncObj) =
+  GC_unref(cast[RootRef](x.env))
+
+proc `=copy`(x: var FuncObj, y: FuncObj) {.error.}
+
+# bug #18433
+
+proc main =
+  var fs: seq[FuncObj]
+
+  proc wrap(p: proc()) =
+    proc closeOver() = p()
+    let env = rawEnv closeOver
+    GC_ref(cast[RootRef](env))
+    fs.add(FuncObj(fn: cast[proc(env: pointer){.cdecl.}](rawProc closeOver), env: env))
+
+  wrap(proc() {.closure.} = echo "f1")
+  wrap(proc() {.closure.} = echo "f2")
+  wrap(proc() {.closure.} = echo "f3")
+
+  for a in fs:
+    a.fn(a.env)
+
+main()
+

From 12da32a891b08f34acbb676001f0c211ecfe1af8 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 13 Jul 2021 14:17:59 +0200
Subject: [PATCH 0219/1654] fixes #17893 (#18485)

* fixes #17893
---
 compiler/liftdestructors.nim    |  7 +---
 tests/arc/tshared_ptr_crash.nim | 67 +++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 6 deletions(-)
 create mode 100644 tests/arc/tshared_ptr_crash.nim

diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim
index b2b70edee..cc7e36cdb 100644
--- a/compiler/liftdestructors.nim
+++ b/compiler/liftdestructors.nim
@@ -416,12 +416,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
       body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
-  of attachedAsgn, attachedSink:
-    var op = getAttachedOp(c.g, t, c.kind)
-    result = considerAsgnOrSink(c, t, body, x, y, op)
-    if op != nil:
-      setAttachedOp(c.g, c.idgen.module, t, c.kind, op)
-  of attachedTrace:
+  of attachedAsgn, attachedSink, attachedTrace:
     var op = getAttachedOp(c.g, t, c.kind)
     if op != nil and sfOverriden in op.flags:
       if op.ast.isGenericRoutine:
diff --git a/tests/arc/tshared_ptr_crash.nim b/tests/arc/tshared_ptr_crash.nim
new file mode 100644
index 000000000..1794834db
--- /dev/null
+++ b/tests/arc/tshared_ptr_crash.nim
@@ -0,0 +1,67 @@
+discard """
+  cmd: "nim c --threads:on --gc:arc $file"
+  action: compile
+"""
+
+# bug #17893
+
+type
+  SharedPtr*[T] = object
+    val: ptr tuple[value: T, atomicCounter: int]
+
+proc `=destroy`*[T](p: var SharedPtr[T]) =
+  mixin `=destroy`
+  if p.val != nil:
+    if atomicLoadN(addr p.val[].atomicCounter, AtomicConsume) == 0:
+      `=destroy`(p.val[])
+      deallocShared(p.val)
+    else:
+      discard atomicDec(p.val[].atomicCounter)
+
+proc `=copy`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) =
+  if src.val != nil:
+    discard atomicInc(src.val[].atomicCounter)
+  if dest.val != nil:
+    `=destroy`(dest)
+  dest.val = src.val
+
+proc newSharedPtr*[T](val: sink T): SharedPtr[T] {.nodestroy.} =
+  result.val = cast[typeof(result.val)](allocShared(sizeof(result.val[])))
+  result.val.atomicCounter = 0
+  result.val.value = val
+
+proc isNil*[T](p: SharedPtr[T]): bool {.inline.} =
+  p.val == nil
+
+proc `[]`*[T](p: SharedPtr[T]): var T {.inline.} =
+  when compileOption("boundChecks"):
+    doAssert(p.val != nil, "deferencing nil shared pointer")
+  result = p.val.value
+
+type
+  Sender*[T] = object
+    queue: SharedPtr[seq[T]]
+
+proc newSender*[T](queue: sink SharedPtr[seq[T]]): Sender[T] =
+  result = Sender[T](queue: queue)
+
+proc send*[T](self: Sender[T]; t: sink T) =
+  self.queue[].add t
+
+proc newChannel*(): Sender[int] =
+  let queue = newSharedPtr(newSeq[int]())
+  result = newSender(queue)
+
+
+var
+  p: Thread[Sender[int]]
+
+proc threadFn(tx: Sender[int]) =
+  send tx, 0
+
+proc multiThreadedChannel =
+  let tx = newChannel()
+  createThread(p, threadFn, tx)
+  joinThread(p)
+
+multiThreadedChannel()

From a6ff6e7871d757821ba2b84cb8f175f48721bb9a Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 14 Jul 2021 10:52:09 -0700
Subject: [PATCH 0220/1654] fix https://github.com/dom96/choosenim/issues/256
 WSL CRLF (#18452)

* fix https://github.com/dom96/choosenim/issues/256 WSL CRLF

* fixup
---
 .gitattributes | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitattributes b/.gitattributes
index 674da7939..2c9ff103c 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -4,3 +4,6 @@
 # duplicated, which is easily identifiable and fixable.
 /changelog.md merge=union
 
+# bug https://github.com/dom96/choosenim/issues/256 for WSL CRLF
+*.sh text eol=lf
+/config/build_config.txt text eol=lf

From 8c6dd2b9a9157c0e0bd822d8a80d1014fc87ecd7 Mon Sep 17 00:00:00 2001
From: GordonBGood 
Date: Thu, 15 Jul 2021 19:35:53 +0700
Subject: [PATCH 0221/1654] Improve description of how =trace is used (#18491)

* [skip ci] thamming_orc test created/destroyed counts match

The thamming_orc.nim code now counts all created objects being tested, not just the ones following the "first 20" test, and the position of the `destroyed += 1` counter has been adjusted so it counts all the calls that are as a result of `=trace` tracing and not just the original destruction calls.

* Improve description of how `=trace` is used

The following nuances weren't previously fully explained:

1. That `=trace` is only used by `--gc:orc`.
2. That `=trace` is almost certainly used along with `=destroy` when manual resource allocation has been used, but it is only required if there is a possibility of cyclic references in the wrapped types (ie. generic types).
3. That, currently, a forward definition is required for the second of the pair to avoid an auto compiler generation conflict.

The pattern of the use of `=trace` has also been made more extensive, showing how both a custom `=destroy` and `=trace` are used for manual allocation of resources when there is any possibility of a cyclic reference in the resource-wrapped values.

* Update doc/destructors.rst

Co-authored-by: Andreas Rumpf 
---
 doc/destructors.rst | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/doc/destructors.rst b/doc/destructors.rst
index a94ccb9e7..a3fd7514e 100644
--- a/doc/destructors.rst
+++ b/doc/destructors.rst
@@ -222,16 +222,33 @@ The prototype of this hook for a type `T` needs to be:
 `env` is used by ORC to keep track of its internal state, it should be passed around
 to calls of the built-in `=trace` operation.
 
-The general pattern in `=trace` looks like:
+Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates manually allocated resources is also used, and then only when there is a chance of cyclic references from items within the manually allocated resources when it is desired that `--gc:orc` be able to break and collect these cyclic referenced resources.  Currently however, there is a mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically create a version of the other which will then conflict with the creation of the second of the pair.  The work around for this problem is to forward declare the second of the "hooks" to prevent the automatic creation.
+
+The general pattern in using `=destroy` with `=trace` looks like:
 
 .. code-block:: nim
 
-  proc `=trace`(dest: var T; env: pointer) =
-    for child in childrenThatCanContainPointers(dest):
-      `=trace`(child, env)
+  type
+    Test[T] = object
+      size: Natural
+      arr: ptr UncheckedArray[T] # raw pointer field
+
+  proc makeTest[T](size: Natural): Test[T] = # custom allocation...
+    Test[T](size: size, arr: cast[ptr UncheckedArray[T]](alloc0(sizeof(T) * size)))
 
 
-**Note**: The `=trace` hooks is currently more experimental and less refined
+  proc `=destroy`[T](dest: var Test[T]) =
+    if dest.arr != nil:
+      for i in 0 ..< dest.size: dest.arr[i].`=destroy`
+      dest.arr.dealloc
+
+  proc `=trace`[T](dest: var Test[T]; env: pointer) =
+    if dest.arr != nil: # trace the `T`'s which may be cyclic
+      for i in 0 ..< dest.size: dest.arr[i].`=trace` env
+
+  # following may be other custom "hooks" as required...
+
+**Note**: The `=trace` hooks (which are only used by `--gc:orc`) are currently more experimental and less refined
 than the other hooks.
 
 

From af3d2d8ad90c31161c34f0d4f82cd0ac4a9b3d00 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 15 Jul 2021 17:58:47 +0200
Subject: [PATCH 0222/1654] added `nimAllocPagesViaMalloc` switch (#18490)

* added  switch

* alloc.nim needs page aligned memory blocks
---
 changelog.md           |  3 +++
 doc/nimc.rst           | 14 +++++++++++++-
 lib/system/alloc.nim   |  3 ++-
 lib/system/osalloc.nim | 31 +++++++++++++++++++++++++++----
 tests/arc/tarcmisc.nim |  2 +-
 5 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index bc64e9579..d44f28b94 100644
--- a/changelog.md
+++ b/changelog.md
@@ -391,6 +391,9 @@
 - The `gc:orc` algorithm was refined so that custom container types can participate in the
   cycle collection process.
 
+- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`.
+  This is only supported for `--gc:orc` or `--gc:arc`.
+
 
 ## Compiler changes
 
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 2b9f33cb5..070956860 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -176,7 +176,7 @@ directories (in this order; later files overwrite previous settings):
    ``%APPDATA%/nim/nim.cfg`` (Windows).
    This file can be skipped with the `--skipUserCfg`:option: command line
    option.
-3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent 
+3) ``$parentDir/nim.cfg`` where ``$parentDir`` stands for any parent
    directory of the project file's path.
    These files can be skipped with the `--skipParentCfg`:option:
    command-line option.
@@ -634,6 +634,18 @@ optimization in the compiler and linker.
 Check the `Cross-compilation`_ section for instructions on how to compile the
 program for your target.
 
+
+nimAllocPagesViaMalloc
+----------------------
+
+Nim's default allocator is based on TLSF, this algorithm was designed for embedded
+devices. This allocator gets blocks/pages of memory via a currently undocumented
+`osalloc` API which usually uses POSIX's `mmap` call. On many environments `mmap`
+is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc`
+define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently
+only supported with `--gc:arc` or `--gc:orc`. (Since version 1.6)
+
+
 Nim for realtime systems
 ========================
 
diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim
index c598b8250..479458950 100644
--- a/lib/system/alloc.nim
+++ b/lib/system/alloc.nim
@@ -749,8 +749,9 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
     inc(a.allocCounter)
   sysAssert(allocInv(a), "rawAlloc: begin")
   sysAssert(roundup(65, 8) == 72, "rawAlloc: roundup broken")
-  sysAssert(requestedSize >= sizeof(FreeCell), "rawAlloc: requested size too small")
   var size = roundup(requestedSize, MemAlign)
+  sysAssert(size >= sizeof(FreeCell), "rawAlloc: requested size too small")
+
   sysAssert(size >= requestedSize, "insufficient allocated size!")
   #c_fprintf(stdout, "alloc; size: %ld; %ld\n", requestedSize, size)
   if size <= SmallChunkSize-smallChunkOverhead():
diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim
index 32d3b166d..2830adb48 100644
--- a/lib/system/osalloc.nim
+++ b/lib/system/osalloc.nim
@@ -28,7 +28,30 @@ const doNotUnmap = not (defined(amd64) or defined(i386)) or
                    defined(windows) or defined(nimAllocNoUnmap)
 
 
-when defined(emscripten) and not defined(StandaloneHeapSize):
+when defined(nimAllocPagesViaMalloc):
+  when not defined(gcArc) and not defined(gcOrc):
+    {.error: "-d:nimAllocPagesViaMalloc is only supported with --gc:arc or --gc:orc".}
+
+  proc osTryAllocPages(size: int): pointer {.inline.} =
+    let base = c_malloc(csize_t size + PageSize - 1 + sizeof(uint32))
+    if base == nil: raiseOutOfMem()
+    # memory layout: padding + offset (4 bytes) + user_data
+    # in order to deallocate: read offset at user_data - 4 bytes,
+    # then deallocate user_data - offset
+    let offset = PageSize - (cast[int](base) and (PageSize - 1))
+    cast[ptr uint32](base +! (offset - sizeof(uint32)))[] = uint32(offset)
+    result = base +! offset
+
+  proc osAllocPages(size: int): pointer {.inline.} =
+    result = osTryAllocPages(size)
+    if result == nil: raiseOutOfMem()
+
+  proc osDeallocPages(p: pointer, size: int) {.inline.} =
+    # read offset at p - 4 bytes, then deallocate (p - offset) pointer
+    let offset = cast[ptr uint32](p -! sizeof(uint32))[]
+    c_free(p -! offset)
+
+elif defined(emscripten) and not defined(StandaloneHeapSize):
   const
     PROT_READ  = 1             # page can be read
     PROT_WRITE = 2             # page can be written
@@ -197,11 +220,11 @@ elif defined(posix) and not defined(StandaloneHeapSize):
     PROT_WRITE = 2             # page can be written
 
   when defined(netbsd) or defined(openbsd):
-      # OpenBSD security for setjmp/longjmp coroutines
-      var MAP_STACK {.importc: "MAP_STACK", header: "".}: cint
+    # OpenBSD security for setjmp/longjmp coroutines
+    var MAP_STACK {.importc: "MAP_STACK", header: "".}: cint
   else:
     const MAP_STACK = 0             # avoid sideeffects
-    
+
   when defined(macosx) or defined(freebsd):
     const MAP_ANONYMOUS = 0x1000
     const MAP_PRIVATE = 0x02        # Changes are private
diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim
index 5d5d8e914..7daea62c8 100644
--- a/tests/arc/tarcmisc.nim
+++ b/tests/arc/tarcmisc.nim
@@ -34,7 +34,7 @@ closed
 destroying variable: 20
 destroying variable: 10
 '''
-  cmd: "nim c --gc:arc --deepcopy:on $file"
+  cmd: "nim c --gc:arc --deepcopy:on -d:nimAllocPagesViaMalloc $file"
 """
 
 proc takeSink(x: sink string): bool = true

From a9701f6531288ab3ba2c2a7ff0124802ae78122c Mon Sep 17 00:00:00 2001
From: quantimnot <54247259+quantimnot@users.noreply.github.com>
Date: Thu, 15 Jul 2021 14:43:57 -0400
Subject: [PATCH 0223/1654] Extended side effect error messages (#18418)

* Extended side effect error messages

* Applied feedback:
- refactored `markSideEffect`
- refactored string interpolations
- single message
- skip diagnostics in `system.compiles` context

Other:
- started a test of diagnostic messages

[ci skip] Tests aren't updated yet because messaging isn't nailed down.

* - Added hints of where for side effect call locations.
- Tried to clarify the reasons.

* fix tests

* Applied PR review feedback:
  - moved collection of side effects from TSym to TContext
  - used pragma shorthand form `.sideEffect` and `.noSideEffect` in messages
  - added leading '>' to structured messages for readability
  - changed `sempass2.markSideEffect` to a proc
  - replaced `system.echo` in the test to make the test compatible with Windows

* Applied NEP1 formatting suggestion

Co-authored-by: quantimnot 
---
 compiler/semdata.nim                   |  1 +
 compiler/sempass2.nim                  | 74 ++++++++++++++++++++------
 tests/effects/tdiagnostic_messages.nim | 37 +++++++++++++
 3 files changed, 95 insertions(+), 17 deletions(-)
 create mode 100644 tests/effects/tdiagnostic_messages.nim

diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index bd863505c..12bb22ff9 100644
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -159,6 +159,7 @@ type
     exportIndirections*: HashSet[(int, int)] # (module.id, symbol.id)
     importModuleMap*: Table[int, int] # (module.id, module.id)
     lastTLineInfo*: TLineInfo
+    sideEffects*: Table[int, seq[(TLineInfo, PSym)]] # symbol.id index
 
 template config*(c: PContext): ConfigRef = c.graph.config
 
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index e124baf26..1e1672cad 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -10,7 +10,7 @@
 import
   intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
   wordrecg, strutils, options, guards, lineinfos, semfold, semdata,
-  modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling
+  modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling, tables
 
 when defined(useDfa):
   import dfa
@@ -223,13 +223,21 @@ proc markGcUnsafe(a: PEffects; reason: PNode) =
         a.owner.gcUnsafetyReason = newSym(skUnknown, a.owner.name, nextSymId a.c.idgen,
                                           a.owner, reason.info, {})
 
-when true:
-  template markSideEffect(a: PEffects; reason: typed) =
-    if not a.inEnforcedNoSideEffects: a.hasSideEffect = true
-else:
-  template markSideEffect(a: PEffects; reason: typed) =
-    if not a.inEnforcedNoSideEffects: a.hasSideEffect = true
-    markGcUnsafe(a, reason)
+proc markSideEffect(a: PEffects; reason: PNode | PSym; useLoc: TLineInfo) =
+  if not a.inEnforcedNoSideEffects:
+    a.hasSideEffect = true
+    if a.owner.kind in routineKinds:
+      var sym: PSym
+      when reason is PNode:
+        if reason.kind == nkSym:
+          sym = reason.sym
+        else:
+          let kind = if reason.kind == nkHiddenDeref: skParam else: skUnknown
+          sym = newSym(kind, a.owner.name, nextSymId a.c.idgen, a.owner, reason.info, {})
+      else:
+        sym = reason
+      a.c.sideEffects.mgetOrPut(a.owner.id, @[]).add (useLoc, sym)
+    when false: markGcUnsafe(a, reason)
 
 proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet; conf: ConfigRef) =
   let u = s.gcUnsafetyReason
@@ -259,6 +267,31 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; conf: ConfigRef) =
   var cycleCheck = initIntSet()
   listGcUnsafety(s, onlyWarning, cycleCheck, conf)
 
+proc listSideEffects(result: var string; s: PSym; cycleCheck: var IntSet;
+                     conf: ConfigRef; context: PContext; indentLevel: int) =
+  template addHint(msg; lineInfo; sym; level = indentLevel) =
+    result.addf("$# $# Hint: '$#' $#\n", repeat(">", level), conf $ lineInfo, sym, msg)
+  if context.sideEffects.hasKey(s.id):
+    for (useLineInfo, u) in context.sideEffects[s.id]:
+      if u != nil and not cycleCheck.containsOrIncl(u.id):
+        case u.kind
+        of skLet, skVar:
+          addHint("accesses global state '$#'" % u.name.s, useLineInfo, s.name.s)
+          addHint("accessed by '$#'" % s.name.s, u.info, u.name.s, indentLevel + 1)
+        of routineKinds:
+          addHint("calls `.sideEffect` '$#'" % u.name.s, useLineInfo, s.name.s)
+          addHint("called by '$#'" % s.name.s, u.info, u.name.s, indentLevel + 1)
+          listSideEffects(result, u, cycleCheck, conf, context, indentLevel + 2)
+        of skParam, skForVar:
+          addHint("calls routine via hidden pointer indirection", useLineInfo, s.name.s)
+        else:
+          addHint("calls routine via pointer indirection", useLineInfo, s.name.s)
+
+proc listSideEffects(result: var string; s: PSym; conf: ConfigRef; context: PContext) =
+  var cycleCheck = initIntSet()
+  result.addf("'$#' can have side effects\n", s.name.s)
+  listSideEffects(result, s, cycleCheck, conf, context, 1)
+
 proc useVarNoInitCheck(a: PEffects; n: PNode; s: PSym) =
   if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet} and
       s.magic != mNimvm:
@@ -267,9 +300,7 @@ proc useVarNoInitCheck(a: PEffects; n: PNode; s: PSym) =
         (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem):
       #if a.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n)
       markGcUnsafe(a, s)
-      markSideEffect(a, s)
-    else:
-      markSideEffect(a, s)
+    markSideEffect(a, s, n.info)
   if s.owner != a.owner and s.kind in {skVar, skLet, skForVar, skResult, skParam} and
      {sfGlobal, sfThread} * s.flags == {}:
     a.isInnerProc = true
@@ -502,7 +533,7 @@ proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) =
     if tracked.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n, tracked.config)
     markGcUnsafe(tracked, s)
   if tfNoSideEffect notin s.typ.flags:
-    markSideEffect(tracked, s)
+    markSideEffect(tracked, s, n.info)
   mergeLockLevels(tracked, n, s.getLockLevel)
 
 proc procVarCheck(n: PNode; conf: ConfigRef) =
@@ -584,7 +615,7 @@ proc trackOperandForIndirectCall(tracked: PEffects, n: PNode, paramType: PType;
         if tracked.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n, tracked.config)
         markGcUnsafe(tracked, a)
       elif tfNoSideEffect notin op.flags and not isOwnedProcVar(a, tracked.owner):
-        markSideEffect(tracked, a)
+        markSideEffect(tracked, a, n.info)
     else:
       mergeRaises(tracked, effectList[exceptionEffects], n)
       mergeTags(tracked, effectList[tagEffects], n)
@@ -592,7 +623,7 @@ proc trackOperandForIndirectCall(tracked: PEffects, n: PNode, paramType: PType;
         if tracked.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n, tracked.config)
         markGcUnsafe(tracked, a)
       elif tfNoSideEffect notin op.flags:
-        markSideEffect(tracked, a)
+        markSideEffect(tracked, a, n.info)
   if paramType != nil and paramType.kind in {tyVar}:
     invalidateFacts(tracked.guards, n)
     if n.kind == nkSym and isLocalVar(tracked, n.sym):
@@ -749,7 +780,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
     if tfNoSideEffect notin op.flags and not importedFromC(a):
       # and it's not a recursive call:
       if not (a.kind == nkSym and a.sym == tracked.owner):
-        markSideEffect(tracked, a)
+        markSideEffect(tracked, a, n.info)
 
   # p's effects are ours too:
   var a = n[0]
@@ -771,7 +802,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
         if a.sym == tracked.owner: tracked.isRecursive = true
         # even for recursive calls we need to check the lock levels (!):
         mergeLockLevels(tracked, n, a.sym.getLockLevel)
-        if sfSideEffect in a.sym.flags: markSideEffect(tracked, a)
+        if sfSideEffect in a.sym.flags: markSideEffect(tracked, a, n.info)
       else:
         mergeLockLevels(tracked, n, op.lockLevel)
       var effectList = op.n[0]
@@ -1336,6 +1367,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     effects[ensuresEffects] = ensuresSpec
 
   var mutationInfo = MutationInfo()
+  var hasMutationSideEffect = false
   if {strictFuncs, views} * c.features != {}:
     var goals: set[Goal] = {}
     if strictFuncs in c.features: goals.incl constParameters
@@ -1343,6 +1375,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     var partitions = computeGraphPartitions(s, body, g, goals)
     if not t.hasSideEffect and t.hasDangerousAssign:
       t.hasSideEffect = varpartitions.hasSideEffect(partitions, mutationInfo)
+      hasMutationSideEffect = t.hasSideEffect
     if views in c.features:
       checkBorrowedLocations(partitions, body, g.config)
 
@@ -1357,7 +1390,14 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) =
     when false:
       listGcUnsafety(s, onlyWarning=false, g.config)
     else:
-      localError(g.config, s.info, ("'$1' can have side effects" % s.name.s) & (g.config $ mutationInfo))
+      if hasMutationSideEffect:
+        localError(g.config, s.info, "'$1' can have side effects$2" % [s.name.s, g.config $ mutationInfo])
+      elif c.compilesContextId == 0: # don't render extended diagnostic messages in `system.compiles` context
+        var msg = ""
+        listSideEffects(msg, s, g.config, t.c)
+        message(g.config, s.info, errGenerated, msg)
+      else:
+        localError(g.config, s.info, "") # simple error for `system.compiles` context
   if not t.gcUnsafe:
     s.typ.flags.incl tfGcSafe
   if not t.hasSideEffect and sfSideEffect notin s.flags:
diff --git a/tests/effects/tdiagnostic_messages.nim b/tests/effects/tdiagnostic_messages.nim
new file mode 100644
index 000000000..2ce4895a3
--- /dev/null
+++ b/tests/effects/tdiagnostic_messages.nim
@@ -0,0 +1,37 @@
+discard """
+  nimoutFull: true
+  action: "reject"
+  cmd: "nim r --hint:Conf:off $file"
+  nimout: '''
+tdiagnostic_messages.nim(36, 6) Error: 'a' can have side effects
+> tdiagnostic_messages.nim(37, 30) Hint: 'a' calls `.sideEffect` 'callWithSideEffects'
+>> tdiagnostic_messages.nim(29, 6) Hint: 'callWithSideEffects' called by 'a'
+>>> tdiagnostic_messages.nim(31, 34) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaVarParam'
+>>>> tdiagnostic_messages.nim(25, 6) Hint: 'indirectCallViaVarParam' called by 'callWithSideEffects'
+>>>>> tdiagnostic_messages.nim(26, 7) Hint: 'indirectCallViaVarParam' calls routine via hidden pointer indirection
+>>> tdiagnostic_messages.nim(32, 33) Hint: 'callWithSideEffects' calls `.sideEffect` 'indirectCallViaPointer'
+>>>> tdiagnostic_messages.nim(27, 6) Hint: 'indirectCallViaPointer' called by 'callWithSideEffects'
+>>>>> tdiagnostic_messages.nim(28, 32) Hint: 'indirectCallViaPointer' calls routine via pointer indirection
+>>> tdiagnostic_messages.nim(33, 10) Hint: 'callWithSideEffects' calls `.sideEffect` 'myEcho'
+>>>> tdiagnostic_messages.nim(24, 6) Hint: 'myEcho' called by 'callWithSideEffects'
+>>> tdiagnostic_messages.nim(34, 3) Hint: 'callWithSideEffects' accesses global state 'globalVar'
+>>>> tdiagnostic_messages.nim(23, 5) Hint: 'globalVar' accessed by 'callWithSideEffects'
+
+'''
+"""
+
+var globalVar = 0
+proc myEcho(a: string) {.sideEffect.} = discard
+proc indirectCallViaVarParam(call: var proc(): int {.nimcall.}): int =
+  call()
+proc indirectCallViaPointer(call: pointer): int =
+  cast[ptr proc(): int](call)[]()
+proc callWithSideEffects(): int =
+  var p = proc (): int {.nimcall.} = 0
+  discard indirectCallViaVarParam(p)
+  discard indirectCallViaPointer(addr p)
+  myEcho ""
+  globalVar
+
+func a: int =
+  discard callWithSideEffects()

From 1d6863a7899fd87fd9eb017ae370ef37db18ad32 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Thu, 15 Jul 2021 11:54:07 -0700
Subject: [PATCH 0224/1654] deprecate `sequtils.delete` and add an overload
 with saner semantics consistent with `system.delete` (#18487)

* deprecate sequtils.delete and add an overload with saner semantics
* AssertionDefect => IndexDefect
* improve tsequtils
* add tests; use splice in js for optimization
---
 changelog.md                      |   3 +
 lib/pure/collections/sequtils.nim |  45 +++++++++-
 tests/stdlib/tsequtils.nim        | 144 ++++++++++++++++++++----------
 3 files changed, 144 insertions(+), 48 deletions(-)

diff --git a/changelog.md b/changelog.md
index d44f28b94..d07ddddb4 100644
--- a/changelog.md
+++ b/changelog.md
@@ -346,6 +346,9 @@
 
 - Added `dom.setInterval`, `dom.clearInterval` overloads.
 
+- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect
+  if the slice is out of bounds.
+
 ## Language changes
 
 - `nimscript` now handles `except Exception as e`.
diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim
index 6facfcc32..5b41ea00d 100644
--- a/lib/pure/collections/sequtils.nim
+++ b/lib/pure/collections/sequtils.nim
@@ -509,12 +509,51 @@ proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.})
       inc(pos)
   setLen(s, pos)
 
-func delete*[T](s: var seq[T]; first, last: Natural) =
+func delete*[T](s: var seq[T]; slice: Slice[int]) =
+  ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains
+  ## elements out of range.
+  ##
+  ## This operation moves all elements after `s[slice]` in linear time.
+  runnableExamples:
+    var a = @[10, 11, 12, 13, 14]
+    doAssertRaises(IndexDefect): a.delete(4..5)
+    assert a == @[10, 11, 12, 13, 14]
+    a.delete(4..4)
+    assert a == @[10, 11, 12, 13]
+    a.delete(1..2)
+    assert a == @[10, 13]
+    a.delete(1..<1) # empty slice
+    assert a == @[10, 13]
+  when compileOption("boundChecks"):
+    if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len):
+      raise newException(IndexDefect, $(slice: slice, len: s.len))
+  if slice.b >= slice.a:
+    template defaultImpl =
+      var i = slice.a
+      var j = slice.b + 1
+      var newLen = s.len - j + i
+      while i < newLen:
+        when defined(gcDestructors):
+          s[i] = move(s[j])
+        else:
+          s[i].shallowCopy(s[j])
+        inc(i)
+        inc(j)
+      setLen(s, newLen)
+    when nimvm: defaultImpl()
+    else:
+      when defined(js):
+        let n = slice.b - slice.a + 1
+        let first = slice.a
+        {.emit: "`s`.splice(`first`, `n`);".}
+      else:
+        defaultImpl()
+
+func delete*[T](s: var seq[T]; first, last: Natural) {.deprecated: "use `delete(s, first..last)`".} =
   ## Deletes the items of a sequence `s` at positions `first..last`
   ## (including both ends of the range).
   ## This modifies `s` itself, it does not return a copy.
-  ##
-  runnableExamples:
+  runnableExamples("--warning:deprecated:off"):
     let outcome = @[1, 1, 1, 1, 1, 1, 1, 1]
     var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1]
     dest.delete(3, 8)
diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim
index 385e6e651..6c9c92873 100644
--- a/tests/stdlib/tsequtils.nim
+++ b/tests/stdlib/tsequtils.nim
@@ -1,3 +1,9 @@
+discard """
+  targets: "c js"
+"""
+
+# xxx move all tests under `main`
+
 import std/sequtils
 import strutils
 from algorithm import sorted
@@ -190,17 +196,6 @@ block: # keepIf test
   keepIf(floats, proc(x: float): bool = x > 10)
   doAssert floats == @[13.0, 12.5, 10.1]
 
-block: # delete tests
-  let outcome = @[1, 1, 1, 1, 1, 1, 1, 1]
-  var dest = @[1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1]
-  dest.delete(3, 8)
-  doAssert outcome == dest, """\
-  Deleting range 3-9 from [1,1,1,2,2,2,2,2,2,1,1,1,1,1]
-  is [1,1,1,1,1,1,1,1]"""
-  var x = @[1, 2, 3]
-  x.delete(100, 100)
-  doAssert x == @[1, 2, 3]
-
 block: # insert tests
   var dest = @[1, 1, 1, 1, 1, 1, 1, 1]
   let
@@ -298,43 +293,45 @@ block: # toSeq test
     doAssert myIter.toSeq == @[1, 2]
     doAssert toSeq(myIter) == @[1, 2]
 
-  block:
-    iterator myIter(): int {.closure.} =
-      yield 1
-      yield 2
+  when not defined(js):
+    # pending #4695
+    block:
+        iterator myIter(): int {.closure.} =
+          yield 1
+          yield 2
 
-    doAssert myIter.toSeq == @[1, 2]
-    doAssert toSeq(myIter) == @[1, 2]
-
-  block:
-    proc myIter(): auto =
-      iterator ret(): int {.closure.} =
-        yield 1
-        yield 2
-      result = ret
-
-    doAssert myIter().toSeq == @[1, 2]
-    doAssert toSeq(myIter()) == @[1, 2]
-
-  block:
-    proc myIter(n: int): auto =
-      var counter = 0
-      iterator ret(): int {.closure.} =
-        while counter < n:
-          yield counter
-          counter.inc
-      result = ret
+        doAssert myIter.toSeq == @[1, 2]
+        doAssert toSeq(myIter) == @[1, 2]
 
     block:
-      let myIter3 = myIter(3)
-      doAssert myIter3.toSeq == @[0, 1, 2]
+      proc myIter(): auto =
+        iterator ret(): int {.closure.} =
+          yield 1
+          yield 2
+        result = ret
+
+      doAssert myIter().toSeq == @[1, 2]
+      doAssert toSeq(myIter()) == @[1, 2]
+
     block:
-      let myIter3 = myIter(3)
-      doAssert toSeq(myIter3) == @[0, 1, 2]
-    block:
-      # makes sure this does not hang forever
-      doAssert myIter(3).toSeq == @[0, 1, 2]
-      doAssert toSeq(myIter(3)) == @[0, 1, 2]
+      proc myIter(n: int): auto =
+        var counter = 0
+        iterator ret(): int {.closure.} =
+          while counter < n:
+            yield counter
+            counter.inc
+        result = ret
+
+      block:
+        let myIter3 = myIter(3)
+        doAssert myIter3.toSeq == @[0, 1, 2]
+      block:
+        let myIter3 = myIter(3)
+        doAssert toSeq(myIter3) == @[0, 1, 2]
+      block:
+        # makes sure this does not hang forever
+        doAssert myIter(3).toSeq == @[0, 1, 2]
+        doAssert toSeq(myIter(3)) == @[0, 1, 2]
 
 block:
   # tests https://github.com/nim-lang/Nim/issues/7187
@@ -452,4 +449,61 @@ block:
       for i in 0..
Date: Thu, 15 Jul 2021 23:13:01 +0300
Subject: [PATCH 0225/1654] Add changes required by Nimble lock file support
 (#12104)

Implemented support for Nimble local cache with package directories with
a checksum of the package at the end of their names. Now the compiler
supports package paths in the form:

 * /path_to_nimble_cache_dir/pkgs/package_name-1.2.3-
FEBADEAEA2345E777F0F6F8433F7F0A52EDD5D1B

 * /path_to_nimble_cache_dir/pkgs/package_name-#head-
042D4BE2B90ED0672E717D71850ABDB0A2D19CD2

 * /path_to_nimble_cache_dir/pkgs/package_name-#branch-name-
DBC1F902CB79946E990E38AF51F0BAD36ACFABD9

Related to nim-lang/nimble#127
---
 compiler/modulepaths.nim      |  2 +-
 compiler/nimblecmd.nim        | 78 +++++++++++++++++++-----------
 lib/std/sha1.nim              |  4 ++
 tests/compiler/tnimblecmd.nim | 91 +++++++++++++++++++++++++++--------
 tests/stdlib/tsha1.nim        | 10 ++++
 5 files changed, 135 insertions(+), 50 deletions(-)

diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim
index 8511b1592..a16b669c4 100644
--- a/compiler/modulepaths.nim
+++ b/compiler/modulepaths.nim
@@ -21,7 +21,7 @@ when false:
     for k, p in os.walkDir(dir, relative=true):
       if k == pcDir and p.len > pkg.len+1 and
           p[pkg.len] == '-' and p.startsWith(pkg):
-        let (_, a) = getPathVersion(p)
+        let (_, a, _) = getPathVersionChecksum(p)
         if bestv.len == 0 or bestv < a:
           bestv = a
           best = dir / p
diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim
index 28398289c..9cd2941ba 100644
--- a/compiler/nimblecmd.nim
+++ b/compiler/nimblecmd.nim
@@ -9,8 +9,8 @@
 
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
-import parseutils, strutils, strtabs, os, options, msgs, sequtils,
-  lineinfos, pathutils
+import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils,
+  std/sha1, tables
 
 proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if not conf.searchPaths.contains(path):
@@ -18,6 +18,7 @@ proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
 
 type
   Version* = distinct string
+  PackageInfo = Table[string, tuple[version, checksum: string]]
 
 proc `$`*(ver: Version): string {.borrow.}
 
@@ -62,43 +63,64 @@ proc `<`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
-proc getPathVersion*(p: string): tuple[name, version: string] =
-  ## Splits path ``p`` in the format ``/home/user/.nimble/pkgs/package-0.1``
-  ## into ``(/home/user/.nimble/pkgs/package, 0.1)``
-  result.name = ""
-  result.version = ""
+proc getPathVersionChecksum*(p: string): tuple[name, version, checksum: string] =
+  ## Splits path ``p`` in the format
+  ## ``/home/user/.nimble/pkgs/package-0.1-febadeaea2345e777f0f6f8433f7f0a52edd5d1b`` into
+  ## ``("/home/user/.nimble/pkgs/package", "0.1", "febadeaea2345e777f0f6f8433f7f0a52edd5d1b")``
 
-  const specialSeparator = "-#"
-  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
-  var sepIdx = p.find(specialSeparator, start = last)
-  if sepIdx == -1:
-    sepIdx = p.rfind('-', start = last)
+  const checksumSeparator = '-'
+  const versionSeparator = '-'
+  const specialVersionSepartator = "-#"
+  const separatorNotFound = -1
 
-  if sepIdx == -1:
-    result.name = p
-    return
+  var checksumSeparatorIndex = p.rfind(checksumSeparator)
+  if checksumSeparatorIndex != separatorNotFound:
+    result.checksum = p.substr(checksumSeparatorIndex + 1)
+    if not result.checksum.isValidSha1Hash():
+      result.checksum = ""
+      checksumSeparatorIndex = p.len()
+  else:
+    checksumSeparatorIndex = p.len()
 
-  for i in sepIdx..
Date: Fri, 16 Jul 2021 07:42:35 +0200
Subject: [PATCH 0226/1654] added Atlas helper tool (#18497)

* added Atlas helper tool
* further improvements
---
 koch.nim                       |   8 +-
 tools/atlas/atlas.md           |  77 +++++++
 tools/atlas/atlas.nim          | 390 +++++++++++++++++++++++++++++++++
 tools/atlas/osutils.nim        |  59 +++++
 tools/atlas/packagesjson.nim   | 117 ++++++++++
 tools/atlas/parse_requires.nim | 101 +++++++++
 6 files changed, 751 insertions(+), 1 deletion(-)
 create mode 100644 tools/atlas/atlas.md
 create mode 100644 tools/atlas/atlas.nim
 create mode 100644 tools/atlas/osutils.nim
 create mode 100644 tools/atlas/packagesjson.nim
 create mode 100644 tools/atlas/parse_requires.nim

diff --git a/koch.nim b/koch.nim
index 4ba0d0eb5..da7576e65 100644
--- a/koch.nim
+++ b/koch.nim
@@ -222,7 +222,13 @@ proc buildTools(args: string = "") =
   # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler
   # `-d:debug` should be changed to a flag that doesn't require re-compiling nim
   # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces
-  nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, outputName = "nim_dbg")
+  nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options =
+      "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args,
+      outputName = "nim_dbg")
+
+  nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args,
+      outputName = "atlas")
+
 
 proc nsis(latest: bool; args: string) =
   bundleNimbleExe(latest, args)
diff --git a/tools/atlas/atlas.md b/tools/atlas/atlas.md
new file mode 100644
index 000000000..a36817dc5
--- /dev/null
+++ b/tools/atlas/atlas.md
@@ -0,0 +1,77 @@
+# Atlas Package Cloner
+
+Atlas is a simple package cloner tool that automates some of the
+workflows and needs for Nim's stdlib evolution.
+
+Atlas is compatible with Nimble in the sense that it supports the Nimble
+file format.
+
+
+## How it works
+
+Atlas uses git commits internally; version requirements are translated
+to git commits via `git show-ref --tags`.
+
+Atlas uses URLs internally; Nimble package names are translated to URLs
+via Nimble's  `packages.json` file.
+
+Atlas does not call the Nim compiler for a build, instead it creates/patches
+a `nim.cfg` file for the compiler. For example:
+
+```
+############# begin Atlas config section ##########
+--noNimblePath
+--path:"../nimx"
+--path:"../sdl2/src"
+--path:"../opengl/src"
+############# end Atlas config section   ##########
+```
+
+The version selection is deterministic, it picks up the *minimum* required
+version. Thanks to this design, lock files are not required.
+
+
+## Dependencies
+
+Dependencies are neither installed globally, nor locally into the current
+project. Instead a "workspace" is used. The workspace is the nearest parent
+directory of the current directory that does not contain a `.git` subdirectory.
+Dependencies are managed as **siblings**, not as children. Dependencies are
+kept as git repositories.
+
+Thanks to this setup, it's easy to develop multiple projects at the same time.
+
+A project plus its dependencies are stored in a workspace:
+
+  $workspace / main project
+  $workspace / dependency A
+  $workspace / dependency B
+
+
+No attempts are being made at keeping directory hygiene inside the
+workspace, you're supposed to create appropriate `$workspace` directories
+at your own leisure.
+
+
+## Commands
+
+Atlas supports the following commands:
+
+
+### Clone 
+
+Clones a URL and all of its dependencies (recursively) into the workspace.
+Creates or patches a `nim.cfg` file with the required `--path` entries.
+
+
+### Clone 
+
+The `` is translated into an URL via `packages.json` and
+then `clone ` is performed.
+
+
+### Search 
+
+Search the package index `packages.json` for a package that the given terms
+in its description (or name or list of tags).
+
diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
new file mode 100644
index 000000000..bc1cc6ae3
--- /dev/null
+++ b/tools/atlas/atlas.nim
@@ -0,0 +1,390 @@
+#
+#           Atlas Package Cloner
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Simple tool to automate frequent workflows: Can "clone"
+## a Nimble dependency and its dependencies recursively.
+
+import std/[parseopt, strutils, os, osproc, sequtils, unicode, tables, sets]
+import parse_requires, osutils, packagesjson
+
+const
+  Version = "0.1"
+  Usage = "atlas - Nim Package Manager Version " & Version & """
+
+  (c) 2021 Andreas Rumpf
+Usage:
+  atlas [options] [command] [arguments]
+Command:
+  clone url|pkgname     clone a package and all of its dependencies
+  search keyw keywB...  search for package that contains the given keywords
+
+Options:
+  --keepCommits         do not perform any `git checkouts`
+  --version             show the version
+  --help                show this help
+"""
+
+proc writeHelp() =
+  stdout.write(Usage)
+  stdout.flushFile()
+  quit(0)
+
+proc writeVersion() =
+  stdout.write(Version & "\n")
+  stdout.flushFile()
+  quit(0)
+
+type
+  PackageName = distinct string
+  DepRelation = enum
+    normal, strictlyLess, strictlyGreater
+
+  Dependency = object
+    name: PackageName
+    url, commit: string
+    rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too.
+  AtlasContext = object
+    projectDir, workspace: string
+    hasPackageList: bool
+    keepCommits: bool
+    p: Table[string, string] # name -> url mapping
+    processed: HashSet[string] # the key is (url / commit)
+    errors: int
+
+const
+  InvalidCommit = ""
+
+proc toDepRelation(s: string): DepRelation =
+  case s
+  of "<": strictlyLess
+  of ">": strictlyGreater
+  else: normal
+
+proc isCleanGit(dir: string): string =
+  result = ""
+  let (outp, status) = osproc.execCmdEx("git diff")
+  if outp.len != 0:
+    result = "'git diff' not empty"
+  elif status != 0:
+    result = "'git diff' returned non-zero"
+
+proc message(c: var AtlasContext; category: string; p: PackageName; args: varargs[string]) =
+  var msg = category & "(" & p.string & ")"
+  for a in args:
+    msg.add ' '
+    msg.add a
+  stdout.writeLine msg
+  inc c.errors
+
+proc warn(c: var AtlasContext; p: PackageName; args: varargs[string]) =
+  message(c, "[Warning] ", p, args)
+
+proc error(c: var AtlasContext; p: PackageName; args: varargs[string]) =
+  message(c, "[Error] ", p, args)
+
+proc sameVersionAs(tag, ver: string): bool =
+  const VersionChars = {'0'..'9', '.'}
+
+  proc safeCharAt(s: string; i: int): char {.inline.} =
+    if i >= 0 and i < s.len: s[i] else: '\0'
+
+  let idx = find(tag, ver)
+  if idx >= 0:
+    # we found the version as a substring inside the `tag`. But we
+    # need to watch out the the boundaries are not part of a
+    # larger/different version number:
+    result = safeCharAt(tag, idx-1) notin VersionChars and
+      safeCharAt(tag, idx+ver.len) notin VersionChars
+
+proc versionToCommit(d: Dependency): string =
+  let (outp, status) = osproc.execCmdEx("git show-ref --tags")
+  if status == 0:
+    var useNextOne = false
+    for line in splitLines(outp):
+      let commitsAndTags = strutils.splitWhitespace(line)
+      if commitsAndTags.len == 2:
+        case d.rel
+        of normal:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyLess:
+          if d.commit == InvalidCommit or not commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyGreater:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            useNextOne = true
+          elif useNextOne:
+            return commitsAndTags[0]
+
+  return ""
+
+proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
+  let (outp, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
+  if status != 0:
+    error(c, p, "could not checkout commit", commit)
+
+proc gitPull(c: var AtlasContext; p: PackageName) =
+  let (_, status) = osproc.execCmdEx("git pull")
+  if status != 0:
+    error(c, p, "could not 'git pull'")
+
+proc updatePackages(c: var AtlasContext) =
+  if dirExists(c.workspace / PackagesDir):
+    withDir(c.workspace / PackagesDir):
+      gitPull(c, PackageName PackagesDir)
+  else:
+    withDir c.workspace:
+      let err = cloneUrl("https://github.com/nim-lang/packages", PackagesDir, false)
+      if err != "":
+        error c, PackageName(PackagesDir), err
+
+proc fillPackageLookupTable(c: var AtlasContext) =
+  if not c.hasPackageList:
+    c.hasPackageList = true
+    updatePackages(c)
+    let plist = getPackages(c.workspace)
+    for entry in plist:
+      c.p[unicode.toLower entry.name] = entry.url
+
+proc toUrl(c: var AtlasContext; p: string): string =
+  if p.isUrl:
+    result = p
+  else:
+    fillPackageLookupTable(c)
+    result = c.p.getOrDefault(unicode.toLower p)
+  if result.len == 0:
+    inc c.errors
+
+proc toName(p: string): PackageName =
+  if p.isUrl:
+    result = PackageName splitFile(p).name
+  else:
+    result = PackageName p
+
+proc needsCommitLookup(commit: string): bool {.inline} =
+  '.' in commit or commit == InvalidCommit
+
+proc checkoutCommit(c: var AtlasContext; w: Dependency) =
+  let dir = c.workspace / w.name.string
+  withDir dir:
+    if w.commit.len == 0 or cmpIgnoreCase(w.commit, "#head") == 0:
+      gitPull(c, w.name)
+    else:
+      let err = isCleanGit(dir)
+      if err != "":
+        warn c, w.name, err
+      else:
+        let requiredCommit = if needsCommitLookup(w.commit): versionToCommit(w) else: w.commit
+        let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
+        let currentCommit = strutils.strip(cc)
+        if requiredCommit == "" or status != 0:
+          if requiredCommit == "" and w.commit == InvalidCommit:
+            warn c, w.name, "package has no tagged releases"
+          else:
+            warn c, w.name, "cannot find specified version/commit", w.commit
+        else:
+          if currentCommit != requiredCommit:
+            # checkout the later commit:
+            # git merge-base --is-ancestor  
+            let (mergeBase, status) = osproc.execCmdEx("git merge-base " &
+                currentCommit.quoteShell & " " & requiredCommit.quoteShell)
+            if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
+              # conflict resolution: pick the later commit:
+              if mergeBase == currentCommit:
+                checkoutGitCommit(c, w.name, requiredCommit)
+            else:
+              checkoutGitCommit(c, w.name, requiredCommit)
+              when false:
+                warn c, w.name, "do not know which commit is more recent:",
+                  currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
+
+proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
+  result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
+  if not fileExists(result):
+    result = ""
+    for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
+      if result.len == 0:
+        result = x
+      else:
+        # ambiguous .nimble file
+        return ""
+
+proc addUniqueDep(c: var AtlasContext; work: var seq[Dependency];
+                  tokens: seq[string]) =
+  let oldErrors = c.errors
+  let url = toUrl(c, tokens[0])
+  if oldErrors != c.errors:
+    warn c, toName(tokens[0]), "cannot resolve package name"
+  elif not c.processed.containsOrIncl(url / tokens[2]):
+    work.add Dependency(name: toName(tokens[0]), url: url, commit: tokens[2],
+                        rel: toDepRelation(tokens[1]))
+
+proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
+                    dep: Dependency; result: var seq[string];
+                    isMainProject: bool) =
+  let nimbleFile = findNimbleFile(c, dep)
+  if nimbleFile != "":
+    let nimbleInfo = extractRequiresInfo(nimbleFile)
+    for r in nimbleInfo.requires:
+      var tokens: seq[string] = @[]
+      for token in tokenizeRequires(r):
+        tokens.add token
+      if tokens.len == 1:
+        # nimx uses dependencies like 'requires "sdl2"'.
+        # Via this hack we map them to the first tagged release.
+        # (See the `isStrictlySmallerThan` logic.)
+        tokens.add "<"
+        tokens.add InvalidCommit
+      elif tokens.len == 2 and tokens[1].startsWith("#"):
+        # Dependencies can also look like 'requires "sdl2#head"
+        var commit = tokens[1]
+        tokens[1] = "=="
+        tokens.add commit
+
+      if tokens.len >= 3 and cmpIgnoreCase(tokens[0], "nim") != 0:
+        c.addUniqueDep work, tokens
+
+    result.add dep.name.string / nimbleInfo.srcDir
+  else:
+    result.add dep.name.string
+
+proc clone(c: var AtlasContext; start: string): seq[string] =
+  # non-recursive clone.
+  let oldErrors = c.errors
+  var work = @[Dependency(name: toName(start), url: toUrl(c, start), commit: "")]
+
+  if oldErrors != c.errors:
+    error c, toName(start), "cannot resolve package name"
+    return
+
+  c.projectDir = work[0].name.string
+  result = @[]
+  var i = 0
+  while i < work.len:
+    let w = work[i]
+    let oldErrors = c.errors
+    if not dirExists(c.workspace / w.name.string):
+      withDir c.workspace:
+        let err = cloneUrl(w.url, w.name.string, false)
+        if err != "":
+          error c, w.name, err
+    if oldErrors == c.errors:
+      if not c.keepCommits: checkoutCommit(c, w)
+      # even if the checkout fails, we can make use of the somewhat
+      # outdated .nimble file to clone more of the most likely still relevant
+      # dependencies:
+      collectNewDeps(c, work, w, result, i == 0)
+    inc i
+
+const
+  configPatternBegin = "############# begin Atlas config section ##########\n"
+  configPatternEnd =   "############# end Atlas config section   ##########\n"
+
+proc patchNimCfg(c: AtlasContext; deps: seq[string]) =
+  var paths = "--noNimblePath\n"
+  for d in deps:
+    paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n"
+
+  let cfg = c.projectDir / "nim.cfg"
+  var cfgContent = configPatternBegin & paths & configPatternEnd
+  if not fileExists(cfg):
+    writeFile(cfg, cfgContent)
+  else:
+    let content = readFile(cfg)
+    let start = content.find(configPatternBegin)
+    if start >= 0:
+      cfgContent = content.substr(0, start-1) & cfgContent
+      let theEnd = content.find(configPatternEnd, start)
+      if theEnd >= 0:
+        cfgContent.add content.substr(theEnd+len(configPatternEnd))
+    else:
+      cfgContent = content & "\n" & cfgContent
+    if cfgContent != content:
+      # do not touch the file if nothing changed
+      # (preserves the file date information):
+      writeFile(cfg, cfgContent)
+
+proc error*(msg: string) =
+  when defined(debug):
+    writeStackTrace()
+  quit "[Error] " & msg
+
+proc main =
+  var action = ""
+  var args: seq[string] = @[]
+  template singleArg() =
+    if args.len != 1:
+      error action & " command takes a single package name"
+
+  template noArgs() =
+    if args.len != 0:
+      error action & " command takes no arguments"
+
+  var c = AtlasContext(
+    projectDir: getCurrentDir(),
+    workspace: getCurrentDir())
+
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      if action.len == 0:
+        action = key.normalize
+      else:
+        args.add key
+    of cmdLongOption, cmdShortOption:
+      case normalize(key)
+      of "help", "h": writeHelp()
+      of "version", "v": writeVersion()
+      of "keepcommits": c.keepCommits = true
+      else: writeHelp()
+    of cmdEnd: assert false, "cannot happen"
+
+  while c.workspace.len > 0 and dirExists(c.workspace / ".git"):
+    c.workspace = c.workspace.parentDir()
+
+  case action
+  of "":
+    error "No action."
+  of "clone":
+    singleArg()
+    let deps = clone(c, args[0])
+    patchNimCfg c, deps
+    if c.errors > 0:
+      error "There were problems."
+  of "refresh":
+    noArgs()
+    updatePackages(c)
+  of "search", "list":
+    updatePackages(c)
+    search getPackages(c.workspace), args
+  else:
+    error "Invalid action: " & action
+
+when isMainModule:
+  main()
+
+when false:
+  # some testing code for the `patchNimCfg` logic:
+  var c = AtlasContext(
+    projectDir: getCurrentDir(),
+    workspace: getCurrentDir().parentDir)
+
+  patchNimCfg(c, @[PackageName"abc", PackageName"xyz"])
+
+when false:
+  assert sameVersionAs("v0.2.0", "0.2.0")
+  assert sameVersionAs("v1", "1")
+
+  assert sameVersionAs("1.90", "1.90")
+
+  assert sameVersionAs("v1.2.3-zuzu", "1.2.3")
+  assert sameVersionAs("foo-1.2.3.4", "1.2.3.4")
+
+  assert not sameVersionAs("foo-1.2.3.4", "1.2.3")
+  assert not sameVersionAs("foo", "1.2.3")
+  assert not sameVersionAs("", "1.2.3")
diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim
new file mode 100644
index 000000000..31392b113
--- /dev/null
+++ b/tools/atlas/osutils.nim
@@ -0,0 +1,59 @@
+## OS utilities like 'withDir'.
+## (c) 2021 Andreas Rumpf
+
+import os, strutils, osproc
+
+template withDir*(dir, body) =
+  let oldDir = getCurrentDir()
+  try:
+    setCurrentDir(dir)
+    body
+  finally:
+    setCurrentDir(oldDir)
+
+proc isUrl*(x: string): bool =
+  x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://")
+
+proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string =
+  ## Returns an error message on error or else "".
+  result = ""
+  var modUrl =
+    if url.startsWith("git://") and cloneUsingHttps:
+      "https://" & url[6 .. ^1]
+    else: url
+
+  # github + https + trailing url slash causes a
+  # checkout/ls-remote to fail with Repository not found
+  var isGithub = false
+  if modUrl.contains("github.com") and modUrl.endswith("/"):
+    modUrl = modUrl[0 .. ^2]
+    isGithub = true
+
+  let (_, exitCode) = execCmdEx("git ls-remote --quiet --tags " & modUrl)
+  var xcode = exitCode
+  if isGithub and exitCode != QuitSuccess:
+    # retry multiple times to avoid annoying github timeouts:
+    for i in 0..4:
+      os.sleep(4000)
+      xcode = execCmdEx("git ls-remote --quiet --tags " & modUrl)[1]
+      if xcode == QuitSuccess: break
+
+  if xcode == QuitSuccess:
+    # retry multiple times to avoid annoying github timeouts:
+    let cmd = "git clone " & modUrl & " " & dest
+    for i in 0..4:
+      if execShellCmd(cmd) == 0: return ""
+      os.sleep(4000)
+    result = "exernal program failed: " & cmd
+  elif not isGithub:
+    let (_, exitCode) = execCmdEx("hg identify " & modUrl)
+    if exitCode == QuitSuccess:
+      let cmd = "hg clone " & modUrl & " " & dest
+      for i in 0..4:
+        if execShellCmd(cmd) == 0: return ""
+        os.sleep(4000)
+      result = "exernal program failed: " & cmd
+    else:
+      result = "Unable to identify url: " & modUrl
+  else:
+    result = "Unable to identify url: " & modUrl
diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim
new file mode 100644
index 000000000..e1e23ee29
--- /dev/null
+++ b/tools/atlas/packagesjson.nim
@@ -0,0 +1,117 @@
+
+import std / [json, os, sets, strutils]
+import osutils
+
+type
+  Package* = ref object
+    # Required fields in a package.
+    name*: string
+    url*: string # Download location.
+    license*: string
+    downloadMethod*: string
+    description*: string
+    tags*: seq[string] # \
+    # From here on, optional fields set to the empty string if not available.
+    version*: string
+    dvcsTag*: string
+    web*: string # Info url for humans.
+
+proc optionalField(obj: JsonNode, name: string, default = ""): string =
+  if hasKey(obj, name) and obj[name].kind == JString:
+    result = obj[name].str
+  else:
+    result = default
+
+proc requiredField(obj: JsonNode, name: string): string =
+  result = optionalField(obj, name, "")
+
+proc fromJson*(obj: JSonNode): Package =
+  result = Package()
+  result.name = obj.requiredField("name")
+  if result.name.len == 0: return nil
+  result.version = obj.optionalField("version")
+  result.url = obj.requiredField("url")
+  if result.url.len == 0: return nil
+  result.downloadMethod = obj.requiredField("method")
+  if result.downloadMethod.len == 0: return nil
+  result.dvcsTag = obj.optionalField("dvcs-tag")
+  result.license = obj.optionalField("license")
+  result.tags = @[]
+  for t in obj["tags"]:
+    result.tags.add(t.str)
+  result.description = obj.requiredField("description")
+  result.web = obj.optionalField("web")
+
+const PackagesDir* = "packages"
+
+proc getPackages*(workspaceDir: string): seq[Package] =
+  result = @[]
+  var uniqueNames = initHashSet[string]()
+  var jsonFiles = 0
+  for kind, path in walkDir(workspaceDir / PackagesDir):
+    if kind == pcFile and path.endsWith(".json"):
+      inc jsonFiles
+      let packages = json.parseFile(path)
+      for p in packages:
+        let pkg = p.fromJson()
+        if pkg != nil and not uniqueNames.containsOrIncl(pkg.name):
+          result.add(pkg)
+
+proc `$`*(pkg: Package): string =
+  result = pkg.name & ":\n"
+  result &= "  url:         " & pkg.url & " (" & pkg.downloadMethod & ")\n"
+  result &= "  tags:        " & pkg.tags.join(", ") & "\n"
+  result &= "  description: " & pkg.description & "\n"
+  result &= "  license:     " & pkg.license & "\n"
+  if pkg.web.len > 0:
+    result &= "  website:     " & pkg.web & "\n"
+
+proc search*(pkgList: seq[Package]; terms: seq[string]) =
+  var found = false
+  template onFound =
+    echo pkg
+    echo("")
+    found = true
+    break forPackage
+
+  for pkg in pkgList:
+    if terms.len > 0:
+      block forPackage:
+        for term in terms:
+          let word = term.toLower
+          # Search by name.
+          if word in pkg.name.toLower:
+            onFound()
+          # Search by tag.
+          for tag in pkg.tags:
+            if word in tag.toLower:
+              onFound()
+    else:
+      echo(pkg)
+      echo(" ")
+
+  if not found and terms.len > 0:
+    echo("No package found.")
+
+type PkgCandidates* = array[3, seq[Package]]
+
+proc determineCandidates*(pkgList: seq[Package];
+                         terms: seq[string]): PkgCandidates =
+  result[0] = @[]
+  result[1] = @[]
+  result[2] = @[]
+  for pkg in pkgList:
+    block termLoop:
+      for term in terms:
+        let word = term.toLower
+        if word == pkg.name.toLower:
+          result[0].add pkg
+          break termLoop
+        elif word in pkg.name.toLower:
+          result[1].add pkg
+          break termLoop
+        else:
+          for tag in pkg.tags:
+            if word in tag.toLower:
+              result[2].add pkg
+              break termLoop
diff --git a/tools/atlas/parse_requires.nim b/tools/atlas/parse_requires.nim
new file mode 100644
index 000000000..7e26a1656
--- /dev/null
+++ b/tools/atlas/parse_requires.nim
@@ -0,0 +1,101 @@
+## Utility API for Nim package managers.
+## (c) 2021 Andreas Rumpf
+
+import std / strutils
+import ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils]
+
+type
+  NimbleFileInfo* = object
+    requires*: seq[string]
+    srcDir*: string
+    tasks*: seq[(string, string)]
+
+proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extract(child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "requires":
+        for i in 1.. 0: ch = ch.lastSon
+          if ch.kind in {nkStrLit..nkTripleStrLit}:
+            result.requires.add ch.strVal
+          else:
+            localError(conf, ch.info, "'requires' takes string literals")
+      of "task":
+        if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}:
+          result.tasks.add((n[1].ident.s, n[2].strVal))
+      else: discard
+  of nkAsgn, nkFastAsgn:
+    if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0:
+      if n[1].kind in {nkStrLit..nkTripleStrLit}:
+        result.srcDir = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
+  else:
+    discard
+
+proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
+  ## Extract the `requires` information from a Nimble file. This does **not**
+  ## evaluate the Nimble file. Errors are produced on stderr/stdout and are
+  ## formatted as the Nim compiler does it. The parser uses the Nim compiler
+  ## as an API. The result can be empty, this is not an error, only parsing
+  ## errors are reported.
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extract(parseAll(parser), conf, result)
+    closeParser(parser)
+
+const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
+
+proc token(s: string; idx: int; lit: var string): int =
+  var i = idx
+  if i >= s.len: return i
+  while s[i] in Whitespace: inc(i)
+  case s[i]
+  of Letters, '#':
+    lit.add s[i]
+    inc i
+    while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
+      lit.add s[i]
+      inc i
+  of '0'..'9':
+    while i < s.len and s[i] in {'0'..'9', '.'}:
+      lit.add s[i]
+      inc i
+  of '"':
+    inc i
+    while i < s.len and s[i] != '"':
+      lit.add s[i]
+      inc i
+    inc i
+  of Operators:
+    while i < s.len and s[i] in Operators:
+      lit.add s[i]
+      inc i
+  else:
+    lit.add s[i]
+    inc i
+  result = i
+
+iterator tokenizeRequires*(s: string): string =
+  var start = 0
+  var tok = ""
+  while start < s.len:
+    tok.setLen 0
+    start = token(s, start, tok)
+    yield tok
+
+when isMainModule:
+  for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
+    echo x

From 96a7f9b31c84f14906b201d5e3b17c93d3878af6 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Fri, 16 Jul 2021 15:50:02 +0200
Subject: [PATCH 0227/1654] deprecate cuchar, don't redefine it (#18505)

---
 changelog.md                       |  3 ++-
 lib/posix/posix_linux_amd64.nim    |  2 +-
 lib/posix/posix_nintendoswitch.nim | 12 ++++-----
 lib/pure/bitops.nim                | 42 +++++++++++++++---------------
 lib/pure/net.nim                   |  6 ++---
 lib/pure/terminal.nim              |  2 +-
 lib/std/sysrand.nim                |  2 +-
 lib/system.nim                     |  4 +--
 lib/windows/winlean.nim            |  4 +--
 lib/wrappers/openssl.nim           | 24 ++++++++---------
 tests/stdlib/topenssl.nim          |  8 +++---
 11 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/changelog.md b/changelog.md
index d07ddddb4..568ff0bba 100644
--- a/changelog.md
+++ b/changelog.md
@@ -6,7 +6,8 @@
 
 - Deprecated `std/mersenne`
 
--  `cuchar` now aliases `uint8` instead of `char`
+- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
+  Please use `char` or `uint8` instead.
 
 - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
   see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim
index 7f6a589f0..5c0b4d6cf 100644
--- a/lib/posix/posix_linux_amd64.nim
+++ b/lib/posix/posix_linux_amd64.nim
@@ -47,7 +47,7 @@ type
     d_ino*: Ino
     d_off*: Off
     d_reclen*: cushort
-    d_type*: int8  # cuchar really!
+    d_type*: int8  # uint8 really!
     d_name*: array[256, cchar]
 
   Tflock* {.importc: "struct flock", final, pure,
diff --git a/lib/posix/posix_nintendoswitch.nim b/lib/posix/posix_nintendoswitch.nim
index 44e431437..4cef80bb7 100644
--- a/lib/posix/posix_nintendoswitch.nim
+++ b/lib/posix/posix_nintendoswitch.nim
@@ -33,7 +33,7 @@ type
   Dirent* {.importc: "struct dirent",
             header: "", final, pure.} = object ## dirent_t struct
     d_ino*: Ino
-    d_type*: int8  # cuchar really!
+    d_type*: int8  # uint8 really!
     d_name*: array[256, cchar]
 
   Tflock* {.importc: "struct flock", final, pure,
@@ -347,20 +347,20 @@ type
 
   SockAddr* {.importc: "struct sockaddr", header: "",
               pure, final.} = object ## struct sockaddr
-    sa_len: cuchar
+    sa_len: uint8
     sa_family*: TSa_Family         ## Address family.
     sa_data*: array[14, char] ## Socket address (variable-length data).
 
   Sockaddr_storage* {.importc: "struct sockaddr_storage",
                        header: "",
                        pure, final.} = object ## struct sockaddr_storage
-    ss_len: cuchar
+    ss_len: uint8
     ss_family*: TSa_Family ## Address family.
-    ss_padding1: array[64 - sizeof(cuchar) - sizeof(cshort), char]
+    ss_padding1: array[64 - sizeof(uint8) - sizeof(cshort), char]
     ss_align: clonglong
     ss_padding2: array[
-      128 - sizeof(cuchar) - sizeof(cshort) -
-      (64 - sizeof(cuchar) - sizeof(cshort)) - 64, char]
+      128 - sizeof(uint8) - sizeof(cshort) -
+      (64 - sizeof(uint8) - sizeof(cshort)) - 64, char]
 
   Tif_nameindex* {.importc: "struct if_nameindex", final,
                    pure, header: "".} = object ## struct if_nameindex
diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim
index b5b08985e..6d3865e5d 100644
--- a/lib/pure/bitops.nim
+++ b/lib/pure/bitops.nim
@@ -450,15 +450,15 @@ when useGCC_builtins:
 
 elif useVCC_builtins:
   # Search the mask data from most significant bit (MSB) to least significant bit (LSB) for a set bit (1).
-  func bitScanReverse(index: ptr culong, mask: culong): cuchar {.
+  func bitScanReverse(index: ptr culong, mask: culong): uint8 {.
       importc: "_BitScanReverse", header: "".}
-  func bitScanReverse64(index: ptr culong, mask: uint64): cuchar {.
+  func bitScanReverse64(index: ptr culong, mask: uint64): uint8 {.
       importc: "_BitScanReverse64", header: "".}
 
   # Search the mask data from least significant bit (LSB) to the most significant bit (MSB) for a set bit (1).
-  func bitScanForward(index: ptr culong, mask: culong): cuchar {.
+  func bitScanForward(index: ptr culong, mask: culong): uint8 {.
       importc: "_BitScanForward", header: "".}
-  func bitScanForward64(index: ptr culong, mask: uint64): cuchar {.
+  func bitScanForward64(index: ptr culong, mask: uint64): uint8 {.
       importc: "_BitScanForward64", header: "".}
 
   template vcc_scan_impl(fnc: untyped; v: untyped): int =
@@ -468,15 +468,15 @@ elif useVCC_builtins:
 
 elif useICC_builtins:
   # Returns the number of trailing 0-bits in x, starting at the least significant bit position. If x is 0, the result is undefined.
-  func bitScanForward(p: ptr uint32, b: uint32): cuchar {.
+  func bitScanForward(p: ptr uint32, b: uint32): uint8 {.
       importc: "_BitScanForward", header: "".}
-  func bitScanForward64(p: ptr uint32, b: uint64): cuchar {.
+  func bitScanForward64(p: ptr uint32, b: uint64): uint8 {.
       importc: "_BitScanForward64", header: "".}
 
   # Returns the number of leading 0-bits in x, starting at the most significant bit position. If x is 0, the result is undefined.
-  func bitScanReverse(p: ptr uint32, b: uint32): cuchar {.
+  func bitScanReverse(p: ptr uint32, b: uint32): uint8 {.
       importc: "_BitScanReverse", header: "".}
-  func bitScanReverse64(p: ptr uint32, b: uint64): cuchar {.
+  func bitScanReverse64(p: ptr uint32, b: uint64): uint8 {.
       importc: "_BitScanReverse64", header: "".}
 
   template icc_scan_impl(fnc: untyped; v: untyped): int =
@@ -664,7 +664,7 @@ when useBuiltinsRotate:
   when defined(gcc):
     # GCC was tested until version 4.8.1 and intrinsics were present. Not tested
     # in previous versions.
-    func builtin_rotl8(value: cuchar, shift: cint): cuchar
+    func builtin_rotl8(value: uint8, shift: cint): uint8
                       {.importc: "__rolb", header: "".}
     func builtin_rotl16(value: cushort, shift: cint): cushort
                        {.importc: "__rolw", header: "".}
@@ -674,7 +674,7 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: cint): culonglong
                          {.importc: "__rolq", header: "".}
 
-    func builtin_rotr8(value: cuchar, shift: cint): cuchar
+    func builtin_rotr8(value: uint8, shift: cint): uint8
                       {.importc: "__rorb", header: "".}
     func builtin_rotr16(value: cushort, shift: cint): cushort
                        {.importc: "__rorw", header: "".}
@@ -690,7 +690,7 @@ when useBuiltinsRotate:
     # https://releases.llvm.org/8.0.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release
     # https://releases.llvm.org/8.0.0/tools/clang/docs/LanguageExtensions.html#builtin-rotateleft
     # source for correct declarations: https://github.com/llvm/llvm-project/blob/main/clang/include/clang/Basic/Builtins.def
-    func builtin_rotl8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotl8(value: uint8, shift: uint8): uint8
                       {.importc: "__builtin_rotateleft8", nodecl.}
     func builtin_rotl16(value: cushort, shift: cushort): cushort
                        {.importc: "__builtin_rotateleft16", nodecl.}
@@ -700,7 +700,7 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: culonglong): culonglong
                          {.importc: "__builtin_rotateleft64", nodecl.}
 
-    func builtin_rotr8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotr8(value: uint8, shift: uint8): uint8
                       {.importc: "__builtin_rotateright8", nodecl.}
     func builtin_rotr16(value: cushort, shift: cushort): cushort
                        {.importc: "__builtin_rotateright16", nodecl.}
@@ -716,9 +716,9 @@ when useBuiltinsRotate:
     # https://docs.microsoft.com/en-us/cpp/intrinsics/rotl8-rotl16?view=msvc-160
     # https://docs.microsoft.com/en-us/cpp/intrinsics/rotr8-rotr16?view=msvc-160
     # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/rotl-rotl64-rotr-rotr64?view=msvc-160
-    func builtin_rotl8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotl8(value: uint8, shift: uint8): uint8
                       {.importc: "_rotl8", header: "".}
-    func builtin_rotl16(value: cushort, shift: cuchar): cushort
+    func builtin_rotl16(value: cushort, shift: uint8): cushort
                        {.importc: "_rotl16", header: "".}
     func builtin_rotl32(value: cuint, shift: cint): cuint
                        {.importc: "_rotl", header: "".}
@@ -726,9 +726,9 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: cint): culonglong
                          {.importc: "_rotl64", header: "".}
 
-    func builtin_rotr8(value: cuchar, shift: cuchar): cuchar
+    func builtin_rotr8(value: uint8, shift: uint8): uint8
                       {.importc: "_rotr8", header: "".}
-    func builtin_rotr16(value: cushort, shift: cuchar): cushort
+    func builtin_rotr16(value: cushort, shift: uint8): cushort
                        {.importc: "_rotr16", header: "".}
     func builtin_rotr32(value: cuint, shift: cint): cuint
                        {.importc: "_rotr", header: "".}
@@ -738,7 +738,7 @@ when useBuiltinsRotate:
   elif defined(icl):
     # Tested on Intel(R) C++ Intel(R) 64 Compiler Classic Version 2021.1.2 Build
     # 20201208_000000 x64 and x86. Not tested in previous versions.
-    func builtin_rotl8(value: cuchar, shift: cint): cuchar
+    func builtin_rotl8(value: uint8, shift: cint): uint8
                       {.importc: "__rolb", header: "".}
     func builtin_rotl16(value: cushort, shift: cint): cushort
                        {.importc: "__rolw", header: "".}
@@ -748,7 +748,7 @@ when useBuiltinsRotate:
       func builtin_rotl64(value: culonglong, shift: cint): culonglong
                          {.importc: "__rolq", header: "".}
 
-    func builtin_rotr8(value: cuchar, shift: cint): cuchar
+    func builtin_rotr8(value: uint8, shift: cint): uint8
                       {.importc: "__rorb", header: "".}
     func builtin_rotr16(value: cushort, shift: cint): cushort
                        {.importc: "__rorw", header: "".}
@@ -777,7 +777,7 @@ func shiftTypeTo(size: static int, shift: int): auto {.inline.} =
   when (defined(vcc) and (size in [4, 8])) or defined(gcc) or defined(icl):
     cint(shift)
   elif (defined(vcc) and (size in [1, 2])) or (defined(clang) and size == 1):
-    cuchar(shift)
+    uint8(shift)
   elif defined(clang):
     when size == 2:
       cushort(shift)
@@ -802,7 +802,7 @@ func rotateLeftBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) *
     when useBuiltinsRotate:
       const size = sizeof(T)
       when size == 1:
-        builtin_rotl8(value.cuchar, shiftTypeTo(size, shift)).T
+        builtin_rotl8(value.uint8, shiftTypeTo(size, shift)).T
       elif size == 2:
         builtin_rotl16(value.cushort, shiftTypeTo(size, shift)).T
       elif size == 4:
@@ -830,7 +830,7 @@ func rotateRightBits*[T: SomeUnsignedInt](value: T, shift: range[0..(sizeof(T) *
     when useBuiltinsRotate:
       const size = sizeof(T)
       when size == 1:
-        builtin_rotr8(value.cuchar, shiftTypeTo(size, shift)).T
+        builtin_rotr8(value.uint8, shiftTypeTo(size, shift)).T
       elif size == 2:
         builtin_rotr16(value.cushort, shiftTypeTo(size, shift)).T
       elif size == 4:
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 9c3827751..934dd034e 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -503,7 +503,7 @@ when defineSsl:
     ## Retrieve the ssl pointer of `socket`.
     ## Useful for interfacing with `openssl`.
     self.sslHandle
-    
+
   proc raiseSSLError*(s = "") =
     ## Raises a new SSL error.
     if s != "":
@@ -688,7 +688,7 @@ when defineSsl:
     return ctx.getExtraInternal().clientGetPskFunc
 
   proc pskClientCallback(ssl: SslPtr; hint: cstring; identity: cstring;
-      max_identity_len: cuint; psk: ptr cuchar;
+      max_identity_len: cuint; psk: ptr uint8;
       max_psk_len: cuint): cuint {.cdecl.} =
     let ctx = SslContext(context: ssl.SSL_get_SSL_CTX)
     let hintString = if hint == nil: "" else: $hint
@@ -714,7 +714,7 @@ when defineSsl:
   proc serverGetPskFunc*(ctx: SslContext): SslServerGetPskFunc =
     return ctx.getExtraInternal().serverGetPskFunc
 
-  proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr cuchar;
+  proc pskServerCallback(ssl: SslCtx; identity: cstring; psk: ptr uint8;
       max_psk_len: cint): cuint {.cdecl.} =
     let ctx = SslContext(context: ssl.SSL_get_SSL_CTX)
     let pskString = (ctx.serverGetPskFunc)($identity)
diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim
index 283652bb3..ce5660557 100644
--- a/lib/pure/terminal.nim
+++ b/lib/pure/terminal.nim
@@ -761,7 +761,7 @@ template styledWriteLine*(f: File, args: varargs[untyped]) =
   runnableExamples:
     proc error(msg: string) =
       styledWriteLine(stderr, fgRed, "Error: ", resetStyle, msg)
-  
+
   styledWrite(f, args)
   write(f, "\n")
 
diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim
index 4c51577ac..75983e3c6 100644
--- a/lib/std/sysrand.nim
+++ b/lib/std/sysrand.nim
@@ -132,7 +132,7 @@ elif defined(windows):
   type
     PVOID = pointer
     BCRYPT_ALG_HANDLE = PVOID
-    PUCHAR = ptr cuchar
+    PUCHAR = ptr uint8
     NTSTATUS = clong
     ULONG = culong
 
diff --git a/lib/system.nim b/lib/system.nim
index 8aa9bd718..57d620e57 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -1427,8 +1427,8 @@ type # these work for most platforms:
     ## This is the same as the type `long double` in *C*.
     ## This C type is not supported by Nim's code generator.
 
-  cuchar* {.importc: "unsigned char", nodecl.} = uint8
-    ## This is the same as the type `unsigned char` in *C*.
+  cuchar* {.importc: "unsigned char", nodecl, deprecated: "use `char` or `uint8` instead".} = char
+    ## Deprecated: Use `uint8` instead.
   cushort* {.importc: "unsigned short", nodecl.} = uint16
     ## This is the same as the type `unsigned short` in *C*.
   cuint* {.importc: "unsigned int", nodecl.} = uint32
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index a71dd97d8..d901af244 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -42,7 +42,7 @@ type
   PULONG_PTR* = ptr uint
   HDC* = Handle
   HGLRC* = Handle
-  BYTE* = cuchar
+  BYTE* = uint8
 
   SECURITY_ATTRIBUTES* {.final, pure.} = object
     nLength*: int32
@@ -717,7 +717,7 @@ const
   FILE_WRITE_DATA* = 0x00000002 # file & pipe
 
 # Error Constants
-const 
+const
   ERROR_FILE_NOT_FOUND* = 2 ## https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
   ERROR_PATH_NOT_FOUND* = 3
   ERROR_ACCESS_DENIED* = 5
diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim
index 1a8893957..c756fa938 100644
--- a/lib/wrappers/openssl.nim
+++ b/lib/wrappers/openssl.nim
@@ -520,16 +520,16 @@ proc OPENSSL_sk_num*(stack: PSTACK): int {.cdecl, dynlib: DLLSSLName, importc.}
 proc OPENSSL_sk_value*(stack: PSTACK, index: int): pointer {.cdecl,
     dynlib: DLLSSLName, importc.}
 
-proc d2i_X509*(px: ptr PX509, i: ptr ptr cuchar, len: cint): PX509 {.cdecl,
+proc d2i_X509*(px: ptr PX509, i: ptr ptr uint8, len: cint): PX509 {.cdecl,
     dynlib: DLLUtilName, importc.}
 
-proc i2d_X509*(cert: PX509; o: ptr ptr cuchar): cint {.cdecl,
+proc i2d_X509*(cert: PX509; o: ptr ptr uint8): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
 
 proc d2i_X509*(b: string): PX509 =
   ## decode DER/BER bytestring into X.509 certificate struct
   var bb = b.cstring
-  let i = cast[ptr ptr cuchar](addr bb)
+  let i = cast[ptr ptr uint8](addr bb)
   let ret = d2i_X509(addr result, i, b.len.cint)
   if ret.isNil:
     raise newException(Exception, "X.509 certificate decoding failed")
@@ -539,7 +539,7 @@ proc i2d_X509*(cert: PX509): string =
   let encoded_length = i2d_X509(cert, nil)
   result = newString(encoded_length)
   var q = result.cstring
-  let o = cast[ptr ptr cuchar](addr q)
+  let o = cast[ptr ptr uint8](addr q)
   let length = i2d_X509(cert, o)
   if length.int <= 0:
     raise newException(Exception, "X.509 certificate encoding failed")
@@ -599,11 +599,11 @@ proc SSL_CTX_set_tlsext_servername_arg*(ctx: SslCtx, arg: pointer): int =
 
 type
   PskClientCallback* = proc (ssl: SslPtr;
-    hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr cuchar;
+    hint: cstring; identity: cstring; max_identity_len: cuint; psk: ptr uint8;
     max_psk_len: cuint): cuint {.cdecl.}
 
   PskServerCallback* = proc (ssl: SslPtr;
-    identity: cstring; psk: ptr cuchar; max_psk_len: cint): cuint {.cdecl.}
+    identity: cstring; psk: ptr uint8; max_psk_len: cint): cuint {.cdecl.}
 
 proc SSL_CTX_set_psk_client_callback*(ctx: SslCtx; callback: PskClientCallback) {.cdecl, dynlib: DLLSSLName, importc.}
   ## Set callback called when OpenSSL needs PSK (for client).
@@ -672,13 +672,13 @@ proc PEM_read_bio_RSAPublicKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: po
     dynlib: DLLUtilName, importc.}
 proc PEM_read_bio_RSAPrivateKey*(bp: BIO, x: ptr PRSA, cb: pem_password_cb, u: pointer): PRSA {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_private_encrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_private_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_public_encrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_public_encrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_private_decrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_private_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
-proc RSA_public_decrypt*(flen: cint, fr: ptr cuchar, to: ptr cuchar, rsa: PRSA, padding: PaddingType): cint {.cdecl,
+proc RSA_public_decrypt*(flen: cint, fr: ptr uint8, to: ptr uint8, rsa: PRSA, padding: PaddingType): cint {.cdecl,
     dynlib: DLLUtilName, importc.}
 proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.}
 proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.}
@@ -744,8 +744,8 @@ type
 proc md5_Init*(c: var MD5_CTX): cint{.importc: "MD5_Init".}
 proc md5_Update*(c: var MD5_CTX; data: pointer; len: csize_t): cint{.importc: "MD5_Update".}
 proc md5_Final*(md: cstring; c: var MD5_CTX): cint{.importc: "MD5_Final".}
-proc md5*(d: ptr cuchar; n: csize_t; md: ptr cuchar): ptr cuchar{.importc: "MD5".}
-proc md5_Transform*(c: var MD5_CTX; b: ptr cuchar){.importc: "MD5_Transform".}
+proc md5*(d: ptr uint8; n: csize_t; md: ptr uint8): ptr uint8{.importc: "MD5".}
+proc md5_Transform*(c: var MD5_CTX; b: ptr uint8){.importc: "MD5_Transform".}
 {.pop.}
 
 from strutils import toHex, toLowerAscii
diff --git a/tests/stdlib/topenssl.nim b/tests/stdlib/topenssl.nim
index 75e1ba868..4c38b4216 100644
--- a/tests/stdlib/topenssl.nim
+++ b/tests/stdlib/topenssl.nim
@@ -12,8 +12,8 @@ proc rsaPublicEncrypt(fr: string): string =
   doAssert rsa != nil
   doAssert BIO_free(bio) >= 0
   result = newString(RSA_size(rsa))
-  let frdata = cast[ptr cuchar](fr.cstring)
-  var todata = cast[ptr cuchar](result.cstring)
+  let frdata = cast[ptr uint8](fr.cstring)
+  var todata = cast[ptr uint8](result.cstring)
   doAssert RSA_public_encrypt(fr.len.cint, frdata, todata, rsa, RSA_PKCS1_PADDING) != -1
   RSA_free(rsa)
 
@@ -26,8 +26,8 @@ proc rasPrivateDecrypt(fr: string): string =
   doAssert BIO_free(bio) >= 0
   let rsaLen = RSA_size(rsa)
   result = newString(rsaLen)
-  let frdata = cast[ptr cuchar](fr.cstring)
-  var todata = cast[ptr cuchar](result.cstring)
+  let frdata = cast[ptr uint8](fr.cstring)
+  var todata = cast[ptr uint8](result.cstring)
   let lenOrig = RSA_private_decrypt(rsaLen, frdata, todata, rsa, RSA_PKCS1_PADDING)
   doAssert lenOrig >= 0 and lenOrig < result.len
   doAssert result[lenOrig] == '\0'

From 25efb5386293540b0542833625d3fb6e22f3cfbc Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Fri, 16 Jul 2021 18:29:49 +0200
Subject: [PATCH 0228/1654] make dragonbox opt-in via -d:nimFpRoundtrips
 (#18504)

* make dragonbox opt-in via -d:nimFpRoundtrips

* make tests green again

* make tests green again
---
 changelog.md                    | 12 ++++++------
 lib/system/formatfloat.nim      |  2 +-
 lib/system/strmantle.nim        |  2 +-
 tests/errmsgs/treportunused.nim | 14 ++++++++------
 tests/float/nim.cfg             |  1 +
 tests/float/tfloats.nim         |  4 ++++
 tests/stdlib/tjson.nim          |  6 +++---
 tests/stdlib/tjsonutils.nim     | 21 +++++++++++----------
 8 files changed, 35 insertions(+), 27 deletions(-)
 create mode 100644 tests/float/nim.cfg

diff --git a/changelog.md b/changelog.md
index 568ff0bba..289202587 100644
--- a/changelog.md
+++ b/changelog.md
@@ -76,9 +76,6 @@
 - `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that
   `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json.
 
-- `system.addFloat` now uses the "Dragonbox" algorithm, which ensures correct roundtrips of floating point
-  numbers, that the minimum length representation of a floating point number is used and correct rounding.
-  Use `-d:nimLegacyAddFloat` for a transition period.
 
 - `strformat` is now part of `include std/prelude`.
 
@@ -102,6 +99,12 @@
   added support for parenthesized expressions.
   added support for const string's instead of just string literals
 
+
+- `system.addFloat` and `system.$` now can produce string representations of floating point numbers
+  that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has
+  to be enabled via `-d:nimFpRoundtrips`. It is expected that this behavior becomes the new default
+  in upcoming versions.
+
 - Fixed buffer overflow bugs in `net`
 
 - Exported `sslHandle` from `net` and `asyncnet`.
@@ -116,8 +119,6 @@
   the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
   now also need to ship `cacert.pem` with your `.exe` file.
 
-- Make `{.requiresInit.}` pragma to work for `distinct` types.
-
 - `typetraits`:
   `distinctBase` now is identity instead of error for non distinct types.
   Added `enumLen` to return the number of elements in an enum.
@@ -468,7 +469,6 @@
   with `--styleCheck:error` or `--styleCheck:hint`.
 
 
-
 ## Tool changes
 
 - Latex doc generation is revised: output `.tex` files should be compiled
diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim
index 41bc56adc..772f0a848 100644
--- a/lib/system/formatfloat.nim
+++ b/lib/system/formatfloat.nim
@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-when not defined(nimLegacyAddFloat) and not defined(nimscript) and
+when defined(nimFpRoundtrips) and not defined(nimscript) and
     not defined(js) and defined(nimHasDragonBox):
   import dragonbox
 
diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim
index 63e341763..041272175 100644
--- a/lib/system/strmantle.nim
+++ b/lib/system/strmantle.nim
@@ -97,7 +97,7 @@ proc nimFloatToStr(f: float): string {.compilerproc.} =
   result = newStringOfCap(8)
   result.addFloat f
 
-when not defined(nimLegacyAddFloat) and not defined(nimscript) and
+when defined(nimFpRoundtrips) and not defined(nimscript) and
     not defined(js) and defined(nimHasDragonBox):
   import schubfach
 
diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim
index 65e00163b..f9b7c3d11 100644
--- a/tests/errmsgs/treportunused.nim
+++ b/tests/errmsgs/treportunused.nim
@@ -13,12 +13,12 @@ treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUse
 treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed]
-treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
-treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 '''
 action: compile
 """
 
+#treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
+#treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 # bug #9764
 iterator s1(a:string): int = discard
 iterator s2(): int = discard
@@ -32,7 +32,9 @@ var s9: int
 type s10 = object
 type s11 = type(1.2)
 
-# https://github.com/nim-lang/Nim/issues/14407
-let
-  `v0.99` = "0.99"
-  `v0.99.99` = "0.99.99"
+when false:
+  # enabled again when Nim bootstraps with -d:nimFpRoundtrips
+  # https://github.com/nim-lang/Nim/issues/14407
+  let
+    `v0.99` = "0.99"
+    `v0.99.99` = "0.99.99"
diff --git a/tests/float/nim.cfg b/tests/float/nim.cfg
new file mode 100644
index 000000000..d27bbf43b
--- /dev/null
+++ b/tests/float/nim.cfg
@@ -0,0 +1 @@
+-d:nimFpRoundtrips
diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim
index 406ddb6d9..30d9c50d6 100644
--- a/tests/float/tfloats.nim
+++ b/tests/float/tfloats.nim
@@ -66,6 +66,8 @@ template main =
     var a = 1.1'f32
     doAssert $a == "1.1", $a # was failing
 
+proc runtimeOnlyTests =
+  # enable for 'static' once -d:nimFpRoundtrips became the default
   block: # bug #7717
     proc test(f: float) =
       let f2 = $f
@@ -82,3 +84,5 @@ template main =
 
 static: main()
 main()
+
+runtimeOnlyTests()
diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim
index 289ef9d05..000f72038 100644
--- a/tests/stdlib/tjson.nim
+++ b/tests/stdlib/tjson.nim
@@ -303,7 +303,7 @@ let jsonNode = %*mynode
 doAssert $jsonNode == """{"kind":"P","pChildren":[{"kind":"Text","textStr":"mychild"},{"kind":"Br"}]}"""
 doAssert $jsonNode.to(ContentNode) == """(kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)])"""
 
-block: # bug #17383
+when defined(nimFpRoundtrips): # bug #17383
   testRoundtrip(int32.high): "2147483647"
   testRoundtrip(uint32.high): "4294967295"
   when int.sizeof == 4:
@@ -316,7 +316,7 @@ block: # bug #17383
     testRoundtrip(int64.high): "9223372036854775807"
     testRoundtrip(uint64.high): "18446744073709551615"
 
-block: # bug #18007
+when defined(nimFpRoundtrips): # bug #18007
   testRoundtrip([NaN, Inf, -Inf, 0.0, -0.0, 1.0]): """["nan","inf","-inf",0.0,-0.0,1.0]"""
   # pending https://github.com/nim-lang/Nim/issues/18025 use:
   # testRoundtrip([float32(NaN), Inf, -Inf, 0.0, -0.0, 1.0])
@@ -332,7 +332,7 @@ block: # bug #18007
     testRoundtripVal(0.0): "0.0"
     testRoundtripVal(-0.0): "-0.0"
 
-block: # bug #15397, bug #13196
+when defined(nimFpRoundtrips): # bug #15397, bug #13196
   testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
   testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
 
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index b0c444cbf..3e9c422e0 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -44,7 +44,7 @@ proc `$`(a: MyEnum): string =
   if a == me2: "me2Modif"
   else: system.`$`(a)
 
-template fn() = 
+template fn() =
   block: # toJson, jsonTo
     type Foo = distinct float
     testRoundtrip('x', """120""")
@@ -161,15 +161,16 @@ template fn() =
     doAssert b[2].signbit
     doAssert not b[3].signbit
 
-  block: # bug #15397, bug #13196
-    let a = 0.1
-    let x = 0.12345678901234567890123456789
-    let b = (a + 0.2, 0.3, x)
-    testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
+  when defined(nimFpRoundtrips):
+    block: # bug #15397, bug #13196
+      let a = 0.1
+      let x = 0.12345678901234567890123456789
+      let b = (a + 0.2, 0.3, x)
+      testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
 
-    testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
-    testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
-    testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
+      testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
+      testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
+      testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
 
   block: # case object
     type Foo = object
@@ -433,7 +434,7 @@ template fn() =
           """{"b": true, "bt": false, "btf": "test"}"""
         testRoundtrip(Variant(b: true, bt: true, btt: 'c')):
           """{"b": true, "bt": true, "btt": "c"}"""
-        
+
         # TODO: Add additional tests with missing and extra JSON keys, both when
         # allowed and forbidden analogous to the tests for the not nested
         # variant objects.

From 923a1c6ea7d9f45b6389680717692690218228fb Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 16 Jul 2021 22:54:47 -0700
Subject: [PATCH 0229/1654] fix nativeToUnixPath (#18501)

---
 lib/std/private/globs.nim | 11 ++++++++---
 tests/stdlib/tglobs.nim   | 20 ++++++++++++++++++++
 2 files changed, 28 insertions(+), 3 deletions(-)
 create mode 100644 tests/stdlib/tglobs.nim

diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim
index 55f6a40bd..b3726c9c3 100644
--- a/lib/std/private/globs.nim
+++ b/lib/std/private/globs.nim
@@ -45,10 +45,15 @@ iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = n
 
 proc nativeToUnixPath*(path: string): string =
   # pending https://github.com/nim-lang/Nim/pull/13265
-  doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive
+  result = path
+  when defined(windows):
+    if path.len >= 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':':
+      result[0] = '/'
+      result[1] = path[0]
+      if path.len > 2 and path[2] != '\\':
+        doAssert false, "paths like `C:foo` are currently unsupported, path: " & path
   when DirSep == '\\':
-    result = replace(path, '\\', '/')
-  else: result = path
+    result = replace(result, '\\', '/')
 
 when isMainModule:
   import sugar
diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim
new file mode 100644
index 000000000..739a127f8
--- /dev/null
+++ b/tests/stdlib/tglobs.nim
@@ -0,0 +1,20 @@
+import std/private/globs
+
+template main =
+  when defined(windows):
+    doAssert nativeToUnixPath("C:") == "/C"
+    doAssert nativeToUnixPath(r"D:\") == "/D/"
+    doAssert nativeToUnixPath(r"E:\a") == "/E/a"
+    doAssert nativeToUnixPath(r"E:\a1\") == "/E/a1/"
+    doAssert nativeToUnixPath(r"E:\a1\bc") == "/E/a1/bc"
+    doAssert nativeToUnixPath(r"\a1\bc") == "/a1/bc"
+    doAssert nativeToUnixPath(r"a1\bc") == "a1/bc"
+    doAssert nativeToUnixPath("a1") == "a1"
+    doAssert nativeToUnixPath("") == ""
+    doAssert nativeToUnixPath(".") == "."
+    doAssert nativeToUnixPath("..") == ".."
+    doAssert nativeToUnixPath(r"..\") == "../"
+    doAssert nativeToUnixPath(r"..\..\.\") == "../.././"
+
+static: main()
+main()

From ac5435ecd0a08d1a0bdd9d6d6cd89577634ecb0f Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Sun, 18 Jul 2021 10:49:03 +0200
Subject: [PATCH 0230/1654] Make error message for empty new-styled concept
 more descriptive (#18506)

* Allow empty new-styled concept

Slightly improve error messages

* Make empty new-styled concepts an error
---
 compiler/concepts.nim    | 15 +++++++--------
 compiler/parser.nim      |  2 ++
 tests/concepts/tspec.nim |  5 ++---
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/compiler/concepts.nim b/compiler/concepts.nim
index de994f1b8..885b69c60 100644
--- a/compiler/concepts.nim
+++ b/compiler/concepts.nim
@@ -11,8 +11,7 @@
 ## for details. Note this is a first implementation and only the "Concept matching"
 ## section has been implemented.
 
-import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer,
-  types, intsets
+import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, intsets
 
 from magicsys import addSonSkipIntLit
 
@@ -23,7 +22,7 @@ const
 ## --------------------------------------
 
 proc declareSelf(c: PContext; info: TLineInfo) =
-  ## adds the magical 'Self' symbols to the current scope.
+  ## Adds the magical 'Self' symbols to the current scope.
   let ow = getCurrOwner(c)
   let s = newSym(skType, getIdent(c.cache, "Self"), nextSymId(c.idgen), ow, info)
   s.typ = newType(tyTypeDesc, nextTypeId(c.idgen), ow)
@@ -32,7 +31,7 @@ proc declareSelf(c: PContext; info: TLineInfo) =
   addDecl(c, s, info)
 
 proc isSelf*(t: PType): bool {.inline.} =
-  ## is this the magical 'Self' type?
+  ## Is this the magical 'Self' type?
   t.kind == tyTypeDesc and tfPacked in t.flags
 
 proc makeTypeDesc*(c: PContext, typ: PType): PType =
@@ -45,8 +44,8 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 
 proc semConceptDecl(c: PContext; n: PNode): PNode =
   ## Recursive helper for semantic checking for the concept declaration.
-  ## Currently we only support lists of statements containing 'proc'
-  ## declarations and the like.
+  ## Currently we only support (possibly empty) lists of statements
+  ## containing 'proc' declarations and the like.
   case n.kind
   of nkStmtList, nkStmtListExpr:
     result = shallowCopy(n)
@@ -60,7 +59,7 @@ proc semConceptDecl(c: PContext; n: PNode): PNode =
       result[i] = n[i]
     result[^1] = semConceptDecl(c, n[^1])
   else:
-    localError(c.config, n.info, "unexpected construct in the new-styled concept " & renderTree(n))
+    localError(c.config, n.info, "unexpected construct in the new-styled concept: " & renderTree(n))
     result = n
 
 proc semConceptDeclaration*(c: PContext; n: PNode): PNode =
@@ -97,7 +96,7 @@ proc existingBinding(m: MatchCon; key: PType): PType =
 proc conceptMatchNode(c: PContext; n: PNode; m: var MatchCon): bool
 
 proc matchType(c: PContext; f, a: PType; m: var MatchCon): bool =
-  ## the heart of the concept matching process. 'f' is the formal parameter of some
+  ## The heart of the concept matching process. 'f' is the formal parameter of some
   ## routine inside the concept that we're looking for. 'a' is the formal parameter
   ## of a routine that might match.
   const
diff --git a/compiler/parser.nim b/compiler/parser.nim
index b6c01e9c6..d00a17e9f 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -2058,6 +2058,8 @@ proc parseTypeClass(p: var Parser): PNode =
     skipComment(p, result)
   # an initial IND{>} HAS to follow:
   if not realInd(p):
+    if result.isNewStyleConcept:
+      parMessage(p, "routine expected, but found '$1' (empty new-styled concepts are not allowed)", p.tok)
     result.add(p.emptyNode)
   else:
     result.add(parseStmt(p))
diff --git a/tests/concepts/tspec.nim b/tests/concepts/tspec.nim
index 1bca3c11b..52f13a40a 100644
--- a/tests/concepts/tspec.nim
+++ b/tests/concepts/tspec.nim
@@ -7,8 +7,7 @@ discard """
 2
 3
 yes int
-string int
-true'''
+string int'''
   joinable: false
 """
 
@@ -102,5 +101,5 @@ type Monoid = concept
 
 proc z(x: typedesc[int]): int = 0
 
-echo(int is Monoid)
+doAssert int is Monoid
 

From 8091545f95143d520c64b14f88bb5ce2818c893c Mon Sep 17 00:00:00 2001
From: Miran 
Date: Sun, 18 Jul 2021 11:04:19 +0200
Subject: [PATCH 0231/1654] Revert #17398 and #17402 (#18480)

* Revert "followup #17398: `getTempDir`, `getConfigDir` now do not have trailing DirSep (#17402)"

This reverts commit 2356d0603f70cad90f76fa57999054bf5c0a9157.

* Revert "fix #17393 getHomeDir and expandTilde should not include trailing `/` (#17398)"

This reverts commit bebf2ce24a43bef4cde5c90c4010631a1e4a7927.

* fix test
---
 changelog.md                |  3 ---
 lib/pure/os.nim             | 40 ++++++++++++++-----------------------
 tests/stdlib/tos.nim        |  6 +++---
 tests/stdlib/ttempfiles.nim |  2 +-
 4 files changed, 19 insertions(+), 32 deletions(-)

diff --git a/changelog.md b/changelog.md
index 289202587..16f2192dc 100644
--- a/changelog.md
+++ b/changelog.md
@@ -39,9 +39,6 @@
   underlying code is also updated the same way.
 - Custom pragma values have now an API for use in macros.
 
-- In `std/os`, `getHomeDir`, `expandTilde`, `getTempDir`, `getConfigDir` now do not include trailing `DirSep`,
-  unless `-d:nimLegacyHomeDir` is specified (for a transition period).
-
 - On POSIX systems, the default signal handlers used for Nim programs (it's
   used for printing the stacktrace on fatal signals) will now re-raise the
   signal for the OS default handlers to handle.
diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 213da2aed..dadc380e6 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -901,14 +901,9 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1",
   ## * `setCurrentDir proc <#setCurrentDir,string>`_
   runnableExamples:
     assert getHomeDir() == expandTilde("~")
-    # `getHomeDir()` doesn't end in `DirSep` even if `$HOME` (on posix) or
-    # `$USERPROFILE` (on windows) does, unless `-d:nimLegacyHomeDir` is specified.
-    from std/strutils import endsWith
-    assert not getHomeDir().endsWith DirSep
 
-  when defined(windows): result = getEnv("USERPROFILE")
-  else: result = getEnv("HOME")
-  result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
+  when defined(windows): return getEnv("USERPROFILE") & "\\"
+  else: return getEnv("HOME") & "/"
 
 proc getConfigDir*(): string {.rtl, extern: "nos$1",
   tags: [ReadEnvEffect, ReadIOEffect].} =
@@ -917,21 +912,23 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1",
   ## On non-Windows OSs, this proc conforms to the XDG Base Directory
   ## spec. Thus, this proc returns the value of the `XDG_CONFIG_HOME` environment
   ## variable if it is set, otherwise it returns the default configuration
-  ## directory ("~/.config").
+  ## directory ("~/.config/").
+  ##
+  ## An OS-dependent trailing slash is always present at the end of the
+  ## returned string: `\\` on Windows and `/` on all other OSs.
   ##
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getTempDir proc <#getTempDir>`_
-  ## * `getCacheDir proc <#getCacheDir>`_
-  runnableExamples:
-    from std/strutils import endsWith
-    # See `getHomeDir` for behavior regarding trailing DirSep.
-    assert not getConfigDir().endsWith DirSep
+  ## * `expandTilde proc <#expandTilde,string>`_
+  ## * `getCurrentDir proc <#getCurrentDir>`_
+  ## * `setCurrentDir proc <#setCurrentDir,string>`_
   when defined(windows):
     result = getEnv("APPDATA")
   else:
     result = getEnv("XDG_CONFIG_HOME", getEnv("HOME") / ".config")
-  result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
+  result.normalizePathEnd(trailingSep = true)
+
 
 proc getCacheDir*(): string =
   ## Returns the cache directory of the current user for applications.
@@ -1006,11 +1003,9 @@ proc getTempDir*(): string {.rtl, extern: "nos$1",
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
-  ## * `getCacheDir proc <#getCacheDir>`_
-  runnableExamples:
-    from std/strutils import endsWith
-    # See `getHomeDir` for behavior regarding trailing DirSep.
-    assert not getTempDir().endsWith(DirSep)
+  ## * `expandTilde proc <#expandTilde,string>`_
+  ## * `getCurrentDir proc <#getCurrentDir>`_
+  ## * `setCurrentDir proc <#setCurrentDir,string>`_
   const tempDirDefault = "/tmp"
   when defined(tempDir):
     const tempDir {.strdefine.}: string = tempDirDefault
@@ -1031,7 +1026,7 @@ proc getTempDir*(): string {.rtl, extern: "nos$1",
         getTempDirImpl(result)
     if result.len == 0:
       result = tempDirDefault
-  result.normalizePathEnd(trailingSep = defined(nimLegacyHomeDir))
+  normalizePathEnd(result, trailingSep=true)
 
 proc expandTilde*(path: string): string {.
   tags: [ReadEnvEffect, ReadIOEffect].} =
@@ -1041,8 +1036,6 @@ proc expandTilde*(path: string): string {.
   ## Windows: this is still supported despite the Windows platform not having this
   ## convention; also, both ``~/`` and ``~\`` are handled.
   ##
-  ## .. warning:: `~bob` and `~bob/` are not yet handled correctly.
-  ##
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
@@ -1053,9 +1046,6 @@ proc expandTilde*(path: string): string {.
     assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg"
     assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar"
     assert expandTilde("/foo/bar") == "/foo/bar"
-    assert expandTilde("~") == getHomeDir()
-    from std/strutils import endsWith
-    assert not expandTilde("~").endsWith(DirSep)
 
   if len(path) == 0 or path[0] != '~':
     result = path
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index a4f030dde..912558461 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -594,12 +594,12 @@ block getTempDir:
       if existsEnv("TMPDIR"):
         let origTmpDir = getEnv("TMPDIR")
         putEnv("TMPDIR", "/mytmp")
-        doAssert getTempDir() == "/mytmp"
+        doAssert getTempDir() == "/mytmp/"
         delEnv("TMPDIR")
-        doAssert getTempDir() == "/tmp"
+        doAssert getTempDir() == "/tmp/"
         putEnv("TMPDIR", origTmpDir)
       else:
-        doAssert getTempDir() == "/tmp"
+        doAssert getTempDir() == "/tmp/"
 
 block: # getCacheDir
   doAssert getCacheDir().dirExists
diff --git a/tests/stdlib/ttempfiles.nim b/tests/stdlib/ttempfiles.nim
index d740e1850..297410686 100644
--- a/tests/stdlib/ttempfiles.nim
+++ b/tests/stdlib/ttempfiles.nim
@@ -40,7 +40,7 @@ block: # createTempDir
 
     doAssert dirExists(dir1)
     doAssert dir1.lastPathPart.contains(re"^D20210502T100442(\w+).tmp$")
-    doAssert dir1.parentDir == getTempDir()
+    doAssert dir1.parentDir == getTempDir().normalizePathEnd()
 
   block:
     let dir3 = createTempDir(prefix, "_mytmp", ".")

From 3723140044b99fde3f47df4f2e6a3ed4abdaba89 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 18 Jul 2021 02:05:35 -0700
Subject: [PATCH 0232/1654] add NIM_STATIC_ASSERT(CHAR_BIT == 8, "") to
 fail-fast where assumption is violated (#18512)

---
 lib/nimbase.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/nimbase.h b/lib/nimbase.h
index cbd35605b..e480c55fd 100644
--- a/lib/nimbase.h
+++ b/lib/nimbase.h
@@ -325,6 +325,9 @@ typedef unsigned char NIM_BOOL; // best effort
 #endif
 
 NIM_STATIC_ASSERT(sizeof(NIM_BOOL) == 1, ""); // check whether really needed
+NIM_STATIC_ASSERT(CHAR_BIT == 8, "");
+  // fail fast for (rare) environments where this doesn't hold, as some implicit
+  // assumptions would need revisiting (e.g. `uint8` or https://github.com/nim-lang/Nim/pull/18505)
 
 #define NIM_TRUE true
 #define NIM_FALSE false

From adba5eb45e0ae0d370aea4d653a4f00a4c075695 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 18 Jul 2021 02:10:08 -0700
Subject: [PATCH 0233/1654] deprecate strutils.delete and add an overload with
 saner semantics consistent with sequtils.delete; follows #18487 (#18510)

---
 changelog.md               |  2 +-
 lib/pure/strutils.nim      | 36 +++++++++++++++++++++++++++++++-----
 tests/stdlib/tstrutils.nim | 26 +++++++++++++++++++++++++-
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/changelog.md b/changelog.md
index 16f2192dc..6dddd8a01 100644
--- a/changelog.md
+++ b/changelog.md
@@ -346,7 +346,7 @@
 - Added `dom.setInterval`, `dom.clearInterval` overloads.
 
 - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect
-  if the slice is out of bounds.
+  if the slice is out of bounds, likewise with `strutils.delete`.
 
 ## Language changes
 
diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 9e0837942..ac078df4e 100644
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -1476,12 +1476,39 @@ func dedent*(s: string, count: Natural = indentation(s)): string {.rtl,
     doAssert x == "Hello\n  There\n"
   unindent(s, count, " ")
 
-func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} =
-  ## Deletes in `s` (must be declared as `var`) the characters at positions
-  ## `first .. last` (both ends included).
+func delete*(s: var string, slice: Slice[int]) =
+  ## Deletes the items `s[slice]`, raising `IndexDefect` if the slice contains
+  ## elements out of range.
   ##
-  ## This modifies `s` itself, it does not return a copy.
+  ## This operation moves all elements after `s[slice]` in linear time, and
+  ## is the string analog to `sequtils.delete`.
   runnableExamples:
+    var a = "abcde"
+    doAssertRaises(IndexDefect): a.delete(4..5)
+    assert a == "abcde"
+    a.delete(4..4)
+    assert a == "abcd"
+    a.delete(1..2)
+    assert a == "ad"
+    a.delete(1..<1) # empty slice
+    assert a == "ad"
+  when compileOption("boundChecks"):
+    if not (slice.a < s.len and slice.a >= 0 and slice.b < s.len):
+      raise newException(IndexDefect, $(slice: slice, len: s.len))
+  if slice.b >= slice.a:
+    var i = slice.a
+    var j = slice.b + 1
+    var newLen = s.len - j + i
+    # if j < s.len: moveMem(addr s[i], addr s[j], s.len - j) # pending benchmark
+    while i < newLen:
+      s[i] = s[j]
+      inc(i)
+      inc(j)
+    setLen(s, newLen)
+
+func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete", deprecated: "use `delete(s, first..last)`".} =
+  ## Deletes in `s` the characters at positions `first .. last` (both ends included).
+  runnableExamples("--warning:deprecated:off"):
     var a = "abracadabra"
 
     a.delete(4, 5)
@@ -1502,7 +1529,6 @@ func delete*(s: var string, first, last: int) {.rtl, extern: "nsuDelete".} =
     inc(j)
   setLen(s, newLen)
 
-
 func startsWith*(s: string, prefix: char): bool {.inline.} =
   ## Returns true if `s` starts with character `prefix`.
   ##
diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim
index 68ee5812b..234991bdb 100644
--- a/tests/stdlib/tstrutils.nim
+++ b/tests/stdlib/tstrutils.nim
@@ -198,7 +198,30 @@ template main() =
     s.removePrefix("")
     doAssert s == "\r\n\r\nhello"
 
-  block: # delete
+  block: # delete(slice)
+    var s = "0123456789ABCDEFGH"
+    delete(s, 4 .. 5)
+    doAssert s == "01236789ABCDEFGH"
+    delete(s, s.len-1 .. s.len-1)
+    doAssert s == "01236789ABCDEFG"
+    delete(s, 0..0)
+    doAssert s == "1236789ABCDEFG"
+    s = ""
+    doAssertRaises(IndexDefect): delete(s, 0..0)
+    doAssert s == ""
+    s = "abc"
+    doAssertRaises(IndexDefect): delete(s, -1 .. -2)
+    doAssertRaises(IndexDefect): delete(s, 2..3)
+    doAssertRaises(IndexDefect): delete(s, 3..2)
+    delete(s, 2..2)
+    doAssert s == "ab"
+    delete(s, 1..0)
+    doAssert s == "ab"
+    delete(s, 0..0)
+    doAssert s == "b"
+
+  block: # delete(first, last)
+    {.push warning[deprecated]:off.}
     var s = "0123456789ABCDEFGH"
     delete(s, 4, 5)
     doAssert s == "01236789ABCDEFGH"
@@ -206,6 +229,7 @@ template main() =
     doAssert s == "01236789ABCDEFG"
     delete(s, 0, 0)
     doAssert s == "1236789ABCDEFG"
+    {.pop.}
 
   block: # find
     doAssert "0123456789ABCDEFGH".find('A') == 10

From 99c4b69097638aca8377ea3746e52cc19c24ced8 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Sun, 18 Jul 2021 15:16:26 +0200
Subject: [PATCH 0234/1654] fixed system.delete (#18507)

---
 changelog.md             | 13 ++++++++++++-
 lib/system.nim           | 17 ++++++++++-------
 tests/stdlib/tsystem.nim |  8 ++++----
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/changelog.md b/changelog.md
index 6dddd8a01..097beac2c 100644
--- a/changelog.md
+++ b/changelog.md
@@ -4,7 +4,18 @@
 
 ## Changes affecting backward compatibility
 
-- Deprecated `std/mersenne`
+- Deprecated `std/mersenne`.
+
+- `system.delete` had a most surprising behavior when the index passed to it was out of
+  bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so
+  that an index error is produced instead. But be aware that your code might depend on
+  this quirky behavior so a review process is required on your part before you can
+  use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete`
+  switch, it pretends that `system.delete` is deprecated so that it is easier to see
+  where it was used in your code.
+
+  `-d:nimStrictDelete` will become the default in upcoming versions.
+
 
 - `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`.
   Please use `char` or `uint8` instead.
diff --git a/lib/system.nim b/lib/system.nim
index 57d620e57..2790187f4 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -2134,7 +2134,12 @@ const
 import system/dollars
 export dollars
 
-proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} =
+when defined(nimAuditDelete):
+  {.pragma: auditDelete, deprecated: "review this call for out of bounds behavior".}
+else:
+  {.pragma: auditDelete.}
+
+proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, auditDelete.} =
   ## Deletes the item at index `i` by moving all `x[i+1..^1]` items by one position.
   ##
   ## This is an `O(n)` operation.
@@ -2147,12 +2152,10 @@ proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} =
     s.delete(2)
     doAssert s == @[1, 2, 4, 5]
 
-    doAssertRaises(IndexDefect):
-      s.delete(4)
-
-  if i > high(x):
-    # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring
-    raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed")
+  when defined(nimStrictDelete):
+    if i > high(x):
+      # xxx this should call `raiseIndexError2(i, high(x))` after some refactoring
+      raise (ref IndexDefect)(msg: "index out of bounds: '" & $i & "' < '" & $x.len & "' failed")
 
   template defaultImpl =
     let xl = x.len
diff --git a/tests/stdlib/tsystem.nim b/tests/stdlib/tsystem.nim
index b48034043..00be16275 100644
--- a/tests/stdlib/tsystem.nim
+++ b/tests/stdlib/tsystem.nim
@@ -53,8 +53,8 @@ template main =
       var s = @["foo", "bar"]
       s.delete(1)
       doAssert s == @["foo"]
-  
-    block:
+
+    when false:
       var s: seq[string]
       doAssertRaises(IndexDefect):
         s.delete(0)
@@ -66,8 +66,8 @@ template main =
       var s = @["foo"]
       s.delete(0)
       doAssert s == @[]
-  
-    block: # bug #16544: deleting out of bounds index should raise
+
+    when false: # bug #16544: deleting out of bounds index should raise
       var s = @["foo"]
       doAssertRaises(IndexDefect):
         s.delete(1)

From 220b55c5d7c40aa93df7879ebc6c9f71147c3eda Mon Sep 17 00:00:00 2001
From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com>
Date: Sun, 18 Jul 2021 16:29:49 +0300
Subject: [PATCH 0235/1654] attempt to support short commit hashes (#18514)

* attempt to support short commit hashes

Not sure if that's the correct way and what happens when the short hash matches "head"

* need to remove #

* Output needs to be stripped of newlines
---
 tools/atlas/atlas.nim | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
index bc1cc6ae3..371a1c13f 100644
--- a/tools/atlas/atlas.nim
+++ b/tools/atlas/atlas.nim
@@ -123,8 +123,12 @@ proc versionToCommit(d: Dependency): string =
 
   return ""
 
+proc shortToCommit(short: string): string =
+  let (cc, status) = osproc.execCmdEx("git rev-parse " & quoteShell(short))
+  result = if status == 0: strutils.strip(cc) else: ""
+
 proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
-  let (outp, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
+  let (_, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
   if status != 0:
     error(c, p, "could not checkout commit", commit)
 
@@ -169,17 +173,23 @@ proc toName(p: string): PackageName =
 proc needsCommitLookup(commit: string): bool {.inline} =
   '.' in commit or commit == InvalidCommit
 
+proc isShortCommitHash(commit: string): bool {.inline.} =
+  commit.len >= 4 and commit.len < 40
+
 proc checkoutCommit(c: var AtlasContext; w: Dependency) =
   let dir = c.workspace / w.name.string
   withDir dir:
-    if w.commit.len == 0 or cmpIgnoreCase(w.commit, "#head") == 0:
+    if w.commit.len == 0 or cmpIgnoreCase(w.commit, "head") == 0:
       gitPull(c, w.name)
     else:
       let err = isCleanGit(dir)
       if err != "":
         warn c, w.name, err
       else:
-        let requiredCommit = if needsCommitLookup(w.commit): versionToCommit(w) else: w.commit
+        let requiredCommit =
+          if needsCommitLookup(w.commit): versionToCommit(w)
+          elif isShortCommitHash(w.commit): shortToCommit(w.commit)
+          else: w.commit
         let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
         let currentCommit = strutils.strip(cc)
         if requiredCommit == "" or status != 0:
@@ -191,8 +201,9 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
           if currentCommit != requiredCommit:
             # checkout the later commit:
             # git merge-base --is-ancestor  
-            let (mergeBase, status) = osproc.execCmdEx("git merge-base " &
+            let (cc, status) = osproc.execCmdEx("git merge-base " &
                 currentCommit.quoteShell & " " & requiredCommit.quoteShell)
+            let mergeBase = strutils.strip(cc)
             if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
               # conflict resolution: pick the later commit:
               if mergeBase == currentCommit:
@@ -242,7 +253,7 @@ proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
         tokens.add InvalidCommit
       elif tokens.len == 2 and tokens[1].startsWith("#"):
         # Dependencies can also look like 'requires "sdl2#head"
-        var commit = tokens[1]
+        var commit = tokens[1][1 .. ^1]
         tokens[1] = "=="
         tokens.add commit
 

From 488e9c2991cd6f4ae946dd719271ea284a051891 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 19 Jul 2021 03:42:45 -0700
Subject: [PATCH 0236/1654] nim doc now correctly renders deprecated pragmas
 for routines and types (#18515)

---
 compiler/ast.nim                             | 31 ++++++++++++++++++++
 compiler/docgen.nim                          | 15 ++--------
 compiler/renderer.nim                        |  4 +--
 nimdoc/testproject/expected/testproject.html | 18 ++++++++++--
 nimdoc/testproject/expected/testproject.idx  |  1 +
 nimdoc/testproject/expected/theindex.html    |  4 +++
 nimdoc/testproject/testproject.nim           |  3 ++
 7 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/compiler/ast.nim b/compiler/ast.nim
index f96e464af..f1f73dcee 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1168,6 +1168,37 @@ template `[]=`*(n: Indexable, i: int; x: Indexable) = n.sons[i] = x
 template `[]`*(n: Indexable, i: BackwardsIndex): Indexable = n[n.len - i.int]
 template `[]=`*(n: Indexable, i: BackwardsIndex; x: Indexable) = n[n.len - i.int] = x
 
+proc getDeclPragma*(n: PNode): PNode =
+  ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found.
+  ## Currently only supports routineDefs + {nkTypeDef}.
+  case n.kind
+  of routineDefs:
+    if n[pragmasPos].kind != nkEmpty: result = n[pragmasPos]
+  of nkTypeDef:
+    #[
+    type F3*{.deprecated: "x3".} = int
+
+    TypeSection
+      TypeDef
+        PragmaExpr
+          Postfix
+            Ident "*"
+            Ident "F3"
+          Pragma
+            ExprColonExpr
+              Ident "deprecated"
+              StrLit "x3"
+        Empty
+        Ident "int"
+    ]#
+    if n[0].kind == nkPragmaExpr:
+      result = n[0][1]
+  else:
+    # support as needed for `nkIdentDefs` etc.
+    result = nil
+  if result != nil:
+    assert result.kind == nkPragma, $(result.kind, n.kind)
+
 when defined(useNodeIds):
   const nodeIdToDebug* = -1 # 2322968
   var gNodeId: int
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 1ae00e22e..9c75c0a6f 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -764,14 +764,6 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
       result.add(defaultParamSeparator)
       result.add(params)
 
-proc isCallable(n: PNode): bool =
-  ## Returns true if `n` contains a callable node.
-  case n.kind
-  of nkProcDef, nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef,
-    nkConverterDef, nkFuncDef: result = true
-  else:
-    result = false
-
 proc docstringSummary(rstText: string): string =
   ## Returns just the first line or a brief chunk of text from a rst string.
   ##
@@ -862,9 +854,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       break
     plainName.add(literal)
 
-  var pragmaNode: PNode = nil
-  if n.isCallable and n[pragmasPos].kind != nkEmpty:
-    pragmaNode = findPragma(n[pragmasPos], wDeprecated)
+  var pragmaNode = getDeclPragma(n)
+  if pragmaNode != nil: pragmaNode = findPragma(pragmaNode, wDeprecated)
 
   inc(d.id)
   let
@@ -920,7 +911,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   # because it doesn't include object fields or documentation comments. So we
   # use the plain one for callable elements, and the complex for the rest.
   var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": "
-  if n.isCallable: linkTitle.add(xmltree.escape(plainName.strip))
+  if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip))
   else: linkTitle.add(xmltree.escape(complexSymbol.strip))
 
   setIndexTerm(d[], external, symbolOrId, name, linkTitle,
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index ac3f47948..3346e629c 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -593,8 +593,8 @@ proc isHideable(config: ConfigRef, n: PNode): bool =
   # xxx compare `ident` directly with `getIdent(cache, wRaises)`, but
   # this requires a `cache`.
   case n.kind
-  of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern"]
-  of nkIdent: result = n.ident.s in ["gcsafe"]
+  of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern", "deprecated"]
+  of nkIdent: result = n.ident.s in ["gcsafe", "deprecated"]
   else: result = false
 
 proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 65d9f83f5..7e5a685d2 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -107,7 +107,9 @@ window.addEventListener('DOMContentLoaded', main);
 
  • 
   Types
   
      
-      
    • FooBuzz
      • 
+  
    • A
      • 
   
    • 
 
      Types
      
 
      
+
+
      FooBuzz {....deprecated: "FooBuzz msg".} = int
      
+
      
+  
      
+    Deprecated: FooBuzz msg
+  
      
+
+
+
+
      
 
 
      A {.inject.} = enum
   aA
      
@@ -544,7 +556,7 @@ This should be visible.
 
 
 
-
      proc baz[T](a, b: T): T {.deprecated.}
      
+
      proc baz[T](a, b: T): T {....deprecated.}
      
 
      
   
      
     Deprecated 
@@ -554,7 +566,7 @@ This is deprecated without message.
 
 
      
 
-
      proc buzz[T](a, b: T): T {.deprecated: "since v0.20".}
      
+
      proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
      
 
      
   
      
     Deprecated: since v0.20
diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx
index a8cdc3be2..4d0e63c0c 100644
--- a/nimdoc/testproject/expected/testproject.idx
+++ b/nimdoc/testproject/expected/testproject.idx
@@ -6,6 +6,7 @@ C_D	testproject.html#C_D	testproject: C_D
 bar	testproject.html#bar,T,T	testproject: bar[T](a, b: T): T	
 baz	testproject.html#baz,T,T	testproject: baz[T](a, b: T): T	
 buzz	testproject.html#buzz,T,T	testproject: buzz[T](a, b: T): T	
+FooBuzz	testproject.html#FooBuzz	testproject: FooBuzz	
 aVariable	testproject.html#aVariable	testproject: aVariable	
 A	testproject.html#A	testproject: A	
 B	testproject.html#B	testproject: B	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index 55f7f33c9..cfaf6856d 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -179,6 +179,10 @@ window.addEventListener('DOMContentLoaded', main);
 
    • testproject: foo(a, b: SomeType)
      • 
           
    
+
    FooBuzz:
    
 
    fromUtils1:
      
 
    • testproject: fromUtils1(): int
      • 
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index b88b41098..57960c69f 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -39,6 +39,9 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} =
   ## This is deprecated with a message.
   result = a + b
 
+type
+  FooBuzz* {.deprecated: "FooBuzz msg".} = int
+
 import std/macros
 
 var aVariable*: array[1, int]

From bdfee32dc8e81dd6c727d711a7b1c1fad1728383 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Mon, 19 Jul 2021 18:43:31 +0800
Subject: [PATCH 0237/1654] close #18489 (#18524)

---
 tests/stdlib/tstrtabs2.nim | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 tests/stdlib/tstrtabs2.nim

diff --git a/tests/stdlib/tstrtabs2.nim b/tests/stdlib/tstrtabs2.nim
new file mode 100644
index 000000000..cb534f198
--- /dev/null
+++ b/tests/stdlib/tstrtabs2.nim
@@ -0,0 +1,15 @@
+discard """
+  targets: "c cpp js"
+"""
+
+import std/strtabs
+
+macro m =
+  var t = {"name": "John"}.newStringTable
+  doAssert t["name"] == "John"
+
+block:
+  var t = {"name": "John"}.newStringTable
+  doAssert t["name"] == "John"
+
+m()

From 1e7a10001a8f823dddabf10b59e0ef9893716787 Mon Sep 17 00:00:00 2001
From: ynfle <23086821+ynfle@users.noreply.github.com>
Date: Mon, 19 Jul 2021 13:44:06 +0300
Subject: [PATCH 0238/1654] Docs(manual): add clarification for default params
 (#18522)

* Docs(manual): add clarification for default params

* Docs(manual): fix wording to refer to call time

* Docs(manual): Clarify default parameter wording

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 doc/manual.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/manual.rst b/doc/manual.rst
index 34781141e..f0bd85638 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -3543,7 +3543,8 @@ separation of types and subsequent identifiers more distinct.
   proc foo(a; b: int; c, d: bool): int
 
 A parameter may be declared with a default value which is used if the caller
-does not provide a value for the argument.
+does not provide a value for the argument. The value will be reevaluated
+every time the function is called.
 
 .. code-block:: nim
   # b is optional with 47 as its default value

From 73ce40aaf71196faa08072216bd826a94b5d40b1 Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Mon, 19 Jul 2021 12:45:13 +0200
Subject: [PATCH 0239/1654] Revert #17321 (#18521)

Use better names in example
---
 lib/pure/collections/lists.nim | 270 ++++++++++++++++-----------------
 1 file changed, 135 insertions(+), 135 deletions(-)

diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim
index c83be7aa3..af53bb68b 100644
--- a/lib/pure/collections/lists.nim
+++ b/lib/pure/collections/lists.nim
@@ -19,15 +19,15 @@
 ##
 ## ## Lists
 runnableExamples:
-  var l = initDoublyLinkedList[int]()
+  var list = initDoublyLinkedList[int]()
   let
     a = newDoublyLinkedNode[int](3)
     b = newDoublyLinkedNode[int](7)
     c = newDoublyLinkedNode[int](9)
 
-  l.add(a)
-  l.add(b)
-  l.prepend(c)
+  list.add(a)
+  list.add(b)
+  list.prepend(c)
 
   assert a.next == b
   assert a.prev == c
@@ -38,15 +38,15 @@ runnableExamples:
 
 ## ## Rings
 runnableExamples:
-  var l = initSinglyLinkedRing[int]()
+  var ring = initSinglyLinkedRing[int]()
   let
     a = newSinglyLinkedNode[int](3)
     b = newSinglyLinkedNode[int](7)
     c = newSinglyLinkedNode[int](9)
 
-  l.add(a)
-  l.add(b)
-  l.prepend(c)
+  ring.add(a)
+  ring.add(b)
+  ring.prepend(c)
 
   assert c.next == a
   assert a.next == b
@@ -189,21 +189,21 @@ func toDoublyLinkedList*[T](elems: openArray[T]): DoublyLinkedList[T] {.since: (
     result.add(elem)
 
 template itemsListImpl() {.dirty.} =
-  var it = list.head
+  var it = L.head
   while it != nil:
     yield it.value
     it = it.next
 
 template itemsRingImpl() {.dirty.} =
-  var it = ring.head
+  var it = L.head
   if it != nil:
     while true:
       yield it.value
       it = it.next
-      if it == ring.head: break
+      if it == L.head: break
 
-iterator items*[T](list: SomeLinkedList[T]): T =
-  ## Yields every value of `list`.
+iterator items*[T](L: SomeLinkedList[T]): T =
+  ## Yields every value of `L`.
   ##
   ## **See also:**
   ## * `mitems iterator <#mitems.i,SomeLinkedList[T]>`_
@@ -218,8 +218,8 @@ iterator items*[T](list: SomeLinkedList[T]): T =
 
   itemsListImpl()
 
-iterator items*[T](ring: SomeLinkedRing[T]): T =
-  ## Yields every value of `ring`.
+iterator items*[T](L: SomeLinkedRing[T]): T =
+  ## Yields every value of `L`.
   ##
   ## **See also:**
   ## * `mitems iterator <#mitems.i,SomeLinkedRing[T]>`_
@@ -234,8 +234,8 @@ iterator items*[T](ring: SomeLinkedRing[T]): T =
 
   itemsRingImpl()
 
-iterator mitems*[T](list: var SomeLinkedList[T]): var T =
-  ## Yields every value of `list` so that you can modify it.
+iterator mitems*[T](L: var SomeLinkedList[T]): var T =
+  ## Yields every value of `L` so that you can modify it.
   ##
   ## **See also:**
   ## * `items iterator <#items.i,SomeLinkedList[T]>`_
@@ -251,8 +251,8 @@ iterator mitems*[T](list: var SomeLinkedList[T]): var T =
 
   itemsListImpl()
 
-iterator mitems*[T](ring: var SomeLinkedRing[T]): var T =
-  ## Yields every value of `ring` so that you can modify it.
+iterator mitems*[T](L: var SomeLinkedRing[T]): var T =
+  ## Yields every value of `L` so that you can modify it.
   ##
   ## **See also:**
   ## * `items iterator <#items.i,SomeLinkedRing[T]>`_
@@ -268,7 +268,7 @@ iterator mitems*[T](ring: var SomeLinkedRing[T]): var T =
 
   itemsRingImpl()
 
-iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] =
+iterator nodes*[T](L: SomeLinkedList[T]): SomeLinkedNode[T] =
   ## Iterates over every node of `x`. Removing the current node from the
   ## list during traversal is supported.
   ##
@@ -287,13 +287,13 @@ iterator nodes*[T](list: SomeLinkedList[T]): SomeLinkedNode[T] =
         x.value = 5 * x.value - 1
     assert $a == "[49, 99, 199, 249]"
 
-  var it = list.head
+  var it = L.head
   while it != nil:
     let nxt = it.next
     yield it
     it = nxt
 
-iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] =
+iterator nodes*[T](L: SomeLinkedRing[T]): SomeLinkedNode[T] =
   ## Iterates over every node of `x`. Removing the current node from the
   ## list during traversal is supported.
   ##
@@ -312,27 +312,27 @@ iterator nodes*[T](ring: SomeLinkedRing[T]): SomeLinkedNode[T] =
         x.value = 5 * x.value - 1
     assert $a == "[49, 99, 199, 249]"
 
-  var it = ring.head
+  var it = L.head
   if it != nil:
     while true:
       let nxt = it.next
       yield it
       it = nxt
-      if it == ring.head: break
+      if it == L.head: break
 
-proc `$`*[T](l: SomeLinkedCollection[T]): string =
+proc `$`*[T](L: SomeLinkedCollection[T]): string =
   ## Turns a list into its string representation for logging and printing.
   runnableExamples:
     let a = [1, 2, 3, 4].toSinglyLinkedList
     assert $a == "[1, 2, 3, 4]"
 
   result = "["
-  for x in nodes(l):
+  for x in nodes(L):
     if result.len > 1: result.add(", ")
     result.addQuoted(x.value)
   result.add("]")
 
-proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
+proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
   ## Searches in the list for a value. Returns `nil` if the value does not
   ## exist.
   ##
@@ -343,10 +343,10 @@ proc find*[T](l: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =
     assert a.find(9).value == 9
     assert a.find(1) == nil
 
-  for x in nodes(l):
+  for x in nodes(L):
     if x.value == value: return x
 
-proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} =
+proc contains*[T](L: SomeLinkedCollection[T], value: T): bool {.inline.} =
   ## Searches in the list for a value. Returns `false` if the value does not
   ## exist, `true` otherwise. This allows the usage of the `in` and `notin`
   ## operators.
@@ -360,7 +360,7 @@ proc contains*[T](l: SomeLinkedCollection[T], value: T): bool {.inline.} =
     assert(not a.contains(1))
     assert 2 notin a
 
-  result = find(l, value) != nil
+  result = find(L, value) != nil
 
 proc prepend*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} =
   ## Prepends a shallow copy of `b` to the beginning of `a`.
@@ -411,8 +411,8 @@ proc prependMoved*[T: SomeLinkedList](a, b: var T) {.since: (1, 5, 1).} =
     (b, a) = (a, b)
   else: swap a, b
 
-proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} =
-  ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value
@@ -426,14 +426,14 @@ proc add*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]) {.inline.} =
     assert a.contains(9)
 
   n.next = nil
-  if list.tail != nil:
-    assert(list.tail.next == nil)
-    list.tail.next = n
-  list.tail = n
-  if list.head == nil: list.head = n
+  if L.tail != nil:
+    assert(L.tail.next == nil)
+    L.tail.next = n
+  L.tail = n
+  if L.head == nil: L.head = n
 
-proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
-  ## Appends (adds to the end) a value to `list`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedList[T], value: T) {.inline.} =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],T>`_ for appending a value
@@ -446,11 +446,11 @@ proc add*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
     a.add(8)
     assert a.contains(9)
 
-  add(list, newSinglyLinkedNode(value))
+  add(L, newSinglyLinkedNode(value))
 
-proc prepend*[T](list: var SinglyLinkedList[T],
+proc prepend*[T](L: var SinglyLinkedList[T],
                  n: SinglyLinkedNode[T]) {.inline.} =
-  ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1).
+  ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_
@@ -463,12 +463,12 @@ proc prepend*[T](list: var SinglyLinkedList[T],
     a.prepend(n)
     assert a.contains(9)
 
-  n.next = list.head
-  list.head = n
-  if list.tail == nil: list.tail = n
+  n.next = L.head
+  L.head = n
+  if L.tail == nil: L.tail = n
 
-proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
-  ## Prepends (adds to the beginning) a node to `list`. Efficiency: O(1).
+proc prepend*[T](L: var SinglyLinkedList[T], value: T) {.inline.} =
+  ## Prepends (adds to the beginning) a node to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedList[T],SinglyLinkedNode[T]>`_
@@ -482,7 +482,7 @@ proc prepend*[T](list: var SinglyLinkedList[T], value: T) {.inline.} =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(list, newSinglyLinkedNode(value))
+  prepend(L, newSinglyLinkedNode(value))
 
 func copy*[T](a: SinglyLinkedList[T]): SinglyLinkedList[T] {.since: (1, 5, 1).} =
   ## Creates a shallow copy of `a`.
@@ -540,8 +540,8 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} =
     b.head = nil
     b.tail = nil
 
-proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
-  ## Appends (adds to the end) a node `n` to `list`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],T>`_ for appending a value
@@ -557,15 +557,15 @@ proc add*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
     assert a.contains(9)
 
   n.next = nil
-  n.prev = list.tail
-  if list.tail != nil:
-    assert(list.tail.next == nil)
-    list.tail.next = n
-  list.tail = n
-  if list.head == nil: list.head = n
+  n.prev = L.tail
+  if L.tail != nil:
+    assert(L.tail.next == nil)
+    L.tail.next = n
+  L.tail = n
+  if L.head == nil: L.head = n
 
-proc add*[T](list: var DoublyLinkedList[T], value: T) =
-  ## Appends (adds to the end) a value to `list`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedList[T], value: T) =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_
@@ -581,10 +581,10 @@ proc add*[T](list: var DoublyLinkedList[T], value: T) =
     a.add(8)
     assert a.contains(9)
 
-  add(list, newDoublyLinkedNode(value))
+  add(L, newDoublyLinkedNode(value))
 
-proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
-  ## Prepends (adds to the beginning) a node `n` to `list`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
+  ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_
@@ -600,15 +600,15 @@ proc prepend*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
     assert a.contains(9)
 
   n.prev = nil
-  n.next = list.head
-  if list.head != nil:
-    assert(list.head.prev == nil)
-    list.head.prev = n
-  list.head = n
-  if list.tail == nil: list.tail = n
+  n.next = L.head
+  if L.head != nil:
+    assert(L.head.prev == nil)
+    L.head.prev = n
+  L.head = n
+  if L.tail == nil: L.tail = n
 
-proc prepend*[T](list: var DoublyLinkedList[T], value: T) =
-  ## Prepends (adds to the beginning) a value to `list`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedList[T], value: T) =
+  ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedList[T],DoublyLinkedNode[T]>`_
@@ -624,7 +624,7 @@ proc prepend*[T](list: var DoublyLinkedList[T], value: T) =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(list, newDoublyLinkedNode(value))
+  prepend(L, newDoublyLinkedNode(value))
 
 func copy*[T](a: DoublyLinkedList[T]): DoublyLinkedList[T] {.since: (1, 5, 1).} =
   ## Creates a shallow copy of `a`.
@@ -705,9 +705,9 @@ proc add*[T: SomeLinkedList](a: var T, b: T) {.since: (1, 5, 1).} =
   var tmp = b.copy
   a.addMoved(tmp)
 
-proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} =
-  ## Removes a node `n` from `list`.
-  ## Returns `true` if `n` was found in `list`.
+proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.discardable.} =
+  ## Removes a node `n` from `L`.
+  ## Returns `true` if `n` was found in `L`.
   ## Efficiency: O(n); the list is traversed until `n` is found.
   ## Attempting to remove an element not contained in the list is a no-op.
   ## When the list is cyclic, the cycle is preserved after removal.
@@ -728,12 +728,12 @@ proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.d
         ai
     assert s == [2, 2, 2, 2]
 
-  if n == list.head:
-    list.head = n.next
-    if list.tail.next == n:
-      list.tail.next = list.head # restore cycle
+  if n == L.head:
+    L.head = n.next
+    if L.tail.next == n:
+      L.tail.next = L.head # restore cycle
   else:
-    var prev = list.head
+    var prev = L.head
     while prev.next != n and prev.next != nil:
       prev = prev.next
     if prev.next == nil:
@@ -741,9 +741,9 @@ proc remove*[T](list: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.d
     prev.next = n.next
   true
 
-proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
-  ## Removes a node `n` from `list`. Efficiency: O(1).
-  ## This function assumes, for the sake of efficiency, that `n` is contained in `list`,
+proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
+  ## Removes a node `n` from `L`. Efficiency: O(1).
+  ## This function assumes, for the sake of efficiency, that `n` is contained in `L`,
   ## otherwise the effects are undefined.
   ## When the list is cyclic, the cycle is preserved after removal.
   runnableExamples:
@@ -763,15 +763,15 @@ proc remove*[T](list: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =
         ai
     assert s == [2, 2, 2, 2]
 
-  if n == list.tail: list.tail = n.prev
-  if n == list.head: list.head = n.next
+  if n == L.tail: L.tail = n.prev
+  if n == L.head: L.head = n.next
   if n.next != nil: n.next.prev = n.prev
   if n.prev != nil: n.prev.next = n.next
 
 
 
-proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
-  ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],T>`_ for appending a value
@@ -784,17 +784,17 @@ proc add*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
     a.add(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    assert(ring.tail != nil)
-    ring.tail.next = n
+  if L.head != nil:
+    n.next = L.head
+    assert(L.tail != nil)
+    L.tail.next = n
   else:
     n.next = n
-    ring.head = n
-  ring.tail = n
+    L.head = n
+  L.tail = n
 
-proc add*[T](ring: var SinglyLinkedRing[T], value: T) =
-  ## Appends (adds to the end) a value to `ring`. Efficiency: O(1).
+proc add*[T](L: var SinglyLinkedRing[T], value: T) =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_
@@ -808,10 +808,10 @@ proc add*[T](ring: var SinglyLinkedRing[T], value: T) =
     a.add(8)
     assert a.contains(9)
 
-  add(ring, newSinglyLinkedNode(value))
+  add(L, newSinglyLinkedNode(value))
 
-proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
-  ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
+  ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_
@@ -824,17 +824,17 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], n: SinglyLinkedNode[T]) =
     a.prepend(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    assert(ring.tail != nil)
-    ring.tail.next = n
+  if L.head != nil:
+    n.next = L.head
+    assert(L.tail != nil)
+    L.tail.next = n
   else:
     n.next = n
-    ring.tail = n
-  ring.head = n
+    L.tail = n
+  L.head = n
 
-proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) =
-  ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var SinglyLinkedRing[T], value: T) =
+  ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,SinglyLinkedRing[T],SinglyLinkedNode[T]>`_
@@ -848,12 +848,12 @@ proc prepend*[T](ring: var SinglyLinkedRing[T], value: T) =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(ring, newSinglyLinkedNode(value))
+  prepend(L, newSinglyLinkedNode(value))
 
 
 
-proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
-  ## Appends (adds to the end) a node `n` to `ring`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
+  ## Appends (adds to the end) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],T>`_ for appending a value
@@ -868,18 +868,18 @@ proc add*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
     a.add(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    n.prev = ring.head.prev
-    ring.head.prev.next = n
-    ring.head.prev = n
+  if L.head != nil:
+    n.next = L.head
+    n.prev = L.head.prev
+    L.head.prev.next = n
+    L.head.prev = n
   else:
     n.prev = n
     n.next = n
-    ring.head = n
+    L.head = n
 
-proc add*[T](ring: var DoublyLinkedRing[T], value: T) =
-  ## Appends (adds to the end) a value to `ring`. Efficiency: O(1).
+proc add*[T](L: var DoublyLinkedRing[T], value: T) =
+  ## Appends (adds to the end) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_
@@ -895,10 +895,10 @@ proc add*[T](ring: var DoublyLinkedRing[T], value: T) =
     a.add(8)
     assert a.contains(9)
 
-  add(ring, newDoublyLinkedNode(value))
+  add(L, newDoublyLinkedNode(value))
 
-proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
-  ## Prepends (adds to the beginning) a node `n` to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
+  ## Prepends (adds to the beginning) a node `n` to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_
@@ -913,18 +913,18 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
     a.prepend(n)
     assert a.contains(9)
 
-  if ring.head != nil:
-    n.next = ring.head
-    n.prev = ring.head.prev
-    ring.head.prev.next = n
-    ring.head.prev = n
+  if L.head != nil:
+    n.next = L.head
+    n.prev = L.head.prev
+    L.head.prev.next = n
+    L.head.prev = n
   else:
     n.prev = n
     n.next = n
-  ring.head = n
+  L.head = n
 
-proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) =
-  ## Prepends (adds to the beginning) a value to `ring`. Efficiency: O(1).
+proc prepend*[T](L: var DoublyLinkedRing[T], value: T) =
+  ## Prepends (adds to the beginning) a value to `L`. Efficiency: O(1).
   ##
   ## **See also:**
   ## * `add proc <#add,DoublyLinkedRing[T],DoublyLinkedNode[T]>`_
@@ -940,11 +940,11 @@ proc prepend*[T](ring: var DoublyLinkedRing[T], value: T) =
     a.prepend(8)
     assert a.contains(9)
 
-  prepend(ring, newDoublyLinkedNode(value))
+  prepend(L, newDoublyLinkedNode(value))
 
-proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
-  ## Removes `n` from `ring`. Efficiency: O(1).
-  ## This function assumes, for the sake of efficiency, that `n` is contained in `ring`,
+proc remove*[T](L: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
+  ## Removes `n` from `L`. Efficiency: O(1).
+  ## This function assumes, for the sake of efficiency, that `n` is contained in `L`,
   ## otherwise the effects are undefined.
   runnableExamples:
     var a = initDoublyLinkedRing[int]()
@@ -956,13 +956,13 @@ proc remove*[T](ring: var DoublyLinkedRing[T], n: DoublyLinkedNode[T]) =
 
   n.next.prev = n.prev
   n.prev.next = n.next
-  if n == ring.head:
-    let p = ring.head.prev
-    if p == ring.head:
+  if n == L.head:
+    let p = L.head.prev
+    if p == L.head:
       # only one element left:
-      ring.head = nil
+      L.head = nil
     else:
-      ring.head = p
+      L.head = p
 
 proc append*[T](a: var (SinglyLinkedList[T] | SinglyLinkedRing[T]),
                 b: SinglyLinkedList[T] | SinglyLinkedNode[T] | T) =

From db6e7fffba021c223fc67e1c0853e5839cd256d2 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Mon, 19 Jul 2021 20:07:47 -0700
Subject: [PATCH 0240/1654] minor clenup in vmops (#18529)

---
 compiler/vmops.nim | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/compiler/vmops.nim b/compiler/vmops.nim
index 67a196b8f..283306ecc 100644
--- a/compiler/vmops.nim
+++ b/compiler/vmops.nim
@@ -150,6 +150,9 @@ when defined(nimHasInvariant):
     of cincludes: copySeq(conf.cIncludes)
     of clibs: copySeq(conf.cLibs)
 
+proc stackTrace2(c: PCtx, msg: string, n: PNode) =
+  stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info)
+
 proc registerAdditionalOps*(c: PCtx) =
   proc gorgeExWrapper(a: VmArgs) =
     let ret = opGorge(getString(a, 0), getString(a, 1), getString(a, 2),
@@ -238,15 +241,13 @@ proc registerAdditionalOps*(c: PCtx) =
   registerCallback c, "stdlib.macros.symBodyHash", proc (a: VmArgs) =
     let n = getNode(a, 0)
     if n.kind != nkSym:
-      stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr,
-                  "symBodyHash() requires a symbol. '" & $n & "' is of kind '" & $n.kind & "'", n.info)
+      stackTrace2(c, "symBodyHash() requires a symbol. '$#' is of kind '$#'" % [$n, $n.kind], n)
     setResult(a, $symBodyDigest(c.graph, n.sym))
 
   registerCallback c, "stdlib.macros.isExported", proc(a: VmArgs) =
     let n = getNode(a, 0)
     if n.kind != nkSym:
-      stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr,
-                  "isExported() requires a symbol. '" & $n & "' is of kind '" & $n.kind & "'", n.info)
+      stackTrace2(c, "isExported() requires a symbol. '$#' is of kind '$#'" % [$n, $n.kind], n)
     setResult(a, sfExported in n.sym.flags)
 
   registerCallback c, "stdlib.vmutils.vmTrace", proc (a: VmArgs) =

From 44c5afe448ecd9a7009982c16ec85fc8cabe3124 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 20 Jul 2021 11:14:04 +0800
Subject: [PATCH 0241/1654] [std/os]document `checkDir` and use
 `runnableExamples` (#18517)

Co-authored-by: Timothee Cour 
---
 lib/pure/os.nim | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index dadc380e6..6289c64f0 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -10,27 +10,14 @@
 ## This module contains basic operating system facilities like
 ## retrieving environment variables, reading command line arguments,
 ## working with directories, running shell commands, etc.
-##
-## .. code-block::
-##   import std/os
-##
-##   let myFile = "/path/to/my/file.nim"
-##
-##   let pathSplit = splitPath(myFile)
-##   assert pathSplit.head == "/path/to/my"
-##   assert pathSplit.tail == "file.nim"
-##
-##   assert parentDir(myFile) == "/path/to/my"
-##
-##   let fileSplit = splitFile(myFile)
-##   assert fileSplit.dir == "/path/to/my"
-##   assert fileSplit.name == "file"
-##   assert fileSplit.ext == ".nim"
-##
-##   assert myFile.changeFileExt("c") == "/path/to/my/file.c"
 
-##
-##
+runnableExamples("-r:off"):
+  let myFile = "/path/to/my/file.nim"
+  assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim")
+  assert parentDir(myFile) == "/path/to/my"
+  assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim")
+  assert myFile.changeFileExt("c") == "/path/to/my/file.c"
+
 ## **See also:**
 ## * `osproc module `_ for process communication beyond
 ##   `execShellCmd proc <#execShellCmd,string>`_
@@ -2286,6 +2273,10 @@ iterator walkDir*(dir: string; relative = false, checkDir = false):
   ##
   ## Walking is not recursive. If ``relative`` is true (default: false)
   ## the resulting path is shortened to be relative to ``dir``.
+  ##
+  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ## doesn't exist.
+  ##
   ## Example: This directory structure::
   ##   dirA / dirB / fileB1.txt
   ##        / dirC
@@ -2385,6 +2376,9 @@ iterator walkDirRec*(dir: string,
   ## If ``relative`` is true (default: false) the resulting path is
   ## shortened to be relative to ``dir``, otherwise the full path is returned.
   ##
+  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ## doesn't exist.
+  ##
   ## .. warning:: Modifying the directory structure while the iterator
   ##   is traversing may result in undefined behavior!
   ##
@@ -2448,7 +2442,7 @@ proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [
   ## in `dir` (recursively).
   ##
   ## If this fails, `OSError` is raised. This does not fail if the directory never
-  ## existed in the first place, unless `checkDir` = true
+  ## existed in the first place, unless `checkDir` = true.
   ##
   ## See also:
   ## * `tryRemoveFile proc <#tryRemoveFile,string>`_

From 8c7ee96457769a8c8aa1d4bab306b53acf306c8e Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Tue, 20 Jul 2021 09:32:22 +0300
Subject: [PATCH 0242/1654] rst: add missing line/column info for some warnings
 (#18383)

* rst: add missing line/column info for some warnings

* add workaround

* use TLineInfo/FileIndex for storing file names

* fix blank lines in include file (rm harmful strip)

* don't use ref TLineInfo

* return `hasToc` as output parameter for uniformity

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour 

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour 

* address review - stylistic things

* Update compiler/docgen.nim

Co-authored-by: Timothee Cour 

* unify RST warnings/errors names

* doAssert + minor name change

* fix a bug caught by doAssert

* apply strbasics.strip to final HTML/Latex

* rm redundant filename

* fix test after rebase

* delete `order` from rnFootnoteRef,

also display errors/warnings properly when footnote references are from
different files

* Update compiler/lineinfos.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour 

* Update lib/packages/docutils/rstast.nim

Co-authored-by: Timothee Cour 

* revert because of error:

Error: cannot prove that it's safe to initialize 'info' with the runtime value for the discriminator 'kind'

* Update lib/packages/docutils/rstgen.nim

Co-authored-by: Timothee Cour 

* apply suggestion

* Update lib/packages/docutils/rst.nim

Co-authored-by: Timothee Cour 

* add Table for string->file name mapping

* do not import compiler/lineinfos

* fix ambiguous calls

Co-authored-by: Timothee Cour 
Co-authored-by: narimiran 
---
 compiler/docgen.nim              |  54 ++++-----
 compiler/lineinfos.nim           |  49 +++++----
 compiler/main.nim                |  11 +-
 lib/packages/docutils/rst.nim    | 181 ++++++++++++++++++++-----------
 lib/packages/docutils/rstast.nim |  20 +++-
 lib/packages/docutils/rstgen.nim |  40 +++----
 tests/stdlib/trst.nim            |  58 +++++++++-
 tests/stdlib/trstgen.nim         | 122 ++++++++++++---------
 8 files changed, 342 insertions(+), 193 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 9c75c0a6f..f9e72850a 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -12,11 +12,12 @@
 
 import
   ast, strutils, strtabs, options, msgs, os, idents,
-  wordrecg, syntaxes, renderer, lexer, packages/docutils/rstast,
+  wordrecg, syntaxes, renderer, lexer,
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
   pathutils, tables, nimpaths, renderverbatim, osproc
+import packages/docutils/rstast except FileIndex, TLineInfo
 
 from uri import encodeUrl
 from std/private/globs import nativeToUnixPath
@@ -159,17 +160,18 @@ template declareClosures =
     case msgKind
     of meCannotOpenFile: k = errCannotOpenFile
     of meExpected: k = errXExpected
-    of meGridTableNotImplemented: k = errGridTableNotImplemented
-    of meMarkdownIllformedTable: k = errMarkdownIllformedTable
-    of meNewSectionExpected: k = errNewSectionExpected
-    of meGeneralParseError: k = errGeneralParseError
-    of meInvalidDirective: k = errInvalidDirectiveX
-    of meInvalidRstField: k = errInvalidRstField
-    of meFootnoteMismatch: k = errFootnoteMismatch
-    of mwRedefinitionOfLabel: k = warnRedefinitionOfLabel
-    of mwUnknownSubstitution: k = warnUnknownSubstitutionX
-    of mwUnsupportedLanguage: k = warnLanguageXNotSupported
-    of mwUnsupportedField: k = warnFieldXNotSupported
+    of meGridTableNotImplemented: k = errRstGridTableNotImplemented
+    of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable
+    of meNewSectionExpected: k = errRstNewSectionExpected
+    of meGeneralParseError: k = errRstGeneralParseError
+    of meInvalidDirective: k = errRstInvalidDirectiveX
+    of meInvalidField: k = errRstInvalidField
+    of meFootnoteMismatch: k = errRstFootnoteMismatch
+    of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel
+    of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX
+    of mwBrokenLink: k = warnRstBrokenLink
+    of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported
+    of mwUnsupportedField: k = warnRstFieldXNotSupported
     of mwRstStyle: k = warnRstStyle
     {.gcsafe.}:
       globalError(conf, newLineInfo(conf, AbsoluteFile filename, line, col), k, arg)
@@ -182,11 +184,9 @@ template declareClosures =
 
 proc parseRst(text, filename: string,
               line, column: int,
-              rstOptions: RstParseOptions;
               conf: ConfigRef, sharedState: PRstSharedState): PRstNode =
   declareClosures()
-  result = rstParsePass1(text, filename, line, column, rstOptions,
-                         sharedState)
+  result = rstParsePass1(text, line, column, sharedState)
 
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
                  ext: string, guessTarget: bool): AbsoluteFile =
@@ -202,20 +202,22 @@ proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
 proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex}
 
 proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
-                    outExt: string = HtmlExt, module: PSym = nil): PDoc =
+                    outExt: string = HtmlExt, module: PSym = nil,
+                    isPureRst = false): PDoc =
   declareClosures()
   new(result)
   result.module = module
   result.conf = conf
   result.cache = cache
   result.outDir = conf.outDir.string
-  const options = {roSupportRawDirective, roSupportMarkdown,
-                   roPreferMarkdown, roNimFile}
+  result.isPureRst = isPureRst
+  var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}
+  if not isPureRst: options.incl roNimFile
   result.sharedState = newRstSharedState(
       options, filename.string,
       docgenFindFile, compilerMsgHandler)
   initRstGenerator(result[], (if conf.isLatexCmd: outLatex else: outHtml),
-                   conf.configVars, filename.string, options,
+                   conf.configVars, filename.string,
                    docgenFindFile, compilerMsgHandler)
 
   if conf.configVars.hasKey("doc.googleAnalytics"):
@@ -299,8 +301,7 @@ proc genComment(d: PDoc, n: PNode): PRstNode =
     result = parseRst(n.comment, toFullPath(d.conf, n.info),
                       toLinenumber(n.info),
                       toColumn(n.info) + DocColOffset,
-                      d.options, d.conf,
-                      d.sharedState)
+                      d.conf, d.sharedState)
 
 proc genRecCommentAux(d: PDoc, n: PNode): PRstNode =
   if n == nil: return nil
@@ -1123,6 +1124,9 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
 
 proc finishGenerateDoc*(d: var PDoc) =
   ## Perform 2nd RST pass for resolution of links/footnotes/headings...
+  # copy file map `filenames` to ``rstgen.nim`` for its warnings
+  d.filenames = d.sharedState.filenames
+
   # Main title/subtitle are allowed only in the first RST fragment of document
   var firstRst = PRstNode(nil)
   for fragment in d.modDescPre:
@@ -1417,14 +1421,10 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
 proc commandRstAux(cache: IdentCache, conf: ConfigRef;
                    filename: AbsoluteFile, outExt: string) =
   var filen = addFileExt(filename, "txt")
-  var d = newDocumentor(filen, cache, conf, outExt)
-
-  d.isPureRst = true
+  var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true)
   let rst = parseRst(readFile(filen.string), filen.string,
                      line=LineRstInit, column=ColRstInit,
-                     {roSupportRawDirective, roSupportMarkdown,
-                      roPreferMarkdown}, conf,
-                     d.sharedState)
+                     conf, d.sharedState)
   d.modDescPre = @[ItemFragment(isRst: true, rst: rst)]
   finishGenerateDoc(d)
   writeOutput(d)
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 801c20d25..8ab52a452 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -32,13 +32,13 @@ type
     # non-fatal errors
     errIllFormedAstX, errCannotOpenFile,
     errXExpected,
-    errGridTableNotImplemented,
-    errMarkdownIllformedTable,
-    errGeneralParseError,
-    errNewSectionExpected,
-    errInvalidDirectiveX,
-    errInvalidRstField,
-    errFootnoteMismatch,
+    errRstGridTableNotImplemented,
+    errRstMarkdownIllformedTable,
+    errRstNewSectionExpected,
+    errRstGeneralParseError,
+    errRstInvalidDirectiveX,
+    errRstInvalidField,
+    errRstFootnoteMismatch,
     errProveInit, # deadcode
     errGenerated,
     errUser,
@@ -47,10 +47,13 @@ type
     warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
     warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
     warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
-    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
-    warnLanguageXNotSupported = "LanguageXNotSupported",
-    warnFieldXNotSupported = "FieldXNotSupported",
-    warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored",
+    warnRstRedefinitionOfLabel = "RedefinitionOfLabel",
+    warnRstUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnRstBrokenLink = "BrokenLink",
+    warnRstLanguageXNotSupported = "LanguageXNotSupported",
+    warnRstFieldXNotSupported = "FieldXNotSupported",
+    warnRstStyle = "warnRstStyle",
+    warnCommentXIgnored = "CommentXIgnored",
     warnTypelessParam = "TypelessParam",
     warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
     warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
@@ -93,13 +96,13 @@ const
     errIllFormedAstX: "illformed AST: $1",
     errCannotOpenFile: "cannot open '$1'",
     errXExpected: "'$1' expected",
-    errGridTableNotImplemented: "grid table is not implemented",
-    errMarkdownIllformedTable: "illformed delimiter row of a markdown table",
-    errGeneralParseError: "general parse error",
-    errNewSectionExpected: "new section expected $1",
-    errInvalidDirectiveX: "invalid directive: '$1'",
-    errInvalidRstField: "invalid field: $1",
-    errFootnoteMismatch: "number of footnotes and their references don't match: $1",
+    errRstGridTableNotImplemented: "grid table is not implemented",
+    errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table",
+    errRstNewSectionExpected: "new section expected $1",
+    errRstGeneralParseError: "general parse error",
+    errRstInvalidDirectiveX: "invalid directive: '$1'",
+    errRstInvalidField: "invalid field: $1",
+    errRstFootnoteMismatch: "number of footnotes and their references don't match: $1",
     errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
@@ -111,10 +114,11 @@ const
     warnConfigDeprecated: "config file '$1' is deprecated",
     warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)",
     warnUnknownMagic: "unknown magic '$1' might crash the compiler",
-    warnRedefinitionOfLabel: "redefinition of label '$1'",
-    warnUnknownSubstitutionX: "unknown substitution '$1'",
-    warnLanguageXNotSupported: "language '$1' not supported",
-    warnFieldXNotSupported: "field '$1' not supported",
+    warnRstRedefinitionOfLabel: "redefinition of label '$1'",
+    warnRstUnknownSubstitutionX: "unknown substitution '$1'",
+    warnRstBrokenLink: "broken link '$1'",
+    warnRstLanguageXNotSupported: "language '$1' not supported",
+    warnRstFieldXNotSupported: "field '$1' not supported",
     warnRstStyle: "RST style: $1",
     warnCommentXIgnored: "comment '$1' ignored",
     warnTypelessParam: "", # deadcode
@@ -196,6 +200,7 @@ const
   warnMax* = pred(hintSuccess)
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
+  rstWarnings* = {warnRstRedefinitionOfLabel..warnRstStyle}
 
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
diff --git a/compiler/main.nim b/compiler/main.nim
index 1c8034c6c..e4ec4d729 100644
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -285,7 +285,7 @@ proc mainCommand*(graph: ModuleGraph) =
   of cmdDoc:
     docLikeCmd():
       conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+      conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # issue #13218
         # because currently generates lots of false positives due to conflation
         # of labels links in doc comments, e.g. for random.rand:
         #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
@@ -295,19 +295,16 @@ proc mainCommand*(graph: ModuleGraph) =
         commandBuildIndex(conf, $conf.outDir)
   of cmdRst2html:
     # XXX: why are warnings disabled by default for rst2html and rst2tex?
-    for warn in [warnUnknownSubstitutionX, warnLanguageXNotSupported,
-                 warnFieldXNotSupported, warnRstStyle]:
+    for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
+    conf.setNoteDefaults(warnRstRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
     else:
       loadConfigs(DocConfig, cache, conf, graph.idgen)
       commandRst2Html(cache, conf)
   of cmdRst2tex, cmdDoc2tex:
-    for warn in [warnRedefinitionOfLabel, warnUnknownSubstitutionX,
-                 warnLanguageXNotSupported,
-                 warnFieldXNotSupported, warnRstStyle]:
+    for warn in rstWarnings:
       conf.setNoteDefaults(warn, true)
     when defined(leanCompiler):
       conf.quitOrRaise "compiler wasn't built with documentation generator"
diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim
index 9abdf3fc3..abea026dd 100644
--- a/lib/packages/docutils/rst.nim
+++ b/lib/packages/docutils/rst.nim
@@ -196,7 +196,7 @@
 
 import
   os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
-  std/private/miscdollars
+  std/private/miscdollars, tables
 from highlite import SourceLanguage, getSourceLanguage
 
 type
@@ -217,6 +217,7 @@ type
     mcWarning = "Warning",
     mcError = "Error"
 
+  # keep the order in sync with compiler/docgen.nim and compiler/lineinfos.nim:
   MsgKind* = enum          ## the possible messages
     meCannotOpenFile = "cannot open '$1'",
     meExpected = "'$1' expected",
@@ -225,10 +226,11 @@ type
     meNewSectionExpected = "new section expected $1",
     meGeneralParseError = "general parse error",
     meInvalidDirective = "invalid directive: '$1'",
-    meInvalidRstField = "invalid field: $1",
+    meInvalidField = "invalid field: $1",
     meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1",
     mwRedefinitionOfLabel = "redefinition of label '$1'",
     mwUnknownSubstitution = "unknown substitution '$1'",
+    mwBrokenLink = "broken link '$1'",
     mwUnsupportedLanguage = "language '$1' not supported",
     mwUnsupportedField = "field '$1' not supported",
     mwRstStyle = "RST style: $1"
@@ -489,7 +491,9 @@ type
     autoNumIdx: int     # order of occurence: fnAutoNumber, fnAutoNumberLabel
     autoSymIdx: int     # order of occurence: fnAutoSymbol
     label: string       # valid for fnAutoNumberLabel
-
+  RstFileTable* = object
+    filenameToIdx*: Table[string, FileIndex]
+    idxToFilename*: seq[string]
   RstSharedState = object
     options: RstParseOptions    # parsing options
     hLevels: LevelMap           # hierarchy of heading styles
@@ -501,15 +505,19 @@ type
     subs: seq[Substitution]     # substitutions
     refs*: seq[Substitution]    # references
     anchors*: seq[AnchorSubst]  # internal target substitutions
-    lineFootnoteNum: seq[int]     # footnote line, auto numbers .. [#]
-    lineFootnoteNumRef: seq[int]  # footnote line, their reference [#]_
-    lineFootnoteSym: seq[int]     # footnote line, auto symbols .. [*]
-    lineFootnoteSymRef: seq[int]  # footnote line, their reference [*]_
+    lineFootnoteNum: seq[TLineInfo]     # footnote line, auto numbers .. [#]
+    lineFootnoteNumRef: seq[TLineInfo]  # footnote line, their reference [#]_
+    currFootnoteNumRef: int             # ... their counter for `resolveSubs`
+    lineFootnoteSym: seq[TLineInfo]     # footnote line, auto symbols .. [*]
+    lineFootnoteSymRef: seq[TLineInfo]  # footnote line, their reference [*]_
+    currFootnoteSymRef: int             # ... their counter for `resolveSubs`
     footnotes: seq[FootnoteSubst] # correspondence b/w footnote label,
                                   # number, order of occurrence
     msgHandler: MsgHandler      # How to handle errors.
     findFile: FindFileHandler   # How to find files.
-    filename: string
+    filenames*: RstFileTable    # map file name <-> FileIndex (for storing
+                                # file names for warnings after 1st stage)
+    currFileIdx: FileIndex      # current index in `filesnames`
     hasToc*: bool
 
   PRstSharedState* = ref RstSharedState
@@ -579,6 +587,25 @@ proc whichRoleAux(sym: string): RstNodeKind =
   else:  # unknown role
     result = rnUnknownRole
 
+proc len(filenames: RstFileTable): int = filenames.idxToFilename.len
+
+proc setCurrFilename(s: PRstSharedState, file1: string) =
+  let nextIdx = s.filenames.len.FileIndex
+  let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx)
+  if v == nextIdx:
+    s.filenames.filenameToIdx[file1] = v
+    s.filenames.idxToFilename.add file1
+  s.currFileIdx = v
+
+proc getFilename(filenames: RstFileTable, fid: FileIndex): string =
+  doAssert(0 <= fid.int and fid.int < filenames.len,
+      "incorrect FileIndex $1 (range 0..$2)" % [
+        $fid.int, $(filenames.len - 1)])
+  result = filenames.idxToFilename[fid.int]
+
+proc currFilename(s: PRstSharedState): string =
+  getFilename(s.filenames, s.currFileIdx)
+
 proc newRstSharedState*(options: RstParseOptions,
                         filename: string,
                         findFile: FindFileHandler,
@@ -589,32 +616,37 @@ proc newRstSharedState*(options: RstParseOptions,
       currRoleKind: whichRoleAux(r),
       options: options,
       msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler,
-      filename: filename,
       findFile: if not isNil(findFile): findFile else: defaultFindFile
   )
+  setCurrFilename(result, filename)
 
 proc curLine(p: RstParser): int = p.line + currentTok(p).line
 
 proc findRelativeFile(p: RstParser; filename: string): string =
-  result = p.s.filename.splitFile.dir / filename
+  result = p.s.currFilename.splitFile.dir / filename
   if not fileExists(result):
     result = p.s.findFile(filename)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) =
-  p.s.msgHandler(p.s.filename, curLine(p),
+  p.s.msgHandler(p.s.currFilename, curLine(p),
                              p.col + currentTok(p).col, msgKind, arg)
 
 proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) =
-  ## Print warnings for footnotes/substitutions.
-  ## TODO: their line/column info is not known, to fix it.
-  s.msgHandler(s.filename, LineRstInit, ColRstInit, msgKind, arg)
+  s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg)
+
+proc rstMessage*(filenames: RstFileTable, f: MsgHandler,
+                 info: TLineInfo, msgKind: MsgKind, arg: string) =
+  ## Print warnings using `info`, i.e. in 2nd-pass warnings for
+  ## footnotes/substitutions/references or from ``rstgen.nim``.
+  let file = getFilename(filenames, info.fileIndex)
+  f(file, info.line.int, info.col.int, msgKind, arg)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string, line, col: int) =
-  p.s.msgHandler(p.s.filename, p.line + line,
+  p.s.msgHandler(p.s.currFilename, p.line + line,
                              p.col + col, msgKind, arg)
 
 proc rstMessage(p: RstParser, msgKind: MsgKind) =
-  p.s.msgHandler(p.s.filename, curLine(p),
+  p.s.msgHandler(p.s.currFilename, curLine(p),
                              p.col + currentTok(p).col, msgKind,
                              currentTok(p).symbol)
 
@@ -827,11 +859,18 @@ proc addFootnoteNumManual(p: var RstParser, num: int) =
       return
   p.s.footnotes.add((fnManualNumber, num, -1, -1, $num))
 
+proc lineInfo(p: RstParser, iTok: int): TLineInfo =
+  result.col = int16(p.col + p.tok[iTok].col)
+  result.line = uint16(p.line + p.tok[iTok].line)
+  result.fileIndex = p.s.currFileIdx
+
+proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx)
+
 proc addFootnoteNumAuto(p: var RstParser, label: string) =
   ## add auto-numbered footnote.
   ## Empty label [#] means it'll be resolved by the occurrence.
   if label == "":  # simple auto-numbered [#]
-    p.s.lineFootnoteNum.add curLine(p)
+    p.s.lineFootnoteNum.add lineInfo(p)
     p.s.footnotes.add((fnAutoNumber, -1, p.s.lineFootnoteNum.len, -1, label))
   else:           # auto-numbered with label [#label]
     for fnote in p.s.footnotes:
@@ -841,7 +880,7 @@ proc addFootnoteNumAuto(p: var RstParser, label: string) =
     p.s.footnotes.add((fnAutoNumberLabel, -1, -1, -1, label))
 
 proc addFootnoteSymAuto(p: var RstParser) =
-  p.s.lineFootnoteSym.add curLine(p)
+  p.s.lineFootnoteSym.add lineInfo(p)
   p.s.footnotes.add((fnAutoSymbol, -1, -1, p.s.lineFootnoteSym.len, ""))
 
 proc orderFootnotes(s: PRstSharedState) =
@@ -850,7 +889,15 @@ proc orderFootnotes(s: PRstSharedState) =
   ## Save the result back to `s.footnotes`.
 
   # Report an error if found any mismatch in number of automatic footnotes
-  proc listFootnotes(lines: seq[int]): string =
+  proc listFootnotes(locations: seq[TLineInfo]): string =
+    var lines: seq[string]
+    for info in locations:
+      if s.filenames.len > 1:
+        let file = getFilename(s.filenames, info.fileIndex)
+        lines.add file & ":"
+      else:  # no need to add file name here if there is only 1
+        lines.add ""
+      lines[^1].add $info.line
     result.add $lines.len & " (lines " & join(lines, ", ") & ")"
   if s.lineFootnoteNum.len != s.lineFootnoteNumRef.len:
     rstMessage(s, meFootnoteMismatch,
@@ -1157,7 +1204,7 @@ proc whichRole(p: RstParser, sym: string): RstNodeKind =
 
 proc toInlineCode(n: PRstNode, language: string): PRstNode =
   ## Creates rnInlineCode and attaches `n` contents as code (in 3rd son).
-  result = newRstNode(rnInlineCode)
+  result = newRstNode(rnInlineCode, info=n.info)
   let args = newRstNode(rnDirArg)
   var lang = language
   if language == "cpp": lang = "c++"
@@ -1179,6 +1226,7 @@ proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
   result = newRstNode(kind, newSons)
 
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
+  ## Finalizes node `n` that was tentatively determined as interpreted text.
   var newKind = n.kind
   var newSons = n.sons
 
@@ -1207,12 +1255,10 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
         newKind = rnHyperlink
         newSons = @[a, b]
         setRef(p, rstnodeToRefname(a), b)
-    elif n.kind == rnInterpretedText:
+      result = newRstNode(newKind, newSons)
+    else:  # some link that will be resolved in `resolveSubs`
       newKind = rnRef
-    else:
-      newKind = rnRef
-      newSons = @[n]
-    result = newRstNode(newKind, newSons)
+      result = newRstNode(newKind, sons=newSons, info=n.info)
   elif match(p, p.idx, ":w:"):
     # a role:
     let (roleName, lastIdx) = getRefname(p, p.idx+1)
@@ -1300,20 +1346,19 @@ proc parseWordOrRef(p: var RstParser, father: PRstNode) =
   else:
     # check for reference (probably, long one like some.ref.with.dots_ )
     var saveIdx = p.idx
-    var isRef = false
+    var reference: PRstNode = nil
     inc p.idx
     while currentTok(p).kind in {tkWord, tkPunct}:
       if currentTok(p).kind == tkPunct:
         if isInlineMarkupEnd(p, "_", exact=true):
-          isRef = true
+          reference = newRstNode(rnRef, info=lineInfo(p, saveIdx))
           break
         if not validRefnamePunct(currentTok(p).symbol):
           break
       inc p.idx
-    if isRef:
-      let r = newRstNode(rnRef)
-      for i in saveIdx..p.idx-1: r.add newLeaf(p.tok[i].symbol)
-      father.add r
+    if reference != nil:
+      for i in saveIdx..p.idx-1: reference.add newLeaf(p.tok[i].symbol)
+      father.add reference
       inc p.idx  # skip final _
     else:  # 1 normal word
       father.add newLeaf(p.tok[saveIdx].symbol)
@@ -1387,6 +1432,8 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string,
     else: rstMessage(p, meExpected, postfix, line, col)
 
 proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
+  result = newRstNodeA(p, rnCodeBlock)
+  result.info = lineInfo(p)
   var args = newRstNode(rnDirArg)
   if currentTok(p).kind == tkWord:
     args.add(newLeaf(p))
@@ -1411,7 +1458,6 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode =
       inc p.idx
   var lb = newRstNode(rnLiteralBlock)
   lb.add(n)
-  result = newRstNodeA(p, rnCodeBlock)
   result.add(args)
   result.add(PRstNode(nil))
   result.add(lb)
@@ -1495,6 +1541,7 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode =
 
 proc parseInline(p: var RstParser, father: PRstNode) =
   var n: PRstNode  # to be used in `if` condition
+  let saveIdx = p.idx
   case currentTok(p).kind
   of tkPunct:
     if isInlineMarkupStart(p, "***"):
@@ -1537,12 +1584,12 @@ proc parseInline(p: var RstParser, father: PRstNode) =
         n = n.toOtherRole(k, roleName)
       father.add(n)
     elif isInlineMarkupStart(p, "`"):
-      var n = newRstNode(rnInterpretedText)
+      var n = newRstNode(rnInterpretedText, info=lineInfo(p, p.idx+1))
       parseUntil(p, n, "`", false) # bug #17260
       n = parsePostfix(p, n)
       father.add(n)
     elif isInlineMarkupStart(p, "|"):
-      var n = newRstNode(rnSubstitutionReferences)
+      var n = newRstNode(rnSubstitutionReferences, info=lineInfo(p, p.idx+1))
       parseUntil(p, n, "|", false)
       father.add(n)
     elif roSupportMarkdown in p.s.options and
@@ -1552,15 +1599,14 @@ proc parseInline(p: var RstParser, father: PRstNode) =
     elif isInlineMarkupStart(p, "[") and nextTok(p).symbol != "[" and
          (n = parseFootnoteName(p, reference=true); n != nil):
       var nn = newRstNode(rnFootnoteRef)
+      nn.info = lineInfo(p, saveIdx+1)
       nn.add n
       let (fnType, _) = getFootnoteType(n)
       case fnType
       of fnAutoSymbol:
-        p.s.lineFootnoteSymRef.add curLine(p)
-        nn.order = p.s.lineFootnoteSymRef.len
+        p.s.lineFootnoteSymRef.add lineInfo(p)
       of fnAutoNumber:
-        p.s.lineFootnoteNumRef.add curLine(p)
-        nn.order = p.s.lineFootnoteNumRef.len
+        p.s.lineFootnoteNumRef.add lineInfo(p)
       else: discard
       father.add(nn)
     else:
@@ -1693,7 +1739,7 @@ proc parseField(p: var RstParser): PRstNode =
   ## Returns a parsed rnField node.
   ##
   ## rnField nodes have two children nodes, a rnFieldName and a rnFieldBody.
-  result = newRstNode(rnField)
+  result = newRstNode(rnField, info=lineInfo(p))
   var col = currentTok(p).col
   var fieldname = newRstNode(rnFieldName)
   parseUntil(p, fieldname, ":", false)
@@ -2469,6 +2515,7 @@ proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode
   ## Both rnDirArg and rnFieldList children nodes might be nil, so you need to
   ## check them before accessing.
   result = newRstNodeA(p, k)
+  if k == rnCodeBlock: result.info = lineInfo(p)
   var args: PRstNode = nil
   var options: PRstNode = nil
   if hasArg in flags:
@@ -2593,14 +2640,16 @@ proc dirInclude(p: var RstParser): PRstNode =
 
       var q: RstParser
       initParser(q, p.s)
-      q.s.filename = path
+      let saveFileIdx = p.s.currFileIdx
+      setCurrFilename(p.s, path)
       getTokens(
-        inputString[startPosition..endPosition].strip(),
+        inputString[startPosition..endPosition],
         q.tok)
       # workaround a GCC bug; more like the interior pointer bug?
       #if find(q.tok[high(q.tok)].symbol, "\0\x01\x02") > 0:
       #  InternalError("Too many binary zeros in include file")
       result = parseDoc(q)
+      p.s.currFileIdx = saveFileIdx
 
 proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
   ## Parses a code block.
@@ -2634,7 +2683,7 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
     if result.sons[1].isNil: result.sons[1] = newRstNode(rnFieldList)
     assert result.sons[1].kind == rnFieldList
     # Hook the extra field and specify the Nim language as value.
-    var extraNode = newRstNode(rnField)
+    var extraNode = newRstNode(rnField, info=lineInfo(p))
     extraNode.add(newRstNode(rnFieldName))
     extraNode.add(newRstNode(rnFieldBody))
     extraNode.sons[0].add newLeaf("default-language")
@@ -2837,9 +2886,8 @@ proc parseDotDot(p: var RstParser): PRstNode =
   else:
     result = parseComment(p, col)
 
-proc rstParsePass1*(fragment, filename: string,
+proc rstParsePass1*(fragment: string,
                     line, column: int,
-                    options: RstParseOptions,
                     sharedState: PRstSharedState): PRstNode =
   ## Parses an RST `fragment`.
   ## The result should be further processed by
@@ -2872,7 +2920,8 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       var key = addNodes(n)
       var e = getEnv(key)
       if e != "": result = newLeaf(e)
-      else: rstMessage(s, mwUnknownSubstitution, key)
+      else: rstMessage(s.filenames, s.msgHandler, n.info,
+                       mwUnknownSubstitution, key)
   of rnHeadline, rnOverline:
     # fix up section levels depending on presence of a title and subtitle
     if s.hTitleCnt == 2:
@@ -2890,12 +2939,14 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       let text = newRstNode(rnInner, n.sons)
       result.sons = @[text, y]
     else:
-      let s = findMainAnchor(s, refn)
-      if s != "":
+      let anchor = findMainAnchor(s, refn)
+      if anchor != "":
         result = newRstNode(rnInternalRef)
         let text = newRstNode(rnInner, n.sons)
-        result.sons = @[text,        # visible text of reference
-                        newLeaf(s)]  # link itself
+        result.sons = @[text,             # visible text of reference
+                        newLeaf(anchor)]  # link itself
+      else:
+        rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
   of rnFootnote:
     var (fnType, num) = getFootnoteType(n.sons[0])
     case fnType
@@ -2922,20 +2973,22 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       result.add(nn)
     var refn = fnType.prefix
     # create new rnFootnoteRef, add final label, and finalize target refn:
-    result = newRstNode(rnFootnoteRef)
+    result = newRstNode(rnFootnoteRef, info = n.info)
     case fnType
     of fnManualNumber:
       addLabel num
       refn.add $num
     of fnAutoNumber:
-      addLabel getFootnoteNum(s, n.order)
-      refn.add $n.order
+      inc s.currFootnoteNumRef
+      addLabel getFootnoteNum(s, s.currFootnoteNumRef)
+      refn.add $s.currFootnoteNumRef
     of fnAutoNumberLabel:
       addLabel getFootnoteNum(s, rstnodeToRefname(n))
       refn.add rstnodeToRefname(n)
     of fnAutoSymbol:
-      addLabel getAutoSymbol(s, n.order)
-      refn.add $n.order
+      inc s.currFootnoteSymRef
+      addLabel getAutoSymbol(s, s.currFootnoteSymRef)
+      refn.add $s.currFootnoteSymRef
     of fnCitation:
       result.add n.sons[0]
       refn.add rstnodeToRefname(n)
@@ -2943,7 +2996,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
     if anch != "":
       result.add newLeaf(anch)     # add link
     else:
-      rstMessage(s, mwUnknownSubstitution, refn)
+      rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn)
       result.add newLeaf(refn)  # add link
   of rnLeaf:
     discard
@@ -2971,14 +3024,18 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode =
       result.sons = newSons
 
 proc rstParse*(text, filename: string,
-               line, column: int, hasToc: var bool,
+               line, column: int,
                options: RstParseOptions,
                findFile: FindFileHandler = nil,
-               msgHandler: MsgHandler = nil): PRstNode =
-  ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`.
+               msgHandler: MsgHandler = nil):
+              tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] =
+  ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`,
+  ## note that 2nd tuple element should be fed to `initRstGenerator`
+  ## argument `filenames` (it is being filled here at least with `filename`
+  ## and possibly with other files from RST ``.. include::`` statement).
   var sharedState = newRstSharedState(options, filename, findFile, msgHandler)
-  let unresolved = rstParsePass1(text, filename, line, column,
-                                 options, sharedState)
+  let unresolved = rstParsePass1(text, line, column, sharedState)
   preparePass2(sharedState, unresolved)
-  result = resolveSubs(sharedState, unresolved)
-  hasToc = sharedState.hasToc
+  result.node = resolveSubs(sharedState, unresolved)
+  result.filenames = sharedState.filenames
+  result.hasToc = sharedState.hasToc
diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim
index 2489ce40c..fa0620f44 100644
--- a/lib/packages/docutils/rstast.nim
+++ b/lib/packages/docutils/rstast.nim
@@ -74,6 +74,11 @@ type
     rnLeaf                    # a leaf; the node's text field contains the
                               # leaf val
 
+  FileIndex* = distinct int32
+  TLineInfo* = object
+    line*: uint16
+    col*: int16
+    fileIndex*: FileIndex
 
   PRstNode* = ref RstNode    ## an RST node
   RstNodeSeq* = seq[PRstNode]
@@ -92,21 +97,32 @@ type
       level*: int             ## level of headings starting from 1 (main
                               ## chapter) to larger ones (minor sub-sections)
                               ## level=0 means it's document title or subtitle
-    of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem:
+    of rnFootnote, rnCitation, rnOptionListItem:
       order*: int             ## footnote order (for auto-symbol footnotes and
                               ## auto-numbered ones without a label)
+    of rnRef, rnSubstitutionReferences,
+        rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef:
+      info*: TLineInfo        ## To have line/column info for warnings at
+                              ## nodes that are post-processed after parsing
     else:
       discard
     anchor*: string           ## anchor, internal link target
                               ## (aka HTML id tag, aka Latex label/hypertarget)
     sons*: RstNodeSeq        ## the node's sons
 
+proc `==`*(a, b: FileIndex): bool {.borrow.}
+
 proc len*(n: PRstNode): int =
   result = len(n.sons)
 
 proc newRstNode*(kind: RstNodeKind, sons: seq[PRstNode] = @[],
                  anchor = ""): PRstNode =
+  result = PRstNode(kind: kind, sons: sons, anchor: anchor)
+
+proc newRstNode*(kind: RstNodeKind, info: TLineInfo,
+                 sons: seq[PRstNode] = @[]): PRstNode =
   result = PRstNode(kind: kind, sons: sons)
+  result.info = info
 
 proc newRstNode*(kind: RstNodeKind, s: string): PRstNode {.deprecated.} =
   assert kind in {rnLeaf, rnSmiley}
@@ -388,7 +404,7 @@ proc renderRstToStr*(node: PRstNode, indent=0): string =
     result.add "  adType=" & node.adType
   of rnHeadline, rnOverline, rnMarkdownHeadline:
     result.add "  level=" & $node.level
-  of rnFootnote, rnCitation, rnFootnoteRef, rnOptionListItem:
+  of rnFootnote, rnCitation, rnOptionListItem:
     result.add (if node.order == 0:   "" else: "  order=" & $node.order)
   else:
     discard
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim
index b9c8f688b..10a229479 100644
--- a/lib/packages/docutils/rstgen.nim
+++ b/lib/packages/docutils/rstgen.nim
@@ -40,7 +40,7 @@
 ##   can be done by simply searching for [footnoteName].
 
 import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils,
-  algorithm, parseutils
+  algorithm, parseutils, std/strbasics
 
 import ../../std/private/since
 
@@ -72,11 +72,11 @@ type
     tocPart*: seq[TocEntry]
     hasToc*: bool
     theIndex: string # Contents of the index file to be dumped at the end.
-    options*: RstParseOptions
     findFile*: FindFileHandler
     msgHandler*: MsgHandler
     outDir*: string      ## output directory, initialized by docgen.nim
     destFile*: string    ## output (HTML) file, initialized by docgen.nim
+    filenames*: RstFileTable
     filename*: string         ## source Nim or Rst file
     meta*: array[MetaEnum, string]
     currentSection: string ## \
@@ -112,9 +112,9 @@ proc init(p: var CodeBlockParams) =
 
 proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
                        config: StringTableRef, filename: string,
-                       options: RstParseOptions,
                        findFile: FindFileHandler = nil,
-                       msgHandler: MsgHandler = nil) =
+                       msgHandler: MsgHandler = nil,
+                       filenames = default(RstFileTable)) =
   ## Initializes a ``RstGenerator``.
   ##
   ## You need to call this before using a ``RstGenerator`` with any other
@@ -160,9 +160,9 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
   g.target = target
   g.tocPart = @[]
   g.filename = filename
+  g.filenames = filenames
   g.splitAfter = 20
   g.theIndex = ""
-  g.options = options
   g.findFile = findFile
   g.currentSection = ""
   g.id = 0
@@ -908,9 +908,8 @@ proc renderSmiley(d: PDoc, n: PRstNode, result: var string) =
     [d.config.getOrDefault"doc.smiley_format" % n.text])
 
 proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int =
-  # TODO: proper column/line info
   template err(msg: string) =
-    d.msgHandler(d.filename, 1, 0, meInvalidRstField, msg)
+    rstMessage(d.filenames, d.msgHandler, n.info, meInvalidField, msg)
   let value = n.getFieldValue
   var number: int
   let nChars = parseInt(value, number)
@@ -958,7 +957,8 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
     params.langStr = n.getFieldValue.strip
     params.lang = params.langStr.getSourceLanguage
   else:
-    d.msgHandler(d.filename, 1, 0, mwUnsupportedField, n.getArgument)
+    rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedField,
+               n.getArgument)
 
 proc parseCodeBlockParams(d: PDoc, n: PRstNode): CodeBlockParams =
   ## Iterates over all code block fields and returns processed params.
@@ -1069,7 +1069,8 @@ proc renderCode(d: PDoc, n: PRstNode, result: var string) =
   dispA(d.target, result, blockStart, blockStart, [])
   if params.lang == langNone:
     if len(params.langStr) > 0:
-      d.msgHandler(d.filename, 1, 0, mwUnsupportedLanguage, params.langStr)
+      rstMessage(d.filenames, d.msgHandler, n.info, mwUnsupportedLanguage,
+                 params.langStr)
     for letter in m.text: escChar(d.target, result, letter, emText)
   else:
     renderCodeLang(result, params.lang, m.text, d.target)
@@ -1564,23 +1565,24 @@ proc rstToHtml*(s: string, options: RstParseOptions,
     result = ""
 
   const filen = "input"
+  let (rst, filenames, _) = rstParse(s, filen,
+                                     line=LineRstInit, column=ColRstInit,
+                                     options, myFindFile, msgHandler)
   var d: RstGenerator
-  initRstGenerator(d, outHtml, config, filen, options, myFindFile, msgHandler)
-  var dummyHasToc = false
-  var rst = rstParse(s, filen, line=LineRstInit, column=ColRstInit,
-                     dummyHasToc, options, myFindFile, msgHandler)
+  initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames)
   result = ""
   renderRstToOut(d, rst, result)
+  strbasics.strip(result)
 
 
 proc rstToLatex*(rstSource: string; options: RstParseOptions): string {.inline, since: (1, 3).} =
   ## Convenience proc for `renderRstToOut` and `initRstGenerator`.
   runnableExamples: doAssert rstToLatex("*Hello* **world**", {}) == """\emph{Hello} \textbf{world}"""
   if rstSource.len == 0: return
-  var option: bool
+  let (rst, filenames, _) = rstParse(rstSource, "",
+                                     line=LineRstInit, column=ColRstInit,
+                                     options)
   var rstGenera: RstGenerator
-  rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", options)
-  rstGenera.renderRstToOut(
-      rstParse(rstSource, "", line=LineRstInit, column=ColRstInit,
-               option, options),
-      result)
+  rstGenera.initRstGenerator(outLatex, defaultConfig(), "input", filenames=filenames)
+  rstGenera.renderRstToOut(rst, result)
+  strbasics.strip(result)
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index fe99d6cfa..fc3ccbf4e 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -5,6 +5,8 @@ discard """
 
 [Suite] RST indentation
 
+[Suite] Warnings
+
 [Suite] RST include directive
 
 [Suite] RST escaping
@@ -51,9 +53,8 @@ proc toAst(input: string,
       # we don't find any files in online mode:
       result = ""
 
-    var dummyHasToc = false
-    var rst = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
-                       dummyHasToc, rstOptions, myFindFile, testMsgHandler)
+    var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit,
+                               rstOptions, myFindFile, testMsgHandler)
     result = renderRstToStr(rst)
   except EParseError as e:
     if e.msg != "":
@@ -356,6 +357,53 @@ suite "RST indentation":
     # "template..." should be parsed as a definition list attached to ":test:":
     check inputWrong.toAst != ast
 
+suite "Warnings":
+  test "warnings for broken footnotes/links/substitutions":
+    let input = dedent"""
+      firstParagraph
+
+      footnoteRef [som]_
+
+      link `a broken Link`_
+
+      substitution |undefined subst|
+
+      link short.link_
+
+      lastParagraph
+      """
+    var warnings = new seq[string]
+    let output = input.toAst(warnings=warnings)
+    check(warnings[] == @[
+        "input(3, 14) Warning: broken link 'citation-som'",
+        "input(5, 7) Warning: broken link 'a-broken-link'",
+        "input(7, 15) Warning: unknown substitution 'undefined subst'",
+        "input(9, 6) Warning: broken link 'shortdotlink'"
+        ])
+
+  test "With include directive and blank lines at the beginning":
+    "other.rst".writeFile(dedent"""
+
+
+        firstParagraph
+
+        here brokenLink_""")
+    let input = ".. include:: other.rst"
+    var warnings = new seq[string]
+    let output = input.toAst(warnings=warnings)
+    check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"]
+    check(output == dedent"""
+      rnInner
+        rnParagraph
+          rnLeaf  'firstParagraph'
+        rnParagraph
+          rnLeaf  'here'
+          rnLeaf  ' '
+          rnRef
+            rnLeaf  'brokenLink'
+      """)
+    removeFile("other.rst")
+
 suite "RST include directive":
   test "Include whole":
     "other.rst".writeFile("**test1**")
@@ -374,7 +422,7 @@ OtherStart
 .. include:: other.rst
              :start-after: OtherStart
 """
-    doAssert "Visible" == rstTohtml(input, {}, defaultConfig())
+    check "Visible" == rstTohtml(input, {}, defaultConfig())
     removeFile("other.rst")
 
   test "Include everything before":
@@ -406,7 +454,7 @@ And this should **NOT** be visible in `docs.html`
              :start-after: OtherStart
              :end-before: OtherEnd
 """
-    doAssert "Visible" == rstTohtml(input, {}, defaultConfig())
+    check "Visible" == rstTohtml(input, {}, defaultConfig())
     removeFile("other.rst")
 
 
diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim
index d5d902e1d..867677404 100644
--- a/tests/stdlib/trstgen.nim
+++ b/tests/stdlib/trstgen.nim
@@ -237,8 +237,7 @@ not in table"""
 
    • 
-
    Precedence levelOperatorsFirst characterTerminal symbol
    D1 """ & id"code" & " " & op"\|" & """D2
    
 
    E1 | text
    
 
    F2 without pipe
    not in table
    
-""")
+
    not in table
    """)
     let input2 = """
 | A1 header | A2 |
 | --- | --- |"""
@@ -420,9 +419,10 @@ Some chapter
             "the following intermediate section level(s) are missing on " &
             "lines 12..15: underline -----)")
 
+  test "RST sections overline":
     # the same as input9good but with overline headings
     # first overline heading has a special meaning: document title
-    let input10 = dedent """
+    let input = dedent """
       ======
       Title0
       ======
@@ -454,22 +454,23 @@ Some chapter
       ~~~~~
 
       """
-    var option: bool
     var rstGenera: RstGenerator
-    var output10: string
-    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera.renderRstToOut(rstParse(input10, "", 1, 1, option, {}), output10)
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames = files)
+    rstGenera.renderRstToOut(rst, output)
     doAssert rstGenera.meta[metaTitle] == "Title0"
     doAssert rstGenera.meta[metaSubTitle] == "SubTitle0"
-    doAssert "
    Level1
    " in output10
-    doAssert "
    Level2
    " in output10
-    doAssert "
    Level3
    " in output10
-    doAssert "
    L1
    " in output10
-    doAssert "
    Another2
    " in output10
-    doAssert "
    More3
    " in output10
+    doAssert "
    Level1
    " in output
+    doAssert "
    Level2
    " in output
+    doAssert "
    Level3
    " in output
+    doAssert "
    L1
    " in output
+    doAssert "
    Another2
    " in output
+    doAssert "
    More3
    " in output
 
+  test "RST sections overline 2":
     # check that a paragraph prevents interpreting overlines as document titles
-    let input11 = dedent """
+    let input = dedent """
       Paragraph
 
       ======
@@ -480,18 +481,19 @@ Some chapter
       SubTitle0
       +++++++++
       """
-    var option11: bool
-    var rstGenera11: RstGenerator
-    var output11: string
-    rstGenera11.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera11.renderRstToOut(rstParse(input11, "", 1, 1, option11, {}), output11)
-    doAssert rstGenera11.meta[metaTitle] == ""
-    doAssert rstGenera11.meta[metaSubTitle] == ""
-    doAssert "
    Title0
    " in output11
-    doAssert "
    SubTitle0
    " in output11
+    var rstGenera: RstGenerator
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
+    rstGenera.renderRstToOut(rst, output)
+    doAssert rstGenera.meta[metaTitle] == ""
+    doAssert rstGenera.meta[metaSubTitle] == ""
+    doAssert "
    Title0
    " in output
+    doAssert "
    SubTitle0
    " in output
 
+  test "RST+Markdown sections":
     # check that RST and Markdown headings don't interfere
-    let input12 = dedent """
+    let input = dedent """
       ======
       Title0
       ======
@@ -509,14 +511,14 @@ Some chapter
       MySection2a
       -----------
       """
-    var option12: bool
-    var rstGenera12: RstGenerator
-    var output12: string
-    rstGenera12.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera12.renderRstToOut(rstParse(input12, "", 1, 1, option12, {roSupportMarkdown}), output12)
-    doAssert rstGenera12.meta[metaTitle] == "Title0"
-    doAssert rstGenera12.meta[metaSubTitle] == ""
-    doAssert output12 ==
+    var rstGenera: RstGenerator
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {roSupportMarkdown})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
+    rstGenera.renderRstToOut(rst, output)
+    doAssert rstGenera.meta[metaTitle] == "Title0"
+    doAssert rstGenera.meta[metaSubTitle] == ""
+    doAssert output ==
              "\n
    MySection1a
    " & # RST
              "\n
    MySection1b
    " & # Markdown
              "\n
    MySection1c
    " & # RST
@@ -628,7 +630,7 @@ let x = 1
     let p2 = """
    Par2 value2.
    """
     let p3 = """
    Par3 """ & id"value3" & ".
    "
     let p4 = """
    Par4 value4.
    """
-    let expected = p1 & p2 & "\n" & p3 & "\n" & p4 & "\n"
+    let expected = p1 & p2 & "\n" & p3 & "\n" & p4
     check(input.toHtml == expected)
 
   test "role directive":
@@ -653,8 +655,8 @@ Check that comment disappears:
     let output1 = input1.toHtml
     doAssert output1 == "Check that comment disappears:"
 
-  test "RST line blocks":
-    let input1 = """
+  test "RST line blocks + headings":
+    let input = """
 =====
 Test1
 =====
@@ -665,19 +667,20 @@ Test1
 | other line
 
 """
-    var option: bool
     var rstGenera: RstGenerator
-    var output1: string
-    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", {})
-    rstGenera.renderRstToOut(rstParse(input1, "", 1, 1, option, {}), output1)
+    var output: string
+    let (rst, files, _) = rstParse(input, "", 1, 1, {})
+    rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files)
+    rstGenera.renderRstToOut(rst, output)
     doAssert rstGenera.meta[metaTitle] == "Test1"
       # check that title was not overwritten to '|'
-    doAssert output1 == "


    line block
    other line
    "
-    let output1l = rstToLatex(input1, {})
+    doAssert output == "


    line block
    other line
    "
+    let output1l = rstToLatex(input, {})
     doAssert "line block\n\n" in output1l
     doAssert "other line\n\n" in output1l
     doAssert output1l.count("\\vspace") == 2 + 2  # +2 surrounding paddings
 
+  test "RST line blocks":
     let input2 = dedent"""
       Paragraph1
       
@@ -686,7 +689,7 @@ Test1
       Paragraph2"""
 
     let output2 = input2.toHtml
-    doAssert "Paragraph1

     
    Paragraph2
    \n" == output2
+    doAssert "Paragraph1

     
    Paragraph2
    " == output2
 
     let input3 = dedent"""
       | xxx
@@ -853,7 +856,7 @@ Test1
     check("input(6, 1) Warning: RST style: \n" &
           "not enough indentation on line 6" in warnings8[0])
     doAssert output8 == "Paragraph.
      " &
-        "
    1. stringA
      2. \n
    2. stringB
      3. \n
    \n
    C. string1 string2 
    \n"
+        "
  • stringA
    • \n
  • stringB
    • \n\n
    C. string1 string2 
    "
 
   test "Markdown enumerated lists":
     let input1 = dedent """
@@ -926,7 +929,7 @@ Test1
       Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_.
       """
     let output2 = input2.toHtml
-    doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_. "
+    doAssert output2 == "Not references[#note]_[1 #]_ [wrong citation]_ and [not&allowed]_."
 
     # check that auto-symbol footnotes work:
     let input3 = dedent """
@@ -1034,9 +1037,7 @@ Test1
       """
     var warnings8 = new seq[string]
     let output8 = input8.toHtml(warnings=warnings8)
-    # TODO: the line 1 is arbitrary because reference lines are not preserved
-    check(warnings8[] == @["input(1, 1) Warning: unknown substitution " &
-            "\'citation-som\'"])
+    check(warnings8[] == @["input(3, 7) Warning: broken link 'citation-som'"])
 
     # check that footnote group does not break parsing of other directives:
     let input9 = dedent """
@@ -1144,10 +1145,33 @@ Test1
       """
     var error = new string
     let output = input.toHtml(error=error)
-    check(error[] == "input(1, 1) Error: invalid field: " &
+    check(error[] == "input(2, 3) Error: invalid field: " &
                      "extra arguments were given to number-lines: ' let a = 1'")
     check "" == output
 
+  test "code-block warning":
+    let input = dedent """
+      .. code:: Nim
+         :unsupportedField: anything
+
+      .. code:: unsupportedLang
+
+         anything
+
+      ```anotherLang
+      someCode
+      ```
+      """
+    let warnings = new seq[string]
+    let output = input.toHtml(warnings=warnings)
+    check(warnings[] == @[
+        "input(2, 4) Warning: field 'unsupportedField' not supported",
+        "input(4, 11) Warning: language 'unsupportedLang' not supported",
+        "input(8, 4) Warning: language 'anotherLang' not supported"
+        ])
+    check(output == "
    anything
    " &
+                    "
    \nsomeCode\n
     
    ")
+
   test "RST admonitions":
     # check that all admonitions are implemented
     let input0 = dedent """
@@ -1477,7 +1501,7 @@ Test1
     check "(3, 15) Warning: " in warnings[1]
     check "language 'py:class' not supported" in warnings[1]
     check("""
    See function spam.
    """ & "\n" &
-          """
    See also egg. 
    """ & "\n" ==
+          """
    See also egg. 
    """ ==
           output)
 
   test "(not) Roles: check escaping 1":

From e38388eb4e92b0e7c7db204fe36d7969967cd609 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Tue, 20 Jul 2021 08:35:53 +0200
Subject: [PATCH 0243/1654] use .} instead of } for code highlighters

---
 lib/system/threads.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/system/threads.nim b/lib/system/threads.nim
index 7fbfe54ab..2b5eb07fd 100644
--- a/lib/system/threads.nim
+++ b/lib/system/threads.nim
@@ -41,7 +41,7 @@
 ##  for i in 0..high(thr):
 ##    createThread(thr[i], threadFunc, (i*10, i*10+5))
 ##  joinThreads(thr)
-## 
+##
 ##  deinitLock(L)
 
 when not declared(ThisIsSystem):
@@ -395,7 +395,7 @@ elif defined(netbsd):
 
 elif defined(freebsd):
   proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "".}
-  var SYS_thr_self {.importc:"SYS_thr_self", header:""}: cint
+  var SYS_thr_self {.importc:"SYS_thr_self", header:"".}: cint
 
   proc getThreadId*(): int =
     ## Gets the ID of the currently running thread.

From 5e011e8e42b2e4e322e9b40d4075da6d4a9103e5 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 20 Jul 2021 10:37:36 +0200
Subject: [PATCH 0244/1654] updated the changelog (#18539)

---
 changelog.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/changelog.md b/changelog.md
index 097beac2c..a633c1fb3 100644
--- a/changelog.md
+++ b/changelog.md
@@ -39,9 +39,6 @@
 - `math.round` now is rounded "away from zero" in JS backend which is consistent
   with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior.
 
-- Instead of deleting the element at the last index,
-  `system.delete()` now raises `IndexDefect` when given index is out of bounds.
-
 - Changed the behavior of `uri.decodeQuery` when there are unencoded `=`
   characters in the decoded values. Prior versions would raise an error. This is
   no longer the case to comply with the HTML spec and other languages

From 6286b5599c7d9fc03f3a6809f5c17fbefc23427e Mon Sep 17 00:00:00 2001
From: Paul Roberts 
Date: Tue, 20 Jul 2021 11:50:43 +0100
Subject: [PATCH 0245/1654] termux ssl (#18520)

---
 lib/pure/net.nim       | 2 +-
 lib/pure/ssl_certs.nim | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index 934dd034e..c2e4810de 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -647,7 +647,7 @@ when defineSsl:
       if verifyMode != CVerifyNone:
         # Use the caDir and caFile parameters if set
         if caDir != "" or caFile != "":
-          if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != VerifySuccess:
+          if newCTX.SSL_CTX_load_verify_locations(if caFile == "": nil else: caFile.cstring, if caDir == "": nil else: caDir.cstring) != VerifySuccess:
             raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).")
 
         else:
diff --git a/lib/pure/ssl_certs.nim b/lib/pure/ssl_certs.nim
index 2d2644ebe..c7ce04ffa 100644
--- a/lib/pure/ssl_certs.nim
+++ b/lib/pure/ssl_certs.nim
@@ -34,6 +34,7 @@ elif defined(linux):
     # Fedora/RHEL
     "/etc/pki/tls/certs",
     # Android
+    "/data/data/com.termux/files/usr/etc/tls/cert.pem",
     "/system/etc/security/cacerts",
   ]
 elif defined(bsd):

From dbf8d0b8946419311ac710b1adc995ad206b52c1 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 20 Jul 2021 20:12:15 +0800
Subject: [PATCH 0246/1654] use more `.}` (#18542)

---
 changelogs/changelog_0_19_0.md  | 2 +-
 doc/manual_experimental.rst     | 4 ++--
 lib/posix/posix_linux_amd64.nim | 2 +-
 lib/system.nim                  | 8 ++++----
 lib/windows/winlean.nim         | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/changelogs/changelog_0_19_0.md b/changelogs/changelog_0_19_0.md
index 18d3ca2b3..45cefe933 100644
--- a/changelogs/changelog_0_19_0.md
+++ b/changelogs/changelog_0_19_0.md
@@ -15,7 +15,7 @@
   at runtime. Use ``ref object`` with inheritance rather than ``object`` with
   inheritance to prevent this issue.
 - The ``not nil`` type annotation now has to be enabled explicitly
-  via ``{.experimental: "notnil"}`` as we are still not pleased with how this
+  via ``{.experimental: "notnil".}`` as we are still not pleased with how this
   feature works with Nim's containers.
 - The parser now warns about inconsistent spacing around binary operators as
   these can easily be confused with unary operators. This warning will likely
diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst
index d10ef8d1b..22db184a8 100644
--- a/doc/manual_experimental.rst
+++ b/doc/manual_experimental.rst
@@ -474,13 +474,13 @@ Not nil annotation
 ==================
 
 **Note:** This is an experimental feature. It can be enabled with
-`{.experimental: "notnil"}`.
+`{.experimental: "notnil".}`.
 
 All types for which `nil` is a valid value can be annotated with the
 `not nil` annotation to exclude `nil` as a valid value:
 
 .. code-block:: nim
-  {.experimental: "notnil"}
+  {.experimental: "notnil".}
 
   type
     PObject = ref TObj not nil
diff --git a/lib/posix/posix_linux_amd64.nim b/lib/posix/posix_linux_amd64.nim
index 5c0b4d6cf..2309e19a9 100644
--- a/lib/posix/posix_linux_amd64.nim
+++ b/lib/posix/posix_linux_amd64.nim
@@ -334,7 +334,7 @@ type
     si_status*: cint   ## Exit value or signal.
     si_band*: int      ## Band event for SIGPOLL.
     si_value*: SigVal  ## Signal value.
-    pad {.importc: "_pad"}: array[128 - 56, uint8]
+    pad {.importc: "_pad".}: array[128 - 56, uint8]
 
   Nl_item* {.importc: "nl_item", header: "".} = cint
   Nl_catd* {.importc: "nl_catd", header: "".} = pointer
diff --git a/lib/system.nim b/lib/system.nim
index 2790187f4..e878f9f2f 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -536,7 +536,7 @@ include "system/arithmetics"
 include "system/comparisons"
 
 const
-  appType* {.magic: "AppType"}: string = ""
+  appType* {.magic: "AppType".}: string = ""
     ## A string that describes the application type. Possible values:
     ## `"console"`, `"gui"`, `"lib"`.
 
@@ -1101,15 +1101,15 @@ const
     ## True only when accessed in the main module. This works thanks to
     ## compiler magic. It is useful to embed testing code in a module.
 
-  CompileDate* {.magic: "CompileDate"}: string = "0000-00-00"
+  CompileDate* {.magic: "CompileDate".}: string = "0000-00-00"
     ## The date (in UTC) of compilation as a string of the form
     ## `YYYY-MM-DD`. This works thanks to compiler magic.
 
-  CompileTime* {.magic: "CompileTime"}: string = "00:00:00"
+  CompileTime* {.magic: "CompileTime".}: string = "00:00:00"
     ## The time (in UTC) of compilation as a string of the form
     ## `HH:MM:SS`. This works thanks to compiler magic.
 
-  cpuEndian* {.magic: "CpuEndian"}: Endianness = littleEndian
+  cpuEndian* {.magic: "CpuEndian".}: Endianness = littleEndian
     ## The endianness of the target CPU. This is a valuable piece of
     ## information for low-level code only. This works thanks to compiler
     ## magic.
diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
index d901af244..b8f3886d7 100644
--- a/lib/windows/winlean.nim
+++ b/lib/windows/winlean.nim
@@ -1143,7 +1143,7 @@ proc setFileTime*(hFile: Handle, lpCreationTime: LPFILETIME,
 type
   # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid_identifier_authority
   SID_IDENTIFIER_AUTHORITY* {.importc, header: "".} = object
-    value* {.importc: "Value"}: array[6, BYTE]
+    value* {.importc: "Value".}: array[6, BYTE]
   # https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-sid
   SID* {.importc, header: "".} = object
     Revision: BYTE

From a8b3e7c05919511db62f1aabd706c46316b4f7b6 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 20 Jul 2021 21:32:55 +0800
Subject: [PATCH 0247/1654] follow up #18517 (#18537)

* follow up #18517

* Update lib/pure/os.nim

Co-authored-by: Timothee Cour 

* Update lib/pure/os.nim

Co-authored-by: Timothee Cour 
---
 lib/pure/os.nim | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 6289c64f0..73e3fcb31 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -11,10 +11,11 @@
 ## retrieving environment variables, reading command line arguments,
 ## working with directories, running shell commands, etc.
 
-runnableExamples("-r:off"):
+runnableExamples:
   let myFile = "/path/to/my/file.nim"
   assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim")
-  assert parentDir(myFile) == "/path/to/my"
+  when defined(posix):
+    assert parentDir(myFile) == "/path/to/my"
   assert splitFile(myFile) == (dir: "/path/to/my", name: "file", ext: ".nim")
   assert myFile.changeFileExt("c") == "/path/to/my/file.c"
 

From cf0cf32d276002e850a87667fff62c4df12999d6 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 20 Jul 2021 13:13:52 -0700
Subject: [PATCH 0248/1654] make -d:nimFpRoundtrips work consistently in vm vs
 rt, fix #18400, etc (#18531)

* compiler/vmhooks: add getVar to allow vmops with var params
* addFloat vmops with var param
* cgen now renders float32 literals in c backend using roundtrip float to string
---
 compiler/ast.nim                |   4 +-
 compiler/ccgexprs.nim           |  12 +--
 compiler/jsgen.nim              |   7 +-
 compiler/nim.cfg                |   1 +
 compiler/rodutils.nim           |  10 +-
 compiler/semfold.nim            |   1 -
 compiler/vmgen.nim              |   3 +-
 compiler/vmhooks.nim            |  32 +++---
 compiler/vmops.nim              |  11 +++
 lib/system.nim                  |   3 -
 lib/system/dollars.nim          |  13 +--
 lib/system/formatfloat.nim      | 166 ++++++++++++++++++++++----------
 lib/system/jssys.nim            |  18 +---
 lib/system/repr_v2.nim          |   6 +-
 lib/system/strmantle.nim        |  41 --------
 tests/config.nims               |   2 +
 tests/errmsgs/treportunused.nim |  14 ++-
 tests/float/nim.cfg             |   1 -
 tests/float/tfloats.nim         | 117 ++++++++++++++++++----
 tests/stdlib/tjson.nim          |   6 +-
 tests/stdlib/tjsonutils.nim     |  17 ++--
 21 files changed, 286 insertions(+), 199 deletions(-)
 delete mode 100644 tests/float/nim.cfg

diff --git a/compiler/ast.nim b/compiler/ast.nim
index f1f73dcee..8b836e088 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -652,7 +652,7 @@ type
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot,
     mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mImplies, mIff, mExists, mForall, mOld,
@@ -720,7 +720,7 @@ const
     mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor,
     mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI,
     mUnaryPlusF64, mUnaryMinusF64,
-    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr,
+    mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr,
     mStrToStr, mEnumToStr,
     mAnd, mOr,
     mEqStr, mLeStr, mLtStr,
diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 09326ceeb..79c73b602 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -94,9 +94,12 @@ proc genLiteral(p: BProc, n: PNode, ty: PType): Rope =
     else:
       result = makeCString(n.strVal)
   of nkFloatLit, nkFloat64Lit:
-    result = rope(n.floatVal.toStrMaxPrecision)
+    if ty.kind == tyFloat32:
+      result = rope(n.floatVal.float32.toStrMaxPrecision)
+    else:
+      result = rope(n.floatVal.toStrMaxPrecision)
   of nkFloat32Lit:
-    result = rope(n.floatVal.toStrMaxPrecision("f"))
+    result = rope(n.floatVal.float32.toStrMaxPrecision)
   else:
     internalError(p.config, n.info, "genLiteral(" & $n.kind & ')')
     result = nil
@@ -2300,11 +2303,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)")
   of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)")
   of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)")
-  of mFloatToStr:
-    if e[1].typ.skipTypes(abstractInst).kind == tyFloat32:
-      genDollar(p, e, d, "#nimFloat32ToStr($1)")
-    else:
-      genDollar(p, e, d, "#nimFloatToStr($1)")
   of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)")
   of mStrToStr, mUnown: expr(p, e[1], d)
   of mIsolate: genCall(p, e, d)
diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim
index 82fba02a6..4b89b0cde 100644
--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -434,7 +434,6 @@ const # magic checked op; magic unchecked op;
     mBoolToStr: ["nimBoolToStr", "nimBoolToStr"],
     mIntToStr: ["cstrToNimstr", "cstrToNimstr"],
     mInt64ToStr: ["cstrToNimstr", "cstrToNimstr"],
-    mFloatToStr: ["cstrToNimstr", "cstrToNimstr"],
     mCStrToStr: ["cstrToNimstr", "cstrToNimstr"],
     mStrToStr: ["", ""]]
 
@@ -656,9 +655,6 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
   of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
   of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")")
-  of mFloatToStr:
-    useMagic(p, "nimFloatToString")
-    applyFormat "cstrToNimstr(nimFloatToString($1))"
   of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)")
   of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1")
   else:
@@ -682,8 +678,7 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
     gen(p, n[1], x)
     gen(p, n[2], y)
     r.res = "($1 >>> $2)" % [x.rdLoc, y.rdLoc]
-  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr,
-      mCStrToStr, mStrToStr, mEnumToStr:
+  of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, mStrToStr, mEnumToStr:
     arithAux(p, n, r, op)
   of mEqRef:
     if mapType(n[1].typ) != etyBaseIndex:
diff --git a/compiler/nim.cfg b/compiler/nim.cfg
index 9ecd00b0b..f10d847ac 100644
--- a/compiler/nim.cfg
+++ b/compiler/nim.cfg
@@ -4,6 +4,7 @@ hint:XDeclaredButNotUsed:off
 
 define:booting
 define:nimcore
+define:nimFpRoundtrips
 
 #import:"$projectpath/testability"
 
diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim
index e13b08e05..a4f7a5146 100644
--- a/compiler/rodutils.nim
+++ b/compiler/rodutils.nim
@@ -39,7 +39,10 @@ when not declared(signbit):
   proc signbit*(x: SomeFloat): bool {.inline.} =
     result = c_signbit(x) != 0
 
-proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
+import system/formatfloat
+
+proc toStrMaxPrecision*(f: BiggestFloat | float32): string =
+  const literalPostfix = when f is float32: "f" else: ""
   case classify(f)
   of fcNan:
     if signbit(f):
@@ -55,9 +58,8 @@ proc toStrMaxPrecision*(f: BiggestFloat, literalPostfix = ""): string =
   of fcNegInf:
     result = "-INF"
   else:
-    result = newString(81)
-    let n = c_snprintf(result.cstring, result.len.uint, "%#.16e%s", f, literalPostfix.cstring)
-    setLen(result, n)
+    result.addFloatRoundtrip(f)
+    result.add literalPostfix
 
 proc encodeStr*(s: string, result: var string) =
   for i in 0..", discardable.}
 
-  proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
-    ## This is the implementation to format floats.
-    ##
-    ## returns the amount of bytes written to `buf` not counting the
-    ## terminating '\0' character.
-    result = toChars(buf, value, forceTrailingDotZero=true)
-    buf[result] = '\0'
+proc addCstringN(result: var string, buf: cstring; buflen: int) =
+  # no nimvm support needed, so it doesn't need to be fast here either
+  let oldLen = result.len
+  let newLen = oldLen + buflen
+  result.setLen newLen
+  c_memcpy(result[oldLen].addr, buf, buflen.csize_t)
 
-else:
-  proc c_sprintf(buf, frmt: cstring): cint {.header: "",
-                                      importc: "sprintf", varargs, noSideEffect.}
+import dragonbox, schubfach
 
-  proc writeToBuffer(buf: var array[65, char]; value: cstring) =
-    var i = 0
-    while value[i] != '\0':
-      buf[i] = value[i]
-      inc i
+proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: BiggestFloat): int =
+  ## This is the implementation to format floats.
+  ##
+  ## returns the amount of bytes written to `buf` not counting the
+  ## terminating '\0' character.
+  result = toChars(buf, value, forceTrailingDotZero=true)
+  buf[result] = '\0'
 
-  proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat): int =
-    ## This is the implementation to format floats.
-    ##
-    ## returns the amount of bytes written to `buf` not counting the
-    ## terminating '\0' character.
-    var n: int = c_sprintf(addr buf, "%.16g", value)
-    var hasDot = false
-    for i in 0..n-1:
-      if buf[i] == ',':
-        buf[i] = '.'
-        hasDot = true
-      elif buf[i] in {'a'..'z', 'A'..'Z', '.'}:
-        hasDot = true
-    if not hasDot:
-      buf[n] = '.'
-      buf[n+1] = '0'
-      buf[n+2] = '\0'
-      result = n + 2
+proc writeFloatToBufferRoundtrip*(buf: var array[65, char]; value: float32): int =
+  result = float32ToChars(buf, value, forceTrailingDotZero=true)
+  buf[result] = '\0'
+
+proc c_sprintf(buf, frmt: cstring): cint {.header: "",
+                                    importc: "sprintf", varargs, noSideEffect.}
+
+proc writeToBuffer(buf: var array[65, char]; value: cstring) =
+  var i = 0
+  while value[i] != '\0':
+    buf[i] = value[i]
+    inc i
+
+proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat): int =
+  ## This is the implementation to format floats.
+  ##
+  ## returns the amount of bytes written to `buf` not counting the
+  ## terminating '\0' character.
+  var n: int = c_sprintf(addr buf, "%.16g", value)
+  var hasDot = false
+  for i in 0..n-1:
+    if buf[i] == ',':
+      buf[i] = '.'
+      hasDot = true
+    elif buf[i] in {'a'..'z', 'A'..'Z', '.'}:
+      hasDot = true
+  if not hasDot:
+    buf[n] = '.'
+    buf[n+1] = '0'
+    buf[n+2] = '\0'
+    result = n + 2
+  else:
+    result = n
+  # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)'
+  # of '-1.#IND' are produced.
+  # We want to get rid of these here:
+  if buf[n-1] in {'n', 'N', 'D', 'd', ')'}:
+    writeToBuffer(buf, "nan")
+    result = 3
+  elif buf[n-1] == 'F':
+    if buf[0] == '-':
+      writeToBuffer(buf, "-inf")
+      result = 4
     else:
-      result = n
-    # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' or 'nan(ind)'
-    # of '-1.#IND' are produced.
-    # We want to get rid of these here:
-    if buf[n-1] in {'n', 'N', 'D', 'd', ')'}:
-      writeToBuffer(buf, "nan")
+      writeToBuffer(buf, "inf")
       result = 3
-    elif buf[n-1] == 'F':
-      if buf[0] == '-':
-        writeToBuffer(buf, "-inf")
-        result = 4
-      else:
-        writeToBuffer(buf, "inf")
-        result = 3
+
+proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} =
+  when defined(nimFpRoundtrips):
+    writeFloatToBufferRoundtrip(buf, value)
+  else:
+    writeFloatToBufferSprintf(buf, value)
+
+proc addFloatRoundtrip*(result: var string; x: float | float32) =
+  when nimvm:
+    doAssert false
+  else:
+    var buffer {.noinit.}: array[65, char]
+    let n = writeFloatToBufferRoundtrip(buffer, x)
+    result.addCstringN(cstring(buffer[0].addr), n)
+
+proc addFloatSprintf*(result: var string; x: float) =
+  when nimvm:
+    doAssert false
+  else:
+    var buffer {.noinit.}: array[65, char]
+    let n = writeFloatToBufferSprintf(buffer, x)
+    result.addCstringN(cstring(buffer[0].addr), n)
+
+proc nimFloatToString(a: float): cstring =
+  ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2
+  # print `-0.0` properly
+  asm """
+    function nimOnlyDigitsOrMinus(n) {
+      return n.toString().match(/^-?\d+$/);
+    }
+    if (Number.isSafeInteger(`a`))
+      `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"
+    else {
+      `result` = `a`+""
+      if(nimOnlyDigitsOrMinus(`result`)){
+        `result` = `a`+".0"
+      }
+    }
+  """
+
+proc addFloat*(result: var string; x: float | float32) {.inline.} =
+  ## Converts float to its string representation and appends it to `result`.
+  runnableExamples:
+    var
+      s = "foo:"
+      b = 45.67
+    s.addFloat(45.67)
+    assert s == "foo:45.67"
+  template impl =
+    when defined(nimFpRoundtrips):
+      addFloatRoundtrip(result, x)
+    else:
+      addFloatSprintf(result, x)
+  when defined(js):
+    when nimvm: impl()
+    else:
+      result.add nimFloatToString(x)
+  else: impl()
diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim
index 7cb5652c5..b42dc3a20 100644
--- a/lib/system/jssys.nim
+++ b/lib/system/jssys.nim
@@ -496,23 +496,6 @@ proc negInt(a: int): int {.compilerproc.} =
 proc negInt64(a: int64): int64 {.compilerproc.} =
   result = a*(-1)
 
-proc nimFloatToString(a: float): cstring {.compilerproc.} =
-  ## ensures the result doesn't print like an integer, i.e. return 2.0, not 2
-  # print `-0.0` properly
-  asm """
-    function nimOnlyDigitsOrMinus(n) {
-      return n.toString().match(/^-?\d+$/);
-    }
-    if (Number.isSafeInteger(`a`))
-      `result` = `a` === 0 && 1 / `a` < 0 ? "-0.0" : `a`+".0"
-    else {
-      `result` = `a`+""
-      if(nimOnlyDigitsOrMinus(`result`)){
-        `result` = `a`+".0"
-      }
-    }
-  """
-
 proc absInt(a: int): int {.compilerproc.} =
   result = if a < 0: a*(-1) else: a
 
@@ -703,6 +686,7 @@ proc addChar(x: string, c: char) {.compilerproc, asmNoStackFrame.} =
 {.pop.}
 
 proc tenToThePowerOf(b: int): BiggestFloat =
+  # xxx deadcode
   var b = b
   var a = 10.0
   result = 1.0
diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim
index 618cc2b40..8471ea148 100644
--- a/lib/system/repr_v2.nim
+++ b/lib/system/repr_v2.nim
@@ -19,9 +19,9 @@ proc repr*(x: uint64): string {.noSideEffect.} =
   ## converted to a decimal string.
   $x #Calls `$` from system/strmantle.nim
 
-proc repr*(x: float): string {.magic: "FloatToStr", noSideEffect.}
-  ## repr for a float argument. Returns `x`
-  ## converted to a decimal string.
+proc repr*(x: float): string =
+  ## Same as $x
+  $x
 
 proc repr*(x: bool): string {.magic: "BoolToStr", noSideEffect.}
   ## repr for a boolean argument. Returns `x`
diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim
index 041272175..81ac5c751 100644
--- a/lib/system/strmantle.nim
+++ b/lib/system/strmantle.nim
@@ -69,47 +69,6 @@ proc nimIntToStr(x: int): string {.compilerRtl.} =
   result = newStringOfCap(sizeof(x)*4)
   result.addInt x
 
-proc addCstringN(result: var string, buf: cstring; buflen: int) =
-  # no nimvm support needed, so it doesn't need to be fast here either
-  let oldLen = result.len
-  let newLen = oldLen + buflen
-  result.setLen newLen
-  copyMem(result[oldLen].addr, buf, buflen)
-
-import formatfloat
-
-proc addFloat*(result: var string; x: float) =
-  ## Converts float to its string representation and appends it to `result`.
-  ##
-  ## .. code-block:: Nim
-  ##   var
-  ##     a = "123"
-  ##     b = 45.67
-  ##   a.addFloat(b) # a <- "12345.67"
-  when nimvm:
-    result.add $x
-  else:
-    var buffer {.noinit.}: array[65, char]
-    let n = writeFloatToBuffer(buffer, x)
-    result.addCstringN(cstring(buffer[0].addr), n)
-
-proc nimFloatToStr(f: float): string {.compilerproc.} =
-  result = newStringOfCap(8)
-  result.addFloat f
-
-when defined(nimFpRoundtrips) and not defined(nimscript) and
-    not defined(js) and defined(nimHasDragonBox):
-  import schubfach
-
-proc nimFloat32ToStr(f: float32): string {.compilerproc.} =
-  when declared(float32ToChars):
-    result = newString(65)
-    let L = float32ToChars(result, f, forceTrailingDotZero=true)
-    setLen(result, L)
-  else:
-    result = newStringOfCap(8)
-    result.addFloat f
-
 proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {.
   importc: "strtod", header: "", noSideEffect.}
 
diff --git a/tests/config.nims b/tests/config.nims
index 539de5e8d..12b303318 100644
--- a/tests/config.nims
+++ b/tests/config.nims
@@ -34,3 +34,5 @@ hint("Processing", off)
 switch("define", "nimExperimentalAsyncjsThen")
 switch("define", "nimExperimentalJsfetch")
 switch("define", "nimExperimentalLinenoiseExtra")
+
+switch("define", "nimFpRoundtrips")
diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim
index f9b7c3d11..3105b35ea 100644
--- a/tests/errmsgs/treportunused.nim
+++ b/tests/errmsgs/treportunused.nim
@@ -13,12 +13,12 @@ treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUse
 treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed]
 treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed]
+treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
+treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 '''
 action: compile
 """
 
-#treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed]
-#treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed]
 # bug #9764
 iterator s1(a:string): int = discard
 iterator s2(): int = discard
@@ -32,9 +32,7 @@ var s9: int
 type s10 = object
 type s11 = type(1.2)
 
-when false:
-  # enabled again when Nim bootstraps with -d:nimFpRoundtrips
-  # https://github.com/nim-lang/Nim/issues/14407
-  let
-    `v0.99` = "0.99"
-    `v0.99.99` = "0.99.99"
+# bug #14407 (requires `compiler/nim.cfg` containing define:nimFpRoundtrips)
+let
+  `v0.99` = "0.99"
+  `v0.99.99` = "0.99.99"
diff --git a/tests/float/nim.cfg b/tests/float/nim.cfg
deleted file mode 100644
index d27bbf43b..000000000
--- a/tests/float/nim.cfg
+++ /dev/null
@@ -1 +0,0 @@
--d:nimFpRoundtrips
diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim
index 30d9c50d6..63987bb8d 100644
--- a/tests/float/tfloats.nim
+++ b/tests/float/tfloats.nim
@@ -1,13 +1,14 @@
 discard """
+  matrix: "-d:nimFpRoundtrips; -u:nimFpRoundtrips"
   targets: "c cpp js"
 """
-# disabled: "windows"
 
 #[
 xxx merge all or most float tests into this file
 ]#
 
 import std/[fenv, math, strutils]
+import stdtest/testutils
 
 proc equalsOrNaNs(a, b: float): bool =
   if isNaN(a): isNaN(b)
@@ -62,27 +63,103 @@ template main =
     reject "1_.0"
     reject "1.0_"
 
-  block: # bug #18148
-    var a = 1.1'f32
-    doAssert $a == "1.1", $a # was failing
+  block: # bugs mentioned in https://github.com/nim-lang/Nim/pull/18504#issuecomment-881635317
+    block: # example 1
+      let a = 0.1+0.2
+      doAssert a != 0.3
+      when defined(nimFpRoundtrips):
+        doAssert $a == "0.30000000000000004"
+      else:
+        whenRuntimeJs: discard
+        do: doAssert $a == "0.3"
+    block: # example 2
+      const a = 0.1+0.2
+      when defined(nimFpRoundtrips):
+        doAssert $($a, a) == """("0.30000000000000004", 0.30000000000000004)"""
+      else:
+        whenRuntimeJs: discard
+        do: doAssert $($a, a) == """("0.3", 0.3)"""
+    block: # example 3
+      const a1 = 0.1+0.2
+      let a2 = a1
+      doAssert a1 != 0.3
+      when defined(nimFpRoundtrips):
+        doAssert $[$a1, $a2] == """["0.30000000000000004", "0.30000000000000004"]"""
+      else:
+        whenRuntimeJs: discard
+        do: doAssert $[$a1, $a2] == """["0.3", "0.3"]"""
 
-proc runtimeOnlyTests =
-  # enable for 'static' once -d:nimFpRoundtrips became the default
-  block: # bug #7717
-    proc test(f: float) =
-      let f2 = $f
-      let f3 = parseFloat(f2)
-      doAssert equalsOrNaNs(f, f3), $(f, f2, f3)
+  when defined(nimFpRoundtrips):
+    block: # bug #18148
+      var a = 1.1'f32
+      doAssert $a == "1.1", $a # was failing
 
-    test 1.0 + epsilon(float64)
-    test 1000000.0000000123
-    test log2(100000.0)
-    test maximumPositiveValue(float32)
-    test maximumPositiveValue(float64)
-    test minimumPositiveValue(float32)
-    test minimumPositiveValue(float64)
+    block: # bug #18400
+      block:
+        let a1 = 0.1'f32
+        let a2 = 0.2'f32
+        let a3 = a1 + a2
+        var s = ""
+        s.addFloat(a3)
+        whenVMorJs: discard # xxx refs #12884
+        do:
+          doAssert a3 == 0.3'f32
+          doAssert $a3 == "0.3"
+
+      block:
+        let a1 = 0.1
+        let a2 = 0.2
+        let a3 = a1 + a2
+        var s = ""
+        s.addFloat(a3)
+        doAssert a3 != 0.3
+        doAssert $a3 == "0.30000000000000004"
+
+      block:
+        var s = [-13.888888'f32]
+        whenRuntimeJs: discard
+        do:
+          doAssert $s == "[-13.888888]"
+          doAssert $s[0] == "-13.888888"
+
+    block: # bug #7717
+      proc test(f: float) =
+        let f2 = $f
+        let f3 = parseFloat(f2)
+        doAssert equalsOrNaNs(f, f3), $(f, f2, f3)
+      test 1.0 + epsilon(float64)
+      test 1000000.0000000123
+      test log2(100000.0)
+      test maximumPositiveValue(float32)
+      test maximumPositiveValue(float64)
+      test minimumPositiveValue(float32)
+      test minimumPositiveValue(float64)
+
+    block: # bug #12884
+      block: # example 1
+        const x0: float32 = 1.32
+        let x1 = 1.32
+        let x2 = 1.32'f32
+        var x3: float32 = 1.32
+        doAssert $(x0, x1, x2, x3) == "(1.32, 1.32, 1.32, 1.32)"
+      block: # example https://github.com/nim-lang/Nim/issues/12884#issuecomment-564967962
+        let x = float(1.32'f32)
+        when nimvm: discard # xxx prints 1.3
+        else:
+          when not defined(js):
+            doAssert $x == "1.3200000524520874"
+        doAssert $1.32 == "1.32"
+        doAssert $1.32'f32 == "1.32"
+        let x2 = 1.32'f32
+        doAssert $x2 == "1.32"
+      block:
+        var x = 1.23456789012345'f32
+        when nimvm:
+          discard # xxx, refs #12884
+        else:
+          when not defined(js):
+            doAssert x == 1.2345679'f32
+            doAssert $x == "1.2345679"
 
 static: main()
 main()
-
-runtimeOnlyTests()
diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim
index 000f72038..289ef9d05 100644
--- a/tests/stdlib/tjson.nim
+++ b/tests/stdlib/tjson.nim
@@ -303,7 +303,7 @@ let jsonNode = %*mynode
 doAssert $jsonNode == """{"kind":"P","pChildren":[{"kind":"Text","textStr":"mychild"},{"kind":"Br"}]}"""
 doAssert $jsonNode.to(ContentNode) == """(kind: P, pChildren: @[(kind: Text, textStr: "mychild"), (kind: Br)])"""
 
-when defined(nimFpRoundtrips): # bug #17383
+block: # bug #17383
   testRoundtrip(int32.high): "2147483647"
   testRoundtrip(uint32.high): "4294967295"
   when int.sizeof == 4:
@@ -316,7 +316,7 @@ when defined(nimFpRoundtrips): # bug #17383
     testRoundtrip(int64.high): "9223372036854775807"
     testRoundtrip(uint64.high): "18446744073709551615"
 
-when defined(nimFpRoundtrips): # bug #18007
+block: # bug #18007
   testRoundtrip([NaN, Inf, -Inf, 0.0, -0.0, 1.0]): """["nan","inf","-inf",0.0,-0.0,1.0]"""
   # pending https://github.com/nim-lang/Nim/issues/18025 use:
   # testRoundtrip([float32(NaN), Inf, -Inf, 0.0, -0.0, 1.0])
@@ -332,7 +332,7 @@ when defined(nimFpRoundtrips): # bug #18007
     testRoundtripVal(0.0): "0.0"
     testRoundtripVal(-0.0): "-0.0"
 
-when defined(nimFpRoundtrips): # bug #15397, bug #13196
+block: # bug #15397, bug #13196
   testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
   testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
 
diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim
index 3e9c422e0..205160471 100644
--- a/tests/stdlib/tjsonutils.nim
+++ b/tests/stdlib/tjsonutils.nim
@@ -161,16 +161,15 @@ template fn() =
     doAssert b[2].signbit
     doAssert not b[3].signbit
 
-  when defined(nimFpRoundtrips):
-    block: # bug #15397, bug #13196
-      let a = 0.1
-      let x = 0.12345678901234567890123456789
-      let b = (a + 0.2, 0.3, x)
-      testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
+  block: # bug #15397, bug #13196
+    let a = 0.1
+    let x = 0.12345678901234567890123456789
+    let b = (a + 0.2, 0.3, x)
+    testRoundtripVal(b): "[0.30000000000000004,0.3,0.12345678901234568]"
 
-      testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
-      testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
-      testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
+    testRoundtripVal(0.12345678901234567890123456789): "0.12345678901234568"
+    testRoundtripVal(epsilon(float64)): "2.220446049250313e-16"
+    testRoundtripVal(1.0 + epsilon(float64)): "1.0000000000000002"
 
   block: # case object
     type Foo = object

From f8519657c43f458db9c915cec62c59022041eb05 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 20 Jul 2021 22:15:06 +0200
Subject: [PATCH 0249/1654] fixes #18469 (#18544)

* fixes #18469

* Update compiler/injectdestructors.nim
---
 compiler/injectdestructors.nim         | 25 ++++++++++++---
 compiler/jsgen.nim                     | 10 ------
 compiler/types.nim                     | 10 ++++++
 tests/arc/tcursor_field_obj_constr.nim | 44 ++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 15 deletions(-)
 create mode 100644 tests/arc/tcursor_field_obj_constr.nim

diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim
index 6e4eaa817..cf09b3ff5 100644
--- a/compiler/injectdestructors.nim
+++ b/compiler/injectdestructors.nim
@@ -774,7 +774,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result = passCopyToSink(n, c, s)
   else:
     case n.kind
-    of nkBracket, nkObjConstr, nkTupleConstr, nkClosure, nkCurly:
+    of nkBracket, nkTupleConstr, nkClosure, nkCurly:
       # Let C(x) be the construction, 'x' the vector of arguments.
       # C(x) either owns 'x' or it doesn't.
       # If C(x) owns its data, we must consume C(x).
@@ -785,13 +785,11 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       # don't destroy it"
       # but if C(x) is a ref it MUST own its data since we must destroy it
       # so then we have no choice but to use 'sinkArg'.
-      let isRefConstr = n.kind == nkObjConstr and n.typ.skipTypes(abstractInst).kind == tyRef
-      let m = if isRefConstr: sinkArg
-              elif mode == normal: normal
+      let m = if mode == normal: normal
               else: sinkArg
 
       result = copyTree(n)
-      for i in ord(n.kind in {nkObjConstr, nkClosure})..
Date: Wed, 21 Jul 2021 08:51:03 +0200
Subject: [PATCH 0250/1654] =?UTF-8?q?undo=20RFC=20#294,=20it=20breaks=20co?=
 =?UTF-8?q?de=20for=20no=20good=20reason,=20the=20compiler=20can=20wa?=
 =?UTF-8?q?=E2=80=A6=20(#18546)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* undo RFC #294, it breaks code for no good reason, the compiler can warn about the construct instead

* Update changelog.md

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* enable test case

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 changelog.md           | 9 ++++-----
 compiler/lineinfos.nim | 2 ++
 compiler/semexprs.nim  | 4 +---
 compiler/types.nim     | 2 --
 tests/misc/tconv.nim   | 8 +++-----
 5 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/changelog.md b/changelog.md
index a633c1fb3..1a904d043 100644
--- a/changelog.md
+++ b/changelog.md
@@ -23,15 +23,14 @@
 - `repr` now doesn't insert trailing newline; previous behavior was very inconsistent,
   see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior.
 
-- An enum now can't be converted to another enum directly, you must use `ord` (or `cast`, but
-  compiler won't help if you misuse it).
+- A type conversion from one enum type to another now produces an `[EnumConv]` warning.
+  You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead.
   ```
   type A = enum a1, a2
   type B = enum b1, b2
-  doAssert not compiles(a1.B)
-  doAssert compiles(a1.ord.B)
+  echo a1.B # produces a warning
+  echo a1.ord.B # produces no warning
   ```
-  for a transition period, use `-d:nimLegacyConvEnumEnum`.
 
 - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
   behavior.
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 8ab52a452..a3243d5fc 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -70,6 +70,7 @@ type
     warnResultUsed = "ResultUsed",
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
+    warnSuspiciousEnumConv = "EnumConv",
     warnUser = "User",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
@@ -153,6 +154,7 @@ const
     warnResultUsed: "used 'result' variable",
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
+    warnSuspiciousEnumConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index f46d44bf0..c25a4067f 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -168,9 +168,7 @@ proc checkConvertible(c: PContext, targetTyp: PType, src: PNode): TConvStatus =
   elif (targetBaseTyp.kind in IntegralTypes) and
       (srcBaseTyp.kind in IntegralTypes):
     if targetTyp.kind == tyEnum and srcBaseTyp.kind == tyEnum:
-      if c.config.isDefined("nimLegacyConvEnumEnum"):
-        message(c.config, src.info, warnUser, "enum to enum conversion is now deprecated")
-      else: result = convNotLegal
+      message(c.config, src.info, warnSuspiciousEnumConv, "suspicious code: enum to enum conversion")
     # `elif` would be incorrect here
     if targetTyp.kind == tyBool:
       discard "convOk"
diff --git a/compiler/types.nim b/compiler/types.nim
index 0a50af56e..798f872c2 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1598,8 +1598,6 @@ proc typeMismatch*(conf: ConfigRef; info: TLineInfo, formal, actual: PType, n: P
       of efLockLevelsDiffer:
         msg.add "\nlock levels differ"
 
-    if formal.kind == tyEnum and actual.kind == tyEnum:
-      msg.add "\nmaybe use `-d:nimLegacyConvEnumEnum` for a transition period"
     localError(conf, info, msg)
 
 proc isTupleRecursive(t: PType, cycleDetector: var IntSet): bool =
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index c93fc57f8..94677b1bf 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -1,7 +1,5 @@
 discard """
-  nimout:'''
-tconv.nim(81, 15) Warning: enum to enum conversion is now deprecated [User]
-'''
+  matrix: "--warningAsError:EnumConv"
 """
 
 template reject(x) =
@@ -77,12 +75,12 @@ block: # https://github.com/nim-lang/RFCs/issues/294
   reject: k2.Goo
   reject: k2.string
 
-  {.define(nimLegacyConvEnumEnum).}
+  {.push warningAsError[EnumConv]:off.}
   discard Goo(k2)
   accept: Goo(k2)
   accept: k2.Goo
   reject: k2.string
-  {.undef(nimLegacyConvEnumEnum).}
+  {.pop.}
 
   reject: Goo(k2)
   reject: k2.Goo

From b6f9f7a33eb550a75350c2bce8423269734bb185 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Wed, 21 Jul 2021 09:09:43 +0200
Subject: [PATCH 0251/1654] atlas: minor changes

---
 tools/atlas/atlas.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
index 371a1c13f..b10ec0358 100644
--- a/tools/atlas/atlas.nim
+++ b/tools/atlas/atlas.nim
@@ -14,7 +14,7 @@ import parse_requires, osutils, packagesjson
 
 const
   Version = "0.1"
-  Usage = "atlas - Nim Package Manager Version " & Version & """
+  Usage = "atlas - Nim Package Cloner Version " & Version & """
 
   (c) 2021 Andreas Rumpf
 Usage:
@@ -170,7 +170,7 @@ proc toName(p: string): PackageName =
   else:
     result = PackageName p
 
-proc needsCommitLookup(commit: string): bool {.inline} =
+proc needsCommitLookup(commit: string): bool {.inline.} =
   '.' in commit or commit == InvalidCommit
 
 proc isShortCommitHash(commit: string): bool {.inline.} =

From 9c2442af94fc3db8486ee16e82ea27752e9ae4f5 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 21 Jul 2021 09:46:18 +0200
Subject: [PATCH 0252/1654] unary slices get a deprecation period (#18549)

---
 changelog.md   |  3 +--
 lib/system.nim | 17 ++++++++---------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/changelog.md b/changelog.md
index 1a904d043..ece88f710 100644
--- a/changelog.md
+++ b/changelog.md
@@ -63,8 +63,7 @@
 - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`,
   `hashes.hash(closure)` has also been improved.
 
-- The unary slice `..b` was removed, use `0..b` instead or use `-d:nimLegacyUnarySlice`
-  for a deprecation period.
+- The unary slice `..b` was deprecated, use `0..b` instead.
 
 - Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`.
 
diff --git a/lib/system.nim b/lib/system.nim
index 1da1444d2..e6a2439df 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -511,15 +511,14 @@ proc `..`*[T, U](a: sink T, b: sink U): HSlice[T, U] {.noSideEffect, inline, mag
   ##   echo a[2 .. 3] # @[30, 40]
   result = HSlice[T, U](a: a, b: b)
 
-when defined(nimLegacyUnarySlice):
-  proc `..`*[T](b: sink T): HSlice[int, T]
-    {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} =
-    ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`.
-    ##
-    ## .. code-block:: Nim
-    ##   let a = [10, 20, 30, 40, 50]
-    ##   echo a[.. 2] # @[10, 20, 30]
-    result = HSlice[int, T](a: 0, b: b)
+proc `..`*[T](b: sink T): HSlice[int, T]
+  {.noSideEffect, inline, magic: "DotDot", deprecated: "replace `..b` with `0..b`".} =
+  ## Unary `slice`:idx: operator that constructs an interval `[default(int), b]`.
+  ##
+  ## .. code-block:: Nim
+  ##   let a = [10, 20, 30, 40, 50]
+  ##   echo a[.. 2] # @[10, 20, 30]
+  result = HSlice[int, T](a: 0, b: b)
 
 when defined(hotCodeReloading):
   {.pragma: hcrInline, inline.}

From f048dad7c3af7d70fbb852a59cb5bdda332e96d7 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 21 Jul 2021 15:47:06 +0800
Subject: [PATCH 0253/1654] add testcase for #6499 #12229 #7172 (#18547)

---
 tests/magics/tmagics.nim |  8 ++++++++
 tests/threads/t7172.nim  | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 tests/threads/t7172.nim

diff --git a/tests/magics/tmagics.nim b/tests/magics/tmagics.nim
index fa138320c..70926a0d7 100644
--- a/tests/magics/tmagics.nim
+++ b/tests/magics/tmagics.nim
@@ -55,3 +55,11 @@ block t9442:
   GC_unref(v2)
   GC_ref(v3)
   GC_unref(v3)
+
+block: # bug #6499
+  let x = (chr, 0)
+  doAssert x[1] == 0
+
+block: # bug #12229
+  proc foo(T: typedesc) = discard
+  foo(ref)
diff --git a/tests/threads/t7172.nim b/tests/threads/t7172.nim
new file mode 100644
index 000000000..983765dba
--- /dev/null
+++ b/tests/threads/t7172.nim
@@ -0,0 +1,32 @@
+discard """
+  output: '''
+In doStuff()
+In initProcess()
+initProcess() done
+TEST
+Crashes before getting here!
+'''
+  joinable: false
+"""
+
+import std/os
+
+proc whatever() {.thread, nimcall.} =
+  echo("TEST")
+
+proc initProcess(): void =
+  echo("In initProcess()")
+  var thread: Thread[void]
+  createThread(thread, whatever)
+  echo("initProcess() done")
+  joinThread(thread)
+
+proc doStuff(): void =
+  echo("In doStuff()")
+  # ...
+  initProcess()
+  sleep(500)
+  # ...
+  echo("Crashes before getting here!")
+
+doStuff()

From f86a530214d56f39664d6aaca3ddddf3b2fdba0c Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 21 Jul 2021 10:22:30 +0200
Subject: [PATCH 0254/1654] breaking changes policies (#18541)

* document some std library evolution policies

In wanting to improve the standard library, it's helpful to have a set
of principles and guidelines to lean back on, that show how to introduce
such improvements while at the same time considering existing users of
the language.

A key idea behind documentation of this sort is that it should highlight
a path forwards in making changes rather than trying to prevent them,
and although the current snippet does include some language for what one
shouldn't do, it also shows a few options for what one can do.

This is a riff on https://github.com/nim-lang/Nim/pull/18468 mainly
based on what helps enable to the use of Nim productively in
environments and teams where the priority and focus is not always on the
tool (Nim in this case) but rather on the codebase itself, and its use
by end users.

We use similar guidance documentation to help coordinate the code of the
teams using Nim in https://status-im.github.io/nim-style-guide/ where it
is applied not as law, but rather as recommendations for the default
approach that should be considered first.

* document the new policies

* typo

* Update doc/contributing.rst

Co-authored-by: Timothee Cour 

* Update doc/contributing.rst

* Update doc/contributing.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Update doc/contributing.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Update doc/contributing.rst

* Update doc/contributing.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* clarify some things

* Update doc/contributing.rst

Co-authored-by: Dominik Picheta 

Co-authored-by: Jacek Sieka 
Co-authored-by: Timothee Cour 
Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Co-authored-by: Dominik Picheta 
---
 doc/contributing.rst | 114 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 114 insertions(+)

diff --git a/doc/contributing.rst b/doc/contributing.rst
index 5a31d85bd..07fa01714 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -671,3 +671,117 @@ Conventions
    Furthermore, module names should use `snake_case` and not use capital
    letters, which cause issues when going from an OS without case
    sensitivity to an OS with it.
+
+
+Breaking Changes
+================
+
+Introducing breaking changes, no matter how well intentioned,
+creates long-term problems for the community, in particular those looking to promote
+reusable Nim code in libraries: In the Nim distribution, critical security and bugfixes,
+language changes and community improvements are bundled in a single distribution - it is
+difficult to make partial upgrades with only benign changes. When one library depends on
+a legacy behavior, it can no longer be used together with another library that does not,
+breaking all downstream applications - the standard library is unique in that it sits at
+the root of **all** dependency trees.
+
+There is a big difference between compile-time breaking changes and run-time breaking
+changes.
+
+
+Run-time breaking changes
+-------------------------
+
+Run-time breaking changes are to be avoided at almost all costs: Nim is used for
+mission critical applications which depend on behaviours that
+are not covered by the test suite. As such, it's important that changes to the
+*stable* parts of the standard library are made avoiding changing the existing
+behaviors, even when the test suite continues to pass.
+
+Examples of run-time breaking changes:
+
+- Raising exceptions of a new type, compared to what's currently being raised.
+
+- Adding unconstrained or poorly constrained generic procs or macros
+  ("hash now works for all `ref T`"): This may cause code to behave differently
+  depending only on which modules are imported - common examples include `==` and `hash`.
+
+- Changing behavior of existing functions like:
+
+  * "Nim's path handling procs like `getXDir` now consistently lack the trailing slash"
+  * "Nim's strformat implementation is now more consistent with Python"
+
+Instead write new code that explicitly announces the feature you think we announced but
+didn't. For example, `strformat` does not say "it's compatible with Python", it
+says "inspired by Python's f-strings". This new code can be submitted to the stdlib
+and the old code can be deprecated or it can be published as a Nimble package.
+
+Sometimes, a run-time breaking change is most desirable: For example, a string
+representation of a floating point number that "roundtrips" is much better than
+a string represenation that doesn't. These run-time breaking changes must start in the
+state "opt-in" via a new `-d:nimPreviewX` or command line flag and then should become
+the new default later, in follow-up versions. This way users can track
+regressions more easily. ("git bisect" is not an acceptable alternative, that's for
+Nim compiler developers, not for Nim users.)
+
+Above all else, additive approaches that don't change existing behaviors
+should be preferred.
+
+
+Compile-time breaking changes
+-----------------------------
+
+Compile-time breaking changes are usually easier to handle, but for large code bases
+it can also be much work and it can hinder the adoption of a new Nim release.
+Additive approaches are to be preferred here as well.
+
+Examples of compile-time breaking changes include (but are not limited to):
+
+* Renaming functions and modules, or moving things. Instead of a direct rename,
+  deprecate the old name and introduce a new one.
+* Renaming the parameter names: Thanks to Nim's "named parameter" calling syntax
+  like `f(x = 0, y = 1)` this is a breaking change. Instead live with the existing
+  parameter names.
+* Adding an enum value to an existing enum. Nim's exhaustive case statements stop
+  compiling after such a change. Instead consider to introduce new `bool`
+  fields/parameters. This can be impractical though, so we use good judgement
+  and our list of "important packages" to see if it doesn't break too much code
+  out there in practice.
+* Adding a new proc to an existing stdlib module. However, for aesthetic reasons
+  this is often preferred over introducing a new module with just a single proc
+  inside. We use good judgement and our list of "important packages" to see if
+  it doesn't break too much code out there in practice. The new procs need to
+  be annotated with a `.since` annotation.
+
+
+Compiler/language spec bugfixes
+-------------------------------
+
+This can even be applied to compiler "bugfixes": If the compiler should have been
+"pickier" in its handling of `typedesc`, instead of "fixing typedesc handling bugs",
+consider the following solution:
+
+- Spec out how `typedesc` should really work and also spec out the cases where it
+  should not be allowed!
+- Deprecate `typedesc` and name the new metatype something new like `typeArg`.
+- Implement the spec.
+
+
+Non-breaking changes
+--------------------
+
+Examples of changes that are considered non-breaking (or acceptable breaking changes) include:
+
+* Creating a new module.
+* Adding an overload to an already overloaded proc.
+* Adding new default parameters to an existing proc. It is assumed that you do not
+  use Nim's stdlib procs's addresses (that you don't use them as first class entities).
+* Changing the calling convention from `nimcall` to `inline`
+  (but first RFC https://github.com/nim-lang/RFCs/issues/396 needs to be implemented).
+* Changing the behavior from "crashing" into some other, well documented result (including
+  raising a Defect, but not raising an exception that does not inherit from Defect).
+* Adding new fields to an existing object.
+
+Nim's introspection facilities imply that strictly speaking almost every addition can
+break somebody's code. It is impractical to care about these cases, a change that only
+affects introspection is not considered to be a breaking change.

From 6f34829ee5a060300859002fd41f0181c0adf754 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Wed, 21 Jul 2021 19:46:17 +0800
Subject: [PATCH 0255/1654] clean unused imports on runnableExamples (#18551)

---
 lib/std/monotimes.nim | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim
index 3bb4fa141..bd078d7c0 100644
--- a/lib/std/monotimes.nim
+++ b/lib/std/monotimes.nim
@@ -15,8 +15,6 @@ meaning that that the following is guaranteed to work:
 ]##
 
 runnableExamples:
-  import std/os
-
   let a = getMonoTime()
   let b = getMonoTime()
   assert a <= b

From 880353c0ad9300f2543c1b16548b4f2ab6784183 Mon Sep 17 00:00:00 2001
From: IterableTrucks <42210451+IterableTrucks@users.noreply.github.com>
Date: Wed, 21 Jul 2021 20:13:50 +0800
Subject: [PATCH 0256/1654] complement notes of two doc strings of tables
 module (#18527)

* complement notes of two doc strings warning the deprecated  procedure's side effect

* Update lib/pure/collections/tables.nim

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

* Update lib/pure/collections/tables.nim

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: BillyZee 
Co-authored-by: Andreas Rumpf 
Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 lib/pure/collections/tables.nim | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index d617e06e4..b845f57cc 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -496,6 +496,8 @@ template tabCellHash(i)  = t.data[i].hcode
 proc del*[A, B](t: var Table[A, B], key: A) =
   ## Deletes `key` from hash table `t`. Does nothing if the key does not exist.
   ##
+  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ##
   ## See also:
   ## * `pop proc<#pop,Table[A,B],A,B>`_
   ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table
@@ -514,6 +516,8 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   ## mapping of the key. Otherwise, returns `false`, and the `val` is
   ## unchanged.
   ##
+  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ##
   ## See also:
   ## * `del proc<#del,Table[A,B],A>`_
   ## * `clear proc<#clear,Table[A,B]>`_ to empty the whole table

From 58080525a1dcad23de6ad0cf9ca6988f103de350 Mon Sep 17 00:00:00 2001
From: itsumura-h <39766805+itsumura-h@users.noreply.github.com>
Date: Wed, 21 Jul 2021 22:03:48 +0900
Subject: [PATCH 0257/1654] add [1..2] for JArray (#18525)

* add [1..2] for JArray
* fix BackwardsIndex to int
* fix for BackwardsIndex
* fix for assert node kind check
* fix variable name
* Update lib/pure/json.nim
* fix for when x.a is BackwardsIndex

Co-authored-by: itsumura-h 
Co-authored-by: Dominik Picheta 
---
 lib/pure/json.nim | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index d5bbb86b8..85c3393b2 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -531,6 +531,24 @@ proc `[]`*(node: JsonNode, index: BackwardsIndex): JsonNode {.inline, since: (1,
 
   `[]`(node, node.len - int(index))
 
+proc `[]`*[U, V](a: JsonNode, x: HSlice[U, V]): JsonNode =
+  ## Slice operation for JArray.
+  ##
+  ## Returns the inclusive range `[a[x.a], a[x.b]]`:
+  runnableExamples:
+    import json
+    let arr = %[0,1,2,3,4,5]
+    doAssert arr[2..4] == %[2,3,4]
+    doAssert arr[2..^2] == %[2,3,4]
+    doAssert arr[^4..^2] == %[2,3,4]
+
+  assert(a.kind == JArray)
+  result = newJArray()
+  let xa = (when x.a is BackwardsIndex: a.len - int(x.a) else: int(x.a))
+  let L = (when x.b is BackwardsIndex: a.len - int(x.b) else: int(x.b)) - xa + 1
+  for i in 0..
Date: Wed, 21 Jul 2021 16:55:50 +0200
Subject: [PATCH 0258/1654] fixes #18550 (#18553)

* fixes #18550

* update the manual to reflect reality
---
 changelog.md            |  4 ++++
 compiler/lineinfos.nim  |  2 ++
 compiler/sempass2.nim   | 16 ++++++++++++++++
 compiler/sighashes.nim  |  4 +++-
 compiler/types.nim      | 15 +++++++++++++++
 doc/manual.rst          |  7 +++----
 lib/system/io.nim       |  8 ++++----
 lib/system/widestrs.nim |  2 +-
 tests/misc/tconv.nim    | 10 +++++++++-
 9 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/changelog.md b/changelog.md
index ece88f710..f2a9ca914 100644
--- a/changelog.md
+++ b/changelog.md
@@ -32,6 +32,10 @@
   echo a1.ord.B # produces no warning
   ```
 
+- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning.
+  This warning will become an error in future versions! Use an explicit conversion
+  like `cstring(x)` in order to silence the warning.
+
 - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
   behavior.
 
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index a3243d5fc..3f31ceb97 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -71,6 +71,7 @@ type
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
     warnSuspiciousEnumConv = "EnumConv",
+    warnCstringConv = "CStringConv",
     warnUser = "User",
     # hints
     hintSuccess = "Success", hintSuccessX = "SuccessX",
@@ -155,6 +156,7 @@ const
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
     warnSuspiciousEnumConv: "$1",
+    warnCstringConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index 1e1672cad..ce21c3ed2 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -951,6 +951,15 @@ proc trackInnerProc(tracked: PEffects, n: PNode) =
   else:
     for ch in n: trackInnerProc(tracked, ch)
 
+proc allowCStringConv(n: PNode): bool =
+  case n.kind
+  of nkStrLit..nkTripleStrLit: result = true
+  of nkSym: result = n.sym.kind in {skConst, skParam}
+  of nkAddr: result = isCharArrayPtr(n.typ, true)
+  of nkCallKinds:
+    result = isCharArrayPtr(n.typ, n[0].kind == nkSym and n[0].sym.magic == mAddr)
+  else: result = isCharArrayPtr(n.typ, false)
+
 proc track(tracked: PEffects, n: PNode) =
   case n.kind
   of nkSym:
@@ -1157,6 +1166,13 @@ proc track(tracked: PEffects, n: PNode) =
       if tracked.owner.kind != skMacro:
         createTypeBoundOps(tracked, n.typ, n.info)
   of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    if n.kind in {nkHiddenStdConv, nkHiddenSubConv} and
+        n.typ.skipTypes(abstractInst).kind == tyCstring and
+        not allowCStringConv(n[1]):
+      message(tracked.config, n.info, warnCstringConv,
+        "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
+          [$n[1]])
+
     if n.len == 2:
       track(tracked, n[1])
       if tracked.owner.kind != skMacro:
diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim
index f6df422e6..0ff97017f 100644
--- a/compiler/sighashes.nim
+++ b/compiler/sighashes.nim
@@ -14,7 +14,9 @@ from hashes import Hash
 import types
 
 proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
-proc `&=`(c: var MD5Context, ch: char) = md5Update(c, unsafeAddr ch, 1)
+proc `&=`(c: var MD5Context, ch: char) =
+  # XXX suspicious code here; relies on ch being zero terminated?
+  md5Update(c, unsafeAddr ch, 1)
 proc `&=`(c: var MD5Context, r: Rope) =
   for l in leaves(r): md5Update(c, l, l.len)
 proc `&=`(c: var MD5Context, i: BiggestInt) =
diff --git a/compiler/types.nim b/compiler/types.nim
index 798f872c2..f0bd99a17 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1664,3 +1664,18 @@ proc lookupFieldAgain*(ty: PType; field: PSym): PSym =
     if result != nil: break
     ty = ty[0]
   if result == nil: result = field
+
+proc isCharArrayPtr*(t: PType; allowPointerToChar: bool): bool =
+  let t = t.skipTypes(abstractInst)
+  if t.kind == tyPtr:
+    let pointsTo = t[0].skipTypes(abstractInst)
+    case pointsTo.kind
+    of tyUncheckedArray:
+      result = pointsTo[0].kind == tyChar
+    of tyArray:
+      result = pointsTo[1].kind == tyChar and firstOrd(nil, pointsTo[0]) == 0 and
+        skipTypes(pointsTo[0], {tyRange}).kind in {tyInt..tyInt64}
+    of tyChar:
+      result = allowPointerToChar
+    else:
+      discard
diff --git a/doc/manual.rst b/doc/manual.rst
index f0bd85638..9f614c2cd 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -1380,10 +1380,9 @@ variadic proc, it is implicitly converted to `cstring` too:
 
 Even though the conversion is implicit, it is not *safe*: The garbage collector
 does not consider a `cstring` to be a root and may collect the underlying
-memory. However, in practice, this almost never happens as the GC considers
-stack roots conservatively. One can use the builtin procs `GC_ref` and
-`GC_unref` to keep the string data alive for the rare cases where it does
-not work.
+memory. For this reason, the implicit conversion will be removed in future
+releases of the Nim compiler. Certain idioms like conversion of a `const` string
+to `cstring` are safe and will remain to be allowed.
 
 A `$` proc is defined for cstrings that returns a string. Thus to get a nim
 string from a cstring:
diff --git a/lib/system/io.nim b/lib/system/io.nim
index 591ea3e92..d8b2ac741 100644
--- a/lib/system/io.nim
+++ b/lib/system/io.nim
@@ -630,8 +630,8 @@ const
         ""
     else:
       ""
-  FormatOpen: array[FileMode, string] = [
-    "rb" & NoInheritFlag, "wb" & NoInheritFlag, "w+b" & NoInheritFlag,
+  FormatOpen: array[FileMode, cstring] = [
+    cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag,
     "r+b" & NoInheritFlag, "ab" & NoInheritFlag
   ]
     #"rt", "wt", "w+t", "r+t", "at"
@@ -678,7 +678,7 @@ proc open*(f: var File, filename: string,
   ## This throws no exception if the file could not be opened.
   ##
   ## The file handle associated with the resulting `File` is not inheritable.
-  var p = fopen(filename, FormatOpen[mode])
+  var p = fopen(filename.cstring, FormatOpen[mode])
   if p != nil:
     var f2 = cast[File](p)
     when defined(posix) and not defined(nimscript):
@@ -711,7 +711,7 @@ proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {.
   ## Default mode is readonly. Returns true if the file could be reopened.
   ##
   ## The file handle associated with `f` won't be inheritable.
-  if freopen(filename, FormatOpen[mode], f) != nil:
+  if freopen(filename.cstring, FormatOpen[mode], f) != nil:
     when not defined(nimInheritHandles) and declared(setInheritable) and
          NoInheritFlag.len == 0:
       if not setInheritable(getOsFileHandle(f), false):
diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim
index a999c9167..8b08959b5 100644
--- a/lib/system/widestrs.nim
+++ b/lib/system/widestrs.nim
@@ -171,7 +171,7 @@ proc newWideCString*(s: cstring): WideCStringObj =
   result = newWideCString(s, s.len)
 
 proc newWideCString*(s: string): WideCStringObj =
-  result = newWideCString(s, s.len)
+  result = newWideCString(cstring s, s.len)
 
 proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string =
   result = newStringOfCap(estimate + estimate shr 2)
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index 94677b1bf..1cb4b4fbf 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -1,5 +1,5 @@
 discard """
-  matrix: "--warningAsError:EnumConv"
+  matrix: "--warningAsError:EnumConv --warningAsError:CStringConv"
 """
 
 template reject(x) =
@@ -84,3 +84,11 @@ block: # https://github.com/nim-lang/RFCs/issues/294
 
   reject: Goo(k2)
   reject: k2.Goo
+
+reject:
+  # bug #18550
+  proc f(c: char): cstring =
+    var x = newString(109*1024*1024)
+    x[0] = c
+    x
+

From d5b5827bc2a3724290d3257b0f2fa70004c37c97 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Wed, 21 Jul 2021 23:00:29 -0700
Subject: [PATCH 0259/1654] fix setCommand so it behaves like regular nim
 invocation (#18555)

---
 compiler/cmdlinehelper.nim |  9 +--------
 compiler/options.nim       | 10 ++++++++++
 compiler/scriptconfig.nim  | 11 +----------
 3 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim
index c6a0f200a..007dc2319 100644
--- a/compiler/cmdlinehelper.nim
+++ b/compiler/cmdlinehelper.nim
@@ -44,14 +44,7 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   elif self.supportsStdinFile and conf.projectName == "-":
     handleStdinInput(conf)
   elif conf.projectName != "":
-    try:
-      conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
-    except OSError:
-      conf.projectFull = AbsoluteFile conf.projectName
-    let p = splitFile(conf.projectFull)
-    let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
-    conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
-    conf.projectName = p.name
+    setFromProjectName(conf, conf.projectName)
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
diff --git a/compiler/options.nim b/compiler/options.nim
index 45ca14e76..f02f02256 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -695,6 +695,16 @@ proc setDefaultLibpath*(conf: ConfigRef) =
 proc canonicalizePath*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
   result = AbsoluteFile path.string.expandFilename
 
+proc setFromProjectName*(conf: ConfigRef; projectName: string) =
+  try:
+    conf.projectFull = canonicalizePath(conf, AbsoluteFile projectName)
+  except OSError:
+    conf.projectFull = AbsoluteFile projectName
+  let p = splitFile(conf.projectFull)
+  let dir = if p.dir.isEmpty: AbsoluteDir getCurrentDir() else: p.dir
+  conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile dir)
+  conf.projectName = p.name
+
 proc removeTrailingDirSep*(path: string): string =
   if (path.len > 0) and (path[^1] == DirSep):
     result = substr(path, 0, path.len - 2)
diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim
index ef1ceb12b..89510f6b2 100644
--- a/compiler/scriptconfig.nim
+++ b/compiler/scriptconfig.nim
@@ -151,18 +151,9 @@ proc setupVM*(module: PSym; cache: IdentCache; scriptName: string;
     setResult(a, strutils.cmpIgnoreCase(a.getString 0, a.getString 1))
   cbconf setCommand:
     conf.setCommandEarly(a.getString 0)
-    # xxx move remaining logic to commands.nim or other
     let arg = a.getString 1
     incl(conf.globalOptions, optWasNimscript)
-    if arg.len > 0:
-      conf.projectName = arg
-      let path =
-        if conf.projectName.isAbsolute: AbsoluteFile(conf.projectName)
-        else: conf.projectPath / RelativeFile(conf.projectName)
-      try:
-        conf.projectFull = canonicalizePath(conf, path)
-      except OSError:
-        conf.projectFull = path
+    if arg.len > 0: setFromProjectName(conf, arg)
   cbconf getCommand:
     setResult(a, conf.command)
   cbconf switch:

From 58e27ebd4a16ca9f2ba24826d7162384d5321101 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 22 Jul 2021 09:37:41 +0200
Subject: [PATCH 0260/1654] fixes #12815 (#18554)

---
 changelog.md           |  3 +++
 compiler/ast.nim       |  2 +-
 compiler/lineinfos.nim |  4 +++-
 compiler/sempass2.nim  |  5 ++++-
 doc/nimc.rst           |  6 ++++++
 tests/misc/tconv.nim   | 13 +++++++++++++
 6 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/changelog.md b/changelog.md
index f2a9ca914..8e8b0a908 100644
--- a/changelog.md
+++ b/changelog.md
@@ -36,6 +36,9 @@
   This warning will become an error in future versions! Use an explicit conversion
   like `cstring(x)` in order to silence the warning.
 
+- There is a new warning for *any* type conversion to enum that can be enabled via
+  `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`.
+
 - Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous
   behavior.
 
diff --git a/compiler/ast.nim b/compiler/ast.nim
index 8b836e088..0eeaa8133 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -1064,7 +1064,7 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
   # which may simplify code.
   case a.kind
   of nkSym: a.sym.name
-  of nkIdent:  a.ident
+  of nkIdent: a.ident
   else: nil
 
 proc getnimblePkg*(a: PSym): PSym =
diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 3f31ceb97..8cb834156 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -71,6 +71,7 @@ type
     warnCannotOpen = "CannotOpen",
     warnFileChanged = "FileChanged",
     warnSuspiciousEnumConv = "EnumConv",
+    warnAnyEnumConv = "AnyEnumConv",
     warnCstringConv = "CStringConv",
     warnUser = "User",
     # hints
@@ -156,6 +157,7 @@ const
     warnCannotOpen: "cannot open: $1",
     warnFileChanged: "file changed: $1",
     warnSuspiciousEnumConv: "$1",
+    warnAnyEnumConv: "$1",
     warnCstringConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
@@ -211,7 +213,7 @@ type
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores, warnResultUsed, warnAnyEnumConv}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext, hintDeclaredLoc, hintProcessingStmt}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index ce21c3ed2..87ed81098 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -1171,7 +1171,10 @@ proc track(tracked: PEffects, n: PNode) =
         not allowCStringConv(n[1]):
       message(tracked.config, n.info, warnCstringConv,
         "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
-          [$n[1]])
+          $n[1])
+
+    if n.typ.skipTypes(abstractInst).kind == tyEnum:
+      message(tracked.config, n.info, warnAnyEnumConv, "enum conversion: $1" % $n[1])
 
     if n.len == 2:
       track(tracked, n[1])
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 070956860..3c451706b 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -70,6 +70,12 @@ SmallLshouldNotBeUsed            The letter 'l' should not be used as an
                                  identifier.
 EachIdentIsTuple                 The code contains a confusing `var`
                                  declaration.
+CStringConv                      Warn about dangerous implicit conversions
+                                 to `cstring`.
+EnumConv                         Warn about conversions from enum to enum.
+AnyEnumConv                      Warn about any conversions to an enum type.
+ResultUsed                       Warn about the usage of the
+                                 built-in `result` variable.
 User                             Some user-defined warning.
 ==========================       ============================================
 
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index 1cb4b4fbf..ff6875021 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -92,3 +92,16 @@ reject:
     x[0] = c
     x
 
+{.push warning[AnyEnumConv]:on, warningAsError[AnyEnumConv]:on.}
+
+reject:
+  # bug #12815
+  type
+    Foo = enum
+      one = 1
+      three = 3
+
+  var va = 2
+  var vb = va.Foo
+
+{.pop.}

From 0c4582c66524d58a3c72827d9546a5c5f1c40286 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Thu, 22 Jul 2021 11:59:21 +0200
Subject: [PATCH 0261/1654] mitigates #12815 (#18557)

* mitigates #12815

* Update doc/nimc.rst

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>

Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
---
 compiler/lineinfos.nim |  2 ++
 compiler/sempass2.nim  |  8 ++++++--
 doc/nimc.rst           |  2 ++
 tests/misc/tconv.nim   | 19 ++++++++++++++++---
 4 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim
index 8cb834156..c64d7b9dd 100644
--- a/compiler/lineinfos.nim
+++ b/compiler/lineinfos.nim
@@ -72,6 +72,7 @@ type
     warnFileChanged = "FileChanged",
     warnSuspiciousEnumConv = "EnumConv",
     warnAnyEnumConv = "AnyEnumConv",
+    warnHoleEnumConv = "HoleEnumConv",
     warnCstringConv = "CStringConv",
     warnUser = "User",
     # hints
@@ -158,6 +159,7 @@ const
     warnFileChanged: "file changed: $1",
     warnSuspiciousEnumConv: "$1",
     warnAnyEnumConv: "$1",
+    warnHoleEnumConv: "$1",
     warnCstringConv: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim
index 87ed81098..e2f85dd38 100644
--- a/compiler/sempass2.nim
+++ b/compiler/sempass2.nim
@@ -1173,8 +1173,12 @@ proc track(tracked: PEffects, n: PNode) =
         "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
           $n[1])
 
-    if n.typ.skipTypes(abstractInst).kind == tyEnum:
-      message(tracked.config, n.info, warnAnyEnumConv, "enum conversion: $1" % $n[1])
+    let t = n.typ.skipTypes(abstractInst)
+    if t.kind == tyEnum:
+      if tfEnumHasHoles in t.flags:
+        message(tracked.config, n.info, warnHoleEnumConv, "conversion to enum with holes is unsafe: $1" % $n)
+      else:
+        message(tracked.config, n.info, warnAnyEnumConv, "enum conversion: $1" % $n)
 
     if n.len == 2:
       track(tracked, n[1])
diff --git a/doc/nimc.rst b/doc/nimc.rst
index 3c451706b..ed9077912 100644
--- a/doc/nimc.rst
+++ b/doc/nimc.rst
@@ -74,6 +74,8 @@ CStringConv                      Warn about dangerous implicit conversions
                                  to `cstring`.
 EnumConv                         Warn about conversions from enum to enum.
 AnyEnumConv                      Warn about any conversions to an enum type.
+HoleEnumConv                     Warn about conversion to an enum with
+                                 holes. These conversions are unsafe.
 ResultUsed                       Warn about the usage of the
                                  built-in `result` variable.
 User                             Some user-defined warning.
diff --git a/tests/misc/tconv.nim b/tests/misc/tconv.nim
index ff6875021..5e8eac729 100644
--- a/tests/misc/tconv.nim
+++ b/tests/misc/tconv.nim
@@ -95,13 +95,26 @@ reject:
 {.push warning[AnyEnumConv]:on, warningAsError[AnyEnumConv]:on.}
 
 reject:
-  # bug #12815
   type
     Foo = enum
-      one = 1
-      three = 3
+      one
+      three
 
   var va = 2
   var vb = va.Foo
 
 {.pop.}
+
+{.push warningAsError[HoleEnumConv]:on.}
+
+reject:
+  # bug #12815
+  type
+    Hole = enum
+      one = 1
+      three = 3
+
+  var va = 2
+  var vb = va.Hole
+
+{.pop.}

From 5386ae75ba91ee2fe280f1fe807676a7c9af4b13 Mon Sep 17 00:00:00 2001
From: Jason Beetham 
Date: Fri, 23 Jul 2021 00:46:13 -0600
Subject: [PATCH 0262/1654] Fixed template const tuple unpacking (#18562)

---
 compiler/semtempl.nim | 40 ++++++++++++++++++----------------------
 tests/vm/tconst.nim   | 10 ++++++++++
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim
index 92cc0be8c..a2b0f99ba 100644
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -209,7 +209,7 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
       let pragmaNode = n[1]
       for i in 0..
Date: Fri, 23 Jul 2021 08:04:29 +0000
Subject: [PATCH 0263/1654] Replace calls to `putenv` with `setenv` (#18530)

* Replace calls to C `putenv` with C `setenv` to remove possible memory leaks

* Add test of correct behaviour on invalid input

* Fix style in tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Update tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Update tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Add comment with bug number to tests/stdlib/tos.nim

Co-authored-by: Timothee Cour 

* Fix possible msvc arch issues

Co-authored-by: Timothee Cour 
---
 lib/pure/includes/osenv.nim | 11 ++++++++---
 tests/stdlib/tos.nim        |  3 +++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim
index d0c92d566..db291c5fd 100644
--- a/lib/pure/includes/osenv.nim
+++ b/lib/pure/includes/osenv.nim
@@ -45,8 +45,10 @@ else:
 
   proc c_getenv(env: cstring): cstring {.
     importc: "getenv", header: "".}
-  proc c_putenv(env: cstring): cint {.
-    importc: "putenv", header: "".}
+  when defined(vcc):
+    proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".}
+  else:
+    proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".}
   proc c_unsetenv(env: cstring): cint {.
     importc: "unsetenv", header: "".}
 
@@ -220,8 +222,11 @@ else:
           if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError())
         else:
           if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError())
+      elif defined(vcc):
+        if c_putenv_s(key, val) != 0'i32:
+          raiseOSError(osLastError())
       else:
-        if c_putenv(environment[indx]) != 0'i32:
+        if c_setenv(key, val, 1'i32) != 0'i32:
           raiseOSError(osLastError())
 
   proc delEnv*(key: string) {.tags: [WriteEnvEffect].} =
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 912558461..8a12b9b90 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -614,6 +614,9 @@ block osenv:
     doAssert existsEnv(dummyEnvVar) == false
     delEnv(dummyEnvVar)         # deleting an already deleted env var
     doAssert existsEnv(dummyEnvVar) == false
+  block: # putEnv, bug #18502
+    doAssertRaises(OSError): putEnv("DUMMY_ENV_VAR_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE")
+    doAssertRaises(OSError): putEnv("", "NEW_DUMMY_VALUE")
   block:
     doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", "") == ""
     doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", " ") == " "

From 76f74fae88fba72f58e43ad5c8fd20a7c4d8b439 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Fri, 23 Jul 2021 04:41:16 -0700
Subject: [PATCH 0264/1654] std/random: fix overflow bugs; fixes #16360; fixes
 #16296; fixes #17670 (#18456)

---
 lib/pure/random.nim      | 32 ++++++++++++++------
 tests/stdlib/trandom.nim | 63 +++++++++++++++++++++++++++++++++++++++-
 2 files changed, 85 insertions(+), 10 deletions(-)

diff --git a/lib/pure/random.nim b/lib/pure/random.nim
index 07db0365c..a292386af 100644
--- a/lib/pure/random.nim
+++ b/lib/pure/random.nim
@@ -204,6 +204,19 @@ proc skipRandomNumbers*(s: var Rand) =
   s.a0 = s0
   s.a1 = s1
 
+proc rand[T: uint | uint64](r: var Rand; max: T): T =
+  # xxx export in future work
+  if max == 0: return
+  else:
+    let max = uint64(max)
+    when T.high.uint64 == uint64.high:
+      if max == uint64.high: return T(next(r))
+    while true:
+      let x = next(r)
+      # avoid `mod` bias
+      if x <= randMax - (randMax mod max):
+        return T(x mod (max + 1))
+
 proc rand*(r: var Rand; max: Natural): int {.benign.} =
   ## Returns a random integer in the range `0..max` using the given state.
   ##
@@ -213,15 +226,13 @@ proc rand*(r: var Rand; max: Natural): int {.benign.} =
   ## * `rand proc<#rand,Rand,HSlice[T: Ordinal or float or float32 or float64,T: Ordinal or float or float32 or float64]>`_
   ##   that accepts a slice
   ## * `rand proc<#rand,typedesc[T]>`_ that accepts an integer or range type
-  runnableExamples("-r:off"):
+  runnableExamples:
     var r = initRand(123)
-    assert r.rand(100) == 96 # implementation defined
-
-  if max == 0: return
-  while true:
-    let x = next(r)
-    if x <= randMax - (randMax mod Ui(max)):
-      return int(x mod (uint64(max) + 1u64))
+    if false:
+      assert r.rand(100) == 96 # implementation defined
+  # bootstrap: can't use `runnableExamples("-r:off")`
+  cast[int](rand(r, uint64(max)))
+    # xxx toUnsigned pending https://github.com/nim-lang/Nim/pull/18445
 
 proc rand*(max: int): int {.benign.} =
   ## Returns a random integer in the range `0..max`.
@@ -306,7 +317,10 @@ proc rand*[T: Ordinal or SomeFloat](r: var Rand; x: HSlice[T, T]): T =
   when T is SomeFloat:
     result = rand(r, x.b - x.a) + x.a
   else: # Integers and Enum types
-    result = T(rand(r, int(x.b) - int(x.a)) + int(x.a))
+    when defined(js):
+      result = cast[T](rand(r, cast[uint](x.b) - cast[uint](x.a)) + cast[uint](x.a))
+    else:
+      result = cast[T](rand(r, cast[uint64](x.b) - cast[uint64](x.a)) + cast[uint64](x.a))
 
 proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T =
   ## For a slice `a..b`, returns a value in the range `a..b`.
diff --git a/tests/stdlib/trandom.nim b/tests/stdlib/trandom.nim
index e47ddad66..39ccca85b 100644
--- a/tests/stdlib/trandom.nim
+++ b/tests/stdlib/trandom.nim
@@ -3,7 +3,9 @@ discard """
   targets: "c js"
 """
 
-import std/[random, math, os, stats, sets, tables]
+import std/[random, math, stats, sets, tables]
+when not defined(js):
+  import std/os
 
 randomize(233)
 
@@ -187,3 +189,62 @@ block: # bug #17467
     doAssert x > 1e-4, $(x, i)
       # This used to fail for each i in 0..<26844, i.e. the 1st produced value
       # was predictable and < 1e-4, skewing distributions.
+
+const withUint = false # pending exporting `proc rand[T: uint | uint64](r: var Rand; max: T): T =`
+
+block: # bug #16360
+  var r = initRand()
+  template test(a) =
+    let a2 = a
+    block:
+      let a3 = r.rand(a2)
+      doAssert a3 <= a2
+      doAssert a3.type is a2.type
+    block:
+      let a3 = rand(a2)
+      doAssert a3 <= a2
+      doAssert a3.type is a2.type
+  when withUint:
+    test cast[uint](int.high)
+    test cast[uint](int.high) + 1
+    when not defined(js):
+      # pending bug #16411
+      test uint64.high
+      test uint64.high - 1
+    test uint.high - 2
+    test uint.high - 1
+    test uint.high
+  test int.high
+  test int.high - 1
+  test int.high - 2
+  test 0
+  when withUint:
+    test 0'u
+    test 0'u64
+
+block: # bug #16296
+  var r = initRand()
+  template test(x) =
+    let a2 = x
+    let a3 = r.rand(a2)
+    doAssert a3 <= a2.b
+    doAssert a3 >= a2.a
+    doAssert a3.type is a2.a.type
+  test(-2 .. int.high-1)
+  test(int.low .. int.high)
+  test(int.low+1 .. int.high)
+  test(int.low .. int.high-1)
+  test(int.low .. 0)
+  test(int.low .. -1)
+  test(int.low .. 1)
+  test(int64.low .. 1'i64)
+  when not defined(js):
+    # pending bug #16411
+    test(10'u64 .. uint64.high)
+
+block: # bug #17670
+  when not defined(js):
+    # pending bug #16411
+    type UInt48 = range[0'u64..2'u64^48-1]
+    let x = rand(UInt48)
+    doAssert x is UInt48

From faabcfa6432cd15ecfee44a061ebf7738b16ad6c Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Fri, 23 Jul 2021 16:15:40 +0200
Subject: [PATCH 0265/1654] fixes #18558 (#18563)

* fixes #18558

* better fix
---
 compiler/ccgcalls.nim | 44 +++++++++++++++++++++++++++----------------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim
index 616028ac6..510b8d9ec 100644
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -317,36 +317,46 @@ proc skipTrivialIndirections(n: PNode): PNode =
   result = n
   while true:
     case result.kind
-    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
+    of nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv:
       result = result[0]
-    of {nkHiddenStdConv, nkHiddenSubConv}:
+    of nkHiddenStdConv, nkHiddenSubConv:
       result = result[1]
     else: break
 
-proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
+proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
   case n.kind:
   of nkLiterals, nkIdent: discard
   of nkSym:
     if mutate: result.add n
   of nkAsgn, nkFastAsgn:
-    result.add getPotentialWrites(n[0], true)
-    result.add getPotentialWrites(n[1], mutate)
+    getPotentialWrites(n[0], true, result)
+    getPotentialWrites(n[1], mutate, result)
   of nkAddr, nkHiddenAddr:
-    result.add getPotentialWrites(n[0], true)
-  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
-    for s in n.sons:
-      result.add getPotentialWrites(s, true)
+    getPotentialWrites(n[0], true, result)
+  of nkCallKinds:
+    case n.getMagic:
+    of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,
+        mAddr, mNew, mNewFinalize, mWasMoved, mDestroy, mReset:
+      getPotentialWrites(n[1], true, result)
+      for i in 2..
Date: Fri, 23 Jul 2021 14:57:47 -0700
Subject: [PATCH 0266/1654] rename nimFpRoundtrips => nimPreviewFloatRoundtrip
 (#18566)

---
 changelog.md                    |  4 ++--
 compiler/nim.cfg                |  2 +-
 lib/system/formatfloat.nim      |  4 ++--
 tests/config.nims               |  2 +-
 tests/errmsgs/treportunused.nim |  2 +-
 tests/float/tfloats.nim         | 10 +++++-----
 6 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/changelog.md b/changelog.md
index 8e8b0a908..d33e24b65 100644
--- a/changelog.md
+++ b/changelog.md
@@ -112,8 +112,8 @@
 
 - `system.addFloat` and `system.$` now can produce string representations of floating point numbers
   that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has
-  to be enabled via `-d:nimFpRoundtrips`. It is expected that this behavior becomes the new default
-  in upcoming versions.
+  to be enabled via `-d:nimPreviewFloatRoundtrip`. It is expected that this behavior becomes the new default
+  in upcoming versions, as with other `nimPreviewX` define flags.
 
 - Fixed buffer overflow bugs in `net`
 
diff --git a/compiler/nim.cfg b/compiler/nim.cfg
index f10d847ac..ec51bd463 100644
--- a/compiler/nim.cfg
+++ b/compiler/nim.cfg
@@ -4,7 +4,7 @@ hint:XDeclaredButNotUsed:off
 
 define:booting
 define:nimcore
-define:nimFpRoundtrips
+define:nimPreviewFloatRoundtrip
 
 #import:"$projectpath/testability"
 
diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim
index cb46c6ea7..3bcd3257b 100644
--- a/lib/system/formatfloat.nim
+++ b/lib/system/formatfloat.nim
@@ -74,7 +74,7 @@ proc writeFloatToBufferSprintf*(buf: var array[65, char]; value: BiggestFloat):
       result = 3
 
 proc writeFloatToBuffer*(buf: var array[65, char]; value: BiggestFloat | float32): int {.inline.} =
-  when defined(nimFpRoundtrips):
+  when defined(nimPreviewFloatRoundtrip):
     writeFloatToBufferRoundtrip(buf, value)
   else:
     writeFloatToBufferSprintf(buf, value)
@@ -121,7 +121,7 @@ proc addFloat*(result: var string; x: float | float32) {.inline.} =
     s.addFloat(45.67)
     assert s == "foo:45.67"
   template impl =
-    when defined(nimFpRoundtrips):
+    when defined(nimPreviewFloatRoundtrip):
       addFloatRoundtrip(result, x)
     else:
       addFloatSprintf(result, x)
diff --git a/tests/config.nims b/tests/config.nims
index 12b303318..f876cb612 100644
--- a/tests/config.nims
+++ b/tests/config.nims
@@ -35,4 +35,4 @@ switch("define", "nimExperimentalAsyncjsThen")
 switch("define", "nimExperimentalJsfetch")
 switch("define", "nimExperimentalLinenoiseExtra")
 
-switch("define", "nimFpRoundtrips")
+switch("define", "nimPreviewFloatRoundtrip")
diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim
index 3105b35ea..f5ee79afa 100644
--- a/tests/errmsgs/treportunused.nim
+++ b/tests/errmsgs/treportunused.nim
@@ -32,7 +32,7 @@ var s9: int
 type s10 = object
 type s11 = type(1.2)
 
-# bug #14407 (requires `compiler/nim.cfg` containing define:nimFpRoundtrips)
+# bug #14407 (requires `compiler/nim.cfg` containing define:nimPreviewFloatRoundtrip)
 let
   `v0.99` = "0.99"
   `v0.99.99` = "0.99.99"
diff --git a/tests/float/tfloats.nim b/tests/float/tfloats.nim
index 63987bb8d..480396e81 100644
--- a/tests/float/tfloats.nim
+++ b/tests/float/tfloats.nim
@@ -1,5 +1,5 @@
 discard """
-  matrix: "-d:nimFpRoundtrips; -u:nimFpRoundtrips"
+  matrix: "-d:nimPreviewFloatRoundtrip; -u:nimPreviewFloatRoundtrip"
   targets: "c cpp js"
 """
 
@@ -67,14 +67,14 @@ template main =
     block: # example 1
       let a = 0.1+0.2
       doAssert a != 0.3
-      when defined(nimFpRoundtrips):
+      when defined(nimPreviewFloatRoundtrip):
         doAssert $a == "0.30000000000000004"
       else:
         whenRuntimeJs: discard
         do: doAssert $a == "0.3"
     block: # example 2
       const a = 0.1+0.2
-      when defined(nimFpRoundtrips):
+      when defined(nimPreviewFloatRoundtrip):
         doAssert $($a, a) == """("0.30000000000000004", 0.30000000000000004)"""
       else:
         whenRuntimeJs: discard
@@ -83,13 +83,13 @@ template main =
       const a1 = 0.1+0.2
       let a2 = a1
       doAssert a1 != 0.3
-      when defined(nimFpRoundtrips):
+      when defined(nimPreviewFloatRoundtrip):
         doAssert $[$a1, $a2] == """["0.30000000000000004", "0.30000000000000004"]"""
       else:
         whenRuntimeJs: discard
         do: doAssert $[$a1, $a2] == """["0.3", "0.3"]"""
 
-  when defined(nimFpRoundtrips):
+  when defined(nimPreviewFloatRoundtrip):
     block: # bug #18148
       var a = 1.1'f32
       doAssert $a == "1.1", $a # was failing

From f4ff276a90091f2b88c191aac5a201dcb3ec9795 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Sat, 24 Jul 2021 00:30:02 +0200
Subject: [PATCH 0267/1654] refactoring: removed dead code (#18567)

---
 compiler/semdata.nim      |  2 +-
 compiler/semexprs.nim     | 19 +++++++------------
 compiler/semobjconstr.nim |  5 ++---
 compiler/semstmts.nim     |  2 +-
 4 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/compiler/semdata.nim b/compiler/semdata.nim
index 12bb22ff9..0a6d09a56 100644
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -65,7 +65,7 @@ type
       # you may be in position to supply a better error message
       # to the user.
     efWantStmt, efAllowStmt, efDetermineType, efExplain,
-    efAllowDestructor, efWantValue, efOperand, efNoSemCheck,
+    efWantValue, efOperand, efNoSemCheck,
     efNoEvaluateGeneric, efInCall, efFromHlo, efNoSem2Check,
     efNoUndeclared
       # Use this if undeclared identifiers should not raise an error during
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index c25a4067f..42c6b465e 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -623,10 +623,10 @@ proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
           localError(c.config, x.info, "invalid order in array constructor")
         x = x[1]
 
-      let xx = semExprWithType(c, x, flags*{efAllowDestructor})
+      let xx = semExprWithType(c, x, {})
       result.add xx
       typ = commonType(c, typ, xx.typ)
-      #n[i] = semExprWithType(c, x, flags*{efAllowDestructor})
+      #n[i] = semExprWithType(c, x, {})
       #result.add fitNode(c, typ, n[i])
       inc(lastIndex)
     addSonSkipIntLit(result.typ, typ, c.idgen)
@@ -1758,13 +1758,9 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
     localError(c.config, a.info, errXCannotBeAssignedTo %
                renderTree(a, {renderNoComments}))
   else:
-    let
-      lhs = n[0]
-      lhsIsResult = lhs.kind == nkSym and lhs.sym.kind == skResult
-    var
-      rhs = semExprWithType(c, n[1],
-        if lhsIsResult: {efAllowDestructor} else: {})
-    if lhsIsResult:
+    let lhs = n[0]
+    let rhs = semExprWithType(c, n[1], {})
+    if lhs.kind == nkSym and lhs.sym.kind == skResult:
       n.typ = c.enforceVoidContext
       if c.p.owner.kind != skMacro and resultTypeIsInferrable(lhs.sym.typ):
         var rhsTyp = rhs.typ
@@ -2512,8 +2508,7 @@ proc semTupleFieldsConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
     let id = considerQuotedIdent(c, n[i][0])
     if containsOrIncl(ids, id.id):
       localError(c.config, n[i].info, errFieldInitTwice % id.s)
-    n[i][1] = semExprWithType(c, n[i][1],
-                                        flags*{efAllowDestructor})
+    n[i][1] = semExprWithType(c, n[i][1], {})
 
     if n[i][1].typ.kind == tyTypeDesc:
       localError(c.config, n[i][1].info, "typedesc not allowed as tuple field.")
@@ -2533,7 +2528,7 @@ proc semTuplePositionsConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   result.transitionSonsKind(nkTupleConstr)
   var typ = newTypeS(tyTuple, c)  # leave typ.n nil!
   for i in 0..
Date: Sun, 25 Jul 2021 09:06:04 -0700
Subject: [PATCH 0268/1654] fixes #18385 (#18571)

needs a more permanent fix by better incorporating IC, but fixes
nimsuggest highlighting for now.
---
 compiler/vmgen.nim | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index be86550ae..4957a3339 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -2254,8 +2254,16 @@ proc optimizeJumps(c: PCtx; start: int) =
     else: discard
 
 proc genProc(c: PCtx; s: PSym): int =
-  let pos = c.procToCodePos.getOrDefault(s.id)
-  if pos == 0:
+  let
+    pos = c.procToCodePos.getOrDefault(s.id)
+    wasNotGenProcBefore = pos == 0
+    noRegistersAllocated = s.offset == -1
+  if wasNotGenProcBefore or noRegistersAllocated:
+    # xxx: the noRegisterAllocated check is required in order to avoid issues
+    #      where nimsuggest can crash due as a macro with pos will be loaded
+    #      but it doesn't have offsets for register allocations see:
+    #      https://github.com/nim-lang/Nim/issues/18385
+    #      Improvements and further use of IC should remove the need for this.
     #if s.name.s == "outterMacro" or s.name.s == "innerProc":
     #  echo "GENERATING CODE FOR ", s.name.s
     let last = c.code.len-1

From e08ec0c6745e54de4e174970e513acef3b0d8f5e Mon Sep 17 00:00:00 2001
From: Juan Carlos 
Date: Sun, 25 Jul 2021 13:07:43 -0300
Subject: [PATCH 0269/1654] Documentation only, Sugar arrow and semicolons
 (#18574)

* Document that sugar arrow do not support semicolon as argument separator
---
 lib/pure/sugar.nim | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim
index 7cfa932e6..f82e1706f 100644
--- a/lib/pure/sugar.nim
+++ b/lib/pure/sugar.nim
@@ -54,6 +54,8 @@ proc createProcType(p, b: NimNode): NimNode =
 
 macro `=>`*(p, b: untyped): untyped =
   ## Syntax sugar for anonymous procedures. It also supports pragmas.
+  ##
+  ## .. warning:: Semicolons can not be used to separate procedure arguments.
   runnableExamples:
     proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2)
 
@@ -133,6 +135,8 @@ macro `=>`*(p, b: untyped): untyped =
 
 macro `->`*(p, b: untyped): untyped =
   ## Syntax sugar for procedure types. It also supports pragmas.
+  ##
+  ## .. warning:: Semicolons can not be used to separate procedure arguments.
   runnableExamples:
     proc passTwoAndTwo(f: (int, int) -> int): int = f(2, 2)
     # is the same as:

From 5f7db652577523bdbb4e9b9157b4fb3bed25e01c Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Sun, 25 Jul 2021 12:55:33 -0700
Subject: [PATCH 0270/1654] followup #18453 (#18582)

---
 tests/stdlib/tos.nim |  4 ++--
 tests/vm/tvmmisc.nim | 30 +++++++++++++++---------------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 8a12b9b90..d598a05b2 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -558,9 +558,9 @@ block ospaths:
   doAssert joinPath("foo","abc") == unixToNativePath"foo/abc"
   doAssert joinPath("","abc") == unixToNativePath"abc"
 
-  doAssert joinPath("gook/.","abc") == unixToNativePath"gook/abc"
+  doAssert joinPath("zook/.","abc") == unixToNativePath"zook/abc"
 
-  # controversial: inconsistent with `joinPath("gook/.","abc")`
+  # controversial: inconsistent with `joinPath("zook/.","abc")`
   # on linux, `./foo` and `foo` are treated a bit differently for executables
   # but not `./foo/bar` and `foo/bar`
   doAssert joinPath(".", "/lib") == unixToNativePath"./lib"
diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim
index a4f6391cc..f6fbde0f9 100644
--- a/tests/vm/tvmmisc.nim
+++ b/tests/vm/tvmmisc.nim
@@ -376,7 +376,7 @@ block: # VM wrong register free causes errors in unrelated code
     in let prc = if not isClosure: bb.sym else: bb[0].sym
     ]#
     proc bar2(head: string): string = "asdf"
-    proc gook(u1: int) = discard
+    proc zook(u1: int) = discard
 
     type PathEntry = object
       kind: int
@@ -385,20 +385,20 @@ block: # VM wrong register free causes errors in unrelated code
     iterator globOpt(): int =
       var u1: int
 
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
-      gook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
+      zook(u1)
 
       var entry = PathEntry()
       entry.path = bar2("")

From 10da888c075acb901b57ba51e4400df6cdf3efd6 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Mon, 26 Jul 2021 00:01:19 +0300
Subject: [PATCH 0271/1654] docgen: sort symbols (fix #17910) (#18560)

* docgen: sort symbols (fix #17910)

* add workaround + change naming

* switch to a dedicated sort comparator

* fix numbers with unequal string lengths

* dedicated `sortName` instead of `plainNameEsc`:

* more compact names for non-overloaded symbols
* more predictable Ascii sort (e.g. `<` instead of `<`)
---
 compiler/docgen.nim                           |  98 ++-
 .../expected/subdir/subdir_b/utils.html       |  10 +-
 nimdoc/testproject/expected/testproject.html  | 686 +++++++++---------
 3 files changed, 432 insertions(+), 362 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index f9e72850a..492668543 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -11,7 +11,7 @@
 # by knowing how the anchors are going to be named.
 
 import
-  ast, strutils, strtabs, options, msgs, os, idents,
+  ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents,
   wordrecg, syntaxes, renderer, lexer,
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, trees, types,
@@ -43,11 +43,15 @@ type
     descRst: ItemPre     ## Description of the item (may contain
                          ## runnableExamples).
     substitutions: seq[string]    ## Variable names in `doc.item`...
+    sortName: string    ## The string used for sorting in output
   ModSection = object  ## Section like Procs, Types, etc.
     secItems: seq[Item]  ## Pre-processed items.
     finalMarkup: string  ## The items, after RST pass 2 and rendering.
   ModSections = array[TSymKind, ModSection]
-  TSections = array[TSymKind, string]
+  TocItem = object  ## HTML TOC item
+    content: string
+    sortName: string
+  TocSectionsFinal = array[TSymKind, string]
   ExampleGroup = ref object
     ## a group of runnableExamples with same rdoccmd
     rdoccmd: string ## from 1st arg in `runnableExamples(rdoccmd): body`
@@ -64,8 +68,13 @@ type
     module: PSym
     modDeprecationMsg: string
     section: ModSections     # entries of ``.nim`` file (for `proc`s, etc)
-    toc, toc2: TSections     # toc2 - grouped TOC
-    tocTable: array[TSymKind, Table[string, string]]
+    tocSimple: array[TSymKind, seq[TocItem]]
+      # TOC entries for non-overloadable symbols (e.g. types, constants)...
+    tocTable:  array[TSymKind, Table[string, seq[TocItem]]]
+      # ...otherwise (e.g. procs)
+    toc2: TocSectionsFinal  # TOC `content`, which is probably wrapped
+                            # in `doc.section.toc2`
+    toc: TocSectionsFinal  # final TOC (wrapped in `doc.section.toc`)
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
     seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
@@ -88,6 +97,51 @@ type
 proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst)
 proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str)
 
+proc cmpDecimalsIgnoreCase(a, b: string): int =
+  ## For sorting with correct handling of cases like 'uint8' and 'uint16'.
+  ## Also handles leading zeros well (however note that leading zeros are
+  ## significant when lengths of numbers mismatch, e.g. 'bar08' > 'bar8' !).
+  runnableExamples:
+    doAssert cmpDecimalsIgnoreCase("uint8", "uint16") < 0
+    doAssert cmpDecimalsIgnoreCase("val00032", "val16suffix") > 0
+    doAssert cmpDecimalsIgnoreCase("val16suffix", "val16") > 0
+    doAssert cmpDecimalsIgnoreCase("val_08_32", "val_08_8") > 0
+    doAssert cmpDecimalsIgnoreCase("val_07_32", "val_08_8") < 0
+    doAssert cmpDecimalsIgnoreCase("ab8", "ab08") < 0
+    doAssert cmpDecimalsIgnoreCase("ab8de", "ab08c") < 0 # sanity check
+  let aLen = a.len
+  let bLen = b.len
+  var
+    iA = 0
+    iB = 0
+  while iA < aLen and iB < bLen:
+    if isDigit(a[iA]) and isDigit(b[iB]):
+      var
+        limitA = iA  # index after the last (least significant) digit
+        limitB = iB
+      while limitA < aLen and isDigit(a[limitA]): inc limitA
+      while limitB < bLen and isDigit(b[limitB]): inc limitB
+      var pos = max(limitA-iA, limitB-iA)
+      while pos > 0:
+        if limitA-pos < iA:  # digit in `a` is 0 effectively
+          result = ord('0') - ord(b[limitB-pos])
+        elif limitB-pos < iB:  # digit in `b` is 0 effectively
+          result = ord(a[limitA-pos]) - ord('0')
+        else:
+          result = ord(a[limitA-pos]) - ord(b[limitB-pos])
+        if result != 0: return
+        dec pos
+      result = (limitA - iA) - (limitB - iB)  # consider 'bar08' > 'bar8'
+      if result != 0: return
+      iA = limitA
+      iB = limitB
+    else:
+      result = ord(toLowerAscii(a[iA])) - ord(toLowerAscii(b[iB]))
+      if result != 0: return
+      inc iA
+      inc iB
+  result = (aLen - iA) - (bLen - iB)
+
 proc prettyString(a: object): string =
   # xxx pending std/prettyprint refs https://github.com/nim-lang/RFCs/issues/203#issuecomment-602534906
   for k, v in fieldPairs(a):
@@ -862,6 +916,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   let
     plainNameEsc = esc(d.target, plainName.strip)
     uniqueName = if k in routineKinds: plainNameEsc else: name
+    sortName = if k in routineKinds: plainName.strip else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
     plainSymbolEnc = encodeUrl(cleanPlainSymbol)
@@ -874,7 +929,10 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
 
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
-  d.section[k].secItems.add Item(descRst: comm, substitutions: @[
+  d.section[k].secItems.add Item(
+    descRst: comm,
+    sortName: sortName,
+    substitutions: @[
      "name", name, "uniqueName", uniqueName,
      "header", result, "itemID", $d.id,
      "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol,
@@ -898,12 +956,15 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       setIndexTerm(d[], external, symbolOrId, plain, nameNode.sym.name.s & '.' & plain,
         xmltree.escape(getPlainDocstring(e).docstringSummary))
 
-  d.toc[k].add(getConfigVar(d.conf, "doc.item.toc") % [
+  d.tocSimple[k].add TocItem(
+    sortName: sortName,
+    content: getConfigVar(d.conf, "doc.item.toc") % [
       "name", name, "header_plain", plainNameEsc,
       "itemSymOrIDEnc", symbolOrIdEnc])
 
-  d.tocTable[k].mgetOrPut(cleanPlainSymbol, "").add(
-    getConfigVar(d.conf, "doc.item.tocTable") % [
+  d.tocTable[k].mgetOrPut(cleanPlainSymbol, newSeq[TocItem]()).add TocItem(
+    sortName: sortName,
+    content: getConfigVar(d.conf, "doc.item.tocTable") % [
       "name", name, "header_plain", plainNameEsc,
       "itemSymOrID", symbolOrId.replace(",", ","),
       "itemSymOrIDEnc", symbolOrIdEnc])
@@ -1143,8 +1204,9 @@ proc finishGenerateDoc*(d: var PDoc) =
         var resolved = resolveSubs(d.sharedState, f.rst)
         renderRstToOut(d[], resolved, result)
       of false: result &= f.str
+  proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
   for k in TSymKind:
-    for item in d.section[k].secItems:
+    for item in d.section[k].secItems.sorted(cmp):
       var itemDesc: string
       renderItemPre(d, item.descRst, itemDesc)
       d.section[k].finalMarkup.add(
@@ -1262,18 +1324,26 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) =
       "sectionid", $ord(kind), "sectionTitle", title,
       "sectionTitleID", $(ord(kind) + 50), "content", d.section[kind].finalMarkup]
 
-  var tocSource = d.toc
+  proc cmp(x, y: TocItem): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName)
   if groupedToc:
-    for p in d.tocTable[kind].keys:
+    let overloadableNames = toSeq(keys(d.tocTable[kind]))
+    for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase):
+      var overloadChoices = d.tocTable[kind][plainName]
+      overloadChoices.sort(cmp)
+      var content: string
+      for item in overloadChoices:
+        content.add item.content
       d.toc2[kind].add getConfigVar(d.conf, "doc.section.toc2") % [
           "sectionid", $ord(kind), "sectionTitle", title,
           "sectionTitleID", $(ord(kind) + 50),
-          "content", d.tocTable[kind][p], "plainName", p]
-    tocSource = d.toc2
+          "content", content, "plainName", plainName]
+  else:
+    for item in d.tocSimple[kind].sorted(cmp):
+      d.toc2[kind].add item.content
 
   d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % [
       "sectionid", $ord(kind), "sectionTitle", title,
-      "sectionTitleID", $(ord(kind) + 50), "content", tocSource[kind]]
+      "sectionTitleID", $(ord(kind) + 50), "content", d.toc2[kind]]
 
 proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): string =
   $relativeTo(outDir / linkto, destFile.splitFile().dir, '/')
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 6aa71e8c2..939495ad1 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -130,16 +130,16 @@ window.addEventListener('DOMContentLoaded', main);
       
  • aEnum(): untyped
    • 
 
-  
-  
   
+  
 
   
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 7e5a685d2..0f6289bbc 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -107,9 +107,7 @@ window.addEventListener('DOMContentLoaded', main);
 
  • 
   Types
   
      
-      
    • FooBuzz
      • 
-  
    • A
      • 
   
    • Foo
      • 
+  
    • FooBuzz
      • 
   
    • Vars
   
 
      • 
@@ -153,35 +153,57 @@ window.addEventListener('DOMContentLoaded', main);
 
    • 
   Procs
   
        
-      
          z10
-      
        • z10()
          • 
+      
-  
            tripleStrLitTest
-      
          • tripleStrLitTest()
            • 
+  
-  
              z17
-      
            • z17()
              • 
+  
+  
   
-  
   
+  
+  
+  
+  
   
-  
-  
   
-  
-  
-  
-  
   
-  
-  
-  
-  
-  
-  
-  
-  
-  
                z4
-      
              • z4()
                • 
+  
   
                  someFunc
@@ -285,29 +235,79 @@ window.addEventListener('DOMContentLoaded', main);
     title="someFunc()">someFunc()
 
   
                
-  
                  z8
-      
                • z8(): int
                  • 
+  
   
+  
+  
+  
   
-  
                    asyncFun2
-      
                  • asyncFun2(): owned(Future[void])
                    • 
+  
-  
                      asyncFun1
-      
                    • asyncFun1(): Future[int]
                      • 
+  
+  
+  
+  
+  
+  
+  
+  
 
@@ -316,7 +316,12 @@ window.addEventListener('DOMContentLoaded', main);
 
                    • 
   Methods
   
@@ -379,10 +379,25 @@ window.addEventListener('DOMContentLoaded', main);
 
                    • 
   Templates
   
                        
-      
@@ -447,16 +447,6 @@ window.addEventListener('DOMContentLoaded', main);
 
                        
 
                        Types
                        
 
                        
-
-
                        FooBuzz {....deprecated: "FooBuzz msg".} = int
                        
-
                        
-  
                        
-    Deprecated: FooBuzz msg
-  
                        
-
-
-
-
                        
 
 
                        A {.inject.} = enum
   aA
                        
@@ -480,6 +470,16 @@ The enum B.
 
 
 
+
+
+
                        FooBuzz {....deprecated: "FooBuzz msg".} = int
                        
+
                        
+  
                        
+    Deprecated: FooBuzz msg
+  
                        
+
+
+
 
                        
 
 
                        Shapes = enum
@@ -496,19 +496,19 @@ Some shapes.
 
                        
 
                        Vars
                        
 
                        
-
-
                        someVariable: bool
                        
-
                        
-
-This should be visible.
-
-
                        
 
 
                        aVariable: array[1, int]
                        
 
                        
 
 
 
+
                        
+
+
                        someVariable: bool
                        
+
                        
+
+This should be visible.
+
 
                        
 
 
                        
@@ -548,12 +548,69 @@ This should be visible.
 
                        
 
                        Procs
                        
 
                        
+
+
                        proc addfBug14485() {....raises: [], tags: [].}
                        
+
                        
+
+Some proc
+
                        Example:
                        
+
                        discard "foo() = " & $[1]
+#[
+0: let's also add some broken html to make sure this won't break in future
+1: </span>
+2: </span>
+3: </span
+4: </script>
+5: </script
+6: </script
+7: end of broken html
+]#
                        
+
+
                        
+
+
                        proc anything() {....raises: [], tags: [].}
                        
+
                        
+
+There is no block quote after blank lines at the beginning.
+
+
                        
+
+
                        proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+                                tags: [RootEffect].}
                        
+
                        
+
+ok1
+
+
                        
+
+
                        proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
+
                        
+
+
+
+
                        
+
+
                        proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
+
                        
+
+
+
                        Example:
                        
+
                        discard
                        ok1
+
+
                        
 
 
                        proc bar[T](a, b: T): T
                        
 
                        
 
 
 
+
                        
+
+
                        proc baz() {....raises: [], tags: [].}
                        
+
                        
+
+
+
 
                        
 
 
                        proc baz[T](a, b: T): T {....deprecated.}
                        
@@ -575,11 +632,20 @@ This is deprecated without message.
 This is deprecated with a message.
 
 
-
-
                        func someFunc() {....raises: [], tags: [].}
                        
+
+
                        proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
+    header: "<stdio.h>", varargs, discardable.}
                        
 
                        
 
-My someFunc. Stuff in quotes here. Some link
+
+
+
                        
+
+
                        proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
+                                     varargs, discardable.}
                        
+
                        
+
+the c printf. etc.
 
 
                        
 
@@ -598,6 +664,103 @@ came form utils but should be shown where 
+
+
                        proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
+
                        
+
+
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
+
                        See also:
                        
+
+
                        low2(2) # => -9223372036854775808
                        
+
                        Example:
                        
+
                        discard "in low2"
                        
+
+
                        
+
+
                        proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
+
                        
+
+
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
+
                        See also:
                        
+
+
                        low(2) # => -9223372036854775808
                        
+
+
                        
+
+
                        proc p1() {....raises: [], tags: [].}
                        
+
                        
+
+cp1
+
                        Example:
                        
+
                        doAssert 1 == 1 # regular comments work here
                        c4
+
                        Example:
                        
+
                        # c5 regular comments before 1st token work
+# regular comment
+#[
+nested regular comment
+]#
+doAssert 2 == 2 # c8
+## this is a non-nested doc comment
+
+##[
+this is a nested doc comment
+]##
+discard "c9"
+# also work after
                        
+
+
                        
+
+
                        func someFunc() {....raises: [], tags: [].}
                        
+
                        
+
+My someFunc. Stuff in quotes here. Some link
+
+
                        
+
+
                        proc tripleStrLitTest() {....raises: [], tags: [].}
                        
+
                        
+
+
+
                        Example: cmd: --hint:XDeclaredButNotUsed:off
                        
+
                        ## mullitline string litterals are tricky as their indentation can span
+## below that of the runnableExamples
+let s1a = """
+should appear at indent 0
+  at indent 2
+at indent 0
+"""
+# make sure this works too
+let s1b = """start at same line
+  at indent 2
+at indent 0
+""" # comment after
+let s2 = """sandwich """
+let s3 = """"""
+when false:
+  let s5 = """
+        in s5 """
+
+let s3b = ["""
+%!? #[...] # inside a multiline ...
+""", "foo"]
+
+## make sure handles trailing spaces
+let s4 = """ 
+"""
+
+let s5 = """ x
+"""
+let s6 = """ ""
+"""
+let s7 = """"""""""
+let s8 = ["""""""""", """
+  """ ]
+discard
+# should be in
                        
+
 
                        
 
 
                        proc z1(): Foo {....raises: [], tags: [].}
                        
@@ -701,13 +864,6 @@ cz13
 
                        Example:
                        
 
                        discard
                        
 
-
-
-
                        proc baz() {....raises: [], tags: [].}
                        
-
                        
-
-
-
 
                        
 
 
                        proc z17() {....raises: [], tags: [].}
                        
@@ -717,162 +873,6 @@ cz17 rest
 
                        Example:
                        
 
                        discard 1
                        rest
 
-
-
-
                        proc p1() {....raises: [], tags: [].}
                        
-
                        
-
-cp1
-
                        Example:
                        
-
                        doAssert 1 == 1 # regular comments work here
                        c4
-
                        Example:
                        
-
                        # c5 regular comments before 1st token work
-# regular comment
-#[
-nested regular comment
-]#
-doAssert 2 == 2 # c8
-## this is a non-nested doc comment
-
-##[
-this is a nested doc comment
-]##
-discard "c9"
-# also work after
                        
-
-
                        
-
-
                        proc addfBug14485() {....raises: [], tags: [].}
                        
-
                        
-
-Some proc
-
                        Example:
                        
-
                        discard "foo() = " & $[1]
-#[
-0: let's also add some broken html to make sure this won't break in future
-1: </span>
-2: </span>
-3: </span
-4: </script>
-5: </script
-6: </script
-7: end of broken html
-]#
                        
-
-
                        
-
-
                        proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
-                                     varargs, discardable.}
                        
-
                        
-
-the c printf. etc.
-
-
                        
-
-
                        proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
-    header: "<stdio.h>", varargs, discardable.}
                        
-
                        
-
-
-
-
                        
-
-
                        proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
-
                        
-
-
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
-
                        See also:
                        
-
-
                        low(2) # => -9223372036854775808
                        
-
-
                        
-
-
                        proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
                        
-
                        
-
-
                        Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
                        
-
                        See also:
                        
-
-
                        low2(2) # => -9223372036854775808
                        
-
                        Example:
                        
-
                        discard "in low2"
                        
-
-
                        
-
-
                        proc tripleStrLitTest() {....raises: [], tags: [].}
                        
-
                        
-
-
-
                        Example: cmd: --hint:XDeclaredButNotUsed:off
                        
-
                        ## mullitline string litterals are tricky as their indentation can span
-## below that of the runnableExamples
-let s1a = """
-should appear at indent 0
-  at indent 2
-at indent 0
-"""
-# make sure this works too
-let s1b = """start at same line
-  at indent 2
-at indent 0
-""" # comment after
-let s2 = """sandwich """
-let s3 = """"""
-when false:
-  let s5 = """
-        in s5 """
-
-let s3b = ["""
-%!? #[...] # inside a multiline ...
-""", "foo"]
-
-## make sure handles trailing spaces
-let s4 = """ 
-"""
-
-let s5 = """ x
-"""
-let s6 = """ ""
-"""
-let s7 = """"""""""
-let s8 = ["""""""""", """
-  """ ]
-discard
-# should be in
                        
-
-
                        
-
-
                        proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
-                                tags: [RootEffect].}
                        
-
                        
-
-ok1
-
-
                        
-
-
                        proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
-
                        
-
-
-
-
                        
-
-
                        proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
                        
-
                        
-
-
-
                        Example:
                        
-
                        discard
                        ok1
-
-
                        
-
-
                        proc anything() {....raises: [], tags: [].}
                        
-
                        
-
-There is no block quote after blank lines at the beginning.
-
 
                        
 
 
                        
@@ -967,6 +967,13 @@ cz18
 
                        
 
                        Templates
                        
 
                        
+
+
                        template foo(a, b: SomeType)
                        
+
                        
+
+This does nothing 
+
+
                        
 
 
                        template fromUtils2()
                        
 
                        
@@ -976,20 +983,6 @@ ok3
 
                        discard """should be shown as examples for fromUtils2
        in module calling fromUtilsGen"""
                        
 
-
                        
-
-
                        template z6t(): int
                        
-
                        
-
-cz6t
-
-
                        
-
-
                        template foo(a, b: SomeType)
                        
-
                        
-
-This does nothing 
-
 
                        
 
 
                        template myfn()
                        
@@ -1012,6 +1005,22 @@ bar
   discard 0xff # elu par cette crapule
 # should be in
                        should be still in
 
+
+
+
                        template testNimDocTrailingExample()
                        
+
                        
+
+
+
                        Example:
                        
+
                        discard 2
                        
+
+
                        
+
+
                        template z6t(): int
                        
+
                        
+
+cz6t
+
 
                        
 
 
                        template z14()
                        
@@ -1038,15 +1047,6 @@ cz15
 
                        Example:
                        
 
                        discard 1
                        in or out?
 
-
-
-
                        template testNimDocTrailingExample()
                        
-
                        
-
-
-
                        Example:
                        
-
                        discard 2
                        
-
 
                        
 
 
                        

From ff280c012731b1978005a743eec96cd885331475 Mon Sep 17 00:00:00 2001
From: Araq 
Date: Mon, 26 Jul 2021 09:57:19 +0200
Subject: [PATCH 0272/1654] added missing .inline for toCChar

---
 compiler/msgs.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/msgs.nim b/compiler/msgs.nim
index 242b06587..20a55daee 100644
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -27,7 +27,7 @@ proc flushDot*(conf: ConfigRef) =
     conf.lastMsgWasDot.excl stdOrrKind
     write(stdOrr, "\n")
 
-proc toCChar*(c: char; result: var string) =
+proc toCChar*(c: char; result: var string) {.inline.} =
   case c
   of '\0'..'\x1F', '\x7F'..'\xFF':
     result.add '\\'

From 158d7c7a7a8977f5937311b98bc381399f31db8d Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Mon, 26 Jul 2021 14:15:55 +0200
Subject: [PATCH 0273/1654] fixes #18558 again (#18586)

---
 compiler/ccgcalls.nim | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim
index 510b8d9ec..749a7a3b6 100644
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -333,6 +333,8 @@ proc getPotentialWrites(n: PNode; mutate: bool; result: var seq[PNode]) =
     getPotentialWrites(n[1], mutate, result)
   of nkAddr, nkHiddenAddr:
     getPotentialWrites(n[0], true, result)
+  of nkBracketExpr, nkDotExpr, nkCheckedFieldExpr:
+    getPotentialWrites(n[0], mutate, result)
   of nkCallKinds:
     case n.getMagic:
     of mIncl, mExcl, mInc, mDec, mAppendStrCh, mAppendStrStr, mAppendSeqElem,

From d4c3a09286d1cbb534ed5dbd21d99a18670b3427 Mon Sep 17 00:00:00 2001
From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com>
Date: Mon, 26 Jul 2021 16:22:42 +0300
Subject: [PATCH 0274/1654] optimize for the non-throwing case (#18587)

---
 lib/pure/collections/tables.nim | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index b845f57cc..403345755 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -227,6 +227,12 @@ template dataLen(t): untyped = len(t.data)
 
 include tableimpl
 
+proc raiseKeyError[T](key: T) {.noinline, noreturn.} =
+  when compiles($key):
+    raise newException(KeyError, "key not found: " & $key)
+  else:
+    raise newException(KeyError, "key not found")
+
 template get(t, key): untyped =
   ## retrieves the value at `t[key]`. The value can be modified.
   ## If `key` is not in `t`, the `KeyError` exception is raised.
@@ -235,10 +241,7 @@ template get(t, key): untyped =
   var index = rawGet(t, key, hc)
   if index >= 0: result = t.data[index].val
   else:
-    when compiles($key):
-      raise newException(KeyError, "key not found: " & $key)
-    else:
-      raise newException(KeyError, "key not found")
+    raiseKeyError(key)
 
 proc enlarge[A, B](t: var Table[A, B]) =
   var n: KeyValuePairSeq[A, B]

From 664dbb6bfbe06ff0ddc4339c0fe80c49d0838ca4 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Mon, 26 Jul 2021 18:00:03 +0200
Subject: [PATCH 0275/1654] atlas tool: testing via mocking (#18588)

* atlas: refactoring
* Atlas: do some basic testing via mocking
---
 koch.nim                                 |   2 +
 tools/atlas/atlas.nim                    | 187 +++++++++++++++++------
 tools/atlas/osutils.nim                  |   8 -
 tools/atlas/testdata.nim                 |  63 ++++++++
 tools/atlas/tests/balls.nimble           |  32 ++++
 tools/atlas/tests/grok.nimble            |   5 +
 tools/atlas/tests/nim-bytes2human.nimble |   7 +
 tools/atlas/tests/nim.cfg                |  11 ++
 tools/atlas/tests/npeg.nimble            |  48 ++++++
 tools/atlas/tests/packages/packages.json |  36 +++++
 tools/atlas/tests/sync.nimble            |  10 ++
 tools/atlas/tests/testes.nimble          |  23 +++
 tools/atlas/tests/ups.nimble             |  13 ++
 13 files changed, 387 insertions(+), 58 deletions(-)
 create mode 100644 tools/atlas/testdata.nim
 create mode 100644 tools/atlas/tests/balls.nimble
 create mode 100644 tools/atlas/tests/grok.nimble
 create mode 100644 tools/atlas/tests/nim-bytes2human.nimble
 create mode 100644 tools/atlas/tests/nim.cfg
 create mode 100644 tools/atlas/tests/npeg.nimble
 create mode 100644 tools/atlas/tests/packages/packages.json
 create mode 100644 tools/atlas/tests/sync.nimble
 create mode 100644 tools/atlas/tests/testes.nimble
 create mode 100644 tools/atlas/tests/ups.nimble

diff --git a/koch.nim b/koch.nim
index da7576e65..4296e4529 100644
--- a/koch.nim
+++ b/koch.nim
@@ -594,6 +594,8 @@ proc runCI(cmd: string) =
     when defined(posix):
       execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
 
+    execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls")
+
   when not defined(bsd):
     if not doUseCpp:
       # the BSDs are overwhelmed already, so only run this test on the other machines:
diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim
index b10ec0358..d0990c1f7 100644
--- a/tools/atlas/atlas.nim
+++ b/tools/atlas/atlas.nim
@@ -39,6 +39,10 @@ proc writeVersion() =
   stdout.flushFile()
   quit(0)
 
+const
+  MockupRun = defined(atlasTests)
+  TestsDir = "tools/atlas/tests"
+
 type
   PackageName = distinct string
   DepRelation = enum
@@ -55,9 +59,79 @@ type
     p: Table[string, string] # name -> url mapping
     processed: HashSet[string] # the key is (url / commit)
     errors: int
+    when MockupRun:
+      currentDir: string
+      step: int
+      mockupSuccess: bool
 
 const
   InvalidCommit = ""
+  ProduceTest = false
+
+type
+  Command = enum
+    GitDiff = "git diff",
+    GitTags = "git show-ref --tags",
+    GitRevParse = "git rev-parse",
+    GitCheckout = "git checkout",
+    GitPull = "git pull",
+    GitCurrentCommit = "git log -n 1 --format=%H"
+    GitMergeBase = "git merge-base"
+
+include testdata
+
+proc exec(c: var AtlasContext; cmd: Command; args: openArray[string]): (string, int) =
+  when MockupRun:
+    assert TestLog[c.step].cmd == cmd
+    case cmd
+    of GitDiff, GitTags, GitRevParse, GitPull, GitCurrentCommit:
+      result = (TestLog[c.step].output, TestLog[c.step].exitCode)
+    of GitCheckout:
+      assert args[0] == TestLog[c.step].output
+    of GitMergeBase:
+      let tmp = TestLog[c.step].output.splitLines()
+      assert tmp.len == 4, $tmp.len
+      assert tmp[0] == args[0]
+      assert tmp[1] == args[1]
+      assert tmp[3] == ""
+      result[0] = tmp[2]
+      result[1] = TestLog[c.step].exitCode
+    inc c.step
+  else:
+    var cmdLine = $cmd
+    for i in 0.. ", result
+
+proc cloneUrl(c: var AtlasContext; url, dest: string; cloneUsingHttps: bool): string =
+  when MockupRun:
+    result = ""
+  else:
+    result = osutils.cloneUrl(url, dest, cloneUsingHttps)
+    when ProduceTest:
+      echo "cloned ", url, " into ", dest
+
+template withDir*(c: var AtlasContext; dir: string; body: untyped) =
+  when MockupRun:
+    c.currentDir = dir
+    body
+  else:
+    let oldDir = getCurrentDir()
+    try:
+      when ProduceTest:
+        echo "Current directory is now ", dir
+      setCurrentDir(dir)
+      body
+    finally:
+      setCurrentDir(oldDir)
+
+proc extractRequiresInfo(c: var AtlasContext; nimbleFile: string): NimbleFileInfo =
+  result = extractRequiresInfo(nimbleFile)
+  when ProduceTest:
+    echo "nimble ", nimbleFile, " info ", result
 
 proc toDepRelation(s: string): DepRelation =
   case s
@@ -65,9 +139,9 @@ proc toDepRelation(s: string): DepRelation =
   of ">": strictlyGreater
   else: normal
 
-proc isCleanGit(dir: string): string =
+proc isCleanGit(c: var AtlasContext; dir: string): string =
   result = ""
-  let (outp, status) = osproc.execCmdEx("git diff")
+  let (outp, status) = exec(c, GitDiff, [])
   if outp.len != 0:
     result = "'git diff' not empty"
   elif status != 0:
@@ -101,8 +175,8 @@ proc sameVersionAs(tag, ver: string): bool =
     result = safeCharAt(tag, idx-1) notin VersionChars and
       safeCharAt(tag, idx+ver.len) notin VersionChars
 
-proc versionToCommit(d: Dependency): string =
-  let (outp, status) = osproc.execCmdEx("git show-ref --tags")
+proc versionToCommit(c: var AtlasContext; d: Dependency): string =
+  let (outp, status) = exec(c, GitTags, [])
   if status == 0:
     var useNextOne = false
     for line in splitLines(outp):
@@ -123,35 +197,36 @@ proc versionToCommit(d: Dependency): string =
 
   return ""
 
-proc shortToCommit(short: string): string =
-  let (cc, status) = osproc.execCmdEx("git rev-parse " & quoteShell(short))
+proc shortToCommit(c: var AtlasContext; short: string): string =
+  let (cc, status) = exec(c, GitRevParse, [short])
   result = if status == 0: strutils.strip(cc) else: ""
 
 proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
-  let (_, status) = osproc.execCmdEx("git checkout " & quoteShell(commit))
+  let (_, status) = exec(c, GitCheckout, [commit])
   if status != 0:
     error(c, p, "could not checkout commit", commit)
 
 proc gitPull(c: var AtlasContext; p: PackageName) =
-  let (_, status) = osproc.execCmdEx("git pull")
+  let (_, status) = exec(c, GitPull, [])
   if status != 0:
     error(c, p, "could not 'git pull'")
 
 proc updatePackages(c: var AtlasContext) =
   if dirExists(c.workspace / PackagesDir):
-    withDir(c.workspace / PackagesDir):
+    withDir(c, c.workspace / PackagesDir):
       gitPull(c, PackageName PackagesDir)
   else:
-    withDir c.workspace:
-      let err = cloneUrl("https://github.com/nim-lang/packages", PackagesDir, false)
+    withDir c, c.workspace:
+      let err = cloneUrl(c, "https://github.com/nim-lang/packages", PackagesDir, false)
       if err != "":
         error c, PackageName(PackagesDir), err
 
 proc fillPackageLookupTable(c: var AtlasContext) =
   if not c.hasPackageList:
     c.hasPackageList = true
-    updatePackages(c)
-    let plist = getPackages(c.workspace)
+    when not MockupRun:
+      updatePackages(c)
+    let plist = getPackages(when MockupRun: TestsDir else: c.workspace)
     for entry in plist:
       c.p[unicode.toLower entry.name] = entry.url
 
@@ -178,19 +253,19 @@ proc isShortCommitHash(commit: string): bool {.inline.} =
 
 proc checkoutCommit(c: var AtlasContext; w: Dependency) =
   let dir = c.workspace / w.name.string
-  withDir dir:
+  withDir c, dir:
     if w.commit.len == 0 or cmpIgnoreCase(w.commit, "head") == 0:
       gitPull(c, w.name)
     else:
-      let err = isCleanGit(dir)
+      let err = isCleanGit(c, dir)
       if err != "":
         warn c, w.name, err
       else:
         let requiredCommit =
-          if needsCommitLookup(w.commit): versionToCommit(w)
-          elif isShortCommitHash(w.commit): shortToCommit(w.commit)
+          if needsCommitLookup(w.commit): versionToCommit(c, w)
+          elif isShortCommitHash(w.commit): shortToCommit(c, w.commit)
           else: w.commit
-        let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H")
+        let (cc, status) = exec(c, GitCurrentCommit, [])
         let currentCommit = strutils.strip(cc)
         if requiredCommit == "" or status != 0:
           if requiredCommit == "" and w.commit == InvalidCommit:
@@ -201,8 +276,7 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
           if currentCommit != requiredCommit:
             # checkout the later commit:
             # git merge-base --is-ancestor  
-            let (cc, status) = osproc.execCmdEx("git merge-base " &
-                currentCommit.quoteShell & " " & requiredCommit.quoteShell)
+            let (cc, status) = exec(c, GitMergeBase, [currentCommit, requiredCommit])
             let mergeBase = strutils.strip(cc)
             if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
               # conflict resolution: pick the later commit:
@@ -215,15 +289,19 @@ proc checkoutCommit(c: var AtlasContext; w: Dependency) =
                   currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
 
 proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
-  result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
-  if not fileExists(result):
-    result = ""
-    for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
-      if result.len == 0:
-        result = x
-      else:
-        # ambiguous .nimble file
-        return ""
+  when MockupRun:
+    result = TestsDir / dep.name.string & ".nimble"
+    doAssert fileExists(result), "file does not exist " & result
+  else:
+    result = c.workspace / dep.name.string / (dep.name.string & ".nimble")
+    if not fileExists(result):
+      result = ""
+      for x in walkFiles(c.workspace / dep.name.string / "*.nimble"):
+        if result.len == 0:
+          result = x
+        else:
+          # ambiguous .nimble file
+          return ""
 
 proc addUniqueDep(c: var AtlasContext; work: var seq[Dependency];
                   tokens: seq[string]) =
@@ -240,7 +318,7 @@ proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency];
                     isMainProject: bool) =
   let nimbleFile = findNimbleFile(c, dep)
   if nimbleFile != "":
-    let nimbleInfo = extractRequiresInfo(nimbleFile)
+    let nimbleInfo = extractRequiresInfo(c, nimbleFile)
     for r in nimbleInfo.requires:
       var tokens: seq[string] = @[]
       for token in tokenizeRequires(r):
@@ -280,8 +358,8 @@ proc clone(c: var AtlasContext; start: string): seq[string] =
     let w = work[i]
     let oldErrors = c.errors
     if not dirExists(c.workspace / w.name.string):
-      withDir c.workspace:
-        let err = cloneUrl(w.url, w.name.string, false)
+      withDir c, c.workspace:
+        let err = cloneUrl(c, w.url, w.name.string, false)
         if err != "":
           error c, w.name, err
     if oldErrors == c.errors:
@@ -296,29 +374,34 @@ const
   configPatternBegin = "############# begin Atlas config section ##########\n"
   configPatternEnd =   "############# end Atlas config section   ##########\n"
 
-proc patchNimCfg(c: AtlasContext; deps: seq[string]) =
+proc patchNimCfg(c: var AtlasContext; deps: seq[string]) =
   var paths = "--noNimblePath\n"
   for d in deps:
     paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n"
 
-  let cfg = c.projectDir / "nim.cfg"
   var cfgContent = configPatternBegin & paths & configPatternEnd
-  if not fileExists(cfg):
-    writeFile(cfg, cfgContent)
+
+  when MockupRun:
+    assert readFile(TestsDir / "nim.cfg") == cfgContent
+    c.mockupSuccess = true
   else:
-    let content = readFile(cfg)
-    let start = content.find(configPatternBegin)
-    if start >= 0:
-      cfgContent = content.substr(0, start-1) & cfgContent
-      let theEnd = content.find(configPatternEnd, start)
-      if theEnd >= 0:
-        cfgContent.add content.substr(theEnd+len(configPatternEnd))
-    else:
-      cfgContent = content & "\n" & cfgContent
-    if cfgContent != content:
-      # do not touch the file if nothing changed
-      # (preserves the file date information):
+    let cfg = c.projectDir / "nim.cfg"
+    if not fileExists(cfg):
       writeFile(cfg, cfgContent)
+    else:
+      let content = readFile(cfg)
+      let start = content.find(configPatternBegin)
+      if start >= 0:
+        cfgContent = content.substr(0, start-1) & cfgContent
+        let theEnd = content.find(configPatternEnd, start)
+        if theEnd >= 0:
+          cfgContent.add content.substr(theEnd+len(configPatternEnd))
+      else:
+        cfgContent = content & "\n" & cfgContent
+      if cfgContent != content:
+        # do not touch the file if nothing changed
+        # (preserves the file date information):
+        writeFile(cfg, cfgContent)
 
 proc error*(msg: string) =
   when defined(debug):
@@ -365,8 +448,12 @@ proc main =
     singleArg()
     let deps = clone(c, args[0])
     patchNimCfg c, deps
-    if c.errors > 0:
-      error "There were problems."
+    when MockupRun:
+      if not c.mockupSuccess:
+        error "There were problems."
+    else:
+      if c.errors > 0:
+        error "There were problems."
   of "refresh":
     noArgs()
     updatePackages(c)
diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim
index 31392b113..1bb414c03 100644
--- a/tools/atlas/osutils.nim
+++ b/tools/atlas/osutils.nim
@@ -3,14 +3,6 @@
 
 import os, strutils, osproc
 
-template withDir*(dir, body) =
-  let oldDir = getCurrentDir()
-  try:
-    setCurrentDir(dir)
-    body
-  finally:
-    setCurrentDir(oldDir)
-
 proc isUrl*(x: string): bool =
   x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://")
 
diff --git a/tools/atlas/testdata.nim b/tools/atlas/testdata.nim
new file mode 100644
index 000000000..f960e3d08
--- /dev/null
+++ b/tools/atlas/testdata.nim
@@ -0,0 +1,63 @@
+
+type
+  PerDirData = object
+    dirname: string
+    cmd: Command
+    exitCode: int
+    output: string
+
+template toData(a, b, c, d): untyped =
+  PerDirData(dirname: a, cmd: b, exitCode: c, output: d)
+
+const
+  TestLog = [
+    toData("balls", GitPull, 0, "Already up to date.\n"),
+    toData("grok", GitDiff, 0, ""),
+    toData("grok", GitTags, 0, "2ca193c31fa2377c1e991a080d60ca3215ff6cf0 refs/tags/0.0.1\n48007554b21ba2f65c726ae2fdda88d621865b4a refs/tags/0.0.2\n7092a0286421c7818cd335cca9ebc72d03d866c2 refs/tags/0.0.3\n62707b8ac684efac35d301dbde57dc750880268e refs/tags/0.0.4\n876f2504e0c2f785ffd2cf65a78e2aea474fa8aa refs/tags/0.0.5\nb7eb1f2501aa2382cb3a38353664a13af62a9888 refs/tags/0.0.6\nf5d818bfd6038884b3d8b531c58484ded20a58a4 refs/tags/0.1.0\n961eaddea49c3144d130d105195583d3f11fb6c6 refs/tags/0.2.0\n15ab8ed8d4f896232a976a9008548bd53af72a66 refs/tags/0.2.1\n426a7d7d4603f77ced658e73ad7f3f582413f6cd refs/tags/0.3.0\n83cf7a39b2fe897786fb0fe01a7a5933c3add286 refs/tags/0.3.1\n8d2e3c900edbc95fa0c036fd76f8e4f814aef2c1 refs/tags/0.3.2\n48b43372f49a3bb4dc0969d82a0fca183fb94662 refs/tags/0.3.3\n9ca947a3009ea6ba17814b20eb953272064eb2e6 refs/tags/0.4.0\n1b5643d04fba6d996a16d1ffc13d034a40003f8f refs/tags/0.5.0\n486b0eb580b1c465453d264ac758cc490c19c33e refs/tags/0.5.1\naedb0d9497390e20b9d2541cef2bb05a5cda7a71 refs/tags/0.5.2\n"),
+    toData("grok", GitCurrentCommit, 0, "349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitMergeBase, 0, "349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n1b5643d04fba6d996a16d1ffc13d034a40003f8f\n349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitCheckout, 0, "1b5643d04fba6d996a16d1ffc13d034a40003f8f"), # watch out!
+
+    toData("ups", GitDiff, 0, ""),
+    toData("ups", GitTags, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1 refs/tags/0.0.2\n19bc490c22b4f5b0628c31cdedead1375b279356 refs/tags/0.0.3\nff34602aaea824cb46d6588cd5fe1178132e9702 refs/tags/0.0.4\n09de599138f20b745133b6e4fe563e204415a7e8 refs/tags/0.0.5\n85fee3b74798311108a105635df31f892150f5d0 refs/tags/0.0.6\nfd303913b22b121dc42f332109e9c44950b9acd4 refs/tags/0.0.7\n"),
+    toData("ups", GitCurrentCommit, 0, "74c31af8030112dac758440aa51ef175992f71f3\n"),
+    toData("ups", GitMergeBase, 0, "74c31af8030112dac758440aa51ef175992f71f3\n4008f9339cd22b30e180bc87a6cca7270fd28ac1\n74c31af8030112dac758440aa51ef175992f71f3\n"),
+    toData("ups", GitCheckout, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1"),
+
+    toData("sync", GitDiff, 0, ""),
+    toData("sync", GitTags, 0, "c9811af2d03dc78a920d33592d0a3daae3d9e68f refs/tags/v0.1.0\nc99d3a73344c4c02335190b489e7ea8aaa4e9584 refs/tags/v0.2.0\n6186a38c97c0f26e1a0c8016979227d26c41717b refs/tags/v0.3.0\n5506e4e09b849bd59f930fcccee3096abb089b1d refs/tags/v0.4.0\n1e4ef046ed0447aec7ca4ab3b2e2a5347ac41448 refs/tags/v0.5.0\na95f7b0b22c77b98ed63c1c4f435d550de8dbb05 refs/tags/v0.6.0\n214b868ace4ebc9140ee56734147fbd16b238c71 refs/tags/v0.7.0\n786698bf9f597834f36cd530d32faee31545a1d7 refs/tags/v0.9.0\n810bd2d75e9f6e182534ae2488670b51a9f13fc3 refs/tags/v1.0.0\na133d37991e3d4e9553c5261eabcb07d471968f7 refs/tags/v1.1.0\n0872a7cd70ea43a26deb94b8ab12cc5b89a1c212 refs/tags/v1.2.0\n19ac734438b6c04fe89711728a97fcde591324eb refs/tags/v1.3.0\nde5c7337ebc22422190e8aeca37d05651735f440 refs/tags/v1.4.0\n"),
+    toData("sync", GitCurrentCommit, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n"),
+    toData("sync", GitMergeBase, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"),
+
+    toData("npeg", GitDiff, 0, ""),
+    toData("npeg", GitTags, 0, "8df2f0c9391995fd086b8aab00e8ab7aded1e8f0 refs/tags/0.1.0\n4c959a72db5283b55eeef491076eefb5e02316f1 refs/tags/0.10.0\n802f47c0f7f4318a4f0858ba5a6a6ed2333bde71 refs/tags/0.11.0\n82c8d92837108dce225358ace2c416bf9a3f30ce refs/tags/0.12.0\n87d2f2c4f6ef7da350d45beb5a336611bde7f518 refs/tags/0.13.0\n39964f0d220bfaade47a568bf03c1cf28aa2bc37 refs/tags/0.14.0\nbe9f03f92304cbeab70572944a8563db9b23b2fb refs/tags/0.14.1\na933fb9832566fc95273e417597bfb4faf564ca6 refs/tags/0.15.0\n6aad2e438c52ff0636c7bfb64338e444ac3e83ba refs/tags/0.16.0\nf4ddffb5848c42c6151743dd9c7eddcaaabc56cc refs/tags/0.17.0\n30b446b39442cdbc53a97018ab8a54149aa7c3b7 refs/tags/0.17.1\n1a9d36aa3b34a6169d4530463f1c17a3fe1e075e refs/tags/0.18.0\ndd34f903a9a63b876cb2db19b7a4ce0bcc252134 refs/tags/0.19.0\nd93d49c81fc8722d7929ac463b435c0f2e10c53b refs/tags/0.2.0\neeae7746c9b1118bcf27744ab2aee26969051256 refs/tags/0.20.0\n8c3471a548129f3bf62df15cd0fd8cca1787d852 refs/tags/0.21.0\nc0e873a17bc713c80e74fec3c30cb62dcd5d194a refs/tags/0.21.1\nbae84c47a1bb259b209b6f6be1582327b784539d refs/tags/0.21.2\nbfcb4bcae76a917c3c88736ca773e4cb67dbb2d8 refs/tags/0.21.3\n0eabb7c462d30932049f0b7e6a030c1562cf9fee refs/tags/0.22.0\n2e75367095f54d4351005078bad98041a55b14c1 refs/tags/0.22.1\n814ea235dd398108d7b18f966694c3d951575701 refs/tags/0.22.2\na812064587d983c129737f8500bf74990e6b8dab refs/tags/0.23.0\nbd969ad3745db0d66022564cac76cf9424651104 refs/tags/0.23.1\na037c646a47623b92718efadc2bb74d03664b360 refs/tags/0.23.2\n078475ccceeaca0fac947492acdd24514da8d863 refs/tags/0.24.0\ne7bd87dc992512fd5825a557a56907647e03c979 refs/tags/0.24.1\n45ea601e1c7f64fb857bc99df984b86673621d2c refs/tags/0.3.0\n1ea9868a3fee3aa487ab7ec9129208a4dd483d0d refs/tags/0.4.0\n39afdb5733d3245386d29d08c5ff61c89268f499 refs/tags/0.5.0\n458c7b5910fcb157af3fc51bc3b3e663fdb3ed4a refs/tags/0.6.0\n06c38bd8563d822455bc237c2a98c153d938ed1b refs/tags/0.7.0\nf446b6056eef6d8dc9d8b47a79aca93d17dc8230 refs/tags/0.8.0\nbb25a195133f9f7af06386d0809793923cc5e8ab refs/tags/0.9.0\n"),
+    toData("npeg", GitCurrentCommit, 0, "5d80f93aa720898936668b3bc47d0fff101ec414\n"),
+    toData("npeg", GitMergeBase, 0, "5d80f93aa720898936668b3bc47d0fff101ec414\na037c646a47623b92718efadc2bb74d03664b360\na037c646a47623b92718efadc2bb74d03664b360\n"),
+
+    toData("testes", GitDiff, 0, ""),
+    toData("testes", GitTags, 0, "3ce9b2968b5f644755a0ced1baa3eece88c2f12e refs/tags/0.1.0\nf73af8318b54737678fab8b54bdcd8a451015e0d refs/tags/0.1.1\nd21d84d37b161a123a43318bae353108755916de refs/tags/0.1.2\n5c36b6095353ed03b08ac939d00aff2d73f79a35 refs/tags/0.1.3\na1220d11237ee8f135f772ff9731c11b2d91ba31 refs/tags/0.1.4\n574f741b90d04a7ce8c9b990e6077708d7ad076e refs/tags/0.1.5\nced0a9e58234b680def6931578e09165a32e6291 refs/tags/0.1.6\nbb248952e8742a6011eb1a45a9d2059aeb0341d7 refs/tags/0.1.7\nabb7d7c552da0a8e0ddc586c15ccf7e74b0d068b refs/tags/0.10.0\n6e42a768a90d6442196b344bcdcb6f834b76e7b7 refs/tags/0.2.0\n9d136c3a0851ca2c021f5fb4f7b63f0a0ef77232 refs/tags/0.2.1\ndcb282b2da863fd2939e1969cec7a99788feb456 refs/tags/0.2.2\nf708a632afaa40a322a1a61c1c13722edac8e8c5 refs/tags/0.3.0\n3213f59e3f9ba052452c59f01d1418360d856af6 refs/tags/0.3.1\nf7bb1743dffd327958dfcebae4cfb6f61cc1cb8c refs/tags/0.3.2\n6b64569ebecad6bc60cc8697713701e7659204f4 refs/tags/0.3.3\nb51c25a4367bd17f419f78cb5a27f319e9d820f5 refs/tags/0.3.4\nb265612710cbd5ddb1b173c94ece8ec5c7ceccac refs/tags/0.3.5\ne404bcfe42e92d7509717a2dfa115cacb4964c5d refs/tags/0.3.6\n5e4d0d5b7e7f314dde701c546c4365c59782d3dc refs/tags/0.3.7\ne13f91c9c913d2b81c59adeaad687efa2b35293a refs/tags/0.3.8\n17599625f09af0ae4b525e63ab726a3002540702 refs/tags/0.3.9\n13e907f70571dd146d8dc29ddec4599b40ba4e85 refs/tags/0.4.0\n155a74cf676495df1e0674dd07b5e4a0291a9a4a refs/tags/0.4.1\nf37abccdc148cb02ca637a6f0bc8821491cce358 refs/tags/0.4.2\n0250d29ebdd02f28f9020445adb5a4e51fd1902c refs/tags/0.5.0\n2fb87db6d9f34109a70205876030c53f815739b7 refs/tags/0.5.1\n629d17ba8d6a1a4eca8145eb089ed5bca4473dfc refs/tags/0.6.0\ne926130f5f1b7903f68be49cc1563225bd9d948d refs/tags/0.7.0\n7365303897e6185796c274425c079916047e3f14 refs/tags/0.7.1\na735c4adabeba637409f41c4325dd8fc5fb91e2d refs/tags/0.7.10\nfe023fd27404889c5122f902456cbba14b767405 refs/tags/0.7.11\n4430e72972c77a5e9c1555d59bba11d840682691 refs/tags/0.7.12\nf0e53eb490a9558c7f594d2e095b70665e36ca88 refs/tags/0.7.13\nf6520e25e7c329c2957cda447f149fc6a930db0d refs/tags/0.7.2\nd509762f7191757c240d3c79c9ecda53f8c0cfe3 refs/tags/0.7.3\nc02e7a783d1c42fd1f91bca7142f7c3733950c05 refs/tags/0.7.4\n8c8a9e496e9b86ba7602709438980ca31e6989d9 refs/tags/0.7.5\n29839c18b4ac83c0111a178322b57ebb8a8d402c refs/tags/0.7.6\n3b62973cf74fafd8ea906644d89ac34d29a8a6cf refs/tags/0.7.7\ne67ff99dc43c391e89a37f97a9d298c3428bbde2 refs/tags/0.7.8\n4b72ecda0d40ed8e5ab8ad4095a0691d30ec6cd0 refs/tags/0.7.9\n2512b8cc3d7f001d277e89978da2049a5feee5c4 refs/tags/0.8.0\n86c47029690bd2731d204245f3f54462227bba0d refs/tags/0.9.0\n9a7f94f78588e9b5ba7ca077e1f7eae0607c6cf6 refs/tags/0.9.1\n08c915dc016d16c1dfa9a77d0b045ec29c9f2074 refs/tags/0.9.2\n3fb658b1ce1e1efa37d6f9f14322bdac8def02a5 refs/tags/0.9.3\n738fda0add962379ffe6aa6ca5f01a6943a98a2e refs/tags/0.9.4\n48d821add361f7ad768ecb35a0b19c38f90c919e refs/tags/0.9.5\nff9ae890f597dac301b2ac6e6805eb9ac5afd49a refs/tags/0.9.6\n483c78f06e60b0ec5e79fc3476df075ee7286890 refs/tags/0.9.7\n416eec87a5ae39a1a6035552e9e9a47d76b13026 refs/tags/1.0.0\na935cfe9445cc5218fbdd7e0afb35aa1587fff61 refs/tags/1.0.1\n4b83863a9181f054bb695b11b5d663406dfd85d2 refs/tags/1.0.2\n295145fddaa4fe29c1e71a5044d968a84f9dbf69 refs/tags/1.1.0\n8f74ea4e5718436c47305b4488842e6458a13dac refs/tags/1.1.1\n4135bb291e53d615a976e997c44fb2bd9e1ad343 refs/tags/1.1.10\n8c09dbcd16612f5989065db02ea2e7a752dd2656 refs/tags/1.1.11\naedfebdb6c016431d84b0c07cf181b957a900640 refs/tags/1.1.12\n2c2e958366ef6998115740bdf110588d730e5738 refs/tags/1.1.2\nbecc77258321e6ec40d89efdddf37bafd0d07fc3 refs/tags/1.1.3\ne070d7c9853bf94c35b81cf0c0a8980c2449bb22 refs/tags/1.1.4\n12c986cbbf65e8571a486e9230808bf887e5f04f refs/tags/1.1.5\n63df8986f5b56913b02d26954fa033eeaf43714c refs/tags/1.1.6\n38e02c9c6bd728b043036fe0d1894d774cab3108 refs/tags/1.1.7\n3c3879fff16450d28ade79a6b08982bf5cefc061 refs/tags/1.1.8\ne32b811b3b2e70a1d189d7a663bc2583e9c18f96 refs/tags/1.1.9\n0c1b4277c08197ce7e7e0aa2bad91d909fcd96ac refs/tags/2.0.0\n"),
+    toData("testes", GitCurrentCommit, 0, "d9db2ad09aa38fc26625341e1b666602959e144f\n"),
+    toData("testes", GitMergeBase, 0, "d9db2ad09aa38fc26625341e1b666602959e144f\n416eec87a5ae39a1a6035552e9e9a47d76b13026\nd9db2ad09aa38fc26625341e1b666602959e144f\n"),
+    toData("testes", GitCheckout, 0, "416eec87a5ae39a1a6035552e9e9a47d76b13026"),
+
+    toData("grok", GitDiff, 0, ""),
+    toData("grok", GitTags, 0, "2ca193c31fa2377c1e991a080d60ca3215ff6cf0 refs/tags/0.0.1\n48007554b21ba2f65c726ae2fdda88d621865b4a refs/tags/0.0.2\n7092a0286421c7818cd335cca9ebc72d03d866c2 refs/tags/0.0.3\n62707b8ac684efac35d301dbde57dc750880268e refs/tags/0.0.4\n876f2504e0c2f785ffd2cf65a78e2aea474fa8aa refs/tags/0.0.5\nb7eb1f2501aa2382cb3a38353664a13af62a9888 refs/tags/0.0.6\nf5d818bfd6038884b3d8b531c58484ded20a58a4 refs/tags/0.1.0\n961eaddea49c3144d130d105195583d3f11fb6c6 refs/tags/0.2.0\n15ab8ed8d4f896232a976a9008548bd53af72a66 refs/tags/0.2.1\n426a7d7d4603f77ced658e73ad7f3f582413f6cd refs/tags/0.3.0\n83cf7a39b2fe897786fb0fe01a7a5933c3add286 refs/tags/0.3.1\n8d2e3c900edbc95fa0c036fd76f8e4f814aef2c1 refs/tags/0.3.2\n48b43372f49a3bb4dc0969d82a0fca183fb94662 refs/tags/0.3.3\n9ca947a3009ea6ba17814b20eb953272064eb2e6 refs/tags/0.4.0\n1b5643d04fba6d996a16d1ffc13d034a40003f8f refs/tags/0.5.0\n486b0eb580b1c465453d264ac758cc490c19c33e refs/tags/0.5.1\naedb0d9497390e20b9d2541cef2bb05a5cda7a71 refs/tags/0.5.2\n"),
+    toData("grok", GitCurrentCommit, 0, "4e6526a91a23eaec778184e16ce9a34d25d48bdc\n"),
+    toData("grok", GitMergeBase, 0, "4e6526a91a23eaec778184e16ce9a34d25d48bdc\n62707b8ac684efac35d301dbde57dc750880268e\n349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitCheckout, 0, "62707b8ac684efac35d301dbde57dc750880268e"),
+
+    toData("nim-bytes2human", GitDiff, 0, ""),
+    toData("nim-bytes2human", GitTags, 0, ""),
+    toData("nim-bytes2human", GitCurrentcommit, 0, "ec2c1a758cabdd4751a06c8ebf2b923f19e32731\n")
+  ]
+
+#[
+Current directory is now E:\atlastest\nim-bytes2human
+cmd git diff args [] --> ("", 0)
+cmd git show-ref --tags args [] --> ("", 1)
+cmd git log -n 1 --format=%H args [] --> (, 0)
+[Warning] (nim-bytes2human) package has no tagged releases
+nimble E:\atlastest\nim-bytes2human\bytes2human.nimble info (requires: @["nim >= 1.0.0"], srcDir: "src", tasks: @[])
+[Error] There were problems.
+Error: execution of an external program failed: 'E:\nim\tools\atlas\atlas.exe clone https://github.com/disruptek/balls'
+]#
diff --git a/tools/atlas/tests/balls.nimble b/tools/atlas/tests/balls.nimble
new file mode 100644
index 000000000..62ab4750b
--- /dev/null
+++ b/tools/atlas/tests/balls.nimble
@@ -0,0 +1,32 @@
+version = "3.4.1"
+author = "disruptek"
+description = "a unittest framework with balls 🔴🟡🟢"
+license = "MIT"
+
+# requires newTreeFrom
+requires "https://github.com/disruptek/grok >= 0.5.0 & < 1.0.0"
+requires "https://github.com/disruptek/ups < 1.0.0"
+requires "https://github.com/planetis-m/sync >= 1.0.0 & < 2.0.0"
+#requires "https://github.com/c-blake/cligen < 2.0.0"
+
+bin = @["balls"]            # build the binary for basic test running
+installExt = @["nim"]       # we need to install balls.nim also
+skipDirs = @["tests"]       # so stupid...  who doesn't want tests?
+#installFiles = @["balls.nim"] # https://github.com/nim-lang/Nim/issues/16661
+
+task test, "run tests for ci":
+  when defined(windows):
+    exec "balls.cmd"
+  else:
+    exec "balls"
+
+task demo, "produce a demo":
+  exec "nim c --define:release balls.nim"
+  when (NimMajor, NimMinor) != (1, 0):
+    echo "due to nim bug #16307, use nim-1.0"
+    quit 1
+  exec """demo docs/demo.svg "nim c --out=\$1 examples/fails.nim""""
+  exec """demo docs/clean.svg "nim c --define:danger -f --out=\$1 tests/test.nim""""
+  exec "nim c --define:release --define:ballsDry balls.nim"
+  exec """demo docs/runner.svg "balls""""
+
diff --git a/tools/atlas/tests/grok.nimble b/tools/atlas/tests/grok.nimble
new file mode 100644
index 000000000..1b6d77c08
--- /dev/null
+++ b/tools/atlas/tests/grok.nimble
@@ -0,0 +1,5 @@
+version = "0.0.4"
+author = "disruptek"
+description = "don't read too much into it"
+license = "MIT"
+requires "nim >= 1.0.0"
diff --git a/tools/atlas/tests/nim-bytes2human.nimble b/tools/atlas/tests/nim-bytes2human.nimble
new file mode 100644
index 000000000..9f3ae2479
--- /dev/null
+++ b/tools/atlas/tests/nim-bytes2human.nimble
@@ -0,0 +1,7 @@
+version     = "0.2.2"
+author      = "Juan Carlos"
+description = "Convert bytes to kilobytes, megabytes, gigabytes, etc."
+license     = "MIT"
+srcDir      = "src"
+
+requires "nim >= 1.0.0"  # https://github.com/juancarlospaco/nim-bytes2human/issues/2#issue-714338524
diff --git a/tools/atlas/tests/nim.cfg b/tools/atlas/tests/nim.cfg
new file mode 100644
index 000000000..5f568569b
--- /dev/null
+++ b/tools/atlas/tests/nim.cfg
@@ -0,0 +1,11 @@
+############# begin Atlas config section ##########
+--noNimblePath
+--path:"../balls"
+--path:"../grok"
+--path:"../ups"
+--path:"../sync"
+--path:"../npeg/src"
+--path:"../testes"
+--path:"../grok"
+--path:"../nim-bytes2human/src"
+############# end Atlas config section   ##########
diff --git a/tools/atlas/tests/npeg.nimble b/tools/atlas/tests/npeg.nimble
new file mode 100644
index 000000000..e71fc5aa5
--- /dev/null
+++ b/tools/atlas/tests/npeg.nimble
@@ -0,0 +1,48 @@
+# Package
+
+version       = "0.24.1"
+author        = "Ico Doornekamp"
+description   = "a PEG library"
+license       = "MIT"
+srcDir        = "src"
+installExt    = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.19.0"
+
+# Test
+
+task test, "Runs the test suite":
+  exec "nimble testc && nimble testcpp && nimble testarc && nimble testjs"
+
+task testc, "C tests":
+  exec "nim c -r tests/tests.nim"
+
+task testcpp, "CPP tests":
+  exec "nim cpp -r tests/tests.nim"
+
+task testjs, "JS tests":
+  exec "nim js -r tests/tests.nim"
+
+task testdanger, "Runs the test suite in danger mode":
+  exec "nim c -d:danger -r tests/tests.nim"
+
+task testwin, "Mingw tests":
+  exec "nim c -d:mingw tests/tests.nim && wine tests/tests.exe"
+
+task test32, "32 bit tests":
+  exec "nim c --cpu:i386 --passC:-m32 --passL:-m32 tests/tests.nim && tests/tests"
+
+task testall, "Test all":
+  exec "nimble test && nimble testcpp && nimble testdanger && nimble testjs && nimble testwin"
+
+when (NimMajor, NimMinor) >= (1, 1):
+  task testarc, "--gc:arc tests":
+    exec "nim c --gc:arc -r tests/tests.nim"
+else:
+  task testarc, "--gc:arc tests":
+    exec "true"
+
+task perf, "Test performance":
+  exec "nim cpp -r -d:danger tests/performance.nim"
diff --git a/tools/atlas/tests/packages/packages.json b/tools/atlas/tests/packages/packages.json
new file mode 100644
index 000000000..d054a201b
--- /dev/null
+++ b/tools/atlas/tests/packages/packages.json
@@ -0,0 +1,36 @@
+[
+  {
+    "name": "bytes2human",
+    "url": "https://github.com/juancarlospaco/nim-bytes2human",
+    "method": "git",
+    "tags": [
+      "bytes",
+      "human",
+      "minimalism",
+      "size"
+    ],
+    "description": "Convert bytes to kilobytes, megabytes, gigabytes, etc.",
+    "license": "LGPLv3",
+    "web": "https://github.com/juancarlospaco/nim-bytes2human"
+  },
+  {
+    "name": "npeg",
+    "url": "https://github.com/zevv/npeg",
+    "method": "git",
+    "tags": [
+      "PEG",
+      "parser",
+      "parsing",
+      "regexp",
+      "regular",
+      "grammar",
+      "lexer",
+      "lexing",
+      "pattern",
+      "matching"
+    ],
+    "description": "PEG (Parsing Expression Grammars) string matching library for Nim",
+    "license": "MIT",
+    "web": "https://github.com/zevv/npeg"
+  }
+]
diff --git a/tools/atlas/tests/sync.nimble b/tools/atlas/tests/sync.nimble
new file mode 100644
index 000000000..a07ae8925
--- /dev/null
+++ b/tools/atlas/tests/sync.nimble
@@ -0,0 +1,10 @@
+# Package
+
+version     = "1.4.0"
+author      = "Antonis Geralis"
+description = "Useful synchronization primitives."
+license     = "MIT"
+
+# Deps
+
+requires "nim >= 1.0.0"
diff --git a/tools/atlas/tests/testes.nimble b/tools/atlas/tests/testes.nimble
new file mode 100644
index 000000000..60fe1d508
--- /dev/null
+++ b/tools/atlas/tests/testes.nimble
@@ -0,0 +1,23 @@
+version = "1.0.0"
+author = "disruptek"
+description = "a cure for salty testes"
+license = "MIT"
+
+#requires "cligen >= 0.9.41 & <= 0.9.45"
+#requires "bump >= 1.8.18 & < 2.0.0"
+requires "https://github.com/disruptek/grok >= 0.0.4 & < 1.0.0"
+requires "https://github.com/juancarlospaco/nim-bytes2human"
+
+bin = @["testes"]           # build the binary for basic test running
+installExt = @["nim"]       # we need to install testes.nim also
+skipDirs = @["tests"]       # so stupid...  who doesn't want tests?
+
+task test, "run tests for ci":
+  exec "nim c --run testes.nim"
+
+task demo, "produce a demo":
+  when (NimMajor, NimMinor) != (1, 0):
+    echo "due to nim bug #16307, use nim-1.0"
+    quit 1
+  exec """demo docs/demo.svg "nim c --out=\$1 examples/balls.nim""""
+  exec """demo docs/clean.svg "nim c --define:danger --out=\$1 tests/testicles.nim""""
diff --git a/tools/atlas/tests/ups.nimble b/tools/atlas/tests/ups.nimble
new file mode 100644
index 000000000..d91abbe60
--- /dev/null
+++ b/tools/atlas/tests/ups.nimble
@@ -0,0 +1,13 @@
+version = "0.0.2"
+author = "disruptek"
+description = "a package handler"
+license = "MIT"
+
+requires "npeg >= 0.23.2 & < 1.0.0"
+requires "https://github.com/disruptek/testes >= 1.0.0 & < 2.0.0"
+
+task test, "run tests":
+  when defined(windows):
+    exec "testes.cmd"
+  else:
+    exec findExe"testes"

From 4eca84cff7a16aa227b1b412d84dcecacc47dad5 Mon Sep 17 00:00:00 2001
From: Miran 
Date: Tue, 27 Jul 2021 01:13:47 +0200
Subject: [PATCH 0276/1654] don't use plus instead of space (#18589)

---
 compiler/docgen.nim | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 492668543..3bc01bbe6 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -919,9 +919,9 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
     sortName = if k in routineKinds: plainName.strip else: name
     cleanPlainSymbol = renderPlainSymbolName(nameNode)
     complexSymbol = complexName(k, n, cleanPlainSymbol)
-    plainSymbolEnc = encodeUrl(cleanPlainSymbol)
+    plainSymbolEnc = encodeUrl(cleanPlainSymbol, usePlus = false)
     symbolOrId = d.newUniquePlainSymbol(complexSymbol)
-    symbolOrIdEnc = encodeUrl(symbolOrId)
+    symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false)
     deprecationMsg = genDeprecationMsg(d, pragmaNode)
 
   nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,

From 22776c410a916b29c05b798523c94c834c7371ad Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Tue, 27 Jul 2021 01:15:21 +0200
Subject: [PATCH 0277/1654] Update documentation for dynlib (#17356)

* Update documentation for dynlib

* Apply suggestions

* Improve error handling in example
---
 lib/pure/dynlib.nim | 78 ++++++++++++++++++++-------------------------
 1 file changed, 34 insertions(+), 44 deletions(-)

diff --git a/lib/pure/dynlib.nim b/lib/pure/dynlib.nim
index 3c1466322..48fd91b8f 100644
--- a/lib/pure/dynlib.nim
+++ b/lib/pure/dynlib.nim
@@ -17,75 +17,64 @@
 ## Loading a simple C function
 ## ---------------------------
 ##
-## The following example demonstrates loading a function called 'greet'
+## The following example demonstrates loading a function called `greet`
 ## from a library that is determined at runtime based upon a language choice.
-## If the library fails to load or the function 'greet' is not found,
+## If the library fails to load or the function `greet` is not found,
 ## it quits with a failure error code.
 ##
-## .. code-block:: Nim
-##
-##   import std/dynlib
-##
-##   type
-##     greetFunction = proc(): cstring {.gcsafe, stdcall.}
-##
-##   let lang = stdin.readLine()
-##
-##   let lib = case lang
-##   of "french":
-##     loadLib("french.dll")
-##   else:
-##     loadLib("english.dll")
-##
-##   if lib == nil:
-##     echo "Error loading library"
-##     quit(QuitFailure)
-##
-##   let greet = cast[greetFunction](lib.symAddr("greet"))
-##
-##   if greet == nil:
-##     echo "Error loading 'greet' function from library"
-##     quit(QuitFailure)
-##
-##   let greeting = greet()
-##
-##   echo greeting
-##
-##   unloadLib(lib)
-##
+runnableExamples:
+  type
+    GreetFunction = proc (): cstring {.gcsafe, stdcall.}
 
-import strutils
+  proc loadGreet(lang: string) =
+    let lib =
+      case lang
+      of "french":
+        loadLib("french.dll")
+      else:
+        loadLib("english.dll")
+    assert lib != nil, "Error loading library"
+
+    let greet = cast[GreetFunction](lib.symAddr("greet"))
+    assert greet != nil, "Error loading 'greet' function from library"
+
+    echo greet()
+
+    unloadLib(lib)
+
+
+import std/strutils
 
 type
-  LibHandle* = pointer ## a handle to a dynamically loaded library
+  LibHandle* = pointer ## A handle to a dynamically loaded library.
 
 proc loadLib*(path: string, globalSymbols = false): LibHandle {.gcsafe.}
-  ## loads a library from `path`. Returns nil if the library could not
+  ## Loads a library from `path`. Returns nil if the library could not
   ## be loaded.
 
 proc loadLib*(): LibHandle {.gcsafe.}
-  ## gets the handle from the current executable. Returns nil if the
+  ## Gets the handle from the current executable. Returns nil if the
   ## library could not be loaded.
 
 proc unloadLib*(lib: LibHandle) {.gcsafe.}
-  ## unloads the library `lib`
+  ## Unloads the library `lib`.
 
 proc raiseInvalidLibrary*(name: cstring) {.noinline, noreturn.} =
-  ## raises an `EInvalidLibrary` exception.
+  ## Raises a `LibraryError` exception.
   raise newException(LibraryError, "could not find symbol: " & $name)
 
 proc symAddr*(lib: LibHandle, name: cstring): pointer {.gcsafe.}
-  ## retrieves the address of a procedure/variable from `lib`. Returns nil
+  ## Retrieves the address of a procedure/variable from `lib`. Returns nil
   ## if the symbol could not be found.
 
 proc checkedSymAddr*(lib: LibHandle, name: cstring): pointer =
-  ## retrieves the address of a procedure/variable from `lib`. Raises
-  ## `EInvalidLibrary` if the symbol could not be found.
+  ## Retrieves the address of a procedure/variable from `lib`. Raises
+  ## `LibraryError` if the symbol could not be found.
   result = symAddr(lib, name)
   if result == nil: raiseInvalidLibrary(name)
 
 proc libCandidates*(s: string, dest: var seq[string]) =
-  ## given a library name pattern `s` write possible library names to `dest`.
+  ## Given a library name pattern `s`, write possible library names to `dest`.
   var le = strutils.find(s, '(')
   var ri = strutils.find(s, ')', le+1)
   if le >= 0 and ri > le:
@@ -97,8 +86,9 @@ proc libCandidates*(s: string, dest: var seq[string]) =
     add(dest, s)
 
 proc loadLibPattern*(pattern: string, globalSymbols = false): LibHandle =
-  ## loads a library with name matching `pattern`, similar to what `dynlib`
+  ## Loads a library with name matching `pattern`, similar to what the `dynlib`
   ## pragma does. Returns nil if the library could not be loaded.
+  ##
   ## .. warning:: this proc uses the GC and so cannot be used to load the GC.
   var candidates = newSeq[string]()
   libCandidates(pattern, candidates)

From 9cb5ab0108c62bfda30b4f9b52b2957fdd364544 Mon Sep 17 00:00:00 2001
From: flywind 
Date: Tue, 27 Jul 2021 13:21:39 +0800
Subject: [PATCH 0278/1654] fix #18578 (#18580)

* fix #18578

* add tests

* tiny

* apply changes

* typo

* add removeStaticFile
---
 compiler/extccomp.nim         | 15 +++++++++------
 tests/compiles/mstaticlib.nim |  1 +
 tests/compiles/tstaticlib.nim | 22 ++++++++++++++++++++++
 3 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 tests/compiles/mstaticlib.nim
 create mode 100644 tests/compiles/tstaticlib.nim

diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim
index 831acae68..a091060ff 100644
--- a/compiler/extccomp.nim
+++ b/compiler/extccomp.nim
@@ -665,9 +665,10 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
   addExternalFileToCompile(conf, c)
 
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
-                objfiles: string, isDllBuild: bool): string =
+                objfiles: string, isDllBuild: bool, removeStaticFile: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    removeFile output # fixes: bug #16947
+    if removeStaticFile:
+      removeFile output # fixes: bug #16947
     result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
                                             "objfiles", objfiles]
   else:
@@ -750,8 +751,9 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
   if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc:
     result.add " /Zi /FS /Od"
 
-template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string =
-  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions)
+template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string,
+                    removeStaticFile = false): string =
+  getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile)
 
 template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) =
   try:
@@ -891,7 +893,7 @@ proc callCCompiler*(conf: ConfigRef) =
         let objFile = conf.getObjFilePath(x)
         let buildDll = idx != mainFileIdx
         let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll)
-        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll))
+        cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true))
         # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache
         # for more info check the comment inside of getLinkCmd() where the /PDB: MSVC flag is used
         if isVSCompatible(conf):
@@ -914,7 +916,8 @@ proc callCCompiler*(conf: ConfigRef) =
         objfiles.add(quoteShell(objFile))
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
                        else: AbsoluteFile(conf.projectName)
-      linkCmd = getLinkCmd(conf, mainOutput, objfiles)
+
+      linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true)
       extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
         const MaxCmdLen = when defined(windows): 8_000 else: 32_000
diff --git a/tests/compiles/mstaticlib.nim b/tests/compiles/mstaticlib.nim
new file mode 100644
index 000000000..6ed593691
--- /dev/null
+++ b/tests/compiles/mstaticlib.nim
@@ -0,0 +1 @@
+echo 1234
\ No newline at end of file
diff --git a/tests/compiles/tstaticlib.nim b/tests/compiles/tstaticlib.nim
new file mode 100644
index 000000000..a18b59204
--- /dev/null
+++ b/tests/compiles/tstaticlib.nim
@@ -0,0 +1,22 @@
+import std/[os, osproc, strformat]
+
+
+const dir = "tests/compiles"
+const fileName = dir / "mstaticlib.nim"
+const nim = getCurrentCompilerExe()
+
+block: # bug #18578
+  const libName = dir / "tstaticlib1.a"
+  let (_, status) = execCmdEx(fmt"{nim} c -o:{libName} --app:staticlib {fileName}")
+  doAssert status == 0
+  doAssert fileExists(libName)
+  removeFile(libName)
+
+block: # bug #16947
+  const libName = dir / "tstaticlib2.a"
+  writeFile(libName, "echo 124")
+  doAssert fileExists(libName)
+  let (_, status) = execCmdEx(fmt"{nim} c -o:{libName} --app:staticlib {fileName}")
+  doAssert status == 0
+  doAssert fileExists(libName)
+  removeFile(libName)

From fa0209609d3bedd3466f162aa350d261907851ce Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 27 Jul 2021 09:36:19 +0200
Subject: [PATCH 0279/1654] fixes #18565 (#18593)

* fixes #18565
---
 compiler/semstmts.nim         | 22 +++++++++++-----------
 compiler/sigmatch.nim         |  4 +++-
 tests/arc/tkeys_lent.nim      | 17 +++++++++++++++++
 tests/stdlib/tgetfileinfo.nim |  1 +
 tests/stdlib/tos.nim          |  1 +
 5 files changed, 33 insertions(+), 12 deletions(-)
 create mode 100644 tests/arc/tkeys_lent.nim

diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index bfce91849..75323a95e 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -2260,20 +2260,20 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode =
   #                                         nkNilLit, nkEmpty}:
   #  dec last
   for i in 0..
Date: Tue, 27 Jul 2021 01:36:59 -0600
Subject: [PATCH 0280/1654] Fixed const tuples in inferred generics (#18598)

---
 compiler/semgnrc.nim                     | 33 ++++++++++++------------
 tests/tuples/tinferred_generic_const.nim | 14 ++++++++++
 2 files changed, 30 insertions(+), 17 deletions(-)
 create mode 100644 tests/tuples/tinferred_generic_const.nim

diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim
index 4f3c50dae..3df1e0c3f 100644
--- a/compiler/semgnrc.nim
+++ b/compiler/semgnrc.nim
@@ -391,16 +391,24 @@ proc semGenericStmt(c: PContext, n: PNode,
       a[^1] = semGenericStmtScope(c, a[^1], flags, ctx)
       closeScope(c)
 
-  of nkVarSection, nkLetSection:
+  of nkVarSection, nkLetSection, nkConstSection:
+    let varKind =
+      case n.kind
+      of nkVarSection: skVar
+      of nkLetSection: skLet
+      else: skConst
     for i in 0..
Date: Tue, 27 Jul 2021 00:37:11 -0700
Subject: [PATCH 0281/1654] add comment to astalgo.debug (#18594)

---
 compiler/astalgo.nim | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim
index 3c97665f0..94fa9da93 100644
--- a/compiler/astalgo.nim
+++ b/compiler/astalgo.nim
@@ -585,6 +585,9 @@ proc value(this: var DebugPrinter; value: PNode) =
   this.openCurly
   this.key "kind"
   this.value  value.kind
+  if value.comment.len > 0:
+    this.key "comment"
+    this.value  value.comment
   when defined(useNodeIds):
     this.key "id"
     this.value value.id

From 37f5f0d7d08d6057b20d3e319b38dfe82ce1ba65 Mon Sep 17 00:00:00 2001
From: GordonBGood 
Date: Tue, 27 Jul 2021 14:38:30 +0700
Subject: [PATCH 0282/1654] Clarify use of import via pseudo dir... (#18569)

---
 doc/manual.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/manual.rst b/doc/manual.rst
index 9f614c2cd..1b65ae9ee 100644
--- a/doc/manual.rst
+++ b/doc/manual.rst
@@ -6190,6 +6190,7 @@ There are two pseudo directories:
    its semantics are: *Use the search path to look for module name but ignore the standard
    library locations*. In other words, it is the opposite of `std`.
 
+It is recommended and preferred but not currently enforced that all stdlib module imports include the std/ "pseudo directory" as part of the import name.
 
 From import statement
 ---------------------

From 493721c16c06b5681dc270679bdcbb41011614b2 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 27 Jul 2021 00:56:16 -0700
Subject: [PATCH 0283/1654] delEnv now works at CT (#18568)

---
 compiler/vmops.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler/vmops.nim b/compiler/vmops.nim
index bc331bbcd..6ee915bd5 100644
--- a/compiler/vmops.nim
+++ b/compiler/vmops.nim
@@ -20,7 +20,7 @@ when declared(math.copySign):
 when declared(math.signbit):
   from std/math import signbit
 
-from std/os import getEnv, existsEnv, dirExists, fileExists, putEnv, walkDir,
+from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir,
                    getAppFilename, raiseOSError, osLastError
 
 from std/md5 import getMD5
@@ -215,6 +215,7 @@ proc registerAdditionalOps*(c: PCtx) =
     wrap2s(getEnv, osop)
     wrap1s(existsEnv, osop)
     wrap2svoid(putEnv, osop)
+    wrap1svoid(delEnv, osop)
     wrap1s(dirExists, osop)
     wrap1s(fileExists, osop)
     wrapDangerous(writeFile, ioop)

From c86f9590fb5ba6512ba33994fa0711341eb8524b Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 27 Jul 2021 12:48:27 +0200
Subject: [PATCH 0284/1654] fixes #18570 (#18599)

---
 compiler/injectdestructors.nim | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim
index cf09b3ff5..985f27878 100644
--- a/compiler/injectdestructors.nim
+++ b/compiler/injectdestructors.nim
@@ -799,7 +799,8 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
           result[i] = p(n[i], c, s, m)
     of nkObjConstr:
       # see also the remark about `nkTupleConstr`.
-      let isRefConstr = n.typ.skipTypes(abstractInst).kind == tyRef
+      let t = n.typ.skipTypes(abstractInst)
+      let isRefConstr = t.kind == tyRef
       let m = if isRefConstr: sinkArg
               elif mode == normal: normal
               else: sinkArg
@@ -807,7 +808,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode): PNode =
       result = copyTree(n)
       for i in 1..
Date: Tue, 27 Jul 2021 14:32:56 +0200
Subject: [PATCH 0285/1654] fixes #18579 (#18600)

---
 lib/system/arc.nim         |  6 ++----
 tests/arc/tunref_cycle.nim | 26 ++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 4 deletions(-)
 create mode 100644 tests/arc/tunref_cycle.nim

diff --git a/lib/system/arc.nim b/lib/system/arc.nim
index ed5e9f5ca..d66f4b997 100644
--- a/lib/system/arc.nim
+++ b/lib/system/arc.nim
@@ -207,10 +207,8 @@ proc nimDecRefIsLast(p: pointer): bool {.compilerRtl, inl.} =
 
 proc GC_unref*[T](x: ref T) =
   ## New runtime only supports this operation for 'ref T'.
-  if nimDecRefIsLast(cast[pointer](x)):
-    # XXX this does NOT work for virtual destructors!
-    `=destroy`(x[])
-    nimRawDispose(cast[pointer](x), T.alignOf)
+  var y {.cursor.} = x
+  `=destroy`(y)
 
 proc GC_ref*[T](x: ref T) =
   ## New runtime only supports this operation for 'ref T'.
diff --git a/tests/arc/tunref_cycle.nim b/tests/arc/tunref_cycle.nim
new file mode 100644
index 000000000..82551b7f7
--- /dev/null
+++ b/tests/arc/tunref_cycle.nim
@@ -0,0 +1,26 @@
+discard """
+  outputsub: '''inside closure
+hello world'''
+  cmd: "nim c --gc:orc -d:useMalloc $file"
+  valgrind: true
+"""
+
+# bug #18579
+
+var fp: proc (env: pointer) {.cdecl.}
+var env: pointer
+
+proc store(f: proc (){.closure.}) =
+  proc closeOver() =
+    echo "inside closure"
+    f()
+  (fp,env) = (cast[proc(env: pointer){.cdecl.}](rawProc closeOver), rawEnv closeOver)
+  GC_ref(cast[RootRef](env))
+
+proc run() =
+  fp(env)
+  GC_unref(cast[RootRef](env))
+
+store(proc() = echo "hello world")
+run()
+GC_fullCollect()

From 4920b06973e69c525405ff3147dc8a823a4c3c58 Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Tue, 27 Jul 2021 19:04:55 +0200
Subject: [PATCH 0286/1654] fixes #18543 (#18601)

* fixes #18543

* make tests green again
---
 changelog.md                     |   1 -
 lib/core/macros.nim              | 264 +++++++++++--------------------
 testament/important_packages.nim |   1 +
 tests/pragmas/tcustom_pragma.nim |  25 +--
 4 files changed, 107 insertions(+), 184 deletions(-)

diff --git a/changelog.md b/changelog.md
index d33e24b65..89e013cff 100644
--- a/changelog.md
+++ b/changelog.md
@@ -51,7 +51,6 @@
   implementations. Old behavior can be obtained with
   `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same
   underlying code is also updated the same way.
-- Custom pragma values have now an API for use in macros.
 
 - On POSIX systems, the default signal handlers used for Nim programs (it's
   used for printing the stacktrace on fatal signals) will now re-raise the
diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index c09fae6b3..d3eaf6298 100644
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -1068,11 +1068,11 @@ proc newEmptyNode*(): NimNode {.noSideEffect.} =
   ## Create a new empty node.
   result = newNimNode(nnkEmpty)
 
-proc newStmtList*(stmts: varargs[NimNode]): NimNode=
+proc newStmtList*(stmts: varargs[NimNode]): NimNode =
   ## Create a new statement list.
   result = newNimNode(nnkStmtList).add(stmts)
 
-proc newPar*(exprs: varargs[NimNode]): NimNode=
+proc newPar*(exprs: varargs[NimNode]): NimNode =
   ## Create a new parentheses-enclosed expression.
   newNimNode(nnkPar).add(exprs)
 
@@ -1494,148 +1494,80 @@ macro expandMacros*(body: typed): untyped =
   echo body.toStrLit
   result = body
 
-proc findPragmaNodeInRecList(arg, fieldSym: NimNode): NimNode =
-  case arg.kind
-  of nnkRecList, nnkRecCase:
-    for it in arg.children:
-      result = findPragmaNodeInRecList(it, fieldSym)
-      if result != nil: return
-  of nnkOfBranch:
-    return findPragmaNodeInRecList(arg[1], fieldSym)
-  of nnkElse:
-    return findPragmaNodeInRecList(arg[0], fieldSym)
-  of nnkIdentDefs:
-    for i in 0.. 0:
-    impl.expectKind nnkTypeDef
-    let pragmaExpr = impl[0]
-    if pragmaExpr.kind == nnkPragmaExpr:
-      result = pragmaExpr[1]
-
-proc getPragmaNodeFromType(typ: NimNode): NimNode =
-  case typ.kind
-  of nnkSym:
-    result = getPragmaNodeFromTypeSym(typ)
-  of nnkProcTy:
-    result = typ[1]
-  else: error("illegal typ kind for argument: " & $typ.kind, typ)
-
-proc getPragmaNodeFromVarLetSym(sym: NimNode): NimNode =
-  sym.expectKind nnkSym
-  if sym.symKind notin {nskVar, nskLet}: error("expected var/let sym", sym)
-  let impl = sym.getImpl
-  impl.expectKind nnkIdentDefs
-  impl.expectLen 3
-  let pragmaExpr = impl[0]
-  if pragmaExpr.kind == nnkPragmaExpr:
-    result = pragmaExpr[1]
-
-proc getPragmasByName(pragmaExpr: NimNode, name: string): seq[NimNode] =
-  if pragmaExpr.kind == nnkPragma:
-    for it in pragmaExpr:
-      if it.kind in nnkPragmaCallKinds:
-        if eqIdent(it[0], name):
-          result.add it
-      elif it.kind == nnkSym:
-        if eqIdent(it, name):
-          result.add it
-
-proc getCustomPragmaNodes(sym: NimNode, name: string): seq[NimNode] =
-  sym.expectKind nnkSym
-  case sym.symKind
-  of nskField:
-    result = getPragmaNodeFromObjFieldSym(sym).getPragmasByName(name)
-  of nskProc:
-    result = getPragmaNodeFromProcSym(sym).getPragmasByName(name)
-  of nskType:
-    result = getPragmaNodeFromTypeSym(sym).getPragmasByName(name)
-  of nskParam:
-    # When a typedesc parameter is passed to the macro, it will be of nskParam.
-    let typeInst = getTypeInst(sym)
-    if typeInst.kind == nnkBracketExpr and eqIdent(typeInst[0], "typeDesc"):
-      result = getPragmaNodeFromTypeSym(typeInst[1]).getPragmasByName(name)
+  if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy:
+    return typ[1][1]
+  elif typ.typeKind == ntyTypeDesc:
+    let impl = typ[1].getImpl()
+    if impl[0].kind == nnkPragmaExpr:
+      return impl[0][1]
     else:
-      error("illegal sym kind for argument: " & $sym.symKind, sym)
-  of nskVar, nskLet:
-    # This checks the type of the sym too, this is consistent with how
-    # field expressions are handled too. If this is changed, make sure to
-    # change it for fields expressions too.
-    result = getPragmaNodeFromType(sym.getTypeInst).getPragmasByName(name)
-    result.add getPragmaNodeFromVarLetSym(sym).getPragmasByName(name)
-  else:
-    error("illegal sym kind for argument: " & $sym.symKind, sym)
+      return impl[0] # handle types which don't have macro at all
 
-since (1, 5):
-  export getCustomPragmaNodes
+  if n.kind == nnkSym: # either an variable or a proc
+    let impl = n.getImpl()
+    if impl.kind in RoutineNodes:
+      return impl.pragma
+    elif impl.kind == nnkIdentDefs and impl[0].kind == nnkPragmaExpr:
+      return impl[0][1]
+    else:
+      let timpl = typ.getImpl()
+      if timpl.len>0 and timpl[0].len>1:
+        return timpl[0][1]
+      else:
+        return timpl
 
-proc hasCustomPragma*(n: NimNode, name: string): bool =
-  n.expectKind nnkSym
-  result = getCustomPragmaNodes(n, name).len > 0
+  if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}:
+    let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1])
+    let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0])
+    var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst)
+    while typDef != nil:
+      typDef.expectKind(nnkTypeDef)
+      let typ = typDef[2]
+      typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy})
+      let isRef = typ.kind in {nnkRefTy, nnkPtrTy}
+      if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X)
+        typDef = getImpl(typ[0])
+      else: # object definition, maybe an object directly defined as a ref type
+        let
+          obj = (if isRef: typ[0] else: typ)
+        var identDefsStack = newSeq[NimNode](obj[2].len)
+        for i in 0.. 0:
+          var identDefs = identDefsStack.pop()
+          if identDefs.kind == nnkRecCase:
+            identDefsStack.add(identDefs[0])
+            for i in 1.. 0)
+  let pragmaNode = customPragmaNode(n)
+  for p in pragmaNode:
+    if (p.kind == nnkSym and p == cp) or
+        (p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp):
+      return newLit(true)
+  return newLit(false)
 
-iterator iterOverFormalArgs(f: NimNode): tuple[name, typ, val: NimNode] =
-  f.expectKind nnkFormalParams
-  for i in 1.. 0 and p[0].kind == nnkSym and p[0] == cp:
+      if p.len == 2:
+        result = p[1]
+      else:
+        let def = p[0].getImpl[3]
+        result = newTree(nnkPar)
+        for i in 1 ..< def.len:
+          let key = def[i][0]
+          let val = p[i]
+          result.add newTree(nnkExprColonExpr, key, val)
+      break
+  if result.kind == nnkEmpty:
+    error(n.repr & " doesn't have a pragma named " & cp.repr()) # returning an empty node results in most cases in a cryptic error,
 
 macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
   ## Calls `callee` with `args` unpacked as individual arguments.
diff --git a/testament/important_packages.nim b/testament/important_packages.nim
index afceffb2e..ab29dcd0d 100644
--- a/testament/important_packages.nim
+++ b/testament/important_packages.nim
@@ -114,6 +114,7 @@ pkg "nimsvg"
 pkg "nimterop", "nimble minitest"
 pkg "nimwc", "nim c nimwc.nim"
 pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true
+pkg "nimYAML", "nim c -r test/tserialization.nim"
 pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter"
 pkg "norm", "nim c -r tests/sqlite/trows.nim"
 pkg "npeg", "nimble testarc"
diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim
index 3ef5bdcb6..b197a7c55 100644
--- a/tests/pragmas/tcustom_pragma.nim
+++ b/tests/pragmas/tcustom_pragma.nim
@@ -384,7 +384,7 @@ block:
   discard Hello(a: 1.0, b: 12)
 
 # issue #11511
-block:
+when false:
   template myAttr {.pragma.}
 
   type TObj = object
@@ -395,23 +395,24 @@ block:
     let recList = objTy[2]
     let sym = recList[0]
     assert sym.kind == nnkSym and sym.eqIdent("a")
-    let hasAttr = sym.hasCustomPragma("myAttr")
+    let hasAttr = sym.hasCustomPragma(myAttr)
     newLit(hasAttr)
 
   doAssert hasMyAttr(TObj)
 
-# misc
-{.pragma: haha.}
-{.pragma: hoho.}
-template hehe(key, val: string, haha) {.pragma.}
+when false:
+  # misc
+  {.pragma: haha.}
+  {.pragma: hoho.}
+  template hehe(key, val: string, haha) {.pragma.}
 
-type A {.haha, hoho, haha, hehe("hi", "hu", "he").} = int
+  type A {.haha, hoho, haha, hehe("hi", "hu", "he").} = int
 
-assert A.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")
+  assert A.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")
 
-template hehe(key, val: int) {.pragma.}
+  template hehe(key, val: int) {.pragma.}
 
-var bb {.haha, hoho, hehe(1, 2), haha, hehe("hi", "hu", "he").} = 3
+  var bb {.haha, hoho, hehe(1, 2), haha, hehe("hi", "hu", "he").} = 3
 
-# left-to-right priority/override order for getCustomPragmaVal
-assert bb.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")
+  # left-to-right priority/override order for getCustomPragmaVal
+  assert bb.getCustomPragmaVal(hehe) == (key: "hi", val: "hu", haha: "he")

From 8d2f6bba3a491bc93677da9590397346e63c2bb2 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 27 Jul 2021 10:50:59 -0700
Subject: [PATCH 0287/1654] support same-line doc comments in routines (#18595)

* support same-line comments in routines
* remove assert as per review comment
---
 compiler/parser.nim                           |   7 ++
 .../expected/subdir/subdir_b/utils.html       | 110 +++++++++++++++++-
 .../expected/subdir/subdir_b/utils.idx        |   9 ++
 nimdoc/testproject/expected/theindex.html     |  36 ++++++
 nimdoc/testproject/subdir/subdir_b/utils.nim  |  17 +++
 tests/stdlib/trepr.nim                        |  27 +++++
 6 files changed, 205 insertions(+), 1 deletion(-)

diff --git a/compiler/parser.nim b/compiler/parser.nim
index d00a17e9f..b089614b2 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -1795,6 +1795,13 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode =
   else:
     result.add(p.emptyNode)
   indAndComment(p, result, maybeMissEquals)
+  let body = result[^1]
+  if body.kind == nkStmtList and body.len > 0 and body[0].comment.len > 0 and body[0].kind != nkCommentStmt:
+    if result.comment.len == 0:
+      # proc fn*(a: int): int = a ## foo
+      # => moves comment `foo` to `fn`
+      swap(result.comment, body[0].comment)
+    else: discard # xxx either `assert false` or issue a warning (otherwise we'll never know of this edge case)
 
 proc newCommentStmt(p: var Parser): PNode =
   #| commentStmt = COMMENT
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 939495ad1..533cb5b9d 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -115,7 +115,52 @@ window.addEventListener('DOMContentLoaded', main);
 
                      • 
   Procs
   
                          
-      
                            someType
+      
+  
+  
+  
+  
+  
+  
+  
+  
+  
                              someType
       
                            • someType(): SomeType
                              • 
 
@@ -181,6 +226,69 @@ window.addEventListener('DOMContentLoaded', main);
 
                              
 
                              Procs
                              
 
                              
+
+
                              proc fn2() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn3(): auto {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn4(): auto {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn5() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn6() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn7() {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              proc fn8(): auto {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              func fn9(a: int): int {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
+
+
                              func fn10(a: int): int {....raises: [], tags: [].}
                              
+
                              
+
+comment
+
+
                              
 
 
                              proc someType(): SomeType {....raises: [], tags: [].}
                              
 
                              
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
index 6dc3953af..a87393f09 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx
@@ -3,6 +3,15 @@ enumValueB	subdir/subdir_b/utils.html#enumValueB	SomeType.enumValueB
 enumValueC	subdir/subdir_b/utils.html#enumValueC	SomeType.enumValueC	
 SomeType	subdir/subdir_b/utils.html#SomeType	utils: SomeType	
 someType	subdir/subdir_b/utils.html#someType_2	utils: someType(): SomeType	
+fn2	subdir/subdir_b/utils.html#fn2	utils: fn2()	
+fn3	subdir/subdir_b/utils.html#fn3	utils: fn3(): auto	
+fn4	subdir/subdir_b/utils.html#fn4	utils: fn4(): auto	
+fn5	subdir/subdir_b/utils.html#fn5	utils: fn5()	
+fn6	subdir/subdir_b/utils.html#fn6	utils: fn6()	
+fn7	subdir/subdir_b/utils.html#fn7	utils: fn7()	
+fn8	subdir/subdir_b/utils.html#fn8	utils: fn8(): auto	
+fn9	subdir/subdir_b/utils.html#fn9,int	utils: fn9(a: int): int	
+fn10	subdir/subdir_b/utils.html#fn10,int	utils: fn10(a: int): int	
 aEnum	subdir/subdir_b/utils.html#aEnum.t	utils: aEnum(): untyped	
 bEnum	subdir/subdir_b/utils.html#bEnum.t	utils: bEnum(): untyped	
 fromUtilsGen	subdir/subdir_b/utils.html#fromUtilsGen.t	utils: fromUtilsGen(): untyped	
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index cfaf6856d..f72f6c37e 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -171,6 +171,42 @@ window.addEventListener('DOMContentLoaded', main);
 
                            • SomeType.enumValueC
                              • 
           
                            
+
                            fn10:
                            
+
                            fn2:
                            
+
                            fn3:
                            
+
                            fn4:
                            
+
                            fn5:
                            
+
                            fn6:
                            
+
                            fn7:
                            
+
                            fn8:
                            
+
                            fn9:
                            
 
                            Foo:
                              
 
                            • testproject: Foo
                              • 
diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim
index 4e3aa1f10..37c91dd3c 100644
--- a/nimdoc/testproject/subdir/subdir_b/utils.nim
+++ b/nimdoc/testproject/subdir/subdir_b/utils.nim
@@ -31,6 +31,23 @@ proc someType*(): SomeType =
   ## constructor.
   SomeType(2)
 
+
+proc fn2*() = discard ## comment
+proc fn3*(): auto = 1 ## comment
+proc fn4*(): auto = 2 * 3 + 4 ## comment
+proc fn5*() ## comment
+proc fn5*() = discard
+proc fn6*() =
+  ## comment
+proc fn7*() =
+  ## comment
+  discard
+proc fn8*(): auto =
+  ## comment
+  1+1
+func fn9*(a: int): int = 42  ## comment
+func fn10*(a: int): int = a  ## comment
+
 # bug #9235
 
 template aEnum*(): untyped =
diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim
index 356764903..83ae7b119 100644
--- a/tests/stdlib/trepr.nim
+++ b/tests/stdlib/trepr.nim
@@ -9,6 +9,7 @@ from strutils import endsWith, contains, strip
 from std/macros import newLit
 
 macro deb(a): string = newLit a.repr.strip
+macro debTyped(a: typed): string = newLit a.repr.strip
 
 template main() =
   doAssert repr({3,5}) == "{3, 5}"
@@ -243,5 +244,31 @@ template bar(): untyped =
     echo "baz"
     4)"""
 
+  block: # one liner doc comments
+    let a1 = deb:
+      func fn1(): int = 1  ## comment
+      func fn2(): int = 1
+        ## comment
+    let a2 = debTyped:
+      func fn1(): int = 1  ## comment
+      func fn2(): int = 1
+        ## comment
+    doAssert a1 == """
+func fn1(): int =
+  ## comment
+  1
+
+func fn2(): int =
+  ## comment
+  1"""
+    doAssert a2 == """
+func fn1(): int =
+  ## comment
+  result = 1
+
+func fn2(): int =
+  ## comment
+  result = 1"""
+
 static: main()
 main()

From 4c1202972abdfe99232e5d15a6169c7b2e0f5d75 Mon Sep 17 00:00:00 2001
From: Timothee Cour 
Date: Tue, 27 Jul 2021 21:38:19 -0700
Subject: [PATCH 0288/1654] fix #17072: add times.dateTime with ISO 8601 order
 (#18590)

* fix #17072: add times.dateTime with ISO 8601 order

* address comments
---
 changelog.md       |  6 +++++-
 lib/pure/times.nim | 29 ++++++++++++++++++-----------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/changelog.md b/changelog.md
index 89e013cff..be94f92c4 100644
--- a/changelog.md
+++ b/changelog.md
@@ -331,9 +331,12 @@
 
 - Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
 
-- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
+- `std/times`:
+  Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
   zone offsets without colons, e.g. `UTC+7 -> +0700`.
 
+  Added `dateTime` and deprecated `initDateTime`.
+
 - Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
 
 - Added dollar `$` and `len` for `jsre.RegExp`.
@@ -357,6 +360,7 @@
 
 - Added `dom.setInterval`, `dom.clearInterval` overloads.
 
+
 - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect
   if the slice is out of bounds, likewise with `strutils.delete`.
 
diff --git a/lib/pure/times.nim b/lib/pure/times.nim
index fcdd4ec8d..b2a22250d 100644
--- a/lib/pure/times.nim
+++ b/lib/pure/times.nim
@@ -1333,14 +1333,13 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} =
   ##    `cpuTime` instead, depending on the use case.
   getTime().local
 
-proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
-                   hour: HourRange, minute: MinuteRange, second: SecondRange,
-                   nanosecond: NanosecondRange,
-                   zone: Timezone = local()): DateTime =
+proc dateTime*(year: int, month: Month, monthday: MonthdayRange, 
+               hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0,
+               nanosecond: NanosecondRange = 0,
+               zone: Timezone = local()): DateTime =
   ## Create a new `DateTime <#DateTime>`_ in the specified timezone.
   runnableExamples:
-    let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc())
-    doAssert $dt1 == "2017-03-30T00:00:00Z"
+    assert $dateTime(2017, mMar, 30, zone = utc()) == "2017-03-30T00:00:00Z"
 
   assertValidDate monthday, month, year
   let dt = DateTime(
@@ -1356,12 +1355,20 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
 
 proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
                    hour: HourRange, minute: MinuteRange, second: SecondRange,
-                   zone: Timezone = local()): DateTime =
+                   nanosecond: NanosecondRange,
+                   zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} =
   ## Create a new `DateTime <#DateTime>`_ in the specified timezone.
-  runnableExamples:
-    let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc())
-    doAssert $dt1 == "2017-03-30T00:00:00Z"
-  initDateTime(monthday, month, year, hour, minute, second, 0, zone)
+  runnableExamples("--warning:deprecated:off"):
+    assert $initDateTime(30, mMar, 2017, 00, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z"
+  dateTime(year, month, monthday, hour, minute, second, nanosecond, zone)
+
+proc initDateTime*(monthday: MonthdayRange, month: Month, year: int,
+                   hour: HourRange, minute: MinuteRange, second: SecondRange,
+                   zone: Timezone = local()): DateTime {.deprecated: "use `dateTime`".} =
+  ## Create a new `DateTime <#DateTime>`_ in the specified timezone.
+  runnableExamples("--warning:deprecated:off"):
+    assert $initDateTime(30, mMar, 2017, 00, 00, 00, utc()) == "2017-03-30T00:00:00Z"
+  dateTime(year, month, monthday, hour, minute, second, 0, zone)
 
 proc `+`*(dt: DateTime, dur: Duration): DateTime =
   runnableExamples:

From a273ea70e8817e3509014a1b3dcd16a360ed400b Mon Sep 17 00:00:00 2001
From: Andreas Rumpf 
Date: Wed, 28 Jul 2021 12:46:28 +0200
Subject: [PATCH 0289/1654] implements overloadable enum values; WIP (#18470)

* implements overloadable enum values
* simpler code
---
 compiler/ast.nim                   |  2 +-
 compiler/lookups.nim               |  7 +++--
 compiler/options.nim               |  3 +-
 compiler/sem.nim                   |  8 +++++
 compiler/semexprs.nim              | 36 +++++++++++++++++++++-
 compiler/semstmts.nim              | 16 +++++++++-
 compiler/semtypes.nim              | 12 ++++++--
 compiler/sigmatch.nim              |  2 +-
 tests/enum/toverloadable_enums.nim | 48 ++++++++++++++++++++++++++++++
 tests/errmsgs/t10251.nim           |  2 +-
 10 files changed, 125 insertions(+), 11 deletions(-)
 create mode 100644 tests/enum/toverloadable_enums.nim

diff --git a/compiler/ast.nim b/compiler/ast.nim
index 0eeaa8133..5b54dd8ef 100644
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -993,7 +993,7 @@ type
 
 const
   OverloadableSyms* = {skProc, skFunc, skMethod, skIterator,
-    skConverter, skModule, skTemplate, skMacro}
+    skConverter, skModule, skTemplate, skMacro, skEnumField}
 
   GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody,
     tyGenericParam}
diff --git a/compiler/lookups.nim b/compiler/lookups.nim
index 44a30eefa..331eef525 100644
--- a/compiler/lookups.nim
+++ b/compiler/lookups.nim
@@ -177,13 +177,16 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
 
 proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
   var marked = initIntSet()
+  var symSet = OverloadableSyms
+  if overloadableEnums notin c.features:
+    symSet.excl skEnumField
   result = nil
   for im in c.imports.mitems:
     for s in symbols(im, marked, name, c.graph):
       if result == nil:
         result = s
       else:
-        if s.kind notin OverloadableSyms or result.kind notin OverloadableSyms:
+        if s.kind notin symSet or result.kind notin symSet:
           ambiguous = true
 
 proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
@@ -384,7 +387,7 @@ proc mergeShadowScope*(c: PContext) =
   ##
   ## Merges:
   ## shadow -> shadow: add symbols to the parent but check for redefinitions etc
-  ## shadow -> non-shadow: the above, but also handle exports and all that 
+  ## shadow -> non-shadow: the above, but also handle exports and all that
   let shadowScope = c.currentScope
   c.rawCloseScope
   for sym in shadowScope.symbols:
diff --git a/compiler/options.nim b/compiler/options.nim
index f02f02256..e37dea1b6 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -203,7 +203,8 @@ type
     vmopsDanger,
     strictFuncs,
     views,
-    strictNotNil
+    strictNotNil,
+    overloadableEnums
 
   LegacyFeature* = enum
     allowSemcheckedAstModification,
diff --git a/compiler/sem.nim b/compiler/sem.nim
index 879f6efc9..804325e56 100644
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -90,6 +90,12 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
     # error correction:
     result = copyTree(arg)
     result.typ = formal
+  elif arg.kind in nkSymChoices and formal.skipTypes(abstractInst).kind == tyEnum:
+    # Pick the right 'sym' from the sym choice by looking at 'formal' type:
+    for ch in arg:
+      if sameType(ch.typ, formal):
+        return getConstExpr(c.module, ch, c.idgen, c.graph)
+    typeMismatch(c.config, info, formal, arg.typ, arg)
   else:
     result = indexTypesMatch(c, formal, arg.typ, arg)
     if result == nil:
@@ -356,6 +362,8 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
   if e == nil:
     localError(c.config, n.info, errConstExprExpected)
     return n
+  if e.kind in nkSymChoices and e[0].typ.skipTypes(abstractInst).kind == tyEnum:
+    return e
   result = getConstExpr(c.module, e, c.idgen, c.graph)
   if result == nil:
     #if e.kind == nkEmpty: globalError(n.info, errConstExprExpected)
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 42c6b465e..f2fe54fad 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -2707,6 +2707,34 @@ proc getNilType(c: PContext): PType =
     result.align = c.config.target.ptrSize.int16
     c.nilTypeCache = result
 
+proc enumFieldSymChoice(c: PContext, n: PNode, s: PSym): PNode =
+  var o: TOverloadIter
+  var i = 0
+  var a = initOverloadIter(o, c, n)
+  while a != nil:
+    if a.kind in OverloadableSyms-{skModule}:
+      inc(i)
+      if i > 1: break
+    a = nextOverloadIter(o, c, n)
+  let info = getCallLineInfo(n)
+  if i <= 1:
+    if sfGenSym notin s.flags:
+      result = newSymNode(s, info)
+      markUsed(c, info, s)
+      onUse(info, s)
+    else:
+      result = n
+  else:
+    result = newNodeIT(nkClosedSymChoice, info, newTypeS(tyNone, c))
+    a = initOverloadIter(o, c, n)
+    while a != nil:
+      if a.kind in OverloadableSyms-{skModule}:
+        incl(a.flags, sfUsed)
+        markOwnerModuleAsUsed(c, a)
+        result.add newSymNode(a, info)
+        onUse(info, a)
+      a = nextOverloadIter(o, c, n)
+
 proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   when defined(nimCompilerStacktraceHints):
     setFrameMsg c.config$n.info & " " & $n.kind
@@ -2730,7 +2758,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
         {checkUndeclared, checkModule, checkAmbiguity, checkPureEnumFields}
     var s = qualifiedLookUp(c, n, checks)
     if c.matchedConcept == nil: semCaptureSym(s, c.p.owner)
-    if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator}:
+    case s.kind
+    of skProc, skFunc, skMethod, skConverter, skIterator:
       #performProcvarCheck(c, n, s)
       result = symChoice(c, n, s, scClosed)
       if result.kind == nkSym:
@@ -2740,6 +2769,11 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       # "procs literals" are 'owned'
       if optOwnedRefs in c.config.globalOptions:
         result.typ = makeVarType(c, result.typ, tyOwned)
+    of skEnumField:
+      if overloadableEnums in c.features:
+        result = enumFieldSymChoice(c, n, s)
+      else:
+        result = semSym(c, n, s, flags)
     else:
       result = semSym(c, n, s, flags)
   of nkSym:
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index 75323a95e..cc09291c5 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -490,6 +490,18 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode =
     ret.add result
     result = semExprNoType(c, ret)
 
+proc errorSymChoiceUseQualifier(c: PContext; n: PNode) =
+  assert n.kind in nkSymChoices
+  var err = "ambiguous identifier: '" & $n[0] & "'"
+  var i = 0
+  for child in n:
+    let candidate = child.sym
+    if i == 0: err.add " -- use one of the following:\n"
+    else: err.add "\n"
+    err.add "  " & candidate.owner.name.s & "." & candidate.name.s
+    inc i
+  localError(c.config, n.info, errGenerated, err)
+
 proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
   if n.len == 1:
     result = semLowerLetVarCustomPragma(c, n[0], n)
@@ -514,7 +526,9 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
     if a[^1].kind != nkEmpty:
       def = semExprWithType(c, a[^1], {})
 
-      if def.kind == nkSym and def.sym.kind in {skTemplate, skMacro}:
+      if def.kind in nkSymChoices and def[0].typ.skipTypes(abstractInst).kind == tyEnum:
+        errorSymChoiceUseQualifier(c, def)
+      elif def.kind == nkSym and def.sym.kind in {skTemplate, skMacro}:
         typFlags.incl taIsTemplateOrMacro
       elif def.typ.kind == tyTypeDesc and c.p.owner.kind != skMacro:
         typFlags.incl taProcContextIsNotMacro
diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim
index 484997e3a..db895cab6 100644
--- a/compiler/semtypes.nim
+++ b/compiler/semtypes.nim
@@ -144,8 +144,13 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
     styleCheckDef(c.config, e)
     onDef(e.info, e)
     if sfGenSym notin e.flags:
-      if not isPure: addInterfaceDecl(c, e)
-      else: declarePureEnumField(c, e)
+      if not isPure:
+        if overloadableEnums in c.features:
+          addInterfaceOverloadableSymAt(c, c.currentScope, e)
+        else:
+          addInterfaceDecl(c, e)
+      else:
+        declarePureEnumField(c, e)
     if isPure and (let conflict = strTableInclReportConflict(symbols, e); conflict != nil):
       wrongRedefinition(c, e.info, e.name.s, conflict.info)
     inc(counter)
@@ -240,7 +245,8 @@ proc semRangeAux(c: PContext, n: PNode, prev: PType): PType =
 
   if not hasUnknownTypes:
     if not sameType(rangeT[0].skipTypes({tyRange}), rangeT[1].skipTypes({tyRange})):
-      localError(c.config, n.info, "type mismatch")
+      typeMismatch(c.config, n.info, rangeT[0], rangeT[1], n)
+
     elif not isOrdinalType(rangeT[0]) and rangeT[0].kind notin {tyFloat..tyFloat128} or
         rangeT[0].kind == tyBool:
       localError(c.config, n.info, "ordinal or float type expected")
diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim
index 6bc641bd6..674dd7e2e 100644
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -2197,7 +2197,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType,
     var best = -1
     for i in 0.. nkSym
+
+proc x(p: int) = discard
+proc x(p: string) = discard
+
+proc takeCallback(param: proc(p: int)) = discard
+
+takeCallback x
+
+echo ord v
diff --git a/tests/errmsgs/t10251.nim b/tests/errmsgs/t10251.nim
index 0c7fe0b3d..19adf02eb 100644
--- a/tests/errmsgs/t10251.nim
+++ b/tests/errmsgs/t10251.nim
@@ -2,7 +2,6 @@ discard """
   action:reject
   cmd: "nim check $options $file"
   nimout: '''
-t10251.nim(15, 5) Error: redefinition of 'foo'; previous declaration here: t10251.nim(13, 5)
 t10251.nim(19, 23) Error: redefinition of 'goo1'; previous declaration here: t10251.nim(19, 11)
 '''
 """
@@ -14,6 +13,7 @@ type
   Enum2 = enum
     foo, bar, baz
 
+
 type
   Enum3 {.pure.} = enum # fixed (by accident?) in https://github.com/nim-lang/Nim/pull/18263
     goo0, goo1, goo2, goo1

From e616675c419552a85bdf0b47f47586852be8b5be Mon Sep 17 00:00:00 2001
From: Miran 
Date: Wed, 28 Jul 2021 14:31:13 +0200
Subject: [PATCH 0290/1654] various small documentation improvements (#18602)

---
 lib/pure/asyncdispatch.nim          | 34 ++++++++++-----------
 lib/pure/collections/deques.nim     | 13 ++++----
 lib/pure/collections/intsets.nim    |  2 +-
 lib/pure/collections/sharedlist.nim |  2 +-
 lib/pure/collections/tables.nim     | 20 ++++++++-----
 lib/pure/net.nim                    |  8 ++---
 lib/pure/strformat.nim              | 46 ++++++++++++++---------------
 lib/pure/strscans.nim               |  2 +-
 lib/std/enumutils.nim               | 31 +++++++++++++++----
 lib/std/packedsets.nim              |  8 ++---
 lib/system/threads.nim              |  2 +-
 lib/system_overview.rst             | 42 +++++++++++++-------------
 tests/stdlib/trst.nim               | 10 +++----
 13 files changed, 123 insertions(+), 97 deletions(-)

diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim
index d98ca77df..955e66f6b 100644
--- a/lib/pure/asyncdispatch.nim
+++ b/lib/pure/asyncdispatch.nim
@@ -42,12 +42,12 @@
 ##
 ## Code to read some data from a socket may look something like this:
 ##
-##   .. code-block:: Nim
-##      var future = socket.recv(100)
-##      future.addCallback(
-##        proc () =
-##          echo(future.read)
-##      )
+## .. code-block:: Nim
+##    var future = socket.recv(100)
+##    future.addCallback(
+##      proc () =
+##        echo(future.read)
+##    )
 ##
 ## All asynchronous functions returning a `Future` will not block. They
 ## will not however return immediately. An asynchronous function will have
@@ -104,20 +104,20 @@
 ## The most reliable way to handle exceptions is to use `yield` on a future
 ## then check the future's `failed` property. For example:
 ##
-##   .. code-block:: Nim
-##     var future = sock.recv(100)
-##     yield future
-##     if future.failed:
-##       # Handle exception
+## .. code-block:: Nim
+##   var future = sock.recv(100)
+##   yield future
+##   if future.failed:
+##     # Handle exception
 ##
 ## The `async` procedures also offer limited support for the try statement.
 ##
-##    .. code-block:: Nim
-##      try:
-##        let data = await sock.recv(100)
-##        echo("Received ", data)
-##      except:
-##        # Handle exception
+## .. code-block:: Nim
+##   try:
+##     let data = await sock.recv(100)
+##     echo("Received ", data)
+##   except:
+##     # Handle exception
 ##
 ## Unfortunately the semantics of the try statement may not always be correct,
 ## and occasionally the compilation may fail altogether.
diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim
index fe9ceca1e..ecdf199d2 100644
--- a/lib/pure/collections/deques.nim
+++ b/lib/pure/collections/deques.nim
@@ -10,8 +10,9 @@
 ## An implementation of a `deque`:idx: (double-ended queue).
 ## The underlying implementation uses a `seq`.
 ##
-## Note that none of the procs that get an individual value from the deque should be used
-## on an empty deque.
+## .. note:: None of the procs that get an individual value from the deque should be used
+##   on an empty deque.
+##
 ## If compiled with the `boundChecks` option, those procs will raise an `IndexDefect`
 ## on such access. This should not be relied upon, as `-d:danger` or `--checks:off` will
 ## disable those checks and then the procs may return garbage or crash the program.
@@ -198,7 +199,7 @@ iterator items*[T](deq: Deque[T]): lent T =
   ## Yields every element of `deq`.
   ##
   ## **See also:**
-  ## * `mitems iterator <#mitems,Deque[T]>`_
+  ## * `mitems iterator <#mitems.i,Deque[T]>`_
   runnableExamples:
     from std/sequtils import toSeq
 
@@ -214,7 +215,7 @@ iterator mitems*[T](deq: var Deque[T]): var T =
   ## Yields every element of `deq`, which can be modified.
   ##
   ## **See also:**
-  ## * `items iterator <#items,Deque[T]>`_
+  ## * `items iterator <#items.i,Deque[T]>`_
   runnableExamples:
     var a = [10, 20, 30, 40, 50].toDeque
     assert $a == "[10, 20, 30, 40, 50]"
@@ -274,7 +275,7 @@ proc addFirst*[T](deq: var Deque[T], item: sink T) =
   ## Adds an `item` to the beginning of `deq`.
   ##
   ## **See also:**
-  ## * `addLast proc <#addLast,Deque[T],T>`_
+  ## * `addLast proc <#addLast,Deque[T],sinkT>`_
   runnableExamples:
     var a = initDeque[int]()
     for i in 1 .. 5:
@@ -290,7 +291,7 @@ proc addLast*[T](deq: var Deque[T], item: sink T) =
   ## Adds an `item` to the end of `deq`.
   ##
   ## **See also:**
-  ## * `addFirst proc <#addFirst,Deque[T],T>`_
+  ## * `addFirst proc <#addFirst,Deque[T],sinkT>`_
   runnableExamples:
     var a = initDeque[int]()
     for i in 1 .. 5:
diff --git a/lib/pure/collections/intsets.nim b/lib/pure/collections/intsets.nim
index 05e5addcc..765a23e97 100644
--- a/lib/pure/collections/intsets.nim
+++ b/lib/pure/collections/intsets.nim
@@ -8,7 +8,7 @@
 #
 
 ## Specialization of the generic `packedsets module `_
-## for ordinal sparse sets.
+## (see its documentation for more examples) for ordinal sparse sets.
 
 import std/private/since
 import std/packedsets
diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim
index 79f2391f6..73e147e05 100644
--- a/lib/pure/collections/sharedlist.nim
+++ b/lib/pure/collections/sharedlist.nim
@@ -35,7 +35,7 @@ template withLock(t, x: untyped) =
   release(t.lock)
 
 proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) =
-  ## Iterates over the list. If 'action' returns true, the
+  ## Iterates over the list. If `action` returns true, the
   ## current item is removed from the list.
   ##
   ## .. warning:: It may not preserve the element order after some modifications.
diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim
index 403345755..2695f3693 100644
--- a/lib/pure/collections/tables.nim
+++ b/lib/pure/collections/tables.nim
@@ -136,6 +136,8 @@ runnableExamples:
 ## a more complex object as a key you will be greeted by a strange compiler
 ## error:
 ##
+## .. code::
+##
 ##   Error: type mismatch: got (Person)
 ##   but expected one of:
 ##   hashes.hash(x: openArray[A]): Hash
@@ -324,7 +326,7 @@ proc `[]`*[A, B](t: Table[A, B], key: A): B =
   ##   a default value (e.g. zero for int) if the key doesn't exist
   ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new
+  ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in
   ##   the table
@@ -345,7 +347,7 @@ proc `[]`*[A, B](t: var Table[A, B], key: A): var B =
   ##   a default value (e.g. zero for int) if the key doesn't exist
   ## * `getOrDefault proc<#getOrDefault,Table[A,B],A,B>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new
+  ## * `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,Table[A,B],A>`_ for checking if a key is in
   ##   the table
@@ -488,7 +490,7 @@ proc add*[A, B](t: var Table[A, B], key: A, val: sink B) {.deprecated:
   ##
   ## **This can introduce duplicate keys into the table!**
   ##
-  ## Use `[]= proc<#[]=,Table[A,B],A,B>`_ for inserting a new
+  ## Use `[]= proc<#[]=,Table[A,B],A,sinkB>`_ for inserting a new
   ## (key, value) pair in the table without introducing duplicates.
   addImpl(enlarge)
 
@@ -499,7 +501,8 @@ template tabCellHash(i)  = t.data[i].hcode
 proc del*[A, B](t: var Table[A, B], key: A) =
   ## Deletes `key` from hash table `t`. Does nothing if the key does not exist.
   ##
-  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `pop proc<#pop,Table[A,B],A,B>`_
@@ -519,7 +522,8 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   ## mapping of the key. Otherwise, returns `false`, and the `val` is
   ## unchanged.
   ##
-  ## .. warning:: If duplicate keys were added, this may need to be called multiple times.
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `del proc<#del,Table[A,B],A>`_
@@ -1028,7 +1032,8 @@ proc add*[A, B](t: TableRef[A, B], key: A, val: sink B) {.deprecated:
 proc del*[A, B](t: TableRef[A, B], key: A) =
   ## Deletes `key` from hash table `t`. Does nothing if the key does not exist.
   ##
-  ## **If duplicate keys were added, this may need to be called multiple times.**
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `pop proc<#pop,TableRef[A,B],A,B>`_
@@ -1048,7 +1053,8 @@ proc pop*[A, B](t: TableRef[A, B], key: A, val: var B): bool =
   ## mapping of the key. Otherwise, returns `false`, and the `val` is
   ## unchanged.
   ##
-  ## **If duplicate keys were added, this may need to be called multiple times.**
+  ## .. warning:: If duplicate keys were added (via the now deprecated `add` proc),
+  ##   this may need to be called multiple times.
   ##
   ## See also:
   ## * `del proc<#del,TableRef[A,B],A>`_
diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index c2e4810de..c0f9e1465 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -585,10 +585,10 @@ when defineSsl:
     ##
     ## CA certificates will be loaded, in the following order, from:
     ##
-    ##  - caFile, caDir, parameters, if set
-    ##  - if `verifyMode` is set to `CVerifyPeerUseEnvVars`,
-    ##    the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used
-    ##  - a set of files and directories from the `ssl_certs `_ file.
+    ## - caFile, caDir, parameters, if set
+    ## - if `verifyMode` is set to `CVerifyPeerUseEnvVars`,
+    ##   the SSL_CERT_FILE and SSL_CERT_DIR environment variables are used
+    ## - a set of files and directories from the `ssl_certs `_ file.
     ##
     ## The last two parameters specify the certificate file path and the key file
     ## path, a server socket will most likely not work without these.
diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim
index 872141383..746f01ace 100644
--- a/lib/pure/strformat.nim
+++ b/lib/pure/strformat.nim
@@ -142,12 +142,12 @@ An expression like `&"{key} is {value:arg} {{z}}"` is transformed into:
   temp
 
 Parts of the string that are enclosed in the curly braces are interpreted
-as Nim code, to escape a `{` or `}`, double it.
+as Nim code. To escape a `{` or `}`, double it.
 
-Within a curly expression,however, '{','}', must be escaped with a backslash.
+Within a curly expression, however, `{`, `}`, must be escaped with a backslash.
 
-To enable evaluating Nim expressions within curlies, inside parentheses
-colons do not need to be escaped.
+To enable evaluating Nim expressions within curlies, colons inside parentheses
+do not need to be escaped.
 ]##
 
 runnableExamples:
@@ -177,28 +177,28 @@ The general form of a standard format specifier is::
 
 The square brackets `[]` indicate an optional element.
 
-The optional 'align' flag can be one of the following:
+The optional `align` flag can be one of the following:
 
-'<'
+`<`
     Forces the field to be left-aligned within the available
     space. (This is the default for strings.)
 
-'>'
+`>`
     Forces the field to be right-aligned within the available space.
     (This is the default for numbers.)
 
-'^'
+`^`
     Forces the field to be centered within the available space.
 
 Note that unless a minimum field width is defined, the field width
 will always be the same size as the data to fill it, so that the alignment
 option has no meaning in this case.
 
-The optional 'fill' character defines the character to be used to pad
+The optional `fill` character defines the character to be used to pad
 the field to the minimum width. The fill character, if present, must be
 followed by an alignment flag.
 
-The 'sign' option is only valid for numeric types, and can be one of the following:
+The `sign` option is only valid for numeric types, and can be one of the following:
 
 =================        ====================================================
   Sign                   Meaning
@@ -211,22 +211,22 @@ The 'sign' option is only valid for numeric types, and can be one of the followi
                          positive numbers.
 =================        ====================================================
 
-If the '#' character is present, integers use the 'alternate form' for formatting.
+If the `#` character is present, integers use the 'alternate form' for formatting.
 This means that binary, octal and hexadecimal output will be prefixed
-with '0b', '0o' and '0x', respectively.
+with `0b`, `0o` and `0x`, respectively.
 
-'width' is a decimal integer defining the minimum field width. If not specified,
+`width` is a decimal integer defining the minimum field width. If not specified,
 then the field width will be determined by the content.
 
-If the width field is preceded by a zero ('0') character, this enables
+If the width field is preceded by a zero (`0`) character, this enables
 zero-padding.
 
-The 'precision' is a decimal number indicating how many digits should be displayed
+The `precision` is a decimal number indicating how many digits should be displayed
 after the decimal point in a floating point conversion. For non-numeric types the
 field indicates the maximum field size - in other words, how many characters will
 be used from the field content. The precision is ignored for integer conversions.
 
-Finally, the 'type' determines how the data should be presented.
+Finally, the `type` determines how the data should be presented.
 
 The available integer presentation types are:
 
@@ -240,7 +240,7 @@ The available integer presentation types are:
                          lower-case letters for the digits above 9.
 `X`                      Hex format. Outputs the number in base 16, using
                          uppercase letters for the digits above 9.
-(None)                   The same as 'd'.
+(None)                   The same as `d`.
 =================        ====================================================
 
 The available floating point presentation types are:
@@ -249,21 +249,21 @@ The available floating point presentation types are:
   Type                   Result
 =================        ====================================================
 `e`                      Exponent notation. Prints the number in scientific
-                         notation using the letter 'e' to indicate the
+                         notation using the letter `e` to indicate the
                          exponent.
-`E`                      Exponent notation. Same as 'e' except it converts
+`E`                      Exponent notation. Same as `e` except it converts
                          the number to uppercase.
 `f`                      Fixed point. Displays the number as a fixed-point
                          number.
-`F`                      Fixed point. Same as 'f' except it converts the
+`F`                      Fixed point. Same as `f` except it converts the
                          number to uppercase.
 `g`                      General format. This prints the number as a
                          fixed-point number, unless the number is too
-                         large, in which case it switches to 'e'
+                         large, in which case it switches to `e`
                          exponent notation.
-`G`                      General format. Same as 'g' except it switches to 'E'
+`G`                      General format. Same as `g` except it switches to `E`
                          if the number gets to large.
-(None)                   Similar to 'g', except that it prints at least one
+(None)                   Similar to `g`, except that it prints at least one
                          digit after the decimal point.
 =================        ====================================================
 
diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim
index 73b53e3d6..291af7408 100644
--- a/lib/pure/strscans.nim
+++ b/lib/pure/strscans.nim
@@ -52,7 +52,7 @@ substrings starting with ``$``. These constructions are available:
 =================   ========================================================
 
 Even though ``$*`` and ``$+`` look similar to the regular expressions ``.*``
-and ``.+`` they work quite differently, there is no non-deterministic
+and ``.+``, they work quite differently. There is no non-deterministic
 state machine involved and the matches are non-greedy. ``[$*]``
 matches ``[xyz]`` via ``parseutils.parseUntil``.
 
diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim
index 09cf24f51..6269950c7 100644
--- a/lib/std/enumutils.nim
+++ b/lib/std/enumutils.nim
@@ -79,8 +79,14 @@ macro enumNames(a: typed): untyped =
 iterator items*[T: HoleyEnum](E: typedesc[T]): T =
   ## Iterates over an enum with holes.
   runnableExamples:
-    type A = enum a0 = 2, a1 = 4, a2
-    type B[T] = enum b0 = 2, b1 = 4
+    type
+      A = enum
+        a0 = 2
+        a1 = 4
+        a2
+      B[T] = enum
+        b0 = 2
+        b1 = 4
     from std/sequtils import toSeq
     assert A.toSeq == [a0, a1, a2]
     assert B[float].toSeq == [B[float].b0, B[float].b1]
@@ -130,9 +136,19 @@ template symbolRank*[T: enum](a: T): int =
   ## for small enums, otherwise is `O(T.enumLen)`.
   runnableExamples:
     type
-      A = enum a0 = -3, a1 = 10, a2, a3 = (20, "f3Alt") # HoleyEnum
-      B = enum b0, b1, b2 # OrdinalEnum
-      C = enum c0 = 10, c1, c2 # OrdinalEnum
+      A = enum # HoleyEnum
+        a0 = -3
+        a1 = 10
+        a2
+        a3 = (20, "f3Alt")
+      B = enum # OrdinalEnum
+        b0
+        b1
+        b2
+      C = enum # OrdinalEnum
+        c0 = 10
+        c1
+        c2
     assert a2.symbolRank == 2
     assert b2.symbolRank == 2
     assert c2.symbolRank == 2
@@ -156,7 +172,10 @@ func symbolName*[T: enum](a: T): string =
     assert b.symbolName == "b0"
     assert $b == "kb0"
     static: assert B.high.symbolName == "b2"
-    type C = enum c0 = -3, c1 = 4, c2 = 20 # HoleyEnum
+    type C = enum # HoleyEnum
+      c0 = -3
+      c1 = 4
+      c2 = 20
     assert c1.symbolName == "c1"
   const names = enumNames(T)
   names[a.symbolRank]
diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim
index e209d2aae..b2ee917eb 100644
--- a/lib/std/packedsets.nim
+++ b/lib/std/packedsets.nim
@@ -12,10 +12,10 @@
 ##
 ## Supports any Ordinal type.
 ##
-## **Note**: Currently the assignment operator `=` for `PackedSet[A]`
-## performs some rather meaningless shallow copy. Since Nim currently does
-## not allow the assignment operator to be overloaded, use the `assign proc
-## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy.
+## .. note:: Currently the assignment operator `=` for `PackedSet[A]`
+##   performs some rather meaningless shallow copy. Since Nim currently does
+##   not allow the assignment operator to be overloaded, use the `assign proc
+##   <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy.
 ##
 ## See also
 ## ========
diff --git a/lib/system/threads.nim b/lib/system/threads.nim
index 2b5eb07fd..a9006b0c5 100644
--- a/lib/system/threads.nim
+++ b/lib/system/threads.nim
@@ -11,7 +11,7 @@
 ##
 ## **Note**: This is part of the system module. Do not import it directly.
 ## To activate thread support you need to compile
-## with the `--threads:on` command line switch.
+## with the `--threads:on`:option: command line switch.
 ##
 ## Nim's memory model for threads is quite different from other common
 ## programming languages (C, Pascal): Each thread has its own
diff --git a/lib/system_overview.rst b/lib/system_overview.rst
index d6cbe1a35..df22697e0 100644
--- a/lib/system_overview.rst
+++ b/lib/system_overview.rst
@@ -48,27 +48,27 @@ Proc                              Usage
 Seqs
 ----
 
-==============================================     ==========================================
-Proc                                               Usage
-==============================================     ==========================================
-`newSeq<#newSeq>`_                                 Create a new sequence of a given length
-`newSeqOfCap<#newSeqOfCap,Natural>`_               Create a new sequence with zero length
-                                                   and a given capacity
-`setLen<#setLen,seq[T],Natural>`_                  Set the length of a sequence
-`len<#len,seq[T]>`_                                Return the length of a sequence
-`@<#@,openArray[T]>`_                              Turn an array into a sequence
-`add<#add,seq[T],sinkT>`_                          Add an item to the sequence
-`insert<#insert,seq[T],sinkT>`_                    Insert an item at a specific position
-`delete<#delete,seq[T],Natural>`_                  Delete an item while preserving the
-                                                   order of elements (`O(n)` operation)
-`del<#del,seq[T],Natural>`_                        `O(1)` removal, doesn't preserve the order
-`pop<#pop,seq[T]>`_                                Remove and return last item of a sequence
-`x & y<#&,seq[T],seq[T]>`_                         Concatenate two sequences
-`x[a .. b]<#[],openArray[T],HSlice[U,V]>`_         Slice of a sequence (both ends included)
-`x[a .. ^b]<#[],openArray[T],HSlice[U,V]>`_        Slice of a sequence but `b` is a 
-                                                   reversed index (both ends included)
-`x[a ..\< b]<#[],openArray[T],HSlice[U,V]>`_       Slice of a sequence (excluded upper bound)
-==============================================     ==========================================
+=============================================================  ==========================================
+Proc                                                           Usage
+=============================================================  ==========================================
+`newSeq<#newSeq>`_                                             Create a new sequence of a given length
+`newSeqOfCap<#newSeqOfCap,Natural>`_                           Create a new sequence with zero length
+                                                               and a given capacity
+`setLen<#setLen,seq[T],Natural>`_                              Set the length of a sequence
+`len<#len,seq[T]>`_                                            Return the length of a sequence
+`@<#@,openArray[T]>`_                                          Turn an array into a sequence
+`add<#add,seq[T],sinkT>`_                                      Add an item to the sequence
+`insert<#insert,seq[T],sinkT>`_                                Insert an item at a specific position
+`delete<#delete,seq[T],Natural>`_                              Delete an item while preserving the
+                                                               order of elements (`O(n)` operation)
+`del<#del,seq[T],Natural>`_                                    `O(1)` removal, doesn't preserve the order
+`pop<#pop,seq[T]>`_                                            Remove and return last item of a sequence
+`x & y<#&,seq[T],seq[T]>`_                                     Concatenate two sequences
+`x[a .. b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_   Slice of a sequence (both ends included)
+`x[a .. ^b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_  Slice of a sequence but `b` is a 
+                                                               reversed index (both ends included)
+`x[a ..< b]<#[],openArray[T],HSlice[U: Ordinal,V: Ordinal]>`_  Slice of a sequence (excluded upper bound)
+=============================================================  ==========================================
 
 **See also:**
 * `sequtils module `_ for operations on container
diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim
index fc3ccbf4e..bc11f219c 100644
--- a/tests/stdlib/trst.nim
+++ b/tests/stdlib/trst.nim
@@ -182,9 +182,9 @@ suite "RST parsing":
   test "using .. as separator b/w directives and block quotes":
     check(dedent"""
         .. note:: someNote
-    
+
         ..
-    
+
           someBlockQuote""".toAst ==
       dedent"""
         rnInner
@@ -300,14 +300,14 @@ suite "RST indentation":
     let input1 = dedent"""
       .. code-block:: nim
           :test: "nim c $1"
-      
+
         template additive(typ: typedesc) =
           discard
       """
     let input2 = dedent"""
       .. code-block:: nim
         :test: "nim c $1"
-      
+
         template additive(typ: typedesc) =
           discard
       """
@@ -320,7 +320,7 @@ suite "RST indentation":
     let inputWrong = dedent"""
       .. code-block:: nim
        :test: "nim c $1"
-      
+
          template additive(typ: typedesc) =
            discard
       """

From 7d3c3e00efe777dcbda15f2125fa77aa7d17cad4 Mon Sep 17 00:00:00 2001
From: Vindaar 
Date: Thu, 29 Jul 2021 07:47:34 +0200
Subject: [PATCH 0291/1654] Allow `nnkAccQuoted` in `genEnumCaseStmt` (#18606)

* [enumutils] provide node kind for `Invalid node type` error

* [enumutils] add support for nnkAccQuoted in `genEnumCaseStmt`

For reasons unknown to me, when running `nim doc` on a file that uses
`parseEnum` with an enum that contains accented quotes errors at CT
with the `Invalid node for type` error. Further errors are raised,
probably because the enum parsing fails?
---
 lib/std/enumutils.nim | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim
index 6269950c7..81e602ad5 100644
--- a/lib/std/enumutils.nim
+++ b/lib/std/enumutils.nim
@@ -36,6 +36,10 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed,
     case f.kind
     of nnkEmpty: continue # skip first node of `enumTy`
     of nnkSym, nnkIdent: fStr = f.strVal
+    of nnkAccQuoted:
+      fStr = ""
+      for ch in f:
+        fStr.add ch.strVal
     of nnkEnumFieldDef:
       case f[1].kind
       of nnkStrLit: fStr = f[1].strVal
@@ -46,7 +50,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed,
         fStr = f[0].strVal
         fNum = f[1].intVal
       else: error("Invalid tuple syntax!", f[1])
-    else: error("Invalid node for enum type!", f)
+    else: error("Invalid node for enum type `" & $f.kind & "`!", f)
     # add field if string not already added
     if fNum >= userMin and fNum <= userMax:
       fStr = normalizer(fStr)

From 1bba641fed99bce61de2470c8bf73cd34480d907 Mon Sep 17 00:00:00 2001
From: konsumlamm <44230978+konsumlamm@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:50:17 +0200
Subject: [PATCH 0292/1654] Update `lib.rst` (#18605)

* Update lib.rst

* Remove "Unstable" category

Add `strbasics`

* Update doc/lib.rst

* Update doc/lib.rst

Co-authored-by: Andreas Rumpf 
---
 doc/lib.rst | 44 ++++++++++++++++++++------------------------
 1 file changed, 20 insertions(+), 24 deletions(-)

diff --git a/doc/lib.rst b/doc/lib.rst
index 988fbf6f0..54728d681 100644
--- a/doc/lib.rst
+++ b/doc/lib.rst
@@ -41,7 +41,7 @@ Automatic imports
   Basic Nim thread support. **Note:** This is part of the system module. Do not
   import it explicitly. Enabled with `--threads:on`:option:.
 
-* `channels `_
+* `channels_builtin `_
   Nim message passing support for threads. **Note:** This is part of the
   system module. Do not import it explicitly. Enabled with `--threads:on`:option:.
 
@@ -49,6 +49,9 @@ Automatic imports
 Core
 ----
 
+* `atomics `_
+  Types and operations for atomic operations and lockless algorithms.
+
 * `bitops `_
   Provides a series of low-level methods for bit manipulation.
 
@@ -66,7 +69,7 @@ Core
 
 * `macrocache `_
   Provides an API for macros to collect compile-time information across modules.
-  
+
 * `macros `_
   Contains the AST API and documentation of Nim for writing macros.
 
@@ -113,7 +116,7 @@ Collections
   The underlying implementation uses a `seq`.
 
 * `heapqueue `_
-  Implementation of a heap data structure that can be used as a priority queue.
+  Implementation of a binary heap data structure that can be used as a priority queue.
 
 * `intsets `_
   Efficient implementation of a set of ints as a sparse bit set.
@@ -129,7 +132,7 @@ Collections
   Efficient implementation of a set of ordinals as a sparse bit set.
 
 * `sets `_
-  Nim hash and bit set support.
+  Nim hash set support.
 
 * `sharedlist `_
   Nim shared linked list support. Contains a shared singly-linked list.
@@ -167,11 +170,14 @@ String handling
 * `ropes `_
   This module contains support for a *rope* data type.
   Ropes can represent very long strings efficiently;
-  especially concatenation is done in O(1) instead of O(n).
+  in particular, concatenation is done in O(1) instead of O(n).
+
+* `strbasics `_
+  This module provides some high performance string operations.
 
 * `strformat `_
   Macro based standard string interpolation/formatting. Inspired by
-  Python's `f`-strings.
+  Python's f-strings.
 
 * `strmisc `_
   This module contains uncommon string handling operations that do not
@@ -266,9 +272,6 @@ Math libraries
 * `math `_
   Mathematical operations like cosine, square root.
 
-* `mersenne `_
-  Mersenne twister random number generator.
-
 * `random `_
   Fast and tiny random number generator.
 
@@ -433,7 +436,7 @@ Hashing
 -------
 
 * `base64 `_
-  This module implements a base64 encoder and decoder.
+  This module implements a Base64 encoder and decoder.
 
 * `hashes `_
   This module implements efficient computations of hash values for diverse
@@ -446,7 +449,7 @@ Hashing
   An OID is a global ID that consists of a timestamp,
   a unique counter, and a random value. This combination should suffice to
   produce a globally distributed unique ID. This implementation was extracted
-  from the Mongodb interface and it thus binary compatible with a Mongo OID.
+  from the MongoDB interface and it thus binary compatible with a MongoDB OID.
 
 * `sha1 `_
   This module implements a sha1 encoder and decoder.
@@ -487,9 +490,8 @@ Miscellaneous
   This module implements the `with` macro for easy function chaining.
 
 
-
-Modules for JS backend
-----------------------
+Modules for the JS backend
+--------------------------
 
 * `asyncjs `_
   Types and macros for writing asynchronous procedures in JavaScript.
@@ -497,6 +499,9 @@ Modules for JS backend
 * `dom `_
   Declaration of the Document Object Model for the JS backend.
 
+* `jsbigints `_
+  Arbitrary precision integers.
+
 * `jsconsole `_
   Wrapper for the `console` object.
 
@@ -519,7 +524,6 @@ Regular expressions
   expressions. The current implementation uses PCRE.
 
 
-
 Database support
 ----------------
 
@@ -583,7 +587,7 @@ Database support
 * `mysql `_
   Contains a wrapper for the mySQL API.
 * `sqlite3 `_
-  Contains a wrapper for SQLite 3 API.
+  Contains a wrapper for the SQLite 3 API.
 * `odbcsql `_
   interface to the ODBC driver.
 
@@ -593,11 +597,3 @@ Network Programming and Internet Protocols
 
 * `openssl `_
   Wrapper for OpenSSL.
-
-
-
-Unstable
-========
-
-* `atomics `_
-  Types and operations for atomic operations and lockless algorithms.

From bbe05c15326c7a62d15c0fe47d1c6e1be3ab11a8 Mon Sep 17 00:00:00 2001
From: Andrey Makarov 
Date: Thu, 29 Jul 2021 14:30:19 +0300
Subject: [PATCH 0293/1654] docgen: draw frame around active anchors (#18607)

---
 config/nimdoc.cfg                             |   3 +-
 doc/nimdoc.css                                |   6 +
 .../expected/index.html                       |   3 +-
 nimdoc/testproject/expected/nimdoc.out.css    |   6 +
 .../expected/subdir/subdir_b/utils.html       |  42 +++--
 nimdoc/testproject/expected/testproject.html  | 177 ++++++++++++------
 6 files changed, 162 insertions(+), 75 deletions(-)

diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg
index 15f887023..162b2b4a3 100644
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -46,13 +46,14 @@ doc.section.toc2 = """
 # * $seeSrc: generated HTML from doc.item.seesrc (if some switches are used).
 
 doc.item = """
-
+
                              
 
                              $header
                              
 
                              
 $deprecationMsg
 $desc
 $seeSrc
 
                              
+
                              
 """
 
 # Chunk of HTML emitted for each entry in the HTML table of contents.
diff --git a/doc/nimdoc.css b/doc/nimdoc.css
index 5c2bf2093..4abea9ce0 100644
--- a/doc/nimdoc.css
+++ b/doc/nimdoc.css
@@ -234,6 +234,12 @@ select:focus {
 }
 
 /* Docgen styles */
+
+:target {
+  border: 2px solid #B5651D;
+  border-style: dotted;
+}
+
 /* Links */
 a {
   color: var(--anchor);
diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html
index fac758c9b..3499b5326 100644
--- a/nimdoc/test_out_index_dot_html/expected/index.html
+++ b/nimdoc/test_out_index_dot_html/expected/index.html
@@ -121,13 +121,14 @@ window.addEventListener('DOMContentLoaded', main);
   
                              
 
                              Procs
                              
 
                              
-
+
                              
 
                              proc foo() {....raises: [], tags: [].}
                              
 
                              
 
 I do foo
 
 
                              
+
                              
 
 
                              
 
diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css
index 5c2bf2093..4abea9ce0 100644
--- a/nimdoc/testproject/expected/nimdoc.out.css
+++ b/nimdoc/testproject/expected/nimdoc.out.css
@@ -234,6 +234,12 @@ select:focus {
 }
 
 /* Docgen styles */
+
+:target {
+  border: 2px solid #B5651D;
+  border-style: dotted;
+}
+
 /* Links */
 a {
   color: var(--anchor);
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index 533cb5b9d..47901ce0d 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -213,7 +213,7 @@ window.addEventListener('DOMContentLoaded', main);
   
                              
 
                              Types
                              
 
                              
-
+
                              
 
                              SomeType = enum
   enumValueA, enumValueB, enumValueC
                              
 
                              
@@ -221,101 +221,114 @@ window.addEventListener('DOMContentLoaded', main);
 
 
 
                              
+
                              
 
 
                              
 
                              
 
                              Procs
                              
 
                              
-
+
                              
 
                              proc fn2() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn3(): auto {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn4(): auto {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn5() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn6() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn7() {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc fn8(): auto {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              func fn9(a: int): int {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              func fn10(a: int): int {....raises: [], tags: [].}
                              
 
                              
 
 comment
 
 
                              
-
+
                              
+
                              
 
                              proc someType(): SomeType {....raises: [], tags: [].}
                              
 
                              
 
 constructor.
 
 
                              
+
                              
 
 
                              
 
                              
 
                              Templates
                              
 
                              
-
+
                              
 
                              template aEnum(): untyped
                              
 
                              
 
 
 
 
                              
-
+
                              
+
                              
 
                              template bEnum(): untyped
                              
 
                              
 
 
 
 
                              
-
+
                              
+
                              
 
                              template fromUtilsGen(): untyped
                              
 
                              
 
@@ -324,6 +337,7 @@ should be shown in utils.html only
 
                              discard "should be in utils.html only, not in module that calls fromUtilsGen"
                              ditto
 
 
                              
+
                              
 
 
                              
 
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 0f6289bbc..d98b1b00f 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -447,7 +447,7 @@ window.addEventListener('DOMContentLoaded', main);
 
                              
 
                              Types
                              
 
                              
-
+
                              
 
                              A {.inject.} = enum
   aA
                              
 
                              
@@ -455,7 +455,8 @@ window.addEventListener('DOMContentLoaded', main);
 The enum A.
 
 
                              
-
+
                              
+
                              
 
                              B {.inject.} = enum
   bB
                              
 
                              
@@ -463,7 +464,8 @@ The enum A.
 The enum B.
 
 
                              
-
+
                              
+
                              
 
                              Foo = enum
   enumValueA2
                              
 
                              
@@ -471,7 +473,8 @@ The enum B.
 
 
 
                              
-
+
                              
+
                              
 
                              FooBuzz {....deprecated: "FooBuzz msg".} = int
                              
 
                              
   
                              
@@ -481,7 +484,8 @@ The enum B.
 
 
 
                              
-
+
                              
+
                              
 
                              Shapes = enum
   Circle,                   ## A circle
   Triangle,                 ## A three-sided shape
@@ -491,64 +495,71 @@ The enum B.
 Some shapes.
 
 
                            
+
 
 
 
                            
 
                            Vars
                            
 
                            
-
+
                            
 
                            aVariable: array[1, int]
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            someVariable: bool
                            
 
                            
 
 This should be visible.
 
 
                            
+
                            
 
 
                            
 
                            
 
                            Consts
                            
 
                            
-
+
                            
 
                            C_A = 0x7FF0000000000000'f64
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            C_B = 0o377'i8
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            C_C = 0o277'i8
                            
 
                            
 
 
 
 
                            
-
+
                            
+
                            
 
                            C_D = 0o177777'i16
                            
 
                            
 
 
 
 
                            
+
                            
 
 
                            
 
                            
 
                            Procs
                            
 
                            
-
+
                            
 
                            proc addfBug14485() {....raises: [], tags: [].}
                            
 
                            
 
@@ -567,14 +578,16 @@ Some proc
 ]#
    • 
 
 
-
+
+
    
 
    proc anything() {....raises: [], tags: [].}
    
 
    
 
 There is no block quote after blank lines at the beginning.
 
 
    
-
+
    
+
    
 
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
    
 
    
@@ -582,14 +595,16 @@ There is no block quote after blank lines at the beginning.
 ok1
 
 
    
-
+
    
+
    
 
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
 
    
 
@@ -598,21 +613,24 @@ ok1
 
    discard
    ok1
 
 
    
-
+
    
+
    
 
    proc bar[T](a, b: T): T
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc baz() {....raises: [], tags: [].}
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc baz[T](a, b: T): T {....deprecated.}
    
 
    
   
    
@@ -622,7 +640,8 @@ ok1
 This is deprecated without message.
 
 
    
-
+
    
+
    
 
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    
 
    
   
    
@@ -632,7 +651,8 @@ This is deprecated without message.
 This is deprecated with a message.
 
 
    
-
+
    
+
    
 
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable.}
    
 
    
@@ -640,7 +660,8 @@ This is deprecated with a message.
 
 
 
    
-
+
    
+
    
 
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable.}
    
 
    
@@ -648,7 +669,8 @@ This is deprecated with a message.
 the c printf. etc.
 
 
    
-
+
    
+
    
 
    proc fromUtils3() {....raises: [], tags: [].}
    
 
    
 
@@ -658,14 +680,16 @@ came form utils but should be shown where 
    
 
 
-
+
+
    
 
    proc isValid[T](x: T): bool
    
 
    
 
 
 
 
    
-
+
    
+
    
 
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    
 
    
 
@@ -678,7 +702,8 @@ came form utils but should be shown where discard "in low2"
    - + +
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    @@ -689,7 +714,8 @@ came form utils but should be shown where low(2) # => -9223372036854775808
    - +
    +
    proc p1() {....raises: [], tags: [].}
    @@ -712,14 +738,16 @@ this is a nested doc comment # also work after
    - +
    +
    func someFunc() {....raises: [], tags: [].}
    My someFunc. Stuff in quotes here. Some link
    - +
    +
    proc tripleStrLitTest() {....raises: [], tags: [].}
    @@ -762,14 +790,16 @@ at indent 0 # should be in
    - +
    +
    proc z1(): Foo {....raises: [], tags: [].}
    cz1
    - +
    +
    proc z2() {....raises: [], tags: [].}
    @@ -778,49 +808,56 @@ cz2 
    discard "in cz2"
    - +
    +
    proc z3() {....raises: [], tags: [].}
    cz3
    - +
    +
    proc z4() {....raises: [], tags: [].}
    cz4
    - +
    +
    proc z5(): int {....raises: [], tags: [].}
    cz5
    - +
    +
    proc z6(): int {....raises: [], tags: [].}
    cz6
    - +
    +
    proc z7(): int {....raises: [], tags: [].}
    cz7
    - +
    +
    proc z8(): int {....raises: [], tags: [].}
    cz8
    - +
    +
    proc z9() {....raises: [], tags: [].}
    @@ -829,7 +866,8 @@ cz8 
    doAssert 1 + 1 == 2
    - +
    +
    proc z10() {....raises: [], tags: [].}
    @@ -838,7 +876,8 @@ cz8 
    discard 1
    cz10
    - +
    +
    proc z11() {....raises: [], tags: [].}
    @@ -847,7 +886,8 @@ cz8 
    discard 1
    - +
    +
    proc z12(): int {....raises: [], tags: [].}
    @@ -856,7 +896,8 @@ cz8 
    discard 1
    - +
    +
    proc z13() {....raises: [], tags: [].}
    @@ -865,7 +906,8 @@ cz13 
    discard
    - +
    +
    proc z17() {....raises: [], tags: [].}
    @@ -874,38 +916,42 @@ cz17 rest 
    discard 1
    rest
    +

    Methods

    - +
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    foo1
    - +
    +
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    foo2
    - +
    +
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    foo3
    +

    Iterators

    - +
    iterator fromUtils1(): int {....raises: [], tags: [].}
    @@ -916,14 +962,16 @@ foo3 # ok2
    - +
    +
    iterator iter1(n: int): int {....raises: [], tags: [].}
    foo1
    - +
    +
    iterator iter2(n: int): int {....raises: [], tags: [].}
    @@ -932,19 +980,21 @@ foo2 
    discard # bar
    +

    Macros

    - +
    macro bar(): untyped
    - +
    +
    macro z16()
    @@ -955,26 +1005,29 @@ foo2 
    doAssert 2 == 1 + 1
    - +
    +
    macro z18(): int
    cz18
    +

    Templates

    - +
    template foo(a, b: SomeType)
    This does nothing
    - +
    +
    template fromUtils2()
    @@ -984,7 +1037,8 @@ ok3 in module calling fromUtilsGen"""
    - +
    +
    template myfn()
    @@ -1006,7 +1060,8 @@ bar # should be inshould be still in
    - +
    +
    template testNimDocTrailingExample()
    @@ -1015,14 +1070,16 @@ bar 
    discard 2
    - +
    +
    template z6t(): int
    cz6t
    - +
    +
    template z14()
    @@ -1031,7 +1088,8 @@ cz14 
    discard
    - +
    +
    template z15()
    @@ -1048,6 +1106,7 @@ cz15 
    discard 1
    in or out?
    +
    From 6b3c77e7f4421267c551b1cd84717a19c65be2c5 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Jul 2021 14:05:26 -0700 Subject: [PATCH 0294/1654] Remove tracking of environment from osenv.nim v2 (#18575) * Remove unnecessary environment tracking * try to fix windows * fix delEnv * make putEnv work on windows even with empty values; improve tests: add tests, add js, vm testing * [skip ci] fix changelog Co-authored-by: Caden Haustein --- changelog.md | 3 + lib/pure/includes/osenv.nim | 237 +++++++++----------------- lib/std/private/win_setenv.nim | 92 ++++++++++ tests/destructor/tnewruntime_misc.nim | 2 +- tests/stdlib/tos.nim | 34 +--- tests/stdlib/tosenv.nim | 58 +++++++ 6 files changed, 240 insertions(+), 186 deletions(-) create mode 100644 lib/std/private/win_setenv.nim create mode 100644 tests/stdlib/tosenv.nim diff --git a/changelog.md b/changelog.md index be94f92c4..a19697c8c 100644 --- a/changelog.md +++ b/changelog.md @@ -97,11 +97,14 @@ The downside is that these defines now have custom logic that doesn't apply for other defines. +- `std/os`: `putEnv` now raises if the 1st argument contains a `=` + - Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. + ## Standard library additions and changes - `strformat`: diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index db291c5fd..6aaafbfda 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -40,112 +40,15 @@ when defined(nodejs): # {.error: "requires -d:nodejs".} else: - when defined(windows): - from parseutils import skipIgnoreCase proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "".} - when defined(vcc): + when defined(windows): proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + from std/private/win_setenv import setEnvImpl else: proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} - proc c_unsetenv(env: cstring): cint {. - importc: "unsetenv", header: "".} - - # Environment handling cannot be put into RTL, because the `envPairs` - # iterator depends on `environment`. - - var - envComputed {.threadvar.}: bool - environment {.threadvar.}: seq[string] - - when defined(nimV2): - proc unpairedEnvAllocs*(): int = - result = environment.len - if result > 0: inc result - - when defined(windows) and not defined(nimscript): - # because we support Windows GUI applications, things get really - # messy here... - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "".} - else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {. - importc: "wcschr", header: "".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {. - importc: "strchr", header: "".} - - proc getEnvVarsC() = - if not envComputed: - environment = @[] - when useWinUnicode: - var - env = getEnvironmentStringsW() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[WideCString](cast[ByteAddress](eend)+2) - if eend[1].int == 0: break - discard freeEnvironmentStringsW(env) - else: - var - env = getEnvironmentStringsA() - e = env - if e == nil: return # an error occurred - while true: - var eend = strEnd(e) - add(environment, $e) - e = cast[cstring](cast[ByteAddress](eend)+1) - if eend[1] == '\0': break - discard freeEnvironmentStringsA(env) - envComputed = true - - else: - const - useNSGetEnviron = (defined(macosx) and not defined(ios) and not defined(emscripten)) or defined(nimscript) - - when useNSGetEnviron: - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # , can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {. - importc: "_NSGetEnviron", header: "".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray - - proc getEnvVarsC() = - # retrieves the variables of char** env of C's main proc - if not envComputed: - environment = @[] - when useNSGetEnviron: - var gEnv = NSGetEnviron()[] - var i = 0 - while gEnv[i] != nil: - add environment, $gEnv[i] - inc(i) - envComputed = true - - proc findEnvVar(key: string): int = - getEnvVarsC() - var temp = key & '=' - for i in 0..high(environment): - when defined(windows): - if skipIgnoreCase(environment[i], temp) == len(temp): return i - else: - if startsWith(environment[i], temp): return i - return -1 + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = ## Returns the value of the `environment variable`:idx: named `key`. @@ -163,16 +66,9 @@ else: assert getEnv("unknownEnv") == "" assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - when nimvm: - discard "built into the compiler" - else: - var i = findEnvVar(key) - if i >= 0: - return substr(environment[i], find(environment[i], '=')+1) - else: - var env = c_getenv(key) - if env == nil: return default - result = $env + let env = c_getenv(key) + if env == nil: return default + result = $env proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = ## Checks whether the environment variable named `key` exists. @@ -186,11 +82,7 @@ else: runnableExamples: assert not existsEnv("unknownEnv") - when nimvm: - discard "built into the compiler" - else: - if c_getenv(key) != nil: return true - else: return findEnvVar(key) >= 0 + return c_getenv(key) != nil proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## Sets the value of the `environment variable`:idx: named `key` to `val`. @@ -201,33 +93,14 @@ else: ## * `existsEnv proc <#existsEnv,string>`_ ## * `delEnv proc <#delEnv,string>`_ ## * `envPairs iterator <#envPairs.i>`_ - - # Note: by storing the string in the environment sequence, - # we guarantee that we don't free the memory before the program - # ends (this is needed for POSIX compliance). It is also needed so that - # the process itself may access its modified environment variables! - when nimvm: - discard "built into the compiler" + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) else: - var indx = findEnvVar(key) - if indx >= 0: - environment[indx] = key & '=' & val - else: - add environment, (key & '=' & val) - indx = high(environment) - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - var v = newWideCString(val) - if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError()) - elif defined(vcc): - if c_putenv_s(key, val) != 0'i32: - raiseOSError(osLastError()) - else: - if c_setenv(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError()) + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = ## Deletes the `environment variable`:idx: named `key`. @@ -238,21 +111,45 @@ else: ## * `existsEnv proc <#existsEnv,string>`_ ## * `putEnv proc <#putEnv,string,string>`_ ## * `envPairs iterator <#envPairs.i>`_ - when nimvm: - discard "built into the compiler" + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + if c_putenv_s(key, "") != 0'i32: bail else: - var indx = findEnvVar(key) - if indx < 0: return # Do nothing if the env var is not already set - when defined(windows) and not defined(nimscript): - when useWinUnicode: - var k = newWideCString(key) - if setEnvironmentVariableW(k, nil) == 0'i32: raiseOSError(osLastError()) - else: - if setEnvironmentVariableA(key, nil) == 0'i32: raiseOSError(osLastError()) + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when useWinUnicode: + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "".} else: - if c_unsetenv(key) != 0'i32: - raiseOSError(osLastError()) - environment.delete(indx) + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "".} + else: + proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", + header: "".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # , can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = ## Iterate over all `environments variables`:idx:. @@ -265,8 +162,30 @@ else: ## * `existsEnv proc <#existsEnv,string>`_ ## * `putEnv proc <#putEnv,string,string>`_ ## * `delEnv proc <#delEnv,string>`_ - getEnvVarsC() - for i in 0..high(environment): - var p = find(environment[i], '=') - yield (substr(environment[i], 0, p-1), - substr(environment[i], p+1)) + when defined(windows): + block: + template impl(get_fun, typ, size, zero, free_fun) = + let env = get_fun() + var e = env + if e == nil: break + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[typ](cast[ByteAddress](eend)+size) + if typeof(zero)(eend[1]) == zero: break + discard free_fun(env) + when useWinUnicode: + impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + else: + impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim new file mode 100644 index 000000000..067e656a3 --- /dev/null +++ b/lib/std/private/win_setenv.nim @@ -0,0 +1,92 @@ +#[ +Copyright (c) Facebook, Inc. and its affiliates. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Adapted `setenv` from https://github.com/facebook/folly/blob/master/folly/portability/Stdlib.cpp +translated from C to nim. +]# + +#[ +Introduced in https://github.com/facebook/folly/commit/5d8ca09a3f96afefb44e35808f03651a096ab9c7 + +TODO: +check errno_t vs cint +]# + +when not defined(windows): discard +else: + proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} + # same as winlean.setEnvironmentVariableA + + proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} + proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "".} + + var errno {.importc, header: "".}: cint + type wchar_t {.importc: "wchar_t".} = int16 + var gWenviron {.importc:"_wenviron".}: ptr ptr wchar_t + # xxx `ptr UncheckedArray[WideCString]` did not work + + proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: WideCString, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} + # xxx cint vs errno_t? + + proc setEnvImpl*(name: cstring, value: cstring, overwrite: cint): cint = + const EINVAL = cint(22) + const MAX_ENV = 32767 + # xxx get it from: `var MAX_ENV {.importc: "_MAX_ENV", header:"".}: cint` + if overwrite == 0 and c_getenv(name) != nil: return 0 + if value[0] != '\0': + let e = c_putenv_s(name, value) + if e != 0: + errno = e + return -1 + return 0 + #[ + We are trying to set the value to an empty string, but `_putenv_s` deletes + entries if the value is an empty string, and just calling + SetEnvironmentVariableA doesn't update `_environ`, + so we have to do these terrible things. + ]# + if c_putenv_s(name, " ") != 0: + errno = EINVAL + return -1 + # Here lies the documentation we blatently ignore to make this work. + var s = c_getenv(name) + s[0] = '\0' + #[ + This would result in a double null termination, which normally signifies the + end of the environment variable list, so we stick a completely empty + environment variable into the list instead. + ]# + s[1] = '=' + #[ + If gWenviron is null, the wide environment has not been initialized + yet, and we don't need to try to update it. We have to do this otherwise + we'd be forcing the initialization and maintenance of the wide environment + even though it's never actually used in most programs. + ]# + if gWenviron != nil: + # var buf: array[MAX_ENV + 1, WideCString] + var buf: array[MAX_ENV + 1, Utf16Char] + let buf2 = cast[WideCString](buf[0].addr) + var len: csize_t + if mbstowcs_s(len.addr, buf2, buf.len.csize_t, name, MAX_ENV) != 0: + errno = EINVAL + return -1 + c_wgetenv(buf2)[0] = '\0'.Utf16Char + c_wgetenv(buf2)[1] = '='.Utf16Char + + # And now, we have to update the outer environment to have a proper empty value. + if setEnvironmentVariableA(name, value) == 0: + errno = EINVAL + return -1 + return 0 diff --git a/tests/destructor/tnewruntime_misc.nim b/tests/destructor/tnewruntime_misc.nim index 48ea36b7c..ac061f2f7 100644 --- a/tests/destructor/tnewruntime_misc.nim +++ b/tests/destructor/tnewruntime_misc.nim @@ -7,7 +7,7 @@ axc ... destroying GenericObj[T] GenericObj[system.int] test -(allocCount: 13, deallocCount: 11) +(allocCount: 12, deallocCount: 10) 3''' """ diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index fedf2b3e3..dcb2d44f4 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -331,7 +331,7 @@ block walkDirRec: doAssert p.startsWith("walkdir_test") var s: seq[string] - for p in walkDirRec("walkdir_test", {pcFile}, {pcDir}, relative=true): + for p in walkDirRec("walkdir_test", {pcFile}, {pcDir}, relative = true): s.add(p) doAssert s.len == 2 @@ -553,19 +553,19 @@ block ospaths: doAssert joinPath("/", "") == unixToNativePath"/" doAssert joinPath("/" / "") == unixToNativePath"/" # weird test case... doAssert joinPath("/", "/a/b/c") == unixToNativePath"/a/b/c" - doAssert joinPath("foo/","") == unixToNativePath"foo/" - doAssert joinPath("foo/","abc") == unixToNativePath"foo/abc" - doAssert joinPath("foo//./","abc/.//") == unixToNativePath"foo/abc/" - doAssert joinPath("foo","abc") == unixToNativePath"foo/abc" - doAssert joinPath("","abc") == unixToNativePath"abc" + doAssert joinPath("foo/", "") == unixToNativePath"foo/" + doAssert joinPath("foo/", "abc") == unixToNativePath"foo/abc" + doAssert joinPath("foo//./", "abc/.//") == unixToNativePath"foo/abc/" + doAssert joinPath("foo", "abc") == unixToNativePath"foo/abc" + doAssert joinPath("", "abc") == unixToNativePath"abc" - doAssert joinPath("zook/.","abc") == unixToNativePath"zook/abc" + doAssert joinPath("zook/.", "abc") == unixToNativePath"zook/abc" # controversial: inconsistent with `joinPath("zook/.","abc")` # on linux, `./foo` and `foo` are treated a bit differently for executables # but not `./foo/bar` and `foo/bar` doAssert joinPath(".", "/lib") == unixToNativePath"./lib" - doAssert joinPath(".","abc") == unixToNativePath"./abc" + doAssert joinPath(".", "abc") == unixToNativePath"./abc" # cases related to issue #13455 doAssert joinPath("foo", "", "") == "foo" @@ -605,24 +605,6 @@ block getTempDir: block: # getCacheDir doAssert getCacheDir().dirExists -block osenv: - block delEnv: - const dummyEnvVar = "DUMMY_ENV_VAR" # This env var wouldn't be likely to exist to begin with - doAssert existsEnv(dummyEnvVar) == false - putEnv(dummyEnvVar, "1") - doAssert existsEnv(dummyEnvVar) == true - delEnv(dummyEnvVar) - doAssert existsEnv(dummyEnvVar) == false - delEnv(dummyEnvVar) # deleting an already deleted env var - doAssert existsEnv(dummyEnvVar) == false - block: # putEnv, bug #18502 - doAssertRaises(OSError): putEnv("DUMMY_ENV_VAR_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE") - doAssertRaises(OSError): putEnv("", "NEW_DUMMY_VALUE") - block: - doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", "") == "" - doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", " ") == " " - doAssert getEnv("DUMMY_ENV_VAR_NONEXISTENT", "Arrakis") == "Arrakis" - block isRelativeTo: doAssert isRelativeTo("/foo", "/") doAssert isRelativeTo("/foo/bar", "/foo") diff --git a/tests/stdlib/tosenv.nim b/tests/stdlib/tosenv.nim new file mode 100644 index 000000000..df5759d0c --- /dev/null +++ b/tests/stdlib/tosenv.nim @@ -0,0 +1,58 @@ +discard """ + matrix: "--threads" + joinable: false + targets: "c js cpp" +""" + +import std/os +from std/sequtils import toSeq +import stdtest/testutils + +template main = + block: # delEnv, existsEnv, getEnv, envPairs + for val in ["val", ""]: # ensures empty val works too + const key = "NIM_TESTS_TOSENV_KEY" + doAssert not existsEnv(key) + putEnv(key, val) + doAssert existsEnv(key) + doAssert getEnv(key) == val + when nimvm: discard + else: + doAssert (key, val) in toSeq(envPairs()) + delEnv(key) + when nimvm: discard + else: + doAssert (key, val) notin toSeq(envPairs()) + doAssert not existsEnv(key) + delEnv(key) # deleting an already deleted env var + doAssert not existsEnv(key) + + block: + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "") == "" + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", " ") == " " + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "defval") == "defval" + + whenVMorJs: discard # xxx improve + do: + doAssertRaises(OSError, putEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE")) + doAssertRaises(OSError, putEnv("", "NEW_DUMMY_VALUE")) + doAssert not existsEnv("") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT") + +static: main() +main() + +when not defined(js): + block: # bug #18533 + proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} + var thr: Thread[void] + proc threadFunc {.thread.} = putEnv("foo", "fooVal2") + + putEnv("foo", "fooVal1") + doAssert getEnv("foo") == "fooVal1" + createThread(thr, threadFunc) + joinThreads(thr) + doAssert getEnv("foo") == $c_getenv("foo") + + doAssertRaises(OSError): delEnv("foo=bar") From afc0259b125f5be032bf4c13721669c5452f9143 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 29 Jul 2021 19:28:33 -0700 Subject: [PATCH 0295/1654] fix #18385 followup, by building nimsuggest with -d:release during testing (#18581) --- koch.nim | 5 +++++ nimsuggest/tester.nim | 15 +++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/koch.nim b/koch.nim index 4296e4529..8dd897028 100644 --- a/koch.nim +++ b/koch.nim @@ -592,6 +592,11 @@ proc runCI(cmd: string) = execFold("Run rst2html tests", "nim r nimdoc/rsttester") execFold("Run nimpretty tests", "nim r nimpretty/tester.nim") when defined(posix): + # refs #18385, build with -d:release instead of -d:danger for testing + # We could also skip building nimsuggest in buildTools, or build it with -d:release + # in bundleNimsuggest depending on some environment variable when we are in CI. One advantage + # of rebuilding is this won't affect bin/nimsuggest when running runCI locally + execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest") execFold("Run nimsuggest tests", "nim r nimsuggest/tester") execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls") diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index b62aa8783..1db33706a 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -6,6 +6,7 @@ # `nim r nimsuggest/tester.nim nimsuggest/tests/tsug_accquote.nim` import os, osproc, strutils, streams, re, sexp, net +from sequtils import toSeq type Test = object @@ -23,7 +24,7 @@ import std/compilesettings proc parseTest(filename: string; epcMode=false): Test = const cursorMarker = "#[!]#" - let nimsug = "bin" / addFileExt("nimsuggest", ExeExt) + let nimsug = "bin" / addFileExt("nimsuggest_testing", ExeExt) doAssert nimsug.fileExists, nimsug const libpath = querySetting(libPath) result.filename = filename @@ -272,6 +273,7 @@ proc runEpcTest(filename: string): int = let a = outp.readAll().strip() let port = parseInt(a) socket.connect("localhost", Port(port)) + for req, resp in items(s.script): if not runCmd(req, s.dest): socket.sendEpcStr(req) @@ -279,8 +281,12 @@ proc runEpcTest(filename: string): int = if not req.startsWith("mod "): let answer = sexpToAnswer(sx) doReport(filename, answer, resp, report) - finally: + socket.sendEpcStr "return arg" + # bugfix: this was in `finally` block, causing the original error to be + # potentially masked by another one in case `socket.sendEpcStr` raises + # (e.g. if socket couldn't connect in the 1st place) + finally: close(p) if report.len > 0: echo "==== EPC ========================================" @@ -336,8 +342,9 @@ proc main() = failures += runTest(xx) failures += runEpcTest(xx) else: - for x in walkFiles(tpath / "t*.nim"): - echo "Test ", x + let files = toSeq(walkFiles(tpath / "t*.nim")) + for i, x in files: + echo "$#/$# test: $#" % [$i, $files.len, x] when defined(i386): if x == "nimsuggest/tests/tmacro_highlight.nim": echo "skipping" # workaround bug #17945 From 9a26f5059be48b19d821c17fe1066c05d167bc7d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 30 Jul 2021 00:30:22 -0700 Subject: [PATCH 0296/1654] refs #16613: check opcWrDeref for nil (#18613) --- compiler/vm.nim | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 9541af701..73a228b6c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -812,11 +812,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = # vmgen generates opcWrDeref, which means that we must dereference # twice. # TODO: This should likely be handled differently in vmgen. - if (nfIsRef notin regs[ra].nodeAddr[].flags and - nfIsRef notin n.flags): - regs[ra].nodeAddr[][] = n[] - else: - regs[ra].nodeAddr[] = n + let nAddr = regs[ra].nodeAddr + if nAddr[] == nil: stackTrace(c, tos, pc, "opcWrDeref internal error") # refs bug #16613 + if (nfIsRef notin nAddr[].flags and nfIsRef notin n.flags): nAddr[][] = n[] + else: nAddr[] = n of rkRegisterAddr: regs[ra].regAddr[] = regs[rc] of rkNode: # xxx: also check for nkRefTy as in opcLdDeref? From c6fadb179975f79ad9914f09ce10f7fa6cbf0735 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 30 Jul 2021 15:32:00 +0800 Subject: [PATCH 0297/1654] [minor] reduce `substr` (#18611) * minor * correct * unify the type of addrLen * Update lib/packages/docutils/rstgen.nim --- lib/packages/docutils/rstgen.nim | 2 +- lib/pure/nativesockets.nim | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 10a229479..3734392c3 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -675,7 +675,7 @@ proc readIndexDir(dir: string): var x = fileEntries[0].link let i = find(x, '#') if i > 0: - x = x.substr(0, i-1) + x.setLen(i) if i != 0: # don't add entries starting with '#' result.modules.add(x.changeFileExt("")) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 124afce56..13c08dd92 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -14,6 +14,7 @@ import os, options import std/private/since +import std/strbasics when hostOS == "solaris": @@ -468,7 +469,7 @@ proc getAddrString*(sockAddr: ptr SockAddr): string = if sockAddr.sa_family.cint == nativeAfInet: result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN + let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int else: 46 # it's actually 46 in both cases result = newString(addrLen) let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr @@ -477,7 +478,7 @@ proc getAddrString*(sockAddr: ptr SockAddr): string = result.len.int32) == nil: raiseOSError(osLastError()) if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result = result.substr("::ffff:".len) + result.setSlice("::ffff:".len.. Date: Fri, 30 Jul 2021 12:05:14 +0200 Subject: [PATCH 0298/1654] fixes #18371 (#18617) --- compiler/vmops.nim | 7 +++++-- lib/std/effecttraits.nim | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 6ee915bd5..a1a4996b2 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -303,12 +303,15 @@ proc registerAdditionalOps*(c: PCtx) = proc getEffectList(c: PCtx; a: VmArgs; effectIndex: int) = let fn = getNode(a, 0) + var list = newNodeI(nkBracket, fn.info) if fn.typ != nil and fn.typ.n != nil and fn.typ.n[0].len >= effectListLen and fn.typ.n[0][effectIndex] != nil: - var list = newNodeI(nkBracket, fn.info) for e in fn.typ.n[0][effectIndex]: list.add opMapTypeInstToAst(c.cache, e.typ.skipTypes({tyRef}), e.info, c.idgen) - setResult(a, list) + else: + list.add newIdentNode(getIdent(c.cache, "UncomputedEffects"), fn.info) + + setResult(a, list) registerCallback c, "stdlib.effecttraits.getRaisesListImpl", proc (a: VmArgs) = getEffectList(c, a, exceptionEffects) diff --git a/lib/std/effecttraits.nim b/lib/std/effecttraits.nim index 358280db0..635ec82d0 100644 --- a/lib/std/effecttraits.nim +++ b/lib/std/effecttraits.nim @@ -11,7 +11,7 @@ ## for Nim's macro system. ## **Since**: Version 1.4. ## -## One can test for the existance of this standard module +## One can test for the existence of this standard module ## via `defined(nimHasEffectTraitsModule)`. import macros From ecfc47b332a67bb373d433f01277bc924b75f77f Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Fri, 30 Jul 2021 17:36:16 +0300 Subject: [PATCH 0299/1654] Modify atlas test for short commit hashes (#18619) * test commit hashes support * Update testdata.nim * remove extra newlines --- tools/atlas/packagesjson.nim | 2 -- tools/atlas/testdata.nim | 2 +- tools/atlas/tests/balls.nimble | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim index e1e23ee29..ed83ce85c 100644 --- a/tools/atlas/packagesjson.nim +++ b/tools/atlas/packagesjson.nim @@ -70,7 +70,6 @@ proc search*(pkgList: seq[Package]; terms: seq[string]) = var found = false template onFound = echo pkg - echo("") found = true break forPackage @@ -88,7 +87,6 @@ proc search*(pkgList: seq[Package]; terms: seq[string]) = onFound() else: echo(pkg) - echo(" ") if not found and terms.len > 0: echo("No package found.") diff --git a/tools/atlas/testdata.nim b/tools/atlas/testdata.nim index f960e3d08..aefaeacd2 100644 --- a/tools/atlas/testdata.nim +++ b/tools/atlas/testdata.nim @@ -25,7 +25,7 @@ const toData("ups", GitCheckout, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1"), toData("sync", GitDiff, 0, ""), - toData("sync", GitTags, 0, "c9811af2d03dc78a920d33592d0a3daae3d9e68f refs/tags/v0.1.0\nc99d3a73344c4c02335190b489e7ea8aaa4e9584 refs/tags/v0.2.0\n6186a38c97c0f26e1a0c8016979227d26c41717b refs/tags/v0.3.0\n5506e4e09b849bd59f930fcccee3096abb089b1d refs/tags/v0.4.0\n1e4ef046ed0447aec7ca4ab3b2e2a5347ac41448 refs/tags/v0.5.0\na95f7b0b22c77b98ed63c1c4f435d550de8dbb05 refs/tags/v0.6.0\n214b868ace4ebc9140ee56734147fbd16b238c71 refs/tags/v0.7.0\n786698bf9f597834f36cd530d32faee31545a1d7 refs/tags/v0.9.0\n810bd2d75e9f6e182534ae2488670b51a9f13fc3 refs/tags/v1.0.0\na133d37991e3d4e9553c5261eabcb07d471968f7 refs/tags/v1.1.0\n0872a7cd70ea43a26deb94b8ab12cc5b89a1c212 refs/tags/v1.2.0\n19ac734438b6c04fe89711728a97fcde591324eb refs/tags/v1.3.0\nde5c7337ebc22422190e8aeca37d05651735f440 refs/tags/v1.4.0\n"), + toData("sync", GitRevParse, 0, "810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"), toData("sync", GitCurrentCommit, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n"), toData("sync", GitMergeBase, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"), diff --git a/tools/atlas/tests/balls.nimble b/tools/atlas/tests/balls.nimble index 62ab4750b..143e757e9 100644 --- a/tools/atlas/tests/balls.nimble +++ b/tools/atlas/tests/balls.nimble @@ -6,7 +6,7 @@ license = "MIT" # requires newTreeFrom requires "https://github.com/disruptek/grok >= 0.5.0 & < 1.0.0" requires "https://github.com/disruptek/ups < 1.0.0" -requires "https://github.com/planetis-m/sync >= 1.0.0 & < 2.0.0" +requires "https://github.com/planetis-m/sync#810bd2d" #requires "https://github.com/c-blake/cligen < 2.0.0" bin = @["balls"] # build the binary for basic test running From 52e276c82daf2b8a33f4e4304eac3074bb37b70d Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 31 Jul 2021 23:54:08 -0700 Subject: [PATCH 0300/1654] exportC => exportc (#18625) --- tests/template/template_pragmas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/template/template_pragmas.nim b/tests/template/template_pragmas.nim index d6c99080d..da532b010 100644 --- a/tests/template/template_pragmas.nim +++ b/tests/template/template_pragmas.nim @@ -3,7 +3,7 @@ proc output_x:string {.compileTime.} = "x" template t = const x = output_x() let - bar {.exportC:"bar" & x.} = 100 + bar {.exportc:"bar" & x.} = 100 static: doAssert(compiles (t())) From 916d0c21af6153338a8064ff953b579ff87c6ba6 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 1 Aug 2021 17:19:43 +0800 Subject: [PATCH 0301/1654] fix #18620 (#18624) * fix #18620 * add testcase --- lib/system/excpt.nim | 5 +---- tests/exception/t18620.nim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 tests/exception/t18620.nim diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 9f65db2fe..4814b77b4 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -524,10 +524,7 @@ proc getStackTrace(e: ref Exception): string = proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = ## Returns the attached stack trace to the exception `e` as ## a `seq`. This is not yet available for the JS backend. - when not defined(nimSeqsV2): - shallowCopy(result, e.trace) - else: - result = move(e.trace) + shallowCopy(result, e.trace) proc getStackTraceEntries*(): seq[StackTraceEntry] = ## Returns the stack trace entries for the current stack trace. diff --git a/tests/exception/t18620.nim b/tests/exception/t18620.nim new file mode 100644 index 000000000..ee23f8bac --- /dev/null +++ b/tests/exception/t18620.nim @@ -0,0 +1,17 @@ +discard """ + matrix: "--gc:arc; --gc:refc" +""" + +proc hello() = + raise newException(ValueError, "You are wrong") + +var flag = false + +try: + hello() +except ValueError as e: + flag = true + doAssert len(getStackTraceEntries(e)) > 0 + doAssert len(getStackTraceEntries(e)) > 0 + +doAssert flag From 2bc07554ccd64d7405d0f386f17f7b1faa95bd6f Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 1 Aug 2021 19:52:24 +0800 Subject: [PATCH 0302/1654] make proc names consistent (#18626) * rename `endswith` to `endsWith` * rename --- compiler/semtypes.nim | 10 +++++----- tests/arc/tasyncawait.nim | 2 +- tests/async/tasyncawait.nim | 2 +- tests/async/tasyncnetudp.nim | 2 +- tests/async/tasyncssl.nim | 2 +- tests/async/tnewasyncudp.nim | 2 +- tests/async/twinasyncrw.nim | 2 +- tools/atlas/osutils.nim | 2 +- tools/trimcc.nim | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index db895cab6..cb7f847a9 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1700,14 +1700,14 @@ proc semStaticType(c: PContext, childNode: PNode, prev: PType): PType = result.rawAddSon(base) result.flags.incl tfHasStatic -proc semTypeof(c: PContext; n: PNode; prev: PType): PType = +proc semTypeOf(c: PContext; n: PNode; prev: PType): PType = openScope(c) let t = semExprWithType(c, n, {efInTypeof}) closeScope(c) fixupTypeOf(c, prev, t) result = t.typ -proc semTypeof2(c: PContext; n: PNode; prev: PType): PType = +proc semTypeOf2(c: PContext; n: PNode; prev: PType): PType = openScope(c) var m = BiggestInt 1 # typeOfIter if n.len == 3: @@ -1731,7 +1731,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of nkTypeOfExpr: # for ``typeof(countup(1,3))``, see ``tests/ttoseq``. checkSonsLen(n, 1, c.config) - result = semTypeof(c, n[0], prev) + result = semTypeOf(c, n[0], prev) if result.kind == tyTypeDesc: result.flags.incl tfExplicit of nkPar: if n.len == 1: result = semTypeNode(c, n[0], prev) @@ -1833,9 +1833,9 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = semAnyRef(c, n, tyRef, prev) elif op.id == ord(wType): checkSonsLen(n, 2, c.config) - result = semTypeof(c, n[1], prev) + result = semTypeOf(c, n[1], prev) elif op.s == "typeof" and n[0].kind == nkSym and n[0].sym.magic == mTypeOf: - result = semTypeof2(c, n, prev) + result = semTypeOf2(c, n, prev) elif op.s == "owned" and optOwnedRefs notin c.config.globalOptions and n.len == 2: result = semTypeExpr(c, n[1], prev) else: diff --git a/tests/arc/tasyncawait.nim b/tests/arc/tasyncawait.nim index 75d9bc9b5..91a7453cd 100644 --- a/tests/arc/tasyncawait.nim +++ b/tests/arc/tasyncawait.nim @@ -40,7 +40,7 @@ proc readMessages(client: AsyncFD) {.async.} = clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index f658a15ed..e86542b2d 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -34,7 +34,7 @@ proc readMessages(client: AsyncFD) {.async.} = clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tests/async/tasyncnetudp.nim b/tests/async/tasyncnetudp.nim index ef6dfc5e1..dade96fb2 100644 --- a/tests/async/tasyncnetudp.nim +++ b/tests/async/tasyncnetudp.nim @@ -59,7 +59,7 @@ proc readMessages(server: AsyncSocket) {.async.} = while i < maxResponses: let (data, fromIp, fromPort) = await recvFrom(server, 16384) - if data.startswith("Message ") and fromIp == "127.0.0.1": + if data.startsWith("Message ") and fromIp == "127.0.0.1": await sendTo(server, fromIp, fromPort, data) inc(msgCount) diff --git a/tests/async/tasyncssl.nim b/tests/async/tasyncssl.nim index a582818eb..222aaa3a1 100644 --- a/tests/async/tasyncssl.nim +++ b/tests/async/tasyncssl.nim @@ -36,7 +36,7 @@ when defined(ssl): inc(clientCount) break else: - if line.startswith("Message "): + if line.startsWith("Message "): inc(msgCount) else: doAssert false diff --git a/tests/async/tnewasyncudp.nim b/tests/async/tnewasyncudp.nim index 76462c21d..ba1e4134e 100644 --- a/tests/async/tnewasyncudp.nim +++ b/tests/async/tnewasyncudp.nim @@ -81,7 +81,7 @@ proc readMessages(server: AsyncFD) {.async.} = addr(slen)) size = 0 var grammString = $cstring(addr buffer) - if grammString.startswith("Message ") and + if grammString.startsWith("Message ") and saddr.sin_addr.s_addr == nativesockets.ntohl(INADDR_LOOPBACK.uint32): await sendTo(server, addr grammString[0], len(grammString), cast[ptr SockAddr](addr saddr), slen) diff --git a/tests/async/twinasyncrw.nim b/tests/async/twinasyncrw.nim index 69b092607..ad27ca38f 100644 --- a/tests/async/twinasyncrw.nim +++ b/tests/async/twinasyncrw.nim @@ -220,7 +220,7 @@ when defined(windows): clientCount.inc break else: - if line.startswith("Message "): + if line.startsWith("Message "): msgCount.inc else: doAssert false diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim index 1bb414c03..6134830b5 100644 --- a/tools/atlas/osutils.nim +++ b/tools/atlas/osutils.nim @@ -17,7 +17,7 @@ proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string = # github + https + trailing url slash causes a # checkout/ls-remote to fail with Repository not found var isGithub = false - if modUrl.contains("github.com") and modUrl.endswith("/"): + if modUrl.contains("github.com") and modUrl.endsWith("/"): modUrl = modUrl[0 .. ^2] isGithub = true diff --git a/tools/trimcc.nim b/tools/trimcc.nim index 959eaf310..41795e1aa 100644 --- a/tools/trimcc.nim +++ b/tools/trimcc.nim @@ -31,7 +31,7 @@ proc processIncludes(dir: string, whitelist: StringTableRef) = if ('.' notin name and "include" in path) or ("c++" in path): let n = whitelist.getOrDefault(name) if n != "processed": whitelist[name] = "found" - if name.endswith(".h"): + if name.endsWith(".h"): let n = whitelist.getOrDefault(name) if n == "found": includes(path, name, whitelist) of pcDir: processIncludes(path, whitelist) From 499d4690fbbb59b6baf1698b1ed1574dde6ea7c5 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Mon, 2 Aug 2021 00:38:15 -0700 Subject: [PATCH 0303/1654] Add Artix to distro list (#18629) --- lib/pure/distros.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index c1bc44eeb..7a9910c1e 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -52,6 +52,7 @@ type CentOS Deepin ArchLinux + Artix Antergos PCLinuxOS Mageia @@ -133,7 +134,7 @@ type const - LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux} + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux, Distribution.Artix} # we cache the result of the 'cmdRelease' # execution for faster platform detections. @@ -180,6 +181,8 @@ proc detectOsImpl(d: Distribution): bool = result = "rhel" in osReleaseID() of Distribution.ArchLinux: result = "arch" in osReleaseID() + of Distribution.Artix: + result = "artix" in osReleaseID() of Distribution.NixOS: # Check if this is a Nix build or NixOS environment result = existsEnv("NIX_BUILD_TOP") or existsEnv("__NIXOS_SET_ENVIRONMENT_DONE") @@ -249,7 +252,7 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("pkg_add " & p, true) elif detectOs(PCLinuxOS): result = ("rpm -ivh " & p, true) - elif detectOs(ArchLinux) or detectOs(Manjaro): + elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) else: result = (" install " & p, true) From c0abdaa2b4587526f235ef41186e84c4f87813e1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 2 Aug 2021 22:22:47 -0700 Subject: [PATCH 0304/1654] improve runnableExamples and docs for std/nre (#18634) * improve runnableExamples and docs for std/nre * avoid too long lines in example --- lib/impure/nre.nim | 117 +++++++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 68 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 7b2d7d3ee..01ef77427 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -23,12 +23,12 @@ when defined(js): ## **Note**: If you love `sequtils.toSeq` we have bad news for you. This ## library doesn't work with it due to documented compiler limitations. As ## a workaround, use this: -## -## .. code-block:: nim -## -## import std/nre except toSeq -## -## +runnableExamples: + # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`: + import std/sequtils + iterator iota(n: int): int = + for i in 0..\w)").get.captures["letter"] == "a"` - ## - `"abc".match(re"(\w)\w").get.captures[-1] == "ab"` + ## `nil` is returned. See examples for `match`. ## ## `captureBounds[]: HSlice[int, int]` ## gets the bounds of the given capture according to the same rules as ## the above. If the capture is not filled, then `None` is returned. - ## The bounds are both inclusive. - ## - ## - `"abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0` - ## - `0 in "abc".match(re"(\w)").get.captureBounds == true` - ## - `"abc".match(re"").get.captureBounds[-1] == 0 .. -1` - ## - `"abc".match(re"abc").get.captureBounds[-1] == 0 .. 2` + ## The bounds are both inclusive. See examples for `match`. ## ## `match: string` ## the full text of the match. @@ -198,7 +184,6 @@ type pattern*: Regex ## The regex doing the matching. ## Not nil. str*: string ## The string that was matched against. - ## Not nil. pcreMatchBounds: seq[HSlice[cint, cint]] ## First item is the bounds of the match ## Other items are the captures ## `a` is inclusive start, `b` is exclusive end @@ -226,18 +211,6 @@ type ## for whatever reason. The message contains the error ## code. -runnableExamples: - # This MUST be kept in sync with the examples in RegexMatch - doAssert "abc".match(re"(\w)").get.captures[0] == "a" - doAssert "abc".match(re"(?\w)").get.captures["letter"] == "a" - doAssert "abc".match(re"(\w)\w").get.captures[-1] == "ab" - - doAssert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 - doAssert 0 in "abc".match(re"(\w)").get.captureBounds == true - doAssert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 - doAssert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 - - proc destroyRegex(pattern: Regex) = pcre.free_substring(cast[cstring](pattern.pcreObj)) if pattern.pcreExtra != nil: @@ -540,18 +513,27 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the ## string. - ## runnableExamples: - doAssert "foo".match(re"f").isSome - doAssert "foo".match(re"o").isNone + assert "foo".match(re"f").isSome + assert "foo".match(re"o").isNone + assert "abc".match(re"(\w)").get.captures[0] == "a" + assert "abc".match(re"(?\w)").get.captures["letter"] == "a" + assert "abc".match(re"(\w)\w").get.captures[-1] == "ab" + + assert "abc".match(re"(\w)").get.captureBounds[0] == 0 .. 0 + assert 0 in "abc".match(re"(\w)").get.captureBounds + assert "abc".match(re"").get.captureBounds[-1] == 0 .. -1 + assert "abc".match(re"abc").get.captureBounds[-1] == 0 .. 2 return str.matchImpl(pattern, start, endpos, pcre.ANCHORED) iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch = ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every - ## non-overlapping match. `"2222".find(re"22")` is `"22", "22"`, not - ## `"22", "22", "22"`. - ## + ## non-overlapping match: + runnableExamples: + import std/sugar + assert collect(for a in "2222".findIter(re"22"): a.match) == @["22", "22"] + # not @["22", "22", "22"] ## Arguments are the same as `find(...)<#find,string,Regex,int>`_ ## ## Variants: @@ -619,11 +601,10 @@ proc contains*(str: string, pattern: Regex, start = 0, endpos = int.high): bool ## Determine if the string contains the given pattern between the end and ## start positions: ## This function is equivalent to `isSome(str.find(pattern, start, endpos))`. - ## runnableExamples: - doAssert "abc".contains(re"bc") - doAssert not "abc".contains(re"cd") - doAssert not "abc".contains(re"a", start = 1) + assert "abc".contains(re"bc") + assert not "abc".contains(re"cd") + assert not "abc".contains(re"a", start = 1) return isSome(str.find(pattern, start, endpos)) @@ -635,16 +616,16 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string] ## runnableExamples: # - If the match is zero-width, then the string is still split: - doAssert "123".split(re"") == @["1", "2", "3"] + assert "123".split(re"") == @["1", "2", "3"] # - If the pattern has a capture in it, it is added after the string # split: - doAssert "12".split(re"(\d)") == @["", "1", "", "2", ""] + assert "12".split(re"(\d)") == @["", "1", "", "2", ""] # - If `maxsplit != -1`, then the string will only be split # `maxsplit - 1` times. This means that there will be `maxsplit` # strings in the output seq. - doAssert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] + assert "1.2.3".split(re"\.", maxsplit = 2) == @["1", "2.3"] result = @[] var lastIdx = start @@ -747,9 +728,9 @@ proc escapeRe*(str: string): string {.gcsafe.} = ## ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -` runnableExamples: - doAssert escapeRe("fly+wind") == "fly\\+wind" - doAssert escapeRe("!") == "\\!" - doAssert escapeRe("nim*") == "nim\\*" + assert escapeRe("fly+wind") == "fly\\+wind" + assert escapeRe("!") == "\\!" + assert escapeRe("nim*") == "nim\\*" #([\\+*?[^\]$(){}=!<>|:-]) const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(', From 6563a685c1076cb5342f65d192b8f2a983220dba Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 3 Aug 2021 13:16:33 -0300 Subject: [PATCH 0305/1654] Documentation only, add 1 example (#18633) --- doc/manual_experimental.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 22db184a8..3272a4d0d 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1157,6 +1157,24 @@ the ordinary AST predicates: a = b inc a, c +Another example: + +.. code-block:: nim + proc somefunc(s: string) = assert s == "variable" + proc somefunc(s: string{nkStrLit}) = assert s == "literal" + proc somefunc(s: string{nkRStrLit}) = assert s == r"raw" + proc somefunc(s: string{nkTripleStrLit}) = assert s == """triple""" + proc somefunc(s: static[string]) = assert s == "constant" + + # Use parameter constraints to provide overloads based on both the input parameter type and form. + var variable = "variable" + somefunc(variable) + const constant = "constant" + somefunc(constant) + somefunc("literal") + somefunc(r"raw") + somefunc("""triple""") + Pattern operators ----------------- From 2cddf7fc96a01bcd0f6029f38de005314333daca Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 8 Aug 2021 14:26:34 -0300 Subject: [PATCH 0306/1654] Documentation only, add 1 example (#18621) * ReSync with Devel * ReSync * Documentation only, add 1 example to For loop macro * Flip it * Update doc/manual.rst Co-authored-by: Andreas Rumpf Co-authored-by: Andreas Rumpf --- doc/manual.rst | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 1b65ae9ee..41ed49715 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5830,6 +5830,27 @@ For loop macro A macro that takes as its only input parameter an expression of the special type `system.ForLoopStmt` can rewrite the entirety of a `for` loop: +.. code-block:: nim + :test: "nim c $1" + + import std/macros + + macro example(loop: ForLoopStmt) = + result = newTree(nnkForStmt) # Create a new For loop. + result.add loop[^3] # This is "item". + result.add loop[^2][^1] # This is "[1, 2, 3]". + result.add newCall(bindSym"echo", loop[0]) + + for item in example([1, 2, 3]): discard + +Expands to: + +.. code-block:: nim + for item in items([1, 2, 3]): + echo item + +Another example: + .. code-block:: nim :test: "nim c $1" @@ -5866,7 +5887,6 @@ type `system.ForLoopStmt` can rewrite the entirety of a `for` loop: echo a, " ", b - Special Types ============= From 9269a72f6f6f19e4bcffce82ad47c68755e7a696 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 9 Aug 2021 01:27:43 +0800 Subject: [PATCH 0307/1654] [nre]fix #17129 (#18632) * fix #17129 * correct * give reference implementaion links * add comment * typo * I'm conservative * change --- lib/impure/nre.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 01ef77427..f346d84cb 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -539,7 +539,7 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R ## Variants: ## ## - `proc findAll(...)` returns a `seq[string]` - # see pcredemo for explanation + # see pcredemo for explanation => https://www.pcre.org/original/doc/html/pcredemo.html let matchesCrLf = pattern.matchesCrLf() let unicode = uint32(getinfo[culong](pattern, pcre.INFO_OPTIONS) and pcre.UTF8) > 0u32 @@ -560,7 +560,7 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R # either the end of the input or the string # cannot be split here - we also need to bail # if we've never matched and we've already tried to... - if offset >= strlen or neverMatched: + if flags == 0 or offset >= strlen or neverMatched: # All matches found break if matchesCrLf and offset < (str.len - 1) and @@ -576,7 +576,6 @@ iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): R else: neverMatched = false offset = match.get.matchBounds.b + 1 - yield match.get proc find*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] = From 24445d31b394a120a8571c0d9dc4ab07b7672cd7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 8 Aug 2021 10:28:49 -0700 Subject: [PATCH 0308/1654] improve several tests in testament (#18635) * silence error output from template_various.nim * any => auto in tests * avoid showing failed for parseSpec since this is expected behavior in 2 cases: tincludefile.nim, tnav1.nim * enforce InheritFromException * fixup --- testament/categories.nim | 2 +- tests/config.nims | 1 + tests/generics/mdotlookup.nim | 4 +- tests/iter/tyieldintry.nim | 60 +++++++++++++--------------- tests/overload/toverload_various.nim | 2 +- tests/template/template_various.nim | 6 ++- tests/template/tparams_gensymed.nim | 2 +- 7 files changed, 38 insertions(+), 39 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index b5e8a5454..6375105ee 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -588,7 +588,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string, options: st except ValueError: # e.g. for `tests/navigator/tincludefile.nim` which have multiple # specs; this will be handled elsewhere - echo "parseSpec failed for: '$1', assuming this will be handled outside of megatest" % file + echo "parseSpec raised ValueError for: '$1', assuming this will be handled outside of megatest" % file continue if isJoinableSpec(spec): specs.add spec diff --git a/tests/config.nims b/tests/config.nims index f876cb612..d7f6ae7f9 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -28,6 +28,7 @@ hint("Processing", off) # switch("define", "nimTestsEnableFlaky") # switch("hint", "ConvFromXtoItselfNotNeeded") +# switch("warningAsError", "InheritFromException") # would require fixing a few tests # experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 # sync with `kochdocs.docDefines` or refactor. diff --git a/tests/generics/mdotlookup.nim b/tests/generics/mdotlookup.nim index 470fa0a51..3112c133f 100644 --- a/tests/generics/mdotlookup.nim +++ b/tests/generics/mdotlookup.nim @@ -1,9 +1,9 @@ -proc baz(o: any): int = 5 # if bar is exported, it works +proc baz(o: auto): int = 5 # if bar is exported, it works type MyObj = object x: int -proc foo*(b: any) = +proc foo*(b: auto) = var o: MyObj echo b.baz, " ", o.x.baz, " ", b.baz() diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim index 3d5260800..9862fe1de 100644 --- a/tests/iter/tyieldintry.nim +++ b/tests/iter/tyieldintry.nim @@ -1,17 +1,15 @@ discard """ -targets: "c" -output: "ok" + targets: "c cpp" """ -var closureIterResult = newSeq[int]() -# XXX Investigate why this fails now for 'nim cpp' +var closureIterResult = newSeq[int]() proc checkpoint(arg: int) = closureIterResult.add(arg) type - TestException = object of Exception - AnotherException = object of Exception + TestError = object of CatchableError + AnotherError = object of CatchableError proc testClosureIterAux(it: iterator(): int, exceptionExpected: bool, expectedResults: varargs[int]) = closureIterResult.setLen(0) @@ -21,7 +19,7 @@ proc testClosureIterAux(it: iterator(): int, exceptionExpected: bool, expectedRe try: for i in it(): closureIterResult.add(i) - except TestException: + except TestError: exceptionCaught = true if closureIterResult != @expectedResults or exceptionCaught != exceptionExpected: @@ -39,8 +37,8 @@ proc test(it: iterator(): int, expectedResults: varargs[int]) = proc testExc(it: iterator(): int, expectedResults: varargs[int]) = testClosureIterAux(it, true, expectedResults) -proc raiseException() = - raise newException(TestException, "Test exception!") +proc raiseTestError() = + raise newException(TestError, "Test exception!") block: iterator it(): int {.closure.} = @@ -58,8 +56,8 @@ block: yield 0 try: checkpoint(1) - raiseException() - except TestException: + raiseTestError() + except TestError: checkpoint(2) yield 3 checkpoint(4) @@ -89,7 +87,7 @@ block: yield 0 try: yield 1 - raiseException() + raiseTestError() yield 2 finally: checkpoint(3) @@ -103,8 +101,8 @@ block: iterator it(): int {.closure.} = try: try: - raiseException() - except AnotherException: + raiseTestError() + except AnotherError: yield 123 finally: checkpoint(3) @@ -117,8 +115,8 @@ block: iterator it(): int {.closure.} = try: yield 1 - raiseException() - except AnotherException: + raiseTestError() + except AnotherError: checkpoint(123) finally: checkpoint(2) @@ -134,12 +132,12 @@ block: yield 1 try: yield 2 - raiseException() - except AnotherException: + raiseTestError() + except AnotherError: yield 123 finally: yield 3 - except AnotherException: + except AnotherError: yield 124 finally: yield 4 @@ -170,10 +168,10 @@ block: try: try: yield 0 - raiseException() + raiseTestError() finally: checkpoint(1) - except TestException: + except TestError: yield 2 return finally: @@ -188,10 +186,10 @@ block: try: try: yield 0 - raiseException() + raiseTestError() finally: return # Return in finally should stop exception propagation - except AnotherException: + except AnotherError: yield 2 return finally: @@ -228,9 +226,9 @@ block: var foo = 123 let i = try: yield 0 - raiseException() + raiseTestError() 1 - except TestException as e: + except TestError as e: assert(e.msg == "Test exception!") case foo of 1: @@ -262,9 +260,9 @@ block: template tryexcept: int = try: yield 1 - raiseException() + raiseTestError() 123 - except TestException: + except TestError: yield 2 checkpoint(3) 4 @@ -323,11 +321,11 @@ block: for i in 0 .. 1: try: yield 1 - raiseException() - except TestException as e: + raiseTestError() + except TestError as e: doAssert(e.msg == "Test exception!") yield 2 - except AnotherException: + except AnotherError: yield 123 except: yield 1234 @@ -497,5 +495,3 @@ block: #17849 - yield in case subject yield 5 test(it, 1, 2, 13, 5) - -echo "ok" diff --git a/tests/overload/toverload_various.nim b/tests/overload/toverload_various.nim index f913ce3ee..55a8d17e6 100644 --- a/tests/overload/toverload_various.nim +++ b/tests/overload/toverload_various.nim @@ -395,7 +395,7 @@ block: template bar2[F,T](x: FooUn[F,T]): int = 1 template bar2[F,T1,T2](x: FooBi[F,T1,T2]): int = 2 - proc test(x: any, n: int) = + proc test(x: auto, n: int) = doAssert(foo1(x) == n) doAssert(foo2(x) == n) doAssert(bar1(x) == n) diff --git a/tests/template/template_various.nim b/tests/template/template_various.nim index 75226ea2d..e7a2be748 100644 --- a/tests/template/template_various.nim +++ b/tests/template/template_various.nim @@ -94,7 +94,7 @@ block generic_templates: var i3: int = t1[int]("xx") - +from strutils import contains block tgetast_typeliar: proc error(s: string) = quit s @@ -111,7 +111,9 @@ block tgetast_typeliar: var message : NimNode = newLit(condition.repr) # echo message result = getAst assertOrReturn2(condition, message) - echo result.repr + # echo result.repr + let s = result.repr + doAssert """error("Assertion failed:""" in s proc point(size: int16): tuple[x, y: int16] = # returns random point in square area with given `size` diff --git a/tests/template/tparams_gensymed.nim b/tests/template/tparams_gensymed.nim index 1444667f4..b68d7e253 100644 --- a/tests/template/tparams_gensymed.nim +++ b/tests/template/tparams_gensymed.nim @@ -72,7 +72,7 @@ proc concreteProc(x: int) = forStatic i, 0..3: echo i -proc genericProc(x: any) = +proc genericProc(x: auto) = forStatic i, 0..3: echo i From eb19db6595846c92e5eb999d96605fe4bb0934f7 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Sun, 8 Aug 2021 10:30:51 -0700 Subject: [PATCH 0309/1654] Nim pretty and add void (#18652) * pretty and add void * distro fixup * Update lib/pure/distros.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * tab to space * requested change to be cleaner Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/distros.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index 7a9910c1e..ddf0fd30c 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -109,7 +109,7 @@ type Clonezilla SteamOS Absolute - NixOS ## NixOS or a Nix build environment + NixOS ## NixOS or a Nix build environment AUSTRUMI Arya Porteus @@ -124,6 +124,7 @@ type ExTiX Rockstor GoboLinux + Void BSD FreeBSD @@ -134,7 +135,9 @@ type const - LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, Distribution.ArchLinux, Distribution.Artix} + LacksDevPackages* = {Distribution.Gentoo, Distribution.Slackware, + Distribution.ArchLinux, Distribution.Artix, Distribution.Antergos, + Distribution.BlackArch, Distribution.ArchBang} # we cache the result of the 'cmdRelease' # execution for faster platform detections. @@ -153,7 +156,8 @@ template hostnamectl(): untyped = cmdRelease("hostnamectl", hostnamectlRes) proc detectOsWithAllCmd(d: Distribution): bool = let dd = toLowerAscii($d) result = dd in toLowerAscii(osReleaseID()) or dd in toLowerAscii(release()) or - dd in toLowerAscii(uname()) or ("operating system: " & dd) in toLowerAscii(hostnamectl()) + dd in toLowerAscii(uname()) or ("operating system: " & dd) in + toLowerAscii(hostnamectl()) proc detectOsImpl(d: Distribution): bool = case d @@ -173,9 +177,9 @@ proc detectOsImpl(d: Distribution): bool = case d of Distribution.Gentoo: result = ("-" & $d & " ") in uname() - of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, Distribution.Fedora, - Distribution.OpenMandriva, Distribution.CentOS, Distribution.Alpine, - Distribution.Mageia, Distribution.Zorin: + of Distribution.Elementary, Distribution.Ubuntu, Distribution.Debian, + Distribution.Fedora, Distribution.OpenMandriva, Distribution.CentOS, + Distribution.Alpine, Distribution.Mageia, Distribution.Zorin, Distribution.Void: result = toLowerAscii($d) in osReleaseID() of Distribution.RedHat: result = "rhel" in osReleaseID() @@ -254,6 +258,8 @@ proc foreignDepInstallCmd*(foreignPackageName: string): (string, bool) = result = ("rpm -ivh " & p, true) elif detectOs(ArchLinux) or detectOs(Manjaro) or detectOs(Artix): result = ("pacman -S " & p, true) + elif detectOs(Void): + result = ("xbps-install " & p, true) else: result = (" install " & p, true) elif defined(haiku): From 0d3af5454b66dfe4a8a7017030708a890207c658 Mon Sep 17 00:00:00 2001 From: Christian Ulrich Date: Sun, 8 Aug 2021 20:11:07 +0200 Subject: [PATCH 0310/1654] Only allow IPv4 literals in strict form (#18656) * Only allow IPv4 literals in strict form The strict form as defined in RFC 6943, section 3.1.1 only allows the dotted form ddd.ddd.ddd.ddd of IPv4 literals, where ddd is a one to three digit decimal number between 0 and 255. Until now octal numbers (with a leading zero) were interpreted as decimal numbers which has security implications, see CVE-2021-29922 and CVE-2021-29923. * Update lib/pure/net.nim Co-authored-by: Dominik Picheta --- lib/pure/net.nim | 25 +++++++++-- tests/stdlib/tnet.nim | 35 +++++++++++++++ tests/stdlib/tparseipv6.nim | 90 ++++++++++++++++++++----------------- 3 files changed, 106 insertions(+), 44 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c0f9e1465..09cdc8afb 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -281,14 +281,20 @@ proc parseIPv4Address(addressStr: string): IpAddress = byteCount = 0 currentByte: uint16 = 0 separatorValid = false + leadingZero = false result = IpAddress(family: IpAddressFamily.IPv4) for i in 0 .. high(addressStr): if addressStr[i] in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers are not allowed") currentByte = currentByte * 10 + cast[uint16](ord(addressStr[i]) - ord('0')) - if currentByte > 255'u16: + if currentByte == 0'u16: + leadingZero = true + elif currentByte > 255'u16: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -300,6 +306,7 @@ proc parseIPv4Address(addressStr: string): IpAddress = currentByte = 0 byteCount.inc separatorValid = false + leadingZero = false else: raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -388,10 +395,16 @@ proc parseIPv6Address(addressStr: string): IpAddress = result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) groupCount.inc() else: # Must parse IPv4 address + var leadingZero = false for i, c in addressStr[v4StartPos..high(addressStr)]: if c in strutils.Digits: # Character is a number + if leadingZero: + raise newException(ValueError, + "Invalid IP address. Octal numbers not allowed") currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: + if currentShort == 0'u32: + leadingZero = true + elif currentShort > 255'u32: raise newException(ValueError, "Invalid IP Address. Value is out of range") separatorValid = true @@ -402,6 +415,7 @@ proc parseIPv6Address(addressStr: string): IpAddress = currentShort = 0 byteCount.inc() separatorValid = false + leadingZero = false else: # Invalid character raise newException(ValueError, "Invalid IP Address. Address contains an invalid character") @@ -431,7 +445,12 @@ proc parseIPv6Address(addressStr: string): IpAddress = proc parseIpAddress*(addressStr: string): IpAddress = ## Parses an IP address - ## Raises ValueError on error + ## + ## Raises ValueError on error. + ## + ## For IPv4 addresses, only the strict form as + ## defined in RFC 6943 is considered valid, see + ## https://datatracker.ietf.org/doc/html/rfc6943#section-3.1.1. if addressStr.len == 0: raise newException(ValueError, "IP Address string is empty") if addressStr.contains(':'): diff --git a/tests/stdlib/tnet.nim b/tests/stdlib/tnet.nim index b19d31f6c..671e0f7bc 100644 --- a/tests/stdlib/tnet.nim +++ b/tests/stdlib/tnet.nim @@ -50,6 +50,41 @@ block: # parseIpAddress tests expect(ValueError): discard parseIpAddress("gggg:cdba:0000:0000:0000:0000:3257:9652") + block: # ipv4-compatible ipv6 address (embedded ipv4 address) + check parseIpAddress("::ffff:10.0.0.23") == parseIpAddress("::ffff:0a00:0017") + + block: # octal number in ipv4 address + expect(ValueError): + discard parseIpAddress("010.8.8.8") + expect(ValueError): + discard parseIpAddress("8.010.8.8") + + block: # hexadecimal number in ipv4 address + expect(ValueError): + discard parseIpAddress("0xc0.168.0.1") + expect(ValueError): + discard parseIpAddress("192.0xa8.0.1") + + block: # less than 4 numbers in ipv4 address + expect(ValueError): + discard parseIpAddress("127.0.1") + + block: # octal number in embedded ipv4 address + expect(ValueError): + discard parseIpAddress("::ffff:010.8.8.8") + expect(ValueError): + discard parseIpAddress("::ffff:8.010.8.8") + + block: # hexadecimal number in embedded ipv4 address + expect(ValueError): + discard parseIpAddress("::ffff:0xc0.168.0.1") + expect(ValueError): + discard parseIpAddress("::ffff:192.0xa8.0.1") + + block: # less than 4 numbers in embedded ipv4 address + expect(ValueError): + discard parseIpAddress("::ffff:127.0.1") + block: # "IpAddress/Sockaddr conversion" proc test(ipaddrstr: string) = var ipaddr_1 = parseIpAddress(ipaddrstr) diff --git a/tests/stdlib/tparseipv6.nim b/tests/stdlib/tparseipv6.nim index 3e1c23e58..dd9abc511 100644 --- a/tests/stdlib/tparseipv6.nim +++ b/tests/stdlib/tparseipv6.nim @@ -8,23 +8,23 @@ const positives = [ "::f:8:a8f:218.17.235.229", "::b:228.19.241.2", - "::8:c:a:f:8.35.8.096", - "::3:e:a:bc:04.19.2.9", + "::8:c:a:f:8.35.8.96", + "::3:e:a:bc:4.19.2.9", "::2:212.242.248.19", "::df:a5f:3.250.208.9", "::8:c:5:e63:250.208.249.0", "::b:f:181.12.9.98", - "::a:f8:77.08.243.232", - "::a:b:85:e4d9:252.9.229.056", + "::a:f8:77.8.243.232", + "::a:b:85:e4d9:252.9.229.56", "941:c:8a:c:e::917", "e8:7a:e:ad:88a:8:203.235.225.46", - "139c:9e::f8:254.08.21.249", + "139c:9e::f8:254.8.21.249", "b38:f0:e::f9:89.6.12.18", "ef::8", "5::ab", "a::8:255.247.96.253", "b:c0::c:254.248.95.254", - "::8c:2:99.251.024.3", + "::8c:2:99.251.24.3", "98::c:247.240.249.57", "9::9", "628::f1ed:f", @@ -106,68 +106,68 @@ const "::315", "::a:a", "::aed3:a", - "f0eb:0:e8:b:c:a:254.098.233.17", + "f0eb:0:e8:b:c:a:254.98.233.17", "bfa:7fc:c66d:15:e9a:ded:254.119.9.9", - "d:ffa8:9:a:879:3:202.39.08.245", + "d:ffa8:9:a:879:3:202.39.8.245", "8e:2:8:fa8a:f1d1:1aa8:252.254.245.81", - "5:d4:a:e9:8:8:06.38.98.253", - "9c5:4:a5c:f:a6:8c9d:05.250.8.2", + "5:d4:a:e9:8:8:6.38.98.253", + "9c5:4:a5c:f:a6:8c9d:5.250.8.2", "d19a:2:f808:be:f:c:98.86.197.249", - "8:26ac:8:8:cb:f:242.00.254.85", + "8:26ac:8:8:cb:f:242.0.254.85", "38:e:1:0b88:f:0:8.89.248.92", - "e7:ff96:a:f:f:b:253.91.052.195", + "e7:ff96:a:f:f:b:253.91.52.195", "d:8:2:5:894:5:254.0.240.199", "2:98:9:8aa:9c8f:fa:252.98.248.17", - "e9:d4f:890:ccbe:5:8:088.200.228.216", + "e9:d4f:890:ccbe:5:8:88.200.228.216", "3:3:9:5:6a:df5:255.251.8.12", - "0280:3:8:8:4:9:255.000.251.249", + "0280:3:8:8:4:9:255.0.251.249", "8:af7:db:aa:0:9:238.248.250.255", - "ff:ee:9a:9252:a:289:059.083.18.255", + "ff:ee:9a:9252:a:289:59.83.18.255", "9f6:5:fc9:b:a89:a:142.1.250.254", "e:981a:da:bf94:9:f8:254.242.18.95", "3c:1:4:f2:89:f:8.91.255.14", - "e::9a2:c:9.050.80.8", + "e::9a2:c:9.50.80.8", "9::4a:07:fb:211.241.254.228", "9be::2:e:215.189.48.188", - "f::f:d:069.148.99.168", + "f::f:d:69.148.99.168", "f::a:97.18.240.47", "c::a98e:1:251.253.252.254", "668::82:214.87.208.9", "9c0::cf0:ecb:253.208.238.255", - "a::0:f1:210.240.238.049", - "8::a:1:251.238.34.09", + "a::0:f1:210.240.238.49", + "8::a:1:251.238.34.9", "81:dfe::b8:8.255.249.248", - "d3::7:b:9:83.189.08.244", - "8::9:8:8:00.7.11.252", + "d3::7:b:9:83.189.8.244", + "8::9:8:8:0.7.11.252", "2:8::c:a8:250.221.9.249", "2::f:99.8.249.247", "c:22f5::5:2c:243.15.79.89", "e:8e::da:251.243.255.2", "f15f:9::a:255.70.247.218", - "f:b::9f38:31.220.94.022", - "9::9a48:03.98.249.119", + "f:b::9f38:31.220.94.22", + "9::9a48:3.98.249.119", "d:d:9b87::2d:a:249.253.38.8", "d86d:99b::a9b:5:242.236.8.244", "eb:3::f:9cf:1.253.1.228", "b::ba2:255.247.114.64", - "2f:ec:bcb::9:219.254.250.094", + "2f:ec:bcb::9:219.254.250.94", "da8a:f6::a:e0:19.251.241.251", - "5e:c1::a:021.250.8.254", + "5e:c1::a:21.250.8.254", "c:9::8c9b:248.219.212.252", "2:a::8d4a:216.255.198.223", - "1f::66:255.30.08.150", + "1f::66:255.30.8.150", "bc2b:8f::2ff9:6.245.99.230", "a:8::a8:9.251.246.255", - "f:7:7::98:06.14.1.208", + "f:7:7::98:6.14.1.208", "e:2::9:218.249.255.254", "79:f::6:250.255.98.246", - "47:9:fb9f::9:038.136.17.251", + "47:9:fb9f::9:38.136.17.251", "ed::a:247.9.23.239", "6f::f1:88.254.119.9", "a::d:218.199.236.0", - "fc88::9:203.196.04.95", - "::8.048.255.85", - "::253.07.255.36", + "fc88::9:203.196.4.95", + "::8.48.255.85", + "::253.7.255.36", "9:d::253.7.178.229", "::250.84.158.253", "::8.55.204.248", @@ -175,31 +175,39 @@ const "df9:88ca::248.255.108.17", "8e9b::250.206.0.82", "::209.8.254.209", - "::247.088.8.8", + "::247.88.8.8", "::cb:f:ba41:250.208.19.249", "::fe:0e8:243.240.229.5", "::c:223.251.5.226", - "::8:08.03.8.250", + "::8:8.3.8.250", "::f:8.88.11.255", - "::fda:48:aa:05.189.07.2", - "::8:c3f:f:240.06.212.255", + "::fda:48:aa:5.189.7.2", + "::8:c3f:f:240.6.212.255", "::f:0aa:244.123.99.16", - "::c9b5:c:034.8.090.196", + "::c9b5:c:34.8.90.196", "::98:c9:254.14.241.81" ] negatives = ["foo.bar", "::::::::::::", "yet another failure", - "de:6:c:ab5:6a::9:252.06.06.249", - "f9:5f7:fa38:9:b::b6:09.255.248.252", + "de:6:c:ab5:6a::9:252.6.6.249", + "f9:5f7:fa38:9:b::b6:9.255.248.252", "97:c:5b:81:8a::f5dd:144.252.250.9", - "9:8:cd:8:a9::f:247.255.09.255", + "9:8:cd:8:a9::f:247.255.9.255", "18:1:8c:2:3::9:8.254.252.139", - "e:c298:3:e:a::bb12:254.246.05.250", + "e:c298:3:e:a::bb12:254.246.5.250", "e:e:c:8e:fd::8:253.8.49.231", "9:97f:f:e929:8a::c9:0.8.252.10", - "0df:b24:7:89:c::2b:16.249.240.092", + "0df:b24:7:89:c::2b:16.249.240.92", "b:8f5f:485:c:9a::84c:178.7.249.34", + "::3:e:a:bc:091.19.2.9", + "::a:f8:77.08.243.232", + "::8c:2:99.251.029.3", + "::8:c:a:f:8.35.8.096", + "d:ffa8:9:a:879:3:0202.39.8.245", + "139c:9e::f8:254.07.21.249", + "f0eb:0:e8:b:c:a:254.233.043.17", + "::a:b:85:e4d9:252.9.229.056", ] proc ok(pos: openarray[string]) = From a7e622267e1049a454f4f522c0dbce5eb4017495 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 9 Aug 2021 02:44:14 +0800 Subject: [PATCH 0311/1654] use lent (#18638) --- lib/system/excpt.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 4814b77b4..3e520570f 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -521,10 +521,10 @@ proc getStackTrace(e: ref Exception): string = else: result = "" -proc getStackTraceEntries*(e: ref Exception): seq[StackTraceEntry] = +proc getStackTraceEntries*(e: ref Exception): lent seq[StackTraceEntry] = ## Returns the attached stack trace to the exception `e` as ## a `seq`. This is not yet available for the JS backend. - shallowCopy(result, e.trace) + e.trace proc getStackTraceEntries*(): seq[StackTraceEntry] = ## Returns the stack trace entries for the current stack trace. From 988c02b7cbef634523b87af04097d06ed453dd61 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 8 Aug 2021 19:24:45 -0700 Subject: [PATCH 0312/1654] renamed: lib/std/private/vmutils.nim -> lib/std/private/bitops_utils.nim to avoid confusion with unrelated std/vmutils (#18660) --- lib/pure/bitops.nim | 4 +--- lib/std/private/{vmutils.nim => bitops_utils.nim} | 0 lib/system/countbits_impl.nim | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) rename lib/std/private/{vmutils.nim => bitops_utils.nim} (100%) diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 6d3865e5d..1b3838621 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -27,9 +27,7 @@ import macros import std/private/since -from std/private/vmutils import forwardImpl, toUnsigned - - +from std/private/bitops_utils import forwardImpl, toUnsigned func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. diff --git a/lib/std/private/vmutils.nim b/lib/std/private/bitops_utils.nim similarity index 100% rename from lib/std/private/vmutils.nim rename to lib/std/private/bitops_utils.nim diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 184f97bd4..020423e01 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -9,7 +9,7 @@ ## Contains the used algorithms for counting bits. -from std/private/vmutils import forwardImpl, toUnsigned +from std/private/bitops_utils import forwardImpl, toUnsigned const useBuiltins* = not defined(noIntrinsicsBitOpts) From 8ce782d463b0bca091be5a34db190dc441c23ce4 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 8 Aug 2021 23:51:39 -0700 Subject: [PATCH 0313/1654] Add optional recursive arg to distinctBase (v2) (#18659) * Add optional recursive arg to distinctBase * Add docs and examples Co-authored-by: ALANVF --- compiler/semmagic.nim | 5 +++-- lib/pure/typetraits.nim | 17 ++++++++++++----- lib/std/jsonutils.nim | 4 ++-- lib/system/repr_v2.nim | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 04f4c4729..87f7da815 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -190,9 +190,10 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym) result = newIntNodeT(toInt128(operand.len), traitCall, c.idgen, c.graph) of "distinctBase": var arg = operand.skipTypes({tyGenericInst}) + let rec = semConstExpr(c, traitCall[2]).intVal != 0 while arg.kind == tyDistinct: - arg = arg.base - arg = arg.skipTypes(skippedTypes + {tyGenericInst}) + arg = arg.base.skipTypes(skippedTypes + {tyGenericInst}) + if not rec: break result = getTypeDescNode(c, arg, operand.owner, traitCall.info) else: localError(c.config, traitCall.info, "unknown trait: " & s) diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index e63585f63..8dc1b8cf2 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -110,25 +110,32 @@ template pointerBase*[T](_: typedesc[ptr T | ref T]): typedesc = assert (var s = "abc"; s[0].addr).typeof.pointerBase is char T -proc distinctBase*(T: typedesc): typedesc {.magic: "TypeTrait".} = +proc distinctBase*(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} = ## Returns the base type for distinct types, or the type itself otherwise. + ## If `recursive` is false, only the immediate distinct base will be returned. ## ## **See also:** - ## * `distinctBase template <#distinctBase.t,T>`_ + ## * `distinctBase template <#distinctBase.t,T,static[bool]>`_ runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert distinctBase(MyInt) is int + doAssert distinctBase(MyOtherInt) is int + doAssert distinctBase(MyOtherInt, false) is MyInt doAssert distinctBase(int) is int since (1, 1): - template distinctBase*[T](a: T): untyped = - ## Overload of `distinctBase <#distinctBase,typedesc>`_ for values. + template distinctBase*[T](a: T, recursive: static bool = true): untyped = + ## Overload of `distinctBase <#distinctBase,typedesc,static[bool]>`_ for values. runnableExamples: type MyInt = distinct int + type MyOtherInt = distinct MyInt doAssert 12.MyInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase == 12 + doAssert 12.MyOtherInt.distinctBase(false) is MyInt doAssert 12.distinctBase == 12 when T is distinct: - distinctBase(typeof(a))(a) + distinctBase(typeof(a), recursive)(a) else: # avoids hint ConvFromXtoItselfNotNeeded a diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index c141985d3..727a75288 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -78,8 +78,8 @@ proc initToJsonOptions*(): ToJsonOptions = ## initializes `ToJsonOptions` with sane options. ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef) -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} -template distinctBase[T](a: T): untyped = distinctBase(typeof(a))(a) +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} +template distinctBase[T](a: T, recursive: static bool = true): untyped = distinctBase(typeof(a), recursive)(a) macro getDiscriminants(a: typedesc): seq[string] = ## return the discriminant keys diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index 8471ea148..f99f09799 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -1,7 +1,7 @@ proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## imported from typetraits -proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".} +proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: "TypeTrait".} ## imported from typetraits proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} From 31fc0f97187cef472cf113629a90c7aa126d3693 Mon Sep 17 00:00:00 2001 From: Kyle Brown Date: Tue, 10 Aug 2021 16:26:42 -0700 Subject: [PATCH 0314/1654] Remove unused imports, and deprecated function usage (#18663) * clean up imports and slice to remove delete * revert buggy code * Replace "delete" with setlen to remove depreciation warning --- compiler/docgen.nim | 4 ++-- compiler/jsgen.nim | 2 +- tools/atlas/atlas.nim | 2 +- tools/atlas/packagesjson.nim | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 3bc01bbe6..d39a20aae 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -835,7 +835,7 @@ proc docstringSummary(rstText: string): string = result = rstText.substr(2).strip var pos = result.find('\L') if pos > 0: - result.delete(pos, result.len - 1) + result.setLen(pos - 1) result.add("…") if pos < maxDocstringChars: return @@ -843,7 +843,7 @@ proc docstringSummary(rstText: string): string = pos = result.find({'.', ',', ':'}) let last = result.len - 1 if pos > 0 and pos < last: - result.delete(pos, last) + result.setLen(pos - 1) result.add("…") proc genDeprecationMsg(d: PDoc, n: PNode): string = diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 85947535e..49a57b4a6 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -29,7 +29,7 @@ implements the required case distinction. import - ast, astalgo, trees, magicsys, options, + ast, trees, magicsys, options, nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index d0990c1f7..761ca5db8 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -9,7 +9,7 @@ ## Simple tool to automate frequent workflows: Can "clone" ## a Nimble dependency and its dependencies recursively. -import std/[parseopt, strutils, os, osproc, sequtils, unicode, tables, sets] +import std/[parseopt, strutils, os, osproc, unicode, tables, sets] import parse_requires, osutils, packagesjson const diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim index ed83ce85c..0b8599769 100644 --- a/tools/atlas/packagesjson.nim +++ b/tools/atlas/packagesjson.nim @@ -1,6 +1,5 @@ import std / [json, os, sets, strutils] -import osutils type Package* = ref object From d0dd923acf8c2ddfa5e0ea7574c8d5200416e9f5 Mon Sep 17 00:00:00 2001 From: RSDuck Date: Wed, 11 Aug 2021 08:54:49 +0200 Subject: [PATCH 0315/1654] Switch maintanance (#18668) * Fix and improve Nintendo Switch support * Document the necessity for nimAllocPagesViaMalloc * update changelog * Use --gc:orc in examples --- changelog.md | 3 + doc/nimc.rst | 21 ++--- lib/nintendoswitch/switch_memory.nim | 36 --------- lib/pure/os.nim | 9 +-- lib/system/osalloc.nim | 111 --------------------------- 5 files changed, 15 insertions(+), 165 deletions(-) delete mode 100644 lib/nintendoswitch/switch_memory.nim diff --git a/changelog.md b/changelog.md index a19697c8c..0e3c54d77 100644 --- a/changelog.md +++ b/changelog.md @@ -104,6 +104,8 @@ - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- The allocator for Nintendo Switch, which was nonfunctional because + of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. ## Standard library additions and changes @@ -363,6 +365,7 @@ - Added `dom.setInterval`, `dom.clearInterval` overloads. +- Allow reading parameters when compiling for Nintendo Switch. - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect if the slice is out of bounds, likewise with `strutils.delete`. diff --git a/doc/nimc.rst b/doc/nimc.rst index ed9077912..ad5ecaa76 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -406,16 +406,18 @@ to your usual `nim c`:cmd: or `nim cpp`:cmd: command and set the `passC`:option: and `passL`:option: command line switches to something like: .. code-block:: cmd - nim c ... --passC="-I$DEVKITPRO/libnx/include" ... + nim c ... --d:nimAllocPagesViaMalloc --gc:orc --passC="-I$DEVKITPRO/libnx/include" ... --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" or setup a ``nim.cfg`` file like so:: #nim.cfg + --gc:orc + --d:nimAllocPagesViaMalloc --passC="-I$DEVKITPRO/libnx/include" --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" -The DevkitPro setup must be the same as the default with their new installer +The devkitPro setup must be the same as the default with their new installer `here for Mac/Linux `_ or `here for Windows `_. @@ -426,20 +428,19 @@ For example, with the above-mentioned config: nim c --os:nintendoswitch switchhomebrew.nim This will generate a file called ``switchhomebrew.elf`` which can then be turned into -an nro file with the `elf2nro`:cmd: tool in the DevkitPro release. Examples can be found at +an nro file with the `elf2nro`:cmd: tool in the devkitPro release. Examples can be found at `the nim-libnx github repo `_. -There are a few things that don't work because the DevkitPro libraries don't support them. +There are a few things that don't work because the devkitPro libraries don't support them. They are: 1. Waiting for a subprocess to finish. A subprocess can be started, but right now it can't be waited on, which sort of makes subprocesses a bit hard to use -2. Dynamic calls. DevkitPro libraries have no dlopen/dlclose functions. -3. Command line parameters. It doesn't make sense to have these for a console - anyways, so no big deal here. -4. mqueue. Sadly there are no mqueue headers. -5. ucontext. No headers for these either. No coroutines for now :( -6. nl_types. No headers for this. +2. Dynamic calls. Switch OS (Horizon) doesn't support dynamic libraries, so dlopen/dlclose are not available. +3. mqueue. Sadly there are no mqueue headers. +4. ucontext. No headers for these either. No coroutines for now :( +5. nl_types. No headers for this. +6. As mmap is not supported, the nimAllocPagesViaMalloc option has to be used. DLL generation ============== diff --git a/lib/nintendoswitch/switch_memory.nim b/lib/nintendoswitch/switch_memory.nim deleted file mode 100644 index f34bd363a..000000000 --- a/lib/nintendoswitch/switch_memory.nim +++ /dev/null @@ -1,36 +0,0 @@ -## All of these library headers and source can be found in the github repo -## https://github.com/switchbrew/libnx. - -const virtMemHeader = "" -const svcHeader = "" -const mallocHeader = "" - -## Aligns a block of memory with request `size` to `bytes` size. For -## example, a request of memalign(0x1000, 0x1001) == 0x2000 bytes allocated -proc memalign*(bytes: csize, size: csize): pointer {.importc: "memalign", - header: mallocHeader.} - -# Should be required, but not needed now because of how -# svcUnmapMemory frees all memory -#proc free*(address: pointer) {.importc: "free", -# header: mallocHeader.} - -## Maps a memaligned block of memory from `src_addr` to `dst_addr`. The -## Nintendo Switch requires this call in order to make use of memory, otherwise -## an invalid memory access occurs. -proc svcMapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {. - importc: "svcMapMemory", header: svcHeader.} - -## Unmaps (frees) all memory from both `dst_addr` and `src_addr`. **Must** be called -## whenever svcMapMemory is used. The Switch will expect all memory to be allocated -## before gfxExit() calls () -proc svcUnmapMemory*(dst_addr: pointer; src_addr: pointer; size: uint64): uint32 {. - importc: "svcUnmapMemory", header: svcHeader.} - -proc virtmemReserveMap*(size: csize): pointer {.importc: "virtmemReserveMap", - header: virtMemHeader.} - -# Should be required, but not needed now because of how -# svcUnmapMemory frees all memory -#proc virtmemFreeMap*(address: pointer; size: csize) {.importc: "virtmemFreeMap", -# header: virtMemHeader.} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 73e3fcb31..fd618e93c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2910,13 +2910,6 @@ elif defined(nodejs): result = $argv[i] else: raise newException(IndexDefect, formatErrorIndexBound(i - 1, argv.len - 2)) -elif defined(nintendoswitch): - proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramStr is not implemented on Nintendo Switch") - - proc paramCount*(): int {.tags: [ReadIOEffect].} = - raise newException(OSError, "paramCount is not implemented on Nintendo Switch") - elif defined(windows): # Since we support GUI applications with Nim, we sometimes generate # a WinMain entry proc. But a WinMain proc has no access to the parsed @@ -3190,7 +3183,7 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW result = getApplAux("/proc/self/exe") elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") - elif defined(genode) or defined(nintendoswitch): + elif defined(genode): raiseOSError(OSErrorCode(-1), "POSIX command line not supported") elif defined(freebsd) or defined(dragonfly) or defined(netbsd): result = getApplFreebsd() diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 2830adb48..39bf65d6c 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -103,117 +103,6 @@ elif defined(emscripten) and not defined(StandaloneHeapSize): elif defined(genode) and not defined(StandaloneHeapSize): include genode/alloc # osAllocPages, osTryAllocPages, osDeallocPages -elif defined(nintendoswitch) and not defined(StandaloneHeapSize): - - import nintendoswitch/switch_memory - - type - PSwitchBlock = ptr NSwitchBlock - ## This will hold the heap pointer data in a separate - ## block of memory that is PageSize bytes above - ## the requested memory. It's the only good way - ## to pass around data with heap allocations - NSwitchBlock {.pure, inheritable.} = object - realSize: int - heap: pointer # pointer to main heap alloc - heapMirror: pointer # pointer to virtmem mapped heap - - proc alignSize(size: int): int {.inline.} = - ## Align a size integer to be in multiples of PageSize - ## The nintendo switch will not allocate memory that is not - ## aligned to 0x1000 bytes and will just crash. - (size + (PageSize - 1)) and not (PageSize - 1) - - proc deallocate(heapMirror: pointer, heap: pointer, size: int) = - # Unmap the allocated memory - discard svcUnmapMemory(heapMirror, heap, size.uint64) - # These should be called (theoretically), but referencing them crashes the switch. - # The above call seems to free all heap memory, so these are not needed. - # virtmemFreeMap(nswitchBlock.heapMirror, nswitchBlock.realSize.csize) - # free(nswitchBlock.heap) - - proc freeMem(p: pointer) = - # Retrieve the switch block data from the pointer we set before - # The data is located just sizeof(NSwitchBlock) bytes below - # the top of the pointer to the heap - let - nswitchDescrPos = cast[ByteAddress](p) -% sizeof(NSwitchBlock) - nswitchBlock = cast[PSwitchBlock](nswitchDescrPos) - - deallocate( - nswitchBlock.heapMirror, nswitchBlock.heap, nswitchBlock.realSize - ) - - proc storeHeapData(address, heapMirror, heap: pointer, size: int) {.inline.} = - ## Store data in the heap for deallocation purposes later - - # the position of our heap pointer data. Since we allocated PageSize extra - # bytes, we should have a buffer on top of the requested size of at least - # PageSize bytes, which is much larger than sizeof(NSwitchBlock). So we - # decrement the address by sizeof(NSwitchBlock) and use that address - # to store our pointer data - let nswitchBlockPos = cast[ByteAddress](address) -% sizeof(NSwitchBlock) - - # We need to store this in a pointer obj (PSwitchBlock) so that the data sticks - # at the address we've chosen. If NSwitchBlock is used here, the data will - # be all 0 when we try to retrieve it later. - var nswitchBlock = cast[PSwitchBlock](nswitchBlockPos) - nswitchBlock.realSize = size - nswitchBlock.heap = heap - nswitchBlock.heapMirror = heapMirror - - proc getOriginalHeapPosition(address: pointer, difference: int): pointer {.inline.} = - ## This function sets the heap back to the originally requested - ## size - let - pos = cast[int](address) - newPos = cast[ByteAddress](pos) +% difference - - return cast[pointer](newPos) - - template allocPages(size: int, outOfMemoryStmt: untyped): untyped = - # This is to ensure we get a block of memory the requested - # size, as well as space to store our structure - let realSize = alignSize(size + sizeof(NSwitchBlock)) - - let heap = memalign(PageSize, realSize) - - if heap.isNil: - outOfMemoryStmt - - let heapMirror = virtmemReserveMap(realSize.csize) - result = heapMirror - - let rc = svcMapMemory(heapMirror, heap, realSize.uint64) - # Any return code not equal 0 means an error in libnx - if rc.uint32 != 0: - deallocate(heapMirror, heap, realSize) - outOfMemoryStmt - - # set result to be the original size requirement - result = getOriginalHeapPosition(result, realSize - size) - - storeHeapData(result, heapMirror, heap, realSize) - - proc osAllocPages(size: int): pointer {.inline.} = - allocPages(size): - raiseOutOfMem() - - proc osTryAllocPages(size: int): pointer = - allocPages(size): - return nil - - proc osDeallocPages(p: pointer, size: int) = - # Note that in order for the Switch not to crash, a call to - # deallocHeap(runFinalizers = true, allowGcAfterwards = false) - # must be run before gfxExit(). The Switch requires all memory - # to be deallocated before the graphics application has exited. - # - # gfxExit() can be found in in the github - # repo https://github.com/switchbrew/libnx - when reallyOsDealloc: - freeMem(p) - elif defined(posix) and not defined(StandaloneHeapSize): const PROT_READ = 1 # page can be read From 854006575498704c42b5ac465fdc74c3efdb5b97 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Aug 2021 02:42:01 -0700 Subject: [PATCH 0316/1654] --cc:env now works correctly to assign linker executable, allowing to cross-compile/run for windows on osx via wine (#18672) * --cc:env now works correctly to assign linker executable, allowing using wine on osx * fixup --- compiler/extccomp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index a091060ff..e7e1bcd51 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -252,7 +252,7 @@ compiler envcc: buildGui: "", buildDll: " -shared ", buildLib: "", # XXX: not supported yet - linkerExe: "cc", + linkerExe: "", linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options", includeCmd: " -I", linkDirCmd: "", # XXX: not supported yet From 6c1bd4bb1cb7a6af38b5929fe02b069b03e39db4 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Aug 2021 03:17:17 -0700 Subject: [PATCH 0317/1654] fix: `var a{.foo.} = expr` inside templates (refs #15920) (except when `foo` is overloaded) (#13869) * fix: `var a{.foo.} = expr` inside templates * add test * improve tdecls test * improve tests * add failing test * PRTEMP * fixup --- compiler/semstmts.nim | 35 ++++++++++++---------- compiler/semtempl.nim | 15 ++++++++-- tests/pragmas/tpragmas_misc.nim | 52 +++++++++++++++++++++++++++++++++ tests/stdlib/tdecls.nim | 23 ++++++++++++++- tests/stdlib/tsince.nim | 2 +- 5 files changed, 107 insertions(+), 20 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index cc09291c5..14dc89781 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -453,23 +453,29 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode = return nil let nodePragma = b[1][0] # see: `singlePragma` - if nodePragma.kind notin {nkIdent, nkAccQuoted}: - return nil - let ident = considerQuotedIdent(c, nodePragma) - var userPragma = strTableGet(c.userPragmas, ident) - if userPragma != nil: return nil - - let w = nodePragma.whichPragma - if n.kind == nkVarSection and w in varPragmas or - n.kind == nkLetSection and w in letPragmas or - n.kind == nkConstSection and w in constPragmas: - return nil var amb = false - let sym = searchInScopes(c, ident, amb) - # XXX what if amb is true? - if sym == nil or sfCustomPragma in sym.flags: return nil + var sym: PSym = nil + case nodePragma.kind + of nkIdent, nkAccQuoted: + let ident = considerQuotedIdent(c, nodePragma) + var userPragma = strTableGet(c.userPragmas, ident) + if userPragma != nil: return nil + let w = nodePragma.whichPragma + if n.kind == nkVarSection and w in varPragmas or + n.kind == nkLetSection and w in letPragmas or + n.kind == nkConstSection and w in constPragmas: + return nil + sym = searchInScopes(c, ident, amb) + # XXX what if amb is true? + # CHECKME: should that test also apply to `nkSym` case? + if sym == nil or sfCustomPragma in sym.flags: return nil + of nkSym: + sym = nodePragma.sym + else: + return nil # skip if not in scope; skip `template myAttr() {.pragma.}` + let lhs = b[0] let clash = strTableGet(c.currentScope.symbols, lhs.ident) if clash != nil: @@ -477,7 +483,6 @@ proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode = wrongRedefinition(c, lhs.info, lhs.ident.s, clash.info) result = newTree(nkCall) - doAssert nodePragma.kind in {nkIdent, nkAccQuoted}, $nodePragma.kind result.add nodePragma result.add lhs if a[1].kind != nkEmpty: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index a2b0f99ba..2636784af 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -208,9 +208,18 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) = if (n.kind == nkPragmaExpr and n.len >= 2 and n[1].kind == nkPragma): let pragmaNode = n[1] for i in 0..= version: + body + when false: # bug + template fun3(): int {.since3: (1, 3).} = 12 + + block: # ditto, w + # case with overload + template since2(version: (int, int), body: untyped) {.dirty.} = + when (NimMajor, NimMinor) >= version: + body + template since2(version: (int, int, int), body: untyped) {.dirty.} = + when (NimMajor, NimMinor, NimPatch) >= version: + body + when false: # bug + template fun3(): int {.since2: (1, 3).} = 12 + +when true: # D20210801T100514:here + from macros import genSym + block: + template fn() = + var ret {.gensym.}: int # must special case template pragmas so it doesn't get confused + discard ret + fn() + static: discard genSym() diff --git a/tests/stdlib/tdecls.nim b/tests/stdlib/tdecls.nim index 3567639e0..53e070bee 100644 --- a/tests/stdlib/tdecls.nim +++ b/tests/stdlib/tdecls.nim @@ -1,6 +1,10 @@ +discard """ + targets: "c cpp js" +""" + import std/decls -block: +template fun() = var s = @[10,11,12] var a {.byaddr.} = s[0] a+=100 @@ -34,6 +38,13 @@ block: doAssert compiles(block: var b2 {.byaddr.}: int = s[2]) +proc fun2() = fun() +fun() +fun2() +static: fun2() +when false: # pending bug #13887 + static: fun() + ## We can define custom pragmas in user code template byUnsafeAddr(lhs, typ, expr) = when typ is type(nil): @@ -68,3 +79,13 @@ block: # nkAccQuoted let a {.`cast`.} = s[0] doAssert a == "foo" doAssert a[0].unsafeAddr == s[0][0].unsafeAddr + +block: # bug #15920 + template foo(lhs, typ, expr) = + let lhs = expr + proc fun1()= + let a {.foo.} = 1 + template fun2()= + let a {.foo.} = 1 + fun1() # ok + fun2() # BUG diff --git a/tests/stdlib/tsince.nim b/tests/stdlib/tsince.nim index 14dd09c15..d0320ff12 100644 --- a/tests/stdlib/tsince.nim +++ b/tests/stdlib/tsince.nim @@ -27,6 +27,6 @@ since (99, 3): doAssert false when false: - # pending https://github.com/timotheecour/Nim/issues/129 + # pending bug #15920 # Error: cannot attach a custom pragma to 'fun3' template fun3(): int {.since: (1, 3).} = 12 From bc14b7735929f764b09446073375174236fab161 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 11 Aug 2021 17:19:23 +0200 Subject: [PATCH 0318/1654] fixes #18665 DFA generator bug (#18676) --- compiler/dfa.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 402945042..65c483e9c 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -769,6 +769,12 @@ proc gen(c: var Con; n: PNode) = of nkCharLit..nkNilLit: discard of nkAsgn, nkFastAsgn: gen(c, n[1]) + + if n[0].kind in PathKinds0: + let a = c.skipTrivials(n[0]) + if a.kind in nkCallKinds: + gen(c, a) + # watch out: 'obj[i].f2 = value' sets 'f2' but # "uses" 'i'. But we are only talking about builtin array indexing so # it doesn't matter and 'x = 34' is NOT a usage of 'x'. From c94933acb7a8fe08e1ef479d02bd20732ea1ea23 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 11 Aug 2021 13:30:51 -0700 Subject: [PATCH 0319/1654] fix #18674 --nimcache now works better with --os:windows (#18675) --- compiler/commands.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/commands.nim b/compiler/commands.nim index 1c3de29ba..82e908152 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -537,6 +537,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.lazyPaths.keepItIf(it != path) of "nimcache": expectArg(conf, switch, arg, pass, info) + var arg = arg + # refs bug #18674, otherwise `--os:windows` messes up with `--nimcache` set + # in config nims files, e.g. via: `import os; switch("nimcache", "/tmp/somedir")` + if conf.target.targetOS == osWindows and DirSep == '/': arg = arg.replace('\\', '/') conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true) of "out", "o": expectArg(conf, switch, arg, pass, info) From 018465a2345b2d11f7d1d711010d97e636af98d0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 12 Aug 2021 08:25:11 +0200 Subject: [PATCH 0320/1654] fixes #18643 [backport:1.0] (#18678) --- compiler/ccgexprs.nim | 2 +- tests/array/tarray.nim | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 79c73b602..acf83f966 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -920,7 +920,7 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = if optBoundsCheck in p.options and ty.kind != tyUncheckedArray: if not isConstExpr(y): # semantic pass has already checked for const index expressions - if firstOrd(p.config, ty) == 0: + if firstOrd(p.config, ty) == 0 and lastOrd(p.config, ty) >= 0: if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)): linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); $3}$n", [rdCharLoc(b), intLiteral(lastOrd(p.config, ty)), raiseInstr(p)]) diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index 2765ad06d..d10011ef2 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -594,3 +594,14 @@ block t17705: a = int(a) var b = array[0, int].high b = int(b) + +block t18643: + # https://github.com/nim-lang/Nim/issues/18643 + let a: array[0, int] = [] + var caught = false + let b = 9999999 + try: + echo a[b] + except IndexDefect: + caught = true + doAssert caught, "IndexDefect not caught!" From 5c1304a4181596e764b05679cd8a0044ab3398dd Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 12 Aug 2021 07:50:08 -0700 Subject: [PATCH 0321/1654] fix #18670 quoteShellCommand, quoteShell, quoteShellWindows on windows (#18671) --- lib/pure/os.nim | 11 ++++++----- tests/stdlib/tos.nim | 13 ++++++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index fd618e93c..cfdacdb3c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1045,15 +1045,12 @@ proc expandTilde*(path: string): string {. # TODO: handle `~bob` and `~bob/` which means home of bob result = path -# TODO: consider whether quoteShellPosix, quoteShellWindows, quoteShell, quoteShellCommand -# belong in `strutils` instead; they are not specific to paths proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote `s`, so it can be safely passed to Windows API. ## ## Based on Python's `subprocess.list2cmdline`. ## See `this link `_ ## for more details. - let needQuote = {' ', '\t'} in s or s.len == 0 result = "" var backslashBuff = "" @@ -1064,8 +1061,8 @@ proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1" if c == '\\': backslashBuff.add(c) elif c == '\"': - result.add(backslashBuff) - result.add(backslashBuff) + for i in 0.. 0: + result.add(backslashBuff) if needQuote: + result.add(backslashBuff) result.add("\"") + proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to POSIX shell. const safeUnixChars = {'%', '+', '-', '.', '/', '_', ':', '=', '@', diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index dcb2d44f4..b7816fd41 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -622,7 +622,18 @@ block: # quoteShellWindows doAssert quoteShellWindows("aaa\"") == "aaa\\\"" doAssert quoteShellWindows("") == "\"\"" -block: # quoteShellWindows +block: # quoteShellCommand + when defined(windows): + doAssert quoteShellCommand(["a b c", "d", "e"]) == """"a b c" d e""" + doAssert quoteShellCommand(["""ab"c""", r"\", "d"]) == """ab\"c \ d""" + doAssert quoteShellCommand(["""ab"c""", """ \""", "d"]) == """ab\"c " \\" d""" + doAssert quoteShellCommand(["""a\\\b""", """de fg""", "h"]) == """a\\\b "de fg" h""" + doAssert quoteShellCommand(["""a\"b""", "c", "d"]) == """a\\\"b c d""" + doAssert quoteShellCommand(["""a\\b c""", "d", "e"]) == """"a\\b c" d e""" + doAssert quoteShellCommand(["""a\\b\ c""", "d", "e"]) == """"a\\b\ c" d e""" + doAssert quoteShellCommand(["ab", ""]) == """ab """"" + +block: # quoteShellPosix doAssert quoteShellPosix("aaa") == "aaa" doAssert quoteShellPosix("aaa a") == "'aaa a'" doAssert quoteShellPosix("") == "''" From f559319a682fb2abbf2945fd3390585a54f044d5 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 13 Aug 2021 00:21:01 +0800 Subject: [PATCH 0322/1654] fix a sqlite bug (#18669) --- lib/impure/db_sqlite.nim | 15 ++++----------- lib/std/private/dbutils.nim | 15 +++++++++++++++ tests/stdlib/tsqlitebindatas.nim | 28 ++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 lib/std/private/dbutils.nim diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 7bd807a12..b600576df 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -171,7 +171,7 @@ import sqlite3, macros import db_common export db_common -import std/private/since +import std/private/[since, dbutils] type DbConn* = PSqlite3 ## Encapsulates a database connection. @@ -211,14 +211,7 @@ proc dbQuote*(s: string): string = add(result, '\'') proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = - result = "" - var a = 0 - for c in items(string(formatstr)): - if c == '?': - add(result, dbQuote(args[a])) - inc(a) - else: - add(result, c) + dbFormatImpl(formatstr, dbQuote, args) proc prepare*(db: DbConn; q: string): SqlPrepared {.since: (1, 3).} = ## Creates a new `SqlPrepared` statement. @@ -642,7 +635,7 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared): string proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [].} = + {.tags: [WriteDbEffect], raises: [DbError].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. ## @@ -699,7 +692,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [], since: (1, 3).} = + {.tags: [WriteDbEffect], raises: [DbError], since: (1, 3).} = ## same as tryInsertID tryInsertID(db, query, args) diff --git a/lib/std/private/dbutils.nim b/lib/std/private/dbutils.nim new file mode 100644 index 000000000..0ae3b3702 --- /dev/null +++ b/lib/std/private/dbutils.nim @@ -0,0 +1,15 @@ +import db_common + + +template dbFormatImpl*(formatstr: SqlQuery, dbQuote: proc (s: string): string, args: varargs[string]): string = + var res = "" + var a = 0 + for c in items(string(formatstr)): + if c == '?': + if a == args.len: + dbError("""The number of "?" given exceeds the number of parameters present in the query.""") + add(res, dbQuote(args[a])) + inc(a) + else: + add(res, c) + res diff --git a/tests/stdlib/tsqlitebindatas.nim b/tests/stdlib/tsqlitebindatas.nim index 643f1e2e6..754c80ae1 100644 --- a/tests/stdlib/tsqlitebindatas.nim +++ b/tests/stdlib/tsqlitebindatas.nim @@ -48,3 +48,31 @@ block tsqlitebindatas: ## db_sqlite binary data db.close() doAssert tryRemoveFile(dbName) + + +block: + block: + const dbName = buildDir / "db.sqlite3" + var db = db_sqlite.open(dbName, "", "", "") + var witness = false + try: + db.exec(sql("CREATE TABLE table1 (url TEXT, other_field INT);")) + db.exec(sql("REPLACE INTO table (url, another_field) VALUES (?, '123');")) + except DbError as e: + witness = true + doAssert e.msg == "The number of \"?\" given exceeds the number of parameters present in the query." + finally: + db.close() + removeFile(dbName) + + doAssert witness + + block: + const dbName = buildDir / "db.sqlite3" + var db = db_sqlite.open(dbName, "", "", "") + try: + db.exec(sql("CREATE TABLE table1 (url TEXT, other_field INT);")) + db.exec(sql("INSERT INTO table1 (url, other_field) VALUES (?, ?);"), "http://domain.com/test?param=1", 123) + finally: + db.close() + removeFile(dbName) From 883c04d444e943ba8e3d071a4dfcaca68ab300f9 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 13 Aug 2021 08:57:12 +0800 Subject: [PATCH 0323/1654] don't use space after proc names (#18681) --- lib/pure/strutils.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index ac078df4e..3179c73d2 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2726,7 +2726,7 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, add s, formatstr[i] inc(i) -func `%` *(formatstr: string, a: openArray[string]): string {.rtl, +func `%`*(formatstr: string, a: openArray[string]): string {.rtl, extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. ## @@ -2774,7 +2774,7 @@ func `%` *(formatstr: string, a: openArray[string]): string {.rtl, result = newStringOfCap(formatstr.len + a.len shl 4) addf(result, formatstr, a) -func `%` *(formatstr, a: string): string {.rtl, +func `%`*(formatstr, a: string): string {.rtl, extern: "nsuFormatSingleElem".} = ## This is the same as `formatstr % [a]` (see ## `% func<#%25,string,openArray[string]>`_). From 3a1109a53be172a92c63ebc5bdb69c45e78e9ca6 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 12 Aug 2021 23:57:29 -0700 Subject: [PATCH 0324/1654] initial fix for compiling Nim on Zephyr RTOS (issue #18684) (#18685) * initial fix for compiling Nim on Zephyr RTOS (issue #18684) Co-authored-by: Jaremy J. Creechley --- lib/nimbase.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index e480c55fd..a83bd3006 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -75,7 +75,8 @@ __AVR__ #endif /* ------------------------------------------------------------------------- */ -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__ZEPHYR__) +/* Zephyr does some magic in it's headers that override the GCC stdlib. This breaks that. */ # define _GNU_SOURCE 1 #endif From b24812df5f952427c1266e48c40b7956bdf3d999 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 13 Aug 2021 00:35:48 -0700 Subject: [PATCH 0325/1654] properly fix #10053 ; `FieldDefect` msg now shows discriminant value + lineinfo, in all backends (c,vm,js) (#11955) * fix #10053 FieldError for vm * fixup * FieldError now also shows runtime value of discriminant * fix field error reporting in vm * also report culprit line info in err msg * fix errors for newruntime 2 * fix for js * fixup * PRTEMP4 * works * works * works perfect * refactor * std/private/repr_impl * suppport --gc:arc * cleanup * refactor * simplify * simplify * simplify * fixup * move out compiler.vmgen.genCustom * fixup * fixup * add tests * revert compiler/debugutils.nim * simplify reprDiscriminant * fixup * lib/std/private/repr_impl.nim -> lib/system/repr_impl.nim * try to fix D20210812T165220 * honor --declaredlocs * control toFileLineCol via --declaredlocs --- compiler/astmsgs.nim | 11 ++++++++++ compiler/ccgexprs.nim | 41 +++++++++++++++++++++++++----------- compiler/ccgtypes.nim | 6 ++++++ compiler/cgen.nim | 2 +- compiler/jsgen.nim | 11 +++++----- compiler/pragmas.nim | 2 +- compiler/renderer.nim | 12 ----------- compiler/vm.nim | 5 ++++- compiler/vmgen.nim | 14 ++++++------ lib/system/chcks.nim | 10 +++++++++ lib/system/indexerrors.nim | 4 ++++ lib/system/jssys.nim | 4 ++-- lib/system/repr.nim | 2 ++ lib/system/repr_impl.nim | 15 +++++++++++++ lib/system/repr_v2.nim | 7 ++++++ lib/system/reprjs.nim | 2 ++ tests/misc/mfield_defect.nim | 30 ++++++++++++++++++++++++++ tests/misc/trunner.nim | 16 +++++++++++--- 18 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 lib/system/repr_impl.nim create mode 100644 tests/misc/mfield_defect.nim diff --git a/compiler/astmsgs.nim b/compiler/astmsgs.nim index d9105b761..a9027126a 100644 --- a/compiler/astmsgs.nim +++ b/compiler/astmsgs.nim @@ -26,3 +26,14 @@ proc addDeclaredLoc*(result: var string, conf: ConfigRef; typ: PType) = proc addDeclaredLocMaybe*(result: var string, conf: ConfigRef; typ: PType) = if optDeclaredLocs in conf.globalOptions: addDeclaredLoc(result, conf, typ) + +template quoteExpr*(a: string): untyped = + ## can be used for quoting expressions in error msgs. + "'" & a & "'" + +proc genFieldDefect*(conf: ConfigRef, field: string, disc: PSym): string = + let obj = disc.owner.name.s # `types.typeToString` might be better, eg for generics + result = "field '$#' is not accessible for type '$#'" % [field, obj] + if optDeclaredLocs in conf.globalOptions: + result.add " [discriminant declared in $#]" % toFileLineCol(conf, disc.info) + result.add " using '$# = " % disc.name.s diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index acf83f966..7eb4a7a1d 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -18,6 +18,8 @@ proc getNullValueAuxT(p: BProc; orig, t: PType; obj, constOrNil: PNode, # -------------------------- constant expressions ------------------------ +proc rdSetElemLoc(conf: ConfigRef; a: TLoc, typ: PType): Rope + proc int64Literal(i: BiggestInt): Rope = if i > low(int64): result = "IL64($1)" % [rope(i)] @@ -873,16 +875,34 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = v.r.add(".") v.r.add(disc.sym.loc.r) genInExprAux(p, it, u, v, test) - let msg = genFieldDefect(field, disc.sym) + var msg = "" + if optDeclaredLocs in p.config.globalOptions: + # xxx this should be controlled by a separate flag, and + # used for other similar defects so that location information is shown + # even without the expensive `--stacktrace`; binary size could be optimized + # by encoding the file names separately from `file(line:col)`, essentially + # passing around `TLineInfo` + the set of files in the project. + msg.add toFileLineCol(p.config, e.info) & " " + msg.add genFieldDefect(p.config, field.name.s, disc.sym) let strLit = genStringLiteral(p.module, newStrNode(nkStrLit, msg)) - if op.magic == mNot: - linefmt(p, cpsStmts, - "if ($1){ #raiseFieldError($2); $3}$n", - [rdLoc(test), strLit, raiseInstr(p)]) + + ## discriminant check + template fun(code) = linefmt(p, cpsStmts, code, [rdLoc(test)]) + if op.magic == mNot: fun("if ($1) ") else: fun("if (!($1)) ") + + ## call raiseFieldError2 on failure + let discIndex = rdSetElemLoc(p.config, v, u.t) + if optTinyRtti in p.config.globalOptions: + # not sure how to use `genEnumToStr` here + const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex]) else: - linefmt(p, cpsStmts, - "if (!($1)){ #raiseFieldError($2); $3}$n", - [rdLoc(test), strLit, raiseInstr(p)]) + # complication needed for signed types + let first = p.config.firstOrd(disc.sym.typ) + let firstLit = int64Literal(cast[int](first)) + let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) + const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName]) proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) = assert e[0].kind == nkDotExpr @@ -1565,10 +1585,7 @@ proc genNewFinalize(p: BProc, e: PNode) = initLocExpr(p, e[1], a) initLocExpr(p, e[2], f) initLoc(b, locExpr, a.lode, OnHeap) - if optTinyRtti in p.config.globalOptions: - ti = genTypeInfoV2(p.module, refType, e.info) - else: - ti = genTypeInfoV1(p.module, refType, e.info) + ti = genTypeInfo(p.config, p.module, refType, e.info) p.module.s[cfsTypeInit3].addf("$1->finalizer = (void*)$2;$n", [ti, rdLoc(f)]) b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [ getTypeDesc(p.module, refType), diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 67e701804..d05c4cd8b 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1514,3 +1514,9 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = proc genTypeSection(m: BModule, n: PNode) = discard + +proc genTypeInfo*(config: ConfigRef, m: BModule, t: PType; info: TLineInfo): Rope = + if optTinyRtti in config.globalOptions: + result = genTypeInfoV2(m, t, info) + else: + result = genTypeInfoV1(m, t, info) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 3b004d399..f55da7f65 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -15,7 +15,7 @@ import ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, rodutils, renderer, cgendata, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf, - injectdestructors + injectdestructors, astmsgs when not defined(leanCompiler): import spawn, semparallel diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 49a57b4a6..3aa5c474e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -33,7 +33,7 @@ import nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, - transf, injectdestructors, sourcemap + transf, injectdestructors, sourcemap, astmsgs import json, sets, math, tables, intsets, strutils @@ -1208,12 +1208,13 @@ proc genCheckedFieldOp(p: PProc, n: PNode, addrTyp: PType, r: var TCompRes) = let tmp = p.getTemp() lineF(p, "var $1 = $2;$n", tmp, obj.res) - useMagic(p, "raiseFieldError") + useMagic(p, "raiseFieldError2") useMagic(p, "makeNimstrLit") - let msg = genFieldDefect(field, disc) - lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError(makeNimstrLit($5)); }$n", + useMagic(p, "reprDiscriminant") # no need to offset by firstOrd unlike for cgen + let msg = genFieldDefect(p.config, field.name.s, disc) + lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError2(makeNimstrLit($5), reprDiscriminant($2.$3, $6)); }$n", setx.res, tmp, disc.loc.r, if negCheck: ~"!==" else: ~"===", - makeJSString(msg)) + makeJSString(msg), genTypeInfo(p, disc.typ)) if addrTyp != nil and mapType(p, addrTyp) == etyBaseIndex: r.typ = etyBaseIndex diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 8ae931d84..6637a8673 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -699,7 +699,7 @@ proc typeBorrow(c: PContext; sym: PSym, n: PNode) = incl(sym.typ.flags, tfBorrowDot) proc markCompilerProc(c: PContext; s: PSym) = - # minor hack ahead: FlowVar is the only generic .compilerProc type which + # minor hack ahead: FlowVar is the only generic .compilerproc type which # should not have an external name set: if s.kind != skType or s.name.s != "FlowVar": makeExternExport(c, s, "$1", s.info) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 3346e629c..393f35289 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1758,15 +1758,3 @@ proc getTokSym*(r: TSrcGen): PSym = result = r.tokens[r.idx-1].sym else: result = nil - -proc quoteExpr*(a: string): string {.inline.} = - ## can be used for quoting expressions in error msgs. - "'" & a & "'" - -proc genFieldDefect*(field: PSym, disc: PSym): string = - ## this needs to be in a module accessible by jsgen, ccgexprs, and vm to - ## provide this error msg FieldDefect; msgs would be better but it does not - ## import ast - result = field.name.s.quoteExpr & " is not accessible using discriminant " & - disc.name.s.quoteExpr & " of type " & - disc.owner.name.s.quoteExpr diff --git a/compiler/vm.nim b/compiler/vm.nim index 73a228b6c..9fa25efbe 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1472,7 +1472,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = else: return TFullReg(kind: rkNone) of opcInvalidField: - stackTrace(c, tos, pc, errFieldXNotFound & regs[ra].node.strVal) + let msg = regs[ra].node.strVal + let disc = regs[instr.regB].regToNode + let msg2 = formatFieldDefect(msg, $disc) + stackTrace(c, tos, pc, msg2) of opcSetLenStr: decodeB(rkNode) #createStrKeepNode regs[ra] diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 4957a3339..725afdd3f 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -31,7 +31,7 @@ import tables import strutils, ast, types, msgs, renderer, vmdef, - intsets, magicsys, options, lowerings, lineinfos, transf + intsets, magicsys, options, lowerings, lineinfos, transf, astmsgs from modulegraphs import getBody @@ -1742,12 +1742,14 @@ proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags let lab1 = c.xjmp(n, if negCheck: opcFJmp else: opcTJmp, rs) c.freeTemp(rs) let strType = getSysType(c.graph, n.info, tyString) - var fieldNameRegister: TDest = c.getTemp(strType) - let strLit = newStrNode($accessExpr[1], accessExpr[1].info) + var msgReg: TDest = c.getTemp(strType) + let fieldName = $accessExpr[1] + let msg = genFieldDefect(c.config, fieldName, disc.sym) + let strLit = newStrNode(msg, accessExpr[1].info) strLit.typ = strType - c.genLit(strLit, fieldNameRegister) - c.gABC(n, opcInvalidField, fieldNameRegister) - c.freeTemp(fieldNameRegister) + c.genLit(strLit, msgReg) + c.gABC(n, opcInvalidField, msgReg, discVal) + c.freeTemp(msgReg) c.patch(lab1) proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) = diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 255d97cb2..8c7c95795 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -26,8 +26,18 @@ proc raiseIndexError() {.compilerproc, noinline.} = sysFatal(IndexDefect, "index out of bounds") proc raiseFieldError(f: string) {.compilerproc, noinline.} = + ## remove after bootstrap > 1.5.1 sysFatal(FieldDefect, f) +when defined(gcdestructors): + proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldError, f & $discVal & "'") +else: + proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noinline.} = + ## raised when field is inaccessible given runtime value of discriminant + sysFatal(FieldError, formatFieldDefect(f, discVal)) + proc raiseRangeErrorI(i, a, b: BiggestInt) {.compilerproc, noinline.} = when defined(standalone): sysFatal(RangeDefect, "value out of range") diff --git a/lib/system/indexerrors.nim b/lib/system/indexerrors.nim index 1b91789bd..6a8cb8a0a 100644 --- a/lib/system/indexerrors.nim +++ b/lib/system/indexerrors.nim @@ -1,4 +1,5 @@ # imported by other modules, unlike helpers.nim which is included +# xxx this is now included instead of imported, we should import instead template formatErrorIndexBound*[T](i, a, b: T): string = when defined(standalone): @@ -9,3 +10,6 @@ template formatErrorIndexBound*[T](i, a, b: T): string = template formatErrorIndexBound*[T](i, n: T): string = formatErrorIndexBound(i, 0, n) + +template formatFieldDefect*(f, discVal): string = + f & discVal & "'" diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index b42dc3a20..63e7b3d9d 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -171,8 +171,8 @@ proc raiseRangeError() {.compilerproc, noreturn.} = proc raiseIndexError(i, a, b: int) {.compilerproc, noreturn.} = raise newException(IndexDefect, formatErrorIndexBound(int(i), int(a), int(b))) -proc raiseFieldError(f: string) {.compilerproc, noreturn.} = - raise newException(FieldDefect, f) +proc raiseFieldError2(f: string, discVal: string) {.compilerproc, noreturn.} = + raise newException(FieldDefect, formatFieldDefect(f, discVal)) proc setConstr() {.varargs, asmNoStackFrame, compilerproc.} = asm """ diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 020c2281f..e049d18fa 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -72,6 +72,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = result = $e & " (invalid data!)" +include system/repr_impl + type PByteArray = ptr UncheckedArray[byte] # array[0xffff, byte] diff --git a/lib/system/repr_impl.nim b/lib/system/repr_impl.nim new file mode 100644 index 000000000..b9ec1890f --- /dev/null +++ b/lib/system/repr_impl.nim @@ -0,0 +1,15 @@ +#[ +other APIs common to system/repr and system/reprjs could be refactored here, eg: +* reprChar +* reprBool +* reprStr + +Another possibility in future work would be to have a single include file instead +of system/repr and system/reprjs, and use `when defined(js)` inside it. +]# + +proc reprDiscriminant*(e: int, typ: PNimType): string {.compilerRtl.} = + case typ.kind + of tyEnum: reprEnum(e, typ) + of tyBool: $(e != 0) + else: $e diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index f99f09799..ba94b881d 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -1,3 +1,5 @@ +include system/inclrtl + proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} ## imported from typetraits @@ -66,6 +68,11 @@ proc repr*[Enum: enum](x: Enum): string {.magic: "EnumToStr", noSideEffect.} ## If a `repr` operator for a concrete enumeration is provided, this is ## used instead. (In other words: *Overwriting* is possible.) +proc reprDiscriminant*(e: int): string {.compilerproc.} = + # repr and reprjs can use `PNimType` to symbolize `e`; making this work here + # would require a way to pass the set of enum stringified values to cgen. + $e + proc repr*(p: pointer): string = ## repr of pointer as its hexadecimal value if p == nil: diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index cdc3403f8..28935a4ef 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -30,6 +30,8 @@ proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = else: result = $e & " (invalid data!)" +include system/repr_impl + proc reprChar(x: char): string {.compilerRtl.} = result = "\'" case x diff --git a/tests/misc/mfield_defect.nim b/tests/misc/mfield_defect.nim new file mode 100644 index 000000000..53bfba40e --- /dev/null +++ b/tests/misc/mfield_defect.nim @@ -0,0 +1,30 @@ +#[ +ran from trunner +]# + + + + + + +# line 10 +type Kind = enum k0, k1, k2, k3, k4 + +type Foo = object + case kind: Kind + of k0: f0: int + of k1: f1: int + of k2: f2: int + of k3: f3: int + of k4: f4: int + +proc main()= + var foo = Foo(kind: k3, f3: 3) + let s1 = foo.f3 + doAssert s1 == 3 + let s2 = foo.f2 + +when defined case1: + static: main() +when defined case2: + main() diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 8426e9aee..f874d38d9 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -37,9 +37,9 @@ proc runNimCmd(file, options = "", rtarg = ""): auto = echo cmd echo result[0] & "\n" & $result[1] -proc runNimCmdChk(file, options = "", rtarg = ""): string = - let (ret, status) = runNimCmd(file, options, rtarg = rtarg) - doAssert status == 0, $(file, options) & "\n" & ret +proc runNimCmdChk(file, options = "", rtarg = "", status = 0): string = + let (ret, status2) = runNimCmd(file, options, rtarg = rtarg) + doAssert status2 == status, $(file, options, status, status2) & "\n" & ret ret proc genShellCmd(filename: string): string = @@ -376,5 +376,15 @@ mused3.nim(13, 8) Warning: imported and not used: 'mused3b' [UnusedImport] mused3.nim(75, 10) Hint: duplicate import of 'mused3a'; previous import here: mused3.nim(74, 10) [DuplicateModuleImport] """ + block: # FieldDefect + proc fn(opt: string, expected: string) = + let output = runNimCmdChk("misc/mfield_defect.nim", fmt"-r --warning:all:off --declaredlocs {opt}", status = 1) + doAssert expected in output, opt & "\noutput:\n" & output & "expected:\n" & expected + fn("-d:case1"): """mfield_defect.nim(25, 15) Error: field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + fn("-d:case2 --gc:refc"): """mfield_defect.nim(25, 15) field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + fn("-d:case1 -b:js"): """mfield_defect.nim(25, 15) Error: field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + fn("-d:case2 -b:js"): """field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = k3'""" + # 3 instead of k3, because of lack of RTTI + fn("-d:case2 --gc:arc"): """mfield_defect.nim(25, 15) field 'f2' is not accessible for type 'Foo' [discriminant declared in mfield_defect.nim(14, 8)] using 'kind = 3'""" else: discard # only during debugging, tests added here will run with `-d:nimTestsTrunnerDebugging` enabled From 4463a3c3d75daabfef52a139221f729e8af9e81e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 13 Aug 2021 12:34:28 +0200 Subject: [PATCH 0326/1654] manual: minor improvements (#18687) --- doc/manual.rst | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 41ed49715..9c7c219e9 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -31,7 +31,7 @@ the `destructors `_ document. This document describes the lexis, the syntax, and the semantics of the Nim language. To learn how to compile Nim programs and generate documentation see -`Compiler User Guide `_ and `DocGen Tools Guide `_. +the `Compiler User Guide `_ and the `DocGen Tools Guide `_. The language constructs are explained using an extended BNF, in which `(a)*` means 0 or more `a`'s, `a+` means 1 or more `a`'s, and `(a)?` means an @@ -583,12 +583,11 @@ In the following examples, `-1` is parsed as two separate tokens "abc"-1 -There exists a literal for each numerical type that is -defined. The suffix starting with an apostrophe ('\'') is called a +The suffix starting with an apostrophe ('\'') is called a `type suffix`:idx:. Literals without a type suffix are of an integer type unless the literal contains a dot or `E|e` in which case it is of type `float`. This integer type is `int` if the literal is in the range -`low(i32)..high(i32)`, otherwise it is `int64`. +`low(int32)..high(int32)`, otherwise it is `int64`. For notational convenience, the apostrophe of a type suffix is optional if it is not ambiguous (only hexadecimal floating-point literals with a type suffix can be ambiguous). @@ -619,10 +618,9 @@ notation: `0B0_10001110100_0000101001000111101011101111111011000101001101001001'f64` is approximately 1.72826e35 according to the IEEE floating-point standard. -Literals are bounds checked so that they fit the datatype. Non-base-10 -literals are used mainly for flags and bit pattern representations, therefore -bounds checking is done on bit width, not value range. If the literal fits in -the bit width of the datatype, it is accepted. +Literals must match the datatype, for example, `333'i8` is an invalid literal. +Non-base-10 literals are used mainly for flags and bit pattern representations, +therefore the checking is done on bit width and not on value range. Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1 instead of causing an overflow error. @@ -1980,8 +1978,7 @@ details like this when mixing garbage-collected data with unmanaged memory. Procedural type --------------- A procedural type is internally a pointer to a procedure. `nil` is -an allowed value for variables of a procedural type. Nim uses procedural -types to achieve `functional`:idx: programming techniques. +an allowed value for a variable of a procedural type. Examples: @@ -2494,8 +2491,8 @@ An expression `b` can be assigned to an expression `a` iff `a` is an `l-value` and `isImplicitlyConvertible(b.typ, a.typ)` holds. -Overloading resolution -====================== +Overload resolution +=================== In a call `p(args)` the routine `p` that matches best is selected. If multiple routines match equally well, the ambiguity is reported during From 52cc0a82cad856fa58ebbb16a9e7ad8c7012e8cf Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 14 Aug 2021 02:13:30 -0700 Subject: [PATCH 0327/1654] fix #18692 AsyncHttpServer was hanging because client.close was not called (#18693) --- lib/pure/asynchttpserver.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index fd0cad5ca..f23c3d3f9 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -362,7 +362,9 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string let retry = await processRequest( server, request, client, address, lineFut, callback ) - if not retry: break + if not retry: + client.close() + break const nimMaxDescriptorsFallback* {.intdefine.} = 16_000 ## fallback value for \ From b3e077863aa6985eebab0c37785ba127b3192c80 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sat, 14 Aug 2021 21:51:41 -0300 Subject: [PATCH 0328/1654] Documentation only, Terminal (#18696) Add simple progress bar example to terminal module --- lib/pure/terminal.nim | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index ce5660557..25b1a4cdd 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -15,6 +15,21 @@ ## code `exitprocs.addExitProc(resetAttributes)` to restore the defaults. ## Similarly, if you hide the cursor, make sure to unhide it with ## `showCursor` before quitting. +## +## Progress bar +## ============ +## +## Basic progress bar example: +runnableExamples("-r:off"): + import std/[os, strutils] + + for i in 0..100: + stdout.styledWriteLine(fgRed, "0% ", fgWhite, '#'.repeat i, if i > 50: fgGreen else: fgYellow, "\t", $i , "%") + sleep 42 + cursorUp 1 + eraseLine() + + stdout.resetAttributes() ##[ ## Playing with colorful and styled text @@ -43,7 +58,6 @@ runnableExamples("-r:off"): stdout.styledWriteLine(fgRed, "red text ", styleBright, "bold red", fgDefault, " bold text") - import macros import strformat from strutils import toLowerAscii, `%` From 1acba63cb725a0eee8f8b02f585057e030ce6295 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 16 Aug 2021 00:32:12 -0700 Subject: [PATCH 0329/1654] cross compilation targetting windows now supports `nim r`: `nim r -d:mingw main` (#18682) * cross compilation targetting windows now supports `nim r`: `nim r -d:mingw main` * quoteShell * address comment: remove `conf.getConfigVar("nimrun.exe")` --- changelog.md | 3 +++ compiler/nim.nim | 20 +++++++++++++++----- doc/nimc.rst | 2 ++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 0e3c54d77..40c6c9ace 100644 --- a/changelog.md +++ b/changelog.md @@ -482,6 +482,9 @@ nim r main # uses cached binary nim r main arg1 arg2 # ditto (runtime arguments are irrelevant) +- `nim r` now supports cross compilation from unix to windows when specifying `-d:mingw` by using wine, + e.g.: `nim r --eval:'import os; echo "a" / "b"'` prints `a\b` + - The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch enforces that every symbol is written as it was declared, not enforcing the official Nim style guide. To be enabled, this has to be combined either diff --git a/compiler/nim.nim b/compiler/nim.nim index 91b1bc2db..b8256d576 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -70,6 +70,13 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = config.arguments.len > 0 and config.cmd notin {cmdTcc, cmdNimscript, cmdCrun}: rawMessage(config, errGenerated, errArgsNeedRunOption) +proc getNimRunExe(conf: ConfigRef): string = + # xxx consider defining `conf.getConfigVar("nimrun.exe")` to allow users to + # customize the binary to run the command with, e.g. for custom `nodejs` or `wine`. + if conf.isDefined("mingw"): + if conf.isDefined("i386"): result = "wine" + elif conf.isDefined("amd64"): result = "wine64" + proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = let self = NimProg( supportsStdinFile: true, @@ -95,18 +102,21 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = let output = conf.absOutFile case conf.cmd of cmdBackends, cmdTcc: - var cmdPrefix = "" + let nimRunExe = getNimRunExe(conf) + var cmdPrefix: string + if nimRunExe.len > 0: cmdPrefix.add nimRunExe.quoteShell case conf.backend of backendC, backendCpp, backendObjc: discard of backendJs: # D20210217T215950:here this flag is needed for node < v15.0.0, otherwise # tasyncjs_fail` would fail, refs https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode - cmdPrefix = findNodeJs() & " --unhandled-rejections=strict " + if cmdPrefix.len == 0: cmdPrefix = findNodeJs().quoteShell + cmdPrefix.add " --unhandled-rejections=strict" else: doAssert false, $conf.backend - # No space before command otherwise on windows you'd get a cryptic: - # `The parameter is incorrect` + if cmdPrefix.len > 0: cmdPrefix.add " " + # without the `cmdPrefix.len > 0` check, on windows you'd get a cryptic: + # `The parameter is incorrect` execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments) - # execExternalProgram(conf, cmdPrefix & ' ' & output.quoteShell & ' ' & conf.arguments) of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing) if conf.arguments.len > 0: # reserved for future use diff --git a/doc/nimc.rst b/doc/nimc.rst index ad5ecaa76..98b754729 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -317,6 +317,8 @@ To cross-compile for Windows from Linux or macOS using the MinGW-w64 toolchain: .. code:: cmd nim c -d:mingw myproject.nim + # `nim r` also works, running the binary via `wine` or `wine64`: + nim r -d:mingw --eval:'import os; echo "a" / "b"' Use `--cpu:i386`:option: or `--cpu:amd64`:option: to switch the CPU architecture. From 901c5ded527bb06c52bfc1fd38c9a5fadee0f49a Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 16 Aug 2021 21:14:35 +0800 Subject: [PATCH 0330/1654] fix #18702(fix `parseutils.parseFloat`) (#18703) [backport:1.0] * fix #18702 * Apply suggestions from code review --- lib/system/strmantle.nim | 9 +++++---- tests/stdlib/tparseutils.nim | 13 +++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 81ac5c751..f55168c01 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -211,15 +211,16 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, var ti = 0 let maxlen = t.high - "e+000".len # reserve enough space for exponent - result = i - start + let endPos = i + result = endPos - start i = start # re-parse without error checking, any error should be handled by the code above. - if i < s.len and s[i] == '.': i.inc - while i < s.len and s[i] in {'0'..'9','+','-'}: + if i < endPos and s[i] == '.': i.inc + while i < endPos and s[i] in {'0'..'9','+','-'}: if ti < maxlen: t[ti] = s[i]; inc(ti) inc(i) - while i < s.len and s[i] in {'.', '_'}: # skip underscore and decimal point + while i < endPos and s[i] in {'.', '_'}: # skip underscore and decimal point inc(i) # insert exponent diff --git a/tests/stdlib/tparseutils.nim b/tests/stdlib/tparseutils.nim index 3bc54dff1..db7a0ac8d 100644 --- a/tests/stdlib/tparseutils.nim +++ b/tests/stdlib/tparseutils.nim @@ -1,5 +1,5 @@ -import parseutils -import sequtils +import std/[parseutils, sequtils, sugar] + let input = "$test{} $this is ${an{ example}} " let expected = @[(ikVar, "test"), (ikStr, "{} "), (ikVar, "this"), @@ -41,3 +41,12 @@ doAssert value == 1_000_000 var i64Value: int64 discard parseBiggestInt("9223372036854775807", i64Value) doAssert i64Value == 9223372036854775807 + +block: + var f: float + let res = collect: + for x in ["9.123456789012345+","11.123456789012345+","9.123456789012345-","8.123456789012345+","9.12345678901234-","9.123456789012345"]: + (parseFloat(x, f, 0), $f) + doAssert res == @[(17, "9.123456789012344"), (18, "11.123456789012344"), + (17, "9.123456789012344"), (17, "8.123456789012344"), + (16, "9.12345678901234"), (17, "9.123456789012344")] From 5fd150b7f792754b0431f2cc2e9830fb23ce15f6 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Wed, 18 Aug 2021 14:25:25 +0200 Subject: [PATCH 0331/1654] asynchttpserver: ipv6 support (#18706) Fix #18705 --- lib/pure/asynchttpserver.nim | 13 +++++++------ tests/errmsgs/tgcsafety.nim | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index f23c3d3f9..d7daacd03 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -41,7 +41,7 @@ runnableExamples("-r:off"): import asyncnet, asyncdispatch, parseutils, uri, strutils import httpcore -from nativesockets import getLocalAddr, AF_INET +from nativesockets import getLocalAddr, Domain, AF_INET, AF_INET6 import std/private/since export httpcore except parseHeader @@ -82,7 +82,7 @@ proc getPort*(self: AsyncHttpServer): Port {.since: (1, 5, 1).} = server.listen(Port(0)) assert server.getPort.uint16 > 0 server.close() - result = getLocalAddr(self.socket.getFd, AF_INET)[1] + result = getLocalAddr(self.socket)[1] proc newAsyncHttpServer*(reuseAddr = true, reusePort = false, maxBody = 8388608): AsyncHttpServer = @@ -372,13 +372,13 @@ const ## This can be set on the command line during compilation ## via `-d:nimMaxDescriptorsFallback=N` -proc listen*(server: AsyncHttpServer; port: Port; address = "") = +proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET) = ## Listen to the given port and address. when declared(maxDescriptors): server.maxFDs = try: maxDescriptors() except: nimMaxDescriptorsFallback else: server.maxFDs = nimMaxDescriptorsFallback - server.socket = newAsyncSocket() + server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) if server.reusePort: @@ -404,7 +404,8 @@ proc acceptRequest*(server: AsyncHttpServer, proc serve*(server: AsyncHttpServer, port: Port, callback: proc (request: Request): Future[void] {.closure, gcsafe.}, address = ""; - assumedDescriptorsPerRequest = -1) {.async.} = + assumedDescriptorsPerRequest = -1; + domain = AF_INET) {.async.} = ## Starts the process of listening for incoming HTTP connections on the ## specified address and port. ## @@ -417,7 +418,7 @@ proc serve*(server: AsyncHttpServer, port: Port, ## ## You should prefer to call `acceptRequest` instead with a custom server ## loop so that you're in control over the error handling and logging. - listen server, port, address + listen server, port, address, domain while true: if shouldAcceptRequest(server, assumedDescriptorsPerRequest): var (address, client) = await server.socket.acceptAddr() diff --git a/tests/errmsgs/tgcsafety.nim b/tests/errmsgs/tgcsafety.nim index 6021d19e1..701adeebf 100644 --- a/tests/errmsgs/tgcsafety.nim +++ b/tests/errmsgs/tgcsafety.nim @@ -6,7 +6,7 @@ tgcsafety.nim(31, 18) Error: type mismatch: got Date: Wed, 18 Aug 2021 21:53:52 +0800 Subject: [PATCH 0332/1654] [wip]better hint message for JS (#18704) * better hint message for JS * both --- compiler/msgs.nim | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 20a55daee..56531eb68 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -669,17 +669,28 @@ proc genSuccessX*(conf: ConfigRef) = else: formatSize(getTotalMem()) & " totmem" let loc = $conf.linesCompiled var build = "" + var flags = "" + const debugModeHints = "none (DEBUG BUILD, `-d:release` generates faster code)" if conf.cmd in cmdBackends: - build.add "gc: $#; " % $conf.selectedGC - if optThreads in conf.globalOptions: build.add "threads: on; " - build.add "opt: " - if optOptimizeSpeed in conf.options: build.add "speed" - elif optOptimizeSize in conf.options: build.add "size" - else: build.add "none (DEBUG BUILD, `-d:release` generates faster code)" - # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html - var flags = "" - if isDefined(conf, "danger"): flags.add " -d:danger" - elif isDefined(conf, "release"): flags.add " -d:release" + if conf.backend != backendJs: + build.add "gc: $#; " % $conf.selectedGC + if optThreads in conf.globalOptions: build.add "threads: on; " + build.add "opt: " + if optOptimizeSpeed in conf.options: build.add "speed" + elif optOptimizeSize in conf.options: build.add "size" + else: build.add debugModeHints + # pending https://github.com/timotheecour/Nim/issues/752, point to optimization.html + if isDefined(conf, "danger"): flags.add " -d:danger" + elif isDefined(conf, "release"): flags.add " -d:release" + else: + build.add "opt: " + if isDefined(conf, "danger"): + build.add "speed" + flags.add " -d:danger" + elif isDefined(conf, "release"): + build.add "speed" + flags.add " -d:release" + else: build.add debugModeHints if flags.len > 0: build.add "; options:" & flags let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3) let project = if conf.filenameOption == foAbs: $conf.projectFull else: $conf.projectName From 7b58dc2de0f606b757a558dfdda9d930ae63f41a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 18 Aug 2021 10:33:11 -0700 Subject: [PATCH 0333/1654] allow building 1.4.0 from devel (#18708) * allow building 1.4.0 from devel * changelog --- changelog.md | 3 +++ compiler/ast.nim | 8 ++++++-- compiler/ccgexprs.nim | 21 +++++++++++++++++---- compiler/jsgen.nim | 1 + compiler/semfold.nim | 1 + compiler/vmgen.nim | 2 +- 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/changelog.md b/changelog.md index 40c6c9ace..77a77f2f5 100644 --- a/changelog.md +++ b/changelog.md @@ -485,6 +485,9 @@ - `nim r` now supports cross compilation from unix to windows when specifying `-d:mingw` by using wine, e.g.: `nim r --eval:'import os; echo "a" / "b"'` prints `a\b` +- `nim` can compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. + `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. + - The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch enforces that every symbol is written as it was declared, not enforcing the official Nim style guide. To be enabled, this has to be combined either diff --git a/compiler/ast.nim b/compiler/ast.nim index 5b54dd8ef..08fbc70e1 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -652,7 +652,9 @@ type mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, + mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, + mFloatToStr, # for -d:nimVersion140 + mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mImplies, mIff, mExists, mForall, mOld, @@ -720,7 +722,9 @@ const mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, + mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, + mFloatToStr, + mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, mEqStr, mLeStr, mLtStr, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7eb4a7a1d..374876367 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -894,15 +894,23 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let discIndex = rdSetElemLoc(p.config, v, u.t) if optTinyRtti in p.config.globalOptions: # not sure how to use `genEnumToStr` here - const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n" - linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex]) + if p.config.isDefined("nimVersion140"): + const code = "{ #raiseFieldError($1); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) + else: + const code = "{ #raiseFieldError2($1, (NI)$3); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex]) else: # complication needed for signed types let first = p.config.firstOrd(disc.sym.typ) let firstLit = int64Literal(cast[int](first)) let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) - const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n" - linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName]) + if p.config.isDefined("nimVersion140"): + const code = "{ #raiseFieldError($1); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) + else: + const code = "{ #raiseFieldError2($1, #reprDiscriminant(((NI)$3) + (NI)$4, $5)); $2} $n" + linefmt(p, cpsStmts, code, [strLit, raiseInstr(p), discIndex, firstLit, discName]) proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) = assert e[0].kind == nkDotExpr @@ -2320,6 +2328,11 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mInt64ToStr: genDollar(p, e, d, "#nimInt64ToStr($1)") of mBoolToStr: genDollar(p, e, d, "#nimBoolToStr($1)") of mCharToStr: genDollar(p, e, d, "#nimCharToStr($1)") + of mFloatToStr: + if e[1].typ.skipTypes(abstractInst).kind == tyFloat32: + genDollar(p, e, d, "#nimFloat32ToStr($1)") + else: + genDollar(p, e, d, "#nimFloatToStr($1)") of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) of mIsolate: genCall(p, e, d) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 3aa5c474e..931e56475 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -434,6 +434,7 @@ const # magic checked op; magic unchecked op; mBoolToStr: ["nimBoolToStr", "nimBoolToStr"], mIntToStr: ["cstrToNimstr", "cstrToNimstr"], mInt64ToStr: ["cstrToNimstr", "cstrToNimstr"], + mFloatToStr: ["cstrToNimstr", "cstrToNimstr"], mCStrToStr: ["cstrToNimstr", "cstrToNimstr"], mStrToStr: ["", ""]] diff --git a/compiler/semfold.nim b/compiler/semfold.nim index e5f0643bc..04ed73209 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -290,6 +290,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P of mBoolToStr: if getOrdValue(a) == 0: result = newStrNodeT("false", n, g) else: result = newStrNodeT("true", n, g) + of mFloatToStr: result = newStrNodeT($getFloat(a), n, g) of mCStrToStr, mCharToStr: if a.kind == nkBracket: var s = "" diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 725afdd3f..f2758ff37 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1127,7 +1127,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) - of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mCStrToStr, mStrToStr, mEnumToStr: + of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: genConv(c, n, n[1], dest) of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr) of mLeStr: genBinaryABC(c, n, dest, opcLeStr) From 394f4ac7bb92fe5aaf902495c6b43b3451667602 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 19 Aug 2021 02:33:52 -0700 Subject: [PATCH 0334/1654] improvements to `addInt` and `$` for integer types (#18592) * improvements to $(SomeInteger) and addInt * remove mIntToStr, mInt64ToStr * improvements * fix tests/pragmas/tinjectstmt.nim; the diff is harmless, cgen code is identical with -d:danger or debug mode * rm tests/system/tstrmantle.nim * revert compiler/jsgen.nim for -d:nimVersion140 --- compiler/ast.nim | 8 +-- lib/std/private/digitsutils.nim | 72 +++++++++++++++----- lib/std/private/miscdollars.nim | 9 ++- lib/system.nim | 9 ++- lib/system/assertions.nim | 5 +- lib/system/dollars.nim | 56 ++++++---------- lib/system/repr_v2.nim | 17 +++-- lib/system/strmantle.nim | 30 --------- nimsuggest/tests/tqualified_highlight.nim | 2 +- tests/pragmas/tinjectstmt.nim | 13 ++-- tests/system/tdollars.nim | 81 +++++++++++++++-------- tests/system/tstrmantle.nim | 46 ------------- 12 files changed, 162 insertions(+), 186 deletions(-) delete mode 100644 tests/system/tstrmantle.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 08fbc70e1..0bdcc0e97 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -652,8 +652,8 @@ type mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, - mFloatToStr, # for -d:nimVersion140 + mCharToStr, mBoolToStr, + mIntToStr, mInt64ToStr, mFloatToStr, # for -d:nimVersion140 mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, @@ -722,8 +722,8 @@ const mEqRef, mEqProc, mLePtr, mLtPtr, mEqCString, mXor, mUnaryMinusI, mUnaryMinusI64, mAbsI, mNot, mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, - mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, - mFloatToStr, + mCharToStr, mBoolToStr, + mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 7aefc36bc..268a3ba7d 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -29,18 +29,34 @@ const # doAssert res == digits100 proc utoa2Digits*(buf: var openArray[char]; pos: int; digits: uint32) {.inline.} = - assert(digits <= 99) buf[pos] = digits100[2 * digits] buf[pos+1] = digits100[2 * digits + 1] #copyMem(buf, unsafeAddr(digits100[2 * digits]), 2 * sizeof((char))) proc trailingZeros2Digits*(digits: uint32): int32 {.inline.} = - assert(digits <= 99) return trailingZeros100[digits] -func addIntImpl*(result: var string, origin: uint64) = +when defined(js): + proc numToString(a: SomeInteger): cstring {.importjs: "((#) + \"\")".} + +func addChars[T](result: var string, x: T, start: int, n: int) {.inline.} = + let old = result.len + result.setLen old + n + template impl = + for i in 0.. 0: result.add "(" - # simplify this after moving moving `include strmantle` above import assertions` - when declared(addInt): result.addInt line - else: result.add $line + addInt(result, line) if col > 0: result.add ", " - when declared(addInt): result.addInt col - else: result.add $col + addInt(result, col) result.add ")" diff --git a/lib/system.nim b/lib/system.nim index e6a2439df..740c4419f 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2463,14 +2463,13 @@ when notJSnotNims: else: {.error: "Only closure iterator is allowed!".} +from std/private/digitsutils import addInt +export addInt + when defined(js): include "system/jssys" include "system/reprjs" -when defined(js) or defined(nimscript): - proc addInt*(result: var string; x: int64) = - result.add $x - proc quit*(errormsg: string, errorcode = QuitFailure) {.noreturn.} = ## A shorthand for `echo(errormsg); quit(errorcode)`. when defined(nimscript) or defined(js) or (hostOS == "standalone"): @@ -2918,7 +2917,7 @@ proc addQuoted*[T](s: var string, x: T) = s.addEscapedChar(x) s.add("'") # prevent temporary string allocation - elif T is SomeSignedInt: + elif T is SomeInteger: s.addInt(x) elif T is SomeFloat: s.addFloat(x) diff --git a/lib/system/assertions.nim b/lib/system/assertions.nim index cd789845b..6f64a55b7 100644 --- a/lib/system/assertions.nim +++ b/lib/system/assertions.nim @@ -12,7 +12,6 @@ import std/private/miscdollars type InstantiationInfo = tuple[filename: string, line: int, column: int] -proc `$`(x: int): string {.magic: "IntToStr", noSideEffect.} proc `$`(info: InstantiationInfo): string = # The +1 is needed here # instead of overriding `$` (and changing its meaning), consider explicit name. @@ -108,7 +107,9 @@ template doAssertRaises*(exception: typedesc, code: untyped) = wrong = true except exception: discard - except Exception as e: raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) + except Exception as e: + mixin `$` # alternatively, we could define $cstring in this module + raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) except: raisedForeign() if wrong: raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/lib/system/dollars.nim b/lib/system/dollars.nim index 8634db382..46085d2aa 100644 --- a/lib/system/dollars.nim +++ b/lib/system/dollars.nim @@ -1,45 +1,31 @@ +## `$` is Nim's general way of spelling `toString`:idx:. +runnableExamples: + assert $0.1 == "0.1" + assert $(-2*3) == "-6" + import std/private/digitsutils import system/formatfloat export addFloat -proc `$`*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. `$` is Nim's general way of - ## spelling `toString`:idx:. +proc `$`*(x: int): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) -template dollarImpl(x: uint | uint64, result: var string) = - addIntImpl(result, x) +proc `$`*(x: int64): string {.raises: [].} = + ## Outplace version of `addInt`. + result.addInt(x) -when defined(js): - import std/private/since - since (1, 3): - proc `$`*(x: uint): string = - ## Caveat: currently implemented as $(cast[int](x)), tied to current - ## semantics of js' Number type. - # for c, see strmantle.`$` - when nimvm: - dollarImpl(x, result) - else: - result = $(int(x)) +proc `$`*(x: uint64): string {.raises: [].} = + ## Outplace version of `addInt`. + addInt(result, x) - proc `$`*(x: uint64): string = - ## Compatibility note: - ## the results may change in future releases if/when js target implements - ## 64bit ints. - # pending https://github.com/nim-lang/RFCs/issues/187 - when nimvm: - dollarImpl(x, result) - else: - result = $(cast[int](x)) -else: - proc `$`*(x: uint64): string {.noSideEffect, raises: [].} = - ## The stringify operator for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - dollarImpl(x, result) - -proc `$`*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## The stringify operator for an integer argument. Returns `x` - ## converted to a decimal string. +# same as old `ctfeWhitelist` behavior, whether or not this is a good idea. +template gen(T) = + # xxx simplify this by supporting this in compiler: int{lit} | uint64{lit} | int64{lit} + func `$`*(x: T{lit}): string {.compileTime.} = result.addInt(x) +gen(int) +gen(uint64) +gen(int64) func `$`*(x: float | float32): string = ## Outplace version of `addFloat`. diff --git a/lib/system/repr_v2.nim b/lib/system/repr_v2.nim index ba94b881d..6ab5f3c3f 100644 --- a/lib/system/repr_v2.nim +++ b/lib/system/repr_v2.nim @@ -8,18 +8,17 @@ proc distinctBase(T: typedesc, recursive: static bool = true): typedesc {.magic: proc repr*(x: NimNode): string {.magic: "Repr", noSideEffect.} -proc repr*(x: int): string {.magic: "IntToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int): string = + ## Same as $x + $x -proc repr*(x: int64): string {.magic: "Int64ToStr", noSideEffect.} - ## repr for an integer argument. Returns `x` - ## converted to a decimal string. +proc repr*(x: int64): string = + ## Same as $x + $x proc repr*(x: uint64): string {.noSideEffect.} = - ## repr for an unsigned integer argument. Returns `x` - ## converted to a decimal string. - $x #Calls `$` from system/strmantle.nim + ## Same as $x + $x proc repr*(x: float): string = ## Same as $x diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index f55168c01..9cf4f9e55 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -43,32 +43,6 @@ proc hashString(s: string): int {.compilerproc.} = h = h + h shl 15 result = cast[int](h) -proc addInt*(result: var string; x: int64) = - ## Converts integer to its string representation and appends it to `result`. - ## - ## .. code-block:: Nim - ## var - ## a = "123" - ## b = 45 - ## a.addInt(b) # a <- "12345" - var num: uint64 - - if x < 0: - if x == low(int64): - num = uint64(x) - else: - num = uint64(-x) - let base = result.len - setLen(result, base + 1) - result[base] = '-' - else: - num = uint64(x) - addIntImpl(result, num) - -proc nimIntToStr(x: int): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x - proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "", noSideEffect.} @@ -240,10 +214,6 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, when defined(nimHasInvariant): {.pop.} # staticBoundChecks -proc nimInt64ToStr(x: int64): string {.compilerRtl.} = - result = newStringOfCap(sizeof(x)*4) - result.addInt x - proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" diff --git a/nimsuggest/tests/tqualified_highlight.nim b/nimsuggest/tests/tqualified_highlight.nim index 3b521ecc1..b83669e72 100644 --- a/nimsuggest/tests/tqualified_highlight.nim +++ b/nimsuggest/tests/tqualified_highlight.nim @@ -10,5 +10,5 @@ highlight;;skProc;;1;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 highlight;;skTemplate;;2;;7;;4 -highlight;;skProc;;3;;8;;1 +highlight;;skFunc;;3;;8;;1 """ diff --git a/tests/pragmas/tinjectstmt.nim b/tests/pragmas/tinjectstmt.nim index bca041e46..c6256bda6 100644 --- a/tests/pragmas/tinjectstmt.nim +++ b/tests/pragmas/tinjectstmt.nim @@ -7,15 +7,18 @@ ok0 ok1 onInject: 3 onInject: 4 -0 onInject: 5 +0 onInject: 6 -1 onInject: 7 onInject: 8 +1 +onInject: 9 +onInject: 10 +onInject: 11 2 ok2 -onInject: 9 +onInject: 12 ''' """ @@ -25,7 +28,7 @@ onInject: 9 {.injectStmt.} pragma can be used to inject a statement before every other statement in the current module. It's now undocumented and may be removed in the future and replaced with something more general and without its limitations. -e.g. (e.g. doesn't work in VM or js backends). +(e.g. doesn't work in VM or js backends). ]# from system/ansi_c import c_printf @@ -44,5 +47,5 @@ proc main()= echo a echo "ok2" -static: main() # xxx injectStmt not honred in VM +static: main() # xxx injectStmt not honored in VM main() diff --git a/tests/system/tdollars.nim b/tests/system/tdollars.nim index 34801d9ee..17d195e76 100644 --- a/tests/system/tdollars.nim +++ b/tests/system/tdollars.nim @@ -102,40 +102,41 @@ block: # #14350, #16674, #16686 for JS doAssert nil2 == cstring("") block: - block: - let x = -1'i8 - let y = uint32(x) - - doAssert $y == "4294967295" - - block: - let x = -1'i16 - let y = uint32(x) - - doAssert $y == "4294967295" - - block: - let x = -1'i32 - let y = uint32(x) - - doAssert $y == "4294967295" + when defined(js): # bug #18591 + let a1 = -1'i8 + let a2 = uint8(a1) + # if `uint8(a1)` changes meaning to `cast[uint8](a1)` in future, update this test; + # until then, this is the correct semantics. + let a3 = $a2 + doAssert a2 < 3 + doAssert a3 == "-1" + proc intToStr(a: uint8): cstring {.importjs: "(# + \"\")".} + doAssert $intToStr(a2) == "-1" + else: + block: + let x = -1'i8 + let y = uint32(x) + doAssert $y == "4294967295" + block: + let x = -1'i16 + let y = uint32(x) + doAssert $y == "4294967295" + block: + let x = -1'i32 + let y = uint32(x) + doAssert $y == "4294967295" + block: + proc foo1(arg: int): string = + let x = uint32(arg) + $x + doAssert $foo1(-1) == "4294967295" block: let x = 4294967295'u32 doAssert $x == "4294967295" - block: doAssert $(4294967295'u32) == "4294967295" - - block: - proc foo1(arg: int): string = - let x = uint32(arg) - $x - - doAssert $foo1(-1) == "4294967295" - - proc main()= block: let a = -0.0 @@ -159,6 +160,32 @@ proc main()= doAssert $uint32.high == "4294967295" + block: # addInt + var res = newStringOfCap(24) + template test2(a, b) = + res.setLen(0) + res.addInt a + doAssert res == b + + for i in 0 .. 9: + res.addInt int64(i) + doAssert res == "0123456789" + + res.setLen(0) + for i in -9 .. 0: + res.addInt int64(i) + doAssert res == "-9-8-7-6-5-4-3-2-10" + + when not defined(js): + test2 high(int64), "9223372036854775807" + test2 low(int64), "-9223372036854775808" + + test2 high(int32), "2147483647" + test2 low(int32), "-2147483648" + test2 high(int16), "32767" + test2 low(int16), "-32768" + test2 high(int8), "127" + test2 low(int8), "-128" static: main() main() diff --git a/tests/system/tstrmantle.nim b/tests/system/tstrmantle.nim deleted file mode 100644 index 1f195adde..000000000 --- a/tests/system/tstrmantle.nim +++ /dev/null @@ -1,46 +0,0 @@ -var res = newStringOfCap(24) - -for i in 0 .. 9: - res.addInt int64(i) - -doAssert res == "0123456789" - -res.setLen(0) - -for i in -9 .. 0: - res.addInt int64(i) - -doAssert res == "-9-8-7-6-5-4-3-2-10" - -res.setLen(0) -res.addInt high(int64) -doAssert res == "9223372036854775807" - -res.setLen(0) -res.addInt low(int64) -doAssert res == "-9223372036854775808" - -res.setLen(0) -res.addInt high(int32) -doAssert res == "2147483647" - -res.setLen(0) -res.addInt low(int32) -doAssert res == "-2147483648" - -res.setLen(0) -res.addInt high(int16) -doAssert res == "32767" - -res.setLen(0) -res.addInt low(int16) -doAssert res == "-32768" - - -res.setLen(0) -res.addInt high(int8) -doAssert res == "127" - -res.setLen(0) -res.addInt low(int8) -doAssert res == "-128" From 373bbd9bb4cbcb06fbfd80ab0e968127dc098873 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Thu, 19 Aug 2021 18:35:40 +0900 Subject: [PATCH 0335/1654] Add ceilDiv to math (#18596) * Use assert in runnableExamples and improve boundary check * Add more tests for ceilDiv * Fix comment in ceilDiv * Calling ceilDiv with int type T such like sizeof(T) > 8 is error --- changelog.md | 2 ++ lib/pure/math.nim | 52 ++++++++++++++++++++++++++++++++++++++++++ tests/stdlib/tmath.nim | 40 ++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/changelog.md b/changelog.md index 77a77f2f5..447401cd0 100644 --- a/changelog.md +++ b/changelog.md @@ -148,6 +148,8 @@ - Added `clamp` in `math` which allows using a `Slice` to clamp to a value. +- Added `ceilDiv` in `math` for round up integer division. + - The JSON module can now handle integer literals and floating point literals of arbitrary length and precision. Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 410660d4a..66f1a2e2c 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -941,6 +941,58 @@ func euclMod*[T: SomeNumber](x, y: T): T {.since: (1, 5, 1).} = if result < 0: result += abs(y) +func ceilDiv*[T: SomeInteger](x, y: T): T {.inline, since: (1, 5, 1).} = + ## Ceil division is conceptually defined as `ceil(x / y)`. + ## + ## Assumes `x >= 0` and `y > 0` (and `x + y - 1 <= high(T)` if T is SomeUnsignedInt). + ## + ## This is different from the `system.div `_ + ## operator, which works like `trunc(x / y)`. + ## That is, `div` rounds towards `0` and `ceilDiv` rounds up. + ## + ## This function has the above input limitation, because that allows the + ## compiler to generate faster code and it is rarely used with + ## negative values or unsigned integers close to `high(T)/2`. + ## If you need a `ceilDiv` that works with any input, see: + ## https://github.com/demotomohiro/divmath. + ## + ## **See also:** + ## * `system.div proc `_ for integer division + ## * `floorDiv func <#floorDiv,T,T>`_ for integer division which rounds down. + runnableExamples: + assert ceilDiv(12, 3) == 4 + assert ceilDiv(13, 3) == 5 + + when sizeof(T) == 8: + type UT = uint64 + elif sizeof(T) == 4: + type UT = uint32 + elif sizeof(T) == 2: + type UT = uint16 + elif sizeof(T) == 1: + type UT = uint8 + else: + {.fatal: "Unsupported int type".} + + assert x >= 0 and y > 0 + when T is SomeUnsignedInt: + assert x + y - 1 >= x + + # If the divisor is const, the backend C/C++ compiler generates code without a `div` + # instruction, as it is slow on most CPUs. + # If the divisor is a power of 2 and a const unsigned integer type, the + # compiler generates faster code. + # If the divisor is const and a signed integer, generated code becomes slower + # than the code with unsigned integers, because division with signed integers + # need to works for both positive and negative value without `idiv`/`sdiv`. + # That is why this code convert parameters to unsigned. + # This post contains a comparison of the performance of signed/unsigned integers: + # https://github.com/nim-lang/Nim/pull/18596#issuecomment-894420984. + # If signed integer arguments were not converted to unsigned integers, + # `ceilDiv` wouldn't work for any positive signed integer value, because + # `x + (y - 1)` can overflow. + ((x.UT + (y.UT - 1.UT)) div y.UT).T + func frexp*[T: float32|float64](x: T): tuple[frac: T, exp: int] {.inline.} = ## Splits `x` into a normalized fraction `frac` and an integral power of 2 `exp`, ## such that `abs(frac) in 0.5..<1` and `x == frac * 2 ^ exp`, except for special diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 2cf544ddb..9bc5b994f 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -106,6 +106,46 @@ template main() = doAssert euclDiv(-9, -3) == 3 doAssert euclMod(-9, -3) == 0 + block: # ceilDiv + doAssert ceilDiv(8, 3) == 3 + doAssert ceilDiv(8, 4) == 2 + doAssert ceilDiv(8, 5) == 2 + doAssert ceilDiv(11, 3) == 4 + doAssert ceilDiv(12, 3) == 4 + doAssert ceilDiv(13, 3) == 5 + doAssert ceilDiv(41, 7) == 6 + doAssert ceilDiv(0, 1) == 0 + doAssert ceilDiv(1, 1) == 1 + doAssert ceilDiv(1, 2) == 1 + doAssert ceilDiv(2, 1) == 2 + doAssert ceilDiv(2, 2) == 1 + doAssert ceilDiv(0, high(int)) == 0 + doAssert ceilDiv(1, high(int)) == 1 + doAssert ceilDiv(0, high(int) - 1) == 0 + doAssert ceilDiv(1, high(int) - 1) == 1 + doAssert ceilDiv(high(int) div 2, high(int) div 2 + 1) == 1 + doAssert ceilDiv(high(int) div 2, high(int) div 2 + 2) == 1 + doAssert ceilDiv(high(int) div 2 + 1, high(int) div 2) == 2 + doAssert ceilDiv(high(int) div 2 + 2, high(int) div 2) == 2 + doAssert ceilDiv(high(int) div 2 + 1, high(int) div 2 + 1) == 1 + doAssert ceilDiv(high(int), 1) == high(int) + doAssert ceilDiv(high(int) - 1, 1) == high(int) - 1 + doAssert ceilDiv(high(int) - 1, 2) == high(int) div 2 + doAssert ceilDiv(high(int) - 1, high(int)) == 1 + doAssert ceilDiv(high(int) - 1, high(int) - 1) == 1 + doAssert ceilDiv(high(int) - 1, high(int) - 2) == 2 + doAssert ceilDiv(high(int), high(int)) == 1 + doAssert ceilDiv(high(int), high(int) - 1) == 2 + doAssert ceilDiv(255'u8, 1'u8) == 255'u8 + doAssert ceilDiv(254'u8, 2'u8) == 127'u8 + when not defined(danger): + doAssertRaises(AssertionDefect): discard ceilDiv(41, 0) + doAssertRaises(AssertionDefect): discard ceilDiv(41, -1) + doAssertRaises(AssertionDefect): discard ceilDiv(-1, 1) + doAssertRaises(AssertionDefect): discard ceilDiv(-1, -1) + doAssertRaises(AssertionDefect): discard ceilDiv(254'u8, 3'u8) + doAssertRaises(AssertionDefect): discard ceilDiv(255'u8, 2'u8) + block: # splitDecimal() tests doAssert splitDecimal(54.674).intpart == 54.0 doAssert splitDecimal(54.674).floatpart ==~ 0.674 From 8fa0decf6b4917e5559b9fd84a18825849dc11df Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 19 Aug 2021 21:54:45 +0800 Subject: [PATCH 0336/1654] fix a typo (#18715) --- compiler/semcall.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index d8dba3e6c..2478420f7 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -280,7 +280,7 @@ const proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = # Gives a detailed error message; this is separated from semOverloadedCall, - # as semOverlodedCall is already pretty slow (and we need this information + # as semOverloadedCall is already pretty slow (and we need this information # only in case of an error). if c.config.m.errorOutputs == {}: # fail fast: From 13b97291835046eb1368ff4cd240cdcc7d0a2899 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 20 Aug 2021 01:56:52 +0800 Subject: [PATCH 0337/1654] fix #18627(Program segfaults with ARC when using openArray[string]) (#18713) * fix #18627 * add testcase * rename * another * remove tyVarargs --- compiler/ast.nim | 2 +- tests/arc/topenarray.nim | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/arc/topenarray.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 0bdcc0e97..e7507c9c6 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1595,7 +1595,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) = if mask != {} and propagateHasAsgn: let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink}) if o2.kind in {tyTuple, tyObject, tyArray, - tySequence, tySet, tyDistinct, tyOpenArray, tyVarargs}: + tySequence, tySet, tyDistinct}: o2.flags.incl mask owner.flags.incl mask diff --git a/tests/arc/topenarray.nim b/tests/arc/topenarray.nim new file mode 100644 index 000000000..03ec7adf2 --- /dev/null +++ b/tests/arc/topenarray.nim @@ -0,0 +1,24 @@ +discard """ + input: "hi" + output: ''' +hi +Nim +''' + matrix: "--gc:arc -d:useMalloc; --gc:arc" +""" +block: # bug 18627 + proc setPosition(params: openArray[string]) = + for i in params.toOpenArray(0, params.len - 1): + echo i + + proc uciLoop() = + let params = @[readLine(stdin)] + setPosition(params) + + uciLoop() + + proc uciLoop2() = + let params = @["Nim"] + for i in params.toOpenArray(0, params.len - 1): + echo i + uciLoop2() From f2910077ac41a614154f3851f5bffc02b06124a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 20 Aug 2021 01:13:03 -0700 Subject: [PATCH 0338/1654] sigmatch: support optional params with last block arg(s) (#18631) * sigmatch: support optional params with last block arg * add tests * works with multiple block args * cleanup * address comment --- compiler/sigmatch.nim | 14 +++++++- tests/misc/trfc405.nim | 81 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 tests/misc/trfc405.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 674dd7e2e..465a9a51d 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2305,6 +2305,17 @@ proc incrIndexType(t: PType) = template isVarargsUntyped(x): untyped = x.kind == tyVarargs and x[0].kind == tyUntyped +proc findFirstArgBlock(m: var TCandidate, n: PNode): int = + # see https://github.com/nim-lang/RFCs/issues/405 + result = int.high + for a2 in countdown(n.len-1, 0): + # checking `nfBlockArg in n[a2].flags` wouldn't work inside templates + if n[a2].kind != nkStmtList: break + let formalLast = m.callee.n[m.callee.n.len - (n.len - a2)] + if formalLast.kind == nkSym and formalLast.sym.ast == nil: + result = a2 + else: break + proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var IntSet) = template noMatch() = @@ -2345,7 +2356,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int formalLen = m.callee.n.len formal = if formalLen > 1: m.callee.n[1].sym else: nil # current routine parameter container: PNode = nil # constructed container - + let firstArgBlock = findFirstArgBlock(m, n) while a < n.len: c.openShadowScope @@ -2441,6 +2452,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int if m.callee.n[f].kind != nkSym: internalError(c.config, n[a].info, "matches") noMatch() + if a >= firstArgBlock: f = max(f, m.callee.n.len - (n.len - a)) formal = m.callee.n[f].sym m.firstMismatch.kind = kTypeMismatch if containsOrIncl(marker, formal.position) and container.isNil: diff --git a/tests/misc/trfc405.nim b/tests/misc/trfc405.nim new file mode 100644 index 000000000..adc39b1df --- /dev/null +++ b/tests/misc/trfc405.nim @@ -0,0 +1,81 @@ +# https://github.com/nim-lang/RFCs/issues/405 + +template main = + template fn1(a = 1, b = 2, body): auto = (a, b, astToStr(body)) + let a1 = fn1(10, 20): + foo + doAssert a1 == (10, 20, "\nfoo") + + template fn2(a = 1, b = 2, body): auto = (a, b, astToStr(body)) + let a2 = fn2(a = 10): foo + doAssert a2 == (10, 2, "\nfoo") + let a2b = fn2(b = 20): foo + doAssert a2b == (1, 20, "\nfoo") + + template fn3(x: int, a = 1, b = 2, body): auto = (a, b, astToStr(body)) + let a3 = fn3(3, 10, 20): foo + doAssert a3 == (10, 20, "\nfoo") + let a3b = fn3(3, a = 10): foo + doAssert a3b == (10, 2, "\nfoo") + + template fn4(x: int, y: int, body): auto = (x, y, astToStr(body)) + let a4 = fn4(1, 2): foo + doAssert a4 == (1, 2, "\nfoo") + + template fn5(x = 1, y = 2, body: untyped = 3): auto = (x, y, astToStr(body)) + doAssert compiles(fn5(1, 2, foo)) + doAssert not compiles(fn5(1, foo)) + + block: + # with an overload + var witness = 0 + template fn6() = discard + template fn6(procname: string, body: untyped): untyped = witness.inc + fn6("abc"): discard + assert witness == 1 + + block: + # with overloads + var witness = 0 + template fn6() = discard + template fn6(a: int) = discard + template fn6(procname: string, body: untyped): untyped = witness.inc + fn6("abc"): discard + assert witness == 1 + + template fn6(b = 1.5, body: untyped): untyped = witness.inc + fn6(1.3): discard + assert witness == 2 + + block: + var witness = 0 + template fn6(a: int) = discard + template fn6(a: string) = discard + template fn6(ignore: string, b = 1.5, body: untyped): untyped = witness.inc + fn6(""): + foobar1 + foobar2 + doAssert witness == 1 + fn6(""): discard + doAssert witness == 2 + + block: # multi block args + template fn8(a = 1, b = 2, body1: untyped, body2: untyped): auto = (a, b, astToStr(body1), astToStr(body2)) + let a1 = fn8(): + foobar1 + foobar2 + do: + foobar3 + foobar4 + doAssert a1 == (1, 2, "\nfoobar1\nfoobar2", "\nfoobar3\nfoobar4") + + let a2 = fn8(b = 20): + foobar1 + foobar2 + do: + foobar3 + foobar4 + doAssert a2 == (1, 20, "\nfoobar1\nfoobar2", "\nfoobar3\nfoobar4") + +static: main() +main() From e52221e2967d7511f26dc020cae5fe62dcae5696 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 21 Aug 2021 02:00:04 +0800 Subject: [PATCH 0339/1654] [docs]fix an item missing (#18717) --- doc/astspec.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/astspec.txt b/doc/astspec.txt index 92212bf8c..6d44f7f7e 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -364,6 +364,7 @@ AST: nnkExprColonExpr(nnkIdent("a"), nnkIntLit(1)), nnkExprColonExpr(nnkIdent("b"), nnkIntLit(2)), nnkExprColonExpr(nnkIdent("c"), nnkIntLit(3))) + nnkTupleConstr() Since the one tuple would be syntactically identical to parentheses with an expression in them, the parser expects a trailing comma for From f0c6593412c117c7f8e19421016303416d6b8bd3 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 21 Aug 2021 14:21:06 +0800 Subject: [PATCH 0340/1654] add testcase for overloadable_enums (#18722) * add testcase for overloadable_enums * link --- tests/enum/toverloadable_enums.nim | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/enum/toverloadable_enums.nim b/tests/enum/toverloadable_enums.nim index 6f3aac3e2..2152d7fff 100644 --- a/tests/enum/toverloadable_enums.nim +++ b/tests/enum/toverloadable_enums.nim @@ -46,3 +46,32 @@ proc takeCallback(param: proc(p: int)) = discard takeCallback x echo ord v + +block: # https://github.com/nim-lang/RFCs/issues/8 + type + Enum1 = enum + A, B, C + Enum2 = enum + A, Z + + proc f(e: Enum1): int = ord(e) + proc g(e: Enum2): int = ord(e) + + proc h(e: Enum1): int = ord(e) + proc h(e: Enum2): int = ord(e) + + let fA = f(A) # Type of A is well defined + let gA = g(A) # Same as above + + let hA1 = h(Enum1.A) # A requires disambiguation + let hA2 = h(Enum2.A) # Similarly + let hA3 = h(B) + let hA4 = h(B) + let x = ord(Enum1.A) # Also + doAssert fA == 0 + doAssert gA == 0 + doAssert hA1 == 0 + doAssert hA2 == 0 + doAssert x == 0 + doAssert hA3 == 1 + doAssert hA4 == 1 From 5b26f2bd81d6fc7d48befbfb4fa3317f713af787 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 21 Aug 2021 14:22:00 +0800 Subject: [PATCH 0341/1654] fix deprecated example (#18721) --- doc/astspec.txt | 16 ++++++++++++---- tests/astspec/tastspec.nim | 22 +++++++++++++++------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/doc/astspec.txt b/doc/astspec.txt index 6d44f7f7e..6d3fa1f8c 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -1082,7 +1082,7 @@ its entirety to see some of the complexities. Concrete syntax: .. code-block:: nim - type Obj[T] = object {.inheritable.} + type Obj[T] {.inheritable.} = object name: string case isFat: bool of true: @@ -1094,10 +1094,18 @@ AST: .. code-block:: nim # ... + nnkPragmaExpr( + nnkIdent("Obj"), + nnkPragma(nnkIdent("inheritable")) + ), + nnkGenericParams( + nnkIdentDefs( + nnkIdent("T"), + nnkEmpty(), + nnkEmpty()) + ), nnkObjectTy( - nnkPragma( - nnkIdent("inheritable") - ), + nnkEmpty(), nnkEmpty(), nnkRecList( # list of object parameters nnkIdentDefs( diff --git a/tests/astspec/tastspec.nim b/tests/astspec/tastspec.nim index 33a245b1b..c99d8ec79 100644 --- a/tests/astspec/tastspec.nim +++ b/tests/astspec/tastspec.nim @@ -871,11 +871,20 @@ static: scope: macro testRecCase(ast: untyped): untyped = - ast.peelOff({nnkStmtList, nnkTypeSection})[2].matchAst: - of nnkObjectTy( - nnkPragma( - ident"inheritable" + ast.peelOff({nnkStmtList, nnkTypeSection}).matchAst: + of nnkTypeDef( + nnkPragmaExpr( + ident"Obj", + nnkPragma(ident"inheritable") ), + nnkGenericParams( + nnkIdentDefs( + ident"T", + nnkEmpty(), + nnkEmpty()) + ), + nnkObjectTy( + nnkEmpty(), nnkEmpty(), nnkRecList( # list of object parameters nnkIdentDefs( @@ -914,6 +923,7 @@ static: ident"T" ), nnkEmpty() + ) ) ) ) @@ -922,10 +932,8 @@ static: ): echo "ok" - - testRecCase: - type Obj[T] = object {.inheritable.} + type Obj[T] {.inheritable.} = object name: string case isFat: bool of true: From 061a9183f7b49289b05cf134893d703760f467a7 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 22 Aug 2021 12:21:53 +0800 Subject: [PATCH 0342/1654] replace wrt with proper word (#18724) * what does wrt mean? * clarify --- compiler/ast.nim | 2 +- compiler/docgen.nim | 2 +- compiler/sighashes.nim | 2 +- lib/core/macros.nim | 6 ++++-- tests/iter/titer3.nim | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index e7507c9c6..446c00f49 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -860,7 +860,7 @@ type case kind*: TSymKind of routineKinds: #procInstCache*: seq[PInstantiation] - gcUnsafetyReason*: PSym # for better error messages wrt gcsafe + gcUnsafetyReason*: PSym # for better error messages regarding gcsafe transformedBody*: PNode # cached body after transf pass of skLet, skVar, skField, skForVar: guard*: PSym diff --git a/compiler/docgen.nim b/compiler/docgen.nim index d39a20aae..706b54bcd 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -172,7 +172,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile): RelativeFile = if result.isEmpty: bail() elif conf.docRoot.len > 0: # we're (currently) requiring `isAbsolute` to avoid confusion when passing - # a relative path (would it be relative wrt $PWD or to projectfile) + # a relative path (would it be relative with regard to $PWD or to projectfile) conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot conf.globalAssert conf.docRoot.dirExists, arg=conf.docRoot # needed because `canonicalizePath` called on `file` diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 0ff97017f..8534f4c0c 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -390,7 +390,7 @@ proc idOrSig*(s: PSym, currentModule: string, sigCollisions: var CountTable[SigHash]): Rope = if s.kind in routineKinds and s.typ != nil: # signatures for exported routines are reliable enough to - # produce a unique name and this means produced C++ is more stable wrt + # produce a unique name and this means produced C++ is more stable regarding # Nim changes: let sig = hashProc(s) result = rope($sig) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index d3eaf6298..645f5067a 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -442,9 +442,11 @@ proc ident*(name: string): NimNode {.magic: "StrToIdent", noSideEffect.} ## Create a new ident node from a string. type - BindSymRule* = enum ## specifies how `bindSym` behaves + BindSymRule* = enum ## Specifies how `bindSym` behaves. The difference + ## between open and closed symbols can be found in + ## ``_ brClosed, ## only the symbols in current scope are bound - brOpen, ## open wrt overloaded symbols, but may be a single + brOpen, ## open for overloaded symbols, but may be a single ## symbol if not ambiguous (the rules match that of ## binding in generics) brForceOpen ## same as brOpen, but it will always be open even diff --git a/tests/iter/titer3.nim b/tests/iter/titer3.nim index 9dcfd7be5..defd56c98 100644 --- a/tests/iter/titer3.nim +++ b/tests/iter/titer3.nim @@ -26,7 +26,7 @@ var x = [[1, 2, 3], [4, 5, 6]] for y in iter1(x[0]): write(stdout, $y) writeLine(stdout, "") -# ensure closure and inline iterators have the same behaviour wrt +# ensure closure and inline iterators have the same behaviour regarding # parameter passing iterator clo(a: int): int {.closure.} = From cc08d5c2c641e9390e9e05012defc9616f5d6ba8 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 22 Aug 2021 13:40:20 +0800 Subject: [PATCH 0343/1654] fix #17898(randomPathName called twice in a row can return the same string on windows) (#18729) * close #17898 * no need to consider js --- lib/std/tempfiles.nim | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index c97db67b6..f1cfd5885 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -17,7 +17,7 @@ See also: * `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html ]# -import os, random +import os, random, std/monotimes const @@ -96,11 +96,25 @@ proc safeOpen(filename: string): File = discard posix.close(fileHandle) # TODO handles failure when closing file raiseOSError(osLastError(), filename) + +type + NimTempPathState = object + state: Rand + isInit: bool + +var nimTempPathState {.threadvar.}: NimTempPathState + template randomPathName(length: Natural): string = var res = newString(length) - var state = initRand() + if not nimTempPathState.isInit: + var time = getMonoTime().ticks + when compileOption("threads"): + time = time xor int64(getThreadId()) + nimTempPathState.isInit = true + nimTempPathState.state = initRand(time) + for i in 0 ..< length: - res[i] = state.sample(letters) + res[i] = nimTempPathState.state.sample(letters) res proc getTempDirImpl(dir: string): string {.inline.} = From e65dc36ddd6bbb1a87a607ee8c8491732730fa95 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 22 Aug 2021 16:32:55 -0300 Subject: [PATCH 0344/1654] Implement RFC-391 (#18585) --- lib/std/jsfetch.nim | 177 ++++++++++++++++++++++---------------------- tests/config.nims | 1 - tools/kochdocs.nim | 2 +- 3 files changed, 89 insertions(+), 91 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 1e7bb4d29..a25fe7ca8 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -1,121 +1,120 @@ ## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API -## .. Note:: jsfetch is Experimental. jsfetch module requires `-d:nimExperimentalJsfetch` +## .. Note:: jsfetch is Experimental. when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} -when defined(nimExperimentalJsfetch): - import std/[asyncjs, jsheaders, jsformdata] - from std/httpcore import HttpMethod - from std/jsffi import JsObject +import std/[asyncjs, jsheaders, jsformdata] +from std/httpcore import HttpMethod +from std/jsffi import JsObject - type - FetchOptions* = ref object of JsRoot ## Options for Fetch API. - keepalive*: bool - metod* {.importjs: "method".}: cstring - body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring +type + FetchOptions* = ref object of JsRoot ## Options for Fetch API. + keepalive*: bool + metod* {.importjs: "method".}: cstring + body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring - FetchModes* = enum ## Mode options. - fmCors = "cors" - fmNoCors = "no-cors" - fmSameOrigin = "same-origin" + FetchModes* = enum ## Mode options. + fmCors = "cors" + fmNoCors = "no-cors" + fmSameOrigin = "same-origin" - FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials - fcInclude = "include" - fcSameOrigin = "same-origin" - fcOmit = "omit" + FetchCredentials* = enum ## Credential options. See https://developer.mozilla.org/en-US/docs/Web/API/Request/credentials + fcInclude = "include" + fcSameOrigin = "same-origin" + fcOmit = "omit" - FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache - fchDefault = "default" - fchNoStore = "no-store" - fchReload = "reload" - fchNoCache = "no-cache" - fchForceCache = "force-cache" + FetchCaches* = enum ## https://developer.mozilla.org/docs/Web/API/Request/cache + fchDefault = "default" + fchNoStore = "no-store" + fchReload = "reload" + fchNoCache = "no-cache" + fchForceCache = "force-cache" - FetchRedirects* = enum ## Redirects options. - frFollow = "follow" - frError = "error" - frManual = "manual" + FetchRedirects* = enum ## Redirects options. + frFollow = "follow" + frError = "error" + frManual = "manual" - FetchReferrerPolicies* = enum ## Referrer Policy options. - frpNoReferrer = "no-referrer" - frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" - frpOrigin = "origin" - frpOriginWhenCrossOrigin = "origin-when-cross-origin" - frpUnsafeUrl = "unsafe-url" + FetchReferrerPolicies* = enum ## Referrer Policy options. + frpNoReferrer = "no-referrer" + frpNoReferrerWhenDowngrade = "no-referrer-when-downgrade" + frpOrigin = "origin" + frpOriginWhenCrossOrigin = "origin-when-cross-origin" + frpUnsafeUrl = "unsafe-url" - Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body - bodyUsed*: bool + Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body + bodyUsed*: bool - Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body + Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: Body - Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request - bodyUsed*, ok*, redirected*: bool - typ* {.importjs: "type".}: cstring - url*, statusText*: cstring - status*: cint - headers*: Headers - body*: Body + Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request + bodyUsed*, ok*, redirected*: bool + typ* {.importjs: "type".}: cstring + url*, statusText*: cstring + status*: cint + headers*: Headers + body*: Body - func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} - ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. +func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} + ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. - func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} - ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. +func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} + ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. - func clone*(self: Response | Request): Response {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone +func clone*(self: Response | Request): Response {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone - proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text +proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text - proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json +proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json - proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData +proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData - proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; - keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: - "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} - ## .. warning:: Unsafe `newfetchOptions`. +proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} + ## .. warning:: Unsafe `newfetchOptions`. - func newfetchOptions*(metod: HttpMethod; body: cstring; - mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = - ## Constructor for `FetchOptions`. - result = FetchOptions( - body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, - keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, - metod: (case metod - of HttpHead: "HEAD".cstring - of HttpGet: "GET".cstring - of HttpPost: "POST".cstring - of HttpPut: "PUT".cstring - of HttpDelete: "DELETE".cstring - of HttpPatch: "PATCH".cstring - else: "GET".cstring - ) +func newfetchOptions*(metod: HttpMethod; body: cstring; + mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; + keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = + ## Constructor for `FetchOptions`. + result = FetchOptions( + body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, + keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, + metod: (case metod + of HttpHead: "HEAD".cstring + of HttpGet: "GET".cstring + of HttpPost: "POST".cstring + of HttpPut: "PUT".cstring + of HttpDelete: "DELETE".cstring + of HttpPatch: "PATCH".cstring + else: "GET".cstring ) + ) - proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} - ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. +proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} + ## `fetch()` API, simple `GET` only, returns a `Future[Response]`. - proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} - ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. +proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} + ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. - func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} +func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} - func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) +func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) -runnableExamples("-d:nimExperimentalJsfetch -r:off"): +runnableExamples("-r:off"): import std/[asyncjs, jsconsole, jsheaders, jsformdata] from std/httpcore import HttpMethod from std/jsffi import JsObject diff --git a/tests/config.nims b/tests/config.nims index d7f6ae7f9..0327f0b76 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -33,7 +33,6 @@ hint("Processing", off) # experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 # sync with `kochdocs.docDefines` or refactor. switch("define", "nimExperimentalAsyncjsThen") -switch("define", "nimExperimentalJsfetch") switch("define", "nimExperimentalLinenoiseExtra") switch("define", "nimPreviewFloatRoundtrip") diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 8e8085f73..fe3335c0f 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -8,7 +8,7 @@ const gaCode* = " --doc.googleAnalytics:UA-48159761-1" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalJsfetch -d:nimExperimentalLinenoiseExtra" + docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalLinenoiseExtra" nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" From 2d84f6e7c836bf18c0ecbe07fb413166861196ac Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 23 Aug 2021 00:13:39 +0200 Subject: [PATCH 0345/1654] Fix #18718 (#18731) Update tests --- lib/pure/stats.nim | 13 +++--- tests/stdlib/tstats.nim | 93 ++++++++++++++++++++++------------------- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index e46dff212..58014e503 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -42,7 +42,7 @@ runnableExamples: template `~=`(a, b: float): bool = almostEqual(a, b) - var statistics: RunningStat ## Must be var + var statistics: RunningStat # must be var statistics.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) doAssert statistics.n == 8 doAssert statistics.mean() ~= 2.0 @@ -76,7 +76,7 @@ type proc clear*(s: var RunningStat) = ## Resets `s`. s.n = 0 - s.min = toBiggestFloat(int.high) + s.min = 0.0 s.max = 0.0 s.sum = 0.0 s.mom1 = 0.0 @@ -86,11 +86,14 @@ proc clear*(s: var RunningStat) = proc push*(s: var RunningStat, x: float) = ## Pushes a value `x` for processing. - if s.n == 0: s.min = x + if s.n == 0: + s.min = x + s.max = x + else: + if s.min > x: s.min = x + if s.max < x: s.max = x inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.min > x: s.min = x - if s.max < x: s.max = x s.sum += x let n = toFloat(s.n) let delta = x - s.mom1 diff --git a/tests/stdlib/tstats.nim b/tests/stdlib/tstats.nim index 37240c884..92a2ed8b8 100644 --- a/tests/stdlib/tstats.nim +++ b/tests/stdlib/tstats.nim @@ -1,46 +1,55 @@ -include stats +import std/stats -proc clean(x: float): float = - result = round(1.0e8*x).float * 1.0e-8 +proc `~=`(x, y: float): bool = + abs(x - y) < 10e-8 -var rs: RunningStat -rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) -doAssert(rs.n == 8) -doAssert(clean(rs.mean) == 2.0) -doAssert(clean(rs.variance()) == 1.5) -doAssert(clean(rs.varianceS()) == 1.71428571) -doAssert(clean(rs.skewness()) == 0.81649658) -doAssert(clean(rs.skewnessS()) == 1.01835015) -doAssert(clean(rs.kurtosis()) == -1.0) -doAssert(clean(rs.kurtosisS()) == -0.7000000000000001) +template main() = + var rs: RunningStat + rs.push(@[1.0, 2.0, 1.0, 4.0, 1.0, 4.0, 1.0, 2.0]) + doAssert(rs.n == 8) + doAssert rs.mean ~= 2.0 + doAssert rs.variance() ~= 1.5 + doAssert rs.varianceS() ~= 1.71428571 + doAssert rs.skewness() ~= 0.81649658 + doAssert rs.skewnessS() ~= 1.01835015 + doAssert rs.kurtosis() ~= -1.0 + doAssert rs.kurtosisS() ~= -0.7000000000000001 -var rs1, rs2: RunningStat -rs1.push(@[1.0, 2.0, 1.0, 4.0]) -rs2.push(@[1.0, 4.0, 1.0, 2.0]) -let rs3 = rs1 + rs2 -doAssert(clean(rs3.mom2) == clean(rs.mom2)) -doAssert(clean(rs3.mom3) == clean(rs.mom3)) -doAssert(clean(rs3.mom4) == clean(rs.mom4)) -rs1 += rs2 -doAssert(clean(rs1.mom2) == clean(rs.mom2)) -doAssert(clean(rs1.mom3) == clean(rs.mom3)) -doAssert(clean(rs1.mom4) == clean(rs.mom4)) -rs1.clear() -rs1.push(@[1.0, 2.2, 1.4, 4.9]) -doAssert(rs1.sum == 9.5) -doAssert(rs1.mean() == 2.375) + var rs1, rs2: RunningStat + rs1.push(@[1.0, 2.0, 1.0, 4.0]) + rs2.push(@[1.0, 4.0, 1.0, 2.0]) + let rs3 = rs1 + rs2 + doAssert rs3.variance ~= rs.variance + doAssert rs3.skewness ~= rs.skewness + doAssert rs3.kurtosis ~= rs.kurtosis + rs1 += rs2 + doAssert rs1.variance ~= rs.variance + doAssert rs1.skewness ~= rs.skewness + doAssert rs1.kurtosis ~= rs.kurtosis + rs1.clear() + rs1.push(@[1.0, 2.2, 1.4, 4.9]) + doAssert(rs1.sum == 9.5) + doAssert(rs1.mean() == 2.375) -when not defined(cpu32): - # XXX For some reason on 32bit CPUs these results differ - var rr: RunningRegress - rr.push(@[0.0, 1.0, 2.8, 3.0, 4.0], @[0.0, 1.0, 2.3, 3.0, 4.0]) - doAssert(rr.slope() == 0.9695585996955861) - doAssert(rr.intercept() == -0.03424657534246611) - doAssert(rr.correlation() == 0.9905100362239381) - var rr1, rr2: RunningRegress - rr1.push(@[0.0, 1.0], @[0.0, 1.0]) - rr2.push(@[2.8, 3.0, 4.0], @[2.3, 3.0, 4.0]) - let rr3 = rr1 + rr2 - doAssert(rr3.correlation() == rr.correlation()) - doAssert(clean(rr3.slope()) == clean(rr.slope())) - doAssert(clean(rr3.intercept()) == clean(rr.intercept())) \ No newline at end of file + when not defined(cpu32): + # XXX For some reason on 32bit CPUs these results differ + var rr: RunningRegress + rr.push(@[0.0, 1.0, 2.8, 3.0, 4.0], @[0.0, 1.0, 2.3, 3.0, 4.0]) + doAssert(rr.slope() == 0.9695585996955861) + doAssert(rr.intercept() == -0.03424657534246611) + doAssert(rr.correlation() == 0.9905100362239381) + var rr1, rr2: RunningRegress + rr1.push(@[0.0, 1.0], @[0.0, 1.0]) + rr2.push(@[2.8, 3.0, 4.0], @[2.3, 3.0, 4.0]) + let rr3 = rr1 + rr2 + doAssert(rr3.correlation() == rr.correlation()) + doAssert rr3.slope() ~= rr.slope() + doAssert rr3.intercept() ~= rr.intercept() + + block: # bug #18718 + var rs: RunningStat + rs.push(-1.0) + doAssert rs.max == -1.0 + +static: main() +main() From 8a8267e2bc36ac8070fa2842fecd9cfe53e0a9ef Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 23 Aug 2021 21:05:22 +0800 Subject: [PATCH 0346/1654] fix a typo (#18736) --- compiler/sighashes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 8534f4c0c..01a0fa784 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -166,7 +166,7 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = if t.n.len > 0: let oldFlags = symWithFlags.flags # Hack to prevent endless recursion - # xxx intead, use a hash table to indicate we've already visited a type, which + # xxx instead, use a hash table to indicate we've already visited a type, which # would also be more efficient. symWithFlags.flags.excl {sfAnon, sfGenSym} hashTree(c, t.n, flags + {CoHashTypeInsideNode}) From 9674ff4361468137afdbedb86196b7082aaa6efd Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 24 Aug 2021 02:13:10 +0800 Subject: [PATCH 0347/1654] make deprecated message better (#18737) --- lib/pure/mersenne.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index 6ec0d66b4..37c5085b1 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -8,7 +8,7 @@ ## random number generator. ## .. note:: The procs in this module work at compile-time. -{.deprecated: "use `std/random` instead.".} +{.deprecated: "use `std/random` instead".} runnableExamples: var rand = newMersenneTwister(uint32.high) ## must be "var" From 24178bf8aa37c179caab9e16682940e8340f2bfc Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 24 Aug 2021 17:15:26 +0200 Subject: [PATCH 0348/1654] get rid of the warnings during bootstrapping (#18741) * fix bootstrapping hints and warnings * revert removals in ccgtypes --- compiler/nimlexbase.nim | 4 ++-- compiler/semfold.nim | 2 +- compiler/sighashes.nim | 2 +- compiler/suggest.nim | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/nimlexbase.nim b/compiler/nimlexbase.nim index 99d07d093..b8cb9f78a 100644 --- a/compiler/nimlexbase.nim +++ b/compiler/nimlexbase.nim @@ -104,7 +104,7 @@ proc fillBuffer(L: var TBaseLexer) = oldBufLen = L.bufLen L.bufLen = L.bufLen * 2 L.bufStorage.setLen(L.bufLen) - L.buf = L.bufStorage + L.buf = L.bufStorage.cstring assert(L.bufLen - oldBufLen == oldBufLen) charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]), oldBufLen) @@ -147,7 +147,7 @@ proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) = L.bufpos = 0 L.offsetBase = 0 L.bufStorage = newString(bufLen) - L.buf = L.bufStorage + L.buf = L.bufStorage.cstring L.bufLen = bufLen L.sentinel = bufLen - 1 L.lineStart = 0 diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 04ed73209..1f8efc4de 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -144,7 +144,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P if a.typ.kind == tyString: result = newIntNodeT(toInt128(a.strVal.len), n, idgen, g) elif a.typ.kind == tyCstring: - result = newIntNodeT(toInt128(nimCStrLen(a.strVal)), n, idgen, g) + result = newIntNodeT(toInt128(nimCStrLen(a.strVal.cstring)), n, idgen, g) else: result = newIntNodeT(toInt128(a.len), n, idgen, g) of mUnaryPlusI, mUnaryPlusF64: result = a # throw `+` away diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 01a0fa784..d02728627 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -18,7 +18,7 @@ proc `&=`(c: var MD5Context, ch: char) = # XXX suspicious code here; relies on ch being zero terminated? md5Update(c, unsafeAddr ch, 1) proc `&=`(c: var MD5Context, r: Rope) = - for l in leaves(r): md5Update(c, l, l.len) + for l in leaves(r): md5Update(c, l.cstring, l.len) proc `&=`(c: var MD5Context, i: BiggestInt) = md5Update(c, cast[cstring](unsafeAddr i), sizeof(i)) proc `&=`(c: var MD5Context, f: BiggestFloat) = diff --git a/compiler/suggest.nim b/compiler/suggest.nim index eaa30040a..84c94d793 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -32,7 +32,7 @@ # included from sigmatch.nim -import algorithm, sets, prefixmatches, lineinfos, parseutils, linter, tables +import algorithm, sets, prefixmatches, parseutils, tables from wordrecg import wDeprecated, wError, wAddr, wYield when defined(nimsuggest): From 3d1bba04ab1092630983695734d6984ddff4688c Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 24 Aug 2021 23:16:12 +0800 Subject: [PATCH 0349/1654] [minor]use `addInt` and `addFloat` (#18733) * unify int and uint * back * minor --- compiler/ic/ic.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index bd3be27f0..7387d165b 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -94,10 +94,10 @@ proc toString*(tree: PackedTree; n: NodePos; m: PackedModule; nesting: int; result.addInt m.numbers[LitId tree.nodes[pos].operand] of externUIntLit: result.add " " - result.add $cast[uint64](m.numbers[LitId tree.nodes[pos].operand]) + result.addInt cast[uint64](m.numbers[LitId tree.nodes[pos].operand]) of nkFloatLit..nkFloat128Lit: result.add " " - result.add $cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand]) + result.addFloat cast[BiggestFloat](m.numbers[LitId tree.nodes[pos].operand]) else: result.add "(\n" for i in 1..(nesting+1)*2: result.add ' ' From 3aa16c1de00c723d48e48fe3fdf07a276d1b4b6a Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 24 Aug 2021 21:50:40 -0700 Subject: [PATCH 0350/1654] fix RFC #341: dot-like operators are now parsed with same precedence as `.` (#18711) * fix RFC #341: dot-like operators are now parsed with same precedence as `.` * fixup * [skip ci] address comment in changelog * address comment * update grammmar * add manual entry * fixup * -d:nimPreviewDotLikeOps * address comment to unblock PR: move nimPreviewDotLikeOps out of config/config.nims --- changelog.md | 4 ++++ compiler/lineinfos.nim | 2 ++ compiler/parser.nim | 34 +++++++++++++++++++++++++++++----- doc/grammar.txt | 2 ++ doc/manual.rst | 9 +++++++++ tests/config.nims | 1 + tests/misc/trfc_341.nim | 27 +++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 tests/misc/trfc_341.nim diff --git a/changelog.md b/changelog.md index 447401cd0..3ba008cf3 100644 --- a/changelog.md +++ b/changelog.md @@ -104,6 +104,10 @@ - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- With `-d:nimPreviewDotLikeOps` (default in devel), dot-like operators (operators starting with `.`, but not with `..`) + now have the same precedence as `.`, so that `a.?b.c` is now parsed as `(a.?b).c` instead of `a.?(b.c)`. + A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`. + - The allocator for Nintendo Switch, which was nonfunctional because of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index c64d7b9dd..fd7ce0c04 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -46,6 +46,7 @@ type warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape", warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit", warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated", + warnDotLikeOps = "DotLikeOps", warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnRstRedefinitionOfLabel = "RedefinitionOfLabel", warnRstUnknownSubstitutionX = "UnknownSubstitutionX", @@ -116,6 +117,7 @@ const warnXmightNotBeenInit: "'$1' might not have been initialized", warnDeprecated: "$1", warnConfigDeprecated: "config file '$1' is deprecated", + warnDotLikeOps: "$1", warnSmallLshouldNotBeUsed: "'l' should not be used as an identifier; may look like '1' (one)", warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnRstRedefinitionOfLabel: "redefinition of label '$1'", diff --git a/compiler/parser.nim b/compiler/parser.nim index b089614b2..30180e545 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -466,6 +466,17 @@ proc dotExpr(p: var Parser, a: PNode): PNode = exprColonEqExprListAux(p, tkParRi, y) result = y +proc dotLikeExpr(p: var Parser, a: PNode): PNode = + #| dotLikeExpr = expr DOTLIKEOP optInd symbol + var info = p.parLineInfo + result = newNodeI(nkInfix, info) + optInd(p, result) + var opNode = newIdentNodeP(p.tok.ident, p) + getTok(p) + result.add(opNode) + result.add(a) + result.add(parseSymbol(p, smAfterDot)) + proc qualifiedIdent(p: var Parser): PNode = #| qualifiedIdent = symbol ('.' optInd symbol)? result = parseSymbol(p) @@ -788,10 +799,15 @@ proc commandExpr(p: var Parser; r: PNode; mode: PrimaryMode): PNode = p.hasProgress = false result.add commandParam(p, isFirstParam, mode) +proc isDotLike(tok: Token): bool = + result = tok.tokType == tkOpr and tok.ident.s.len > 1 and + tok.ident.s[0] == '.' and tok.ident.s[1] != '.' + proc primarySuffix(p: var Parser, r: PNode, baseIndent: int, mode: PrimaryMode): PNode = #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' #| | '.' optInd symbol generalizedLit? + #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax @@ -840,11 +856,19 @@ proc primarySuffix(p: var Parser, r: PNode, # `foo ref` or `foo ptr`. Unfortunately, these two are also # used as infix operators for the memory regions feature and # the current parsing rules don't play well here. - if p.inPragma == 0 and (isUnary(p.tok) or p.tok.tokType notin {tkOpr, tkDotDot}): - # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet - # solution, but pragmas.nim can't handle that - result = commandExpr(p, result, mode) - break + let isDotLike2 = p.tok.isDotLike + if isDotLike2 and p.lex.config.isDefined("nimPreviewDotLikeOps"): + # synchronize with `tkDot` branch + result = dotLikeExpr(p, result) + result = parseGStrLit(p, result) + else: + if isDotLike2: + parMessage(p, warnDotLikeOps, "dot-like operators will be parsed differently with `-d:nimPreviewDotLikeOps`") + if p.inPragma == 0 and (isUnary(p.tok) or p.tok.tokType notin {tkOpr, tkDotDot}): + # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet + # solution, but pragmas.nim can't handle that + result = commandExpr(p, result, mode) + break else: break diff --git a/doc/grammar.txt b/doc/grammar.txt index d4f4a0515..f58621b97 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -29,6 +29,7 @@ exprList = expr ^+ comma exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? dotExpr = expr '.' optInd (symbol | '[:' exprList ']') explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )? +dotLikeExpr = expr DOTLIKEOP optInd symbol qualifiedIdent = symbol ('.' optInd symbol)? setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}' castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') / @@ -56,6 +57,7 @@ tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' primarySuffix = '(' (exprColonEqExpr comma?)* ')' | '.' optInd symbol generalizedLit? + | DOTLIKEOP optInd symbol generalizedLit? | '[' optInd exprColonEqExprList optPar ']' | '{' optInd exprColonEqExprList optPar '}' | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax diff --git a/doc/manual.rst b/doc/manual.rst index 9c7c219e9..72d50a901 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -748,6 +748,7 @@ has the second-lowest precedence. Otherwise, precedence is determined by the first character. + ================ ======================================================= ================== =============== Precedence level Operators First character Terminal symbol ================ ======================================================= ================== =============== @@ -783,6 +784,14 @@ of a call or whether it is parsed as a tuple constructor: .. code-block:: nim echo (1, 2) # pass the tuple (1, 2) to echo +Dot-like operators +------------------ + +Terminal symbol in the grammar: `DOTLIKEOP`. + +Dot-like operators are operators starting with `.`, but not with `..`, for e.g. `.?`; +they have the same precedence as `.`, so that `a.?b.c` is parsed as `(a.?b).c` instead of `a.?(b.c)`. + Grammar ------- diff --git a/tests/config.nims b/tests/config.nims index 0327f0b76..86787db1d 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -36,3 +36,4 @@ switch("define", "nimExperimentalAsyncjsThen") switch("define", "nimExperimentalLinenoiseExtra") switch("define", "nimPreviewFloatRoundtrip") +switch("define", "nimPreviewDotLikeOps") diff --git a/tests/misc/trfc_341.nim b/tests/misc/trfc_341.nim new file mode 100644 index 000000000..37cf675c6 --- /dev/null +++ b/tests/misc/trfc_341.nim @@ -0,0 +1,27 @@ +# test for https://github.com/nim-lang/RFCs/issues/341 +import std/json +import std/jsonutils +import std/macros + +macro fn1(a: untyped): string = newLit a.lispRepr + +doAssert fn1(a.?b.c) == """(DotExpr (Infix (Ident ".?") (Ident "a") (Ident "b")) (Ident "c"))""" + +template `.?`(a: JsonNode, b: untyped{ident}): JsonNode = + a[astToStr(b)] + +proc identity[T](a: T): T = a +proc timesTwo[T](a: T): T = a * 2 + +template main = + let a = (a1: 1, a2: "abc", a3: (a4: 2.5)) + let j = a.toJson + doAssert j.?a1.getInt == 1 + doAssert j.?a3.?a4.getFloat == 2.5 + doAssert j.?a3.?a4.getFloat.timesTwo == 5.0 + doAssert j.?a3.identity.?a4.getFloat.timesTwo == 5.0 + doAssert j.identity.?a3.identity.?a4.identity.getFloat.timesTwo == 5.0 + doAssert j.identity.?a3.?a4.identity.getFloat.timesTwo == 5.0 + +static: main() +main() From c70e4040bd69e1ee32b6c676dbfc763c7cf4df37 Mon Sep 17 00:00:00 2001 From: Dankr4d Date: Wed, 25 Aug 2021 19:27:00 +0200 Subject: [PATCH 0351/1654] fixes #14511 [backport:1.4] (#18732) * fixes #14511 [backport:1.4] Signed-off-by: Dankr4d * Replaced fix with code from alaviss, for better readability, with small changes. Signed-off-by: Dankr4d * - Specified output in test. Signed-off-by: Dankr4d * Replaced case in nnkRecCase with a simpler version, which just adds the last son. Signed-off-by: Dankr4d * Update tests/macros/t14511.nim * Update tests/macros/t14511.nim Co-authored-by: Andreas Rumpf --- lib/core/macros.nim | 21 ++++++++-------- tests/macros/t14511.nim | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 tests/macros/t14511.nim diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 645f5067a..806ea7e31 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1541,18 +1541,17 @@ proc customPragmaNode(n: NimNode): NimNode = for i in 0.. 0: var identDefs = identDefsStack.pop() - if identDefs.kind == nnkRecCase: - identDefsStack.add(identDefs[0]) - for i in 1.. Date: Wed, 25 Aug 2021 22:04:18 -0700 Subject: [PATCH 0352/1654] followup #18711 grammar_nanny now reports unused non-terminals (#18746) * followup #18711 cleanup unused grammar rules * make tools/grammar_nanny.nim report unused terminals * revert removal of some grammar comments --- tools/grammar_nanny.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/grammar_nanny.nim b/tools/grammar_nanny.nim index 397041559..5120ef976 100644 --- a/tools/grammar_nanny.nim +++ b/tools/grammar_nanny.nim @@ -4,7 +4,7 @@ import std / [strutils, sets] import ".." / compiler / [ - llstream, ast, lexer, options, msgs, idents, + llstream, lexer, options, msgs, idents, lineinfos, pathutils] proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = @@ -35,6 +35,10 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = usedSyms.incl word else: rawGetTok(L, tok) + for u in declaredSyms: + if u notin usedSyms: + echo "Unused non-terminal: ", u + for u in usedSyms: if u notin declaredSyms: echo "Undeclared non-terminal: ", u From 041edaa1df96fc7963f01bf5048f2bee86b6b363 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 26 Aug 2021 18:07:54 -0300 Subject: [PATCH 0353/1654] Implement RFC-413 (#18749) * Implement RFC 413 * Implement RFC 413 * Implement RFC 413 * Implement RFC 413 * https://github.com/nim-lang/Nim/pull/18749#discussion_r696320995 * Update lib/js/dom_extensions.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/js/dom_extensions.nim * https://github.com/nim-lang/Nim/pull/18749#discussion_r696913310 Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Andreas Rumpf --- changelog.md | 3 +++ lib/js/dom.nim | 3 +++ lib/js/dom_extensions.nim | 6 ++---- tests/effects/tstrict_funcs_imports_js.nim | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 3ba008cf3..65ae61833 100644 --- a/changelog.md +++ b/changelog.md @@ -371,6 +371,9 @@ - Added `dom.setInterval`, `dom.clearInterval` overloads. +- Merged `dom_extensions` module into `dom` module, + it was a module with a single line, see https://github.com/nim-lang/RFCs/issues/413 + - Allow reading parameters when compiling for Nintendo Switch. - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect diff --git a/lib/js/dom.nim b/lib/js/dom.nim index 3f0b9f630..aca79faca 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1771,3 +1771,6 @@ since (1, 3): ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL proc readAsText*(f: FileReader, b: Blob|File, encoding = cstring"UTF-8") {.importcpp: "#.readAsText(#, #)".} ## https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsText + +since (1, 5): + proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] {.importcpp.} diff --git a/lib/js/dom_extensions.nim b/lib/js/dom_extensions.nim index a1ceff5b4..c15a03195 100644 --- a/lib/js/dom_extensions.nim +++ b/lib/js/dom_extensions.nim @@ -1,5 +1,3 @@ import std/dom - -{.push importcpp.} -proc elementsFromPoint*(n: DocumentOrShadowRoot; x, y: float): seq[Element] -{.pop.} +export elementsFromPoint +{.deprecated: "use `std/dom` instead".} diff --git a/tests/effects/tstrict_funcs_imports_js.nim b/tests/effects/tstrict_funcs_imports_js.nim index b7fcd343a..667887ff0 100644 --- a/tests/effects/tstrict_funcs_imports_js.nim +++ b/tests/effects/tstrict_funcs_imports_js.nim @@ -7,7 +7,6 @@ discard """ import asyncjs, dom, - dom_extensions, jsconsole, jsffi, jsre From 5eba3725707df0f62dcf9c380eaa518635b0f0c0 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 27 Aug 2021 17:50:06 +0800 Subject: [PATCH 0354/1654] [minor] break loops if it is ambiguous (#18745) * [minor] break loops if it is ambiguous * Update compiler/lookups.nim Co-authored-by: Timothee Cour Co-authored-by: Timothee Cour --- compiler/lookups.nim | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 331eef525..4c072eed8 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -181,13 +181,14 @@ proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PS if overloadableEnums notin c.features: symSet.excl skEnumField result = nil - for im in c.imports.mitems: - for s in symbols(im, marked, name, c.graph): - if result == nil: - result = s - else: - if s.kind notin symSet or result.kind notin symSet: + block outer: + for im in c.imports.mitems: + for s in symbols(im, marked, name, c.graph): + if result == nil: + result = s + elif s.kind notin symSet or result.kind notin symSet: ambiguous = true + break outer proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym = for scope in allScopes(c.currentScope): From 719718df2f98f2bd9aa4286ff8aeac3f02e4d46c Mon Sep 17 00:00:00 2001 From: Miran Date: Sat, 28 Aug 2021 08:27:56 +0200 Subject: [PATCH 0355/1654] more minor bootstrapping cleanups (#18759) --- compiler/ccgtypes.nim | 3 --- compiler/cgen.nim | 1 - compiler/vmops.nim | 1 - 3 files changed, 5 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index d05c4cd8b..492e1f181 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -12,7 +12,6 @@ # ------------------------- Name Mangling -------------------------------- import sighashes, modulegraphs -from lowerings import createObj proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope @@ -256,8 +255,6 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) = m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope]) # see `testCodegenABICheck` for example error message it generates -from ccgutils import ccgIntroducedPtr -export ccgIntroducedPtr proc fillResult(conf: ConfigRef; param: PNode) = fillLoc(param.sym.loc, locParam, param, ~"Result", diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f55da7f65..a06a87172 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -23,7 +23,6 @@ when not defined(leanCompiler): import strutils except `%` # collides with ropes.`%` from ic / ic import ModuleBackendFlag -from modulegraphs import ModuleGraph, PPassContext import dynlib when not declared(dynlib.libCandidates): diff --git a/compiler/vmops.nim b/compiler/vmops.nim index a1a4996b2..08225b065 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -29,7 +29,6 @@ from std/hashes import hash from std/osproc import nil from system/formatfloat import addFloatRoundtrip, addFloatSprintf -from sighashes import symBodyDigest # There are some useful procs in vmconv. import vmconv From 014edde36a2f61b0a232ceb4800681a2ce261759 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 28 Aug 2021 16:34:27 -0700 Subject: [PATCH 0356/1654] followup #18759: make compiler build cleanly with 0 warnings/hints (#18764) --- compiler/semtypes.nim | 2 -- compiler/vmops.nim | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index cb7f847a9..9f7a17a53 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -10,8 +10,6 @@ # this module does the semantic checking of type declarations # included from sem.nim -import math - const errStringOrIdentNodeExpected = "string or ident node expected" errStringLiteralExpected = "string literal expected" diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 08225b065..e287dd41d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -15,10 +15,12 @@ from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, lgamma when declared(math.copySign): - from std/math import copySign + # pending bug #18762, avoid renaming math + from std/math as math2 import copySign when declared(math.signbit): - from std/math import signbit + # ditto + from std/math as math3 import signbit from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError From c07d8da7b9d491500096dbd4c224f5762cbf89ac Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 29 Aug 2021 15:18:54 +0800 Subject: [PATCH 0357/1654] add missing docs (#18758) --- lib/nimrtl.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/nimrtl.nim b/lib/nimrtl.nim index 301ffb43b..93349b287 100644 --- a/lib/nimrtl.nim +++ b/lib/nimrtl.nim @@ -26,6 +26,7 @@ batchable: false ## * unicode ## * pegs ## * ropes +## * cstrutils ## when system.appType != "lib": From 0ddd9519c0007b705d70b5a75b634831396dd904 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Sun, 29 Aug 2021 10:42:52 +0200 Subject: [PATCH 0358/1654] Remove `Covariance` section from the experimental manual (#18688) * Remove `Covariance` section * Add blank lines after `.. code-block::` * Fix CI? --- doc/manual_experimental.rst | 164 ++++++++++++++---------------------- 1 file changed, 63 insertions(+), 101 deletions(-) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 3272a4d0d..458512292 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -70,6 +70,7 @@ type `void` are treated as non-existent, `void` as a return type means that the procedure does not return a value: .. code-block:: nim + proc nothing(x, y: void): void = echo "ha" @@ -78,6 +79,7 @@ the procedure does not return a value: The `void` type is particularly useful for generic code: .. code-block:: nim + proc callProc[T](p: proc (x: T), x: T) = when T is void: p() @@ -93,6 +95,7 @@ The `void` type is particularly useful for generic code: However, a `void` type cannot be inferred in generic code: .. code-block:: nim + callProc(emptyProc) # Error: type mismatch: got (proc ()) # but expected one of: @@ -102,106 +105,6 @@ The `void` type is only valid for parameters and return types; other symbols cannot have the type `void`. - -Covariance -========== - -Covariance in Nim can be introduced only through pointer-like types such -as `ptr` and `ref`. Sequence, Array and OpenArray types, instantiated -with pointer-like types will be considered covariant if and only if they -are also immutable. The introduction of a `var` modifier or additional -`ptr` or `ref` indirections would result in invariant treatment of -these types. - -`proc` types are currently always invariant, but future versions of Nim -may relax this rule. - -User-defined generic types may also be covariant with respect to some of -their parameters. By default, all generic params are considered invariant, -but you may choose the apply the prefix modifier `in` to a parameter to -make it contravariant or `out` to make it covariant: - -.. code-block:: nim - type - AnnotatedPtr[out T] = - metadata: MyTypeInfo - p: ref T - - RingBuffer[out T] = - startPos: int - data: seq[T] - - Action {.importcpp: "std::function".} [in T] = object - -When the designated generic parameter is used to instantiate a pointer-like -type as in the case of `AnnotatedPtr` above, the resulting generic type will -also have pointer-like covariance: - -.. code-block:: nim - type - GuiWidget = object of RootObj - Button = object of GuiWidget - ComboBox = object of GuiWidget - - var - widgetPtr: AnnotatedPtr[GuiWidget] - buttonPtr: AnnotatedPtr[Button] - - ... - - proc drawWidget[T](x: AnnotatedPtr[GuiWidget]) = ... - - # you can call procs expecting base types by supplying a derived type - drawWidget(buttonPtr) - - # and you can convert more-specific pointer types to more general ones - widgetPtr = buttonPtr - -Just like with regular pointers, covariance will be enabled only for immutable -values: - -.. code-block:: nim - proc makeComboBox[T](x: var AnnotatedPtr[GuiWidget]) = - x.p = new(ComboBox) - - makeComboBox(buttonPtr) # Error, AnnotatedPtr[Button] cannot be modified - # to point to a ComboBox - -On the other hand, in the `RingBuffer` example above, the designated generic -param is used to instantiate the non-pointer `seq` type, which means that -the resulting generic type will have covariance that mimics an array or -sequence (i.e. it will be covariant only when instantiated with `ptr` and -`ref` types): - -.. code-block:: nim - - type - Base = object of RootObj - Derived = object of Base - - proc consumeBaseValues(b: RingBuffer[Base]) = ... - - var derivedValues: RingBuffer[Derived] - - consumeBaseValues(derivedValues) # Error, Base and Derived values may differ - # in size - - proc consumeBasePointers(b: RingBuffer[ptr Base]) = ... - - var derivedPointers: RingBuffer[ptr Derived] - - consumeBaseValues(derivedPointers) # This is legal - -Please note that Nim will treat the user-defined pointer-like types as -proper alternatives to the built-in pointer types. That is, types such -as `seq[AnnotatedPtr[T]]` or `RingBuffer[AnnotatedPtr[T]]` will also be -considered covariant and you can create new pointer-like types by instantiating -other user-defined pointer-like types. - -The contravariant parameters introduced with the `in` modifier are currently -useful only when interfacing with imported types having such semantics. - - Automatic dereferencing ======================= @@ -209,6 +112,7 @@ Automatic dereferencing is performed for the first argument of a routine call. This feature has to be enabled via `{.experimental: "implicitDeref".}`: .. code-block:: nim + {.experimental: "implicitDeref".} proc depth(x: NodeObj): int = ... @@ -219,6 +123,7 @@ This feature has to be enabled via `{.experimental: "implicitDeref".}`: echo n.depth # no need to write n[].depth either + Code reordering =============== @@ -281,6 +186,7 @@ statement) can have their entire initialization statement reordered. Be wary of what code is executed at the top level: .. code-block:: nim + {.experimental: "codeReordering".} proc a() = @@ -300,6 +206,7 @@ what code is executed at the top level: code is the same as it would be with code reordering disabled. .. code-block:: nim + {.experimental: "codeReordering".} proc x() = @@ -313,6 +220,7 @@ It is important to note that reordering *only* works for symbols at top level scope. Therefore, the following will *fail to compile:* .. code-block:: nim + {.experimental: "codeReordering".} proc a() = @@ -331,6 +239,7 @@ has different names. This does not need an `experimental` switch, but is an unstable feature. .. code-block:: Nim + proc foo(x: int) = echo "Using x: ", x proc foo(y: int) = @@ -349,6 +258,7 @@ As a special more convenient notation, proc expressions involved in procedure calls can use the `do` keyword: .. code-block:: nim + sort(cities) do (x,y: string) -> int: cmp(x.len, y.len) @@ -371,6 +281,7 @@ parentheses is just a block of code. The `do` notation can be used to pass multiple blocks to a macro: .. code-block:: nim + macro performWithUndo(task, undo: untyped) = ... performWithUndo do: @@ -380,7 +291,6 @@ pass multiple blocks to a macro: # code to undo it - Special Operators ================= @@ -404,6 +314,7 @@ the expression, where the unknown field or proc name is passed to an `untyped` parameter: .. code-block:: nim + a.b # becomes `.`(a, b) a.b(c, d) # becomes `.`(a, b, c, d) @@ -411,6 +322,7 @@ The matched dot operators can be symbols of any callable kind (procs, templates and macros), depending on the desired effect: .. code-block:: nim + template `.`(js: PJsonNode, field: untyped): JSON = js[astToStr(field)] var js = parseJson("{ x: 1, y: 2}") @@ -435,6 +347,7 @@ operator `.=` This operator will be matched against assignments to missing fields. .. code-block:: nim + a.b = c # becomes `.=`(a, b, c) Call operator @@ -445,6 +358,7 @@ for existing routines. The experimental `callOperator` switch must be enabled to use this operator. .. code-block:: nim + {.experimental: "callOperator".} template `()`(a: int, b: float): untyped = $(a, b) @@ -480,6 +394,7 @@ All types for which `nil` is a valid value can be annotated with the `not nil` annotation to exclude `nil` as a valid value: .. code-block:: nim + {.experimental: "notnil".} type @@ -502,6 +417,7 @@ here. .. include:: manual_experimental_strictnotnil.rst + Concepts ======== @@ -511,6 +427,7 @@ arbitrary set of requirements that the matched type must satisfy. Concepts are written in the following form: .. code-block:: nim + type Comparable = concept x, y (x < y) is bool @@ -536,6 +453,7 @@ instance. You can also apply the `type` modifier to create a named instance of the type itself: .. code-block:: nim + type MyConcept = concept x, var v, ref r, ptr p, static s, type T ... @@ -545,6 +463,7 @@ and parameters are expected. This provides a more convenient way to check for the presence of callable symbols with specific signatures: .. code-block:: nim + type OutputStream = concept var s s.write(string) @@ -555,6 +474,7 @@ type, following the `concept` keyword is also considered to have the explicit modifier and will be matched only as a type. .. code-block:: nim + type # Let's imagine a user-defined casting framework with operators # such as `val.to(string)` and `val.to(JSonValue)`. We can test @@ -592,6 +512,7 @@ concept and, as a result, a wrong overload is selected, you can apply the `explain` pragma to either the concept body or a particular call-site. .. code-block:: nim + type MyConcept {.explain.} = concept ... @@ -607,6 +528,7 @@ Generic concepts and type binding rules The concept types can be parametric just like the regular generic types: .. code-block:: nim + ### matrixalgo.nim import std/typetraits @@ -680,6 +602,7 @@ Unbound static params will be inferred from expressions involving the `==` operator and also when types dependent on them are being matched: .. code-block:: nim + type MatrixReducer[M, N: static int; T] = concept x x.reduce(SquareMatrix[N, T]) is array[M, int] @@ -694,6 +617,7 @@ matched without permanently inferring it. This may be useful when you need to match several procs accepting the same wide class of types: .. code-block:: nim + type Enumerable[T] = concept e for v in e: @@ -716,6 +640,7 @@ types used in multiple signatures, without actually requiring any concrete types, thus allowing you to encode implementation-defined types: .. code-block:: nim + type MyConcept = concept x type T1 = auto @@ -776,6 +701,7 @@ accessible through the dot operator in procs where the concept was successfully matched to a concrete type: .. code-block:: nim + type DateTime = concept t1, t2, type T const Min = T.MinDate @@ -810,6 +736,7 @@ one. As an alternative way of defining concept refinements, you can use the object inheritance syntax involving the `of` keyword: .. code-block:: nim + type Graph = concept g, type G of EquallyComparable, Copyable type @@ -853,6 +780,7 @@ object inheritance syntax involving the `of` keyword: the concept body: .. code-block:: nim + type Stringable = concept x $x is string @@ -904,6 +832,7 @@ object inheritance syntax involving the `of` keyword: types to the corresponding VTable type. .. code-block:: nim + type IntEnumerable = vtref Enumerable[int] @@ -970,6 +899,7 @@ language may weaken this restriction.) The signature has to be: .. code-block:: nim + proc `=deepCopy`(x: T): T This mechanism will be used by most data structures that support shared memory @@ -1037,6 +967,7 @@ phase of the compiler: This means they provide an easy way to enhance the compilation pipeline with user defined optimizations: .. code-block:: nim + template optMul{`*`(a, 2)}(a: int): int = a+a let x = 3 @@ -1059,6 +990,7 @@ Unfortunately optimizations are hard to get right and even the tiny example is **wrong**: .. code-block:: nim + template optMul{`*`(a, 2)}(a: int): int = a+a proc f(): int = @@ -1071,6 +1003,7 @@ We cannot duplicate 'a' if it denotes an expression that has a side effect! Fortunately Nim supports side effect analysis: .. code-block:: nim + template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a proc f(): int = @@ -1088,11 +1021,13 @@ cannot really do that however as the following code only swaps arguments blindly: .. code-block:: nim + template mulIsCommutative{`*`(a, b)}(a, b: int): int = b*a What optimizers really need to do is a *canonicalization*: .. code-block:: nim + template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a The `int{lit}` parameter pattern matches against an expression of @@ -1152,6 +1087,7 @@ but also to every other bound parameter; syntactically they need to occur after the ordinary AST predicates: .. code-block:: nim + template ex{a = b + c}(a: int{noalias}, b, c: int) = # this transformation is only valid if 'b' and 'c' do not alias 'a': a = b @@ -1160,6 +1096,7 @@ the ordinary AST predicates: Another example: .. code-block:: nim + proc somefunc(s: string) = assert s == "variable" proc somefunc(s: string{nkStrLit}) = assert s == "literal" proc somefunc(s: string{nkRStrLit}) = assert s == r"raw" @@ -1189,6 +1126,7 @@ The `|` operator The `|` operator if used as infix operator creates an ordered choice: .. code-block:: nim + template t{0|1}(): untyped = 3 let a = 1 # outputs 3: @@ -1198,6 +1136,7 @@ The matching is performed after the compiler performed some optimizations like constant folding, so the following does not work: .. code-block:: nim + template t{0|1}(): untyped = 3 # outputs 1: echo 1 @@ -1215,6 +1154,7 @@ A pattern expression can be bound to a pattern parameter via the `expr{param}` notation: .. code-block:: nim + template t{(0|1|2){x}}(x: untyped): untyped = x+1 let a = 1 # outputs 2: @@ -1227,6 +1167,7 @@ The `~` operator The `~` operator is the **not** operator in patterns: .. code-block:: nim + template t{x = (~x){y} and (~x){z}}(x, y, z: bool) = x = y if x: x = z @@ -1246,6 +1187,7 @@ The `*` operator can *flatten* a nested binary expression like `a & b & c` to `&(a, b, c)`: .. code-block:: nim + var calls = 0 @@ -1270,6 +1212,7 @@ which is flattened into a call expression; thus the invocation of `optConc` produces: .. code-block:: nim + `&&`("my", space & "awe", "some ", "concat") @@ -1280,6 +1223,7 @@ The `**` is much like the `*` operator, except that it gathers not only all the arguments, but also the matched operators in reverse polish notation: .. code-block:: nim + import std/macros type @@ -1324,6 +1268,7 @@ parameter is of the type `varargs` it is treated specially and it can match 0 or more arguments in the AST to be matched against: .. code-block:: nim + template optWrite{ write(f, x) ((write|writeLine){w})(f, y) @@ -1339,6 +1284,7 @@ The following example shows how some simple partial evaluation can be implemented with term rewriting: .. code-block:: nim + proc p(x, y: int; cond: bool): int = result = if cond: x + y else: x - y @@ -1352,6 +1298,7 @@ Example: Hoisting The following example shows how some form of hoisting can be implemented: .. code-block:: nim + import std/pegs template optPeg{peg(pattern)}(pattern: string{lit}): Peg = @@ -1375,6 +1322,7 @@ Parameter constraints can also be used for ordinary routine parameters; these constraints affect ordinary overloading resolution then: .. code-block:: nim + proc optLit(a: string{lit|`const`}) = echo "string literal" proc optLit(a: string) = @@ -1426,6 +1374,7 @@ Spawn statement `spawn`:idx: can be used to pass a task to the thread pool: .. code-block:: nim + import std/threadpool proc processLine(line: string) = @@ -1458,6 +1407,7 @@ with the `^` operator is **blocking**. However, one can use `blockUntilAny` to wait on multiple flow variables at the same time: .. code-block:: nim + import std/threadpool, ... # wait until 2 out of 3 servers received the update: @@ -1552,6 +1502,7 @@ Protecting global variables Object fields and global variables can be annotated via a `guard` pragma: .. code-block:: nim + var glock: TLock var gdata {.guard: glock.}: int @@ -1559,6 +1510,7 @@ The compiler then ensures that every access of `gdata` is within a `locks` section: .. code-block:: nim + proc invalid = # invalid: unguarded access: echo gdata @@ -1577,6 +1529,7 @@ semantics and should not be used directly! It should only be used in templates that also implement some form of locking at runtime: .. code-block:: nim + template lock(a: TLock; body: untyped) = pthread_mutex_lock(a) {.locks: [a].}: @@ -1590,6 +1543,7 @@ The guard does not need to be of any particular type. It is flexible enough to model low level lockfree mechanisms: .. code-block:: nim + var dummyLock {.compileTime.}: int var atomicCounter {.guard: dummyLock.}: int @@ -1617,6 +1571,7 @@ Since objects can reside on the heap or on the stack this greatly enhances the expressivity of the language: .. code-block:: nim + type ProtectedCounter = object v {.guard: L.}: int @@ -1631,6 +1586,7 @@ The access to field `x.v` is allowed since its guard `x.L` is active. After template expansion, this amounts to: .. code-block:: nim + proc incCounters(counters: var openArray[ProtectedCounter]) = for i in 0..counters.high: pthread_mutex_lock(counters[i].L) @@ -1651,6 +1607,7 @@ Two paths are considered equivalent if they are syntactically the same. This means the following compiles (for now) even though it really should not: .. code-block:: nim + {.locks: [a[i].L].}: inc i access a[i].v @@ -1671,6 +1628,7 @@ of the same level can only be acquired *at the same time* within a single `locks` section: .. code-block:: nim + var a, b: TLock[2] var x: TLock[1] # invalid locking order: TLock[1] cannot be acquired before TLock[2]: @@ -1697,6 +1655,7 @@ the runtime check is required to ensure a global ordering for two locks `a` and `b` of the same lock level: .. code-block:: nim + template multilock(a, b: ptr TLock; body: untyped) = if cast[ByteAddress](a) < cast[ByteAddress](b): pthread_mutex_lock(a) @@ -1717,6 +1676,7 @@ level. This then means that the routine may acquire locks of up to this level. This is essential so that procs can be called within a `locks` section: .. code-block:: nim + proc p() {.locks: 3.} = discard var a: TLock[4] @@ -1739,6 +1699,7 @@ cannot be inferred statically, leading to compiler warnings. By using having unknown lock level as well: .. code-block:: nim + type SomeBase* = ref object of RootObj type SomeDerived* = ref object of SomeBase memberProc*: proc () @@ -1761,6 +1722,7 @@ e.g. when rewriting term to same term plus extra content. e.g. with given example `echo("ab")` will be rewritten just once: .. code-block:: nim + template pwnEcho{echo(x)}(x: untyped) = {.noRewrite.}: echo("pwned!") From fa7c1aa8647aa97a4a204751db727c3cbeded749 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sun, 29 Aug 2021 06:54:07 -0700 Subject: [PATCH 0359/1654] move PNode.comment to a side channel, reducing memory usage during compilation by a factor 1.25x (#18760) * move PNode.comment so a side channel, reducing memory usage * fix a bug * fixup * use sfHasComment to speedup comment lookups * fix for IC * Update compiler/parser.nim Co-authored-by: Andreas Rumpf --- compiler/ast.nim | 77 +++++++++++++++++++++++++++++++------------ compiler/commands.nim | 3 +- compiler/parser.nim | 14 +++++--- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 446c00f49..5893a5671 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - lineinfos, hashes, options, ropes, idents, int128 + lineinfos, hashes, options, ropes, idents, int128, tables from strutils import toLowerAscii export int128 @@ -497,6 +497,7 @@ type nfExecuteOnReload # A top-level statement that will be executed during reloads nfLastRead # this node is a last read nfFirstWrite# this node is a first write + nfHasComment # node has a comment TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 43) @@ -774,7 +775,6 @@ type ident*: PIdent else: sons*: TNodeSeq - comment*: string TStrTable* = object # a table[PIdent] of PSym counter*: int @@ -991,6 +991,38 @@ type TImplication* = enum impUnknown, impNo, impYes +template nodeId(n: PNode): int = cast[int](n) + +type Gconfig = object + # we put comments in a side channel to avoid increasing `sizeof(TNode)`, which + # reduces memory usage given that `PNode` is the most allocated type by far. + comments: Table[int, string] # nodeId => comment + useIc*: bool + +var gconfig {.threadvar.}: Gconfig + +proc setUseIc*(useIc: bool) = gconfig.useIc = useIc + +proc comment*(n: PNode): string = + if nfHasComment in n.flags and not gconfig.useIc: + # IC doesn't track comments, see `packed_ast`, so this could fail + result = gconfig.comments[n.nodeId] + +proc `comment=`*(n: PNode, a: string) = + let id = n.nodeId + if a.len > 0: + # if needed, we could periodically cleanup gconfig.comments when its size increases, + # to ensure only live nodes (and with nfHasComment) have an entry in gconfig.comments; + # for compiling compiler, the waste is very small: + # num calls to newNodeImpl: 14984160 (num of PNode allocations) + # size of gconfig.comments: 33585 + # num of nodes with comments that were deleted and hence wasted: 3081 + n.flags.incl nfHasComment + gconfig.comments[id] = a + elif nfHasComment in n.flags: + n.flags.excl nfHasComment + gconfig.comments.del(id) + # BUGFIX: a module is overloadable so that a proc can have the # same name as an imported module. This is necessary because of # the poor naming choices in the standard library. @@ -1207,9 +1239,17 @@ when defined(useNodeIds): const nodeIdToDebug* = -1 # 2322968 var gNodeId: int -proc newNode*(kind: TNodeKind): PNode = - ## new node with unknown line info, no type, and no children - result = PNode(kind: kind, info: unknownLineInfo) +template newNodeImpl(info2) = + result = PNode(kind: kind, info: info2) + when false: + # this would add overhead, so we skip it; it results in a small amount of leaked entries + # for old PNode that gets re-allocated at the same address as a PNode that + # has `nfHasComment` set (and an entry in that table). Only `nfHasComment` + # should be used to test whether a PNode has a comment; gconfig.comments + # can contain extra entries for deleted PNode's with comments. + gconfig.comments.del(cast[int](result)) + +template setIdMaybe() = when defined(useNodeIds): result.id = gNodeId if result.id == nodeIdToDebug: @@ -1217,27 +1257,22 @@ proc newNode*(kind: TNodeKind): PNode = writeStackTrace() inc gNodeId +proc newNode*(kind: TNodeKind): PNode = + ## new node with unknown line info, no type, and no children + newNodeImpl(unknownLineInfo) + setIdMaybe() + proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode = ## new node with line info, no type, and no children - result = PNode(kind: kind, info: info) - when defined(useNodeIds): - result.id = gNodeId - if result.id == nodeIdToDebug: - echo "KIND ", result.kind - writeStackTrace() - inc gNodeId + newNodeImpl(info) + setIdMaybe() proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode = ## new node with line info, type, and children - result = PNode(kind: kind, info: info) + newNodeImpl(info) if children > 0: newSeq(result.sons, children) - when defined(useNodeIds): - result.id = gNodeId - if result.id == nodeIdToDebug: - echo "KIND ", result.kind - writeStackTrace() - inc gNodeId + setIdMaybe() proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode = ## new node with line info, type, and no children @@ -1644,8 +1679,8 @@ proc copyNode*(src: PNode): PNode = template transitionNodeKindCommon(k: TNodeKind) = let obj {.inject.} = n[] - n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags, - comment: obj.comment) + n[] = TNode(kind: k, typ: obj.typ, info: obj.info, flags: obj.flags) + # n.comment = obj.comment # shouldn't be needed, the address doesnt' change when defined(useNodeIds): n.id = obj.id diff --git a/compiler/commands.nim b/compiler/commands.nim index 82e908152..079717862 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -8,7 +8,7 @@ # # This module handles the parsing of command line arguments. - +from ast import setUseIc # We do this here before the 'import' statement so 'defined' does not get # confused with 'TGCMode.gcMarkAndSweep' etc. @@ -862,6 +862,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "v2": conf.symbolFiles = v2Sf of "stress": conf.symbolFiles = stressTest else: localError(conf, info, "invalid option for --incremental: " & arg) + setUseIc(conf.symbolFiles != disabledSf) of "skipcfg": processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info) of "skipprojcfg": diff --git a/compiler/parser.nim b/compiler/parser.nim index 30180e545..225493902 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -170,13 +170,15 @@ proc validInd(p: var Parser): bool {.inline.} = proc rawSkipComment(p: var Parser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: + var rhs = node.comment when defined(nimpretty): if p.tok.commentOffsetB > p.tok.commentOffsetA: - node.comment.add fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB) + rhs.add fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB) else: - node.comment.add p.tok.literal + rhs.add p.tok.literal else: - node.comment.add p.tok.literal + rhs.add p.tok.literal + node.comment = move rhs else: parMessage(p, errInternal, "skipComment") getTok(p) @@ -1824,8 +1826,10 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = if result.comment.len == 0: # proc fn*(a: int): int = a ## foo # => moves comment `foo` to `fn` - swap(result.comment, body[0].comment) - else: discard # xxx either `assert false` or issue a warning (otherwise we'll never know of this edge case) + result.comment = body[0].comment + body[0].comment = "" + else: + assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT From f02de25ca1b660b0b6e1ae2d4981106744814fe3 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Tue, 31 Aug 2021 08:14:05 +0200 Subject: [PATCH 0360/1654] Fix #15150 (#18730) --- lib/pure/asyncfile.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index c51b338e3..222a89b97 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -202,10 +202,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = proc read*(f: AsyncFile, size: int): Future[string] = ## Read `size` bytes from the specified file asynchronously starting at - ## the current position of the file pointer. + ## the current position of the file pointer. `size` should be greater than zero. ## ## If the file pointer is past the end of the file then an empty string is ## returned. + assert size > 0 var retFuture = newFuture[string]("asyncfile.read") when defined(windows) or defined(nimdoc): From 8f4bdb3596bb7650f2688b8243dc3164abfc21c9 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 31 Aug 2021 19:32:37 +0800 Subject: [PATCH 0361/1654] [minor]break loops after a candidate is added to seqs (#18770) * [minor]break loops when added * Update compiler/lookups.nim Co-authored-by: Clyybber Co-authored-by: Clyybber --- compiler/lookups.nim | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 4c072eed8..fc30408e5 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -211,14 +211,16 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} = proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] = result = @[] - for scope in allScopes(c.currentScope): - var ti: TIdentIter - var candidate = initIdentIter(ti, scope.symbols, s) - while candidate != nil: - if candidate.kind in filter: - if result.len == 0: + block outer: + for scope in allScopes(c.currentScope): + var ti: TIdentIter + var candidate = initIdentIter(ti, scope.symbols, s) + while candidate != nil: + if candidate.kind in filter: result.add candidate - candidate = nextIdentIter(ti, scope.symbols) + # Break here, because further symbols encountered would be shadowed + break outer + candidate = nextIdentIter(ti, scope.symbols) if result.len == 0: var marked = initIntSet() From 3469f3a39320cf35d02d11b78c7db0be79e992f8 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 31 Aug 2021 20:05:45 +0800 Subject: [PATCH 0362/1654] add Apple Silicon to supporting platforms (#18772) * add Apple Silicon to supporting platforms * Update readme.md Co-authored-by: flywind Co-authored-by: Andreas Rumpf --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 7c6e28be3..b28c25ec1 100644 --- a/readme.md +++ b/readme.md @@ -35,7 +35,7 @@ architecture combinations: * Windows (Windows XP or greater) - x86 and x86_64 * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l - * Mac OS X (10.04 or greater) - x86, x86_64 and ppc64 + * Mac OS X (10.04 or greater) - x86, x86_64, ppc64 and Apple Silicon (based on the ARM64 architecture) More platforms are supported, however, they are not tested regularly and they may not be as stable as the above-listed platforms. From 72fa5833adac590e3c78e2b774cd33f28827594b Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 2 Sep 2021 13:41:14 +0800 Subject: [PATCH 0363/1654] fix #9778 (pairs iterator calling a helper proc with tuple return type will cut the iterator yield into half) (#18767) * test someSymFromImportTable * Update compiler/lookups.nim * test nkTupleConstr * use isConstExpr * add tests * add comments and todo * use todo --- compiler/transf.nim | 55 +++++++-- tests/stdlib/tyield.nim | 256 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+), 11 deletions(-) create mode 100644 tests/stdlib/tyield.nim diff --git a/compiler/transf.nim b/compiler/transf.nim index fb59887b0..80794b581 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -364,7 +364,7 @@ proc transformYield(c: PTransf, n: PNode): PNode = if e.typ.isNil: return result # can happen in nimsuggest for unknown reasons if c.transCon.forStmt.len != 3: e = skipConv(e) - if e.kind in {nkPar, nkTupleConstr}: + if e.kind == nkTupleConstr: for i in 0.. 0: - var changeNode = result[0] - changeNode.info = c.transCon.forStmt.info - for i, child in changeNode: - child.info = changeNode.info + for idx in 0 ..< result.len: + var changeNode = result[idx] + changeNode.info = c.transCon.forStmt.info + for i, child in changeNode: + child.info = changeNode.info proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PNode = result = transformSons(c, n) diff --git a/tests/stdlib/tyield.nim b/tests/stdlib/tyield.nim new file mode 100644 index 000000000..85be97365 --- /dev/null +++ b/tests/stdlib/tyield.nim @@ -0,0 +1,256 @@ +discard """ + targets: "c cpp js" +""" + +import std/[sugar, algorithm] + +block: + var x = @[(6.0, 6, '6'), + (5.0, 5, '5'), + (4.0, 4, '4'), + (3.0, 3, '3'), + (2.0, 2, '2'), + (1.0, 1, '1')] + + let y = x.reversed + + block: + let res = collect: + for (f, i, c) in x: + (f, i, c) + + doAssert res == x + + iterator popAscending[T](q: var seq[T]): T = + while q.len > 0: yield q.pop + + block: + var res = collect: + for f, i, c in popAscending(x): + (f, i, c) + + doAssert res == y + + let z = reversed(res) + let res2 = collect: + for (f, i, c) in popAscending(res): + (f, i, c) + + doAssert res2 == z + + +block: + var visits = 0 + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + iterator foo(): (int, int) = + yield bar() + + for a, b in foo(): + doAssert a == b + + doAssert visits == 1 + + block: + proc iterAux(a: seq[int], i: var int): (int, string) = + result = (a[i], $a[i]) + inc i + + iterator pairs(a: seq[int]): (int, string) = + var i = 0 + while i < a.len: + yield iterAux(a, i) + + var x = newSeq[int](10) + for i in 0 ..< x.len: + x[i] = i + + let res = collect: + for k, v in x: + (k, v) + + let expected = collect: + for i in 0 ..< x.len: + (i, $i) + + doAssert res == expected + + block: + proc bar(): (int, int, int) = + inc visits + (visits, visits, visits) + + iterator foo(): (int, int, int) = + yield bar() + + for a, b, c in foo(): + doAssert a == b + + doAssert visits == 2 + + + block: + + proc bar(): int = + inc visits + visits + + proc car(): int = + inc visits + visits + + iterator foo(): (int, int) = + yield (bar(), car()) + yield (bar(), car()) + + for a, b in foo(): + doAssert b == a + 1 + + doAssert visits == 6 + + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield (12, t2()) + yield bar() + + let res = collect: + for (a, b) in foo(): + (a, b) + + doAssert res == @[(12, 99), (7, 7)] + doAssert visits == 7 + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield ((12, t2())) + yield (bar()) + + let res = collect: + for (a, b) in foo(): + (a, b) + + doAssert res == @[(12, 99), (8, 8)] + doAssert visits == 8 + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t1(): int = 99 + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield (t1(), t2()) + yield bar() + + let res = collect: + for a, b in foo(): + (a, b) + + doAssert res == @[(99, 99), (9, 9)] + doAssert visits == 9 + + + block: + proc bar(): ((int, int), string) = + inc visits + ((visits, visits), $visits) + + proc t2(): int = 99 + + iterator foo(): ((int, int), string) = + yield ((1, 2), $t2()) + yield bar() + + let res = collect: + for a, b in foo(): + (a, b) + + doAssert res == @[((1, 2), "99"), ((10, 10), "10")] + doAssert visits == 10 + + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + iterator foo(): (int, int) = + yield (for i in 0 ..< 10: discard bar(); bar()) + yield (bar()) + + let res = collect: + for (a, b) in foo(): + (a, b) + + doAssert res == @[(21, 21), (22, 22)] + + block: + proc bar(): (int, int) = + inc visits + (visits, visits) + + proc t2(): int = 99 + + iterator foo(): (int, int) = + yield if true: bar() else: (t2(), t2()) + yield (bar()) + + let res = collect: + for a, b in foo(): + (a, b) + + doAssert res == @[(23, 23), (24, 24)] + + +block: + iterator foo(): (int, int, int) = + var time = 777 + yield (1, time, 3) + + let res = collect: + for a, b, c in foo(): + (a, b, c) + + doAssert res == @[(1, 777, 3)] + +block: + iterator foo(): (int, int, int) = + var time = 777 + yield (1, time, 3) + + let res = collect: + for t in foo(): + (t[0], t[1], t[2]) + + doAssert res == @[(1, 777, 3)] + + +block: + proc bar(): (int, int, int) = + (1, 2, 3) + iterator foo(): (int, int, int) = + yield bar() + + let res = collect: + for a, b, c in foo(): + (a, b, c) + + doAssert res == @[(1, 2, 3)] From e0ef859130f429df1891e31a85955daa753346b4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Sep 2021 12:10:14 +0200 Subject: [PATCH 0364/1654] strict effects (#18777) * fixes #17369 * megatest is green for --cpu:arm64 * docgen output includes more tags/raises * implemented 'effectsOf' * algorithm.nim: uses new effectsOf annotation * closes #18376 * closes #17475 * closes #13905 * allow effectsOf: [a, b] * added a test case * parameters that are not ours cannot be declared as .effectsOf * documentation * manual: added the 'sort' example * bootstrap with the new better options --- changelog.md | 17 ++ compiler/ast.nim | 6 +- compiler/ccgcalls.nim | 4 +- compiler/commands.nim | 3 +- compiler/condsyms.nim | 1 + compiler/docgen.nim | 7 +- compiler/lineinfos.nim | 2 + compiler/nim.cfg | 5 + compiler/options.nim | 3 +- compiler/pragmas.nim | 30 ++- compiler/sem.nim | 7 +- compiler/semcall.nim | 2 + compiler/semdata.nim | 4 +- compiler/sempass2.nim | 116 +++++--- compiler/semstmts.nim | 9 +- compiler/types.nim | 17 +- compiler/wordrecg.nim | 2 +- doc/manual.rst | 252 ++++++++++-------- lib/pure/algorithm.nim | 23 +- lib/pure/times.nim | 2 +- lib/std/private/globs.nim | 7 +- lib/system/io.nim | 2 +- lib/system/nimscript.nim | 6 +- .../expected/index.html | 2 +- nimdoc/testproject/expected/testproject.html | 10 +- tests/effects/teffects1.nim | 3 +- tests/effects/teffects2.nim | 3 +- tests/effects/teffects7.nim | 4 +- tests/effects/teffects8.nim | 3 +- tests/effects/tstrict_effects.nim | 27 ++ tests/effects/tstrict_effects2.nim | 28 ++ tests/effects/tstrict_effects3.nim | 17 ++ tests/effects/tstrict_effects_sort.nim | 27 ++ tests/pragmas/tuserpragma2.nim | 3 +- tests/stdlib/tstdlib_various.nim | 31 +-- 35 files changed, 482 insertions(+), 203 deletions(-) create mode 100644 tests/effects/tstrict_effects.nim create mode 100644 tests/effects/tstrict_effects2.nim create mode 100644 tests/effects/tstrict_effects3.nim create mode 100644 tests/effects/tstrict_effects_sort.nim diff --git a/changelog.md b/changelog.md index 65ae61833..10824cfc6 100644 --- a/changelog.md +++ b/changelog.md @@ -427,6 +427,23 @@ - On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. This is only supported for `--gc:orc` or `--gc:arc`. +- The effect system was refined and there is a new `.effectsOf` annotation that does + explicitly what was previously done implicitly. See the manual for details. + To write code that is portable with older Nim versions, use this idiom: + +```nim + +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + +proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} + +``` + + To enable the new effect system, use --experimental:strictEffects. + ## Compiler changes diff --git a/compiler/ast.nim b/compiler/ast.nim index 5893a5671..50d048edd 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -229,7 +229,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # 47 flags! + TSymFlag* = enum # 48 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -299,6 +299,7 @@ type sfUsedInFinallyOrExcept # symbol is used inside an 'except' or 'finally' sfSingleUsedTemp # For temporaries that we know will only be used once sfNoalias # 'noalias' annotation, means C's 'restrict' + sfEffectsDelayed # an 'effectsDelayed' parameter TSymFlags* = set[TSymFlag] @@ -568,6 +569,7 @@ type # sizeof, alignof, offsetof at CT tfExplicitCallConv tfIsConstructor + tfEffectSystemWorkaround TTypeFlags* = set[TTypeFlag] @@ -1781,7 +1783,7 @@ proc containsNode*(n: PNode, kinds: TNodeKinds): bool = proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool = case n.kind - of nkEmpty..nkNilLit: result = n.kind == kind + of nkEmpty..nkNilLit, nkFormalParams: result = n.kind == kind else: for i in 0.. pragma: arg result.transitionSonsKind(n.kind) +proc processEffectsOf(c: PContext, n: PNode; owner: PSym) = + proc processParam(c: PContext; n: PNode) = + let r = c.semExpr(c, n) + if r.kind == nkSym and r.sym.kind == skParam: + if r.sym.owner == owner: + incl r.sym.flags, sfEffectsDelayed + else: + localError(c.config, n.info, errGenerated, "parameter cannot be declared as .effectsOf") + else: + localError(c.config, n.info, errGenerated, "parameter name expected") + + if n.kind notin nkPragmaCallKinds or n.len != 2: + localError(c.config, n.info, errGenerated, "parameter name expected") + else: + let it = n[1] + if it.kind in {nkCurly, nkBracket}: + for x in items(it): processParam(c, x) + else: + processParam(c, it) + proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, validPragmas: TSpecialWords, comesFromPush, isStatement: bool): bool = @@ -895,6 +915,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wNoalias: noVal(c, it) incl(sym.flags, sfNoalias) + of wEffectsOf: + processEffectsOf(c, it, sym) of wThreadVar: noVal(c, it) incl(sym.flags, {sfThread, sfGlobal}) diff --git a/compiler/sem.nim b/compiler/sem.nim index 804325e56..24709cf21 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -400,9 +400,10 @@ when not defined(nimHasSinkInference): include hlo, seminst, semcall proc resetSemFlag(n: PNode) = - excl n.flags, nfSem - for i in 0.. 0: rr = rr.lastSon - localError(g.config, r.info, errGenerated, renderTree(rr) & " " & msg & typeToString(r.typ)) + message(g.config, r.info, if emitWarnings: warnEffect else: errGenerated, + renderTree(rr) & " " & msg & typeToString(r.typ)) popInfoContext(g.config) # hint about unnecessarily listed exception types: if hints: @@ -1258,11 +1293,11 @@ proc checkMethodEffects*(g: ModuleGraph; disp, branch: PSym) = let p = disp.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): - checkRaisesSpec(g, raisesSpec, actual[exceptionEffects], + checkRaisesSpec(g, false, raisesSpec, actual[exceptionEffects], "can raise an unlisted exception: ", hints=off, subtypeRelation) let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): - checkRaisesSpec(g, tagsSpec, actual[tagEffects], + checkRaisesSpec(g, false, tagsSpec, actual[tagEffects], "can have an unlisted effect: ", hints=off, subtypeRelation) if sfThread in disp.flags and notGcSafe(branch.typ): localError(g.config, branch.info, "base method is GC-safe, but '$1' is not" % @@ -1283,7 +1318,7 @@ proc checkMethodEffects*(g: ModuleGraph; disp, branch: PSym) = "base method has lock level $1, but dispatcher has $2" % [$branch.typ.lockLevel, $disp.typ.lockLevel]) -proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode) = +proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode; s: PSym = nil) = var effects = t.n[0] if t.kind != tyProc or effects.kind != nkEffectList: return if n.kind != nkEmpty: @@ -1292,9 +1327,14 @@ proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode) = let raisesSpec = effectSpec(n, wRaises) if not isNil(raisesSpec): effects[exceptionEffects] = raisesSpec + elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}): + effects[exceptionEffects] = newNodeI(nkArgList, effects.info) + let tagsSpec = effectSpec(n, wTags) if not isNil(tagsSpec): effects[tagEffects] = tagsSpec + elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}): + effects[tagEffects] = newNodeI(nkArgList, effects.info) let requiresSpec = propSpec(n, wRequires) if not isNil(requiresSpec): @@ -1304,15 +1344,21 @@ proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode) = effects[ensuresEffects] = ensuresSpec effects[pragmasEffects] = n + if s != nil and s.magic != mNone: + if s.magic != mEcho: + t.flags.incl tfNoSideEffect -proc initEffects(g: ModuleGraph; effects: PNode; s: PSym; t: var TEffects; c: PContext) = +proc rawInitEffects(g: ModuleGraph; effects: PNode) = newSeq(effects.sons, effectListLen) - effects[exceptionEffects] = newNodeI(nkArgList, s.info) - effects[tagEffects] = newNodeI(nkArgList, s.info) + effects[exceptionEffects] = newNodeI(nkArgList, effects.info) + effects[tagEffects] = newNodeI(nkArgList, effects.info) effects[requiresEffects] = g.emptyNode effects[ensuresEffects] = g.emptyNode effects[pragmasEffects] = g.emptyNode +proc initEffects(g: ModuleGraph; effects: PNode; s: PSym; t: var TEffects; c: PContext) = + rawInitEffects(g, effects) + t.exc = effects[exceptionEffects] t.tags = effects[tagEffects] t.owner = s @@ -1341,10 +1387,14 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = if effects.kind != nkEffectList: return # effects already computed? if not s.hasRealBody: return - if effects.len == effectListLen: return + let emitWarnings = tfEffectSystemWorkaround in s.typ.flags + if effects.len == effectListLen and not emitWarnings: return + + var inferredEffects = newNodeI(nkEffectList, s.info) var t: TEffects - initEffects(g, effects, s, t, c) + initEffects(g, inferredEffects, s, t, c) + rawInitEffects g, effects track(t, body) if s.kind != skMacro: @@ -1369,17 +1419,21 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let p = s.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): - checkRaisesSpec(g, raisesSpec, t.exc, "can raise an unlisted exception: ", + checkRaisesSpec(g, emitWarnings, raisesSpec, t.exc, "can raise an unlisted exception: ", hints=on, subtypeRelation, hintsArg=s.ast[0]) # after the check, use the formal spec: effects[exceptionEffects] = raisesSpec + else: + effects[exceptionEffects] = t.exc let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): - checkRaisesSpec(g, tagsSpec, t.tags, "can have an unlisted effect: ", + checkRaisesSpec(g, emitWarnings, tagsSpec, t.tags, "can have an unlisted effect: ", hints=off, subtypeRelation) # after the check, use the formal spec: effects[tagEffects] = tagsSpec + else: + effects[tagEffects] = t.tags let requiresSpec = propSpec(p, wRequires) if not isNil(requiresSpec): diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 14dc89781..df0e2778b 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1953,6 +1953,10 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not hasProto: implicitPragmas(c, s, n.info, validPragmas) + if n[pragmasPos].kind != nkEmpty: + setEffectsForProcType(c.graph, s.typ, n[pragmasPos], s) + s.typ.flags.incl tfEffectSystemWorkaround + # To ease macro generation that produce forwarded .async procs we now # allow a bit redundancy in the pragma declarations. The rule is # a prototype's pragma list must be a superset of the current pragma @@ -2211,8 +2215,9 @@ proc evalInclude(c: PContext, n: PNode): PNode = incMod(c, n, it, result) proc setLine(n: PNode, info: TLineInfo) = - for i in 0..`_ pretends to be free of side effects +so that it can be used for debugging routines marked as `noSideEffect`. + +`func` is syntactic sugar for a proc with no side effects: + +.. code-block:: nim + func `+` (x, y: int): int + + +To override the compiler's side effect analysis a `{.noSideEffect.}` +`cast` pragma block can be used: + +.. code-block:: nim + + func f() = + {.cast(noSideEffect).}: + echo "test" + +**Side effects are usually inferred. The inference for side effects is +analogous to the inference for exception tracking.** + + +GC safety effect +---------------- + +We call a proc `p` `GC safe`:idx: when it doesn't access any global variable +that contains GC'ed memory (`string`, `seq`, `ref` or a closure) either +directly or indirectly through a call to a GC unsafe proc. + +**The GC safety property is usually inferred. The inference for GC safety is +analogous to the inference for exception tracking.** + +The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe, +otherwise this property is inferred by the compiler. Note that `noSideEffect` +implies `gcsafe`. + +Routines that are imported from C are always assumed to be `gcsafe`. + +To override the compiler's gcsafety analysis a `{.cast(gcsafe).}` pragma block can +be used: + +.. code-block:: nim + + var + someGlobal: string = "some string here" + perThread {.threadvar.}: string + + proc setPerThread() = + {.cast(gcsafe).}: + deepCopy(perThread, someGlobal) + + +See also: + +- `Shared heap memory management `_. + + Effects pragma -------------- @@ -6366,55 +6486,6 @@ This pragma can also take in an optional warning string to relay to developers. proc thing(x: bool) {.deprecated: "use thong instead".} -noSideEffect pragma -------------------- - -The `noSideEffect` pragma is used to mark a proc/iterator that can have only -side effects through parameters. This means that the proc/iterator only changes locations that are -reachable from its parameters and the return value only depends on the -parameters. If none of its parameters have the type `var`, `ref`, `ptr`, `cstring`, or `proc`, -then no locations are modified. - -It is a static error to mark a proc/iterator to have no side effect if the compiler cannot verify this. - -As a special semantic rule, the built-in `debugEcho -`_ pretends to be free of side effects -so that it can be used for debugging routines marked as `noSideEffect`. - -`func` is syntactic sugar for a proc with no side effects: - -.. code-block:: nim - func `+` (x, y: int): int - - -To override the compiler's side effect analysis a `{.noSideEffect.}` -`cast` pragma block can be used: - -.. code-block:: nim - - func f() = - {.cast(noSideEffect).}: - echo "test" - -When a `noSideEffect` proc has proc params `bar`, whether it can be used inside a `noSideEffect` context -depends on what the compiler knows about `bar`: - -.. code-block:: nim - :test: "nim c $1" - - func foo(bar: proc(): int): int = bar() - var count = 0 - proc fn1(): int = 1 - proc fn2(): int = (count.inc; count) - func fun1() = discard foo(fn1) # ok because fn1 is inferred as `func` - # func fun2() = discard foo(fn2) # would give: Error: 'fun2' can have side effects - - # with callbacks, the compiler is conservative, ie that bar will have side effects - var foo2: type(foo) = foo - func main() = - discard foo(fn1) # ok - # discard foo2(fn1) # now this errors - compileTime pragma ------------------ @@ -7861,6 +7932,10 @@ collected) heap, and sharing of memory is restricted to global variables. This helps to prevent race conditions. GC efficiency is improved quite a lot, because the GC never has to stop other threads and see what they reference. +The only way to create a thread is via `spawn` or +`createThread`. The invoked proc must not use `var` parameters nor must +any of its parameters contain a `ref` or `closure` type. This enforces +the *no heap sharing restriction*. Thread pragma ------------- @@ -7875,43 +7950,6 @@ A thread proc is passed to `createThread` or `spawn` and invoked indirectly; so the `thread` pragma implies `procvar`. -GC safety ---------- - -We call a proc `p` `GC safe`:idx: when it doesn't access any global variable -that contains GC'ed memory (`string`, `seq`, `ref` or a closure) either -directly or indirectly through a call to a GC unsafe proc. - -The `gcsafe`:idx: annotation can be used to mark a proc to be gcsafe, -otherwise this property is inferred by the compiler. Note that `noSideEffect` -implies `gcsafe`. The only way to create a thread is via `spawn` or -`createThread`. The invoked proc must not use `var` parameters nor must -any of its parameters contain a `ref` or `closure` type. This enforces -the *no heap sharing restriction*. - -Routines that are imported from C are always assumed to be `gcsafe`. -To disable the GC-safety checking the `--threadAnalysis:off`:option: command-line -switch can be used. This is a temporary workaround to ease the porting effort -from old code to the new threading model. - -To override the compiler's gcsafety analysis a `{.cast(gcsafe).}` pragma block can -be used: - -.. code-block:: nim - - var - someGlobal: string = "some string here" - perThread {.threadvar.}: string - - proc setPerThread() = - {.cast(gcsafe).}: - deepCopy(perThread, someGlobal) - - -See also: - -- `Shared heap memory management `_. - Threadvar pragma ---------------- diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index c96f599e8..1ddcc9843 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -147,8 +147,13 @@ proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] {.inline, deprecated: "use: `reversed(toOpenArray(a, first, last))`".} = reversed(toOpenArray(a, first, last)) +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + proc binarySearch*[T, K](a: openArray[T], key: K, - cmp: proc (x: T, y: K): int {.closure.}): int = + cmp: proc (x: T, y: K): int {.closure.}): int {.effectsOf: cmp.} = ## Binary search for `key` in `a`. Return the index of `key` or -1 if not found. ## Assumes that `a` is sorted according to `cmp`. ## @@ -210,7 +215,7 @@ const onlySafeCode = true proc lowerBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is not less than ## (i.e. greater or equal to) `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -260,7 +265,7 @@ proc lowerBound*[T](a: openArray[T], key: T): int = lowerBound(a, key, cmp[T]) ## * `upperBound proc<#upperBound,openArray[T],T>`_ proc upperBound*[T, K](a: openArray[T], key: K, - cmp: proc(x: T, k: K): int {.closure.}): int = + cmp: proc(x: T, k: K): int {.closure.}): int {.effectsOf: cmp.} = ## Returns the index of the first element in `a` that is greater than ## `key`, or last if no such element is found. ## In other words if you have a sorted sequence and you call @@ -318,7 +323,7 @@ template `<-`(a, b) = copyMem(addr(a), addr(b), sizeof(T)) proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, - cmp: proc (x, y: T): int {.closure.}, order: SortOrder) = + cmp: proc (x, y: T): int {.closure.}, order: SortOrder) {.effectsOf: cmp.} = # Optimization: If max(left) <= min(right) there is nothing to do! # 1 2 3 4 ## 5 6 7 8 # -> O(n) for sorted arrays. @@ -358,7 +363,7 @@ proc mergeAlt[T](a, b: var openArray[T], lo, m, hi: int, func sort*[T](a: var openArray[T], cmp: proc (x, y: T): int {.closure.}, - order = SortOrder.Ascending) = + order = SortOrder.Ascending) {.effectsOf: cmp.} = ## Default Nim sort (an implementation of merge sort). The sorting ## is guaranteed to be stable (that is, equal elements stay in the same order) ## and the worst case is guaranteed to be O(n log n). @@ -420,7 +425,7 @@ proc sort*[T](a: var openArray[T], order = SortOrder.Ascending) = sort[T](a, ## * `sortedByIt template<#sortedByIt.t,untyped,untyped>`_ proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): seq[T] = + order = SortOrder.Ascending): seq[T] {.effectsOf: cmp.} = ## Returns `a` sorted by `cmp` in the specified `order`. ## ## **See also:** @@ -497,7 +502,7 @@ template sortedByIt*(seq1, op: untyped): untyped = func isSorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, - order = SortOrder.Ascending): bool = + order = SortOrder.Ascending): bool {.effectsOf: cmp.} = ## Checks to see whether `a` is already sorted in `order` ## using `cmp` for the comparison. The parameters are identical ## to `sort`. Requires O(n) time. @@ -545,7 +550,7 @@ proc isSorted*[T](a: openArray[T], order = SortOrder.Ascending): bool = proc merge*[T]( result: var seq[T], x, y: openArray[T], cmp: proc(x, y: T): int {.closure.} -) {.since: (1, 5, 1).} = +) {.since: (1, 5, 1), effectsOf: cmp.} = ## Merges two sorted `openArray`. `x` and `y` are assumed to be sorted. ## If you do not wish to provide your own `cmp`, ## you may use `system.cmp` or instead call the overloaded @@ -638,7 +643,7 @@ proc product*[T](x: openArray[seq[T]]): seq[seq[T]] = ## Produces the Cartesian product of the array. ## Every element of the result is a combination of one element from each seq in `x`, ## with the ith element coming from `x[i]`. - ## + ## ## .. warning:: complexity may explode. runnableExamples: assert product(@[@[1], @[2]]) == @[@[1, 2]] diff --git a/lib/pure/times.nim b/lib/pure/times.nim index b2a22250d..b41bb28b5 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1333,7 +1333,7 @@ proc now*(): DateTime {.tags: [TimeEffect], benign.} = ## `cpuTime` instead, depending on the use case. getTime().local -proc dateTime*(year: int, month: Month, monthday: MonthdayRange, +proc dateTime*(year: int, month: Month, monthday: MonthdayRange, hour: HourRange = 0, minute: MinuteRange = 0, second: SecondRange = 0, nanosecond: NanosecondRange = 0, zone: Timezone = local()): DateTime = diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index b3726c9c3..190316f93 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -8,13 +8,18 @@ import os when defined(windows): from strutils import replace +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + type PathEntry* = object kind*: PathComponent path*: string iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect], effectsOf: follow.} = ## Improved `os.walkDirRec`. #[ note: a yieldFilter isn't needed because caller can filter at call site, without diff --git a/lib/system/io.nim b/lib/system/io.nim index d8b2ac741..59f070f73 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -78,7 +78,7 @@ proc c_fputs(c: cstring, f: File): cint {. proc c_fgets(c: cstring, n: cint, f: File): cstring {. importc: "fgets", header: "", tags: [ReadIOEffect].} proc c_fgetc(stream: File): cint {. - importc: "fgetc", header: "", tags: [ReadIOEffect].} + importc: "fgetc", header: "", tags: [].} proc c_ungetc(c: cint, f: File): cint {. importc: "ungetc", header: "", tags: [].} proc c_putc(c: cint, stream: File): cint {. diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 9ece10e3e..b4554d778 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -261,7 +261,7 @@ proc cpDir*(`from`, to: string) {.raises: [OSError].} = checkOsError() proc exec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. ## @@ -274,7 +274,7 @@ proc exec*(command: string) {. checkOsError() proc exec*(command: string, input: string, cache = "") {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external process. If the external process terminates with ## a non-zero exit code, an OSError exception is raised. log "exec: " & command: @@ -284,7 +284,7 @@ proc exec*(command: string, input: string, cache = "") {. echo output proc selfExec*(command: string) {. - raises: [OSError], tags: [ExecIOEffect].} = + raises: [OSError], tags: [ExecIOEffect, WriteIOEffect].} = ## Executes an external command with the current nim/nimble executable. ## `Command` must not contain the "nim " part. let c = selfExe() & " " & command diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 3499b5326..458a0394f 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -122,7 +122,7 @@ window.addEventListener('DOMContentLoaded', main);

    Procs

    -
    proc foo() {....raises: [], tags: [].}
    +
    proc foo() {....raises: [], tags: [WriteIOEffect].}
    I do foo diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index d98b1b00f..dbacb57a7 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -654,7 +654,7 @@ This is deprecated with a message. 
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
-    header: "<stdio.h>", varargs, discardable.}
    + header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
    @@ -663,7 +663,7 @@ This is deprecated with a message. 
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
-                                     varargs, discardable.}
    + varargs, discardable, ...raises: [], tags: [].}
    the c printf. etc. @@ -690,7 +690,8 @@ came form utils but should be shown where
    -
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    +
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [].}

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    @@ -704,7 +705,8 @@ came form utils but should be shown where
    -
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect.}
    +
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [].}

    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

    diff --git a/tests/effects/teffects1.nim b/tests/effects/teffects1.nim index 66d6a3518..82efefe77 100644 --- a/tests/effects/teffects1.nim +++ b/tests/effects/teffects1.nim @@ -6,7 +6,7 @@ teffects1.nim(22, 29) Hint: 'lier' cannot raise 'IO2Error' [XCannotRaiseY] teffects1.nim(38, 21) Error: type mismatch: got but expected 'MyProcType = proc (x: int): string{.closure.}' .raise effects differ''' """ - +{.push warningAsError[Effect]: on.} type TObj {.pure, inheritable.} = object TObjB = object of TObj @@ -41,3 +41,4 @@ type mismatch: got but ]# {.pop.} +{.pop.} diff --git a/tests/effects/teffects2.nim b/tests/effects/teffects2.nim index e4b50aba5..ec3c0a887 100644 --- a/tests/effects/teffects2.nim +++ b/tests/effects/teffects2.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: ref IOError" line: 19 """ - +{.push warningAsError[Effect]: on.} type TObj = object {.pure, inheritable.} TObjB = object of TObj @@ -17,3 +17,4 @@ proc lier(): int {.raises: [IOError].} = proc forw: int = raise newException(IOError, "arg") +{.pop.} diff --git a/tests/effects/teffects7.nim b/tests/effects/teffects7.nim index 73865b18d..9b7fbf5f0 100644 --- a/tests/effects/teffects7.nim +++ b/tests/effects/teffects7.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: ref ValueError" line: 10 """ - +{.push warningAsError[Effect]: on.} proc foo() {.raises: [].} = try: discard @@ -12,3 +12,5 @@ proc foo() {.raises: [].} = discard foo() + +{.pop.} diff --git a/tests/effects/teffects8.nim b/tests/effects/teffects8.nim index fb3c088d6..359b3a1df 100644 --- a/tests/effects/teffects8.nim +++ b/tests/effects/teffects8.nim @@ -2,7 +2,7 @@ discard """ errormsg: "can raise an unlisted exception: Exception" line: 10 """ - +{.push warningAsError[Effect]: on.} proc foo() {.raises: [].} = try: discard @@ -10,3 +10,4 @@ proc foo() {.raises: [].} = raise foo() +{.pop.} diff --git a/tests/effects/tstrict_effects.nim b/tests/effects/tstrict_effects.nim new file mode 100644 index 000000000..eee8fb71a --- /dev/null +++ b/tests/effects/tstrict_effects.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "s1 can raise an unlisted exception: CatchableError" + line: 27 +""" + +{.push warningAsError[Effect]: on.} +{.experimental: "strictEffects".} + +# bug #18376 + +{.push raises: [Defect].} +type Call = proc (x: int): int {.gcsafe, raises: [Defect, CatchableError].} + +type Bar* = object + foo*: Call + +proc passOn*(x: Call) = discard + +proc barCal(b: var Bar, s: string, s1: Call) = + #compiler complains that his line can throw CatchableError + passOn s1 + + +proc passOnB*(x: Call) {.effectsOf: x.} = discard + +proc barCal2(b: var Bar, s: string, s1: Call) = + passOnB s1 diff --git a/tests/effects/tstrict_effects2.nim b/tests/effects/tstrict_effects2.nim new file mode 100644 index 000000000..acc0a0540 --- /dev/null +++ b/tests/effects/tstrict_effects2.nim @@ -0,0 +1,28 @@ +discard """ + errormsg: "can raise an unlisted exception: Exception" + line: 23 +""" + +{.push warningAsError[Effect]: on.} +{.experimental: "strictEffects".} + +# bug #13905 + +proc atoi(v: cstring): cint {.importc: "atoi", cdecl, raises: [].} + +type Conv = proc(v: cstring): cint {.cdecl, raises: [].} + +var x: Conv = atoi + +# bug #17475 + +type + Callback = proc() + +proc f(callback: Callback) {.raises: [].} = + callback() + +proc main = + f(proc () = raise newException(IOError, "IO")) + +main() diff --git a/tests/effects/tstrict_effects3.nim b/tests/effects/tstrict_effects3.nim new file mode 100644 index 000000000..1ab15cc80 --- /dev/null +++ b/tests/effects/tstrict_effects3.nim @@ -0,0 +1,17 @@ +discard """ + action: compile +""" + +{.push warningAsError[Effect]: on.} + +{.experimental: "strictEffects".} + +proc fn(a: int, p1, p2: proc()) {.effectsOf: p1.} = + if a == 7: + p1() + if a<0: + raise newException(ValueError, $a) + +proc main() {.raises: [ValueError].} = + fn(1, proc()=discard, proc() = raise newException(IOError, "foo")) +main() diff --git a/tests/effects/tstrict_effects_sort.nim b/tests/effects/tstrict_effects_sort.nim new file mode 100644 index 000000000..8928ed0d3 --- /dev/null +++ b/tests/effects/tstrict_effects_sort.nim @@ -0,0 +1,27 @@ +discard """ + errormsg: "cmpE can raise an unlisted exception: Exception" + line: 27 +""" + +{.push warningAsError[Effect]: on.} + +{.experimental: "strictEffects".} + +import algorithm + +type + MyInt = distinct int + +var toSort = @[MyInt 1, MyInt 2, MyInt 3] + +proc cmpN(a, b: MyInt): int = + cmp(a.int, b.int) + +proc harmless {.raises: [].} = + toSort.sort cmpN + +proc cmpE(a, b: MyInt): int {.raises: [Exception].} = + cmp(a.int, b.int) + +proc harmfull {.raises: [].} = + toSort.sort cmpE diff --git a/tests/pragmas/tuserpragma2.nim b/tests/pragmas/tuserpragma2.nim index ce16c4649..c3f31cd5e 100644 --- a/tests/pragmas/tuserpragma2.nim +++ b/tests/pragmas/tuserpragma2.nim @@ -3,9 +3,10 @@ discard """ file: "tuserpragma2.nim" line: 11 """ - +{.push warningAsError[Effect]: on.} # bug #7216 {.pragma: my_pragma, raises: [].} proc test1 {.my_pragma.} = raise newException(Exception, "msg") +{.pop.} diff --git a/tests/stdlib/tstdlib_various.nim b/tests/stdlib/tstdlib_various.nim index b153fd2ba..bc90d6ef4 100644 --- a/tests/stdlib/tstdlib_various.nim +++ b/tests/stdlib/tstdlib_various.nim @@ -20,12 +20,6 @@ Hi Andreas! How do you feel, Rumpf? @[0, 2, 1] @[0, 1, 2] 055this should be the casehugh@["(", "+", " 1", " 2", ")"] -caught a crash! -caught a crash! -caught a crash! -caught a crash! -caught a crash! -caught a crash! [5] [4, 5] [3, 4, 5] @@ -161,18 +155,21 @@ block tropes: block tsegfaults: - proc main = - try: - var x: ptr int - echo x[] + when not defined(arm64): + var crashes = 0 + proc main = try: - raise newException(ValueError, "not a crash") - except ValueError: - discard - except NilAccessDefect: - echo "caught a crash!" - for i in 0..5: - main() + var x: ptr int + echo x[] + try: + raise newException(ValueError, "not a crash") + except ValueError: + discard + except NilAccessDefect: + inc crashes + for i in 0..5: + main() + assert crashes == 6 From 7c8ea490a2be36d171d362c43212195ea8ce489f Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Thu, 2 Sep 2021 21:12:14 +0900 Subject: [PATCH 0365/1654] Fix initrand to avoid random number sequences overlapping (#18744) * Fix initrand to avoid random number sequences overlapping * Minor fix * Fix compile error on js backend * Disable new test for js backend * Minor fix * tempfiles module uses random.initRand() * Remove unused module import from lib/std/tempfiles.nim * Initialize baseState in initRand() * Run tests/stdlib/trandom.nim from tests/test_nimscript.nims * baseState is initialized only with sysrand.urandom and quit if failed * Add comments --- lib/pure/random.nim | 74 +++++++++++++++++++++++++++++++++------ lib/std/tempfiles.nim | 7 ++-- tests/stdlib/trandom.nim | 32 ++++++++++++++--- tests/test_nimscript.nims | 2 ++ 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/lib/pure/random.nim b/lib/pure/random.nim index a292386af..b8aeb86e0 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -108,10 +108,19 @@ when defined(js): a0: 0x69B4C98Cu32, a1: 0xFED1DD30u32) # global for backwards compatibility else: - # racy for multi-threading but good enough for now: - var state = Rand( + const DefaultRandSeed = Rand( a0: 0x69B4C98CB8530805u64, - a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility + a1: 0xFED1DD3004688D67CAu64) + + # racy for multi-threading but good enough for now: + var state = DefaultRandSeed # global for backwards compatibility + +func isValid(r: Rand): bool {.inline.} = + ## Check whether state of `r` is valid. + ## + ## In `xoroshiro128+`, if all bits of `a0` and `a1` are zero, + ## they are always zero after calling `next(r: var Rand)`. + not (r.a0 == 0 and r.a1 == 0) since (1, 5): template randState*(): untyped = @@ -617,15 +626,28 @@ proc shuffle*[T](x: var openArray[T]) = shuffle(state, x) -when not defined(nimscript) and not defined(standalone): - import times +when not defined(standalone): + when defined(js): + import std/times + else: + when defined(nimscript): + import std/hashes + else: + import std/[hashes, os, sysrand, monotimes] + + when compileOption("threads"): + import locks + var baseSeedLock: Lock + baseSeedLock.initLock + + var baseState: Rand proc initRand(): Rand = - ## Initializes a new Rand state with a seed based on the current time. + ## Initializes a new Rand state. ## ## The resulting state is independent of the default RNG's state. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## See also: ## * `initRand proc<#initRand,int64>`_ that accepts a seed for a new Rand state @@ -635,20 +657,50 @@ when not defined(nimscript) and not defined(standalone): let time = int64(times.epochTime() * 1000) and 0x7fff_ffff result = initRand(time) else: - let now = times.getTime() - result = initRand(convert(Seconds, Nanoseconds, now.toUnix) + now.nanosecond) + proc getRandomState(): Rand = + when defined(nimscript): + result = Rand( + a0: CompileTime.hash.Ui, + a1: CompileDate.hash.Ui) + if not result.isValid: + result = DefaultRandSeed + else: + var urand: array[sizeof(Rand), byte] + + for i in 0 .. 7: + if sysrand.urandom(urand): + copyMem(result.addr, urand[0].addr, sizeof(Rand)) + if result.isValid: + break + + if not result.isValid: + # Don't try to get alternative random values from other source like time or process/thread id, + # because such code would be never tested and is a liability for security. + quit("Failed to initializes baseState in random module as sysrand.urandom doesn't work.") + + when compileOption("threads"): + baseSeedLock.withLock: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers + else: + if not baseState.isValid: + baseState = getRandomState() + result = baseState + baseState.skipRandomNumbers since (1, 5, 1): export initRand proc randomize*() {.benign.} = ## Initializes the default random number generator with a seed based on - ## the current time. + ## random number source. ## ## This proc only needs to be called once, and it should be called before ## the first usage of procs from this module that use the default RNG. ## - ## **Note:** Does not work for NimScript or the compile-time VM. + ## **Note:** Does not work for the compile-time VM. ## ## **See also:** ## * `randomize proc<#randomize,int64>`_ that accepts a seed diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index f1cfd5885..26786d463 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -17,7 +17,7 @@ See also: * `mkstemp` (posix), refs https://man7.org/linux/man-pages/man3/mkstemp.3.html ]# -import os, random, std/monotimes +import os, random const @@ -107,11 +107,8 @@ var nimTempPathState {.threadvar.}: NimTempPathState template randomPathName(length: Natural): string = var res = newString(length) if not nimTempPathState.isInit: - var time = getMonoTime().ticks - when compileOption("threads"): - time = time xor int64(getThreadId()) nimTempPathState.isInit = true - nimTempPathState.state = initRand(time) + nimTempPathState.state = initRand() for i in 0 ..< length: res[i] = nimTempPathState.state.sample(letters) diff --git a/tests/stdlib/trandom.nim b/tests/stdlib/trandom.nim index 39ccca85b..61e858f86 100644 --- a/tests/stdlib/trandom.nim +++ b/tests/stdlib/trandom.nim @@ -23,9 +23,10 @@ proc main() = doAssert a in [[0,1], [1,0]] doAssert rand(0) == 0 - doAssert sample("a") == 'a' + when not defined(nimscript): + doAssert sample("a") == 'a' - when compileOption("rangeChecks"): + when compileOption("rangeChecks") and not defined(nimscript): doAssertRaises(RangeDefect): discard rand(-1) @@ -92,7 +93,7 @@ block: # random int block: # again gives new numbers var rand1 = rand(1000000) - when not defined(js): + when not (defined(js) or defined(nimscript)): os.sleep(200) var rand2 = rand(1000000) @@ -122,7 +123,7 @@ block: # random float block: # again gives new numbers var rand1: float = rand(1000000.0) - when not defined(js): + when not (defined(js) or defined(nimscript)): os.sleep(200) var rand2: float = rand(1000000.0) @@ -248,3 +249,26 @@ block: # bug #17670 type UInt48 = range[0'u64..2'u64^48-1] let x = rand(UInt48) doAssert x is UInt48 + +block: # bug #17898 + # Checks whether `initRand()` generates unique states. + # size should be 2^64, but we don't have time and space. + + # Disable this test for js until js gets proper skipRandomNumbers. + when not defined(js): + const size = 1000 + var + rands: array[size, Rand] + randSet: HashSet[Rand] + for i in 0.. Date: Thu, 2 Sep 2021 14:35:25 +0200 Subject: [PATCH 0366/1654] sequtils now support strict effect tracking (#18782) --- lib/pure/collections/sequtils.nim | 23 ++++++++++++++--------- tests/stdlib/tsequtils.nim | 5 +++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 5b41ea00d..12b961371 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -84,6 +84,11 @@ import std/private/since import macros +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} +else: + {.pragma: effectsOf.} + macro evalOnceAs(expAlias, exp: untyped, letAssigneable: static[bool]): untyped = ## Injects `expAlias` in caller scope, to avoid bugs involving multiple @@ -356,7 +361,7 @@ func distribute*[T](s: seq[T], num: Positive, spread = true): seq[seq[T]] = first = last proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): - seq[S]{.inline.} = + seq[S] {.inline, effectsOf: op.} = ## Returns a new sequence with the results of the `op` proc applied to every ## item in the container `s`. ## @@ -382,7 +387,7 @@ proc map*[T, S](s: openArray[T], op: proc (x: T): S {.closure.}): result[i] = op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s`, modifying it directly. ## ## Note that the container `s` must be declared as a `var`, @@ -401,7 +406,7 @@ proc apply*[T](s: var openArray[T], op: proc (x: var T) {.closure.}) for i in 0 ..< s.len: op(s[i]) proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) - {.inline.} = + {.inline, effectsOf: op.} = ## Applies `op` to every item in `s` modifying it directly. ## ## Note that the container `s` must be declared as a `var` @@ -420,7 +425,7 @@ proc apply*[T](s: var openArray[T], op: proc (x: T): T {.closure.}) for i in 0 ..< s.len: s[i] = op(s[i]) -proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3).} = +proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1, 3), effectsOf: op.} = ## Same as `apply` but for a proc that does not return anything ## and does not mutate `s` directly. runnableExamples: @@ -429,7 +434,7 @@ proc apply*[T](s: openArray[T], op: proc (x: T) {.closure.}) {.inline, since: (1 assert message == "01234" for i in 0 ..< s.len: op(s[i]) -iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = +iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T {.effectsOf: pred.} = ## Iterates through a container `s` and yields every item that fulfills the ## predicate `pred` (a function that returns a `bool`). ## @@ -453,7 +458,7 @@ iterator filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): T = yield s[i] proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] - {.inline.} = + {.inline, effectsOf: pred.} = ## Returns a new sequence with all the items of `s` that fulfill the ## predicate `pred` (a function that returns a `bool`). ## @@ -480,7 +485,7 @@ proc filter*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): seq[T] result.add(s[i]) proc keepIf*[T](s: var seq[T], pred: proc(x: T): bool {.closure.}) - {.inline.} = + {.inline, effectsOf: pred.} = ## Keeps the items in the passed sequence `s` if they fulfill the ## predicate `pred` (a function that returns a `bool`). ## @@ -685,7 +690,7 @@ since (1, 1): if pred: result += 1 result -proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc all*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if every item fulfills the ## predicate. ## @@ -727,7 +732,7 @@ template allIt*(s, pred: untyped): bool = break result -proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool = +proc any*[T](s: openArray[T], pred: proc(x: T): bool {.closure.}): bool {.effectsOf: pred.} = ## Iterates through a container and checks if at least one item ## fulfills the predicate. ## diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim index 6c9c92873..338473407 100644 --- a/tests/stdlib/tsequtils.nim +++ b/tests/stdlib/tsequtils.nim @@ -8,6 +8,9 @@ import std/sequtils import strutils from algorithm import sorted +{.experimental: "strictEffects".} +{.push warningAsError[Effect]: on.} + # helper for testing double substitution side effects which are handled # by `evalOnceAs` var counter = 0 @@ -507,3 +510,5 @@ template main = static: main() main() + +{.pop.} From a7cae2bda24871c55348173816a43144b9e668a5 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Sep 2021 17:15:11 +0200 Subject: [PATCH 0367/1654] fixes #16325 [backport:1.4] (#18784) --- compiler/sigmatch.nim | 3 +-- .../tproc_types_dont_like_subtypes.nim | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/overload/tproc_types_dont_like_subtypes.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 465a9a51d..9cf18277f 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -595,8 +595,7 @@ proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = # if f is metatype. result = typeRel(c, f, a) - # v--- is this correct? - if result <= isIntConv or inconsistentVarTypes(f, a): + if result <= isSubrange or inconsistentVarTypes(f, a): result = isNone #if result == isEqual: diff --git a/tests/overload/tproc_types_dont_like_subtypes.nim b/tests/overload/tproc_types_dont_like_subtypes.nim new file mode 100644 index 000000000..d322f41be --- /dev/null +++ b/tests/overload/tproc_types_dont_like_subtypes.nim @@ -0,0 +1,20 @@ +discard """ + errormsg: "got " + line: 20 +""" + +type + A = ref object of RootObj + B = ref object of A + + P = proc (a: A) + +# bug #16325 + +proc doThings(a: A, p: P) = + p(a) + +var x = proc (b: B) {.closure.} = + echo "B" + +doThings(B(), x) From f46569bafdfdd3290baf132f025d61a0e550dd58 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Sep 2021 17:16:03 +0200 Subject: [PATCH 0368/1654] fixes #18494 (#18783) --- lib/system/alloc.nim | 46 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 479458950..6d83a2c17 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -1053,22 +1053,48 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = inc(result, it.size) it = it.next - proc getFreeMem(): int = - result = allocator.freeMem - #sysAssert(result == countFreeMem()) + when hasThreadSupport: + var sharedHeap: MemRegion + var heapLock: SysLock + initSysLock(heapLock) - proc getTotalMem(): int = return allocator.currMem - proc getOccupiedMem(): int = return allocator.occ #getTotalMem() - getFreeMem() - proc getMaxMem*(): int = return getMaxMem(allocator) + proc getFreeMem(): int = + #sysAssert(result == countFreeMem()) + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = sharedHeap.freeMem + releaseSys(heapLock) + else: + result = allocator.freeMem + + proc getTotalMem(): int = + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = sharedHeap.currMem + releaseSys(heapLock) + else: + result = allocator.currMem + + proc getOccupiedMem(): int = + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = sharedHeap.occ + releaseSys(heapLock) + else: + result = allocator.occ #getTotalMem() - getFreeMem() + + proc getMaxMem*(): int = + when hasThreadSupport and defined(gcDestructors): + acquireSys(heapLock) + result = getMaxMem(sharedHeap) + releaseSys(heapLock) + else: + result = getMaxMem(allocator) when defined(nimTypeNames): proc getMemCounters*(): (int, int) = getMemCounters(allocator) # -------------------- shared heap region ---------------------------------- - when hasThreadSupport: - var sharedHeap: MemRegion - var heapLock: SysLock - initSysLock(heapLock) proc allocSharedImpl(size: Natural): pointer = when hasThreadSupport: From 64165f03fc96d6e8293f1b065a2e1e45f9fc1bf8 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 2 Sep 2021 22:15:08 -0700 Subject: [PATCH 0369/1654] testament --megatest:off now makes tests non-joinable so that they all run (#18787) --- testament/categories.nim | 2 +- testament/testament.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 6375105ee..6702d24a2 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -541,7 +541,7 @@ proc processSingleTest(r: var TResults, cat: Category, options, test: string, ta proc isJoinableSpec(spec: TSpec): bool = # xxx simplify implementation using a whitelist of fields that are allowed to be # set to non-default values (use `fieldPairs`), to avoid issues like bug #16576. - result = not spec.sortoutput and + result = useMegatest and not spec.sortoutput and spec.action == actionRun and not fileExists(spec.file.changeFileExt("cfg")) and not fileExists(spec.file.changeFileExt("nims")) and diff --git a/testament/testament.nim b/testament/testament.nim index ab6620c84..105f00156 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -28,6 +28,7 @@ var useColors = true var backendLogging = true var simulate = false var optVerbose = false +var useMegatest = true proc verboseCmd(cmd: string) = if optVerbose: @@ -645,7 +646,6 @@ proc main() = var targetsStr = "" var isMainProcess = true var skipFrom = "" - var useMegatest = true var p = initOptParser() p.next() From 0887dcc398a38c91e99a043acbb095afc7200460 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 09:03:26 +0200 Subject: [PATCH 0370/1654] fixes #18786 (#18788) --- compiler/sempass2.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index b584bb232..cae2f926c 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -296,12 +296,11 @@ proc useVarNoInitCheck(a: PEffects; n: PNode; s: PSym) = if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet} and s.magic != mNimvm: if s.guard != nil: guardGlobal(a, n, s.guard) - if strictEffects notin a.c.features: - if {sfGlobal, sfThread} * s.flags == {sfGlobal} and - (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): - #if a.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n) - markGcUnsafe(a, s) - markSideEffect(a, s, n.info) + if {sfGlobal, sfThread} * s.flags == {sfGlobal} and + (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): + #if a.config.hasWarn(warnGcUnsafe): warnAboutGcUnsafe(n) + markGcUnsafe(a, s) + markSideEffect(a, s, n.info) if s.owner != a.owner and s.kind in {skVar, skLet, skForVar, skResult, skParam} and {sfGlobal, sfThread} * s.flags == {}: a.isInnerProc = true From 0635697a89b3aef2a188a0bddb46965bae7ee70f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Fri, 3 Sep 2021 14:20:57 +0200 Subject: [PATCH 0371/1654] Document `-d:async_backend` flag (#18700) * Document `-d:asyncBackend` flag Since libraries may want to support multiple async backends, it's useful to standardize the way this is done, so as to simplify usage of any async backend within the ecosystem. A similar text is being added to chronos here: https://github.com/status-im/nim-chronos/pull/216 See also https://github.com/status-im/nim-chronos/issues/207 * typos, add none * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim Co-authored-by: Andreas Rumpf --- lib/pure/asyncdispatch.nim | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 955e66f6b..d87e2360f 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -165,6 +165,34 @@ ## ================ ## ## * The effect system (`raises: []`) does not work with async procedures. +## +## +## Multiple async backend support +## ============================== +## +## Thanks to its powerful macro support, Nim allows ``async``/``await`` to be +## implemented in libraries with only minimal support from the language - as +## such, multiple ``async`` libraries exist, including ``asyncdispatch`` and +## ``chronos``, and more may come to be developed in the future. +## +## Libraries built on top of async/await may wish to support multiple async +## backends - the best way to do so is to create separate modules for each backend +## that may be imported side-by-side. +## +## An alternative way is to select backend using a global compile flag - this +## method makes it difficult to compose applications that use both backends as may +## happen with transitive dependencies, but may be appropriate in some cases - +## libraries choosing this path should call the flag `asyncBackend`, allowing +## applications to choose the backend with `-d:asyncBackend=`. +## +## Known `async` backends include: +## +## * `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely +## * `asyncdispatch -``-d:asyncBackend=asyncdispatch`` +## * `chronos ` - ``-d:asyncBackend=chronos`` +## +## ``none`` can be used when a library supports both a synchronous and +## asynchronous API, to disable the latter. import os, tables, strutils, times, heapqueue, options, asyncstreams import options, math, std/monotimes From 06ff0e96249884f4ac6c49dadc96147be1c31097 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 15:18:55 +0200 Subject: [PATCH 0372/1654] fixes #18769 (#18790) --- compiler/semgnrc.nim | 8 +++++++- tests/enum/toverloadable_enums.nim | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 3df1e0c3f..e249d88e8 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -107,6 +107,12 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, else: result = n onUse(n.info, s) + of skEnumField: + if overloadableEnums in c.features: + result = symChoice(c, n, s, scOpen) + else: + result = newSymNode(s, n.info) + onUse(n.info, s) else: result = newSymNode(s, n.info) onUse(n.info, s) @@ -407,7 +413,7 @@ proc semGenericStmt(c: PContext, n: PNode, a[^1] = semGenericStmt(c, a[^1], flags, ctx) for j in 0.. Date: Fri, 3 Sep 2021 22:41:00 +0800 Subject: [PATCH 0373/1654] remove unnecessary if statement (#18792) --- compiler/transf.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 80794b581..3250f2ec4 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -434,12 +434,12 @@ proc transformYield(c: PTransf, n: PNode): PNode = else: # we need to introduce new local variables: result.add(introduceNewLocalVars(c, c.transCon.forLoopBody)) - if result.len > 0: - for idx in 0 ..< result.len: - var changeNode = result[idx] - changeNode.info = c.transCon.forStmt.info - for i, child in changeNode: - child.info = changeNode.info + + for idx in 0 ..< result.len: + var changeNode = result[idx] + changeNode.info = c.transCon.forStmt.info + for i, child in changeNode: + child.info = changeNode.info proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PNode = result = transformSons(c, n) From c2b20516d33520b1d339b447ece32ade8625fefc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 17:31:16 +0200 Subject: [PATCH 0374/1654] implemented Unicode operators (#18789) * implemented Unicode operators; refs https://github.com/nim-lang/RFCs/issues/388 * bugfix * better test * arguably more elegant implementation * Update changelog.md Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 7 +++ compiler/lexer.nim | 80 +++++++++++++++++++++++++++--- compiler/options.nim | 3 +- doc/manual.rst | 19 +++++++ tests/lexer/nim.cfg | 1 + tests/lexer/tunicode_operators.nim | 14 ++++++ 6 files changed, 117 insertions(+), 7 deletions(-) create mode 100644 tests/lexer/nim.cfg create mode 100644 tests/lexer/tunicode_operators.nim diff --git a/changelog.md b/changelog.md index 10824cfc6..bc4898c9d 100644 --- a/changelog.md +++ b/changelog.md @@ -445,6 +445,13 @@ proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} To enable the new effect system, use --experimental:strictEffects. +- Nim now supports a small subset of Unicode operators as operator symbols. + The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". + To enable this feature, use `--experimental:unicodeOperators`. Note that due + to parser limitations you **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably. + + ## Compiler changes - Added `--declaredLocs` to show symbol declaration location in messages. diff --git a/compiler/lexer.nim b/compiler/lexer.nim index c7a9aa47f..ddf98661a 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -882,15 +882,66 @@ proc endOperator(L: var Lexer, tok: var Token, pos: int, else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon)) L.bufpos = pos +const + UnicodeOperatorStartChars = {'\226', '\194', '\195'} + # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔") + # all start with one of these. + +type + UnicodeOprPred = enum + Mul, Add + +proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) = + template m(len): untyped = (int8(len), Mul) + template a(len): untyped = (int8(len), Add) + result = 0.m + case buf[pos] + of '\226': + if buf[pos+1] == '\136': + if buf[pos+2] == '\152': result = 3.m # ∘ + elif buf[pos+2] == '\153': result = 3.m # ∙ + elif buf[pos+2] == '\167': result = 3.m # ∧ + elif buf[pos+2] == '\168': result = 3.a # ∨ + elif buf[pos+2] == '\169': result = 3.m # ∩ + elif buf[pos+2] == '\170': result = 3.a # ∪ + elif buf[pos+1] == '\138': + if buf[pos+2] == '\147': result = 3.m # ⊓ + elif buf[pos+2] == '\148': result = 3.a # ⊔ + elif buf[pos+2] == '\149': result = 3.a # ⊕ + elif buf[pos+2] == '\150': result = 3.a # ⊖ + elif buf[pos+2] == '\151': result = 3.m # ⊗ + elif buf[pos+2] == '\152': result = 3.m # ⊘ + elif buf[pos+2] == '\153': result = 3.m # ⊙ + elif buf[pos+2] == '\155': result = 3.m # ⊛ + elif buf[pos+2] == '\158': result = 3.a # ⊞ + elif buf[pos+2] == '\159': result = 3.a # ⊟ + elif buf[pos+2] == '\160': result = 3.m # ⊠ + elif buf[pos+2] == '\161': result = 3.m # ⊡ + elif buf[pos+1] == '\152' and buf[pos+2] == '\133': result = 3.m # ★ + of '\194': + if buf[pos+1] == '\177': result = 2.a # ± + of '\195': + if buf[pos+1] == '\151': result = 2.m # × + else: + discard + proc getOperator(L: var Lexer, tok: var Token) = var pos = L.bufpos tokenBegin(tok, pos) var h: Hash = 0 while true: var c = L.buf[pos] - if c notin OpChars: break - h = h !& ord(c) - inc(pos) + if c in OpChars: + h = h !& ord(c) + inc(pos) + elif c in UnicodeOperatorStartChars and unicodeOperators in L.config.features: + let oprLen = unicodeOprLen(L.buf, pos)[0] + if oprLen == 0: break + for i in 0..', '!': result = 5 of '.': considerAsgn(6) of '?': result = 2 + of UnicodeOperatorStartChars: + if tok.ident.s[^1] == '=': + result = 1 + else: + let (len, pred) = unicodeOprLen(cstring(tok.ident.s), 0) + if len != 0: + result = if pred == Mul: MulPred else: PlusPred + else: + result = 2 else: considerAsgn(2) of tkDiv, tkMod, tkShl, tkShr: result = 9 of tkDotDot: result = 6 @@ -1167,10 +1230,15 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = var c = L.buf[L.bufpos] tok.line = L.lineNumber tok.col = getColNumber(L, L.bufpos) - if c in SymStartChars - {'r', 'R'}: + if c in SymStartChars - {'r', 'R'} - UnicodeOperatorStartChars: getSymbol(L, tok) else: case c + of UnicodeOperatorStartChars: + if unicodeOperators in L.config.features and unicodeOprLen(L.buf, L.bufpos)[0] != 0: + getOperator(L, tok) + else: + getSymbol(L, tok) of '#': scanComment(L, tok) of '*': diff --git a/compiler/options.nim b/compiler/options.nim index 89a16a49c..ea302aed6 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -205,7 +205,8 @@ type views, strictNotNil, overloadableEnums, - strictEffects + strictEffects, + unicodeOperators LegacyFeature* = enum allowSemcheckedAstModification, diff --git a/doc/manual.rst b/doc/manual.rst index 02c688968..997253f16 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -700,6 +700,25 @@ contain a dot: `{..}` are the three tokens `{`:tok:, `..`:tok:, `}`:tok: and not the two tokens `{.`:tok:, `.}`:tok:. +Unicode Operators +----------------- + +Under the `--experimental:unicodeOperators` switch these Unicode operators are +also parsed as operators:: + + ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ # same priority as * (multiplication) + ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ # same priority as + (addition) + + +If enabled, Unicode operators can be combined with non-Unicode operator +symbols. The usual precedence extensions then apply, for example, `⊠=` is an +assignment like operator just like `*=` is. + +No Unicode normalization step is performed. + +**Note**: Due to parser limitations one **cannot** enable this feature via a +pragma `{.experimental: "unicodeOperators".}` reliably. + Syntax ====== diff --git a/tests/lexer/nim.cfg b/tests/lexer/nim.cfg new file mode 100644 index 000000000..f7a301a10 --- /dev/null +++ b/tests/lexer/nim.cfg @@ -0,0 +1 @@ +--experimental:unicodeOperators diff --git a/tests/lexer/tunicode_operators.nim b/tests/lexer/tunicode_operators.nim new file mode 100644 index 000000000..74fcbb763 --- /dev/null +++ b/tests/lexer/tunicode_operators.nim @@ -0,0 +1,14 @@ +#{.experimental: "unicodeOperators".} + +proc `⊙`(x, y: int): int = x * y +proc `⊙=`(x: var int, y: int) = x *= y + +proc `⊞++`(x, y: int): int = x + y + +var x = 45 +x ⊙= 9 ⊞++ 4 ⊙ 3 + +var y = 45 +y *= 9 + 4 * 3 + +assert x == y From cddf8ec6f684e5636a114e0e286bc6609e01f228 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 3 Sep 2021 21:52:24 +0200 Subject: [PATCH 0375/1654] implements https://github.com/nim-lang/RFCs/issues/407 (#18793) --- compiler/semdata.nim | 1 + compiler/semexprs.nim | 16 ++++++++++++---- compiler/semobjconstr.nim | 31 ++++++++++++++++-------------- compiler/sempass2.nim | 2 ++ compiler/semstmts.nim | 14 ++++++++++++++ compiler/sigmatch.nim | 14 ++++++++++++-- compiler/wordrecg.nim | 1 + doc/manual.rst | 35 ++++++++++++++++++++++++++++++++++ tests/objvariant/treassign.nim | 9 +++++++++ 9 files changed, 103 insertions(+), 20 deletions(-) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 31affb24f..422d1223a 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -160,6 +160,7 @@ type importModuleMap*: Table[int, int] # (module.id, module.id) lastTLineInfo*: TLineInfo sideEffects*: Table[int, seq[(TLineInfo, PSym)]] # symbol.id index + inUncheckedAssignSection*: int template config*(c: PContext): ConfigRef = c.graph.config diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f2fe54fad..3edeae324 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -691,8 +691,12 @@ proc newHiddenAddrTaken(c: PContext, n: PNode): PNode = else: result = newNodeIT(nkHiddenAddr, n.info, makeVarType(c, n.typ)) result.add n - if isAssignable(c, n) notin {arLValue, arLocalLValue}: - localError(c.config, n.info, errVarForOutParamNeededX % renderNotLValue(n)) + let aa = isAssignable(c, n) + if aa notin {arLValue, arLocalLValue}: + if aa == arDiscriminant and c.inUncheckedAssignSection > 0: + discard "allow access within a cast(unsafeAssign) section" + else: + localError(c.config, n.info, errVarForOutParamNeededX % renderNotLValue(n)) proc analyseIfAddressTaken(c: PContext, n: PNode): PNode = result = n @@ -738,9 +742,13 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) = if i < t.len and t[i] != nil and skipTypes(t[i], abstractInst-{tyTypeDesc}).kind in {tyVar}: let it = n[i] - if isAssignable(c, it) notin {arLValue, arLocalLValue}: + let aa = isAssignable(c, it) + if aa notin {arLValue, arLocalLValue}: if it.kind != nkHiddenAddr: - localError(c.config, it.info, errVarForOutParamNeededX % $it) + if aa == arDiscriminant and c.inUncheckedAssignSection > 0: + discard "allow access within a cast(unsafeAssign) section" + else: + localError(c.config, it.info, errVarForOutParamNeededX % $it) # bug #5113: disallow newSeq(result) where result is a 'var T': if n[0].sym.magic in {mNew, mNewFinalize, mNewSeq}: var arg = n[1] #.skipAddr diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 1c97d32c8..1563e5c16 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -200,20 +200,22 @@ proc semConstructFields(c: PContext, n: PNode, if selectedBranch != -1: template badDiscriminatorError = - let fields = fieldsPresentInBranch(selectedBranch) - localError(c.config, constrCtx.initExpr.info, - ("cannot prove that it's safe to initialize $1 with " & - "the runtime value for the discriminator '$2' ") % - [fields, discriminator.sym.name.s]) + if c.inUncheckedAssignSection == 0: + let fields = fieldsPresentInBranch(selectedBranch) + localError(c.config, constrCtx.initExpr.info, + ("cannot prove that it's safe to initialize $1 with " & + "the runtime value for the discriminator '$2' ") % + [fields, discriminator.sym.name.s]) mergeInitStatus(result, initNone) template wrongBranchError(i) = - let fields = fieldsPresentInBranch(i) - localError(c.config, constrCtx.initExpr.info, - "a case selecting discriminator '$1' with value '$2' " & - "appears in the object construction, but the field(s) $3 " & - "are in conflict with this value." % - [discriminator.sym.name.s, discriminatorVal.renderTree, fields]) + if c.inUncheckedAssignSection == 0: + let fields = fieldsPresentInBranch(i) + localError(c.config, constrCtx.initExpr.info, + ("a case selecting discriminator '$1' with value '$2' " & + "appears in the object construction, but the field(s) $3 " & + "are in conflict with this value.") % + [discriminator.sym.name.s, discriminatorVal.renderTree, fields]) template valuesInConflictError(valsDiff) = localError(c.config, discriminatorVal.info, ("possible values " & @@ -251,9 +253,10 @@ proc semConstructFields(c: PContext, n: PNode, badDiscriminatorError() elif discriminatorVal.sym.kind notin {skLet, skParam} or discriminatorVal.sym.typ.kind in {tyVar}: - localError(c.config, discriminatorVal.info, - "runtime discriminator must be immutable if branch fields are " & - "initialized, a 'let' binding is required.") + if c.inUncheckedAssignSection == 0: + localError(c.config, discriminatorVal.info, + "runtime discriminator must be immutable if branch fields are " & + "initialized, a 'let' binding is required.") elif ctorCase[ctorIdx].kind == nkElifBranch: localError(c.config, discriminatorVal.info, "branch initialization " & "with a runtime discriminator is not supported inside of an " & diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index cae2f926c..2df6327b8 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -961,6 +961,8 @@ proc castBlock(tracked: PEffects, pragma: PNode, bc: var PragmaBlockContext) = else: bc.exc = newNodeI(nkArgList, pragma.info) bc.exc.add n + of wUncheckedAssign: + discard "handled in sempass1" else: localError(tracked.config, pragma.info, "invalid pragma block: " & $pragma) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index df0e2778b..2c5f3ba54 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2223,7 +2223,21 @@ proc semPragmaBlock(c: PContext, n: PNode): PNode = checkSonsLen(n, 2, c.config) let pragmaList = n[0] pragma(c, nil, pragmaList, exprPragmas, isStatement = true) + + var inUncheckedAssignSection = 0 + for p in pragmaList: + if whichPragma(p) == wCast: + case whichPragma(p[1]) + of wGcSafe, wNoSideEffect, wTags, wRaises: + discard "handled in sempass2" + of wUncheckedAssign: + inUncheckedAssignSection = 1 + else: + localError(c.config, p.info, "invalid pragma block: " & $p) + + inc c.inUncheckedAssignSection, inUncheckedAssignSection n[1] = semExpr(c, n[1]) + dec c.inUncheckedAssignSection, inUncheckedAssignSection result = n result.typ = n[1].typ for i in 0.. 0 + else: + result = false + proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType, arg: PNode): PNode = result = nil @@ -1895,7 +1905,7 @@ proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType, let constraint = c.converters[i].typ.n[1].sym.constraint if not constraint.isNil and not matchNodeKinds(constraint, arg): continue - if src.kind in {tyVar, tyLent} and not arg.isLValue: + if src.kind in {tyVar, tyLent} and not isLValue(c, arg): continue let destIsGeneric = containsGenericType(dest) @@ -2338,7 +2348,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int if argConverter.typ.kind notin {tyVar}: m.firstMismatch.kind = kVarNeeded noMatch() - elif not n.isLValue: + elif not isLValue(c, n): m.firstMismatch.kind = kVarNeeded noMatch() diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 6ff68be75..4ba807518 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -36,6 +36,7 @@ type wMemTracker = "memtracker", wObjChecks = "objchecks", wIntDefine = "intdefine", wStrDefine = "strdefine", wBoolDefine = "booldefine", wCursor = "cursor", wNoalias = "noalias", wEffectsOf = "effectsOf", + wUncheckedAssign = "uncheckedAssign", wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor", wDelegator = "delegator", wOverride = "override", wImportCpp = "importcpp", diff --git a/doc/manual.rst b/doc/manual.rst index 997253f16..714f98bdc 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1854,6 +1854,41 @@ A small example: let unknownKindBounded = range[nkAdd..nkSub](unknownKind) z = Node(kind: unknownKindBounded, leftOp: Node(), rightOp: Node()) + +cast uncheckedAssign +-------------------- + +Some restrictions for case objects can be disabled via a `{.cast(unsafeAssign).}` section: + +.. code-block:: nim + :test: "nim c $1" + + type + TokenKind* = enum + strLit, intLit + Token = object + case kind*: TokenKind + of strLit: + s*: string + of intLit: + i*: int64 + + proc passToVar(x: var TokenKind) = discard + + var t = Token(kind: strLit, s: "abc") + + {.cast(uncheckedAssign).}: + # inside the 'cast' section it is allowed to pass 't.kind' to a 'var T' parameter: + passToVar(t.kind) + + # inside the 'cast' section it is allowed to set field 's' even though the + # constructed 'kind' field has an unknown value: + t = Token(kind: t.kind, s: "abc") + + # inside the 'cast' section it is allowed to assign to the 't.kind' field directly: + t.kind = intLit + + Set type -------- diff --git a/tests/objvariant/treassign.nim b/tests/objvariant/treassign.nim index 2938b30a3..9549cb29c 100644 --- a/tests/objvariant/treassign.nim +++ b/tests/objvariant/treassign.nim @@ -25,3 +25,12 @@ t.curr = TokenObject(kind: Token.bar, bar: BasicNumber(value: 12.34)) t.curr = TokenObject(kind: Token.foo, foo: "foo") echo "SUCCESS" + +proc passToVar(x: var Token) = discard + +{.cast(uncheckedAssign).}: + passToVar(t.curr.kind) + + t.curr = TokenObject(kind: t.curr.kind, foo: "abc") + + t.curr.kind = Token.foo From ac7acd827cd05ea916c53513e68d13be0d63a604 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 08:18:00 +0200 Subject: [PATCH 0376/1654] =?UTF-8?q?we=20need=20something=20better=20than?= =?UTF-8?q?=20warningAsError=20for=20effect=20handling=20viol=E2=80=A6=20(?= =?UTF-8?q?#18796)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * we need something better than warningAsError for effect handling violations --- compiler/sempass2.nim | 4 ++-- lib/system.nim | 3 +-- nimdoc/test_out_index_dot_html/expected/index.html | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 2df6327b8..6a0e8772a 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1420,7 +1420,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let p = s.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): - checkRaisesSpec(g, emitWarnings, raisesSpec, t.exc, "can raise an unlisted exception: ", + checkRaisesSpec(g, false, raisesSpec, t.exc, "can raise an unlisted exception: ", hints=on, subtypeRelation, hintsArg=s.ast[0]) # after the check, use the formal spec: effects[exceptionEffects] = raisesSpec @@ -1429,7 +1429,7 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = let tagsSpec = effectSpec(p, wTags) if not isNil(tagsSpec): - checkRaisesSpec(g, emitWarnings, tagsSpec, t.tags, "can have an unlisted effect: ", + checkRaisesSpec(g, false, tagsSpec, t.tags, "can have an unlisted effect: ", hints=off, subtypeRelation) # after the check, use the formal spec: effects[tagEffects] = tagsSpec diff --git a/lib/system.nim b/lib/system.nim index 740c4419f..c93af0854 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2004,8 +2004,7 @@ elif hasAlloc: inc(i) {.pop.} -proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", tags: [WriteIOEffect], - benign, sideEffect.} +proc echo*(x: varargs[typed, `$`]) {.magic: "Echo", benign, sideEffect.} ## Writes and flushes the parameters to the standard output. ## ## Special built-in that takes a variable number of arguments. Each argument diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 458a0394f..3499b5326 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -122,7 +122,7 @@ window.addEventListener('DOMContentLoaded', main);

    Procs

    -
    proc foo() {....raises: [], tags: [WriteIOEffect].}
    +
    proc foo() {....raises: [], tags: [].}
    I do foo From 686096a912107e620142e78ef98a380bb8a4764f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 12:52:24 +0200 Subject: [PATCH 0377/1654] documented overloadable enums and changelog improvements (#18797) --- changelog.md | 51 ++++++++++++++++++++++++++++---------------------- doc/manual.rst | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 22 deletions(-) diff --git a/changelog.md b/changelog.md index bc4898c9d..d553864b6 100644 --- a/changelog.md +++ b/changelog.md @@ -114,16 +114,18 @@ ## Standard library additions and changes - `strformat`: - added support for parenthesized expressions. - added support for const string's instead of just string literals + - added support for parenthesized expressions. + - added support for const string's instead of just string literals +- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; + use `-d:nimLegacyCopyFile` for OSX < 10.5. - `system.addFloat` and `system.$` now can produce string representations of floating point numbers that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has to be enabled via `-d:nimPreviewFloatRoundtrip`. It is expected that this behavior becomes the new default in upcoming versions, as with other `nimPreviewX` define flags. -- Fixed buffer overflow bugs in `net` +- Fixed buffer overflow bugs in `net`. - Exported `sslHandle` from `net` and `asyncnet`. @@ -187,10 +189,11 @@ - Added `std/sysrand` module to get random numbers from a secure source provided by the operating system. -- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates case statement to parse string to enum. - Added `items` for enums with holes. - Added `symbolName` to return the enum symbol name ignoring the human readable name. - Added `symbolRank` to return the index in which an enum member is listed in an enum. +- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates + case statement to parse string to enum. +- Added `items` for enums with holes. +- Added `symbolName` to return the enum symbol name ignoring the human readable name. +- Added `symbolRank` to return the index in which an enum member is listed in an enum. - Removed deprecated `iup` module from stdlib, it has already moved to [nimble](https://github.com/nim-lang/iup). @@ -381,15 +384,10 @@ ## Language changes -- `nimscript` now handles `except Exception as e`. - - The `cstring` doesn't support `[]=` operator in JS backend. - nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. -- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; - use `-d:nimLegacyCopyFile` for OSX < 10.5. - - The required name of case statement macros for the experimental `caseStmtMacros` feature has changed from `match` to `` `case` ``. @@ -403,18 +401,11 @@ - Tuple expressions are now parsed consistently as `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. -- `nim e` now accepts arbitrary file extensions for the nimscript file, - although `.nims` is still the preferred extension in general. - - Added `iterable[T]` type class to match called iterators, which enables writing: `template fn(a: iterable)` instead of `template fn(a: untyped)` - A new import syntax `import foo {.all.}` now allows to import all symbols (public or private) - from `foo`. It works in combination with all pre-existing import features. - This reduces or eliminates the need for workarounds such as using `include` (which has known issues) - when you need a private symbol for testing or making some internal APIs public just because - another internal module needs those. - It also helps mitigate the lack of cyclic imports in some cases. + from `foo`. This can be useful for testing purposes. - Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields for an object type in the current scope. @@ -422,7 +413,7 @@ - `typeof(voidStmt)` now works and returns `void`. - The `gc:orc` algorithm was refined so that custom container types can participate in the - cycle collection process. + cycle collection process. See the documentation of `=trace` for more details. - On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. This is only supported for `--gc:orc` or `--gc:arc`. @@ -449,7 +440,18 @@ proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". To enable this feature, use `--experimental:unicodeOperators`. Note that due to parser limitations you **cannot** enable this feature via a - pragma `{.experimental: "unicodeOperators".}` reliably. + pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable + it via the command line or in a configuration file. + +- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some + compiler checks regarding `case objects`. This allows serialization libraries + to avoid ugly, non-portable solutions. See https://github.com/nim-lang/RFCs/issues/407 + for more details. + +- Enum values can now be overloaded. This needs to be enabled + via `{.experimental: "overloadableEnums".}`. We hope that this feature allows for the + development of more fluent (less ugly) APIs. See https://github.com/nim-lang/RFCs/issues/373 + for more details. ## Compiler changes @@ -529,6 +531,11 @@ proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} ## Tool changes +- `nimscript` now handles `except Exception as e`. + +- `nim e` now accepts arbitrary file extensions for the nimscript file, + although `.nims` is still the preferred extension in general. + - Latex doc generation is revised: output `.tex` files should be compiled by `xelatex` (not by `pdflatex` as before). Now default Latex settings provide support for Unicode and do better job for avoiding margin overflows. diff --git a/doc/manual.rst b/doc/manual.rst index 714f98bdc..e9c730c7b 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1323,6 +1323,47 @@ as `MyEnum.value`: To implement bit fields with enums see `Bit fields <#set-type-bit-fields>`_ +Overloadable enum field names +----------------------------- + +To be enabled via `{.experimental: "overloadableEnums".}`. + +Enum field names are overloadable much like routines. When an overloaded +enum field is used, it produces a closed sym choice construct, here +written as `(E|E)`. +During overload resolution the right `E` is picked, if possible. +For (array/object...) constructors the right `E` is picked, comparable to +how `[byte(1), 2, 3]` works, one needs to use `[T.E, E2, E3]`. Ambiguous +enum fields produce a static error: + +.. code-block:: nim + :test: "nim c $1" + + {.experimental: "overloadableEnums".} + + type + E1 = enum + value1, + value2 + E2 = enum + value1, + value2 = 4 + + const + Lookuptable = [ + E1.value1: "1", + value2: "2" + ] + + proc p(e: E1) = + # disambiguation in 'case' statements: + case e + of value1: echo "A" + of value2: echo "B" + + p value2 + + String type ----------- All string literals are of the type `string`. A string in Nim is very From 3c2edd142b527c34163cbd7ea9ad94518f46e9ae Mon Sep 17 00:00:00 2001 From: Sven Keller Date: Sat, 4 Sep 2021 12:53:02 +0200 Subject: [PATCH 0378/1654] fix for js strict mode (#18799) * Assignments, which would accidentally create global variables, instead throw an error in strict mode * Assignment to a getter-only property Co-authored-by: Sven Keller --- compiler/jsgen.nim | 2 +- lib/system/jssys.nim | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 931e56475..f009c83f2 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1769,7 +1769,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = inc p.extraIndent elif useGlobalPragmas: lineF(p, "if (globalThis.$1 === undefined) {$n", varName) - varCode = $varName + varCode = "globalThis." & $varName inc p.extraIndent else: varCode = "var $2" diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 63e7b3d9d..cd4f378be 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -593,12 +593,9 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = `result` = null; } else { - if (`dest` === null || `dest` === undefined) { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { `dest` = new Array(`src`.length); } - else { - `dest`.length = `src`.length; - } `result` = `dest`; for (var i = 0; i < `src`.length; ++i) { `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); From b3ad68edea72f04275c9b1e330cafb77254d945c Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 4 Sep 2021 21:57:02 +0800 Subject: [PATCH 0379/1654] remove channels (#18801) * remove channels * test --- changelog.md | 2 - doc/manual.rst | 2 +- lib/std/channels.nim | 498 ----------------------------- tests/stdlib/tchannels.nim | 33 -- tests/stdlib/tchannels_pthread.nim | 322 ------------------- tests/stdlib/tchannels_simple.nim | 67 ---- 6 files changed, 1 insertion(+), 923 deletions(-) delete mode 100644 lib/std/channels.nim delete mode 100644 tests/stdlib/tchannels.nim delete mode 100644 tests/stdlib/tchannels_pthread.nim delete mode 100644 tests/stdlib/tchannels_simple.nim diff --git a/changelog.md b/changelog.md index d553864b6..3658fb105 100644 --- a/changelog.md +++ b/changelog.md @@ -341,8 +341,6 @@ - Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). -- Added `std/channels`. - - Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). - `std/times`: diff --git a/doc/manual.rst b/doc/manual.rst index e9c730c7b..7976ffe0f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -8016,7 +8016,7 @@ Threads To enable thread support the `--threads:on`:option: command-line switch needs to be used. The system_ module then contains several threading primitives. -See the `threads `_ and `channels `_ modules +See the `channels `_ modules for the low-level thread API. There are also high-level parallelism constructs available. See `spawn `_ for further details. diff --git a/lib/std/channels.nim b/lib/std/channels.nim deleted file mode 100644 index 5ff141384..000000000 --- a/lib/std/channels.nim +++ /dev/null @@ -1,498 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Prell, Mamy André-Ratsimbazafy & Nim Contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# Based on https://github.com/mratsim/weave/blob/5696d94e6358711e840f8c0b7c684fcc5cbd4472/unused/channels/channels_legacy.nim -# Those are translations of @aprell (Andreas Prell) original channels from C to Nim -# (https://github.com/aprell/tasking-2.0/blob/master/src/channel_shm/channel.c) -# And in turn they are an implementation of Michael & Scott lock-based queues -# (note the paper has 2 channels: lock-free and lock-based) with additional caching: -# Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms -# Maged M. Michael, Michael L. Scott, 1996 -# https://www.cs.rochester.edu/~scott/papers/1996_PODC_queues.pdf - -## This module only works with `--gc:arc` or `--gc:orc`. -## -## .. warning:: This module is experimental and its interface may change. -## -## The following is a simple example of two different ways to use channels: -## blocking and non-blocking. -## - -runnableExamples("--threads:on --gc:orc"): - import std/os - - # In this example a channel is declared at module scope. - # Channels are generic, and they include support for passing objects between - # threads. - # Note that isolated data passed through channels is moved around. - var chan = newChannel[string]() - - # This proc will be run in another thread using the threads module. - proc firstWorker() = - chan.send("Hello World!") - - # This is another proc to run in a background thread. This proc takes a while - # to send the message since it sleeps for 2 seconds (or 2000 milliseconds). - proc secondWorker() = - sleep(2000) - chan.send("Another message") - - # Launch the worker. - var worker1: Thread[void] - createThread(worker1, firstWorker) - - # Block until the message arrives, then print it out. - let dest = chan.recv() - assert dest == "Hello World!" - - # Wait for the thread to exit before moving on to the next example. - worker1.joinThread() - - # Launch the other worker. - var worker2: Thread[void] - createThread(worker2, secondWorker) - # This time, use a non-blocking approach with tryRecv. - # Since the main thread is not blocked, it could be used to perform other - # useful work while it waits for data to arrive on the channel. - var messages: seq[string] - while true: - var msg = "" - if chan.tryRecv(msg): - messages.add msg # "Another message" - break - - messages.add "Pretend I'm doing useful work..." - # For this example, sleep in order not to flood stdout with the above - # message. - sleep(400) - - # Wait for the second thread to exit before cleaning up the channel. - worker2.joinThread() - - # Clean up the channel. - assert chan.close() - - assert messages[^1] == "Another message" - assert messages.len >= 2 - - -when not defined(gcArc) and not defined(gcOrc) and not defined(nimdoc): - {.error: "This channel implementation requires --gc:arc or --gc:orc".} - -import std/[locks, atomics, isolation] -import system/ansi_c - -# Channel (Shared memory channels) -# ---------------------------------------------------------------------------------- - -const - cacheLineSize {.intdefine.} = 64 # TODO: some Samsung phone have 128 cache-line - nimChannelCacheSize* {.intdefine.} = 100 - -type - ChannelRaw = ptr ChannelObj - ChannelObj = object - headLock, tailLock: Lock - notFullCond, notEmptyCond: Cond - closed: Atomic[bool] - size: int - itemsize: int # up to itemsize bytes can be exchanged over this channel - head {.align: cacheLineSize.} : int # Items are taken from head and new items are inserted at tail - tail: int - buffer: ptr UncheckedArray[byte] - atomicCounter: Atomic[int] - - ChannelCache = ptr ChannelCacheObj - ChannelCacheObj = object - next: ChannelCache - chanSize: int - chanN: int - numCached: int - cache: array[nimChannelCacheSize, ChannelRaw] - -# ---------------------------------------------------------------------------------- - -proc numItems(chan: ChannelRaw): int {.inline.} = - result = chan.tail - chan.head - if result < 0: - inc(result, 2 * chan.size) - - assert result <= chan.size - -template isFull(chan: ChannelRaw): bool = - abs(chan.tail - chan.head) == chan.size - -template isEmpty(chan: ChannelRaw): bool = - chan.head == chan.tail - -# Unbuffered / synchronous channels -# ---------------------------------------------------------------------------------- - -template numItemsUnbuf(chan: ChannelRaw): int = - chan.head - -template isFullUnbuf(chan: ChannelRaw): bool = - chan.head == 1 - -template isEmptyUnbuf(chan: ChannelRaw): bool = - chan.head == 0 - -# ChannelRaw kinds -# ---------------------------------------------------------------------------------- - -func isUnbuffered(chan: ChannelRaw): bool = - chan.size - 1 == 0 - -# ChannelRaw status and properties -# ---------------------------------------------------------------------------------- - -proc isClosed(chan: ChannelRaw): bool {.inline.} = load(chan.closed, moRelaxed) - -proc peek(chan: ChannelRaw): int {.inline.} = - (if chan.isUnbuffered: numItemsUnbuf(chan) else: numItems(chan)) - -# Per-thread channel cache -# ---------------------------------------------------------------------------------- - -var channelCache {.threadvar.}: ChannelCache -var channelCacheLen {.threadvar.}: int - -proc allocChannelCache(size, n: int): bool = - ## Allocate a free list for storing channels of a given type - var p = channelCache - - # Avoid multiple free lists for the exact same type of channel - while not p.isNil: - if size == p.chanSize and n == p.chanN: - return false - p = p.next - - p = cast[ptr ChannelCacheObj](c_malloc(csize_t sizeof(ChannelCacheObj))) - if p.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - p.chanSize = size - p.chanN = n - p.numCached = 0 - - p.next = channelCache - channelCache = p - inc channelCacheLen - result = true - -proc freeChannelCache*() = - ## Frees the entire channel cache, including all channels - var p = channelCache - var q: ChannelCache - - while not p.isNil: - q = p.next - for i in 0 ..< p.numCached: - let chan = p.cache[i] - if not chan.buffer.isNil: - c_free(chan.buffer) - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - c_free(chan) - c_free(p) - dec channelCacheLen - p = q - - assert(channelCacheLen == 0) - channelCache = nil - -# Channels memory ops -# ---------------------------------------------------------------------------------- - -proc allocChannel(size, n: int): ChannelRaw = - when nimChannelCacheSize > 0: - var p = channelCache - - while not p.isNil: - if size == p.chanSize and n == p.chanN: - # Check if free list contains channel - if p.numCached > 0: - dec p.numCached - result = p.cache[p.numCached] - assert(result.isEmpty) - return - else: - # All the other lists in cache won't match - break - p = p.next - - result = cast[ChannelRaw](c_malloc(csize_t sizeof(ChannelObj))) - if result.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - # To buffer n items, we allocate for n - result.buffer = cast[ptr UncheckedArray[byte]](c_malloc(csize_t n*size)) - if result.buffer.isNil: - raise newException(OutOfMemDefect, "Could not allocate memory") - - initLock(result.headLock) - initLock(result.tailLock) - initCond(result.notFullCond) - initCond(result.notEmptyCond) - - result.closed.store(false, moRelaxed) # We don't need atomic here, how to? - result.size = n - result.itemsize = size - result.head = 0 - result.tail = 0 - result.atomicCounter.store(0, moRelaxed) - - when nimChannelCacheSize > 0: - # Allocate a cache as well if one of the proper size doesn't exist - discard allocChannelCache(size, n) - -proc freeChannel(chan: ChannelRaw) = - if chan.isNil: - return - - when nimChannelCacheSize > 0: - var p = channelCache - while not p.isNil: - if chan.itemsize == p.chanSize and - chan.size == p.chanN: - if p.numCached < nimChannelCacheSize: - # If space left in cache, cache it - p.cache[p.numCached] = chan - inc p.numCached - return - else: - # All the other lists in cache won't match - break - p = p.next - - if not chan.buffer.isNil: - c_free(chan.buffer) - - deinitLock(chan.headLock) - deinitLock(chan.tailLock) - deinitCond(chan.notFullCond) - deinitCond(chan.notEmptyCond) - - c_free(chan) - -# MPMC Channels (Multi-Producer Multi-Consumer) -# ---------------------------------------------------------------------------------- - -proc sendUnbufferedMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isFullUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isFullUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isFullUnbuf: - wait(chan.notFullcond, chan.headLock) - - assert chan.isEmptyUnbuf - assert size <= chan.itemsize - copyMem(chan.buffer, data, size) - - chan.head = 1 - - signal(chan.notEmptyCond) - release(chan.headLock) - result = true - -proc sendMpmc(chan: ChannelRaw, data: sink pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return sendUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isFull: - return false - - acquire(chan.tailLock) - - if nonBlocking and chan.isFull: - # Another thread was faster - release(chan.tailLock) - return false - - while chan.isFull: - wait(chan.notFullcond, chan.tailLock) - - assert not chan.isFull - assert size <= chan.itemsize - - let writeIdx = if chan.tail < chan.size: chan.tail - else: chan.tail - chan.size - - copyMem(chan.buffer[writeIdx * chan.itemsize].addr, data, size) - - inc chan.tail - if chan.tail == 2 * chan.size: - chan.tail = 0 - - signal(chan.notEmptyCond) - release(chan.tailLock) - result = true - -proc recvUnbufferedMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - if nonBlocking and chan.isEmptyUnbuf: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmptyUnbuf: - # Another thread was faster - release(chan.headLock) - return false - - while chan.isEmptyUnbuf: - wait(chan.notEmptyCond, chan.headLock) - - assert chan.isFullUnbuf - assert size <= chan.itemsize - - copyMem(data, chan.buffer, size) - - chan.head = 0 - - signal(chan.notFullCond) - release(chan.headLock) - result = true - -proc recvMpmc(chan: ChannelRaw, data: pointer, size: int, nonBlocking: bool): bool = - assert not chan.isNil - assert not data.isNil - - if isUnbuffered(chan): - return recvUnbufferedMpmc(chan, data, size, nonBlocking) - - if nonBlocking and chan.isEmpty: - return false - - acquire(chan.headLock) - - if nonBlocking and chan.isEmpty: - # Another thread took the last data - release(chan.headLock) - return false - - while chan.isEmpty: - wait(chan.notEmptyCond, chan.headLock) - - assert not chan.isEmpty - assert size <= chan.itemsize - - let readIdx = if chan.head < chan.size: chan.head - else: chan.head - chan.size - - copyMem(data, chan.buffer[readIdx * chan.itemsize].addr, size) - - inc chan.head - if chan.head == 2 * chan.size: - chan.head = 0 - - signal(chan.notFullCond) - release(chan.headLock) - result = true - -proc channelCloseMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if chan.isClosed: - # ChannelRaw already closed - return false - - store(chan.closed, true, moRelaxed) - result = true - -proc channelOpenMpmc(chan: ChannelRaw): bool = - # Unsynchronized - - if not chan.isClosed: - # ChannelRaw already open - return false - - store(chan.closed, false, moRelaxed) - result = true - -# Public API -# ---------------------------------------------------------------------------------- - -type - Channel*[T] = object ## Typed channels - d: ChannelRaw - -proc `=destroy`*[T](c: var Channel[T]) = - if c.d != nil: - if load(c.d.atomicCounter, moAcquire) == 0: - if c.d.buffer != nil: - freeChannel(c.d) - else: - atomicDec(c.d.atomicCounter) - -proc `=copy`*[T](dest: var Channel[T], src: Channel[T]) = - ## Shares `Channel` by reference counting. - if src.d != nil: - atomicInc(src.d.atomicCounter) - - if dest.d != nil: - `=destroy`(dest) - dest.d = src.d - -func trySend*[T](c: Channel[T], src: var Isolated[T]): bool {.inline.} = - ## Sends item to the channel(non blocking). - var data = src.extract - result = sendMpmc(c.d, data.addr, sizeof(T), true) - if result: - wasMoved(data) - -template trySend*[T](c: Channel[T], src: T): bool = - ## Helper templates for `trySend`. - trySend(c, isolate(src)) - -func tryRecv*[T](c: Channel[T], dst: var T): bool {.inline.} = - ## Receives item from the channel(non blocking). - recvMpmc(c.d, dst.addr, sizeof(T), true) - -func send*[T](c: Channel[T], src: sink Isolated[T]) {.inline.} = - ## Sends item to the channel(blocking). - var data = src.extract - discard sendMpmc(c.d, data.addr, sizeof(T), false) - wasMoved(data) - -template send*[T](c: Channel[T]; src: T) = - ## Helper templates for `send`. - send(c, isolate(src)) - -func recv*[T](c: Channel[T]): T {.inline.} = - ## Receives item from the channel(blocking). - discard recvMpmc(c.d, result.addr, sizeof(result), false) - -func open*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelOpenMpmc() - -func close*[T](c: Channel[T]): bool {.inline.} = - result = c.d.channelCloseMpmc() - -func peek*[T](c: Channel[T]): int {.inline.} = peek(c.d) - -proc newChannel*[T](elements = 30): Channel[T] = - ## Returns a new `Channel`. `elements` should be positive. - ## `elements` is used to specify whether a channel is buffered or not. - ## If `elements` = 1, the channel is unbuffered. If `elements` > 1, the - ## channel is buffered. - assert elements >= 1, "Elements must be positive!" - result = Channel[T](d: allocChannel(sizeof(T), elements)) diff --git a/tests/stdlib/tchannels.nim b/tests/stdlib/tchannels.nim deleted file mode 100644 index 492faf500..000000000 --- a/tests/stdlib/tchannels.nim +++ /dev/null @@ -1,33 +0,0 @@ -discard """ - timeout: 60.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime) - disabled: "freebsd" - matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger" -""" - -when true: - # bug #17380: this was either blocking (without -d:danger) or crashing with SIGSEGV (with -d:danger) - import std/[channels, isolation] - const - N1 = 10 - N2 = 100 - var - sender: array[N1, Thread[void]] - receiver: array[5, Thread[void]] - - var chan = newChannel[seq[string]](N1 * N2) # large enough to not block - proc sendHandler() = - chan.send(isolate(@["Hello, Nim"])) - proc recvHandler() = - template fn = - let x = chan.recv() - fn() - - template benchmark() = - for t in mitems(sender): - t.createThread(sendHandler) - joinThreads(sender) - for t in mitems(receiver): - t.createThread(recvHandler) - joinThreads(receiver) - for i in 0.. 0 - -when defined(blockingTest): - const nonBlocking = false -else: - const nonBlocking = true - -type - Pthread {.importc: "pthread_t", header: "".} = distinct culong - PthreadAttr* {.byref, importc: "pthread_attr_t", header: "".} = object - Errno* = distinct cint - -proc pthread_create[T]( - thread: var Pthread, - attr: ptr PthreadAttr, # In Nim this is a var and how Nim sets a custom stack - fn: proc (x: ptr T): pointer {.thread, noconv.}, - arg: ptr T - ): Errno {.header: "".} - -proc pthread_join( - thread: Pthread, - thread_exit_status: ptr pointer - ): Errno {.header: "".} - -template channel_send_loop(chan: ChannelRaw, - data: sink pointer, - size: int, - body: untyped): untyped = - while not sendMpmc(chan, data, size, nonBlocking): - body - -template channel_receive_loop(chan: ChannelRaw, - data: pointer, - size: int, - body: untyped): untyped = - while not recvMpmc(chan, data, size, nonBlocking): - body - - -# Without threads:on or release, -# worker threads will crash on popFrame - -import std/unittest - -type ThreadArgs = object - ID: int - chan: ChannelRaw - -template Worker(id: int, body: untyped): untyped {.dirty.} = - if args.ID == id: - body - - -const Sender = 1 -const Receiver = 0 - -proc runSuite( - name: string, - fn: proc(args: ptr ThreadArgs): pointer {.noconv, gcsafe.} - ) = - var chan: ChannelRaw - - for i in Unbuffered .. Buffered: - if i == Unbuffered: - chan = allocChannel(size = 32, n = 1) - check: - peek(chan) == 0 - capacity(chan) == 1 - isBuffered(chan) == false - isUnbuffered(chan) == true - - else: - chan = allocChannel(size = int.sizeof.int, n = 7) - check: - peek(chan) == 0 - capacity(chan) == 7 - isBuffered(chan) == true - isUnbuffered(chan) == false - - var threads: array[2, Pthread] - var args = [ - ThreadArgs(ID: 0, chan: chan), - ThreadArgs(ID: 1, chan: chan) - ] - - discard pthread_create(threads[0], nil, fn, args[0].addr) - discard pthread_create(threads[1], nil, fn, args[1].addr) - - discard pthread_join(threads[0], nil) - discard pthread_join(threads[1], nil) - - freeChannel(chan) - -# ---------------------------------------------------------------------------------- - -proc thread_func(args: ptr ThreadArgs): pointer {.noconv.} = - - # Worker RECEIVER: - # --------- - # <- chan - # <- chan - # <- chan - # - # Worker SENDER: - # --------- - # chan <- 42 - # chan <- 53 - # chan <- 64 - # - - Worker(Receiver): - var val: int - for j in 0 ..< 3: - channel_receive_loop(args.chan, val.addr, val.sizeof.int): - # Busy loop, normally it should yield - discard - check: val == 42 + j*11 - - Worker(Sender): - var val: int - check: peek(args.chan) == 0 - for j in 0 ..< 3: - val = 42 + j*11 - channel_send_loop(args.chan, val.addr, val.sizeof.int): - # Busy loop, normally it should yield - discard - - return nil - -runSuite("[ChannelRaw] 2 threads can send data", thread_func) - -# ---------------------------------------------------------------------------------- - -iterator pairs(chan: ChannelRaw, T: typedesc): (int, T) = - var i = 0 - var x: T - while not isClosed(chan) or peek(chan) > 0: - let r = recvMpmc(chan, x.addr, x.sizeof.int, true) - # printf("x: %d, r: %d\n", x, r) - if r: - yield (i, x) - inc i - -proc thread_func_2(args: ptr ThreadArgs): pointer {.noconv.} = - # Worker RECEIVER: - # --------- - # <- chan until closed and empty - # - # Worker SENDER: - # --------- - # chan <- 42, 53, 64, ... - - const N = 100 - - Worker(Receiver): - for j, val in pairs(args.chan, int): - # TODO: Need special handling that doesn't allocate - # in thread with no GC - # when check fails - # - check: val == 42 + j*11 - - Worker(Sender): - var val: int - check: peek(args.chan) == 0 - for j in 0 ..< N: - val = 42 + j*11 - channel_send_loop(args.chan, val.addr, int.sizeof.int): - discard - discard channelCloseMpmc(args.chan) - - return nil - -runSuite("[ChannelRaw] channel_close, freeChannel, channelCache", thread_func_2) - -# ---------------------------------------------------------------------------------- - -proc isCached(chan: ChannelRaw): bool = - assert not chan.isNil - - var p = channelCache - while not p.isNil: - if chan.itemsize == p.chanSize and - chan.size == p.chanN: - for i in 0 ..< p.numCached: - if chan == p.cache[i]: - return true - # No more channel in cache can match - return false - p = p.next - return false - -block: # [ChannelRaw] ChannelRaw caching implementation - - # Start from clean cache slate - freeChannelCache() - - block: # Explicit caches allocation - check: - allocChannelCache(int sizeof(char), 4) - allocChannelCache(int sizeof(int), 8) - allocChannelCache(int sizeof(ptr float64), 16) - - # Don't create existing channel cache - not allocChannelCache(int sizeof(char), 4) - not allocChannelCache(int sizeof(int), 8) - not allocChannelCache(int sizeof(ptr float64), 16) - - check: - channelCacheLen == 3 - - # --------------------------------- - var chan, stash: array[10, ChannelRaw] - - block: # Implicit caches allocation - - chan[0] = allocChannel(sizeof(char), 4) - chan[1] = allocChannel(sizeof(int32), 8) - chan[2] = allocChannel(sizeof(ptr float64), 16) - - chan[3] = allocChannel(sizeof(char), 5) - chan[4] = allocChannel(sizeof(int64), 8) - chan[5] = allocChannel(sizeof(ptr float32), 24) - - # We have caches ready to store specific channel kinds - check: channelCacheLen == 6 # Cumulated with previous test - # But they are not in cache while in use - check: - not chan[0].isCached - not chan[1].isCached - not chan[2].isCached - not chan[3].isCached - not chan[4].isCached - not chan[5].isCached - - block: # Freed channels are returned to cache - stash[0..5] = chan.toOpenArray(0, 5) - for i in 0 .. 5: - # Free the channels - freeChannel(chan[i]) - - check: - stash[0].isCached - stash[1].isCached - stash[2].isCached - stash[3].isCached - stash[4].isCached - stash[5].isCached - - block: # Cached channels are being reused - - chan[6] = allocChannel(sizeof(char), 4) - chan[7] = allocChannel(sizeof(int32), 8) - chan[8] = allocChannel(sizeof(ptr float32), 16) - chan[9] = allocChannel(sizeof(ptr float64), 16) - - # All (itemsize, queue size, implementation) were already allocated - check: channelCacheLen == 6 - - # We reused old channels from cache - check: - chan[6] == stash[0] - chan[7] == stash[1] - chan[8] == stash[2] - # chan[9] - required a fresh alloc - - block: # Clearing the cache - - stash[6..9] = chan.toOpenArray(6, 9) - - for i in 6 .. 9: - freeChannel(chan[i]) - - check: - stash[6].isCached - stash[7].isCached - stash[8].isCached - stash[9].isCached - - freeChannelCache() - - # Check that nothing is cached anymore - for i in 0 .. 9: - check: not stash[i].isCached - # And length is reset to 0 - check: channelCacheLen == 0 - - # Cache can grow again - chan[0] = allocChannel(sizeof((int, float, int32, uint)), 1) - chan[1] = allocChannel(sizeof(int32), 0) - chan[2] = allocChannel(sizeof(int32), 0) - - check: channelCacheLen == 2 - - # Interleave cache clear and channel free - freeChannelCache() - check: channelCacheLen == 0 - - freeChannel(chan[0]) - freeChannel(chan[1]) - freeChannel(chan[2]) diff --git a/tests/stdlib/tchannels_simple.nim b/tests/stdlib/tchannels_simple.nim deleted file mode 100644 index 56e5fb8f1..000000000 --- a/tests/stdlib/tchannels_simple.nim +++ /dev/null @@ -1,67 +0,0 @@ -discard """ - matrix: "--threads:on --gc:orc; --threads:on --gc:arc" - disabled: "freebsd" -""" - -import std/channels -import std/os - -var chan = newChannel[string]() - -# This proc will be run in another thread using the threads module. -proc firstWorker() = - chan.send("Hello World!") - -# This is another proc to run in a background thread. This proc takes a while -# to send the message since it sleeps for 2 seconds (or 2000 milliseconds). -proc secondWorker() = - sleep(2000) - chan.send("Another message") - - -# Launch the worker. -var worker1: Thread[void] -createThread(worker1, firstWorker) - -# Block until the message arrives, then print it out. -let dest = chan.recv() -doAssert dest == "Hello World!" - -# Wait for the thread to exit before moving on to the next example. -worker1.joinThread() - -# Launch the other worker. -var worker2: Thread[void] -createThread(worker2, secondWorker) -# This time, use a non-blocking approach with tryRecv. -# Since the main thread is not blocked, it could be used to perform other -# useful work while it waits for data to arrive on the channel. - -var messages: seq[string] -var msg = "" -while true: - let tried = chan.tryRecv(msg) - if tried: - messages.add move(msg) - break - - messages.add "Pretend I'm doing useful work..." - # For this example, sleep in order not to flood stdout with the above - # message. - sleep(400) - -# Wait for the second thread to exit before cleaning up the channel. -worker2.joinThread() - -# Clean up the channel. -doAssert chan.close() -doAssert messages[^1] == "Another message" -doAssert messages.len >= 2 - - -block: - let chan0 = newChannel[int]() - let chan1 = chan0 - block: - let chan3 = chan0 - let chan4 = chan0 From e8dad482a309b1c33056aba22550d691845414d7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 15:57:16 +0200 Subject: [PATCH 0380/1654] fixes #16246 (#18800) --- compiler/seminst.nim | 3 ++- compiler/semtypinst.nim | 2 +- compiler/types.nim | 8 ++++++++ tests/arc/tarcmisc.nim | 11 +++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/compiler/seminst.nim b/compiler/seminst.nim index e1fad236c..456e40a94 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -91,7 +91,8 @@ proc sameInstantiation(a, b: TInstantiation): bool = for i in 0..a.concreteTypes.high: if not compareTypes(a.concreteTypes[i], b.concreteTypes[i], flags = {ExactTypeDescValues, - ExactGcSafety}): return + ExactGcSafety, + PickyCAliases}): return result = true proc genericCacheGet(g: ModuleGraph; genericSym: PSym, entry: TInstantiation; diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 18febb52e..9f9eb9489 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -51,7 +51,7 @@ proc searchInstTypes*(g: ModuleGraph; key: PType): PType = for j in 1..high(key.sons): # XXX sameType is not really correct for nested generics? if not compareTypes(inst[j], key[j], - flags = {ExactGenericParams}): + flags = {ExactGenericParams, PickyCAliases}): break matchType return inst diff --git a/compiler/types.nim b/compiler/types.nim index 61f563514..cf6579317 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -896,6 +896,7 @@ type ExactConstraints ExactGcSafety AllowCommonBase + PickyCAliases # be picky about the distinction between 'cint' and 'int32' TTypeCmpFlags* = set[TTypeCmpFlag] @@ -1144,6 +1145,13 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCstring, tyInt..tyUInt64, tyTyped, tyUntyped, tyVoid: result = sameFlags(a, b) + if result and PickyCAliases in c.flags: + # additional requirement for the caching of generics for importc'ed types: + # the symbols must be identical too: + let symFlagsA = if a.sym != nil: a.sym.flags else: {} + let symFlagsB = if b.sym != nil: b.sym.flags else: {} + if (symFlagsA+symFlagsB) * {sfImportc, sfExportc} != {}: + result = symFlagsA == symFlagsB of tyStatic, tyFromExpr: result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b) if result and a.len == b.len and a.len == 1: diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 7daea62c8..951ae25f3 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -463,3 +463,14 @@ proc putValue[T](n: T) = echo b.n useForward() + + +# bug #16246 + +proc testWeirdTypeAliases() = + var values = newSeq[cuint](8) + # var values: seq[cuint] does not produce codegen error + var drawCb = proc(): seq[uint32] = + result = newSeq[uint32](10) + +testWeirdTypeAliases() From 5c85e480a6a13ac12c69f7e5500322eb5474be11 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 17:49:27 +0200 Subject: [PATCH 0381/1654] unicode operator bugfixes (#18802) --- compiler/lexer.nim | 106 ++++++++++++++++------------- tests/lexer/tunicode_operators.nim | 4 +- 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ddf98661a..ede16fdf1 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -838,50 +838,6 @@ proc getCharacter(L: var Lexer; tok: var Token) = lexMessage(L, errGenerated, "missing closing ' for character literal") tokenEndIgnore(tok, L.bufpos) -proc getSymbol(L: var Lexer, tok: var Token) = - var h: Hash = 0 - var pos = L.bufpos - tokenBegin(tok, pos) - var suspicious = false - while true: - var c = L.buf[pos] - case c - of 'a'..'z', '0'..'9', '\x80'..'\xFF': - h = h !& ord(c) - inc(pos) - of 'A'..'Z': - c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() - h = h !& ord(c) - inc(pos) - suspicious = true - of '_': - if L.buf[pos+1] notin SymChars: - lexMessage(L, errGenerated, "invalid token: trailing underscore") - break - inc(pos) - suspicious = true - else: break - tokenEnd(tok, pos-1) - h = !$h - tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) - if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or - (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)): - tok.tokType = tkSymbol - else: - tok.tokType = TokType(tok.ident.id + ord(tkSymbol)) - if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}: - lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s) - L.bufpos = pos - - -proc endOperator(L: var Lexer, tok: var Token, pos: int, - hash: Hash) {.inline.} = - var h = !$hash - tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) - if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr - else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon)) - L.bufpos = pos - const UnicodeOperatorStartChars = {'\226', '\194', '\195'} # the allowed unicode characters ("∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔") @@ -925,6 +881,56 @@ proc unicodeOprLen(buf: cstring; pos: int): (int8, UnicodeOprPred) = else: discard +proc getSymbol(L: var Lexer, tok: var Token) = + var h: Hash = 0 + var pos = L.bufpos + tokenBegin(tok, pos) + var suspicious = false + while true: + var c = L.buf[pos] + case c + of 'a'..'z', '0'..'9': + h = h !& ord(c) + inc(pos) + of 'A'..'Z': + c = chr(ord(c) + (ord('a') - ord('A'))) # toLower() + h = h !& ord(c) + inc(pos) + suspicious = true + of '_': + if L.buf[pos+1] notin SymChars: + lexMessage(L, errGenerated, "invalid token: trailing underscore") + break + inc(pos) + suspicious = true + of '\x80'..'\xFF': + if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and unicodeOprLen(L.buf, pos)[0] != 0: + break + else: + h = h !& ord(c) + inc(pos) + else: break + tokenEnd(tok, pos-1) + h = !$h + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or + (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)): + tok.tokType = tkSymbol + else: + tok.tokType = TokType(tok.ident.id + ord(tkSymbol)) + if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}: + lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s) + L.bufpos = pos + + +proc endOperator(L: var Lexer, tok: var Token, pos: int, + hash: Hash) {.inline.} = + var h = !$hash + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr + else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon)) + L.bufpos = pos + proc getOperator(L: var Lexer, tok: var Token) = var pos = L.bufpos tokenBegin(tok, pos) @@ -1346,7 +1352,11 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = getNumber(L, tok) let c = L.buf[L.bufpos] if c in SymChars+{'_'}: - lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") + if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and + unicodeOprLen(L.buf, L.bufpos)[0] != 0: + discard + else: + lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") of '-': if L.buf[L.bufpos+1] in {'0'..'9'} and (L.bufpos-1 == 0 or L.buf[L.bufpos-1] in UnaryMinusWhitelist): @@ -1357,7 +1367,11 @@ proc rawGetTok*(L: var Lexer, tok: var Token) = getNumber(L, tok) let c = L.buf[L.bufpos] if c in SymChars+{'_'}: - lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") + if c in UnicodeOperatorStartChars and unicodeOperators in L.config.features and + unicodeOprLen(L.buf, L.bufpos)[0] != 0: + discard + else: + lexMessage(L, errGenerated, "invalid token: no whitespace between number and identifier") else: getOperator(L, tok) else: diff --git a/tests/lexer/tunicode_operators.nim b/tests/lexer/tunicode_operators.nim index 74fcbb763..6ad40beab 100644 --- a/tests/lexer/tunicode_operators.nim +++ b/tests/lexer/tunicode_operators.nim @@ -5,8 +5,10 @@ proc `⊙=`(x: var int, y: int) = x *= y proc `⊞++`(x, y: int): int = x + y +const a = 9 + var x = 45 -x ⊙= 9 ⊞++ 4 ⊙ 3 +x ⊙= a⊞++4⊙3 var y = 45 y *= 9 + 4 * 3 From 2e2d9288465709d84a5d91abbb035583c23efe45 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Sep 2021 19:52:57 +0200 Subject: [PATCH 0382/1654] formal grammar updates [backport:1.2] (#18803) --- compiler/parser.nim | 50 +++++++++++++++++++---------------------- doc/grammar.txt | 37 +++++++++++++----------------- tools/grammar_nanny.nim | 1 + 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 225493902..7d3c913da 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -275,12 +275,6 @@ proc isRightAssociative(tok: Token): bool {.inline.} = result = tok.tokType == tkOpr and tok.ident.s[0] == '^' # or (tok.ident.s.len > 1 and tok.ident.s[^1] == '>') -proc isOperator(tok: Token): bool = - ## Determines if the given token is an operator type token. - tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, - tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, - tkOr, tkXor} - proc isUnary(tok: Token): bool = ## Check if the given token is a unary operator tok.tokType in {tkOpr, tkDotDot} and @@ -323,6 +317,14 @@ proc checkBinary(p: Parser) {.inline.} = #| mulExpr = dollarExpr (OP9 optInd dollarExpr)* #| dollarExpr = primary (OP10 optInd primary)* +proc isOperator(tok: Token): bool = + #| operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 | + #| 'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' | + #| 'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor' + tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, + tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, + tkOr, tkXor} + proc colcom(p: var Parser, n: PNode) = eat(p, tkColon) skipComment(p, n) @@ -447,8 +449,6 @@ proc exprColonEqExprList(p: var Parser, kind: TNodeKind, exprColonEqExprListAux(p, endTok, result) proc dotExpr(p: var Parser, a: PNode): PNode = - #| dotExpr = expr '.' optInd (symbol | '[:' exprList ']') - #| explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )? var info = p.parLineInfo getTok(p) result = newNodeI(nkDotExpr, info) @@ -469,7 +469,6 @@ proc dotExpr(p: var Parser, a: PNode): PNode = result = y proc dotLikeExpr(p: var Parser, a: PNode): PNode = - #| dotLikeExpr = expr DOTLIKEOP optInd symbol var info = p.parLineInfo result = newNodeI(nkInfix, info) optInd(p, result) @@ -661,7 +660,7 @@ proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = #| | NIL #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT #| identOrLiteral = generalizedLit | symbol | literal - #| | par | arrayConstr | setOrTableConstr + #| | par | arrayConstr | setOrTableConstr | tupleConstr #| | castExpr #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' @@ -808,7 +807,7 @@ proc isDotLike(tok: Token): bool = proc primarySuffix(p: var Parser, r: PNode, baseIndent: int, mode: PrimaryMode): PNode = #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' - #| | '.' optInd symbol generalizedLit? + #| | '.' optInd symbol ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit? #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' @@ -1013,11 +1012,9 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode = result.add(newNodeP(nkEmpty, p)) proc parseTuple(p: var Parser, indentAllowed = false): PNode = - #| inlTupleDecl = 'tuple' - #| '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' - #| extTupleDecl = 'tuple' + #| tupleDecl = 'tuple' + #| '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' | #| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? - #| tupleClass = 'tuple' result = newNodeP(nkTupleTy, p) getTok(p) if p.tok.tokType == tkBracketLe: @@ -1123,7 +1120,7 @@ proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = - #| procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)? + #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? # either a proc type or a anonymous proc let info = parLineInfo(p) getTok(p) @@ -1165,7 +1162,6 @@ proc parseSymbolList(p: var Parser, result: PNode) = proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, mode: PrimaryMode): PNode = - #| distinct = 'distinct' optInd typeDesc result = newNodeP(kind, p) getTok(p) if p.tok.indent != -1 and p.tok.indent <= p.currInd: return @@ -1254,12 +1250,14 @@ proc parseObject(p: var Parser): PNode proc parseTypeClass(p: var Parser): PNode proc primary(p: var Parser, mode: PrimaryMode): PNode = - #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' - #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' - #| primary = typeKeyw optInd typeDesc + #| primary = operatorB primary primarySuffix* | + #| tupleDecl | routineExpr | enumDecl + #| objectDecl | conceptDecl | ('bind' primary) + #| ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary #| / prefixOperator* identOrLiteral primarySuffix* - #| / 'bind' primary if isOperator(p.tok): + # Note 'sigil like' operators are currently not reflected in the grammar + # and should be removed for Nim 2.0, I don't think anybody uses them. let isSigil = isSigilLike(p.tok) result = newNodeP(nkPrefix, p) var a = newIdentNodeP(p.tok.ident, p) @@ -1342,7 +1340,6 @@ proc parseTypeDesc(p: var Parser): PNode = proc parseTypeDefAux(p: var Parser): PNode = #| typeDefAux = simpleExpr ('not' expr)? - #| | 'concept' typeClass result = simpleExpr(p, pmTypeDef) result = binaryNot(p, result) @@ -1695,7 +1692,6 @@ proc parseTry(p: var Parser; isExpr: bool): PNode = if b == nil: parMessage(p, "expected 'except'") proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode = - #| exceptBlock = 'except' colcom stmt result = newNodeP(kind, p) getTok(p) colcom(p, result) @@ -1867,7 +1863,7 @@ proc parseSection(p: var Parser, kind: TNodeKind, parMessage(p, errIdentifierExpected, p.tok) proc parseEnum(p: var Parser): PNode = - #| enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ + #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ result = newNodeP(nkEnumTy, p) getTok(p) result.add(p.emptyNode) @@ -2015,7 +2011,7 @@ proc parseObjectPart(p: var Parser): PNode = result = p.emptyNode proc parseObject(p: var Parser): PNode = - #| object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart + #| objectDecl = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart result = newNodeP(nkObjectTy, p) getTok(p) if p.tok.tokType == tkCurlyDotLe and p.validInd: @@ -2057,8 +2053,8 @@ proc parseTypeClassParam(p: var Parser): PNode = result = p.parseSymbol proc parseTypeClass(p: var Parser): PNode = - #| typeClassParam = ('var' | 'out')? symbol - #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? + #| conceptParam = ('var' | 'out')? symbol + #| conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? #| &IND{>} stmt result = newNodeP(nkTypeClassTy, p) getTok(p) diff --git a/doc/grammar.txt b/doc/grammar.txt index f58621b97..8f86dd98c 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -22,14 +22,14 @@ ampExpr = plusExpr (OP7 optInd plusExpr)* plusExpr = mulExpr (OP8 optInd mulExpr)* mulExpr = dollarExpr (OP9 optInd dollarExpr)* dollarExpr = primary (OP10 optInd primary)* +operatorB = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 | + 'div' | 'mod' | 'shl' | 'shr' | 'in' | 'notin' | + 'is' | 'isnot' | 'not' | 'of' | 'as' | 'from' | '..' | 'and' | 'or' | 'xor' symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' | IDENT | KEYW exprColonEqExpr = expr (':'|'=' expr)? exprList = expr ^+ comma exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? -dotExpr = expr '.' optInd (symbol | '[:' exprList ']') -explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )? -dotLikeExpr = expr DOTLIKEOP optInd symbol qualifiedIdent = symbol ('.' optInd symbol)? setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}' castExpr = 'cast' ('[' optInd typeDesc optPar ']' '(' optInd expr optPar ')') / @@ -51,12 +51,12 @@ literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT | NIL generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT identOrLiteral = generalizedLit | symbol | literal - | par | arrayConstr | setOrTableConstr + | par | arrayConstr | setOrTableConstr | tupleConstr | castExpr tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' primarySuffix = '(' (exprColonEqExpr comma?)* ')' - | '.' optInd symbol generalizedLit? + | '.' optInd symbol ('[:' exprList ']' ( '(' exprColonEqExpr ')' )?)? generalizedLit? | DOTLIKEOP optInd symbol generalizedLit? | '[' optInd exprColonEqExprList optPar ']' | '{' optInd exprColonEqExprList optPar '}' @@ -70,17 +70,14 @@ declColonEquals = identWithPragma (comma identWithPragma)* comma? (':' optInd typeDesc)? ('=' optInd expr)? identColonEquals = IDENT (comma IDENT)* comma? (':' optInd typeDesc)? ('=' optInd expr)?) -inlTupleDecl = 'tuple' - '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' -extTupleDecl = 'tuple' +tupleDecl = 'tuple' + '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' | COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? -tupleClass = 'tuple' paramList = '(' declColonEquals ^* (comma/semicolon) ')' paramListArrow = paramList? ('->' optInd typeDesc)? paramListColon = paramList? (':' optInd typeDesc)? doBlock = 'do' paramListArrow pragma? colcom stmt -procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)? -distinct = 'distinct' optInd typeDesc +routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt forExpr = forStmt expr = (blockExpr @@ -90,14 +87,13 @@ expr = (blockExpr | forExpr | tryExpr) / simpleExpr -typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' - | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' -primary = typeKeyw optInd typeDesc +primary = operatorB primary primarySuffix* | + tupleDecl | routineExpr | enumDecl + objectDecl | conceptDecl | ('bind' primary) + ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary / prefixOperator* identOrLiteral primarySuffix* - / 'bind' primary typeDesc = simpleExpr ('not' expr)? typeDefAux = simpleExpr ('not' expr)? - | 'concept' typeClass postExprBlocks = ':' stmt? ( IND{=} doBlock | IND{=} 'of' exprList ':' stmt | IND{=} 'elif' expr ':' stmt @@ -147,7 +143,6 @@ tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') tryExpr = 'try' colcom stmt &(optInd 'except'|'finally') (optInd 'except' exprList colcom stmt)* (optInd 'finally' colcom stmt)? -exceptBlock = 'except' colcom stmt blockStmt = 'block' symbol? colcom stmt blockExpr = 'block' symbol? colcom stmt staticStmt = 'static' colcom stmt @@ -162,7 +157,7 @@ routine = optInd identVis pattern? genericParamList? paramListColon pragma? ('=' COMMENT? stmt)? indAndComment commentStmt = COMMENT section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED) -enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ +enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ objectWhen = 'when' expr colcom objectPart COMMENT? ('elif' expr colcom objectPart COMMENT?)* ('else' colcom objectPart COMMENT?)? @@ -175,9 +170,9 @@ objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT? | IND{=} objectBranches) objectPart = IND{>} objectPart^+IND{=} DED / objectWhen / objectCase / 'nil' / 'discard' / declColonEquals -object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart -typeClassParam = ('var' | 'out')? symbol -typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? +objectDecl = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart +conceptParam = ('var' | 'out')? symbol +conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? &IND{>} stmt typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux indAndComment? / identVisDot genericParamList? pragma '=' optInd typeDefAux diff --git a/tools/grammar_nanny.nim b/tools/grammar_nanny.nim index 5120ef976..502412c3c 100644 --- a/tools/grammar_nanny.nim +++ b/tools/grammar_nanny.nim @@ -13,6 +13,7 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = var stream = llStreamOpen(data) var declaredSyms = initHashSet[string]() var usedSyms = initHashSet[string]() + usedSyms.incl "module" # 'module' is the start rule. if stream != nil: declaredSyms.incl "section" # special case for 'section(RULE)' in the grammar var From 4320b158dd2e5d7420cc813a1e9e2aecc946de7d Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 5 Sep 2021 16:50:54 +0800 Subject: [PATCH 0383/1654] add weave to important packages (#17665) * add weave to important packages * Update testament/important_packages.nim --- testament/important_packages.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index ab29dcd0d..a91246562 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -158,6 +158,7 @@ pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" +pkg "weave", "nimble test_gc_arc", allowFailure = true pkg "websocket", "nim c websocket.nim" pkg "winim", allowFailure = true pkg "with" From f373c17ad926b669bb3b5819ae1dff4bde1da88a Mon Sep 17 00:00:00 2001 From: Miran Date: Mon, 6 Sep 2021 09:40:31 +0200 Subject: [PATCH 0384/1654] use new Nimble, with lockfiles (#18810) --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 8dd897028..e42ea53de 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master + NimbleStableCommit = "795704833ddfd0cdaefb45c60551d3ea205279ef" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From 7ae52d779184cd8a16b760e384777a75ed54464b Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 6 Sep 2021 01:43:26 -0600 Subject: [PATCH 0385/1654] Fix recursive generic typed defs (#18809) --- compiler/semtypinst.nim | 2 +- tests/generics/trecursivegenerics.nim | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/generics/trecursivegenerics.nim diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 9f9eb9489..01b21498b 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -378,7 +378,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = cl.typeMap = newTypeMapLayer(cl) for i in 1.. Date: Mon, 6 Sep 2021 13:12:14 +0200 Subject: [PATCH 0386/1654] fixes #14165, fixes #18739, fix the second example of #6269 (#18812) --- compiler/transf.nim | 11 ++++++---- tests/iter/titer.nim | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 3250f2ec4..5846e6e3b 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -738,7 +738,7 @@ proc transformFor(c: PTransf, n: PNode): PNode = pushInfoContext(c.graph.config, n.info) inc(c.inlining) stmtList.add(transform(c, body)) - #findWrongOwners(c, stmtList.pnode) + #findWrongOwners(c, stmtList.PNode) dec(c.inlining) popInfoContext(c.graph.config) popTransCon(c) @@ -1019,10 +1019,11 @@ proc transform(c: PTransf, n: PNode): PNode = of nkAsgn: result = transformAsgn(c, n) of nkIdentDefs, nkConstDef: - result = n + result = newTransNode(n) result[0] = transform(c, n[0]) # Skip the second son since it only contains an unsemanticized copy of the # variable type used by docgen + result[1] = n[1] result[2] = transform(c, n[2]) # XXX comment handling really sucks: if importantComments(c.graph.config): @@ -1033,8 +1034,10 @@ proc transform(c: PTransf, n: PNode): PNode = # (bug #2604). We need to patch this environment here too: let a = n[1] if a.kind == nkSym: - n[1] = transformSymAux(c, a) - return n + result = copyTree(n) + result[1] = transformSymAux(c, a) + else: + result = n of nkExceptBranch: result = transformExceptBranch(c, n) of nkCheckedFieldExpr: diff --git a/tests/iter/titer.nim b/tests/iter/titer.nim index 8e1c13e82..3a2450ae3 100644 --- a/tests/iter/titer.nim +++ b/tests/iter/titer.nim @@ -62,3 +62,53 @@ type Rule[T] = (int, T) var t: seq[Rule[int]] for (c, t) in t: discard + + + +import std/sugar + +# bug #14165 +iterator log_nodups_hamming(): int {.inline.} = + let lb3 = 1 + let lb4 = 123 + proc mul3(): int = lb3 + lb4 + yield mul3() + +for h in log_nodups_hamming(): + break +for h in log_nodups_hamming(): + break +for h in log_nodups_hamming(): + break + +# bug #18536 +iterator envPairs*(): int = + var foo: seq[int] + proc fun() = + foo = @[] + fun() + yield 3 + +proc main() = + for a in envPairs(): + discard + for a in envPairs(): + discard +static: main() +main() + +# bug #6269 +iterator makeFn(outer_val: int): proc(a: int): int = + for i in 0..1: + yield proc(a:int): int = + return a + i.int + +let v1 = 42 + +let res = collect: + for fn1 in makeFn(v1): + let v2 = fn1(v1) + for fn2 in makeFn(v2): + fn2(v2) + +doAssert res == @[42, 43, 43, 44] From cc5422ae5070a2cdd69dbd5b78663305623a0fbc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 6 Sep 2021 13:19:11 +0200 Subject: [PATCH 0387/1654] Atlas: added 'extract' command, as promised (#18813) --- tools/atlas/atlas.nim | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index 761ca5db8..370138029 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -9,7 +9,7 @@ ## Simple tool to automate frequent workflows: Can "clone" ## a Nimble dependency and its dependencies recursively. -import std/[parseopt, strutils, os, osproc, unicode, tables, sets] +import std/[parseopt, strutils, os, osproc, unicode, tables, sets, json, jsonutils] import parse_requires, osutils, packagesjson const @@ -22,6 +22,8 @@ Usage: Command: clone url|pkgname clone a package and all of its dependencies search keyw keywB... search for package that contains the given keywords + extract file.nimble extract the requirements and custom commands from + the given Nimble file Options: --keepCommits do not perform any `git checkouts` @@ -460,6 +462,12 @@ proc main = of "search", "list": updatePackages(c) search getPackages(c.workspace), args + of "extract": + singleArg() + if fileExists(args[0]): + echo toJson(extractRequiresInfo(args[0])) + else: + error "File does not exist: " & args[0] else: error "Invalid action: " & action From 90bfd342504fd1a6a9e4c8f232bf7c35eab92f82 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 6 Sep 2021 07:30:49 -0600 Subject: [PATCH 0388/1654] '[]' can now be used for iterators (#18814) --- compiler/semexprs.nim | 2 +- tests/iter/tarrayiter.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/iter/tarrayiter.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 3edeae324..c0a2a346a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1596,7 +1596,7 @@ proc semArrayAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = result = semSubscript(c, n, flags) if result == nil: # overloaded [] operator: - result = semExpr(c, buildOverloadedSubscripts(n, getIdent(c.cache, "[]"))) + result = semExpr(c, buildOverloadedSubscripts(n, getIdent(c.cache, "[]")), flags) proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode = var id = considerQuotedIdent(c, a[1], a) diff --git a/tests/iter/tarrayiter.nim b/tests/iter/tarrayiter.nim new file mode 100644 index 000000000..eb7ba591a --- /dev/null +++ b/tests/iter/tarrayiter.nim @@ -0,0 +1,14 @@ +block: + iterator `[]`(a: int, r: int): int = + for q in 0 .. r: + yield a + + for val in 10[2]: discard + + type Custom = distinct string + + iterator `[]`(a: Custom, r: int): char = + for q in 0 .. r: + yield a.string[q] + + for val in Custom("test")[2]: discard \ No newline at end of file From 34a53e804943ceaf3900feb6e89194aa03236c0a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 6 Sep 2021 17:43:03 +0200 Subject: [PATCH 0389/1654] fixes #12642 (#18811) * fixes #12642 * update important packages; refs #18804 * fixes #18805; refs #18806 * fixes a regression * Update testament/categories.nim Co-authored-by: flywind * progress * progress Co-authored-by: flywind --- compiler/ccgcalls.nim | 1 + compiler/sem.nim | 6 ++++++ compiler/transf.nim | 5 +++-- testament/categories.nim | 6 ++++-- testament/important_packages.nim | 5 ++--- tests/effects/teffects6.nim | 20 ++++++++++++++++++++ 6 files changed, 36 insertions(+), 7 deletions(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 828e666a8..f6cc6da74 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -369,6 +369,7 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope = if ri[i].skipTrivialIndirections.kind == nkSym: needTmp[i - 1] = potentialAlias(ri[i], potentialWrites) else: + #if not ri[i].typ.isCompileTimeOnly: var potentialReads: seq[PNode] getPotentialReads(ri[i], potentialReads) for n in potentialReads: diff --git a/compiler/sem.nim b/compiler/sem.nim index 24709cf21..ad0ba1f7c 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -174,6 +174,12 @@ proc commonType*(c: PContext; x, y: PType): PType = result = b #.skipIntLit elif a.kind in IntegralTypes and a.n != nil: result = a #.skipIntLit + elif a.kind == tyProc and b.kind == tyProc: + if a.callConv == ccClosure and b.callConv != ccClosure: + result = x + elif compatibleEffects(a, b) != efCompat or + (b.flags * {tfNoSideEffect, tfGcSafe}) < (a.flags * {tfNoSideEffect, tfGcSafe}): + result = y else: var k = tyNone if a.kind in {tyRef, tyPtr}: diff --git a/compiler/transf.nim b/compiler/transf.nim index 5846e6e3b..89fa89701 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -1023,8 +1023,9 @@ proc transform(c: PTransf, n: PNode): PNode = result[0] = transform(c, n[0]) # Skip the second son since it only contains an unsemanticized copy of the # variable type used by docgen - result[1] = n[1] - result[2] = transform(c, n[2]) + let last = n.len-1 + for i in 1.. Date: Mon, 6 Sep 2021 12:11:21 -0600 Subject: [PATCH 0390/1654] Fixes implicit and explicit generics in procedures (#18808) * Fixes implicit and explicit generics * moved block logic into 'maybeInstantiateGeneric' * Added more tests * Update compiler/semexprs.nim Co-authored-by: Andreas Rumpf --- compiler/semexprs.nim | 28 +++++++++++--- tests/generics/timplicit_and_explicit.nim | 45 +++++++++++++++++++++++ 2 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 tests/generics/timplicit_and_explicit.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c0a2a346a..0f6f71cc2 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1493,6 +1493,28 @@ proc semDeref(c: PContext, n: PNode): PNode = else: result = nil #GlobalError(n[0].info, errCircumNeedsPointer) +proc maybeInstantiateGeneric(c: PContext, n: PNode, s: PSym): PNode = + ## Instantiates generic if not lacking implicit generics, + ## otherwise returns n. + let + neededGenParams = s.ast[genericParamsPos].len + heldGenParams = n.len - 1 + var implicitParams = 0 + for x in s.ast[genericParamsPos]: + if tfImplicitTypeParam in x.typ.flags: + inc implicitParams + if heldGenParams != neededGenParams and implicitParams + heldGenParams == neededGenParams: + # This is an implicit + explicit generic procedure without all args passed, + # kicking back the sem'd symbol fixes #17212 + # Uncertain the hackiness of this solution. + result = n + else: + result = explicitGenericInstantiation(c, n, s) + if result == n: + n[0] = copyTree(result) + else: + n[0] = result + proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = ## returns nil if not a built-in subscript operator; also called for the ## checking of assignments @@ -1568,11 +1590,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = of skProc, skFunc, skMethod, skConverter, skIterator: # type parameters: partial generic specialization n[0] = semSymGenericInstantiation(c, n[0], s) - result = explicitGenericInstantiation(c, n, s) - if result == n: - n[0] = copyTree(result) - else: - n[0] = result + result = maybeInstantiateGeneric(c, n, s) of skMacro, skTemplate: if efInCall in flags: # We are processing macroOrTmpl[] in macroOrTmpl[](...) call. diff --git a/tests/generics/timplicit_and_explicit.nim b/tests/generics/timplicit_and_explicit.nim new file mode 100644 index 000000000..ba24b79e9 --- /dev/null +++ b/tests/generics/timplicit_and_explicit.nim @@ -0,0 +1,45 @@ + +block: # basic test + proc doStuff[T](a: SomeInteger): T = discard + proc doStuff[T;Y](a: SomeInteger, b: Y): Y = discard + assert typeof(doStuff[int](100)) is int + assert typeof(doStuff[int](100, 1.0)) is float + assert typeof(doStuff[int](100, "Hello")) is string + + proc t[T](x: T; z: int | float): seq[T] = result.add(x & $z) + + assert t[string]("Hallo", 2.0) == @["Hallo" & $2.0] + + proc t2[T](z: int | float): seq[T] = result.add($z) + + assert t2[string](2.0) == @[$2.0] + +block: # template test + template someThing[T;Y](a: SomeFloat, b: SomeOrdinal): (T, Y) = (a, b) + assert typeof(someThing[float64, int](1.0, 100)) is (float64, int) + +block: # static test + proc t[T](s: static bool) = discard + proc t2[T](s: static string) = discard + t[string](true) + t2[int]("hello") + t2[string]("world") + t2[float]("test222222") + +block: #11152 + proc f[T](X: typedesc) = discard + f[int](string) + +block: #15622 + proc test1[T](a: T, b: static[string] = "") = discard + test1[int64](123) + proc test2[T](a: T, b: static[string] = "") = discard + test2[int64, static[string]](123) + +block: #4688 + proc convertTo[T](v: int or float): T = (T)(v) + discard convertTo[float](1) + +block: #4164 + proc printStr[T](s: static[string]): T = discard + discard printStr[int]("hello static") \ No newline at end of file From ee2eb5cae2585c5cd0a476814bd5879c130e5d30 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 7 Sep 2021 09:11:08 -0600 Subject: [PATCH 0391/1654] Fix subranges of distinct types (#18816) [backport] --- compiler/typeallowed.nim | 2 +- tests/distinct/tdistinct.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 2f0c039a4..d00aa538f 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -124,7 +124,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedAux(marker, lastSon(t), kind, c, flags) of tyRange: if skipTypes(t[0], abstractInst-{tyTypeDesc}).kind notin - {tyChar, tyEnum, tyInt..tyFloat128, tyInt..tyUInt64}: result = t + {tyChar, tyEnum, tyInt..tyFloat128, tyInt..tyUInt64, tyRange}: result = t of tyOpenArray: # you cannot nest openArrays/sinks/etc. if (kind != skParam or taIsOpenArray in flags) and views notin c.features: diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index 876975a7c..fd60b4ac0 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -167,6 +167,20 @@ template main() = s.Foo.add('c') doAssert s.string == "c" # was failing test() + block: #18061 + type + A = distinct (0..100) + B = A(0) .. A(10) + proc test(b: B) = discard + let + a = A(10) + b = B(a) + test(b) + + proc test(a: A) = discard + discard cast[B](A(1)) + var c: B + static: main() main() From c56008450a86daf454fa0e2e0722711831cd78ae Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 8 Sep 2021 17:47:14 +0800 Subject: [PATCH 0392/1654] fix CI failure(upstream changes) (#18820) * fix upstream changes * Update tests/js/tunittest_error2.nim Co-authored-by: flywind Co-authored-by: Andreas Rumpf --- tests/js/tunittest_error2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/tunittest_error2.nim b/tests/js/tunittest_error2.nim index 273e39d9d..9c5af7529 100644 --- a/tests/js/tunittest_error2.nim +++ b/tests/js/tunittest_error2.nim @@ -1,7 +1,7 @@ discard """ exitcode: 1 outputsub: ''' -Unhandled exception: Cannot read property 'charCodeAt' of null [] +[] [FAILED] Bad test ''' matrix: "-d:nodejs" From 23e10ea85ea35c1a5963bfd12fb69efe6358076a Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 9 Sep 2021 14:46:55 +0800 Subject: [PATCH 0393/1654] active some packages (#18825) --- testament/important_packages.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 025121990..657ddb2d1 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -63,7 +63,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget", allowFailure = true # pending https://github.com/treeform/fidget/issues/155 +pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" @@ -76,7 +76,7 @@ pkg "gnuplot", "nim c gnuplot.nim" pkg "hts", "nim c -o:htss src/hts.nim" pkg "httpauth" pkg "illwill", "nimble examples" -pkg "inim", allowFailure=true +pkg "inim" pkg "itertools", "nim doc src/itertools.nim" pkg "iterutils" pkg "jstin" @@ -100,7 +100,7 @@ pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround need pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" -pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim", allowFailure = true +pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" pkg "nimlsp" @@ -145,7 +145,7 @@ pkg "strslice" pkg "strunicode", "nim c -r src/strunicode.nim" pkg "supersnappy" pkg "synthesis" -pkg "telebot", "nim c -o:tbot -r src/telebot.nim", allowFailure = true # pending https://github.com/ba0f3/telebot.nim/issues/59 +pkg "telebot", "nim c -o:tbot -r src/telebot.nim" pkg "tempdir" pkg "templates" pkg "tensordsl", "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git" From a896f9f19eec221b8651849297bdbc8be72b00af Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 9 Sep 2021 12:57:21 +0100 Subject: [PATCH 0394/1654] Clarify unknown processor error (#18829) --- tools/niminst/makefile.nimf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index d14960ac5..a0e9a40f7 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -164,7 +164,7 @@ ifeq ($(ucpu),riscv64) mycpu = riscv64 endif ifndef mycpu - $(error unknown processor: $(ucpu)) + $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif # for osA in 1..c.oses.len: From 0ef830577b6f6ee2314ecdfe56a631ffd4f8e4c7 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 10 Sep 2021 05:20:32 -0300 Subject: [PATCH 0395/1654] distros.foreignDeps made public (#18830) * Deprecate distros.echoForeignDeps * Deprecate distros.echoForeignDeps * https://github.com/nim-lang/Nim/pull/18830#discussion_r705364162 * https://github.com/nim-lang/Nim/pull/18830#discussion_r705364162 --- changelog.md | 1 + lib/pure/distros.nim | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 3658fb105..9e449dcaa 100644 --- a/changelog.md +++ b/changelog.md @@ -380,6 +380,7 @@ - Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect if the slice is out of bounds, likewise with `strutils.delete`. + ## Language changes - The `cstring` doesn't support `[]=` operator in JS backend. diff --git a/lib/pure/distros.nim b/lib/pure/distros.nim index ddf0fd30c..797698d61 100644 --- a/lib/pure/distros.nim +++ b/lib/pure/distros.nim @@ -27,11 +27,11 @@ ## ## See `packaging `_ for hints on distributing Nim using OS packages. -from strutils import contains, toLowerAscii +from std/strutils import contains, toLowerAscii when not defined(nimscript): - from osproc import execProcess - from os import existsEnv + from std/osproc import execProcess + from std/os import existsEnv type Distribution* {.pure.} = enum ## the list of known distributions @@ -211,7 +211,7 @@ template detectOs*(d: untyped): bool = detectOsImpl(Distribution.d) when not defined(nimble): - var foreignDeps: seq[string] = @[] + var foreignDeps*: seq[string] = @[] ## Registered foreign deps. proc foreignCmd*(cmd: string; requiresSudo = false) = ## Registers a foreign command to the internal list of commands From 6c2f041368e36bb7a6b217c7ac0c5de94c7b4a7a Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 11 Sep 2021 09:25:48 +0800 Subject: [PATCH 0396/1654] fix code-block which causes missing docs --- lib/system/channels_builtin.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim index c7a445766..30267e375 100644 --- a/lib/system/channels_builtin.nim +++ b/lib/system/channels_builtin.nim @@ -26,7 +26,7 @@ ## The following is a simple example of two different ways to use channels: ## blocking and non-blocking. ## -## .. code-block :: Nim +## .. code-block:: Nim ## # Be sure to compile with --threads:on. ## # The channels and threads modules are part of system and should not be ## # imported. @@ -112,7 +112,7 @@ ## using e.g. `system.allocShared0` and pass these pointers through thread ## arguments: ## -## .. code-block :: Nim +## .. code-block:: Nim ## proc worker(channel: ptr Channel[string]) = ## let greeting = channel[].recv() ## echo greeting From 1f68f71ec24d3ec6b8e83411a6f1604277f9d493 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 11 Sep 2021 17:33:52 +0800 Subject: [PATCH 0397/1654] [minor] fix docs (#18834) --- lib/pure/collections/sets.nim | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 3f91d9030..7b1c58ac8 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -603,12 +603,10 @@ proc isValid*[A](s: HashSet[A]): bool {.deprecated: ## Returns `true` if the set has been initialized (with `initHashSet proc ## <#initHashSet>`_ or `init proc <#init,HashSet[A]>`_). ## - ## **Examples:** - ## - ## .. code-block :: - ## proc savePreferences(options: HashSet[string]) = - ## assert options.isValid, "Pass an initialized set!" - ## # Do stuff here, may crash in release builds! + runnableExamples: + proc savePreferences(options: HashSet[string]) = + assert options.isValid, "Pass an initialized set!" + # Do stuff here, may crash in release builds! result = s.data.len > 0 From 66e53bdd7b465edd9045314d4d6a60ef6e0b5e32 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 11 Sep 2021 05:05:53 -0600 Subject: [PATCH 0398/1654] Fixed type inference for 'set` and 'tuple' (#18827) * improved built in typeclass inference * Smarter logic to fit node * Forgot the untyped check --- compiler/sem.nim | 3 ++- tests/metatype/typeclassinference.nim | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index ad0ba1f7c..bdecbe602 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -77,7 +77,8 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode = let x = arg.skipConv - if x.kind in {nkPar, nkTupleConstr, nkCurly} and formal.kind != tyUntyped: + if (x.kind == nkCurly and formal.kind == tySet and formal.base.kind != tyGenericParam) or + (x.kind in {nkPar, nkTupleConstr}) and formal.kind notin {tyUntyped, tyBuiltInTypeClass}: changeType(c, x, formal, check=true) result = arg result = skipHiddenSubConv(result, c.graph, c.idgen) diff --git a/tests/metatype/typeclassinference.nim b/tests/metatype/typeclassinference.nim index c845e04f7..b3f197718 100644 --- a/tests/metatype/typeclassinference.nim +++ b/tests/metatype/typeclassinference.nim @@ -19,3 +19,25 @@ var ptr1: ptr = addr(str1) var str2: string = "hello, world!" var ptr2: ptr = str2 +block: # built in typeclass inference + proc tupleA(): tuple = return (1, 2) + proc tupleB(): tuple = (1f, 2f) + assert typeof(tupleA()) is (int, int) + assert typeof(tupleB()) is (float32, float32) + + proc a(val: int or float): tuple = + when typeof(val) is int: + (10, 10) + else: + (30f, 30f) + + assert typeof(a(10)) is (int, int) + assert typeof(a(10.0)) is (float32, float32) + + proc b(val: int or float): set = + when typeof(val) is int: + {10u8, 3} + else: + {'a', 'b'} + assert typeof(b(10)) is set[uint8] + assert typeof(b(10.0)) is set[char] \ No newline at end of file From 5d1608c9764416e74b36f6a772f16a66d58ddc28 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 11 Sep 2021 14:20:22 -0600 Subject: [PATCH 0399/1654] Generic pointer procs now error if no types supplied (#18832) * more precise logic for pointer procs * added test for generic pointer procs * Fixed generic getting bracket expr if erroring --- compiler/semexprs.nim | 2 +- compiler/semstmts.nim | 2 +- tests/generics/tpointerprocs.nim | 28 ++++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/generics/tpointerprocs.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 0f6f71cc2..41381b057 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1511,7 +1511,7 @@ proc maybeInstantiateGeneric(c: PContext, n: PNode, s: PSym): PNode = else: result = explicitGenericInstantiation(c, n, s) if result == n: - n[0] = copyTree(result) + n[0] = copyTree(result[0]) else: n[0] = result diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 2c5f3ba54..364444c98 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -553,7 +553,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = typ = typ.lastSon if hasEmpty(typ): localError(c.config, def.info, errCannotInferTypeOfTheLiteral % typ.kind.toHumanStr) - elif typ.kind == tyProc and hasUnresolvedParams(def, {}): + elif typ.kind == tyProc and def.kind == nkSym and isGenericRoutine(def.sym.ast): # tfUnresolved in typ.flags: localError(c.config, def.info, errProcHasNoConcreteType % def.renderTree) when false: diff --git a/tests/generics/tpointerprocs.nim b/tests/generics/tpointerprocs.nim new file mode 100644 index 000000000..2bcaf15b3 --- /dev/null +++ b/tests/generics/tpointerprocs.nim @@ -0,0 +1,28 @@ +discard """ +cmd: "nim check $options --hints:off $file" +action: "reject" +nimout:''' +tpointerprocs.nim(15, 11) Error: 'foo' doesn't have a concrete type, due to unspecified generic parameters. +tpointerprocs.nim(27, 11) Error: cannot instantiate: 'foo[int]'; got 1 typeof(s) but expected 2 +tpointerprocs.nim(27, 14) Error: expression 'foo[int]' has no type (or is ambiguous) +tpointerprocs.nim(28, 11) Error: expression 'bar' has no type (or is ambiguous) +''' +""" + +block: + proc foo(x: int | float): float = result = 1.0 + let + bar = foo + baz = bar + +block: + proc foo(x: int | float): float = result = 1.0 + let + bar = foo[int] + baz = bar + +block: + proc foo(x: int | float, y: int or string): float = result = 1.0 + let + bar = foo[int] + baz = bar \ No newline at end of file From 3f3e0fa303a2729d0d6a9ffe54936c1c830e2d7e Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 13 Sep 2021 01:35:19 -0600 Subject: [PATCH 0400/1654] Fixed #18838 (#18841) [backport] --- compiler/semtypinst.nim | 6 ++++- tests/generics/trecursivegenerics.nim | 36 ++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 01b21498b..cb1ead027 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -344,7 +344,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = x = lookupTypeVar(cl, x) if x != nil: if header == t: header = instCopyType(cl, t) - header[i] = x + header[i] = + if x.kind == tyGenericInst: + t[i] + else: + x propagateToOwner(header, x) else: propagateToOwner(header, x) diff --git a/tests/generics/trecursivegenerics.nim b/tests/generics/trecursivegenerics.nim index 56c074e3f..e4c0e720f 100644 --- a/tests/generics/trecursivegenerics.nim +++ b/tests/generics/trecursivegenerics.nim @@ -1,6 +1,7 @@ block: # Replicates #18728 type FlipFlop[A, B] = ref object + val: A next: FlipFlop[B, A] Trinary[A, B, C] = ref object @@ -9,4 +10,37 @@ block: # Replicates #18728 assert typeof(FlipFlop[int, string]().next) is FlipFlop[string, int] assert typeof(FlipFlop[string, int]().next) is FlipFlop[int, string] assert typeof(Trinary[int, float, string]().next) is Trinary[float, string, int] - assert typeof(Trinary[int, float, string]().next.next) is Trinary[string, int, float] \ No newline at end of file + assert typeof(Trinary[int, float, string]().next.next) is Trinary[string, int, float] + var a = FlipFlop[int, string](val: 100, next: FlipFlop[string, int](val: "Hello")) + assert a.val == 100 + assert a.next.val == "Hello" + +block: # 18838 + type + DoublyLinkedNodeObj[T] = object + value: T + + DoublyLinkedNode[T] = ref DoublyLinkedNodeObj[T] + + Item[T] = ref object + link: DoublyLinkedNode[Item[T]] + + Box = object + + proc newDoublyLinkedNode[T](value: T): DoublyLinkedNode[T] = + new(result) + result.value = value + + let link = newDoublyLinkedNode(Item[Box]()) + +import lists +block: + type + Box = object + Item[T] = ref object + link:DoublyLinkedNode[ Item[T] ] + + ItemSimple = ref object + link:DoublyLinkedNode[ ItemSimple ] + + let link = newDoublyLinkedNode( Item[Box]() ) \ No newline at end of file From 179fad934d52641c5e064efce4bdd98578cc5e9e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 13 Sep 2021 11:05:09 +0200 Subject: [PATCH 0401/1654] doc improvements (#18843) * cleaned up destructors documentation [backport] * Spec updates [backport:1.0] --- doc/destructors.rst | 53 +++++--------------- doc/manual.rst | 118 ++++++++------------------------------------ 2 files changed, 33 insertions(+), 138 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index a3fd7514e..28d8fa5f9 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -222,7 +222,14 @@ The prototype of this hook for a type `T` needs to be: `env` is used by ORC to keep track of its internal state, it should be passed around to calls of the built-in `=trace` operation. -Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates manually allocated resources is also used, and then only when there is a chance of cyclic references from items within the manually allocated resources when it is desired that `--gc:orc` be able to break and collect these cyclic referenced resources. Currently however, there is a mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically create a version of the other which will then conflict with the creation of the second of the pair. The work around for this problem is to forward declare the second of the "hooks" to prevent the automatic creation. +Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates +manually allocated resources is also used, and then only when there is a chance of cyclic +references from items within the manually allocated resources when it is desired that `--gc:orc` +be able to break and collect these cyclic referenced resources. Currently however, there is a +mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically +create a version of the other which will then conflict with the creation of the second of the +pair. The work around for this problem is to forward declare the second of the "hooks" to +prevent the automatic creation. The general pattern in using `=destroy` with `=trace` looks like: @@ -696,43 +703,22 @@ should eventually be replaced by a better solution. Copy on write ============= -String literals are implemented as [copy-on-write](https://en.wikipedia.org/wiki/Copy-on-write). +String literals are implemented as "copy on write". When assigning a string literal to a variable, a copy of the literal won't be created. Instead the variable simply points to the literal. The literal is shared between different variables which are pointing to it. The copy operation is deferred until the first write. -.. code-block:: nim - var x = "abc" # no copy - var y = x # no copy - -The string literal "abc" is stored in static memory and not allocated on the heap. -The variable `x` points to the literal and the variable `y` points to the literal too. -There is no copy during assigning operations. +For example: .. code-block:: nim var x = "abc" # no copy var y = x # no copy y[0] = 'h' # copy -The program above shows when the copy operations happen. -When mutating the variable `y`, the Nim compiler creates a fresh copy of `x`, -the variable `y` won't point to the string literal anymore. -Instead it points to the copy of `x` of which the memory can be mutated -and the variable `y` becomes a mutable string. -.. Note:: The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. - -Let's look at a silly example demonstrating this behaviour: - -.. code-block:: nim - var x = "abc" - var y = x - - moveMem(addr y[0], addr x[0], 3) - -The program fails because we need to prepare a fresh copy for the variable `y`. -`prepareMutation` should be called before the address operation. +The abstraction fails for `addr x` because whether the address is going to be used for mutations is unknown. +`prepareMutation` needs to be called before the "address of" operation. For example: .. code-block:: nim var x = "abc" @@ -741,18 +727,3 @@ The program fails because we need to prepare a fresh copy for the variable `y`. prepareMutation(y) moveMem(addr y[0], addr x[0], 3) assert y == "abc" - -Now `prepareMutation` solves the problem. -It manually creates a fresh copy and makes the variable `y` mutable. - -.. code-block:: nim - var x = "abc" - var y = x - - prepareMutation(y) - moveMem(addr y[0], addr x[0], 3) - moveMem(addr y[0], addr x[0], 3) - moveMem(addr y[0], addr x[0], 3) - assert y == "abc" - -No matter how many times `moveMem` is called, the program compiles and runs. diff --git a/doc/manual.rst b/doc/manual.rst index 7976ffe0f..af163bceb 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2400,121 +2400,38 @@ describe the type checking done by the compiler. Type equality ------------- + Nim uses structural type equivalence for most types. Only for objects, -enumerations and distinct types name equivalence is used. The following -algorithm, *in pseudo-code*, determines type equality: - -.. code-block:: nim - proc typeEqualsAux(a, b: PType, - s: var HashSet[(PType, PType)]): bool = - if (a,b) in s: return true - incl(s, (a,b)) - if a.kind == b.kind: - case a.kind - of int, intXX, float, floatXX, char, string, cstring, pointer, - bool, nil, void: - # leaf type: kinds identical; nothing more to check - result = true - of ref, ptr, var, set, seq, openarray: - result = typeEqualsAux(a.baseType, b.baseType, s) - of range: - result = typeEqualsAux(a.baseType, b.baseType, s) and - (a.rangeA == b.rangeA) and (a.rangeB == b.rangeB) - of array: - result = typeEqualsAux(a.baseType, b.baseType, s) and - typeEqualsAux(a.indexType, b.indexType, s) - of tuple: - if a.tupleLen == b.tupleLen: - for i in 0..a.tupleLen-1: - if not typeEqualsAux(a[i], b[i], s): return false - result = true - of object, enum, distinct: - result = a == b - of proc: - result = typeEqualsAux(a.parameterTuple, b.parameterTuple, s) and - typeEqualsAux(a.resultType, b.resultType, s) and - a.callingConvention == b.callingConvention - - proc typeEquals(a, b: PType): bool = - var s: HashSet[(PType, PType)] = {} - result = typeEqualsAux(a, b, s) - -Since types are graphs which can have cycles, the above algorithm needs an -auxiliary set `s` to detect this case. - - -Type equality modulo type distinction -------------------------------------- - -The following algorithm (in pseudo-code) determines whether two types -are equal with no respect to `distinct` types. For brevity the cycle check -with an auxiliary set `s` is omitted: - -.. code-block:: nim - proc typeEqualsOrDistinct(a, b: PType): bool = - if a.kind == b.kind: - case a.kind - of int, intXX, float, floatXX, char, string, cstring, pointer, - bool, nil, void: - # leaf type: kinds identical; nothing more to check - result = true - of ref, ptr, var, set, seq, openarray: - result = typeEqualsOrDistinct(a.baseType, b.baseType) - of range: - result = typeEqualsOrDistinct(a.baseType, b.baseType) and - (a.rangeA == b.rangeA) and (a.rangeB == b.rangeB) - of array: - result = typeEqualsOrDistinct(a.baseType, b.baseType) and - typeEqualsOrDistinct(a.indexType, b.indexType) - of tuple: - if a.tupleLen == b.tupleLen: - for i in 0..a.tupleLen-1: - if not typeEqualsOrDistinct(a[i], b[i]): return false - result = true - of distinct: - result = typeEqualsOrDistinct(a.baseType, b.baseType) - of object, enum: - result = a == b - of proc: - result = typeEqualsOrDistinct(a.parameterTuple, b.parameterTuple) and - typeEqualsOrDistinct(a.resultType, b.resultType) and - a.callingConvention == b.callingConvention - elif a.kind == distinct: - result = typeEqualsOrDistinct(a.baseType, b) - elif b.kind == distinct: - result = typeEqualsOrDistinct(a, b.baseType) +enumerations and distinct types and for generic types name equivalence is used. Subtype relation ---------------- -If object `a` inherits from `b`, `a` is a subtype of `b`. This subtype -relation is extended to the types `var`, `ref`, `ptr`: -.. code-block:: nim - proc isSubtype(a, b: PType): bool = - if a.kind == b.kind: - case a.kind - of object: - var aa = a.baseType - while aa != nil and aa != b: aa = aa.baseType - result = aa == b - of var, ref, ptr: - result = isSubtype(a.baseType, b.baseType) +If object `a` inherits from `b`, `a` is a subtype of `b`. -.. XXX nil is a special value! +This subtype relation is extended to the types `var`, `ref`, `ptr`. +If `A` is a subtype of `B` and `A` and `B` are `object` types then: +- `var A` is a subtype of `var B` +- `ref A` is a subtype of `ref B` +- `ptr A` is a subtype of `ptr B`. +**Note**: In later versions of the language the subtype relation might +be changed to *require* the pointer indirection in order to prevent +"object slicing". Convertible relation -------------------- + A type `a` is **implicitly** convertible to type `b` iff the following algorithm returns true: .. code-block:: nim proc isImplicitlyConvertible(a, b: PType): bool = - if isSubtype(a, b) or isCovariant(a, b): + if isSubtype(a, b): return true if isIntLiteral(a): return b in {int8, int16, int32, int64, int, uint, uint8, uint16, @@ -2540,7 +2457,12 @@ algorithm returns true: result = b == pointer of string: result = b == cstring + of proc: + result = typeEquals(a, b) or compatibleParametersAndEffects(a, b) +We used the predicate `typeEquals(a, b)` for the "type equality" property +and the predicate `isSubtype(a, b)` for the "subtype relation". +`compatibleParametersAndEffects(a, b)` is currently not specified. Implicit conversions are also performed for Nim's `range` type constructor. @@ -2563,7 +2485,9 @@ algorithm returns true: proc isExplicitlyConvertible(a, b: PType): bool = result = false if isImplicitlyConvertible(a, b): return true - if typeEqualsOrDistinct(a, b): return true + if typeEquals(a, b): return true + if a == distinct and typeEquals(a.baseType, b): return true + if b == distinct and typeEquals(b.baseType, a): return true if isIntegralType(a) and isIntegralType(b): return true if isSubtype(a, b) or isSubtype(b, a): return true From ef390e6a68db74a61137d7690dd2b10ec3dee050 Mon Sep 17 00:00:00 2001 From: Christian Ulrich Date: Mon, 13 Sep 2021 18:47:33 +0200 Subject: [PATCH 0402/1654] add changelog entry for #18656 (#18839) [backport] --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 9e449dcaa..ab037c328 100644 --- a/changelog.md +++ b/changelog.md @@ -111,6 +111,11 @@ - The allocator for Nintendo Switch, which was nonfunctional because of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. +- `net.parseIpAddress` now only allows IPv4 addresses in strict form as defined + in [RFC 6943](https://www.rfc-editor.org/rfc/rfc6943#section-3.1.1). + Specifically, octal numbers in IPv4 addresses are no longer accepted (before + they were parsed as decimal numbers). + ## Standard library additions and changes - `strformat`: From 172253cb551a475d7d32093e562521990e71a1ed Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 14 Sep 2021 11:34:52 -0600 Subject: [PATCH 0403/1654] Dotborrow now works with generic distincts (#18848) --- compiler/semexprs.nim | 4 ++-- compiler/semstmts.nim | 3 ++- tests/borrow/tborrow.nim | 20 ++++++++++++++++++++ tests/borrow/tinvalidborrow.nim | 17 +++++++++++++---- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 41381b057..ebdfe02f0 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1280,7 +1280,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if p != nil and p.selfSym != nil: var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) var check: PNode = nil if ty.kind == tyObject: while true: @@ -1412,7 +1412,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) var check: PNode = nil if ty.kind == tyObject: while true: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 364444c98..c2f05ccce 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1296,7 +1296,8 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) = incl a[2].flags, nfSem # bug #10548 if sfExportc in s.flags and s.typ.kind == tyAlias: localError(c.config, name.info, "{.exportc.} not allowed for type aliases") - if tfBorrowDot in s.typ.flags and s.typ.kind != tyDistinct: + + if tfBorrowDot in s.typ.flags and s.typ.skipTypes({tyGenericBody}).kind != tyDistinct: excl s.typ.flags, tfBorrowDot localError(c.config, name.info, "only a 'distinct' type can borrow `.`") let aa = a[2] diff --git a/tests/borrow/tborrow.nim b/tests/borrow/tborrow.nim index 9c403021e..2fc788fa0 100644 --- a/tests/borrow/tborrow.nim +++ b/tests/borrow/tborrow.nim @@ -33,3 +33,23 @@ let b = Radians(1.0) a -= b echo a.float64 + +block: #14449 + type + Foo[T] = object + foo: T + + Bar[T] {.borrow:`.`.} = distinct Foo[T] + SomeThing {.borrow:`.`.} = distinct Foo[float] + OtherThing {.borrow:`.`.} = distinct SomeThing + + var + a: Bar[int] + b: SomeThing + c: OtherThing + a.foo = 300 + b.foo = 400 + c.foo = 42 + assert a.foo == 300 + assert b.foo == 400d + assert c.foo == 42d \ No newline at end of file diff --git a/tests/borrow/tinvalidborrow.nim b/tests/borrow/tinvalidborrow.nim index 89aa4e2e8..08148608d 100644 --- a/tests/borrow/tinvalidborrow.nim +++ b/tests/borrow/tinvalidborrow.nim @@ -1,16 +1,25 @@ discard """ - errormsg: "no symbol to borrow from found" - line: 11 + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout:''' +tinvalidborrow.nim(18, 3) Error: only a 'distinct' type can borrow `.` +tinvalidborrow.nim(19, 3) Error: only a 'distinct' type can borrow `.` +tinvalidborrow.nim(20, 1) Error: no symbol to borrow from found +''' """ # bug #516 type TAtom = culong - + Test {.borrow:`.`.} = distinct int + Foo[T] = object + a: int + Bar[T] {.borrow:`.`.} = Foo[T] + OtherFoo {.borrow:`.`.} = Foo[int] proc `==`*(a, b: TAtom): bool {.borrow.} var d, e: TAtom -echo( $(d == e) ) +discard( $(d == e) ) From bf1700bab17da3c8776914ac5bdfff016eb70442 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 15 Sep 2021 01:39:55 +0800 Subject: [PATCH 0404/1654] add testcase for #7308 (#18849) --- tests/arc/tarcmisc.nim | 11 ---------- tests/ccgbugs/tctypes.nim | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 tests/ccgbugs/tctypes.nim diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 951ae25f3..7daea62c8 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -463,14 +463,3 @@ proc putValue[T](n: T) = echo b.n useForward() - - -# bug #16246 - -proc testWeirdTypeAliases() = - var values = newSeq[cuint](8) - # var values: seq[cuint] does not produce codegen error - var drawCb = proc(): seq[uint32] = - result = newSeq[uint32](10) - -testWeirdTypeAliases() diff --git a/tests/ccgbugs/tctypes.nim b/tests/ccgbugs/tctypes.nim new file mode 100644 index 000000000..be6009115 --- /dev/null +++ b/tests/ccgbugs/tctypes.nim @@ -0,0 +1,43 @@ +discard """ + targets: "c cpp" + matrix: "--gc:refc; --gc:arc" +""" + +# bug #7308 +proc foo(x: seq[int32]) = + var y = newSeq[cint](1) + +proc bar = + var t = newSeq[int32](1) + foo(t) + +bar() + + +# bug #16246 + +proc testWeirdTypeAliases() = + var values = newSeq[cuint](8) + # var values: seq[cuint] does not produce codegen error + var drawCb = proc(): seq[uint32] = + result = newSeq[uint32](10) + +testWeirdTypeAliases() + +block: # bug #11797 + block: + type cdouble2 = cdouble + type Foo1 = seq[cdouble] + type Foo2 = seq[cdouble2] + static: doAssert Foo1 is Foo2 + var a1: Foo1 + var a2: Foo2 + doAssert a1 == @[] + doAssert a2 == @[] + + block: + proc foo[T: int|cint](fun: proc(): T) = discard + proc foo1(): cint = 1 + proc foo3(): int32 = 2 + foo(proc(): cint = foo1()) + foo(proc(): int32 = foo3()) From cebf7cdc1ede7385e8d774b83c52d17e7021899b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 15 Sep 2021 01:40:42 +0800 Subject: [PATCH 0405/1654] fix #10128 (#18852) --- compiler/ccgexprs.nim | 6 +++--- tests/ccgbugs/t10128.nim | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 tests/ccgbugs/t10128.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 374876367..7a3d769aa 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1006,14 +1006,14 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = # emit range check: if optBoundsCheck in p.options: linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n", - [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! + [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) else: if optBoundsCheck in p.options: linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n", - [rdLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! + [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1.Field0[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) @@ -1028,7 +1028,7 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = if optBoundsCheck in p.options: linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n", - [rdLoc(b), lenExpr(p, a), raiseInstr(p)]) + [rdCharLoc(b), lenExpr(p, a), raiseInstr(p)]) if d.k == locNone: d.storage = OnHeap if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: a.r = ropecg(p.module, "(*$1)", [a.r]) diff --git a/tests/ccgbugs/t10128.nim b/tests/ccgbugs/t10128.nim new file mode 100644 index 000000000..48970916f --- /dev/null +++ b/tests/ccgbugs/t10128.nim @@ -0,0 +1,18 @@ +# bug #10128 +let data = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" +var seq2 = newSeq[char](data.len) +for i in 0.. Date: Thu, 16 Sep 2021 00:48:58 -0600 Subject: [PATCH 0406/1654] Fixed borrowing dot from aliases (#18854) --- compiler/semexprs.nim | 4 ++-- tests/borrow/tborrow.nim | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ebdfe02f0..5a1426f2b 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1280,7 +1280,7 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if p != nil and p.selfSym != nil: var ty = skipTypes(p.selfSym.typ, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyAlias, tySink, tyOwned}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias}) var check: PNode = nil if ty.kind == tyObject: while true: @@ -1412,7 +1412,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) - while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst}) + while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias}) var check: PNode = nil if ty.kind == tyObject: while true: diff --git a/tests/borrow/tborrow.nim b/tests/borrow/tborrow.nim index 2fc788fa0..35652e2e0 100644 --- a/tests/borrow/tborrow.nim +++ b/tests/borrow/tborrow.nim @@ -52,4 +52,40 @@ block: #14449 c.foo = 42 assert a.foo == 300 assert b.foo == 400d - assert c.foo == 42d \ No newline at end of file + assert c.foo == 42d + +block: # Borrow from muliple aliasses #16666 + type + AImpl = object + i: int + + A = AImpl + + B {.borrow: `.`.} = distinct A + C = B + D {.borrow: `.`.} = distinct C + E {.borrow: `.`.} = distinct D + + let + b = default(B) + d = default(D) + e = default(E) + + assert b.i == 0 + assert d.i == 0 + assert e.i == 0 + +block: # Borrow from generic alias + type + AImpl[T] = object + i: T + B[T] = AImpl[T] + C {.borrow: `.`.} = distinct B[int] + D = B[float] + E {.borrow: `.`.} = distinct D + + let + c = default(C) + e = default(E) + assert c.i == int(0) + assert e.i == 0d \ No newline at end of file From 6cd219c3a38c5f1a0712b0b5d07a3ea4ab02ff74 Mon Sep 17 00:00:00 2001 From: alaviss Date: Fri, 17 Sep 2021 04:51:26 +0000 Subject: [PATCH 0407/1654] semtypinst: don't wrap type nodes from expressions in static[T] (#18860) --- compiler/semtypinst.nim | 2 +- tests/generics/t18859.nim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/generics/t18859.nim diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index cb1ead027..2ca355e7d 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -548,7 +548,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = result = n.typ.skipTypes({tyTypeDesc}) # result = n.typ.base else: - if n.typ.kind != tyStatic: + if n.typ.kind != tyStatic and n.kind != nkType: # XXX: In the future, semConstExpr should # return tyStatic values to let anyone make # use of this knowledge. The patching here diff --git a/tests/generics/t18859.nim b/tests/generics/t18859.nim new file mode 100644 index 000000000..ca6c3d10b --- /dev/null +++ b/tests/generics/t18859.nim @@ -0,0 +1,17 @@ +import macros + +macro symFromDesc(T: typedesc): untyped = + let typ = getType(T) + typ[1] + +template produceType(T: typedesc): untyped = + type + XT = object + x: symFromDesc(T) + + XT + +type + X[T] = produceType(T) + +var x: X[int] From ca51bb8fd97e36e001afc7e9ad150dc77e9ca83b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 19 Sep 2021 18:31:55 +0200 Subject: [PATCH 0408/1654] documented overload disambiguation [backport] (#18865) * documented overload disambiguation [backport] * Update doc/manual.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * documented overload disambiguation [backport] * documented overload disambiguation [backport] Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/manual.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index af163bceb..09826441c 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2708,6 +2708,24 @@ a parameter typed as `untyped` (for unresolved expressions) or the type class assert toSeq2(items(@[1,2])) == @[1, 2] # but items(@[1,2]) is +Overload disambiguation +======================= + +For routine calls "overload resolution" is performed. There is a weaker form of +overload resolution called *overload disambiguation* that is performed when an +overloaded symbol is used in a context where there is additional type information +available. Let `p` be an overloaded symbol. These contexts are: + +- In a function call `q(..., p, ...)` when the corresponding formal parameter + of `q` is a `proc` type. If `q` itself is overloaded then the cartesian product + of every interpretation of `q` and `p` must be considered. +- In an object constructor `Obj(..., field: p, ...)` when `field` is a `proc` + type. Analogous rules exist for array/set/tuple constructors. +- In a declaration like `x: T = p` when `T` is a `proc` type. + +As usual, ambiguous matches produce a compile-time error. + + Statements and expressions ========================== @@ -4414,7 +4432,7 @@ would. For example: if n > 0: yield n for e in toItr(recCountDown(n - 1)): - yield e + yield e for i in toItr(recCountDown(6)): # Emits: 6 5 4 3 2 1 echo i From 3241df2a13662e0158ec9d0b0c07996b22fb0939 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 19 Sep 2021 23:35:50 +0200 Subject: [PATCH 0409/1654] fixes #18858 [backport] (#18868) * fixes #18858 [backport] * ensure async tests work with --experimental:strictEffects [backport] * ensure async tests work with --experimental:strictEffects [backport] --- lib/pure/asyncfutures.nim | 2 +- lib/pure/osproc.nim | 6 +++++- lib/pure/unicode.nim | 5 ++++- tests/async/nim.cfg | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/async/nim.cfg diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index af3fbb3ec..6906d03fd 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -93,7 +93,7 @@ proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) = ## Change current implementation of `callSoon`. This is normally called when dispatcher from `asyncdispatcher` is initialized. callSoonProc = p -proc callSoon*(cbproc: proc ()) = +proc callSoon*(cbproc: proc () {.gcsafe.}) = ## Call `cbproc` "soon". ## ## If async dispatcher is running, `cbproc` will be executed during next dispatcher tick. diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 1ac37661b..6a0ac9a8b 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -330,12 +330,16 @@ proc countProcessors*(): int {.rtl, extern: "nosp$1".} = ## It is implemented just calling `cpuinfo.countProcessors`. result = cpuinfo.countProcessors() +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc execProcesses*(cmds: openArray[string], options = {poStdErrToStdOut, poParentStreams}, n = countProcessors(), beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", - tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect].} = + tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], + effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. ## Creates `n` processes that execute in parallel. ## diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index dda631633..660be7814 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -705,8 +705,11 @@ proc capitalize*(s: string): string {.noSideEffect, fastRuneAt(s, i, rune, doInc = true) result = $toUpper(rune) & substr(s, i) +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc translate*(s: string, replacements: proc(key: string): string): string {. - rtl, extern: "nuc$1".} = + rtl, extern: "nuc$1", effectsOf: replacements.} = ## Translates words in a string using the ``replacements`` proc to substitute ## words inside ``s`` with their replacements. ## diff --git a/tests/async/nim.cfg b/tests/async/nim.cfg new file mode 100644 index 000000000..be50f572c --- /dev/null +++ b/tests/async/nim.cfg @@ -0,0 +1 @@ +--experimental:strictEffects From c56ba3f06a3d77ff5eabb3b48ecd6a752e5a3ea8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 20 Sep 2021 09:09:53 +0200 Subject: [PATCH 0410/1654] fixes #18847 [backport] (#18870) --- compiler/parser.nim | 5 +++-- tests/parser/tdoc_comments.nim | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 7d3c913da..5e9a7424a 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1824,8 +1824,9 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = # => moves comment `foo` to `fn` result.comment = body[0].comment body[0].comment = "" - else: - assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. + #else: + # assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. + # Yeah, that worked so well. There IS a bug in this logic, now what? proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT diff --git a/tests/parser/tdoc_comments.nim b/tests/parser/tdoc_comments.nim index fa1374b45..3c86e69ea 100644 --- a/tests/parser/tdoc_comments.nim +++ b/tests/parser/tdoc_comments.nim @@ -69,3 +69,7 @@ type MyEnum3* = enum value5 ## only document the enum value + +# bug #18847 +proc close*() = ## asdfasdfsdfa + discard ## adsfasdfads From ee9795f76f1125e0addbf8cf5471cf188ad7f57a Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 20 Sep 2021 15:50:20 +0800 Subject: [PATCH 0411/1654] fix #18844 (#18851) --- lib/pure/concurrency/atomics.nim | 8 ++++---- tests/stdlib/concurrency/tatomics.nim | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index bdf1e8cc2..b252789fb 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -288,10 +288,10 @@ else: type # Atomic* {.importcpp: "_Atomic('0)".} [T] = object - AtomicInt8 {.importc: "_Atomic NI8", size: 1.} = object - AtomicInt16 {.importc: "_Atomic NI16", size: 2.} = object - AtomicInt32 {.importc: "_Atomic NI32", size: 4.} = object - AtomicInt64 {.importc: "_Atomic NI64", size: 8.} = object + AtomicInt8 {.importc: "_Atomic NI8".} = int8 + AtomicInt16 {.importc: "_Atomic NI16".} = int16 + AtomicInt32 {.importc: "_Atomic NI32".} = int32 + AtomicInt64 {.importc: "_Atomic NI64".} = int64 template atomicType*(T: typedesc[Trivial]): untyped = # Maps the size of a trivial type to it's internal atomic type diff --git a/tests/stdlib/concurrency/tatomics.nim b/tests/stdlib/concurrency/tatomics.nim index 260d00990..beae0ed6d 100644 --- a/tests/stdlib/concurrency/tatomics.nim +++ b/tests/stdlib/concurrency/tatomics.nim @@ -1,6 +1,6 @@ # test atomic operations -import atomics, bitops +import std/[atomics, bitops] type Object = object @@ -607,3 +607,23 @@ block clear: doAssert not location.testAndSet location.clear(moRelease) doAssert not location.testAndSet + +block: # bug #18844 + when not defined(cpp): # cpp pending pr #18836 + type + Deprivation = object of RootObj + memes: Atomic[int] + Zoomer = object + dopamine: Deprivation + + block: + var x = Deprivation() + var y = Zoomer() + doAssert x.memes.load == 0 + doAssert y.dopamine.memes.load == 0 + + block: + var x: Deprivation + var y: Zoomer + doAssert x.memes.load == 0 + doAssert y.dopamine.memes.load == 0 From 14ced06bb1ef7f23fc6391d16763f1421a3edc80 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 20 Sep 2021 11:37:54 +0200 Subject: [PATCH 0412/1654] fixes #18863 [backport] (#18871) --- compiler/semstmts.nim | 2 +- compiler/semtempl.nim | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c2f05ccce..81d6eeda2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1905,7 +1905,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime) if n[patternPos].kind != nkEmpty: - n[patternPos] = semPattern(c, n[patternPos]) + n[patternPos] = semPattern(c, n[patternPos], s) if s.kind == skIterator: s.typ.flags.incl(tfIterator) elif s.kind == skFunc: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 2636784af..eda4ca6cc 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -334,7 +334,7 @@ proc semTemplSomeDecl(c: var TemplCtx, n: PNode, symKind: TSymKind; start = 0) = illFormedAst(a, c.c.config) -proc semPattern(c: PContext, n: PNode): PNode +proc semPattern(c: PContext, n: PNode; s: PSym): PNode proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode = result = n @@ -645,7 +645,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = if allUntyped: incl(s.flags, sfAllUntyped) if n[patternPos].kind != nkEmpty: - n[patternPos] = semPattern(c, n[patternPos]) + n[patternPos] = semPattern(c, n[patternPos], s) var ctx: TemplCtx ctx.toBind = initIntSet() @@ -798,7 +798,7 @@ proc semPatternBody(c: var TemplCtx, n: PNode): PNode = for i in 0.. Date: Mon, 20 Sep 2021 13:24:44 +0200 Subject: [PATCH 0413/1654] [backport] Revert "use new Nimble, with lockfiles (#18810)" (#18872) This reverts commit f373c17ad926b669bb3b5819ae1dff4bde1da88a. For more details, see: - https://github.com/nim-lang/nimble/issues/940 - https://github.com/nim-lang/Nim/issues/18840 - https://forum.nim-lang.org/t/8404 --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index e42ea53de..8dd897028 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "795704833ddfd0cdaefb45c60551d3ea205279ef" # master + NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From 928ea6bb4c0cc791f1b81e55dbebd14d5c6a7315 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 20 Sep 2021 13:32:33 +0200 Subject: [PATCH 0414/1654] atlas: added --cfgHere switch [backport] (#18873) --- tools/atlas/atlas.nim | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim index 370138029..e604c3167 100644 --- a/tools/atlas/atlas.nim +++ b/tools/atlas/atlas.nim @@ -27,6 +27,8 @@ Command: Options: --keepCommits do not perform any `git checkouts` + --cfgHere also create/maintain a nim.cfg in the current + working directory --version show the version --help show this help """ @@ -58,6 +60,7 @@ type projectDir, workspace: string hasPackageList: bool keepCommits: bool + cfgHere: bool p: Table[string, string] # name -> url mapping processed: HashSet[string] # the key is (url / commit) errors: int @@ -353,7 +356,7 @@ proc clone(c: var AtlasContext; start: string): seq[string] = error c, toName(start), "cannot resolve package name" return - c.projectDir = work[0].name.string + c.projectDir = c.workspace / work[0].name.string result = @[] var i = 0 while i < work.len: @@ -376,10 +379,16 @@ const configPatternBegin = "############# begin Atlas config section ##########\n" configPatternEnd = "############# end Atlas config section ##########\n" -proc patchNimCfg(c: var AtlasContext; deps: seq[string]) = +proc patchNimCfg(c: var AtlasContext; deps: seq[string]; cfgHere: bool) = var paths = "--noNimblePath\n" - for d in deps: - paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n" + if cfgHere: + let cwd = getCurrentDir() + for d in deps: + let x = relativePath(c.workspace / d, cwd, '/') + paths.add "--path:\"" & x & "\"\n" + else: + for d in deps: + paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n" var cfgContent = configPatternBegin & paths & configPatternEnd @@ -387,7 +396,8 @@ proc patchNimCfg(c: var AtlasContext; deps: seq[string]) = assert readFile(TestsDir / "nim.cfg") == cfgContent c.mockupSuccess = true else: - let cfg = c.projectDir / "nim.cfg" + let cfg = if cfgHere: getCurrentDir() / "nim.cfg" + else: c.projectDir / "nim.cfg" if not fileExists(cfg): writeFile(cfg, cfgContent) else: @@ -437,6 +447,7 @@ proc main = of "help", "h": writeHelp() of "version", "v": writeVersion() of "keepcommits": c.keepCommits = true + of "cfghere": c.cfgHere = true else: writeHelp() of cmdEnd: assert false, "cannot happen" @@ -449,7 +460,9 @@ proc main = of "clone": singleArg() let deps = clone(c, args[0]) - patchNimCfg c, deps + patchNimCfg c, deps, false + if c.cfgHere: + patchNimCfg c, deps, true when MockupRun: if not c.mockupSuccess: error "There were problems." From 0ad601d3c1657216d27be839173861594e3e2421 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 22 Sep 2021 09:43:06 +0200 Subject: [PATCH 0415/1654] fixes #18856 [backport] (#18879) --- compiler/semexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 5a1426f2b..ee2626922 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1661,7 +1661,7 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = localError(c.config, n.info, errXStackEscape % renderTree(n, {renderNoComments})) else: localError(c.config, n.info, errExprHasNoAddress) - result = newNodeIT(nkHiddenAddr, n.info, makePtrType(c, n.typ)) + result = newNodeIT(nkHiddenAddr, n.info, if n.typ.kind in {tyVar, tyLent}: n.typ else: makePtrType(c, n.typ)) result.add(n) proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = From 6163bdd279a2c33d4290223cc7f466c8070b8ce4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 22 Sep 2021 15:07:36 +0200 Subject: [PATCH 0416/1654] closes #16132 [backport] (#18880) * closes #16132 [backport] * fixes #16132 [backport] --- compiler/semexprs.nim | 13 ++++++++----- tests/views/tviews1.nim | 14 +++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ee2626922..9134c1301 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1667,11 +1667,14 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} = if le.kind == nkHiddenDeref: var x = le[0] - if x.kind == nkSym and x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): - n[0] = x # 'result[]' --> 'result' - n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) - x.typ.flags.incl tfVarIsPtr - #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info + if x.kind == nkSym: + if x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView): + n[0] = x # 'result[]' --> 'result' + n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent) + x.typ.flags.incl tfVarIsPtr + #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info + elif sfGlobal in x.sym.flags: + x.typ.flags.incl tfVarIsPtr proc borrowCheck(c: PContext, n, le, ri: PNode) = const diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 51f17b9d6..49d79c5b5 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -5,7 +5,8 @@ discard """ 3 2 3 -3''' +3 +15''' targets: "c cpp" """ @@ -26,3 +27,14 @@ proc main(s: seq[int]) = take x main(@[11, 22, 33]) + +var x: int + +proc foo(x: var int): var int = + once: x = 42 + return x + +var y: var int = foo(x) +y = 15 +echo foo(x) +# bug #16132 From 7d2a4c0880223c2c80a5c30227c26fa497898f39 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 23 Sep 2021 11:00:36 +0200 Subject: [PATCH 0417/1654] update csources hash (#18885) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 75dbb1da3..aee15e695 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v1 nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git nim_csourcesBranch=master -nim_csourcesHash=9a7f751d23c49c75a0b6f63a234c575dc0df7231 +nim_csourcesHash=5e9be0383f842c9c22ae86e41946a896c43e95d0 From 90a2b5afd8368777e5da9ab97c28130ba683e1d3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 23 Sep 2021 16:47:24 +0200 Subject: [PATCH 0418/1654] correct effect tracking for .borrowed procs [backport] (#18882) * correct effect tracking for .borrowed procs [backport] * progress * fix error message in a test * correctly fix it Co-authored-by: narimiran --- compiler/ast.nim | 2 +- compiler/ccgexprs.nim | 4 ++-- compiler/jsgen.nim | 2 +- compiler/sempass2.nim | 13 +++++++++---- compiler/semstmts.nim | 5 ++++- compiler/vmgen.nim | 2 +- lib/system.nim | 2 +- tests/effects/tgcsafe3.nim | 2 +- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 50d048edd..0a3b1b72d 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -673,7 +673,7 @@ type mSwap, mIsNil, mArrToSeq, mNewString, mNewStringOfCap, mParseBiggestFloat, mMove, mWasMoved, mDestroy, mTrace, - mDefault, mUnown, mIsolate, mAccessEnv, mReset, + mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, mIterableType, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7a3d769aa..ac4a26bd6 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2335,7 +2335,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = genDollar(p, e, d, "#nimFloatToStr($1)") of mCStrToStr: genDollar(p, e, d, "#cstrToNimstr($1)") of mStrToStr, mUnown: expr(p, e[1], d) - of mIsolate: genCall(p, e, d) + of mIsolate, mFinished: genCall(p, e, d) of mEnumToStr: if optTinyRtti in p.config.globalOptions: genEnumToStr(p, e, d) @@ -2929,7 +2929,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if n[genericParamsPos].kind == nkEmpty: var prc = n[namePos].sym if useAliveDataFromDce in p.module.flags: - if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate}: + if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate, mFinished}: genProc(p.module, prc) elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags: if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f009c83f2..ff4d2839e 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -657,7 +657,7 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mIntToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")") of mInt64ToStr: applyFormat("cstrToNimstr(($1) + \"\")", "cstrToNimstr(($1) + \"\")") of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)") - of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1") + of mStrToStr, mUnown, mIsolate, mFinished: applyFormat("$1", "$1") else: assert false, $op diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 6a0e8772a..9a27d14fa 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -245,9 +245,14 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet; conf: Co let msgKind = if onlyWarning: warnGcUnsafe2 else: errGenerated case u.kind of skLet, skVar: - message(conf, s.info, msgKind, - ("'$#' is not GC-safe as it accesses '$#'" & - " which is a global using GC'ed memory") % [s.name.s, u.name.s]) + if u.typ.skipTypes(abstractInst).kind == tyProc: + message(conf, s.info, msgKind, + "'$#' is not GC-safe as it calls '$#'" % + [s.name.s, u.name.s]) + else: + message(conf, s.info, msgKind, + ("'$#' is not GC-safe as it accesses '$#'" & + " which is a global using GC'ed memory") % [s.name.s, u.name.s]) of routineKinds: # recursive call *always* produces only a warning so the full error # message is printed: @@ -850,7 +855,7 @@ proc trackCall(tracked: PEffects; n: PNode) = mergeRaises(tracked, effectList[exceptionEffects], n) mergeTags(tracked, effectList[tagEffects], n) gcsafeAndSideeffectCheck() - if a.kind != nkSym or a.sym.magic != mNBindSym: + if a.kind != nkSym or a.sym.magic notin {mNBindSym, mFinished}: for i in 1.. 0: + s.typ.n[0] = b.typ.n[0] + s.typ.flags = b.typ.flags else: localError(c.config, n.info, errNoSymbolToBorrowFromFound) @@ -1954,7 +1957,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if not hasProto: implicitPragmas(c, s, n.info, validPragmas) - if n[pragmasPos].kind != nkEmpty: + if n[pragmasPos].kind != nkEmpty and sfBorrow notin s.flags: setEffectsForProcType(c.graph, s.typ, n[pragmasPos], s) s.typ.flags.incl tfEffectSystemWorkaround diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index f2758ff37..bb095d3ee 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1029,7 +1029,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = c.genAsgnPatch(n[1], d) c.freeTemp(d) of mOrd, mChr, mArrToSeq, mUnown: c.gen(n[1], dest) - of mIsolate: + of mIsolate, mFinished: genCall(c, n, dest) of mNew, mNewFinalize: unused(c, n, dest) diff --git a/lib/system.nim b/lib/system.nim index c93af0854..a5356cb54 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2453,7 +2453,7 @@ when notJSnotNims: else: {.error: "Only closure function and iterator are allowed!".} - proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = + proc finished*[T: proc](x: T): bool {.noSideEffect, inline, magic: "Finished".} = ## It can be used to determine if a first class iterator has finished. when T is "iterator": {.emit: """ diff --git a/tests/effects/tgcsafe3.nim b/tests/effects/tgcsafe3.nim index 5137efe4c..36ea5112c 100644 --- a/tests/effects/tgcsafe3.nim +++ b/tests/effects/tgcsafe3.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'myproc' is not GC-safe as it accesses 'global_proc' which is a global using GC'ed memory" + errormsg: "'myproc' is not GC-safe as it calls 'global_proc'" line: 12 cmd: "nim $target --hints:on --threads:on $options $file" """ From 561b01004cfeb572f82327cd8a3de55d076cb5ff Mon Sep 17 00:00:00 2001 From: Miran Date: Fri, 24 Sep 2021 11:24:55 +0200 Subject: [PATCH 0419/1654] [backport] change some examples in tut1.rst (#18884) * [backport] change some examples in tut1.rst * Update doc/tut1.rst * Update doc/tut1.rst Co-authored-by: Andreas Rumpf --- doc/tut1.rst | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 210ea324e..f492f12b0 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1643,9 +1643,25 @@ variables! For example: echo badname echo badext +Tuple unpacking is also supported in for-loops: + +.. code-block:: nim + :test: "nim c $1" + let a = [(10, 'a'), (20, 'b'), (30, 'c')] + + for (x, c) in a: + echo x + # This will output: 10; 20; 30 + + # Accessing the index is also possible: + for i, (x, c) in a: + echo i, c + # This will output: 0a; 1b; 2c + Fields of tuples are always public, they don't need to be explicity marked to be exported, unlike for example fields in an object type. + Reference and pointer types --------------------------- @@ -1675,13 +1691,18 @@ operators perform implicit dereferencing operations for reference types: Node = ref object le, ri: Node data: int - var - n: Node - new(n) - n.data = 9 + + var n = Node(data: 9) + echo n.data # no need to write n[].data; in fact n[].data is highly discouraged! -To allocate a new traced object, the built-in procedure `new` must be used. +To allocate a new traced object, the built-in procedure `new` can be used: + +.. code-block:: nim + + var n: Node + new(n) + To deal with untraced memory, the procedures `alloc`, `dealloc` and `realloc` can be used. The `system `_ module's documentation contains further details. @@ -1701,15 +1722,17 @@ Example: .. code-block:: nim :test: "nim c $1" - proc echoItem(x: int) = echo x + proc greet(name: string): string = + "Hello, " & name & "!" - proc forEach(action: proc (x: int)) = - const - data = [2, 3, 5, 7, 11] - for d in items(data): - action(d) + proc bye(name: string): string = + "Goodbye, " & name & "." - forEach(echoItem) + proc communicate(greeting: proc (x: string): string, name: string) = + echo greeting(name) + + communicate(greet, "John") + communicate(bye, "Mary") A subtle issue with procedural types is that the calling convention of the procedure influences the type compatibility: procedural types are only compatible From 5d315ebcc2d4f46b4a74c6ab10146466c894b9de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 24 Sep 2021 13:24:41 +0200 Subject: [PATCH 0420/1654] ported Nim to proprietary CrossOS [backport] (#18889) --- ci/funs.sh | 2 ++ compiler/installer.ini | 1 + compiler/options.nim | 6 ++++-- compiler/platform.nim | 6 +++++- lib/pure/os.nim | 2 +- tools/niminst/buildsh.nimf | 5 +++++ 6 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ci/funs.sh b/ci/funs.sh index e25922c8c..1638931c6 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -74,6 +74,8 @@ _nimBuildCsourcesIfNeeded(){ makeX=gmake elif [ "$unamestr" = 'NetBSD' ]; then makeX=gmake + elif [ "$unamestr" = 'CROSSOS' ]; then + makeX=gmake else makeX=make fi diff --git a/compiler/installer.ini b/compiler/installer.ini index 3f1630a92..9e31b4e3f 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -13,6 +13,7 @@ Platforms: """ netbsd: i386;amd64 openbsd: i386;amd64;arm;arm64 dragonfly: i386;amd64 + crossos: amd64 haiku: i386;amd64 android: i386;arm;arm64 nintendoswitch: arm64 diff --git a/compiler/options.nim b/compiler/options.nim index ea302aed6..eafcd816d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -577,11 +577,13 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osQnx, osAtari, osAix, osHaiku, osVxWorks, osSolaris, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos, - osAndroid, osNintendoSwitch, osFreeRTOS} + osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos} of "linux": result = conf.target.targetOS in {osLinux, osAndroid} of "bsd": - result = conf.target.targetOS in {osNetbsd, osFreebsd, osOpenbsd, osDragonfly} + result = conf.target.targetOS in {osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos} + of "freebsd": + result = conf.target.targetOS in {osFreebsd, osCrossos} of "emulatedthreadvars": result = platform.OS[conf.target.targetOS].props.contains(ospLacksThreadVars) of "msdos": result = conf.target.targetOS == osDos diff --git a/compiler/platform.nim b/compiler/platform.nim index 7bc77d809..1bc00b629 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -20,7 +20,7 @@ type TSystemOS* = enum # Also add OS in initialization section and alias # conditionals to condsyms (end of module). osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris, - osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx, + osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx, osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny @@ -105,6 +105,10 @@ const scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", props: {ospNeedsPIC, ospPosix}), + (name: "CROSSOS", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", + objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", + scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", + props: {ospNeedsPIC, ospPosix}), (name: "AIX", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", diff --git a/lib/pure/os.nim b/lib/pure/os.nim index cfdacdb3c..dd1f9f0b6 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -3007,7 +3007,7 @@ when not weirdTarget and (defined(freebsd) or defined(dragonfly) or defined(netb proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t, newp: pointer, newplen: csize_t): cint {.importc: "sysctl",header: """#include - #include """} + #include """.} const CTL_KERN = 1 KERN_PROC = 14 diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index f74413c09..7b30cd86c 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -109,6 +109,11 @@ case $uos in CC="clang" LINK_FLAGS="$LINK_FLAGS -lm" ;; + *crossos* ) + myos="crossos" + CC="clang" + LINK_FLAGS="$LINK_FLAGS -lm" + ;; *openbsd* ) myos="openbsd" CC="clang" From f7d642f2f3925feb5addc315eb59caf326cad470 Mon Sep 17 00:00:00 2001 From: Clyybber Date: Fri, 24 Sep 2021 14:59:48 +0200 Subject: [PATCH 0421/1654] [backport] arc: improve compile time of (nested) loops (#18890) --- compiler/dfa.nim | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 65c483e9c..0539f6699 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -308,6 +308,11 @@ when true: # We unroll every loop 3 times. We emulate 0, 1, 2 iterations # through the loop. We need to prove this is correct for our # purposes. But Herb Sutter claims it is. (Proof by authority.) + # + # EDIT: Actually, we only need to unroll 2 times + # because Nim doesn't have a way of breaking/goto-ing into + # a loop iteration. Unrolling 2 times is much better for compile + # times of nested loops than 3 times, so we do that here. #[ while cond: body @@ -347,12 +352,12 @@ when true: # 'while true' is an idiom in Nim and so we produce # better code for it: withBlock(nil): - for i in 0..2: + for i in 0..1: c.gen(n[1]) else: withBlock(nil): - var endings: array[3, TPosition] - for i in 0..2: + var endings: array[2, TPosition] + for i in 0..1: c.gen(n[0]) endings[i] = c.forkI(n) c.gen(n[1]) From 7e5eab571e6e8e57928b40f535802aec04e04633 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 24 Sep 2021 16:27:34 +0200 Subject: [PATCH 0422/1654] closes #18690; make view types stricter [backport] (#18891) * closes #18690 * don't allow capturing of view types [backport] --- compiler/lambdalifting.nim | 6 ++---- tests/views/tviews1.nim | 13 ++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index c555cedfe..a622f6de6 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -12,7 +12,7 @@ import intsets, strutils, options, ast, astalgo, msgs, idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos, - transf, liftdestructors + transf, liftdestructors, typeallowed discard """ The basic approach is that captured vars need to be put on the heap and @@ -191,9 +191,7 @@ proc interestingVar(s: PSym): bool {.inline.} = s.typ.kind notin {tyStatic, tyTypeDesc} proc illegalCapture(s: PSym): bool {.inline.} = - result = skipTypes(s.typ, abstractInst).kind in - {tyVar, tyOpenArray, tyVarargs, tyLent} or - s.kind == skResult + result = classifyViewType(s.typ) != noView or s.kind == skResult proc isInnerProc(s: PSym): bool = if s.kind in {skProc, skFunc, skMethod, skConverter, skIterator} and s.magic == mNone: diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 49d79c5b5..3dbf664fc 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -6,7 +6,8 @@ discard """ 2 3 3 -15''' +15 +(oa: [1, 3, 4])''' targets: "c cpp" """ @@ -38,3 +39,13 @@ var y: var int = foo(x) y = 15 echo foo(x) # bug #16132 + +# bug #18690 + +type + F = object + oa: openarray[int] + +let s1 = @[1,3,4,5,6] +var test = F(oa: toOpenArray(s1, 0, 2)) +echo test From 4adada0d8016016104d6259a1167a7f8b37f1611 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 24 Sep 2021 22:34:12 +0800 Subject: [PATCH 0423/1654] fix a typo (rename `temporal` to `temporary` (#18892) [backport] --- lib/pure/strutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 3179c73d2..873949ca6 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2039,7 +2039,7 @@ func countLines*(s: string): int {.rtl, extern: "nsuCountLines".} = ## Returns the number of lines in the string `s`. ## ## This is the same as `len(splitLines(s))`, but much more efficient - ## because it doesn't modify the string creating temporal objects. Every + ## because it doesn't modify the string creating temporary objects. Every ## `character literal `_ ## newline combination (CR, LF, CR-LF) is supported. ## From 8bdb98539570fdf3c41ee5509d5b90913ac1803e Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 25 Sep 2021 19:22:00 +0800 Subject: [PATCH 0424/1654] fix wrong name (rnimsyn => renderer; pnimsyn => parser; scanner => lexer) (#18895) * fix wrong module name * rephrase more word --- compiler/filter_tmpl.nim | 2 +- compiler/lexer.nim | 4 ++-- compiler/nimconf.nim | 2 +- compiler/renderer.nim | 4 ++-- doc/manual.rst | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim index 6165ff2f3..84ed99164 100644 --- a/compiler/filter_tmpl.nim +++ b/compiler/filter_tmpl.nim @@ -22,7 +22,7 @@ type info: TLineInfo indent, emitPar: int x: string # the current input line - outp: PLLStream # the output will be parsed by pnimsyn + outp: PLLStream # the output will be parsed by parser subsChar, nimDirective: char emit, conc, toStr: string curly, bracket, par: int diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ede16fdf1..506b0e924 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -7,12 +7,12 @@ # distribution, for details about the copyright. # -# This scanner is handwritten for efficiency. I used an elegant buffering +# This lexer is handwritten for efficiency. I used an elegant buffering # scheme which I have not seen anywhere else: # We guarantee that a whole line is in the buffer. Thus only when scanning # the \n or \r character we have to check whether we need to read in the next # chunk. (\n or \r already need special handling for incrementing the line -# counter; choosing both \n and \r allows the scanner to properly read Unix, +# counter; choosing both \n and \r allows the lexer to properly read Unix, # DOS or Macintosh text files, even when it is not the native format. import diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 94c93a283..1cf22e20a 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -14,7 +14,7 @@ import options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig # ---------------- configuration file parser ----------------------------- -# we use Nim's scanner here to save space and work +# we use Nim's lexer here to save space and work proc ppGetTok(L: var Lexer, tok: var Token) = # simple filter diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 393f35289..22a2d4cbd 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -410,7 +410,7 @@ proc atom(g: TSrcGen; n: PNode): string = if (n.typ != nil) and (n.typ.sym != nil): result = n.typ.sym.name.s else: result = "[type node]" else: - internalError(g.config, "rnimsyn.atom " & $n.kind) + internalError(g.config, "renderer.atom " & $n.kind) result = "" proc lcomma(g: TSrcGen; n: PNode, start: int = 0, theEnd: int = - 1): int = @@ -1699,7 +1699,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = gsub(g, n[0], c) else: #nkNone, nkExplicitTypeListCall: - internalError(g.config, n.info, "rnimsyn.gsub(" & $n.kind & ')') + internalError(g.config, n.info, "renderer.gsub(" & $n.kind & ')') proc renderTree*(n: PNode, renderFlags: TRenderFlags = {}): string = if n == nil: return "" diff --git a/doc/manual.rst b/doc/manual.rst index 09826441c..dfcf0ba54 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -216,7 +216,7 @@ comment: .. code-block:: nim i = 0 # This is a single comment over multiple lines. - # The scanner merges these two pieces. + # The lexer merges these two pieces. # The comment continues here. From e3b19cbe52e0578de315d259fbb066bfe991a60f Mon Sep 17 00:00:00 2001 From: Aditya Siram Date: Sat, 25 Sep 2021 07:17:41 -0500 Subject: [PATCH 0425/1654] fixes #18878 (#18883) --- lib/core/typeinfo.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index d66dc1898..18d2091b9 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -216,7 +216,8 @@ proc extendSeq*(x: Any) = when defined(gcDestructors): var s = cast[ptr NimSeqV2Reimpl](x.value) let elem = x.rawType.base - s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAdd(s.len, s.p, 1, elem.size, elem.align)) + if s.p == nil or s.p.cap < s.len+1: + s.p = cast[ptr NimSeqPayloadReimpl](prepareSeqAdd(s.len, s.p, 1, elem.size, elem.align)) inc s.len else: var y = cast[ptr PGenSeq](x.value)[] From 57eb763c65c2d517e2b6ca7b2ca2d3bb5ab6c3d5 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 27 Sep 2021 19:13:02 +0800 Subject: [PATCH 0426/1654] activate winim (#18907) `winim` is dependent on windows API. --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 657ddb2d1..0c47104ac 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -159,7 +159,7 @@ pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" pkg "weave", "nimble test_gc_arc", allowFailure = true pkg "websocket", "nim c websocket.nim" -pkg "winim", allowFailure = true +pkg "winim", "nim c winim.nim" pkg "with" pkg "ws", allowFailure = true pkg "yaml", "nim c -r test/tserialization.nim" From 5325a366e79b6e34fcd2b2dfe675897212f8ca2d Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 27 Sep 2021 16:35:14 +0100 Subject: [PATCH 0427/1654] Add armv8l support (Closes: #18898) (#18901) [backport] --- tools/niminst/buildsh.nimf | 2 +- tools/niminst/makefile.nimf | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 7b30cd86c..9850abd89 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -213,7 +213,7 @@ case $ucpu in mycpu="alpha" ;; *aarch64*|*arm64* ) mycpu="arm64" ;; - *arm*|*armv6l*|*armv71* ) + *arm*|*armv6l*|*armv7l*|*armv8l* ) mycpu="arm" ;; *riscv64|riscv* ) mycpu="riscv64" ;; diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index a0e9a40f7..69e87b837 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -157,6 +157,9 @@ endif ifeq ($(ucpu),armv7hl) mycpu = arm endif +ifeq ($(ucpu),armv8l) + mycpu = arm +endif ifeq ($(ucpu),aarch64) mycpu = arm64 endif From cdf9ac675b4348a7b5239186637c54920bb92619 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 27 Sep 2021 19:25:00 +0200 Subject: [PATCH 0428/1654] this ensures libp2p continues to compile [backport] (#18908) --- compiler/cgen.nim | 11 ++++++++--- compiler/guards.nim | 10 ++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index a06a87172..43410dc60 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -855,10 +855,15 @@ proc closureSetup(p: BProc, prc: PSym) = [rdLoc(env.loc), getTypeDesc(p.module, env.typ)]) proc containsResult(n: PNode): bool = - if n.kind == nkSym and n.sym.kind == skResult: - result = true + result = false + case n.kind + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkFormalParams: + discard + of nkSym: + if n.sym.kind == skResult: + result = true else: - for i in 0.. Date: Mon, 27 Sep 2021 22:23:31 +0200 Subject: [PATCH 0429/1654] fixes 'lent T' inside object constructor [backport] (#18911) * fixes 'lent T' inside object constructor [backport] * progress --- compiler/sem.nim | 3 ++- compiler/semobjconstr.nim | 2 +- compiler/semstmts.nim | 2 +- tests/views/tviews1.nim | 17 +++++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index bdecbe602..70c57864c 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -107,12 +107,13 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = else: result = fitNodePostMatch(c, formal, result) -proc fitNodeForLocalVar(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = +proc fitNodeConsiderViewType(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode = let a = fitNode(c, formal, arg, info) if formal.kind in {tyVar, tyLent}: #classifyViewType(formal) != noView: result = newNodeIT(nkHiddenAddr, a.info, formal) result.add a + formal.flags.incl tfVarIsPtr else: result = a diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 1563e5c16..001956ede 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -79,7 +79,7 @@ proc semConstrField(c: PContext, flags: TExprFlags, var initValue = semExprFlagDispatched(c, assignment[1], flags) if initValue != nil: - initValue = fitNode(c, field.typ, initValue, assignment.info) + initValue = fitNodeConsiderViewType(c, field.typ, initValue, assignment.info) assignment[0] = newSymNode(field) assignment[1] = initValue assignment.flags.incl nfSem diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3b699f6c8..4607e857a 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -545,7 +545,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = else: # BUGFIX: ``fitNode`` is needed here! # check type compatibility between def.typ and typ - def = fitNodeForLocalVar(c, typ, def, def.info) + def = fitNodeConsiderViewType(c, typ, def, def.info) #changeType(def.skipConv, typ, check=true) else: typ = def.typ.skipTypes({tyStatic, tySink}).skipIntLit(c.idgen) diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 3dbf664fc..ced487ce8 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -49,3 +49,20 @@ type let s1 = @[1,3,4,5,6] var test = F(oa: toOpenArray(s1, 0, 2)) echo test + +type + Foo = object + x: string + y: seq[int] + data: array[10000, byte] + + View[T] = object + x: lent T + +proc mainB = + let f = Foo(y: @[1, 2, 3]) + let foo = View[Foo](x: f) + assert foo.x.x == "" + assert foo.x.y == @[1, 2, 3] + +mainB() From 5eb357f5a2db81d7027eaf5430cda74370e45a67 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 28 Sep 2021 12:45:25 +0800 Subject: [PATCH 0430/1654] change comment (#18913) --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index a5356cb54..358aae938 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -160,7 +160,7 @@ when defined(nimHashOrdinalFixed): ## as well as their subtypes. See also ## `SomeOrdinal`. else: - # bootstrap <= 0.20.0 + # bootstrap < 1.2.0 type OrdinalImpl[T] {.magic: Ordinal.} Ordinal* = OrdinalImpl | uint | uint64 From 56bd5cbb877b69f5c4a49470845621267a316f91 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 28 Sep 2021 15:05:22 +0800 Subject: [PATCH 0431/1654] fix a typo (canonilization => canonicalization) (#18914) --- compiler/patterns.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 1ef903161..4b39de3ba 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -36,7 +36,7 @@ proc putLazy(c: PPatternContext, sym: PSym, n: PNode) = proc matches(c: PPatternContext, p, n: PNode): bool proc canonKind(n: PNode): TNodeKind = - ## nodekind canonilization for pattern matching + ## nodekind canonicalization for pattern matching result = n.kind case result of nkCallKinds: result = nkCall From 08cf4cb1e58fad35be3846e98a308500ac2cfdb0 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 28 Sep 2021 17:35:04 +0800 Subject: [PATCH 0432/1654] test ord (#18909) --- compiler/semmagic.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 87f7da815..874098294 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -216,10 +216,6 @@ proc semOrd(c: PContext, n: PNode): PNode = let parType = n[1].typ if isOrdinalType(parType, allowEnumWithHoles=true): discard - elif parType.kind == tySet: - let a = toInt64(firstOrd(c.config, parType)) - let b = toInt64(lastOrd(c.config, parType)) - result.typ = makeRangeType(c, a, b, n.info) else: localError(c.config, n.info, errOrdinalTypeExpected) result.typ = errorType(c) From f061971a9be5222f0f158f605220f3bda42f7488 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 29 Sep 2021 00:32:39 -0700 Subject: [PATCH 0433/1654] envPairs works in vm, nims (#18615) * envPairs works in vm, nims * fixup --- compiler/vmops.nim | 14 +- lib/pure/includes/osenv.nim | 349 ++++++++++++++-------------- lib/pure/os.nim | 3 +- testament/lib/stdtest/testutils.nim | 7 +- tests/js/tos.nim | 28 +-- tests/stdlib/tosenv.nim | 18 +- tests/test_nimscript.nims | 4 + 7 files changed, 213 insertions(+), 210 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index e287dd41d..b9801234d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -13,7 +13,7 @@ from std/math import sqrt, ln, log10, log2, exp, round, arccos, arcsin, arctan, arctan2, cos, cosh, hypot, sinh, sin, tan, tanh, pow, trunc, floor, ceil, `mod`, cbrt, arcsinh, arccosh, arctanh, erf, erfc, gamma, lgamma - +from std/sequtils import toSeq when declared(math.copySign): # pending bug #18762, avoid renaming math from std/math as math2 import copySign @@ -22,8 +22,8 @@ when declared(math.signbit): # ditto from std/math as math3 import signbit -from std/os import getEnv, existsEnv, delEnv, putEnv, dirExists, fileExists, walkDir, - getAppFilename, raiseOSError, osLastError +from std/os import getEnv, existsEnv, delEnv, putEnv, envPairs, + dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError from std/md5 import getMD5 from std/times import cpuTime @@ -156,6 +156,12 @@ proc stackTrace2(c: PCtx, msg: string, n: PNode) = stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info) proc registerAdditionalOps*(c: PCtx) = + + template wrapIterator(fqname: string, iter: untyped) = + registerCallback c, fqname, proc(a: VmArgs) = + setResult(a, toLit(toSeq(iter))) + + proc gorgeExWrapper(a: VmArgs) = let ret = opGorge(getString(a, 0), getString(a, 1), getString(a, 2), a.currentLineInfo, c.config) @@ -341,3 +347,5 @@ proc registerAdditionalOps*(c: PCtx) = let p = a.getVar(0) let x = a.getFloat(1) addFloatSprintf(p.strVal, x) + + wrapIterator("stdlib.os.envPairsImplSeq"): envPairs() diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 6aaafbfda..00a82327c 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -3,189 +3,200 @@ when not declared(os) and not declared(ospaths): {.error: "This is an include file for os.nim!".} -when defined(nodejs): - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - var ret = default.cstring - let key2 = key.cstring - {.emit: "const value = process.env[`key2`];".} - {.emit: "if (value !== undefined) { `ret` = value };".} - result = $ret +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - var key2 = key.cstring - var ret: bool - {.emit: "`ret` = `key2` in process.env;".} - result = ret + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - var val2 = val.cstring - {.emit: "process.env[`key2`] = `val2`;".} + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - var key2 = key.cstring - {.emit: "delete process.env[`key2`];".} + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - var num: int - var keys: RootObj - {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} - for i in 0..".} - when defined(windows): - proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} - from std/private/win_setenv import setEnvImpl else: - proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} - proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} - proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = - ## Returns the value of the `environment variable`:idx: named `key`. - ## - ## If the variable does not exist, `""` is returned. To distinguish - ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. - ## - ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert getEnv("unknownEnv") == "" - assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - - let env = c_getenv(key) - if env == nil: return default - result = $env - - proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = - ## Checks whether the environment variable named `key` exists. - ## Returns true if it exists, false otherwise. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - runnableExamples: - assert not existsEnv("unknownEnv") - - return c_getenv(key) != nil - - proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = - ## Sets the value of the `environment variable`:idx: named `key` to `val`. - ## If an error occurs, `OSError` is raised. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + proc c_getenv(env: cstring): cstring {. + importc: "getenv", header: "".} when defined(windows): - if key.len == 0 or '=' in key: - raise newException(OSError, "invalid key, got: " & $(key, val)) - if setEnvImpl(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError(), $(key, val)) + proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + from std/private/win_setenv import setEnvImpl else: - if c_setenv(key, val, 1'i32) != 0'i32: - raiseOSError(osLastError(), $(key, val)) + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} - proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = - ## Deletes the `environment variable`:idx: named `key`. - ## If an error occurs, `OSError` is raised. - ## - ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ - template bail = raiseOSError(osLastError(), key) - when defined(windows): - #[ - # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 - > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string - note that nil is not legal - ]# - if key.len == 0 or '=' in key: - raise newException(OSError, "invalid key, got: " & key) - if c_putenv_s(key, "") != 0'i32: bail - else: - if c_unsetenv(key) != 0'i32: bail + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc <#existsEnv,string>`_. + ## + ## See also: + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" - when defined(windows): - when useWinUnicode: - when defined(cpp): - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", - header: "".} + let env = c_getenv(key) + if env == nil: return default + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + return c_getenv(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) else: - proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", - header: "".} - else: - proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", - header: "".} - elif defined(macosx) and not defined(ios) and not defined(emscripten): - # From the manual: - # Shared libraries and bundles don't have direct access to environ, - # which is only available to the loader ld(1) when a complete program - # is being linked. - # The environment routines can still be used, but if direct access to - # environ is needed, the _NSGetEnviron() routine, defined in - # , can be used to retrieve the address of environ - # at runtime. - proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", - header: "".} - elif defined(haiku): - var gEnv {.importc: "environ", header: "".}: cstringArray - else: - var gEnv {.importc: "environ".}: cstringArray + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `envPairs iterator <#envPairs.i>`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + if c_putenv_s(key, "") != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail - iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = - ## Iterate over all `environments variables`:idx:. - ## - ## In the first component of the tuple is the name of the current variable stored, - ## in the second its value. - ## - ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ when defined(windows): - block: - template impl(get_fun, typ, size, zero, free_fun) = - let env = get_fun() - var e = env - if e == nil: break - while true: - let eend = strEnd(e) - let kv = $e - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[ByteAddress](eend)+size) - if typeof(zero)(eend[1]) == zero: break - discard free_fun(env) - when useWinUnicode: - impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + when useWinUnicode: + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "".} else: - impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "".} + else: + proc strEnd(cstr: cstring, c = 0'i32): cstring {.importc: "strchr", + header: "".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # , can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "".}: cstringArray else: - var i = 0 - when defined(macosx) and not defined(ios) and not defined(emscripten): - var gEnv = NSGetEnviron()[] - while gEnv[i] != nil: - let kv = $gEnv[i] - inc(i) - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + block: + template impl(get_fun, typ, size, zero, free_fun) = + let env = get_fun() + var e = env + if e == nil: break + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[typ](cast[ByteAddress](eend)+size) + if typeof(zero)(eend[1]) == zero: break + discard free_fun(env) + when useWinUnicode: + impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + else: + impl(getEnvironmentStringsA, cstring, 1, '\0', freeEnvironmentStringsA) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc <#getEnv,string,string>`_ + ## * `existsEnv proc <#existsEnv,string>`_ + ## * `putEnv proc <#putEnv,string,string>`_ + ## * `delEnv proc <#delEnv,string>`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/lib/pure/os.nim b/lib/pure/os.nim index dd1f9f0b6..e3b8e9f1c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -871,8 +871,7 @@ proc unixToNativePath*(path: string, drive=""): string {. inc(i) include "includes/oserr" -when not defined(nimscript): - include "includes/osenv" +include "includes/osenv" proc getHomeDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index a5476b8d7..12e6a56ab 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -1,5 +1,8 @@ import std/private/miscdollars -from std/os import getEnv +when defined(nimscript): + import std/os # xxx investigate why needed +else: + from std/os import getEnv import std/[macros, genasts] template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = @@ -26,7 +29,7 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = msg2.add $expr & " " & msg echo msg2 -when not defined(js): +when not defined(js) and not defined(nimscript): import std/strutils proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool = diff --git a/tests/js/tos.nim b/tests/js/tos.nim index 07eb3aaa3..bfe3cd9b4 100644 --- a/tests/js/tos.nim +++ b/tests/js/tos.nim @@ -1,3 +1,5 @@ +# xxx consider merging this in tests/stdlib/tos.nim for increased coverage (with selecting disabling) + static: doAssert defined(nodejs) import os @@ -19,29 +21,3 @@ block: if not isWindows: doAssert cwd.isAbsolute doAssert relativePath(getCurrentDir() / "foo", "bar") == "../foo" - -import std/sequtils - -template main = - putEnv("foo", "bar") - doAssert getEnv("foo") == "bar" - doAssert existsEnv("foo") - - putEnv("foo", "") - doAssert existsEnv("foo") - putEnv("foo", "bar2") - doAssert getEnv("foo") == "bar2" - - when nimvm: - discard - else: - # need support in vmops: envPairs, delEnv - let s = toSeq(envPairs()) - doAssert ("foo", "bar2") in s - doAssert ("foo", "bar") notin s - - delEnv("foo") - doAssert not existsEnv("foo") - -static: main() -main() diff --git a/tests/stdlib/tosenv.nim b/tests/stdlib/tosenv.nim index df5759d0c..0a50031a1 100644 --- a/tests/stdlib/tosenv.nim +++ b/tests/stdlib/tosenv.nim @@ -13,16 +13,18 @@ template main = for val in ["val", ""]: # ensures empty val works too const key = "NIM_TESTS_TOSENV_KEY" doAssert not existsEnv(key) - putEnv(key, val) + + putEnv(key, "tempval") + doAssert existsEnv(key) + doAssert getEnv(key) == "tempval" + + putEnv(key, val) # change a key that already exists doAssert existsEnv(key) doAssert getEnv(key) == val - when nimvm: discard - else: - doAssert (key, val) in toSeq(envPairs()) + + doAssert (key, val) in toSeq(envPairs()) delEnv(key) - when nimvm: discard - else: - doAssert (key, val) notin toSeq(envPairs()) + doAssert (key, val) notin toSeq(envPairs()) doAssert not existsEnv(key) delEnv(key) # deleting an already deleted env var doAssert not existsEnv(key) @@ -43,7 +45,7 @@ template main = static: main() main() -when not defined(js): +when not defined(js) and not defined(nimscript): block: # bug #18533 proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} var thr: Thread[void] diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 90b6181aa..ecb409677 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -71,7 +71,11 @@ import std/[ decls, compilesettings, with, wrapnils ] +# non-std imports +import stdtest/testutils +# tests (increase coverage via code reuse) import stdlib/trandom +import stdlib/tosenv echo "Nimscript imports are successful." From b74b914e4f36199797b77e98bd2d771b4056a0d6 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 29 Sep 2021 11:14:47 +0200 Subject: [PATCH 0434/1654] [backport] use old style hints in .cfg files (#18917) refs #18085 --- compiler/nim.cfg | 4 ++-- compiler/nimfix/nimfix.nim.cfg | 2 +- config/nim.cfg | 4 ++-- drnim/nim.cfg | 2 +- nimsuggest/nimsuggest.nim.cfg | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/nim.cfg b/compiler/nim.cfg index d6ac6a937..4a0287cb5 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -1,6 +1,6 @@ # Special configuration file for the Nim project -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off define:booting define:nimcore @@ -23,7 +23,7 @@ define:useStdoutAsStdmsg #gc:markAndSweep @if nimHasWarningObservableStores: - warning:ObservableStores: off + warning[ObservableStores]: off @end @if nimHasEffectsOf: diff --git a/compiler/nimfix/nimfix.nim.cfg b/compiler/nimfix/nimfix.nim.cfg index 27ebf10b9..0d9dbfa4b 100644 --- a/compiler/nimfix/nimfix.nim.cfg +++ b/compiler/nimfix/nimfix.nim.cfg @@ -1,7 +1,7 @@ # Special configuration file for the Nim project # gc:markAndSweep -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off path:"$projectPath/.." path:"$lib/packages/docutils" diff --git a/config/nim.cfg b/config/nim.cfg index c0a844e03..3b964d124 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -13,8 +13,8 @@ cc = gcc # additional options always passed to the compiler: --parallel_build: "0" # 0 to auto-detect number of processors -hint:LineTooLong:off -#hint:XDeclaredButNotUsed:off +hint[LineTooLong]=off +#hint[XDeclaredButNotUsed]=off # Examples of how to setup a cross-compiler: diff --git a/drnim/nim.cfg b/drnim/nim.cfg index 57dca974c..58c1725d9 100644 --- a/drnim/nim.cfg +++ b/drnim/nim.cfg @@ -1,6 +1,6 @@ # Special configuration file for the Nim project -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off define:booting define:nimcore diff --git a/nimsuggest/nimsuggest.nim.cfg b/nimsuggest/nimsuggest.nim.cfg index 6e5ee8c71..394449740 100644 --- a/nimsuggest/nimsuggest.nim.cfg +++ b/nimsuggest/nimsuggest.nim.cfg @@ -2,7 +2,7 @@ gc:markAndSweep -hint:XDeclaredButNotUsed:off +hint[XDeclaredButNotUsed]:off path:"$lib/packages/docutils" From f35e9a5752bb4cd4e30105cd27b6bc56cbf22329 Mon Sep 17 00:00:00 2001 From: Jonas Schubert Erlandsson Date: Wed, 29 Sep 2021 19:14:31 +0300 Subject: [PATCH 0435/1654] Adds source to single tags list (#18920) --- lib/pure/htmlparser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index c572146a7..21dff69ff 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -215,7 +215,7 @@ const tagMenu, tagNoframes} SingleTags* = {tagArea, tagBase, tagBasefont, tagBr, tagCol, tagFrame, tagHr, tagImg, tagIsindex, - tagLink, tagMeta, tagParam, tagWbr} + tagLink, tagMeta, tagParam, tagWbr, tagSource} proc allLower(s: string): bool = for c in s: From 6bb32da4aecbe80a9cc0cce90322c0e741ee8b85 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 30 Sep 2021 13:46:30 +0800 Subject: [PATCH 0436/1654] [std/tasks] more gcsafe stuffs [backport] (#18926) --- lib/std/tasks.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index 97c4df3f7..e2ea5377f 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -61,12 +61,12 @@ type Task* = object ## `Task` contains the callback and its arguments. callback: proc (args: pointer) {.nimcall, gcsafe.} args: pointer - destroy: proc (args: pointer) {.nimcall.} + destroy: proc (args: pointer) {.nimcall, gcsafe.} proc `=copy`*(x: var Task, y: Task) {.error.} -proc `=destroy`*(t: var Task) {.inline.} = +proc `=destroy`*(t: var Task) {.inline, gcsafe.} = ## Frees the resources allocated for a `Task`. if t.args != nil: if t.destroy != nil: @@ -219,7 +219,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC let `objTemp` = cast[ptr `scratchObjType`](args) `functionStmtList` - proc `destroyName`(args: pointer) {.nimcall.} = + proc `destroyName`(args: pointer) {.gcsafe, nimcall.} = let `objTemp2` = cast[ptr `scratchObjType`](args) `tempNode` From c3dd53bb2cf492029ae256f64f64d2577621b995 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 30 Sep 2021 11:35:42 +0200 Subject: [PATCH 0437/1654] typos [backport] (#18929) --- doc/destructors.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 28d8fa5f9..195dc8f97 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -145,7 +145,7 @@ not free the resources afterward by setting the object to its default value default value is written as `wasMoved(x)`. When not provided the compiler is using a combination of `=destroy` and `copyMem` instead. This is efficient hence users rarely need to implement their own `=sink` operator, it is enough to -provide `=destroy` and `=copy`, compiler will take care of the rest. +provide `=destroy` and `=copy`, the compiler will take care of the rest. The prototype of this hook for a type `T` needs to be: @@ -169,7 +169,7 @@ How self-assignments are handled is explained later in this document. `=copy` hook ---------------- +------------ The ordinary assignment in Nim conceptually copies the values. The `=copy` hook is called for assignments that couldn't be transformed into `=sink` @@ -206,11 +206,11 @@ by the compiler. Notice that there is no `=` before the `{.error.}` pragma. `=trace` hook ---------------- +------------- A custom **container** type can support Nim's cycle collector `--gc:orc` via the `=trace` hook. If the container does not implement `=trace`, cyclic data -structure which are constructed with the help of the container might leak +structures which are constructed with the help of the container might leak memory or resources, but memory safety is not compromised. The prototype of this hook for a type `T` needs to be: @@ -225,10 +225,10 @@ to calls of the built-in `=trace` operation. Usually there will only be a need for a custom `=trace` when a custom `=destroy` that deallocates manually allocated resources is also used, and then only when there is a chance of cyclic references from items within the manually allocated resources when it is desired that `--gc:orc` -be able to break and collect these cyclic referenced resources. Currently however, there is a +is able to break and collect these cyclic referenced resources. Currently however, there is a mutual use problem in that whichever of `=destroy`/`=trace` is used first will automatically create a version of the other which will then conflict with the creation of the second of the -pair. The work around for this problem is to forward declare the second of the "hooks" to +pair. The workaround for this problem is to forward declare the second of the "hooks" to prevent the automatic creation. The general pattern in using `=destroy` with `=trace` looks like: @@ -250,8 +250,9 @@ The general pattern in using `=destroy` with `=trace` looks like: dest.arr.dealloc proc `=trace`[T](dest: var Test[T]; env: pointer) = - if dest.arr != nil: # trace the `T`'s which may be cyclic - for i in 0 ..< dest.size: dest.arr[i].`=trace` env + if dest.arr != nil: + # trace the `T`'s which may be cyclic + for i in 0 ..< dest.size: `=trace`(dest.arr[i], env) # following may be other custom "hooks" as required... @@ -265,7 +266,7 @@ Move semantics A "move" can be regarded as an optimized copy operation. If the source of the copy operation is not used afterward, the copy can be replaced by a move. This document uses the notation `lastReadOf(x)` to describe that `x` is not -used afterwards. This property is computed by a static control flow analysis +used afterward. This property is computed by a static control flow analysis but can also be enforced by using `system.move` explicitly. @@ -601,7 +602,7 @@ a form of copy elision. To see how and when we can do that, think about this question: In `dest = src` when do we really have to *materialize* the full copy? - Only if `dest` or `src` are mutated -afterwards. If `dest` is a local variable that is simple to analyze. And if `src` is a +afterward. If `dest` is a local variable that is simple to analyze. And if `src` is a location derived from a formal parameter, we also know it is not mutated! In other words, we do a compile-time copy-on-write analysis. From c38ab3e257fbb08622f9ec6f0068a0fd3a5d82d7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 30 Sep 2021 12:09:42 +0200 Subject: [PATCH 0438/1654] fixes #18921 [backport] (#18930) --- compiler/commands.nim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 9b7e35791..a8caad916 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -482,18 +482,20 @@ proc setCommandEarly*(conf: ConfigRef, command: string) = else: conf.foreignPackageNotes = foreignPackageNotesDefault -proc specialDefine(conf: ConfigRef, key: string) = +proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) = # Keep this syncronized with the default config/nim.cfg! if cmpIgnoreStyle(key, "nimQuirky") == 0: conf.exc = excQuirky elif cmpIgnoreStyle(key, "release") == 0 or cmpIgnoreStyle(key, "danger") == 0: - conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize} - conf.globalOptions.excl {optExcessiveStackTrace, optCDebug} - conf.options.incl optOptimizeSpeed + if pass in {passCmd1, passPP}: + conf.options.excl {optStackTrace, optLineTrace, optLineDir, optOptimizeSize} + conf.globalOptions.excl {optExcessiveStackTrace, optCDebug} + conf.options.incl optOptimizeSpeed if cmpIgnoreStyle(key, "danger") == 0 or cmpIgnoreStyle(key, "quick") == 0: - conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, - optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} - conf.globalOptions.excl {optCDebug} + if pass in {passCmd1, passPP}: + conf.options.excl {optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck, + optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} + conf.globalOptions.excl {optCDebug} proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = @@ -565,10 +567,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectArg(conf, switch, arg, pass, info) if {':', '='} in arg: splitSwitch(conf, arg, key, val, pass, info) - specialDefine(conf, key) + specialDefine(conf, key, pass) defineSymbol(conf.symbols, key, val) else: - specialDefine(conf, arg) + specialDefine(conf, arg, pass) defineSymbol(conf.symbols, arg) of "undef", "u": expectArg(conf, switch, arg, pass, info) From f915b3aa86929d87e162b6ddd7589e6337a30397 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Thu, 30 Sep 2021 08:55:43 -0600 Subject: [PATCH 0439/1654] Alternative to #18928 (#18931) * fixed #18841 * Added test --- compiler/semtypinst.nim | 12 +++---- tests/generics/trecursivegenerics.nim | 52 ++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 2ca355e7d..0bb53e04d 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -344,11 +344,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = x = lookupTypeVar(cl, x) if x != nil: if header == t: header = instCopyType(cl, t) - header[i] = - if x.kind == tyGenericInst: - t[i] - else: - x + header[i] = x propagateToOwner(header, x) else: propagateToOwner(header, x) @@ -382,7 +378,11 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType = cl.typeMap = newTypeMapLayer(cl) for i in 1.. Date: Thu, 30 Sep 2021 22:57:06 +0800 Subject: [PATCH 0440/1654] alternative to #18918 (#18927) * fix #16558 * add testcase --- compiler/semexprs.nim | 2 ++ tests/arc/t16558.nim | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/arc/t16558.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 9134c1301..342b8b05f 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -217,6 +217,8 @@ proc isCastable(c: PContext; dst, src: PType): bool = let s = skipTypes(src, abstractInst) if d.kind == tyRef and s.kind == tyRef and s[0].isFinal != d[0].isFinal: return false + elif d.kind in IntegralTypes and s.kind in {tyString, tySequence}: + return false var dstSize, srcSize: BiggestInt dstSize = computeSize(conf, dst) diff --git a/tests/arc/t16558.nim b/tests/arc/t16558.nim new file mode 100644 index 000000000..7b6eb4669 --- /dev/null +++ b/tests/arc/t16558.nim @@ -0,0 +1,9 @@ +discard """ + matrix: "--gc:arc" + errormsg: "expression cannot be cast to int" +""" + +block: # bug #16558 + var value = "hi there" + var keepInt: int + keepInt = cast[int](value) From 4b764f4c7cff02b1f698d90d4605ae8fa85c3ee6 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 1 Oct 2021 16:19:02 +0800 Subject: [PATCH 0441/1654] close #17319; add testcase (#18934) --- tests/arc/tarcmisc.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 7daea62c8..d259e1197 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -463,3 +463,22 @@ proc putValue[T](n: T) = echo b.n useForward() + + +# bug #17319 +type + BrokenObject = ref object + brokenType: seq[int] + +proc use(obj: BrokenObject) = + discard + +method testMethod(self: BrokenObject) {.base.} = + iterator testMethodIter() {.closure.} = + use(self) + + var nameIterVar = testMethodIter + nameIterVar() + +let mikasa = BrokenObject() +mikasa.testMethod() From 2aeac26f088c5170f7f0c4d8e73d099a32e8895a Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 1 Oct 2021 19:57:06 +0800 Subject: [PATCH 0442/1654] correct licence header (#18935) * rename licence * spaces --- lib/pure/lexbase.nim | 2 +- lib/std/compilesettings.nim | 2 +- lib/std/setutils.nim | 2 +- lib/std/sha1.nim | 2 +- lib/std/strbasics.nim | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index 166d8286b..bbc0a38ae 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2009 Andreas Rumpf # # See the file "copying.txt", included in this diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index e1ffc954a..5bb0bd673 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this diff --git a/lib/std/setutils.nim b/lib/std/setutils.nim index 84cc93623..4664d6dcc 100644 --- a/lib/std/setutils.nim +++ b/lib/std/setutils.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2020 Nim Contributors # # See the file "copying.txt", included in this diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 3cc8a0249..120a81702 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2015 Nim Contributors # # See the file "copying.txt", included in this diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index 9232204ab..0d4c96f44 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -1,6 +1,6 @@ # # -# The Nim Compiler +# Nim's Runtime Library # (c) Copyright 2021 Nim Contributors # # See the file "copying.txt", included in this From 7577ea9e4ce4d9966edc1f1cc27bde1b2ec9f369 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 2 Oct 2021 02:14:10 +0800 Subject: [PATCH 0443/1654] [std/tempfiles] docs improvement (#18936) * unify comments * more --- lib/std/tempfiles.nim | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/std/tempfiles.nim b/lib/std/tempfiles.nim index 26786d463..ce84c2a37 100644 --- a/lib/std/tempfiles.nim +++ b/lib/std/tempfiles.nim @@ -122,23 +122,24 @@ proc getTempDirImpl(dir: string): string {.inline.} = proc genTempPath*(prefix, suffix: string, dir = ""): string = ## Generates a path name in `dir`. ## - ## If `dir` is empty, (`getTempDir `_) will be used. ## The path begins with `prefix` and ends with `suffix`. + ## + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir `_). let dir = getTempDirImpl(dir) result = dir / (prefix & randomPathName(nimTempPathLength) & suffix) proc createTempFile*(prefix, suffix: string, dir = ""): tuple[cfile: File, path: string] = ## Creates a new temporary file in the directory `dir`. - ## + ## ## This generates a path name using `genTempPath(prefix, suffix, dir)` and ## returns a file handle to an open file and the path of that file, possibly after ## retrying to ensure it doesn't already exist. - ## + ## ## If failing to create a temporary file, `OSError` will be raised. ## ## .. note:: It is the caller's responsibility to close `result.cfile` and ## remove `result.file` when no longer needed. - ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir()`). + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir `_). runnableExamples: import std/os doAssertRaises(OSError): discard createTempFile("", "", "nonexistent") @@ -170,7 +171,7 @@ proc createTempDir*(prefix, suffix: string, dir = ""): string = ## If failing to create a temporary directory, `OSError` will be raised. ## ## .. note:: It is the caller's responsibility to remove the directory when no longer needed. - ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir()`). + ## .. note:: `dir` must exist (empty `dir` will resolve to `getTempDir `_). runnableExamples: import std/os doAssertRaises(OSError): discard createTempDir("", "", "nonexistent") From 5c4692fad4c807a010d58d553e92360128699197 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 2 Oct 2021 02:26:29 +0800 Subject: [PATCH 0444/1654] rename nimLegacyJsonutilsHoleyEnum [backport] (#18938) * enable nimPreviewJsonutilsHoleyEnum [backport] * docuement nimPreviewJsonutilsHoleyEnum --- changelog.md | 3 +-- lib/std/jsonutils.nim | 6 +++++- tests/config.nims | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index ab037c328..38ace19ed 100644 --- a/changelog.md +++ b/changelog.md @@ -77,8 +77,7 @@ initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition time, see PR #17467. -- `jsonutils` now serializes/deserializes holey enums as regular enums (via `ord`) instead of as strings. - Use `-d:nimLegacyJsonutilsHoleyEnum` for a transition period. `toJson` now serializes `JsonNode` +- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. It is expected that this behavior becomes the new default in upcoming versions. `toJson` now serializes `JsonNode` as is via reference (without a deep copy) instead of treating `JsonNode` as a regular ref object, this can be customized via `jsonNodeMode`. diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 727a75288..19384b5d1 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -298,6 +298,10 @@ proc jsonTo*(b: JsonNode, T: typedesc, opt = Joptions()): T = proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to ## customize serialization, see strtabs.toJsonHook for an example. + ## + ## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can + ## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. + ## It is expected that this behavior becomes the new default in upcoming versions. when compiles(toJsonHook(a)): result = toJsonHook(a) elif T is object | tuple: when T is object or isNamedTuple(T): @@ -328,7 +332,7 @@ proc toJson*[T](a: T, opt = initToJsonOptions()): JsonNode = elif T is enum: case opt.enumMode of joptEnumOrd: - when T is Ordinal or not defined(nimLegacyJsonutilsHoleyEnum): %(a.ord) + when T is Ordinal or defined(nimPreviewJsonutilsHoleyEnum): %(a.ord) else: toJson($a, opt) of joptEnumSymbol: when T is OrdinalEnum: diff --git a/tests/config.nims b/tests/config.nims index 86787db1d..ed2a1b7e9 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -30,10 +30,12 @@ hint("Processing", off) # switch("hint", "ConvFromXtoItselfNotNeeded") # switch("warningAsError", "InheritFromException") # would require fixing a few tests -# experimental API's are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 +# experimental APIs are enabled in testament, refs https://github.com/timotheecour/Nim/issues/575 # sync with `kochdocs.docDefines` or refactor. switch("define", "nimExperimentalAsyncjsThen") switch("define", "nimExperimentalLinenoiseExtra") +# preview APIs are expected to be the new default in upcoming versions switch("define", "nimPreviewFloatRoundtrip") switch("define", "nimPreviewDotLikeOps") +switch("define", "nimPreviewJsonutilsHoleyEnum") From e9268b52d9a34e7e851105d569bccced4b7261fd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Oct 2021 07:21:08 +0200 Subject: [PATCH 0445/1654] with this patch :idx: can be used for the index generation for LaTeX (#18946) --- config/nimdoc.tex.cfg | 6 ++++-- lib/packages/docutils/rstgen.nim | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 75382ce26..4aff9b379 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -3,7 +3,7 @@ # (c) 2012 Andreas Rumpf # Feel free to edit the templates as you need. -split.item.toc = "20" +split.item.toc = "20" # too long entries in the table of contents wrap around # after this number of characters @@ -97,12 +97,14 @@ doc.file = """ \usepackage{parskip} % paragraphs delimited by vertical space, no indent \usepackage{graphicx} +\newcommand{\nimindexterm}[2]{#2\label{#1}} + \usepackage{dingbat} % for \carriagereturn, etc \usepackage{fvextra} % for code blocks (works better than original fancyvrb) \fvset{ breaklines, breakafter={=}:|\_\{\}[](){,}.;+-*/'", - breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, + breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, breaksymbolright=\color{red}{\small\carriagereturn} } \fvinlineset{% diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 3734392c3..7d7eb4fb8 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -406,7 +406,7 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) - dispA(d.target, result, "$2", "$2\\label{$1}", + dispA(d.target, result, "$2", "\\nimindexterm{$2}{$1}", [id, term]) type From 45c4332c81521224a4db90fe9bf46efc0f93aaf7 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 3 Oct 2021 13:22:53 +0800 Subject: [PATCH 0446/1654] document -d:nimStrictDelete [backport] (#18939) --- lib/system.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 358aae938..caa036f37 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2142,6 +2142,10 @@ proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect, auditDelete.} = ## ## This is an `O(n)` operation. ## + ## .. note:: With `-d:nimStrictDelete`, an index error is produced when the index passed + ## to it was out of bounds. `-d:nimStrictDelete` will become the default + ## in upcoming versions. + ## ## See also: ## * `del <#del,seq[T],Natural>`_ for O(1) operation ## From 6b0db5be1f21cec1b75822c021ccee79035a6f74 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 3 Oct 2021 06:25:15 +0100 Subject: [PATCH 0447/1654] Fixes build_all.sh failing on Nimble builds (#18945) When running `build_all.sh` I was getting: ``` bin/nim c -o:bin/nimsuggest -d:danger --skipUserCfg --skipParentCfg --hints:off nimsuggest/nimsuggest.nim bin/nim c -o:bin/nimgrep -d:release --skipUserCfg --skipParentCfg --hints:off tools/nimgrep.nim bin/nim c -o:bin/nimpretty -d:release --skipUserCfg --skipParentCfg --hints:off nimpretty/nimpretty.nim bin/nim c -o:bin/testament -d:release --skipUserCfg --skipParentCfg --hints:off testament/testament.nim bin/nim c -o:bin/nim_dbg --opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints --skipUserCfg --skipParentCfg --hints:off compiler/nim.nim bin/nim c -o:bin/atlas -d:release --skipUserCfg --skipParentCfg --hints:off tools/atlas/atlas.nim /home/dom/.choosenim/toolchains/nim-#devel/koch.nim(722) koch /home/dom/.choosenim/toolchains/nim-#devel/koch.nim(149) bundleNimbleExe /home/dom/.choosenim/toolchains/nim-#devel/tools/deps.nim(32) cloneDependency /home/dom/.choosenim/toolchains/nim-#devel/lib/pure/os.nim(1438) setCurrentDir /home/dom/.choosenim/toolchains/nim-#devel/lib/pure/includes/oserr.nim(95) raiseOSError Error: unhandled exception: No such file or directory Additional info: '/home/dom/.choosenim/toolchains/nim-#devel/dist/nimble' [OSError] ``` With this patch it builds. --- tools/deps.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/deps.nim b/tools/deps.nim index 95b2916a1..59872070d 100644 --- a/tools/deps.nim +++ b/tools/deps.nim @@ -22,14 +22,14 @@ proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, let name = p.splitFile.name var destDir = destDirBase if appendRepoName: destDir = destDir / name - let destDir2 = destDir.quoteShell + let quotedDestDir = destDir.quoteShell if not dirExists(destDir): # note: old code used `destDir / .git` but that wouldn't prevent git clone # from failing - execRetry fmt"git clone -q {url} {destDir2}" + execRetry fmt"git clone -q {url} {quotedDestDir}" if isGitRepo(destDir): let oldDir = getCurrentDir() - setCurrentDir(destDir2) + setCurrentDir(destDir) try: execRetry "git fetch -q" exec fmt"git checkout -q {commit}" From 97c24dd54892a1f256707bc73230fe5443be36ce Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 4 Oct 2021 15:19:20 +0800 Subject: [PATCH 0448/1654] correct changelog [backport] (#18940) --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 38ace19ed..f8dc9087c 100644 --- a/changelog.md +++ b/changelog.md @@ -103,7 +103,7 @@ - In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. -- With `-d:nimPreviewDotLikeOps` (default in devel), dot-like operators (operators starting with `.`, but not with `..`) +- With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`, but not with `..`) now have the same precedence as `.`, so that `a.?b.c` is now parsed as `(a.?b).c` instead of `a.?(b.c)`. A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`. From 500d5f0f38dc59895908f25f2c43c2a4c61d7874 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Oct 2021 12:34:29 +0200 Subject: [PATCH 0449/1654] typo [backport] (#18948) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index dfcf0ba54..4b212fe19 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1201,7 +1201,7 @@ The only operations that are affected by the `floatChecks` pragma are the `+`, `-`, `*`, `/` operators for floating-point types. An implementation should always use the maximum precision available to evaluate -floating pointer values during semantic analysis; this means expressions like +floating-point values during semantic analysis; this means expressions like `0.09'f32 + 0.01'f32 == 0.09'f64 + 0.01'f64` that are evaluating during constant folding are true. From f017eadc97b041e678196cb2b72c3c66767e7ab0 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 4 Oct 2021 15:27:13 -0400 Subject: [PATCH 0450/1654] Add support for `strictEffects` to `std/pegs` (#18949) * Add support for `strictEffects` to `std/pegs` * Fixed support of older Nim versions Co-authored-by: quantimnot --- lib/pure/pegs.nim | 5 ++++- tests/stdlib/tpegs.nim | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index b6b05cdc1..11270e34c 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1285,9 +1285,12 @@ proc parallelReplace*(s: string, subs: varargs[ # copy the rest: add(result, substr(s, i)) +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} + proc replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. - rtl, extern: "npegs$1cb".} = + rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. ## The callback proc receives the index of the current match (starting with 0), ## the count of captures and an open array with the captures of each match. Examples: diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index 1261d55b8..c3d8942cf 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -51,6 +51,9 @@ Event parser output ''' """ +when defined(nimHasEffectsOf): + {.experimental: "strictEffects".} + import std/[strutils, streams, pegs] const From e7bac9177335c75b4eeed18ec5f8a0b0489c0561 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 5 Oct 2021 03:28:13 +0800 Subject: [PATCH 0451/1654] rename nimEnableHashRef [backport] (#18941) * rename nimEnableHashRef [backport] * Apply suggestions from code review Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 3 ++- lib/pure/hashes.nim | 18 +++++++++++++----- tests/collections/ttables.nim | 1 - tests/config.nims | 1 + tests/stdlib/thashes.nim | 1 - 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/changelog.md b/changelog.md index f8dc9087c..1925878bf 100644 --- a/changelog.md +++ b/changelog.md @@ -64,7 +64,8 @@ for previous behavior. - `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), - if `-d:nimEnableHashRef` is used. + if `-d:nimPreviewHashRef` is used. It is expected that this behavior + becomes the new default in upcoming versions. - `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, `hashes.hash(closure)` has also been improved. diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 1c49b0317..d9f534670 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -51,8 +51,12 @@ runnableExamples: h = h !& hash(x.bar) result = !$h -## **Note:** If the type has a `==` operator, the following must hold: -## If two values compare equal, their hashes must also be equal. +## .. important:: Use `-d:nimPreviewHashRef` to +## enable hashing `ref`s. It is expected that this behavior +## becomes the new default in upcoming versions. +## +## .. note:: If the type has a `==` operator, the following must hold: +## If two values compare equal, their hashes must also be equal. ## ## See also ## ======== @@ -236,10 +240,14 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} = assert cast[pointer](a[0].addr).hash == a[0].addr.hash hash(cast[pointer](x)) -when defined(nimEnableHashRef): +when defined(nimPreviewHashRef) or defined(nimdoc): proc hash*[T](x: ref[T]): Hash {.inline.} = ## Efficient `hash` overload. - runnableExamples: + ## + ## .. important:: Use `-d:nimPreviewHashRef` to + ## enable hashing `ref`s. It is expected that this behavior + ## becomes the new default in upcoming versions. + runnableExamples("-d:nimPreviewHashRef"): type A = ref object x: int let a = A(x: 3) @@ -247,7 +255,7 @@ when defined(nimEnableHashRef): assert ha != A(x: 3).hash # A(x: 3) is a different ref object from `a`. a.x = 4 assert ha == a.hash # the hash only depends on the address - runnableExamples: + runnableExamples("-d:nimPreviewHashRef"): # you can overload `hash` if you want to customize semantics type A[T] = ref object x, y: T diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 751509062..638f4241b 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -8,7 +8,6 @@ And we get here ''' joinable: false targets: "c cpp js" -matrix: "-d:nimEnableHashRef" """ # xxx wrap in a template to test in VM, see https://github.com/timotheecour/Nim/issues/534#issuecomment-769565033 diff --git a/tests/config.nims b/tests/config.nims index ed2a1b7e9..894c4bea0 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -39,3 +39,4 @@ switch("define", "nimExperimentalLinenoiseExtra") switch("define", "nimPreviewFloatRoundtrip") switch("define", "nimPreviewDotLikeOps") switch("define", "nimPreviewJsonutilsHoleyEnum") +switch("define", "nimPreviewHashRef") diff --git a/tests/stdlib/thashes.nim b/tests/stdlib/thashes.nim index 1e9b02d0c..46576ef12 100644 --- a/tests/stdlib/thashes.nim +++ b/tests/stdlib/thashes.nim @@ -1,6 +1,5 @@ discard """ targets: "c cpp js" - matrix: "-d:nimEnableHashRef" """ import std/hashes From f1f1e85ec673b9ea2ea5d3a97aa27993be153602 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Oct 2021 12:29:59 +0200 Subject: [PATCH 0452/1654] fixes #18954 (#18955) --- lib/pure/sugar.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/sugar.nim b/lib/pure/sugar.nim index f82e1706f..d7beef8d9 100644 --- a/lib/pure/sugar.nim +++ b/lib/pure/sugar.nim @@ -121,9 +121,9 @@ macro `=>`*(p, b: untyped): untyped = else: error("Incorrect procedure parameter.", c) params.add(identDefs) - of nnkIdent: + of nnkIdent, nnkOpenSymChoice, nnkClosedSymChoice, nnkSym: var identDefs = newNimNode(nnkIdentDefs) - identDefs.add(p) + identDefs.add(ident $p) identDefs.add(ident"auto") identDefs.add(newEmptyNode()) params.add(identDefs) From 9fc2918314a75fd1923af049966e4c4e9d8cf710 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Wed, 6 Oct 2021 03:41:42 -0400 Subject: [PATCH 0453/1654] Document how to compile a reproducible build (#18953) * Make compiler build more reproducible Removed the compile time from the version output. Added Git commit's author's datetime in UTC timezone. Fixes #18508 See https://reproducible-builds.org/docs/timestamps/ See https://reproducible-builds.org/docs/source-date-epoch/ * Revert "Make compiler build more reproducible" This reverts commit 2f9359ae89622c2d95ef243d13251a310e94a199. Documented how to use compile reproducible builds with the `SOURCE_DATE_EPOCH` environment variable. * Corrected `readme.md` formatting Co-authored-by: quantimnot --- doc/intern.rst | 11 +++++++++++ doc/packaging.rst | 4 ++++ readme.md | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/intern.rst b/doc/intern.rst index ab74c4a3a..fe0d333fd 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -68,6 +68,17 @@ More information about its options can be found in the `koch `_ documentation. +Reproducible builds +------------------- + +Set the compilation timestamp with the `SOURCE_DATE_EPOCH` environment variable. + +.. code:: cmd + + export SOURCE_DATE_EPOCH=$(git log -n 1 --format=%at) + koch boot # or `./build_all.sh` + + Developing the compiler ======================= diff --git a/doc/packaging.rst b/doc/packaging.rst index fcdc4ae04..7976dfe92 100644 --- a/doc/packaging.rst +++ b/doc/packaging.rst @@ -6,6 +6,9 @@ This page provide hints on distributing Nim using OS packages. See `distros `_ for tools to detect Linux distribution at runtime. +See `here `_ for how to +compile reproducible builds. + Supported architectures ----------------------- @@ -71,3 +74,4 @@ What to install: - Optionally: manpages, documentation, shell completion - When installing documentation, .idx files are not required - The "compiler" directory contains compiler sources and should not be part of the compiler binary package + diff --git a/readme.md b/readme.md index b28c25ec1..7f333d5d4 100644 --- a/readme.md +++ b/readme.md @@ -71,7 +71,6 @@ the installation instructions on the website to do so: https://nim-lang.org/inst For package maintainers: see [packaging guidelines](https://nim-lang.github.io/Nim/packaging.html). - First, get Nim from github: ``` @@ -89,6 +88,8 @@ should add the ``bin`` directory to your PATH. See also [rebuilding the compiler](doc/intern.rst#rebuilding-the-compiler). +See also [reproducible builds](doc/intern.rst#reproducible-builds). + ## Koch ``koch`` is the build tool used to build various parts of Nim and to generate From cb30dc51701f0d4bc5dedc2615de68e4c1e5ba71 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 6 Oct 2021 15:42:35 +0800 Subject: [PATCH 0454/1654] enable tests for #2710 (#18961) --- tests/metatype/ttypedesc3.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/metatype/ttypedesc3.nim b/tests/metatype/ttypedesc3.nim index b691d3488..98a59f613 100644 --- a/tests/metatype/ttypedesc3.nim +++ b/tests/metatype/ttypedesc3.nim @@ -3,6 +3,7 @@ output: ''' proc Base proc Child method Base +method Child yield Base yield Child 12 @@ -24,8 +25,7 @@ Base.pr Child.pr Base.me -when false: - Child.me #<- bug #2710 +Child.me #<- bug #2710 for s in Base.it: echo s for s in Child.it: echo s #<- bug #2662 From b2873f0f63f7741ee995363a6bb225a228d03d18 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 6 Oct 2021 18:47:46 +0800 Subject: [PATCH 0455/1654] [tools] use the right parameter [backport:1.0] (#18957) --- tools/finish.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/finish.nim b/tools/finish.nim index 4b5131045..69838829c 100644 --- a/tools/finish.nim +++ b/tools/finish.nim @@ -41,7 +41,7 @@ proc downloadMingw(): DownloadResult = let curl = findExe"curl" var cmd: string if curl.len > 0: - cmd = quoteShell(curl) & " --out " & "dist" / mingw & " " & url + cmd = quoteShell(curl) & " --output " & "dist" / mingw & " " & url elif fileExists"bin/nimgrab.exe": cmd = r"bin\nimgrab.exe " & url & " dist" / mingw if cmd.len > 0: From f03872d99ec2d25de829415a75162f9d8297bb19 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 6 Oct 2021 14:53:01 +0300 Subject: [PATCH 0456/1654] rst: minor fixes (#18960) --- lib/packages/docutils/rst.nim | 1 + lib/packages/docutils/rstast.nim | 4 ++-- lib/packages/docutils/rstgen.nim | 2 +- tests/stdlib/trst.nim | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index abea026dd..29234f28b 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -172,6 +172,7 @@ ## ``header``, ``footer``, ``meta``, ``class`` ## - no ``role`` directives and no custom interpreted text roles ## - some standard roles are not supported (check `RST roles list`_) +## - no generic admonition support ## * inline markup ## - no simple-inline-markup ## - no embedded aliases diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index fa0620f44..e1ed7c099 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -381,7 +381,7 @@ proc renderRstToText*(node: PRstNode): string = result.add renderRstToText(node.sons[i]) if node.kind in code: result.add "`" -proc renderRstToStr*(node: PRstNode, indent=0): string = +proc treeRepr*(node: PRstNode, indent=0): string = ## Writes the parsed RST `node` into an AST tree with compact string ## representation in the format (one line per every sub-node): ## ``indent - kind - [text|level|order|adType] - anchor (if non-zero)`` @@ -411,4 +411,4 @@ proc renderRstToStr*(node: PRstNode, indent=0): string = result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") result.add "\n" for son in node.sons: - result.add renderRstToStr(son, indent=indent+2) + result.add treeRepr(son, indent=indent+2) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 7d7eb4fb8..8eac37307 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1164,7 +1164,7 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = case n.adType of "hint", "note", "tip": htmlCls = "admonition-info"; texSz = "\\normalsize"; texColor = "green" - of "attention", "admonition", "important", "warning": + of "attention", "admonition", "important", "warning", "caution": htmlCls = "admonition-warning"; texSz = "\\large"; texColor = "orange" of "danger", "error": htmlCls = "admonition-error"; texSz = "\\Large"; texColor = "red" diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index bc11f219c..e9f67324c 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -55,7 +55,7 @@ proc toAst(input: string, var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, rstOptions, myFindFile, testMsgHandler) - result = renderRstToStr(rst) + result = treeRepr(rst) except EParseError as e: if e.msg != "": result = e.msg From eede2bfb5de30f7f411d9437e19f253f8dac629b Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 7 Oct 2021 10:48:09 +0200 Subject: [PATCH 0457/1654] update csources hash (#18969) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index aee15e695..15f44bb57 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v1 nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git nim_csourcesBranch=master -nim_csourcesHash=5e9be0383f842c9c22ae86e41946a896c43e95d0 +nim_csourcesHash=561b417c65791cd8356b5f73620914ceff845d10 From 8eef55715725a9dba2ff0afbdbb564383f3a29a6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Oct 2021 12:01:06 +0200 Subject: [PATCH 0458/1654] fixes a 'mixin' statement handling regression [backport:1.2] (#18968) --- compiler/semtempl.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index eda4ca6cc..c10917bcb 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -110,9 +110,14 @@ proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode = proc semMixinStmt(c: PContext, n: PNode, toMixin: var IntSet): PNode = result = copyNode(n) + var count = 0 for i in 0.. Date: Thu, 7 Oct 2021 18:01:27 +0800 Subject: [PATCH 0459/1654] typo [backport] (#18967) --- lib/pure/strscans.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/strscans.nim b/lib/pure/strscans.nim index 291af7408..99442075a 100644 --- a/lib/pure/strscans.nim +++ b/lib/pure/strscans.nim @@ -35,7 +35,7 @@ substrings starting with ``$``. These constructions are available: ``$o`` Matches an octal integer. This uses ``parseutils.parseOct``. ``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``. ``$h`` Matches a hex integer. This uses ``parseutils.parseHex``. -``$f`` Matches a floating pointer number. Uses ``parseFloat``. +``$f`` Matches a floating-point number. Uses ``parseFloat``. ``$w`` Matches an ASCII identifier: ``[A-Za-z_][A-Za-z_0-9]*``. ``$c`` Matches a single ASCII character. ``$s`` Skips optional whitespace. From 19774a72e7fcc7c5b7649bb956b149362c05ea15 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Thu, 7 Oct 2021 06:16:58 -0400 Subject: [PATCH 0460/1654] Fixed `strictFuncs` support for `std/pegs` (#18951) * Fixed `strictFuncs` support for `std/pegs` Enabled `std/pegs` in the `strictFuncs` import test. Fixes #18057 Fixes #16892 See #18111 * Rebased from `devel` * Conditionally compile `std/pegs` in `koch` This is for supporting `csources` bootstrap. Co-authored-by: quantimnot --- lib/pure/pegs.nim | 280 ++++++++++++------------ tests/effects/tstrict_funcs_imports.nim | 2 +- tools/kochdocs.nim | 11 +- 3 files changed, 151 insertions(+), 142 deletions(-) diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 11270e34c..bac8b1a0e 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -87,26 +87,26 @@ type else: sons: seq[Peg] NonTerminal* = ref NonTerminalObj -proc kind*(p: Peg): PegKind = p.kind +func kind*(p: Peg): PegKind = p.kind ## Returns the *PegKind* of a given *Peg* object. -proc term*(p: Peg): string = p.term +func term*(p: Peg): string = p.term ## Returns the *string* representation of a given *Peg* variant object ## where present. -proc ch*(p: Peg): char = p.ch +func ch*(p: Peg): char = p.ch ## Returns the *char* representation of a given *Peg* variant object ## where present. -proc charChoice*(p: Peg): ref set[char] = p.charChoice +func charChoice*(p: Peg): ref set[char] = p.charChoice ## Returns the *charChoice* field of a given *Peg* variant object ## where present. -proc nt*(p: Peg): NonTerminal = p.nt +func nt*(p: Peg): NonTerminal = p.nt ## Returns the *NonTerminal* object of a given *Peg* variant object ## where present. -proc index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index +func index*(p: Peg): range[-MaxSubpatterns..MaxSubpatterns-1] = p.index ## Returns the back-reference index of a captured sub-pattern in the ## *Captures* object for a given *Peg* variant object where present. @@ -120,59 +120,59 @@ iterator pairs*(p: Peg): (int, Peg) {.inline.} = for i in 0 ..< p.sons.len: yield (i, p.sons[i]) -proc name*(nt: NonTerminal): string = nt.name +func name*(nt: NonTerminal): string = nt.name ## Gets the name of the symbol represented by the parent *Peg* object variant ## of a given *NonTerminal*. -proc line*(nt: NonTerminal): int = nt.line +func line*(nt: NonTerminal): int = nt.line ## Gets the line number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc col*(nt: NonTerminal): int = nt.col +func col*(nt: NonTerminal): int = nt.col ## Gets the column number of the definition of the parent *Peg* object variant ## of a given *NonTerminal*. -proc flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags +func flags*(nt: NonTerminal): set[NonTerminalFlag] = nt.flags ## Gets the *NonTerminalFlag*-typed flags field of the parent *Peg* variant ## object of a given *NonTerminal*. -proc rule*(nt: NonTerminal): Peg = nt.rule +func rule*(nt: NonTerminal): Peg = nt.rule ## Gets the *Peg* object representing the rule definition of the parent *Peg* ## object variant of a given *NonTerminal*. -proc term*(t: string): Peg {.noSideEffect, rtl, extern: "npegs$1Str".} = +func term*(t: string): Peg {.rtl, extern: "npegs$1Str".} = ## constructs a PEG from a terminal string if t.len != 1: result = Peg(kind: pkTerminal, term: t) else: result = Peg(kind: pkChar, ch: t[0]) -proc termIgnoreCase*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreCase*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore case for matching result = Peg(kind: pkTerminalIgnoreCase, term: t) -proc termIgnoreStyle*(t: string): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func termIgnoreStyle*(t: string): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG from a terminal string; ignore style for matching result = Peg(kind: pkTerminalIgnoreStyle, term: t) -proc term*(t: char): Peg {.noSideEffect, rtl, extern: "npegs$1Char".} = +func term*(t: char): Peg {.rtl, extern: "npegs$1Char".} = ## constructs a PEG from a terminal char assert t != '\0' result = Peg(kind: pkChar, ch: t) -proc charSet*(s: set[char]): Peg {.noSideEffect, rtl, extern: "npegs$1".} = +func charSet*(s: set[char]): Peg {.rtl, extern: "npegs$1".} = ## constructs a PEG from a character set `s` assert '\0' notin s result = Peg(kind: pkCharChoice) new(result.charChoice) result.charChoice[] = s -proc len(a: Peg): int {.inline.} = return a.sons.len -proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) +func len(a: Peg): int {.inline.} = return a.sons.len +func add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s) -proc addChoice(dest: var Peg, elem: Peg) = +func addChoice(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkCharChoice: # caution! Do not introduce false aliasing here! @@ -195,12 +195,12 @@ template multipleOp(k: PegKind, localOpt: untyped) = if result.len == 1: result = result.sons[0] -proc `/`*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegsOrderedChoice".} = +func `/`*(a: varargs[Peg]): Peg {. + rtl, extern: "npegsOrderedChoice".} = ## constructs an ordered choice with the PEGs in `a` multipleOp(pkOrderedChoice, addChoice) -proc addSequence(dest: var Peg, elem: Peg) = +func addSequence(dest: var Peg, elem: Peg) = var L = dest.len-1 if L >= 0 and dest.sons[L].kind == pkTerminal: # caution! Do not introduce false aliasing here! @@ -212,12 +212,12 @@ proc addSequence(dest: var Peg, elem: Peg) = else: add(dest, elem) else: add(dest, elem) -proc sequence*(a: varargs[Peg]): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func sequence*(a: varargs[Peg]): Peg {. + rtl, extern: "npegs$1".} = ## constructs a sequence with all the PEGs from `a` multipleOp(pkSequence, addSequence) -proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = +func `?`*(a: Peg): Peg {.rtl, extern: "npegsOptional".} = ## constructs an optional for the PEG `a` if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, pkGreedyRepSet}: @@ -227,7 +227,7 @@ proc `?`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsOptional".} = else: result = Peg(kind: pkOption, sons: @[a]) -proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = +func `*`*(a: Peg): Peg {.rtl, extern: "npegsGreedyRep".} = ## constructs a "greedy repetition" for the PEG `a` case a.kind of pkGreedyRep, pkGreedyRepChar, pkGreedyRepSet, pkGreedyAny, pkOption: @@ -242,94 +242,94 @@ proc `*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyRep".} = else: result = Peg(kind: pkGreedyRep, sons: @[a]) -proc `!*`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsSearch".} = +func `!*`*(a: Peg): Peg {.rtl, extern: "npegsSearch".} = ## constructs a "search" for the PEG `a` result = Peg(kind: pkSearch, sons: @[a]) -proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, +func `!*\`*(a: Peg): Peg {.rtl, extern: "npgegsCapturedSearch".} = ## constructs a "captured search" for the PEG `a` result = Peg(kind: pkCapturedSearch, sons: @[a]) -proc `+`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsGreedyPosRep".} = +func `+`*(a: Peg): Peg {.rtl, extern: "npegsGreedyPosRep".} = ## constructs a "greedy positive repetition" with the PEG `a` return sequence(a, *a) -proc `&`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsAndPredicate".} = +func `&`*(a: Peg): Peg {.rtl, extern: "npegsAndPredicate".} = ## constructs an "and predicate" with the PEG `a` result = Peg(kind: pkAndPredicate, sons: @[a]) -proc `!`*(a: Peg): Peg {.noSideEffect, rtl, extern: "npegsNotPredicate".} = +func `!`*(a: Peg): Peg {.rtl, extern: "npegsNotPredicate".} = ## constructs a "not predicate" with the PEG `a` result = Peg(kind: pkNotPredicate, sons: @[a]) -proc any*: Peg {.inline.} = +func any*: Peg {.inline.} = ## constructs the PEG `any character`:idx: (``.``) result = Peg(kind: pkAny) -proc anyRune*: Peg {.inline.} = +func anyRune*: Peg {.inline.} = ## constructs the PEG `any rune`:idx: (``_``) result = Peg(kind: pkAnyRune) -proc newLine*: Peg {.inline.} = +func newLine*: Peg {.inline.} = ## constructs the PEG `newline`:idx: (``\n``) result = Peg(kind: pkNewLine) -proc unicodeLetter*: Peg {.inline.} = +func unicodeLetter*: Peg {.inline.} = ## constructs the PEG ``\letter`` which matches any Unicode letter. result = Peg(kind: pkLetter) -proc unicodeLower*: Peg {.inline.} = +func unicodeLower*: Peg {.inline.} = ## constructs the PEG ``\lower`` which matches any Unicode lowercase letter. result = Peg(kind: pkLower) -proc unicodeUpper*: Peg {.inline.} = +func unicodeUpper*: Peg {.inline.} = ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. result = Peg(kind: pkUpper) -proc unicodeTitle*: Peg {.inline.} = +func unicodeTitle*: Peg {.inline.} = ## constructs the PEG ``\title`` which matches any Unicode title letter. result = Peg(kind: pkTitle) -proc unicodeWhitespace*: Peg {.inline.} = +func unicodeWhitespace*: Peg {.inline.} = ## constructs the PEG ``\white`` which matches any Unicode ## whitespace character. result = Peg(kind: pkWhitespace) -proc startAnchor*: Peg {.inline.} = +func startAnchor*: Peg {.inline.} = ## constructs the PEG ``^`` which matches the start of the input. result = Peg(kind: pkStartAnchor) -proc endAnchor*: Peg {.inline.} = +func endAnchor*: Peg {.inline.} = ## constructs the PEG ``$`` which matches the end of the input. result = !any() -proc capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.noSideEffect, rtl, extern: "npegsCapture".} = +func capture*(a: Peg = Peg(kind: pkEmpty)): Peg {.rtl, extern: "npegsCapture".} = ## constructs a capture with the PEG `a` result = Peg(kind: pkCapture, sons: @[a]) -proc backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backref*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. `reverse` specifies wether indexing starts from the end of the ## capture list. result = Peg(kind: pkBackRef, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreCase*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. `reverse` specifies wether indexing starts from the end of the ## capture list. Ignores case for matching. result = Peg(kind: pkBackRefIgnoreCase, index: (if reverse: -index else: index - 1)) -proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func backrefIgnoreStyle*(index: range[1..MaxSubpatterns], reverse: bool = false): Peg {. + rtl, extern: "npegs$1".} = ## constructs a back reference of the given `index`. `index` starts counting ## from 1. `reverse` specifies wether indexing starts from the end of the ## capture list. Ignores style for matching. result = Peg(kind: pkBackRefIgnoreStyle, index: (if reverse: -index else: index - 1)) -proc spaceCost(n: Peg): int = +func spaceCost(n: Peg): int = case n.kind of pkEmpty: discard of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, @@ -344,8 +344,8 @@ proc spaceCost(n: Peg): int = inc(result, spaceCost(n.sons[i])) if result >= InlineThreshold: break -proc nonterminal*(n: NonTerminal): Peg {. - noSideEffect, rtl, extern: "npegs$1".} = +func nonterminal*(n: NonTerminal): Peg {. + rtl, extern: "npegs$1".} = ## constructs a PEG that consists of the nonterminal symbol assert n != nil if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: @@ -354,8 +354,8 @@ proc nonterminal*(n: NonTerminal): Peg {. else: result = Peg(kind: pkNonTerminal, nt: n) -proc newNonTerminal*(name: string, line, column: int): NonTerminal {. - noSideEffect, rtl, extern: "npegs$1".} = +func newNonTerminal*(name: string, line, column: int): NonTerminal {. + rtl, extern: "npegs$1".} = ## constructs a nonterminal symbol result = NonTerminal(name: name, line: line, col: column) @@ -390,7 +390,7 @@ template natural*: Peg = # ------------------------- debugging ----------------------------------------- -proc esc(c: char, reserved = {'\0'..'\255'}): string = +func esc(c: char, reserved = {'\0'..'\255'}): string = case c of '\b': result = "\\b" of '\t': result = "\\t" @@ -406,14 +406,14 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string = elif c in reserved: result = '\\' & c else: result = $c -proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" +func singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" -proc singleQuoteEsc(str: string): string = +func singleQuoteEsc(str: string): string = result = "'" for c in items(str): add result, esc(c, {'\''}) add result, '\'' -proc charSetEscAux(cc: set[char]): string = +func charSetEscAux(cc: set[char]): string = const reserved = {'^', '-', ']'} result = "" var c1 = 0 @@ -430,13 +430,13 @@ proc charSetEscAux(cc: set[char]): string = c1 = c2 inc(c1) -proc charSetEsc(cc: set[char]): string = +func charSetEsc(cc: set[char]): string = if card(cc) >= 128+64: result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' else: result = '[' & charSetEscAux(cc) & ']' -proc toStrAux(r: Peg, res: var string) = +func toStrAux(r: Peg, res: var string) = case r.kind of pkEmpty: add(res, "()") of pkAny: add(res, '.') @@ -522,7 +522,7 @@ proc toStrAux(r: Peg, res: var string) = of pkStartAnchor: add(res, '^') -proc `$` *(r: Peg): string {.noSideEffect, rtl, extern: "npegsToString".} = +func `$` *(r: Peg): string {.rtl, extern: "npegsToString".} = ## converts a PEG to its string representation result = "" toStrAux(r, result) @@ -535,7 +535,7 @@ type ml: int origStart: int -proc bounds*(c: Captures, +func bounds*(c: Captures, i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = ## returns the bounds ``[first..last]`` of the `i`'th capture. result = c.matches[i] @@ -548,11 +548,11 @@ when not useUnicode: inc(i) template runeLenAt(s, i): untyped = 1 - proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} - proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} - proc isLower(a: char): bool {.inline.} = return a in {'a'..'z'} - proc isTitle(a: char): bool {.inline.} = return false - proc isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} + func isAlpha(a: char): bool {.inline.} = return a in {'a'..'z', 'A'..'Z'} + func isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'} + func isLower(a: char): bool {.inline.} = return a in {'a'..'z'} + func isTitle(a: char): bool {.inline.} = return false + func isWhiteSpace(a: char): bool {.inline.} = return a in {' ', '\9'..'\13'} template matchOrParse(mopProc: untyped) = # Used to make the main matcher proc *rawMatch* as well as event parser @@ -860,8 +860,8 @@ template matchOrParse(mopProc: untyped) = leave(pkStartAnchor, s, p, start, result) of pkRule, pkList: assert false -proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int - {.noSideEffect, rtl, extern: "npegs$1".} = +func rawMatch*(s: string, p: Peg, start: int, c: var Captures): int + {.rtl, extern: "npegs$1".} = ## low-level matching proc that implements the PEG interpreter. Use this ## for maximum efficiency (every other PEG operation ends up calling this ## proc). @@ -873,7 +873,11 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int template leave(pk, s, p, start, length) = discard matchOrParse(matchIt) - result = matchIt(s, p, start, c) + {.cast(noSideEffect).}: + # This cast is allowed because the `matchOrParse` template is used for + # both matching and parsing, but side effects are only possible when it's + # used by `eventParser`. + result = matchIt(s, p, start, c) macro mkHandlerTplts(handlers: untyped): untyped = # Transforms the handler spec in *handlers* into handler templates. @@ -900,7 +904,7 @@ macro mkHandlerTplts(handlers: untyped): untyped = # StmtList # # ... - proc mkEnter(hdName, body: NimNode): NimNode = + func mkEnter(hdName, body: NimNode): NimNode = template helper(hdName, body) {.dirty.} = template hdName(s, p, start) = let s {.inject.} = s @@ -1069,8 +1073,8 @@ template fillMatches(s, caps, c) = else: caps[k] = "" -proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func matchLen*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1080,8 +1084,8 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string], result = rawMatch(s, pattern, start, c) if result >= 0: fillMatches(s, matches, c) -proc matchLen*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func matchLen*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## the same as ``match``, but it returns the length of the match, ## if there is no match, -1 is returned. Note that a match length ## of zero can happen. It's possible that a suffix of `s` remains @@ -1090,22 +1094,22 @@ proc matchLen*(s: string, pattern: Peg, c.origStart = start result = rawMatch(s, pattern, start, c) -proc match*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func match*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and ## the captured substrings in the array ``matches``. If it does not ## match, nothing is written into ``matches`` and ``false`` is ## returned. result = matchLen(s, pattern, matches, start) != -1 -proc match*(s: string, pattern: Peg, - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1".} = +func match*(s: string, pattern: Peg, + start = 0): bool {.rtl, extern: "npegs$1".} = ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. result = matchLen(s, pattern, start) != -1 -proc find*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): int {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func find*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): int {.rtl, extern: "npegs$1Capture".} = ## returns the starting position of ``pattern`` in ``s`` and the captured ## substrings in the array ``matches``. If it does not match, nothing ## is written into ``matches`` and -1 is returned. @@ -1119,9 +1123,9 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string], return -1 # could also use the pattern here: (!P .)* P -proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], +func findBounds*(s: string, pattern: Peg, matches: var openArray[string], start = 0): tuple[first, last: int] {. - noSideEffect, rtl, extern: "npegs$1Capture".} = + rtl, extern: "npegs$1Capture".} = ## returns the starting position and end position of ``pattern`` in ``s`` ## and the captured ## substrings in the array ``matches``. If it does not match, nothing @@ -1136,8 +1140,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], return (i, i+L-1) return (-1, 0) -proc find*(s: string, pattern: Peg, - start = 0): int {.noSideEffect, rtl, extern: "npegs$1".} = +func find*(s: string, pattern: Peg, + start = 0): int {.rtl, extern: "npegs$1".} = ## returns the starting position of ``pattern`` in ``s``. If it does not ## match, -1 is returned. var c: Captures @@ -1160,8 +1164,8 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string = yield substr(s, i, i+L-1) inc(i, L) -proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func findAll*(s: string, pattern: Peg, start = 0): seq[string] {. + rtl, extern: "npegs$1".} = ## returns all matching *substrings* of `s` that match `pattern`. ## If it does not match, @[] is returned. result = @[] @@ -1192,31 +1196,31 @@ template `=~`*(s: string, pattern: Peg): bool = # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func contains*(s: string, pattern: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## same as ``find(s, pattern, start) >= 0`` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: Peg, matches: var openArray[string], - start = 0): bool {.noSideEffect, rtl, extern: "npegs$1Capture".} = +func contains*(s: string, pattern: Peg, matches: var openArray[string], + start = 0): bool {.rtl, extern: "npegs$1Capture".} = ## same as ``find(s, pattern, matches, start) >= 0`` return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func startsWith*(s: string, prefix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix, start) >= 0 -proc endsWith*(s: string, suffix: Peg, start = 0): bool {. - noSideEffect, rtl, extern: "npegs$1".} = +func endsWith*(s: string, suffix: Peg, start = 0): bool {. + rtl, extern: "npegs$1".} = ## returns true if `s` ends with the pattern `suffix` var c: Captures c.origStart = start for i in start .. s.len-1: if rawMatch(s, suffix, i, c) == s.len - i: return true -proc replacef*(s: string, sub: Peg, by: string): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replacef*(s: string, sub: Peg, by: string): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## @@ -1244,8 +1248,8 @@ proc replacef*(s: string, sub: Peg, by: string): string {. inc(i, x) add(result, substr(s, i)) -proc replace*(s: string, sub: Peg, by = ""): string {. - noSideEffect, rtl, extern: "npegs$1".} = +func replace*(s: string, sub: Peg, by = ""): string {. + rtl, extern: "npegs$1".} = ## Replaces `sub` in `s` by the string `by`. Captures cannot be accessed ## in `by`. result = "" @@ -1261,9 +1265,9 @@ proc replace*(s: string, sub: Peg, by = ""): string {. inc(i, x) add(result, substr(s, i)) -proc parallelReplace*(s: string, subs: varargs[ +func parallelReplace*(s: string, subs: varargs[ tuple[pattern: Peg, repl: string]]): string {. - noSideEffect, rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1".} = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" @@ -1288,7 +1292,7 @@ proc parallelReplace*(s: string, subs: varargs[ when not defined(nimHasEffectsOf): {.pragma: effectsOf.} -proc replace*(s: string, sub: Peg, cb: proc( +func replace*(s: string, sub: Peg, cb: proc( match: int, cnt: int, caps: openArray[string]): string): string {. rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. @@ -1297,7 +1301,7 @@ proc replace*(s: string, sub: Peg, cb: proc( ## ## .. code-block:: nim ## - ## proc handleMatches*(m: int, n: int, c: openArray[string]): string = + ## func handleMatches*(m: int, n: int, c: openArray[string]): string = ## result = "" ## if m > 0: ## result.add ", " @@ -1380,8 +1384,8 @@ iterator split*(s: string, sep: Peg): string = if first < last: yield substr(s, first, last-1) -proc split*(s: string, sep: Peg): seq[string] {. - noSideEffect, rtl, extern: "npegs$1".} = +func split*(s: string, sep: Peg): seq[string] {. + rtl, extern: "npegs$1".} = ## Splits the string `s` into substrings. result = @[] for it in split(s, sep): result.add it @@ -1445,20 +1449,20 @@ const "@", "built-in", "escaped", "$", "$", "^" ] -proc handleCR(L: var PegLexer, pos: int): int = +func handleCR(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\c') inc(L.lineNumber) result = pos+1 if result < L.buf.len and L.buf[result] == '\L': inc(result) L.lineStart = result -proc handleLF(L: var PegLexer, pos: int): int = +func handleLF(L: var PegLexer, pos: int): int = assert(L.buf[pos] == '\L') inc(L.lineNumber) result = pos+1 L.lineStart = result -proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = +func init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.buf = input L.bufpos = 0 L.lineNumber = line @@ -1466,18 +1470,18 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = L.lineStart = 0 L.filename = filename -proc getColumn(L: PegLexer): int {.inline.} = +func getColumn(L: PegLexer): int {.inline.} = result = abs(L.bufpos - L.lineStart) + L.colOffset -proc getLine(L: PegLexer): int {.inline.} = +func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber -proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = +func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = var line = if line < 0: getLine(L) else: line var col = if col < 0: getColumn(L) else: col result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] -proc getEscapedChar(c: var PegLexer, tok: var Token) = +func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) if c.bufpos >= len(c.buf): tok.kind = tkInvalid @@ -1537,7 +1541,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc skip(c: var PegLexer) = +func skip(c: var PegLexer) = var pos = c.bufpos while pos < c.buf.len: case c.buf[pos] @@ -1554,7 +1558,7 @@ proc skip(c: var PegLexer) = break # EndOfFile also leaves the loop c.bufpos = pos -proc getString(c: var PegLexer, tok: var Token) = +func getString(c: var PegLexer, tok: var Token) = tok.kind = tkStringLit var pos = c.bufpos + 1 var quote = c.buf[pos-1] @@ -1575,7 +1579,7 @@ proc getString(c: var PegLexer, tok: var Token) = inc(pos) c.bufpos = pos -proc getDollar(c: var PegLexer, tok: var Token) = +func getDollar(c: var PegLexer, tok: var Token) = var pos = c.bufpos + 1 var neg = false if pos < c.buf.len and c.buf[pos] == '^': @@ -1595,7 +1599,7 @@ proc getDollar(c: var PegLexer, tok: var Token) = tok.kind = tkDollar c.bufpos = pos -proc getCharSet(c: var PegLexer, tok: var Token) = +func getCharSet(c: var PegLexer, tok: var Token) = tok.kind = tkCharSet tok.charset = {} var pos = c.bufpos + 1 @@ -1652,7 +1656,7 @@ proc getCharSet(c: var PegLexer, tok: var Token) = c.bufpos = pos if caret: tok.charset = {'\1'..'\xFF'} - tok.charset -proc getSymbol(c: var PegLexer, tok: var Token) = +func getSymbol(c: var PegLexer, tok: var Token) = var pos = c.bufpos while pos < c.buf.len: add(tok.literal, c.buf[pos]) @@ -1661,7 +1665,7 @@ proc getSymbol(c: var PegLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getBuiltin(c: var PegLexer, tok: var Token) = +func getBuiltin(c: var PegLexer, tok: var Token) = if c.bufpos+1 < c.buf.len and c.buf[c.bufpos+1] in strutils.Letters: inc(c.bufpos) getSymbol(c, tok) @@ -1670,7 +1674,7 @@ proc getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -proc getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) @@ -1792,7 +1796,7 @@ proc getTok(c: var PegLexer, tok: var Token) = add(tok.literal, c.buf[c.bufpos]) inc(c.bufpos) -proc arrowIsNextTok(c: PegLexer): bool = +func arrowIsNextTok(c: PegLexer): bool = # the only look ahead we need var pos = c.bufpos while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos) @@ -1813,23 +1817,23 @@ type identIsVerbatim: bool skip: Peg -proc pegError(p: PegParser, msg: string, line = -1, col = -1) = +func pegError(p: PegParser, msg: string, line = -1, col = -1) = var e: ref EInvalidPeg new(e) e.msg = errorStr(p, msg, line, col) raise e -proc getTok(p: var PegParser) = +func getTok(p: var PegParser) = getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") -proc eat(p: var PegParser, kind: TokKind) = +func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -proc parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.gcsafe.} -proc getNonTerminal(p: var PegParser, name: string): NonTerminal = +func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): result = p.nonterms[i] if cmpIgnoreStyle(result.name, name) == 0: return @@ -1837,13 +1841,13 @@ proc getNonTerminal(p: var PegParser, name: string): NonTerminal = result = newNonTerminal(name, getLine(p), getColumn(p)) add(p.nonterms, result) -proc modifiedTerm(s: string, m: Modifier): Peg = +func modifiedTerm(s: string, m: Modifier): Peg = case m of modNone, modVerbatim: result = term(s) of modIgnoreCase: result = termIgnoreCase(s) of modIgnoreStyle: result = termIgnoreStyle(s) -proc modifiedBackref(s: int, m: Modifier): Peg = +func modifiedBackref(s: int, m: Modifier): Peg = var reverse = s < 0 index = if reverse: -s else: s @@ -1852,7 +1856,7 @@ proc modifiedBackref(s: int, m: Modifier): Peg = of modIgnoreCase: result = backrefIgnoreCase(index, reverse) of modIgnoreStyle: result = backrefIgnoreStyle(index, reverse) -proc builtin(p: var PegParser): Peg = +func builtin(p: var PegParser): Peg = # do not use "y", "skip" or "i" as these would be ambiguous case p.tok.literal of "n": result = newLine() @@ -1872,11 +1876,11 @@ proc builtin(p: var PegParser): Peg = of "white": result = unicodeWhitespace() else: pegError(p, "unknown built-in: " & p.tok.literal) -proc token(terminal: Peg, p: PegParser): Peg = +func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -proc primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg = case p.tok.kind of tkAmp: getTok(p) @@ -1968,7 +1972,7 @@ proc primary(p: var PegParser): Peg = getTok(p) else: break -proc seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg = result = primary(p) while true: case p.tok.kind @@ -1982,13 +1986,13 @@ proc seqExpr(p: var PegParser): Peg = else: break else: break -proc parseExpr(p: var PegParser): Peg = +func parseExpr(p: var PegParser): Peg = result = seqExpr(p) while p.tok.kind == tkBar: getTok(p) result = result / seqExpr(p) -proc parseRule(p: var PegParser): NonTerminal = +func parseRule(p: var PegParser): NonTerminal = if p.tok.kind == tkIdentifier and arrowIsNextTok(p): result = getNonTerminal(p, p.tok.literal) if ntDeclared in result.flags: @@ -2002,7 +2006,7 @@ proc parseRule(p: var PegParser): NonTerminal = else: pegError(p, "rule expected, but found: " & p.tok.literal) -proc rawParse(p: var PegParser): Peg = +func rawParse(p: var PegParser): Peg = ## parses a rule or a PEG expression while p.tok.kind == tkBuiltin: case p.tok.literal @@ -2032,7 +2036,7 @@ proc rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -2047,14 +2051,14 @@ proc parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -proc peg*(pattern: string): Peg = +func peg*(pattern: string): Peg = ## constructs a Peg object from the `pattern`. The short name has been ## chosen to encourage its use as a raw string modifier:: ## ## peg"{\ident} \s* '=' \s* {.*}" result = parsePeg(pattern, "pattern") -proc escapePeg*(s: string): string = +func escapePeg*(s: string): string = ## escapes `s` so that it is matched verbatim when used as a peg. result = "" var inQuote = false diff --git a/tests/effects/tstrict_funcs_imports.nim b/tests/effects/tstrict_funcs_imports.nim index 4e9b9fe66..264771a1a 100644 --- a/tests/effects/tstrict_funcs_imports.nim +++ b/tests/effects/tstrict_funcs_imports.nim @@ -84,7 +84,7 @@ import parseutils, parsexml, pathnorm, - # pegs, + pegs, posix_utils, prelude, punycode, diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index fe3335c0f..0c5c8f4a3 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,6 +1,9 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets, pathnorm, pegs, sequtils +import os, strutils, osproc, sets, pathnorm, sequtils +# XXX: Remove this feature check once the csources supports it. +when defined(nimHasCastPragmaBlocks): + import std/pegs from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry import "../compiler/nimpaths" @@ -338,7 +341,9 @@ proc buildDocs*(args: string, localOnly = false, localOutDir = "") = if not localOnly: buildDocsDir(args, webUploadOutput / NimVersion) - let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )" - args = args.replace(gaFilter) + # XXX: Remove this feature check once the csources supports it. + when defined(nimHasCastPragmaBlocks): + let gaFilter = peg"@( y'--doc.googleAnalytics:' @(\s / $) )" + args = args.replace(gaFilter) buildDocsDir(args, localOutDir) From 6f15af41a7ea7aeb09b9c9b9c1d7ca3b505107c9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 7 Oct 2021 15:07:24 +0200 Subject: [PATCH 0461/1654] fixes a regression caused by overloadable enums even though they're opt-in (#18970) --- compiler/semtempl.nim | 10 +++++++--- tests/enum/mregression.nim | 4 ++++ tests/enum/tregression.nim | 13 +++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tests/enum/mregression.nim create mode 100644 tests/enum/tregression.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index c10917bcb..49ce55a64 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -250,7 +250,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = of skUnknown: # Introduced in this pass! Leave it as an identifier. result = n - of OverloadableSyms: + of OverloadableSyms-{skEnumField}: result = symChoice(c, n, s, scOpen, isField) of skGenericParam: if isField and sfGenSym in s.flags: result = n @@ -261,8 +261,12 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = if isField and sfGenSym in s.flags: result = n else: result = newSymNodeTypeDesc(s, c.idgen, n.info) else: - if isField and sfGenSym in s.flags: result = n - else: result = newSymNode(s, n.info) + if s.kind == skEnumField and overloadableEnums in c.features: + result = symChoice(c, n, s, scOpen, isField) + elif isField and sfGenSym in s.flags: + result = n + else: + result = newSymNode(s, n.info) # Issue #12832 when defined(nimsuggest): suggestSym(c.graph, n.info, s, c.graph.usageSym, false) diff --git a/tests/enum/mregression.nim b/tests/enum/mregression.nim new file mode 100644 index 000000000..9c3e3cf8b --- /dev/null +++ b/tests/enum/mregression.nim @@ -0,0 +1,4 @@ + +type + OtherEnum* = enum + Success, Failed, More diff --git a/tests/enum/tregression.nim b/tests/enum/tregression.nim new file mode 100644 index 000000000..597f0c80a --- /dev/null +++ b/tests/enum/tregression.nim @@ -0,0 +1,13 @@ +discard """ + action: "compile" +""" +import options, mregression + +type + MyEnum = enum + Success + +template t = + echo some(Success) + +t() From 08107c2192e6aebb386df2711f2935731ee599fd Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Oct 2021 00:54:21 +0800 Subject: [PATCH 0462/1654] [minor] give more friendly description (#18973) --- nimdoc/tester.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index 1e438b97b..32dce0709 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -80,4 +80,4 @@ let testNimDoc(test2Dir, test2DocsDir, test2Switches, fixup) if failures > 0: - quit "$# failures occurred; see note in tester.nim regarding -d:nimTestsNimdocFixup" % $failures + quit "$# failures occurred; see note in nimdoc/tester.nim regarding -d:nimTestsNimdocFixup" % $failures From 787def271b1cabc6f898caa42f892125de9fa908 Mon Sep 17 00:00:00 2001 From: antonl05 <88133270+antonl05@users.noreply.github.com> Date: Sat, 9 Oct 2021 14:17:07 +0530 Subject: [PATCH 0463/1654] add OpenIndiana to list (#18972) * add OpenIndiana support * point sunos to solaris --- tools/niminst/buildsh.nimf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 9850abd89..5f68a6a17 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -139,6 +139,11 @@ case $uos in myos="solaris" LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" ;; + *SunOS* ) + myos="solaris" + LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" + isOpenIndiana="yes" + ;; *haiku* ) myos="haiku" LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" @@ -159,7 +164,12 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - mycpu="i386" ;; + if [ isOpenIndiana -eq "yes" ] ; then + mycpu="amd64" + else + mycpu="i386" + fi + ;; *amd*64* | *x86-64* | *x86_64* ) mycpu="amd64" ;; *sparc*|*sun* ) From 83128f217f63045974a48e61b65386abbfc97352 Mon Sep 17 00:00:00 2001 From: narimiran Date: Mon, 11 Oct 2021 14:38:59 +0200 Subject: [PATCH 0464/1654] disable testing of `fidget`, to make CIs green --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 0c47104ac..e037cc609 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -63,7 +63,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget" +pkg "fidget", allowFailure = true pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" From 0ae2d1ea88fdc7b6910ddbc960c7db086be383e4 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 12 Oct 2021 06:41:30 -0700 Subject: [PATCH 0465/1654] Improve error message when NodeJS is not installed and 'nim js -r' is run (#18978) --- compiler/nodejs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/nodejs.nim b/compiler/nodejs.nim index 781035bb7..283643e8d 100644 --- a/compiler/nodejs.nim +++ b/compiler/nodejs.nim @@ -3,5 +3,8 @@ import os proc findNodeJs*(): string {.inline.} = ## Find NodeJS executable and return it as a string. result = findExe("nodejs") - if result == "": + if result.len == 0: result = findExe("node") + if result.len == 0: + echo "Please install NodeJS first, see https://nodejs.org/en/download" + raise newException(IOError, "NodeJS not found in PATH: " & result) From 2aa97a228aa3a34b48f1af9901f29379fb131a31 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 13 Oct 2021 10:09:45 +0100 Subject: [PATCH 0466/1654] Removes deprecated {.injectStmt.}. Fixes #18666 (#18984) --- compiler/ccgcalls.nim | 1 - compiler/ccgstmts.nim | 5 ---- compiler/cgen.nim | 3 --- compiler/cgendata.nim | 1 - compiler/pragmas.nim | 8 +----- compiler/wordrecg.nim | 2 +- lib/system/gc_ms.nim | 10 ------- tests/pragmas/tinjectstmt.nim | 51 ----------------------------------- 8 files changed, 2 insertions(+), 79 deletions(-) delete mode 100644 tests/pragmas/tinjectstmt.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index f6cc6da74..12f366a53 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -801,6 +801,5 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = genNamedParamCall(p, ri, d) else: genPrefixCall(p, le, ri, d) - postStmtActions(p) proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 6cbff6ee9..559849f0d 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1509,11 +1509,6 @@ proc genPragma(p: BProc, n: PNode) = for it in n.sons: case whichPragma(it) of wEmit: genEmit(p, it) - of wInjectStmt: - var p = newProc(nil, p.module) - p.options.excl {optLineTrace, optStackTrace} - genStmts(p, it[1]) - p.module.injectStmt = p.s(cpsStmts) else: discard diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 43410dc60..412a428da 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -271,9 +271,6 @@ proc genLineDir(p: BProc, t: PNode) = linefmt(p, cpsStmts, "nimln_($1, $2);$n", [line, quotedFilename(p.config, t.info)]) -proc postStmtActions(p: BProc) {.inline.} = - p.s(cpsStmts).add(p.module.injectStmt) - proc accessThreadLocalVar(p: BProc, s: PSym) proc emulatedThreadVars(conf: ConfigRef): bool {.inline.} proc genProc(m: BModule, prc: PSym) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 3678adacf..88b8d4090 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -170,7 +170,6 @@ type labels*: Natural # for generating unique module-scope names extensionLoaders*: array['0'..'9', Rope] # special procs for the # OpenGL wrapper - injectStmt*: Rope sigConflicts*: CountTable[SigHash] g*: BModuleList ndi*: NdiFile diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index e9f52c71f..a6e6131d7 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -55,7 +55,7 @@ const wDeprecated, wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll, wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto, - wInjectStmt, wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert} + wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert} lambdaPragmas* = {FirstCallConv..LastCallConv, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader, wThread, wAsmNoStackFrame, @@ -1204,12 +1204,6 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wExportNims: if sym == nil: invalidPragma(c, it) else: magicsys.registerNimScriptSymbol(c.graph, sym) - of wInjectStmt: - warningDeprecated(c.config, it.info, "'.injectStmt' pragma is deprecated") - if it.kind notin nkPragmaCallKinds or it.len != 2: - localError(c.config, it.info, "expression expected") - else: - it[1] = c.semExpr(c, it[1]) of wExperimental: if not isTopLevel(c): localError(c.config, n.info, "'experimental' pragma only valid as toplevel statement or in a 'push' environment") diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 4ba807518..ce06effab 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -80,7 +80,7 @@ type wLocalPassc = "localPassC", wBorrow = "borrow", wDiscardable = "discardable", wFieldChecks = "fieldChecks", wSubsChar = "subschar", wAcyclic = "acyclic", wShallow = "shallow", wUnroll = "unroll", wLinearScanEnd = "linearScanEnd", - wComputedGoto = "computedGoto", wInjectStmt = "injectStmt", wExperimental = "experimental", + wComputedGoto = "computedGoto", wExperimental = "experimental", wWrite = "write", wGensym = "gensym", wInject = "inject", wDirty = "dirty", wInheritable = "inheritable", wThreadVar = "threadvar", wEmit = "emit", wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic", diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index 9e306c497..0675b9f2e 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -428,16 +428,6 @@ proc sweep(gch: var GcHeap) = if c.refcount == rcBlack: c.refcount = rcWhite else: freeCyclicCell(gch, c) -when false: - # meant to be used with the now-deprected `.injectStmt`: {.injectStmt: newGcInvariant().} - proc newGcInvariant*() = - for x in allObjects(gch.region): - if isCell(x): - var c = cast[PCell](x) - if c.typ == nil: - writeStackTrace() - quit 1 - proc markGlobals(gch: var GcHeap) = if gch.gcThreadId == 0: when defined(nimTracing): diff --git a/tests/pragmas/tinjectstmt.nim b/tests/pragmas/tinjectstmt.nim deleted file mode 100644 index c6256bda6..000000000 --- a/tests/pragmas/tinjectstmt.nim +++ /dev/null @@ -1,51 +0,0 @@ -discard """ - joinable: false - output:''' -onInject: 1 -onInject: 2 -ok0 -ok1 -onInject: 3 -onInject: 4 -onInject: 5 -0 -onInject: 6 -onInject: 7 -onInject: 8 -1 -onInject: 9 -onInject: 10 -onInject: 11 -2 -ok2 -onInject: 12 -''' -""" - -# test {.injectStmt.} - -#[ -{.injectStmt.} pragma can be used to inject a statement before every -other statement in the current module. It's now undocumented and may be removed -in the future and replaced with something more general and without its limitations. -(e.g. doesn't work in VM or js backends). -]# - -from system/ansi_c import c_printf - -var count = 0 -proc onInject*() = - count.inc - # echo count # xxx would fail, probably infinite recursion - c_printf("onInject: %d\n", cast[int](count)) - -{.injectStmt: onInject().} -echo "ok0" -proc main()= - echo "ok1" - for a in 0..<3: - echo a - echo "ok2" - -static: main() # xxx injectStmt not honored in VM -main() From e645be4d0c6c53cd9483d4fde24c4ddc7bd1bf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E4=BA=A6=E6=81=A9?= Date: Wed, 13 Oct 2021 17:35:47 +0800 Subject: [PATCH 0467/1654] add ghci like type annotation buildEchoStmt (1049) (#18875) * add ghci like type annotation buildEchoStmt (1049) * Update compiler/semexprs.nim * Update compiler/semexprs.nim Co-authored-by: flywind Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- compiler/semexprs.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 342b8b05f..c11cecfa9 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1057,6 +1057,7 @@ proc buildEchoStmt(c: PContext, n: PNode): PNode = else: result.add localErrorNode(c, n, "system needs: echo") result.add(n) + result.add(newStrNode(nkStrLit, ": " & n.typ.typeToString)) result = semExpr(c, result) proc semExprNoType(c: PContext, n: PNode): PNode = From 2ac3ba713b61e4e6b70c98da59dc05925f8b5e6b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 13 Oct 2021 20:57:25 +0800 Subject: [PATCH 0468/1654] fix #18985 (#18988) --- compiler/semtypes.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 9f7a17a53..d03fa88a8 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -2076,7 +2076,10 @@ proc processMagicType(c: PContext, m: PSym) = setMagicType(c.config, m, tySequence, szUncomputedSize) if optSeqDestructors in c.config.globalOptions: incl m.typ.flags, tfHasAsgn - assert c.graph.sysTypes[tySequence] == nil + if defined(nimsuggest) or c.config.cmd == cmdCheck: # bug #18985 + discard + else: + assert c.graph.sysTypes[tySequence] == nil c.graph.sysTypes[tySequence] = m.typ of mOrdinal: setMagicIntegral(c.config, m, tyOrdinal, szUncomputedSize) From e2b19bb2deba0932cf324c0127ffa7f2cbed8b5d Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 13 Oct 2021 20:58:31 +0800 Subject: [PATCH 0469/1654] remove deprecated stuffs from std/times (#18937) --- lib/pure/times.nim | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index b41bb28b5..113f73d2a 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -488,7 +488,7 @@ proc fromEpochDay(epochday: int64): proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): YeardayRange {.tags: [], raises: [], benign.} = ## Returns the day of the year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).yearday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).yearday`. runnableExamples: doAssert getDayOfYear(1, mJan, 2000) == 0 doAssert getDayOfYear(10, mJan, 2000) == 9 @@ -508,7 +508,7 @@ proc getDayOfYear*(monthday: MonthdayRange, month: Month, year: int): proc getDayOfWeek*(monthday: MonthdayRange, month: Month, year: int): WeekDay {.tags: [], raises: [], benign.} = ## Returns the day of the week enum from day, month and year. - ## Equivalent with `initDateTime(monthday, month, year, 0, 0, 0).weekday`. + ## Equivalent with `dateTime(year, month, monthday, 0, 0, 0, 0).weekday`. runnableExamples: doAssert getDayOfWeek(13, mJun, 1990) == dWed doAssert $getDayOfWeek(13, mJun, 1990) == "Wednesday" @@ -1060,13 +1060,13 @@ proc isLeapDay*(dt: DateTime): bool {.since: (1, 1).} = ## Returns whether `t` is a leap day, i.e. Feb 29 in a leap year. This matters ## as it affects time offset calculations. runnableExamples: - let dt = initDateTime(29, mFeb, 2020, 00, 00, 00, utc()) + let dt = dateTime(2020, mFeb, 29, 00, 00, 00, 00, utc()) doAssert dt.isLeapDay doAssert dt+1.years-1.years != dt - let dt2 = initDateTime(28, mFeb, 2020, 00, 00, 00, utc()) + let dt2 = dateTime(2020, mFeb, 28, 00, 00, 00, 00, utc()) doAssert not dt2.isLeapDay doAssert dt2+1.years-1.years == dt2 - doAssertRaises(Exception): discard initDateTime(29, mFeb, 2021, 00, 00, 00, utc()) + doAssertRaises(Exception): discard dateTime(2021, mFeb, 29, 00, 00, 00, 00, utc()) assertDateTimeInitialized dt dt.year.isLeapYear and dt.month == mFeb and dt.monthday == 29 @@ -1372,7 +1372,7 @@ proc initDateTime*(monthday: MonthdayRange, month: Month, year: int, proc `+`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(hours = 5) doAssert $(dt + dur) == "2017-03-30T05:00:00Z" @@ -1380,7 +1380,7 @@ proc `+`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt: DateTime, dur: Duration): DateTime = runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) let dur = initDuration(days = 5) doAssert $(dt - dur) == "2017-03-25T00:00:00Z" @@ -1389,8 +1389,8 @@ proc `-`*(dt: DateTime, dur: Duration): DateTime = proc `-`*(dt1, dt2: DateTime): Duration = ## Compute the duration between `dt1` and `dt2`. runnableExamples: - let dt1 = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) - let dt2 = initDateTime(25, mMar, 2017, 00, 00, 00, utc()) + let dt1 = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) + let dt2 = dateTime(2017, mMar, 25, 00, 00, 00, 00, utc()) doAssert dt1 - dt2 == initDuration(days = 5) @@ -1998,10 +1998,10 @@ proc toDateTime(p: ParsedTime, zone: Timezone, f: TimeFormat, if p.utcOffset.isNone: # No timezone parsed - assume timezone is `zone` - result = initDateTime(monthday, month, year, hour, minute, second, nanosecond, zone) + result = dateTime(year, month, monthday, hour, minute, second, nanosecond, zone) else: # Otherwise convert to `zone` - result = (initDateTime(monthday, month, year, hour, minute, second, nanosecond, utc()).toTime + + result = (dateTime(year, month, monthday, hour, minute, second, nanosecond, utc()).toTime + initDuration(seconds = p.utcOffset.get())).inZone(zone) proc format*(dt: DateTime, f: TimeFormat, @@ -2009,7 +2009,7 @@ proc format*(dt: DateTime, f: TimeFormat, ## Format `dt` using the format specified by `f`. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == dt.format(f) assertDateTimeInitialized dt result = "" @@ -2034,7 +2034,7 @@ proc format*(dt: DateTime, f: string, loc: DateTimeLocale = DefaultLocale): stri ## See `Parsing and formatting dates`_ for documentation of the ## `format` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert "2000-01-01" == format(dt, "yyyy-MM-dd") let dtFormat = initTimeFormat(f) result = dt.format(dtFormat, loc) @@ -2057,7 +2057,7 @@ proc format*(time: Time, f: string, zone: Timezone = local()): string ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - var dt = initDateTime(01, mJan, 1970, 00, 00, 00, utc()) + var dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, utc()) var tm = dt.toTime() doAssert format(tm, "yyyy-MM-dd'T'HH:mm:ss", utc()) == "1970-01-01T00:00:00" time.inZone(zone).format(f) @@ -2083,7 +2083,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), ## Month and day names from the passed in `loc` are used. runnableExamples: let f = initTimeFormat("yyyy-MM-dd") - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == "2000-01-01".parse(f, utc()) var inpIdx = 0 # Input index var patIdx = 0 # Pattern index @@ -2125,7 +2125,7 @@ proc parse*(input, f: string, tz: Timezone = local(), ## See `Parsing and formatting dates`_ for documentation of the ## `f` argument. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 00, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 00, 00, 00, 00, utc()) doAssert dt == parse("2000-01-01", "yyyy-MM-dd", utc()) let dtFormat = initTimeFormat(f) result = input.parse(dtFormat, tz, loc = loc) @@ -2159,7 +2159,7 @@ proc `$`*(dt: DateTime): string {.tags: [], raises: [], benign.} = ## Converts a `DateTime` object to a string representation. ## It uses the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $dt == "2000-01-01T12:00:00Z" doAssert $default(DateTime) == "Uninitialized DateTime" if not dt.isInitialized: @@ -2171,7 +2171,7 @@ proc `$`*(time: Time): string {.tags: [], raises: [], benign.} = ## Converts a `Time` value to a string representation. It will use the local ## time zone and use the format `yyyy-MM-dd'T'HH:mm:sszzz`. runnableExamples: - let dt = initDateTime(01, mJan, 1970, 00, 00, 00, local()) + let dt = dateTime(1970, mJan, 01, 00, 00, 00, 00, local()) let tm = dt.toTime() doAssert $tm == "1970-01-01T00:00:00" & format(dt, "zzz") $time.local @@ -2193,7 +2193,7 @@ proc initTimeInterval*(nanoseconds, microseconds, milliseconds, ## `seconds`, `minutes`, `hours`, `days`, `months`, and `years`. runnableExamples: let day = initTimeInterval(hours = 24) - let dt = initDateTime(01, mJan, 2000, 12, 00, 00, utc()) + let dt = dateTime(2000, mJan, 01, 12, 00, 00, 00, utc()) doAssert $(dt + day) == "2000-01-02T12:00:00Z" doAssert initTimeInterval(hours = 24) != initTimeInterval(days = 1) result.nanoseconds = nanoseconds @@ -2279,8 +2279,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = ## - If `startDt.timezone != endDt.timezone`, then the result will be ## equivalent to `between(startDt.utc, endDt.utc)`. runnableExamples: - var a = initDateTime(25, mMar, 2015, 12, 0, 0, utc()) - var b = initDateTime(1, mApr, 2017, 15, 0, 15, utc()) + var a = dateTime(2015, mMar, 25, 12, 0, 0, 00, utc()) + var b = dateTime(2017, mApr, 1, 15, 0, 15, 00, utc()) var ti = initTimeInterval(years = 2, weeks = 1, hours = 3, seconds = 15) doAssert between(a, b) == ti doAssert between(a, b) == -between(b, a) @@ -2360,8 +2360,8 @@ proc between*(startDt, endDt: DateTime): TimeInterval = startDate = endDate # Handle hours, minutes, seconds, milliseconds, microseconds and nanoseconds - let newStartDt = initDateTime(startDate.monthday, startDate.month.Month, - startDate.year, startDt.hour, startDt.minute, startDt.second, + let newStartDt = dateTime(startDate.year, startDate.month.Month, + startDate.monthday, startDt.hour, startDt.minute, startDt.second, startDt.nanosecond, startDt.timezone) let dur = endDt - newStartDt let parts = toParts(dur) @@ -2512,7 +2512,7 @@ proc `+`*(dt: DateTime, interval: TimeInterval): DateTime = ## So adding one month to `31 October` will result in `31 November`, which ## will overflow and result in `1 December`. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt + 1.months) == "2017-04-30T00:00:00Z" # This is correct and happens due to monthday overflow. doAssert $(dt - 1.months) == "2017-03-02T00:00:00Z" @@ -2535,7 +2535,7 @@ proc `-`*(dt: DateTime, interval: TimeInterval): DateTime = ## then the `months` component and so on. The returned `DateTime` will ## have the same timezone as the input. runnableExamples: - let dt = initDateTime(30, mMar, 2017, 00, 00, 00, utc()) + let dt = dateTime(2017, mMar, 30, 00, 00, 00, 00, utc()) doAssert $(dt - 5.days) == "2017-03-25T00:00:00Z" dt + (-interval) From f93bfc0a329f193c8b83062e10140c2fb55a7873 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 13 Oct 2021 20:31:04 +0200 Subject: [PATCH 0470/1654] [backport] add v1.6 changelog (#18932) --- changelog.md | 538 +------------------- changelogs/changelog_1_6_0.md | 927 ++++++++++++++++++++++++++++++++++ 2 files changed, 928 insertions(+), 537 deletions(-) create mode 100644 changelogs/changelog_1_6_0.md diff --git a/changelog.md b/changelog.md index 1925878bf..ab63bd8b5 100644 --- a/changelog.md +++ b/changelog.md @@ -1,560 +1,24 @@ -# v1.6.x - yyyy-mm-dd +# v1.8.x - yyyy-mm-dd ## Changes affecting backward compatibility -- Deprecated `std/mersenne`. -- `system.delete` had a most surprising behavior when the index passed to it was out of - bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so - that an index error is produced instead. But be aware that your code might depend on - this quirky behavior so a review process is required on your part before you can - use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete` - switch, it pretends that `system.delete` is deprecated so that it is easier to see - where it was used in your code. - - `-d:nimStrictDelete` will become the default in upcoming versions. - - -- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`. - Please use `char` or `uint8` instead. - -- `repr` now doesn't insert trailing newline; previous behavior was very inconsistent, - see #16034. Use `-d:nimLegacyReprWithNewline` for previous behavior. - -- A type conversion from one enum type to another now produces an `[EnumConv]` warning. - You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead. - ``` - type A = enum a1, a2 - type B = enum b1, b2 - echo a1.B # produces a warning - echo a1.ord.B # produces no warning - ``` - -- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning. - This warning will become an error in future versions! Use an explicit conversion - like `cstring(x)` in order to silence the warning. - -- There is a new warning for *any* type conversion to enum that can be enabled via - `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`. - -- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for previous - behavior. - -- `math.round` now is rounded "away from zero" in JS backend which is consistent - with other backends. See #9125. Use `-d:nimLegacyJsRound` for previous behavior. - -- Changed the behavior of `uri.decodeQuery` when there are unencoded `=` - characters in the decoded values. Prior versions would raise an error. This is - no longer the case to comply with the HTML spec and other languages - implementations. Old behavior can be obtained with - `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same - underlying code is also updated the same way. - -- On POSIX systems, the default signal handlers used for Nim programs (it's - used for printing the stacktrace on fatal signals) will now re-raise the - signal for the OS default handlers to handle. - - This lets the OS perform its default actions, which might include core - dumping (on select signals) and notifying the parent process about the cause - of termination. - -- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` - for previous behavior. - -- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), - if `-d:nimPreviewHashRef` is used. It is expected that this behavior - becomes the new default in upcoming versions. - -- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`, - `hashes.hash(closure)` has also been improved. - -- The unary slice `..b` was deprecated, use `0..b` instead. - -- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. - -- `random.initRand(seed)` now produces non-skewed values for the 1st call to `rand()` after - initialization with a small (< 30000) seed. Use `-d:nimLegacyRandomInitRand` to restore - previous behavior for a transition time, see PR #17467. - -- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings. It is expected that this behavior becomes the new default in upcoming versions. `toJson` now serializes `JsonNode` - as is via reference (without a deep copy) instead of treating `JsonNode` as a regular ref object, - this can be customized via `jsonNodeMode`. - -- `json` and `jsonutils` now serialize NaN, Inf, -Inf as strings, so that - `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` which was invalid json. - - -- `strformat` is now part of `include std/prelude`. - -- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` in `std/algorithm`. - -- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes in a collapsed way, - use `-d:nimLegacyMacrosCollapseSymChoice` to get previous behavior. - -- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. - The downside is that these defines now have custom logic that doesn't apply for - other defines. - -- `std/os`: `putEnv` now raises if the 1st argument contains a `=` - -- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints`. - -- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. Definitions of `setTimeout`, - `clearTimeout`, `setInterval`, `clearInterval` were updated. - -- With `-d:nimPreviewDotLikeOps`, dot-like operators (operators starting with `.`, but not with `..`) - now have the same precedence as `.`, so that `a.?b.c` is now parsed as `(a.?b).c` instead of `a.?(b.c)`. - A warning is generated when a dot-like operator is used without `-d:nimPreviewDotLikeOps`. - -- The allocator for Nintendo Switch, which was nonfunctional because - of breaking changes in libnx, was removed, in favour of the new `-d:nimAllocPagesViaMalloc` option. - -- `net.parseIpAddress` now only allows IPv4 addresses in strict form as defined - in [RFC 6943](https://www.rfc-editor.org/rfc/rfc6943#section-3.1.1). - Specifically, octal numbers in IPv4 addresses are no longer accepted (before - they were parsed as decimal numbers). ## Standard library additions and changes -- `strformat`: - - added support for parenthesized expressions. - - added support for const string's instead of just string literals - -- `os.copyFile` is now 2.5x faster on OSX, by using `copyfile` from `copyfile.h`; - use `-d:nimLegacyCopyFile` for OSX < 10.5. - -- `system.addFloat` and `system.$` now can produce string representations of floating point numbers - that are minimal in size and that "roundtrip" (via the "Dragonbox" algorithm). This currently has - to be enabled via `-d:nimPreviewFloatRoundtrip`. It is expected that this behavior becomes the new default - in upcoming versions, as with other `nimPreviewX` define flags. - -- Fixed buffer overflow bugs in `net`. - -- Exported `sslHandle` from `net` and `asyncnet`. - -- Added `sections` iterator in `parsecfg`. - -- Make custom op in macros.quote work for all statements. - -- On Windows the SSL library now checks for valid certificates. - It uses the `cacert.pem` file for this purpose which was extracted - from `https://curl.se/ca/cacert.pem`. Besides - the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you - now also need to ship `cacert.pem` with your `.exe` file. - -- `typetraits`: - `distinctBase` now is identity instead of error for non distinct types. - Added `enumLen` to return the number of elements in an enum. - Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. - Added `hasClosure`. - Added `pointerBase` to return `T` for `ref T | ptr T`. - -- `prelude` now works with the JavaScript target. - Added `sequtils` import to `prelude`. - `prelude` can now be used via `include std/prelude`, but `include prelude` still works. - -- Added `almostEqual` in `math` for comparing two float values using a machine epsilon. - -- Added `clamp` in `math` which allows using a `Slice` to clamp to a value. - -- Added `ceilDiv` in `math` for round up integer division. - -- The JSON module can now handle integer literals and floating point literals of - arbitrary length and precision. - Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are - kept as string literals and one can use external BigNum libraries to handle these. - The `parseFloat` family of functions also has now optional `rawIntegers` and - `rawFloats` parameters that can be used to enforce that all integer or float - literals remain in the "raw" string form so that client code can easily treat - small and large numbers uniformly. - -- Added `BackwardsIndex` overload for `JsonNode`. - -- `json.%`,`json.to`, `jsonutils.formJson`,`jsonutils.toJson` now work with `uint|uint64` - instead of raising (as in 1.4) or giving wrong results (as in 1.2). - -- `jsonutils` now handles `cstring` (including as Table key), and `set`. - -- added `jsonutils.jsonTo` overload with `opt = Joptions()` param. - -- `jsonutils.toJson` now supports customization via `ToJsonOptions`. - -- Added an overload for the `collect` macro that inferes the container type based - on the syntax of the last expression. Works with std seqs, tables and sets. - -- Added `randState` template that exposes the default random number generator. - Useful for library authors. - -- Added `random.initRand()` overload with no argument which uses the current time as a seed. - -- `random.initRand(seed)` now allows `seed == 0`. - -- Added `std/sysrand` module to get random numbers from a secure source - provided by the operating system. - -- Added `std/enumutils` module. Added `genEnumCaseStmt` macro that generates - case statement to parse string to enum. -- Added `items` for enums with holes. -- Added `symbolName` to return the enum symbol name ignoring the human readable name. -- Added `symbolRank` to return the index in which an enum member is listed in an enum. - -- Removed deprecated `iup` module from stdlib, it has already moved to - [nimble](https://github.com/nim-lang/iup). - -- various functions in `httpclient` now accept `url` of type `Uri`. Moreover `request` function's - `httpMethod` argument of type `string` was deprecated in favor of `HttpMethod` enum type. - -- `nodejs` backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. - -- Added `cmpMem` to `system`. - -- `doAssertRaises` now correctly handles foreign exceptions. - -- Added `asyncdispatch.activeDescriptors` that returns the number of currently - active async event handles/file descriptors. - -- Added `getPort` to `asynchttpserver`. - -- `--gc:orc` is now 10% faster than previously for common workloads. If - you have trouble with its changed behavior, compile with `-d:nimOldOrc`. - -- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`, - determining preferred I/O block size for this file object. - -- Added `os.getCacheDir()` to return platform specific cache directory. - -- Added a simpler to use `io.readChars` overload. - -- Added `**` to jsffi. - -- `writeStackTrace` is available in JS backend now. - -- Added `decodeQuery` to `std/uri`. - -- `strscans.scanf` now supports parsing single characters. - -- `strscans.scanTuple` added which uses `strscans.scanf` internally, - returning a tuple which can be unpacked for easier usage of `scanf`. - -- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`, - if the iterable yields a built-in settable type. - -- Added `setutils.fullSet` which returns a full built-in `set` for a valid type. - -- Added `setutils.complement` which returns the complement of a built-in `set`. - -- Added `setutils.[]=`. - -- Added `math.isNaN`. - -- Added `jsbigints` module, arbitrary precision integers for JavaScript target. - -- Added `math.copySign`. - -- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList` - and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements - shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes - its argument, `addMoved`, is also supplied. - -- Added `euclDiv` and `euclMod` to `math`. - -- Added `httpcore.is1xx` and missing HTTP codes. - -- Added `jsconsole.jsAssert` for JavaScript target. - -- Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs. - https://www.freedesktop.org/software/systemd/man/os-release.html - -- Added `socketstream` module that wraps sockets in the stream interface - -- Added `sugar.dumpToString` which improves on `sugar.dump`. - -- Added `math.signbit`. - -- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` iterators (it never worked reliably) - -- In `lists`: renamed `append` to `add` and retained `append` as an alias; - added `prepend` and `prependMoved` analogously to `add` and `addMoved`; - added `remove` for `SinglyLinkedList`s. - -- Deprecated `any`. See https://github.com/nim-lang/RFCs/issues/281 - -- Added optional `options` argument to `copyFile`, `copyFileToDir`, and - `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are - followed (copy files symlinks point to); on Windows, `options` argument is - ignored and symlinks are skipped. - -- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as - symlinks (instead of skipping them as it was before); on Windows symlinks are - skipped. - -- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks - (instead of skipping them sometimes as it was before). - -- Added optional `followSymlinks` argument to `setFilePermissions`. - -- Added `os.isAdmin` to tell whether the caller's process is a member of the - Administrators local group (on Windows) or a root (on POSIX). - -- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C). - -- Added `compilesettings.SingleValueSetting.libPath`. - -- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding - issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages) - for modules importing `std/wrapnils`. - Added `??.` macro which returns an `Option`. - `std/wrapnils` can now be used to protect against `FieldDefect` errors in - case objects, generates optimal code (no overhead compared to manual - if-else branches), and preserves lvalue semantics which allows modifying - an expression. - -- Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. - -- `parseopt.initOptParser` has been made available and `parseopt` has been - added back to `prelude` for all backends. Previously `initOptParser` was - unavailable if the `os` module did not have `paramCount` or `paramStr`, - but the use of these in `initOptParser` were conditionally to the runtime - arguments passed to it, so `initOptParser` has been changed to raise - `ValueError` when the real command line is not available. `parseopt` was - previously excluded from `prelude` for JS, as it could not be imported. - -- Added `system.prepareStrMutation` for better support of low - level `moveMem`, `copyMem` operations for Orc's copy-on-write string - implementation. - -- Added `std/strbasics` for high performance string operations. - Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. - -- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` - and `$none(int)` to `"none(int)"` instead of `"None[int]"`. - -- Added `algorithm.merge`. - -- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target. - -- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target. - -- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) wrapper for JavaScript target. - -- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible - with most other languages. - -- Removed support for named procs in `sugar.=>`. - -- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). - -- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). - -- `std/times`: - Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time - zone offsets without colons, e.g. `UTC+7 -> +0700`. - - Added `dateTime` and deprecated `initDateTime`. - -- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. - -- Added dollar `$` and `len` for `jsre.RegExp`. - -- Added `std/tasks`. - -- Added `hasDataBuffered` to `asyncnet`. - -- Added `std/tempfiles`. - -- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can - be used as a replacement. - -- Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin). - -- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place. - -- Added setCurrentException for JS backend. - -- Added `dom.scrollIntoView` proc with options - -- Added `dom.setInterval`, `dom.clearInterval` overloads. - -- Merged `dom_extensions` module into `dom` module, - it was a module with a single line, see https://github.com/nim-lang/RFCs/issues/413 - -- Allow reading parameters when compiling for Nintendo Switch. - -- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises a defect - if the slice is out of bounds, likewise with `strutils.delete`. ## Language changes -- The `cstring` doesn't support `[]=` operator in JS backend. - -- nil dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. - -- The required name of case statement macros for the experimental - `caseStmtMacros` feature has changed from `match` to `` `case` ``. - -- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. - -- The unary minus in `-1` is now part of the integer literal, it is now parsed as a single token. - This implies that edge cases like `-128'i8` finally work correctly. - -- Custom numeric literals (e.g. `-128'bignum`) are now supported. - -- Tuple expressions are now parsed consistently as - `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. - -- Added `iterable[T]` type class to match called iterators, which enables writing: - `template fn(a: iterable)` instead of `template fn(a: untyped)` - -- A new import syntax `import foo {.all.}` now allows to import all symbols (public or private) - from `foo`. This can be useful for testing purposes. - -- Added a new module `std/importutils`, and an API `privateAccess`, which allows access to private fields - for an object type in the current scope. - -- `typeof(voidStmt)` now works and returns `void`. - -- The `gc:orc` algorithm was refined so that custom container types can participate in the - cycle collection process. See the documentation of `=trace` for more details. - -- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. - This is only supported for `--gc:orc` or `--gc:arc`. - -- The effect system was refined and there is a new `.effectsOf` annotation that does - explicitly what was previously done implicitly. See the manual for details. - To write code that is portable with older Nim versions, use this idiom: - -```nim - -when defined(nimHasEffectsOf): - {.experimental: "strictEffects".} -else: - {.pragma: effectsOf.} - -proc mysort(s: seq; cmp: proc(a, b: T): int) {.effectsOf: cmp.} - -``` - - To enable the new effect system, use --experimental:strictEffects. - - -- Nim now supports a small subset of Unicode operators as operator symbols. - The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". - To enable this feature, use `--experimental:unicodeOperators`. Note that due - to parser limitations you **cannot** enable this feature via a - pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable - it via the command line or in a configuration file. - -- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some - compiler checks regarding `case objects`. This allows serialization libraries - to avoid ugly, non-portable solutions. See https://github.com/nim-lang/RFCs/issues/407 - for more details. - -- Enum values can now be overloaded. This needs to be enabled - via `{.experimental: "overloadableEnums".}`. We hope that this feature allows for the - development of more fluent (less ugly) APIs. See https://github.com/nim-lang/RFCs/issues/373 - for more details. ## Compiler changes -- Added `--declaredLocs` to show symbol declaration location in messages. - -- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. - -- Deprecated `TaintedString` and `--taintmode`. - -- Deprecated `--nilseqs` which is now a noop. - -- Added `--spellSuggest` to show spelling suggestions on typos. - -- Added `--filenames:abs|canonical|legacyRelProj` which replaces --listFullPaths:on|off - -- Added `--processing:dots|filenames|off` which customizes `hintProcessing` - -- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` before a newline - for every generated message (potentially multiline), so tooling can tell when messages start and end. - -- Source+Edit links now appear on top of every docgen'd page when - `nim doc --git.url:url ...` is given. - -- Added `nim --eval:cmd` to evaluate a command directly, see `nim --help`. - -- VM now supports `addr(mystring[ind])` (index + index assignment) - -- Added `--hintAsError` with similar semantics as `--warningAsError`. - -- TLS: OSX now uses native TLS (`--tlsEmulation:off`), TLS now works with importcpp non-POD types, - such types must use `.cppNonPod` and `--tlsEmulation:off`should be used. - -- Now array literals(JS backend) uses JS typed arrays when the corresponding js typed array exists, - for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`. - -- docgen: rst files can now use single backticks instead of double backticks and correctly render - in both rst2html (as before) as well as common tools rendering rst directly (e.g. github), by - adding: `default-role:: code` directive inside the rst file, which is now handled by rst2html. - -- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have certain hints/warnings - -- Added `then`, `catch` to `asyncjs`, for now hidden behind `-d:nimExperimentalAsyncjsThen`. - -- `--newruntime` and `--refchecks` are deprecated. - -- Added `unsafeIsolate` and `extract` to `std/isolation`. - -- `--hint:CC` now goes to stderr (like all other hints) instead of stdout. - -- `--hint:all:on|off` is now supported to select or deselect all hints; it - differs from `--hints:on|off` which acts as a (reversible) gate. - Likewise with `--warning:all:on|off`. - -- json build instructions are now generated in `$nimcache/outFileBasename.json` - instead of `$nimcache/projectName.json`. This allows avoiding recompiling a given project - compiled with different options if the output file differs. - -- `--usenimcache` (implied by `nim r main`) now generates an output file that includes a hash of - some of the compilation options, which allows caching generated binaries: - nim r main # recompiles - nim r -d:foo main # recompiles - nim r main # uses cached binary - nim r main arg1 arg2 # ditto (runtime arguments are irrelevant) - -- `nim r` now supports cross compilation from unix to windows when specifying `-d:mingw` by using wine, - e.g.: `nim r --eval:'import os; echo "a" / "b"'` prints `a\b` - -- `nim` can compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. - `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. - -- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch - enforces that every symbol is written as it was declared, not enforcing - the official Nim style guide. To be enabled, this has to be combined either - with `--styleCheck:error` or `--styleCheck:hint`. ## Tool changes -- `nimscript` now handles `except Exception as e`. -- `nim e` now accepts arbitrary file extensions for the nimscript file, - although `.nims` is still the preferred extension in general. -- Latex doc generation is revised: output `.tex` files should be compiled - by `xelatex` (not by `pdflatex` as before). Now default Latex settings - provide support for Unicode and do better job for avoiding margin overflows. - -- Implemented `doc2tex` compiler command which converts documentation in - `.nim` files to Latex. - -- The rst parser now supports markdown table syntax. - Known limitations: - - cell alignment is not supported, i.e. alignment annotations in a delimiter - row (`:---`, `:--:`, `---:`) are ignored, - - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`. - -- `fusion` is now un-bundled from nim, `./koch fusion` will - install it via nimble at a fixed hash. - -- testament: added `nimoutFull: bool` spec to compare full output of compiler - instead of a subset. diff --git a/changelogs/changelog_1_6_0.md b/changelogs/changelog_1_6_0.md new file mode 100644 index 000000000..1ce6c2093 --- /dev/null +++ b/changelogs/changelog_1_6_0.md @@ -0,0 +1,927 @@ +# v1.6.0 - 2021-10-14 + + + +# Major new features + +With so many new features, pinpointing the most salient ones is a subjective exercise, +but here are a select few: + + +## `iterable[T]` + +The `iterable[T]` type class was added to match called iterators, +which solves a number of long-standing issues related to iterators. +Example: + +```nim +iterator iota(n: int): int = + for i in 0.. response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) +discard fn() +``` + + +## New `std/tasks` module + +Provides basic primitives for creating parallel programs. +Example: +```nim +import std/tasks +var num = 0 +proc hello(a: int) = num+=a + +let b = toTask hello(13) # arguments must be isolated, see `std/isolation` +b.invoke() +assert num == 13 +b.invoke() # can be called again +assert num == 26 +``` + + +## New module: `std/genasts` + +Provides an API `genAst` that avoids the problems inherent with `quote do` and can +be used as a replacement. +Example showing how this could be used for writing a simplified version of `unittest.check`: +```nim +import std/[genasts, macros, strutils] +macro check2(cond: bool): untyped = + assert cond.kind == nnkInfix, "$# not implemented" % $cond.kind + result = genAst(cond, s = repr(cond), lhs = cond[1], rhs = cond[2]): + # each local symbol we access must be explicitly captured + if not cond: + doAssert false, "'$#'' failed: lhs: '$#', rhs: '$#'" % [s, $lhs, $rhs] +let a = 3 +check2 a*2 == a+3 +if false: check2 a*2 < a+1 # would error with: 'a * 2 < a + 1'' failed: lhs: '6', rhs: '4' +``` + +See PR [#17426](https://github.com/nim-lang/Nim/pull/17426) for details. + + +## New module: `std/setutils` + +- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`, + if the iterable yields a built-in settable type. +- Added `setutils.fullSet` which returns a full built-in `set` for a valid type. +- Added `setutils.complement` which returns the complement of a built-in `set`. +- Added `setutils.[]=`. + + +## New module: `std/enumutils` + +- Added `genEnumCaseStmt` macro that generates + case statement to parse string to enum. +- Added `items` for enums with holes. +- Added `symbolName` to return the `enum` symbol name ignoring the human-readable name. +- Added `symbolRank` to return the index in which an `enum` member is listed in an enum. + + +## `system` + +- Added `system.prepareMutation` for better support of low + level `moveMem`, `copyMem` operations for `gc:orc`'s copy-on-write string + implementation. +- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible + with most other languages. +- Added `cmpMem` to `system`. +- `doAssertRaises` now correctly handles foreign exceptions. +- `addInt` now supports unsigned integers. + +Compatibility notes: +- `system.delete` had surprising behavior when the index passed to it was out of + bounds (it would delete the last entry then). Compile with `-d:nimStrictDelete` so + that an index error is produced instead. Be aware however that your code might depend on + this quirky behavior so a review process is required on your part before you can + use `-d:nimStrictDelete`. To make this review easier, use the `-d:nimAuditDelete` + switch, which pretends that `system.delete` is deprecated so that it is easier + to see where it was used in your code. + `-d:nimStrictDelete` will become the default in upcoming versions. +- `cuchar` is now deprecated as it aliased `char` where arguably it should have aliased `uint8`. + Please use `char` or `uint8` instead. +- `repr` now doesn't insert trailing newlines; the previous behavior was very inconsistent, + see [#16034](https://github.com/nim-lang/Nim/pull/16034). + Use `-d:nimLegacyReprWithNewline` for the previous behavior. + `repr` now also renders ASTs correctly for user defined literals, setters, `do`, etc. +- Deprecated `any`. See RFC [#281](https://github.com/nim-lang/RFCs/issues/281). +- The unary slice `..b` was deprecated, use `0..b` instead. + + +## `std/math` + +- Added `almostEqual` for comparing two float values using a machine epsilon. +- Added `clamp` which allows using a `Slice` to clamp to a value. +- Added `ceilDiv` for integer division that rounds up. +- Added `isNaN`. +- Added `copySign`. +- Added `euclDiv` and `euclMod`. +- Added `signbit`. +- Added `frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead. + +Compatibility notes: +- `math.round` now rounds "away from zero" in the JS backend, which is consistent + with other backends. See [#9125](https://github.com/nim-lang/Nim/pull/9125). + Use `-d:nimLegacyJsRound` for the previous behavior. + + +## Random number generators: `std/random`, `std/sysrand`, `std/oids` + +- Added `std/sysrand` module (see details above). +- Added `randState` template that exposes the default random number generator. + Useful for library authors. +- Added `initRand()` overload with no argument which uses the current time as a seed. +- `initRand(seed)` now allows `seed == 0`. +- Fixed overflow bugs. +- Fix `initRand` to avoid random number sequences overlapping, refs + [#18744](https://github.com/nim-lang/Nim/pull/18744). +- `std/oids` now uses `std/random`. + +Compatibility notes: +- Deprecated `std/mersenne`. +- `random.initRand(seed)` now produces non-skewed values for the first call to + `rand()` after initialization with a small (< 30000) seed. + Use `-d:nimLegacyRandomInitRand` to restore previous behavior for a transition + time, see PR [#17467](https://github.com/nim-lang/Nim/pull/17467). + + +## `std/json`, `std/jsonutils` + +- With `-d:nimPreviewJsonutilsHoleyEnum`, `jsonutils` now can serialize/deserialize + holey enums as regular enums (via `ord`) instead of as strings. + It is expected that this behavior becomes the new default in upcoming versions. + `toJson` now serializes `JsonNode` as is via reference (without a deep copy) + instead of treating `JsonNode` as a regular ref object, + this can be customized via `jsonNodeMode`. +- `std/json` and `std/jsonutils` now serialize `NaN`, `Inf`, `-Inf` as strings, + so that `%[NaN, -Inf]` is the string `["nan","-inf"]` instead of `[nan,-inf]` + which was invalid JSON. +- `std/json` can now handle integer literals and floating point literals of + arbitrary length and precision. + Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are + kept as string literals and one can use external BigNum libraries to handle these. + The `parseFloat` family of functions also has now optional `rawIntegers` and + `rawFloats` parameters that can be used to enforce that all integer or float + literals remain in the "raw" string form so that client code can easily treat + small and large numbers uniformly. +- Added `BackwardsIndex` overload for `JsonNode`. +- `json.%`,`json.to`, `jsonutils.fromJson`,`jsonutils.toJson` now work with `uint|uint64` + instead of raising (as in 1.4) or giving wrong results (as in 1.2). +- `std/jsonutils` now handles `cstring` (including as Table key), and `set`. +- Added `jsonutils.jsonTo` overload with `opt = Joptions()` param. +- `jsonutils.toJson` now supports customization via `ToJsonOptions`. +- `std/json`, `std/jsonutils` now support round-trip serialization when + `-d:nimPreviewFloatRoundtrip` is used. + + +## `std/typetraits`, `std/compilesettings` + +- `distinctBase` now is identity instead of error for non distinct types. +- `distinctBase` now allows controlling whether to be recursive or not. +- Added `enumLen` to return the number of elements in an enum. +- Added `HoleyEnum` for enums with holes, `OrdinalEnum` for enums without holes. +- Added `hasClosure`. +- Added `pointerBase` to return `T` for `ref T | ptr T`. +- Added `compilesettings.SingleValueSetting.libPath`. + + +## networking: `std/net`, `std/asyncnet`, `std/htmlgen`, `std/httpclient`, `std/asyncdispatch`, `std/asynchttpserver`, `std/httpcore` + +- Fixed buffer overflow bugs in `std/net`. +- Exported `sslHandle` from `std/net` and `std/asyncnet`. +- Added `hasDataBuffered` to `std/asyncnet`. +- Various functions in `std/httpclient` now accept `url` of type `Uri`. + Moreover, the `request` function's `httpMethod` argument of type `string` was + deprecated in favor of `HttpMethod` `enum` type; see + [#15919](https://github.com/nim-lang/Nim/pull/15919). +- Added `asyncdispatch.activeDescriptors` that returns the number of currently + active async event handles/file descriptors. +- Added `getPort` to `std/asynchttpserver` to resolve OS-assigned `Port(0)`; + this is usually recommended instead of hardcoding ports which can lead to + "Address already in use" errors. +- Fixed premature garbage collection in `std/asyncdispatch`, when a stacktrace + override is in place. +- Added `httpcore.is1xx` and missing HTTP codes. +- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals). + +Compatibility notes: +- On Windows, the SSL library now checks for valid certificates. + For this purpose it uses the `cacert.pem` file, which was extracted + from `https://curl.se/ca/cacert.pem`. Besides + the OpenSSL DLLs (e.g. `libssl-1_1-x64.dll`, `libcrypto-1_1-x64.dll`) you + now also need to ship `cacert.pem` with your `.exe` file. + + +## `std/hashes` + +- `hashes.hash` can now support `object` and `ref` (can be overloaded in user code), + if `-d:nimPreviewHashRef` is used. It is expected that this behavior + becomes the new default in upcoming versions. +- `hashes.hash(proc|ptr|ref|pointer)` now calls `hash(int)` and honors `-d:nimIntHash1`. + `hashes.hash(closure)` has also been improved. + + +## OS: `std/os`, `std/io`, `std/socketstream`, `std/linenoise`, `std/tempfiles` + +- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`, + determining preferred I/O block size for this file object. +- Added `os.getCacheDir()` to return platform specific cache directory. +- Improved `os.getTempDir()`, see PR [#16914](https://github.com/nim-lang/Nim/pull/16914). +- Added `os.isAdmin` to tell whether the caller's process is a member of the + Administrators local group (on Windows) or a root (on POSIX). +- Added optional `options` argument to `copyFile`, `copyFileToDir`, and + `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are + followed (copy files symlinks point to); on Windows, `options` argument is + ignored and symlinks are skipped. +- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as + symlinks (instead of skipping them as it was before); on Windows symlinks are + skipped. +- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks + (instead of skipping them sometimes as it was before). +- Added optional `followSymlinks` argument to `setFilePermissions`. +- Added a simpler to use `io.readChars` overload. +- Added `socketstream` module that wraps sockets in the stream interface. +- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C). + + +## Environment variable handling + +- Empty environment variable values are now supported across OS's and backends. +- Environment variable APIs now work in multithreaded scenarios, by delegating to + direct OS calls instead of trying to keep track of the environment. +- `putEnv`, `delEnv` now work at compile time. +- NodeJS backend now supports osenv: `getEnv`, `putEnv`, `envPairs`, `delEnv`, `existsEnv`. + +Compatibility notes: +- `std/os`: `putEnv` now raises if the first argument contains a `=`. + + +## POSIX + +- On POSIX systems, the default signal handlers used for Nim programs (it's + used for printing the stacktrace on fatal signals) will now re-raise the + signal for the OS default handlers to handle. + This lets the OS perform its default actions, which might include core + dumping (on select signals) and notifying the parent process about the cause + of termination. +- On POSIX systems, we now ignore `SIGPIPE` signals, use `-d:nimLegacySigpipeHandler` + for previous behavior. +- Added `posix_utils.osReleaseFile` to get system identification from `os-release` + file on Linux and the BSDs. + ([link](https://www.freedesktop.org/software/systemd/man/os-release.html)) +- Removed undefined behavior for `posix.open`. + + +## `std/prelude` + +- `std/strformat` is now part of `include std/prelude`. +- Added `std/sequtils` import to `std/prelude`. +- `std/prelude` now works with the JS target. +- `std/prelude` can now be used via `include std/prelude`, but `include prelude` still works. + + +## String manipulation: `std/strformat`, `std/strbasics` + +- Added support for parenthesized expressions. +- Added support for const strings instead of just string literals. +- Added `std/strbasics` for high-performance string operations. +- Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`. + + +## `std/wrapnils` + +- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding + issues like bug [#13063](https://github.com/nim-lang/Nim/issues/13063) + (which affected error messages) for modules importing `std/wrapnils`. +- Added `??.` macro which returns an `Option`. +- `std/wrapnils` can now be used to protect against `FieldDefect` errors in + case objects, generates optimal code (no overhead compared to manual + if-else branches), and preserves lvalue semantics which allows modifying + an expression. + + +## Containers: `std/algorithm`, `std/lists`, `std/sequtils`, `std/options`, `std/packedsets` + +- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` + iterators (it never worked reliably). +- Added `algorithm.merge`. +- In `std/lists`: renamed `append` to `add` and retained `append` as an alias; + added `prepend` and `prependMoved` analogously to `add` and `addMoved`; + added `remove` for `SinglyLinkedList`s. +- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList` + and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements + shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes + its argument, `addMoved`, is also supplied. + See PRs [#16362](https://github.com/nim-lang/Nim/pull/16362), + [#16536](https://github.com/nim-lang/Nim/pull/16536). +- New module: `std/packedsets`. + Generalizes `std/intsets`, see PR [#15564](https://github.com/nim-lang/Nim/pull/15564). + +Compatibility notes: +- Deprecated `sequtils.delete` and added an overload taking a `Slice` that raises + a defect if the slice is out of bounds, likewise with `strutils.delete`. +- Deprecated `proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T]` + in `std/algorithm`. +- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"` + and `$none(int)` to `"none(int)"` instead of `"None[int]"`. + + +## `std/times` + +- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time + zone offsets without colons, e.g. `UTC+7 -> +0700`. +- Added `dateTime` and deprecated `initDateTime`. + + +## `std/macros` and AST + +- New module `std/genasts`, see description above. +- The required name of case statement macros for the experimental + `caseStmtMacros` feature has changed from `match` to `` `case` ``. +- Tuple expressions are now parsed consistently as + `nnkTupleConstr` node. Will affect macros expecting nodes to be of `nnkPar`. +- In `std/macros`, `treeRepr,lispRepr,astGenRepr` now represent SymChoice nodes + in a collapsed way. + Use `-d:nimLegacyMacrosCollapseSymChoice` to get the previous behavior. +- Made custom op in `macros.quote` work for all statements. + + +## `std/sugar` + +- Added `sugar.dumpToString` which improves on `sugar.dump`. +- Added an overload for the `collect` macro that infers the container type based + on the syntax of the last expression. Works with std seqs, tables and sets. + +Compatibility notes: +- Removed support for named procs in `sugar.=>`. + + +## Parsing: `std/parsecfg`, `std/strscans`, `std/uri` + +- Added `sections` iterator in `parsecfg`. +- `strscans.scanf` now supports parsing single characters. +- Added `strscans.scanTuple` which uses `strscans.scanf` internally, + returning a tuple which can be unpacked for easier usage of `scanf`. +- Added `decodeQuery` to `std/uri`. +- `parseopt.initOptParser` has been made available and `parseopt` has been + added back to `std/prelude` for all backends. Previously `initOptParser` was + unavailable if the `std/os` module did not have `paramCount` or `paramStr`, + but the use of these in `initOptParser` were conditionally to the runtime + arguments passed to it, so `initOptParser` has been changed to raise + `ValueError` when the real command line is not available. `parseopt` was + previously excluded from `std/prelude` for JS, as it could not be imported. + +Compatibility notes: +- Changed the behavior of `uri.decodeQuery` when there are unencoded `=` + characters in the decoded values. Prior versions would raise an error. This is + no longer the case to comply with the HTML spec and other languages' + implementations. Old behavior can be obtained with + `-d:nimLegacyParseQueryStrict`. `cgi.decodeData` which uses the same + underlying code is also updated the same way. + + +## JS stdlib changes + +- Added `std/jsbigints` module, which provides arbitrary precision integers for the JS target. +- Added `setCurrentException` for the JS backend. +- `writeStackTrace` is available in the JS backend now. +- Added `then`, `catch` to `std/asyncjs` for promise pipelining, for now hidden + behind `-d:nimExperimentalAsyncjsThen`. +- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) + wrapper for the JS target. +- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) + wrapper for the JS target. +- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) + wrapper for the JS target. +- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger). +- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`. +- Added dollar `$` and `len` for `jsre.RegExp`. +- Added `jsconsole.jsAssert` for the JS target. +- Added `**` to `std/jsffi`. +- Added `copyWithin` [for `seq` and `array` for JS targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin). +- In `std/dom`, `Interval` is now a `ref object`, same as `Timeout`. + Definitions of `setTimeout`, `clearTimeout`, `setInterval`, `clearInterval` were updated. +- Added `dom.scrollIntoView` proc with options. +- Added `dom.setInterval`, `dom.clearInterval` overloads. +- Merged `std/dom_extensions` into the `std/dom` module, + as it was a module with a single line, see RFC [#413](https://github.com/nim-lang/RFCs/issues/413). +- `$` now gives more correct results on the JS backend. + + +## JS compiler changes + +- `cstring` doesn't support the `[]=` operator anymore in the JS backend. +- Array literals now use JS typed arrays when the corresponding JS typed array exists, + for example `[byte(1), 2, 3]` generates `new Uint8Array([1, 2, 3])`. + + +## VM and nimscript backend + +- VM now supports `addr(mystring[ind])` (index + index assignment). +- `nimscript` now handles `except Exception as e`. +- `nil` dereference is not allowed at compile time. `cast[ptr int](nil)[]` is rejected at compile time. +- `static[T]` now works better, refs [#17590](https://github.com/nim-lang/Nim/pull/17590), + [#15853](https://github.com/nim-lang/Nim/pull/15853). +- `distinct T` conversions now work in VM. +- `items(cstring)` now works in VM. +- Fix `addr`, `len`, `high` in VM ([#16002](https://github.com/nim-lang/Nim/pull/16002), + [#16610](https://github.com/nim-lang/Nim/pull/16610)). +- `std/cstrutils` now works in VM. + + +## OS/platform-specific notes + +- Support for Apple silicon/M1. +- Support for 32-bit RISC-V, refs [#16231](https://github.com/nim-lang/Nim/pull/16231). +- Support for armv8l, refs [#18901](https://github.com/nim-lang/Nim/pull/18901). +- Support for CROSSOS, refs [#18889](https://github.com/nim-lang/Nim/pull/18889). +- The allocator for Nintendo Switch, which was nonfunctional because + of breaking changes in libnx, was removed, in favor of the new `-d:nimAllocPagesViaMalloc` option. +- Allow reading parameters when compiling for Nintendo Switch. +- `--nimcache` now correctly works in a cross-compilation setting. +- Cross compilation targeting Windows was improved. +- This now works from macOS/Linux: `nim r -d:mingw main` + + + +## Performance / memory optimizations + +- The comment field in PNode AST was moved to a side channel, reducing overall + memory usage during compilation by a factor 1.25x +- `std/jsonutils` deserialization is now up to 20x faster. +- `os.copyFile` is now 2.5x faster on macOS, by using `copyfile` from `copyfile.h`; + use `-d:nimLegacyCopyFile` for macOS < 10.5. +- Float to string conversion is now 10x faster thanks to the Dragonbox algorithm, + with `-d:nimPreviewFloatRoundtrip`. +- `newSeqWith` is 3x faster. +- CI now supports batching (making Windows CI 2.3X faster). +- Sets now uses the optimized `countSetBits` proc, see PR + [#17334](https://github.com/nim-lang/Nim/pull/17334). + + +## Debugging + +- You can now enable/disable VM tracing in user code via `vmutils.vmTrace`. +- `koch tools` now builds `bin/nim_dbg` which allows easy access to a debug version + of Nim without recompiling. +- Added new module `compiler/debugutils` to help with debugging Nim compiler. +- Renamed `-d:nimCompilerStackraceHints` to `-d:nimCompilerStacktraceHints` and + used it in more contexts; this flag which works in tandem with `--stackTraceMsgs` + to show user code context in compiler stacktraces. + + +## Type system + +- `typeof(voidStmt)` now works and returns `void`. +- `enum` values can now be overloaded. This needs to be enabled + via `{.experimental: "overloadableEnums".}`. We hope that this feature allows + for the development of more fluent (less ugly) APIs. + See RFC [#373](https://github.com/nim-lang/RFCs/issues/373) for more details. +- A type conversion from one `enum` type to another now produces an `[EnumConv]` warning. + You should use `ord` (or `cast`, but the compiler won't help, if you misuse it) instead. + ```nim + type A = enum a1, a2 + type B = enum b1, b2 + echo a1.B # produces a warning + echo a1.ord.B # produces no warning + ``` +- A dangerous implicit conversion to `cstring` now triggers a `[CStringConv]` warning. + This warning will become an error in future versions! Use an explicit conversion + like `cstring(x)` in order to silence the warning. +- There is a new warning for *any* type conversion to `enum` that can be enabled via + `.warning[AnyEnumConv]:on` or `--warning:AnyEnumConv:on`. +- Reusing a type name in a different scope now works, refs [#17710](https://github.com/nim-lang/Nim/pull/17710). +- Fixed implicit and explicit generics in procedures, refs [#18808](https://github.com/nim-lang/Nim/pull/18808). + + +## New-style concepts + +Example: +```nim +type + Comparable = concept # no T, an atom + proc cmp(a, b: Self): int +``` +The new design does not rely on `system.compiles` and may compile faster. +See PR [#15251](https://github.com/nim-lang/Nim/pull/15251) +and RFC [#168](https://github.com/nim-lang/RFCs/issues/168) for details. + + +## Lexical / syntactic + +- Nim now supports a small subset of Unicode operators as operator symbols. + The supported symbols are: "∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔". + To enable this feature, use `--experimental:unicodeOperators`. Note that due + to parser limitations you **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably, you need to enable + it via the command line or in a configuration file. +- `var a {.foo.} = expr` now works inside templates (except when `foo` is overloaded). + + +## Compiler messages, error messages, hints, warnings + +- Significant improvement to error messages involving effect mismatches, + see PRs [#18384](https://github.com/nim-lang/Nim/pull/18384), + [#18418](https://github.com/nim-lang/Nim/pull/18418). +- Added `--declaredLocs` to show symbol declaration location in error messages. +- Added `--spellSuggest` to show spelling suggestions on typos. +- Added `--processing:dots|filenames|off` which customizes `hintProcessing`; + `--processing:filenames` shows which include/import modules are being compiled as an import stack. +- `FieldDefect` messages now shows discriminant value + lineinfo, in all backends (C, JS, VM) +- Added `--hintAsError` with similar semantics as `--warningAsError`. +- Added `--unitsep:on|off` to control whether to add ASCII unit separator `\31` + before a newline for every generated message (potentially multiline), + so tooling can tell when messages start and end. +- Added `--filenames:abs|canonical|legacyRelProj` which replaces `--listFullPaths:on|off` +- `--hint:all:on|off` is now supported to select or deselect all hints; it + differs from `--hints:on|off` which acts as a (reversible) gate. + Likewise with `--warning:all:on|off`. +- The style checking of the compiler now supports a `--styleCheck:usages` switch. + This switch enforces that every symbol is written as it was declared, not enforcing + the official Nim style guide. To be enabled, this has to be combined either + with `--styleCheck:error` or `--styleCheck:hint`. +- Type mismatch errors now show more context, use `-d:nimLegacyTypeMismatch` for + previous behavior. +- `typedesc[Foo]` now renders as such instead of `type Foo` in compiler messages. +- `runnableExamples` now show originating location in stacktraces on failure. +- `SuccessX` message now shows more useful information. +- New `DuplicateModuleImport` warning; improved `UnusedImport` and + `XDeclaredButNotUsed` accuracy. + +Compatibility notes: +- `--hint:CC` now prints to stderr (like all other hints) instead of stdout. + + +## Building and running Nim programs, configuration system + +- JSON build instructions are now generated in `$nimcache/outFileBasename.json` + instead of `$nimcache/projectName.json`. This allows avoiding recompiling a + given project compiled with different options if the output file differs. +- `--usenimcache` (implied by `nim r main`) now generates an output file that includes + a hash of some of the compilation options, which allows caching generated binaries: + ```bash + nim r main # recompiles + nim r -d:foo main # recompiles + nim r main # uses cached binary + nim r main arg1 arg2 # likewise (runtime arguments are irrelevant) + ``` +- `nim r` now supports cross compilation from unix to windows when specifying + `-d:mingw` by using Wine, e.g.: + `nim r --eval:'import os; echo "a" / "b"'` prints `a\b`. +- `nim` can compile version 1.4.0 as follows: + `nim c --lib:lib --stylecheck:off -d:nimVersion140 compiler/nim`. + `-d:nimVersion140` is not needed for bootstrapping, only for building 1.4.0 from devel. +- `nim e` now accepts arbitrary file extensions for the nimscript file, + although `.nims` is still the preferred extension in general. +- The configuration subsystem now allows for `-d:release` and `-d:danger` to work as expected. + The downside is that these defines now have custom logic that doesn't apply for + other defines. + + +## Multithreading + +- TLS: macOS now uses native TLS (`--tlsEmulation:off`). TLS now works with + `importcpp` non-POD types; such types must use `.cppNonPod` and + `--tlsEmulation:off`should be used. +- Added `unsafeIsolate` and `extract` to `std/isolation`. +- Added `std/tasks`, see description above. + + +## Memory management + +- `--gc:arc` now bootstraps (PR [#17342](https://github.com/nim-lang/Nim/pull/17342)). +- Lots of improvements to `gc:arc`, `gc:orc`, see PR + [#15697](https://github.com/nim-lang/Nim/pull/15697), + [#16849](https://github.com/nim-lang/Nim/pull/16849), + [#17993](https://github.com/nim-lang/Nim/pull/17993). +- `--gc:orc` is now 10% faster than previously for common workloads. + If you have trouble with its changed behavior, compile with `-d:nimOldOrc`. +- The `--gc:orc` algorithm was refined so that custom container types can participate in the + cycle collection process. See the documentation of `=trace` for more details. +- On embedded devices `malloc` can now be used instead of `mmap` via `-d:nimAllocPagesViaMalloc`. + This is only supported for `--gc:orc` or `--gc:arc`. + +Compatibility notes: +- `--newruntime` and `--refchecks` are deprecated, + use `--gc:arc`, `--gc:orc`, or `--gc:none` as appropriate instead. + + +## Docgen + +- docgen: RST files can now use single backticks instead of double backticks and + correctly render in both `nim rst2html` (as before) as well as common tools rendering + RST directly (e.g. GitHub). + This is done by adding the `default-role:: code` directive inside the RST file + (which is now handled by `nim rst2html`). +- Source+Edit links now appear on top of every docgen'd page when + `nim doc --git.url:url ...` is given. +- Latex doc generation is revised: output `.tex` files should be compiled + by `xelatex` (not by `pdflatex` as before). Now default Latex settings + provide support for Unicode and better avoid margin overflows. + The minimum required version is TeXLive 2018 (or an equivalent MikTeX version). +- The RST parser now supports footnotes, citations, admonitions, and short style + references with symbols. +- The RST parser now supports Markdown table syntax. + Known limitations: + - cell alignment is not supported, i.e. alignment annotations in a delimiter + row (`:---`, `:--:`, `---:`) are ignored + - every table row must start with `|`, e.g. `| cell 1 | cell 2 |`. +- Implemented `doc2tex` compiler command which converts documentation in + `.nim` files to Latex. +- docgen now supports syntax highlighting for inline code. +- docgen now supports same-line doc comments: + ```nim + func fn*(a: int): int = 42 ## Doc comment + ``` +- docgen now renders `deprecated` and other pragmas. +- `runnableExamples` now works with templates and nested templates. +- `runnableExamples: "-r:off"` now works for examples that should compile but not run. +- `runnableExamples` now renders code verbatim, and produces correct code in all cases. +- docgen now shows correct, canonical import paths in docs. +- docgen now shows all routines in sidebar, and the proc signature is now shown in sidebar. + + +## Effects and checks + +- Significant improvement to error messages involving effect mismatches +- There is a new `cast` section `{.cast(uncheckedAssign).}: body` that disables some + compiler checks regarding `case objects`. This allows serialization libraries + to avoid ugly, non-portable solutions. See RFC + [#407](https://github.com/nim-lang/RFCs/issues/407) for more details. + + +## Tools + +- Major improvements to `nimgrep`, see PR [#15612 +](https://github.com/nim-lang/Nim/pull/15612). +- `fusion` is now un-bundled from Nim, `./koch fusion` will + install it via Nimble at a fixed hash. +- `testament`: added `nimoutFull: bool` spec to compare full output of compiler + instead of a subset; many bugfixes to testament. + + +## Misc/cleanups + +- Deprecated `TaintedString` and `--taintmode`. +- Deprecated `--nilseqs` which is now a noop. +- Added `-d:nimStrictMode` in CI in several places to ensure code doesn't have + certain hints/warnings. +- Removed `.travis.yml`, `appveyor.yml.disabled`, `.github/workflows/ci.yml.disabled`. +- `[skip ci]` now works in azure and CI pipelines, see detail in PR + [#17561](https://github.com/nim-lang/Nim/pull/17561). From c4c9876785a17f9b43bb1e0748eff67a71a7487f Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 14 Oct 2021 15:23:01 +0800 Subject: [PATCH 0471/1654] fix a potential bug (#18900) --- lib/std/private/digitsutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 268a3ba7d..9be2ab3ef 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -95,7 +95,7 @@ proc addInt*(result: var string; x: int64) = var num: uint64 if x < 0: if x == low(int64): - num = uint64(x) + num = cast[uint64](x) else: num = uint64(-x) let base = result.len From 3493783d0f3f90b2a90c87b93c92360168272f34 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 14 Oct 2021 15:23:36 +0800 Subject: [PATCH 0472/1654] test whether it is unnecessary (#18893) --- compiler/semtypinst.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 0bb53e04d..5d300fc58 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -35,7 +35,7 @@ proc checkConstructedType*(conf: ConfigRef; info: TLineInfo, typ: PType) = proc searchInstTypes*(g: ModuleGraph; key: PType): PType = let genericTyp = key[0] if not (genericTyp.kind == tyGenericBody and - key[0] == genericTyp and genericTyp.sym != nil): return + genericTyp.sym != nil): return for inst in typeInstCacheItems(g, genericTyp.sym): if inst.id == key.id: return inst From 8a4eeba218b22d4258f95a279771851082a2e197 Mon Sep 17 00:00:00 2001 From: Miran Date: Thu, 14 Oct 2021 09:56:58 +0200 Subject: [PATCH 0473/1654] try to fix broken ssl test (#18991) --- tests/untestable/thttpclient_ssl_remotenetwork.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim index f56fe274b..d8137e516 100644 --- a/tests/untestable/thttpclient_ssl_remotenetwork.nim +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -48,7 +48,7 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win ("https://sha256.badssl.com/", good, "sha256"), ("https://sha384.badssl.com/", good, "sha384"), ("https://sha512.badssl.com/", good, "sha512"), - ("https://1000-sans.badssl.com/", good, "1000-sans"), + ("https://1000-sans.badssl.com/", bad, "1000-sans"), ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), ("https://ecc256.badssl.com/", good, "ecc256"), ("https://ecc384.badssl.com/", good, "ecc384"), From 73330711a3d753d25b073fb1dc24cb16e559724d Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 14 Oct 2021 23:55:05 +0800 Subject: [PATCH 0474/1654] make choosenim work on windows [backport] (#18993) --- bin/nim-gdb.bash | 1 - compiler/installer.ini | 1 - 2 files changed, 2 deletions(-) delete mode 120000 bin/nim-gdb.bash diff --git a/bin/nim-gdb.bash b/bin/nim-gdb.bash deleted file mode 120000 index f6eba4b0c..000000000 --- a/bin/nim-gdb.bash +++ /dev/null @@ -1 +0,0 @@ -nim-gdb \ No newline at end of file diff --git a/compiler/installer.ini b/compiler/installer.ini index 9e31b4e3f..41eee09f1 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -120,7 +120,6 @@ Files: "bin/nim" InstallScript: "yes" UninstallScript: "yes" Files: "bin/nim-gdb" -Files: "bin/nim-gdb.bash" [InnoSetup] From f4525efcf3b915be8e17c9730e9e0ffd0afe3a0f Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Fri, 15 Oct 2021 09:10:27 +0200 Subject: [PATCH 0475/1654] changelog_1_6_0: mention breaking change in effect tracking (#18995) --- changelogs/changelog_1_6_0.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/changelogs/changelog_1_6_0.md b/changelogs/changelog_1_6_0.md index 1ce6c2093..4a1047d20 100644 --- a/changelogs/changelog_1_6_0.md +++ b/changelogs/changelog_1_6_0.md @@ -905,6 +905,36 @@ Compatibility notes: to avoid ugly, non-portable solutions. See RFC [#407](https://github.com/nim-lang/RFCs/issues/407) for more details. +Compatibility notes: +- Fixed effect tracking for borrowed procs (see [#18882](https://github.com/nim-lang/Nim/pull/18882)). + One consequence is that, under some circumstances, Nim could previously permit a procedure with side effects to be written with `func` - you may need to change some occurrences of `func` to `proc`. + To illustrate, Nim versions before 1.6.0 compile the below without error + ```nim + proc print(s: string) = + echo s + + type + MyString = distinct string + + proc print(s: MyString) {.borrow.} + + func foo(s: MyString) = + print(s) + ``` + but Nim 1.6.0 produces the error + ``` + Error: 'foo' can have side effects + ``` + similar to how we expect that + ```nim + func print(s: string) = + echo s + ``` + produces + ``` + Error: 'print' can have side effects + ``` + ## Tools From 3b1a601fe1ad6ac27a21b5f28442474d47c60a2f Mon Sep 17 00:00:00 2001 From: ee7 <45465154+ee7@users.noreply.github.com> Date: Sat, 16 Oct 2021 11:25:05 +0200 Subject: [PATCH 0476/1654] sequtils: fix errors from `strictFuncs` use (#18998) Nim 1.4.x compiled the below code without error when using `--experimental:strictFuncs` import std/sequtils type Foo = ref object let foo1 = Foo() let foo2 = Foo() let foos = @[foo1, foo2] let fooTuples = @[(foo1, 1), (foo2, 2)] discard repeat(foo1, 3) discard zip(foos, foos) discard unzip(fooTuples) However, since 2020-12-09, devel Nim produced errors like /tmp/bar.nim(11, 15) template/generic instantiation of `repeat` from here /foo/nim/pure/collections/sequtils.nim(172, 6) Error: 'repeat' can have side effects an object reachable from 'x' is potentially mutated /foo/nim/pure/collections/sequtils.nim(183, 15) the mutation is here /foo/nim/pure/collections/sequtils.nim(183, 15) is the statement that connected the mutation to the parameter This commit reverts some `proc` to `func` changes so that code that: - calls `repeat`, `zip`, or `unzip` - and instantiates them with types containing `ref` can once again be compiled with `strictFuncs`. Otherwise, a user might be forced to drop or alter their `strictFuncs` use when upgrading from Nim 1.4.x, or when writing new code that uses these procedures (at least for now, with the current `strictFuncs` implementation). This commit also adds tests to assert that the remaining funcs in this module can be compiled with `strictFuncs` when used with types containing `ref`. The original batch of `proc` to `func` changes in `sequtils.nim` was in commit 6f57ebae349f, which was partially reverted in 38eb021f8158. See also: https://github.com/nim-lang/Nim/issues/16305 --- lib/pure/collections/sequtils.nim | 6 +++--- tests/stdlib/tsequtils.nim | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 12b961371..64a7be7a9 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -169,7 +169,7 @@ func cycle*[T](s: openArray[T], n: Natural): seq[T] = result[o] = e inc o -func repeat*[T](x: T, n: Natural): seq[T] = +proc repeat*[T](x: T, n: Natural): seq[T] = ## Returns a new sequence with the item `x` repeated `n` times. ## `n` must be a non-negative number (zero or more). ## @@ -246,7 +246,7 @@ func maxIndex*[T](s: openArray[T]): int {.since: (1, 1).} = template zipImpl(s1, s2, retType: untyped): untyped = - func zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = + proc zip*[S, T](s1: openArray[S], s2: openArray[T]): retType = ## Returns a new sequence with a combination of the two input containers. ## ## The input containers can be of different types. @@ -289,7 +289,7 @@ when (NimMajor, NimMinor) <= (1, 0): else: zipImpl(s1, s2, seq[(S, T)]) -func unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = +proc unzip*[S, T](s: openArray[(S, T)]): (seq[S], seq[T]) {.since: (1, 1).} = ## Returns a tuple of two sequences split out from a sequence of 2-field tuples. runnableExamples: let diff --git a/tests/stdlib/tsequtils.nim b/tests/stdlib/tsequtils.nim index 338473407..179f619f0 100644 --- a/tests/stdlib/tsequtils.nim +++ b/tests/stdlib/tsequtils.nim @@ -10,6 +10,7 @@ from algorithm import sorted {.experimental: "strictEffects".} {.push warningAsError[Effect]: on.} +{.experimental: "strictFuncs".} # helper for testing double substitution side effects which are handled # by `evalOnceAs` @@ -456,6 +457,31 @@ block: # xxx: obscure CT error: basic_types.nim(16, 16) Error: internal error: symbol has no generated name: true doAssert: iter(3).mapIt(2*it).foldl(a + b) == 6 +block: # strictFuncs tests with ref object + type Foo = ref object + + let foo1 = Foo() + let foo2 = Foo() + let foos = @[foo1, foo2] + + # Procedures that are `func` + discard concat(foos, foos) + discard count(foos, foo1) + discard cycle(foos, 3) + discard deduplicate(foos) + discard minIndex(foos) + discard maxIndex(foos) + discard distribute(foos, 2) + var mutableFoos = foos + mutableFoos.delete(0..1) + mutableFoos.insert(foos) + + # Some procedures that are `proc`, but were reverted from `func` + discard repeat(foo1, 3) + discard zip(foos, foos) + let fooTuples = @[(foo1, 1), (foo2, 2)] + discard unzip(fooTuples) + template main = # xxx move all tests here block: # delete tests From 162b07d72cbe79bad1ddf99413ba508f07a831b1 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 17 Oct 2021 06:38:07 +0300 Subject: [PATCH 0477/1654] fix parsesql.treeRepr when nil is in input (#18974) * fix parsesql.treeRepr when nil is in input * return newNode(nkNone) as default value, not nil --- lib/pure/parsesql.nim | 6 +++--- tests/stdlib/tparsesql.nim | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index eb5d7c2cc..9d74d08d0 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -982,7 +982,7 @@ proc parseInsert(p: var SqlParser): SqlNode = parseParIdentList(p, n) result.add n else: - result.add(nil) + result.add(newNode(nkNone)) if isKeyw(p, "default"): getTok(p) eat(p, "values") @@ -1017,7 +1017,7 @@ proc parseUpdate(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseDelete(p: var SqlParser): SqlNode = getTok(p) @@ -1029,7 +1029,7 @@ proc parseDelete(p: var SqlParser): SqlNode = if isKeyw(p, "where"): result.add(parseWhere(p)) else: - result.add(nil) + result.add(newNode(nkNone)) proc parseSelect(p: var SqlParser): SqlNode = getTok(p) diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index ba9e601a1..b440dc90f 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -3,6 +3,17 @@ discard """ """ import parsesql +doAssert treeRepr(parseSql("INSERT INTO STATS VALUES (10, 5.5); ") +) == """ + +nkStmtList + nkInsert + nkIdent STATS + nkNone + nkValueList + nkIntegerLit 10 + nkNumericLit 5.5""" + doAssert $parseSQL("SELECT foo FROM table;") == "select foo from table;" doAssert $parseSQL(""" SELECT @@ -183,8 +194,31 @@ SET ContactName = 'Alfred Schmidt', City= 'Frankfurt' WHERE CustomerID = 1; """) == "update Customers set ContactName = 'Alfred Schmidt' , City = 'Frankfurt' where CustomerID = 1;" +doAssert treeRepr(parseSql("""UPDATE Customers + SET ContactName = 'Alice', City= 'Frankfurt';""") +) == """ + +nkStmtList + nkUpdate + nkIdent Customers + nkAsgn + nkIdent ContactName + nkStringLit Alice + nkAsgn + nkIdent City + nkStringLit Frankfurt + nkNone""" + doAssert $parseSQL("DELETE FROM table_name;") == "delete from table_name;" +doAssert treeRepr(parseSQL("DELETE FROM table_name;") +) == """ + +nkStmtList + nkDelete + nkIdent table_name + nkNone""" + doAssert $parseSQL("DELETE * FROM table_name;") == "delete from table_name;" doAssert $parseSQL(""" From f77dea01fd980734a3ed4acb812575827540aff9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 16 Oct 2021 23:37:52 -0700 Subject: [PATCH 0478/1654] define `nimVersion` automatically and avoid needing -d:nimVersion140 (#18726) * define `nimVersion` and avoid needing -d:nimVersion140 * fix changelog --- changelog.md | 3 ++- compiler/ast.nim | 2 +- compiler/ccgexprs.nim | 4 ++-- compiler/options.nim | 18 ++++++++++++++++++ config/config.nims | 2 ++ 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index ab63bd8b5..fbda70582 100644 --- a/changelog.md +++ b/changelog.md @@ -1,7 +1,6 @@ # v1.8.x - yyyy-mm-dd - ## Changes affecting backward compatibility @@ -16,6 +15,8 @@ ## Compiler changes +- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, + without requiring `-d:nimVersion140` which is now a noop. ## Tool changes diff --git a/compiler/ast.nim b/compiler/ast.nim index 0a3b1b72d..abd2ff01e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -656,7 +656,7 @@ type mUnaryPlusI, mBitnotI, mUnaryPlusF64, mUnaryMinusF64, mCharToStr, mBoolToStr, - mIntToStr, mInt64ToStr, mFloatToStr, # for -d:nimVersion140 + mIntToStr, mInt64ToStr, mFloatToStr, # for compiling nimStdlibVersion < 1.5.1 (not bootstrapping) mCStrToStr, mStrToStr, mEnumToStr, mAnd, mOr, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ac4a26bd6..fc09fefc3 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -894,7 +894,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let discIndex = rdSetElemLoc(p.config, v, u.t) if optTinyRtti in p.config.globalOptions: # not sure how to use `genEnumToStr` here - if p.config.isDefined("nimVersion140"): + if p.config.getStdlibVersion < (1,5,1): const code = "{ #raiseFieldError($1); $2} $n" linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) else: @@ -905,7 +905,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) = let first = p.config.firstOrd(disc.sym.typ) let firstLit = int64Literal(cast[int](first)) let discName = genTypeInfo(p.config, p.module, disc.sym.typ, e.info) - if p.config.isDefined("nimVersion140"): + if p.config.getStdlibVersion < (1,5,1): const code = "{ #raiseFieldError($1); $2} $n" linefmt(p, cpsStmts, code, [strLit, raiseInstr(p)]) else: diff --git a/compiler/options.nim b/compiler/options.nim index eafcd816d..d3f868316 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -165,6 +165,7 @@ const cmdCtags, cmdBuildindex} type + NimVer* = tuple[major: int, minor: int, patch: int] TStringSeq* = seq[string] TGCMode* = enum # the selected GC gcUnselected = "unselected" @@ -346,6 +347,7 @@ type outDir*: AbsoluteDir jsonBuildFile*: AbsoluteFile prefixDir*, libpath*, nimcacheDir*: AbsoluteDir + nimStdlibVersion*: NimVer dllOverrides, moduleOverrides*, cfileSpecificOptions*: StringTableRef projectName*: string # holds a name like 'nim' projectPath*: AbsoluteDir # holds a path like /home/alice/projects/nim/compiler/ @@ -389,6 +391,16 @@ type cppCustomNamespace*: string vmProfileData*: ProfileData +proc parseNimVersion*(a: string): NimVer = + # could be moved somewhere reusable + if a.len > 0: + let b = a.split(".") + assert b.len == 3, a + template fn(i) = result[i] = b[i].parseInt # could be optimized if needed + fn(0) + fn(1) + fn(2) + proc assignIfDefault*[T](result: var T, val: T, def = default(T)) = ## if `result` was already assigned to a value (that wasn't `def`), this is a noop. if result == def: result = val @@ -560,6 +572,12 @@ proc newPartialConfigRef*(): ConfigRef = proc cppDefine*(c: ConfigRef; define: string) = c.cppDefines.incl define +proc getStdlibVersion*(conf: ConfigRef): NimVer = + if conf.nimStdlibVersion == (0,0,0): + let s = conf.symbols.getOrDefault("nimVersion", "") + conf.nimStdlibVersion = s.parseNimVersion + result = conf.nimStdlibVersion + proc isDefined*(conf: ConfigRef; symbol: string): bool = if conf.symbols.hasKey(symbol): result = true diff --git a/config/config.nims b/config/config.nims index 2ae86012c..893d6960f 100644 --- a/config/config.nims +++ b/config/config.nims @@ -14,3 +14,5 @@ when defined(nimStrictMode): # switch("hint", "ConvFromXtoItselfNotNeeded") switch("hintAsError", "ConvFromXtoItselfNotNeeded") # future work: XDeclaredButNotUsed + +switch("define", "nimVersion:" & NimVersion) From f0af4a36b932af3932b611c1f728ebfbfe3a2749 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sun, 17 Oct 2021 08:24:43 -0600 Subject: [PATCH 0479/1654] Added setGlobalValue to VM api (#19007) --- compiler/nimeval.nim | 4 ++++ compiler/vm.nim | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 577c7b586..82e2f0812 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -57,6 +57,10 @@ proc callRoutine*(i: Interpreter; routine: PSym; args: openArray[PNode]): PNode proc getGlobalValue*(i: Interpreter; letOrVar: PSym): PNode = result = vm.getGlobalValue(PCtx i.graph.vm, letOrVar) +proc setGlobalValue*(i: Interpreter; letOrVar: PSym, val: PNode) = + ## Sets a global value to a given PNode, does not do any type checking. + vm.setGlobalValue(PCtx i.graph.vm, letOrVar, val) + proc implementRoutine*(i: Interpreter; pkg, module, name: string; impl: proc (a: VmArgs) {.closure, gcsafe.}) = assert i != nil diff --git a/compiler/vm.nim b/compiler/vm.nim index 9fa25efbe..1b443aff1 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2162,6 +2162,11 @@ proc getGlobalValue*(c: PCtx; s: PSym): PNode = internalAssert c.config, s.kind in {skLet, skVar} and sfGlobal in s.flags result = c.globals[s.position-1] +proc setGlobalValue*(c: PCtx; s: PSym, val: PNode) = + ## Does not do type checking so ensure the `val` matches the `s.typ` + internalAssert c.config, s.kind in {skLet, skVar} and sfGlobal in s.flags + c.globals[s.position-1] = val + include vmops proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) = From c239db4817586f769327a6d27c244682111f903a Mon Sep 17 00:00:00 2001 From: narimiran Date: Tue, 19 Oct 2021 17:01:56 +0200 Subject: [PATCH 0480/1654] bump NimVersion to 1.7.1 --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index caa036f37..012aefe6a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2121,7 +2121,7 @@ const ## when (NimMajor, NimMinor, NimPatch) >= (1, 3, 1): discard # see also std/private/since - NimMinor* {.intdefine.}: int = 5 + NimMinor* {.intdefine.}: int = 7 ## is the minor number of Nim's version. ## Odd for devel, even for releases. From 62701cd3b92677c2014cf70b1cf5a5ba6e0468bf Mon Sep 17 00:00:00 2001 From: John Novak Date: Thu, 21 Oct 2021 01:06:00 +1000 Subject: [PATCH 0481/1654] Fix isInvalidFilename & always operate on the full passed in string (#19004) * Fix isInvalidFilename segfault & deprecate the method * Update lib/pure/os.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/pure/os.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/os.nim | 91 +++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index e3b8e9f1c..5320aa87e 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -39,8 +39,8 @@ const weirdTarget = defined(nimscript) or defined(js) since (1, 1): const invalidFilenameChars* = {'/', '\\', ':', '*', '?', '"', '<', '>', '|', '^', '\0'} ## \ - ## Characters that may produce invalid filenames across Linux, Windows, Mac, etc. - ## You can check if your filename contains these char and strip them for safety. + ## Characters that may produce invalid filenames across Linux, Windows and Mac. + ## You can check if your filename contains any of these chars and strip them for safety. ## Mac bans ``':'``, Linux bans ``'/'``, Windows bans all others. invalidFilenames* = [ "CON", "PRN", "AUX", "NUL", @@ -297,7 +297,7 @@ proc isAbsolute*(path: string): bool {.rtl, noSideEffect, extern: "nos$1", raise result = path[0] == '$' elif defined(posix) or defined(js): # `or defined(js)` wouldn't be needed pending https://github.com/nim-lang/Nim/issues/13469 - # This works around the problem for posix, but windows is still broken with nim js -d:nodejs + # This works around the problem for posix, but Windows is still broken with nim js -d:nodejs result = path[0] == '/' else: doAssert false # if ever hits here, adapt as needed @@ -321,7 +321,7 @@ when doslikeFileSystem: proc sameRoot(path1, path2: string): bool {.noSideEffect, raises: [].} = ## Return true if path1 and path2 have a same root. ## - ## Detail of windows path formats: + ## Detail of Windows path formats: ## https://docs.microsoft.com/en-us/dotnet/standard/io/file-path-formats assert(isAbsolute(path1)) @@ -370,7 +370,7 @@ proc relativePath*(path, base: string, sep = DirSep): string {. ## this can be useful to ensure the relative path only contains `'/'` ## so that it can be used for URL constructions. ## - ## On windows, if a root of `path` and a root of `base` are different, + ## On Windows, if a root of `path` and a root of `base` are different, ## returns `path` as is because it is impossible to make a relative path. ## That means an absolute path can be returned. ## @@ -558,27 +558,25 @@ iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = ## See also: ## * `parentDir proc <#parentDir,string>`_ ## - ## **Examples:** - ## - ## .. code-block:: - ## let g = "a/b/c" - ## - ## for p in g.parentDirs: - ## echo p - ## # a/b/c - ## # a/b - ## # a - ## - ## for p in g.parentDirs(fromRoot=true): - ## echo p - ## # a/ - ## # a/b/ - ## # a/b/c - ## - ## for p in g.parentDirs(inclusive=false): - ## echo p - ## # a/b - ## # a + runnableExamples: + let g = "a/b/c" + + for p in g.parentDirs: + echo p + # a/b/c + # a/b + # a + + for p in g.parentDirs(fromRoot=true): + echo p + # a/ + # a/b/ + # a/b/c + + for p in g.parentDirs(inclusive=false): + echo p + # a/b + # a if not fromRoot: var current = path @@ -944,8 +942,7 @@ proc getCacheDir*(): string = proc getCacheDir*(app: string): string = ## Returns the cache directory for an application `app`. ## - ## * On windows, this uses: `getCacheDir() / app / "cache"` - ## + ## * On Windows, this uses: `getCacheDir() / app / "cache"` ## * On other platforms, this uses: `getCacheDir() / app` when defined(windows): getCacheDir() / app / "cache" @@ -1988,7 +1985,7 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n proc tryMoveFSObject(source, dest: string, isDir: bool): bool {.noWeirdTarget.} = ## Moves a file (or directory if `isDir` is true) from `source` to `dest`. ## - ## Returns false in case of `EXDEV` error or `AccessDeniedError` on windows (if `isDir` is true). + ## Returns false in case of `EXDEV` error or `AccessDeniedError` on Windows (if `isDir` is true). ## In case of other errors `OSError` is raised. ## Returns true in case of success. when defined(windows): @@ -2144,7 +2141,7 @@ iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdT ## * `walkDirRec iterator <#walkDirRec.i,string>`_ runnableExamples: import std/sequtils - let paths = toSeq(walkPattern("lib/pure/*")) # works on windows too + let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too assert "lib/pure/concurrency".unixToNativePath in paths assert "lib/pure/os.nim".unixToNativePath in paths walkCommon(pattern, defaultWalkFilter) @@ -2163,7 +2160,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTar ## * `walkDirRec iterator <#walkDirRec.i,string>`_ runnableExamples: import std/sequtils - assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on windows too + assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too walkCommon(pattern, isFile) iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarget.} = @@ -2180,7 +2177,7 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarg ## * `walkDirRec iterator <#walkDirRec.i,string>`_ runnableExamples: import std/sequtils - let paths = toSeq(walkDirs("lib/pure/*")) # works on windows too + let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too assert "lib/pure/concurrency".unixToNativePath in paths walkCommon(pattern, isDir) @@ -2278,7 +2275,9 @@ iterator walkDir*(dir: string; relative = false, checkDir = false): ## If `checkDir` is true, `OSError` is raised when `dir` ## doesn't exist. ## - ## Example: This directory structure:: + ## **Example:** + ## + ## This directory structure:: ## dirA / dirB / fileB1.txt ## / dirC ## / fileA1.txt @@ -2291,7 +2290,6 @@ iterator walkDir*(dir: string; relative = false, checkDir = false): # this also works at compile time assert collect(for k in walkDir("dirA"): k.path).join(" ") == "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" - ## ## See also: ## * `walkPattern iterator <#walkPattern.i,string>`_ ## * `walkFiles iterator <#walkFiles.i,string>`_ @@ -3271,7 +3269,7 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = formalInfo.lastAccessTime = fromWinTime(rdFileTime(rawInfo.ftLastAccessTime)) formalInfo.lastWriteTime = fromWinTime(rdFileTime(rawInfo.ftLastWriteTime)) formalInfo.creationTime = fromWinTime(rdFileTime(rawInfo.ftCreationTime)) - formalInfo.blockSize = 8192 # xxx use windows API instead of hardcoding + formalInfo.blockSize = 8192 # xxx use Windows API instead of hardcoding # Retrieve basic permissions if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_READONLY) != 0'i32: @@ -3494,27 +3492,30 @@ proc setLastModificationTime*(file: string, t: times.Time) {.noWeirdTarget.} = discard h.closeHandle if res == 0'i32: raiseOSError(osLastError(), file) -func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1).} = +func isValidFilename*(filename: string, maxLen = 259.Positive): bool {.since: (1, 1), deprecated: "Deprecated since v1.5.1".} = ## Returns true if ``filename`` is valid for crossplatform use. ## ## This is useful if you want to copy or save files across Windows, Linux, Mac, etc. ## You can pass full paths as argument too, but func only checks filenames. - ## It uses ``invalidFilenameChars``, ``invalidFilenames`` and ``maxLen`` to verify the specified ``filename``. ## - ## .. code-block:: nim - ## assert not isValidFilename(" foo") ## Leading white space - ## assert not isValidFilename("foo ") ## Trailing white space - ## assert not isValidFilename("foo.") ## Ends with Dot - ## assert not isValidFilename("con.txt") ## "CON" is invalid (Windows) - ## assert not isValidFilename("OwO:UwU") ## ":" is invalid (Mac) - ## assert not isValidFilename("aux.bat") ## "AUX" is invalid (Windows) + ## It uses `invalidFilenameChars`, `invalidFilenames` and `maxLen` to verify the specified `filename`. ## + runnableExamples: + assert not isValidFilename(" foo") # Leading white space + assert not isValidFilename("foo ") # Trailing white space + assert not isValidFilename("foo.") # Ends with dot + assert not isValidFilename("con.txt") # "CON" is invalid (Windows) + assert not isValidFilename("OwO:UwU") # ":" is invalid (Mac) + assert not isValidFilename("aux.bat") # "AUX" is invalid (Windows) + assert not isValidFilename("") # Empty string + assert not isValidFilename("foo/") # Filename is empty + # https://docs.microsoft.com/en-us/dotnet/api/system.io.pathtoolongexception # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247%28v=vs.85%29.aspx result = true let f = filename.splitFile() - if unlikely(f.name.len + f.ext.len > maxLen or + if unlikely(f.name.len + f.ext.len > maxLen or f.name.len == 0 or f.name[0] == ' ' or f.name[^1] == ' ' or f.name[^1] == '.' or find(f.name, invalidFilenameChars) != -1): return false for invalid in invalidFilenames: From 1a45da9150c6cf5da324e754f49dc4cf957f1180 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 21 Oct 2021 16:53:15 +0200 Subject: [PATCH 0482/1654] use two underscores for easy demangling [backport:1.6] (#19028) --- compiler/ccgtypes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 492e1f181..23e1a5867 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -39,7 +39,7 @@ proc mangleName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: result = s.name.s.mangle.rope - result.add "_" + result.add "__" result.add m.g.graph.ifaces[s.itemId.module].uniqueName result.add "_" result.add rope s.itemId.item From 5e2b9341f343c3f80f92cfd160a0186f23521a42 Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Thu, 21 Oct 2021 18:01:48 +0300 Subject: [PATCH 0483/1654] Add Elbrus 2000 architecture (#19024) * Add Elbrus 2000 architecture * Add e2k to niminst * Update compiler/installer.ini Co-authored-by: Andreas Rumpf --- compiler/platform.nim | 6 ++++-- lib/system/platforms.nim | 4 +++- tools/nim.zsh-completion | 1 + tools/niminst/buildsh.nimf | 2 ++ tools/niminst/makefile.nimf | 3 +++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compiler/platform.nim b/compiler/platform.nim index 1bc00b629..18d43f006 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -197,7 +197,8 @@ type cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64, cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430, - cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32 + cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32, + cpuE2k type TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness, @@ -232,7 +233,8 @@ const (name: "riscv32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), - (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32)] + (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), + (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] type Target* = object diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index bd6541b6a..5bca8cb1c 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -35,7 +35,8 @@ type msp430, ## TI MSP430 microcontroller riscv32, ## RISC-V 32-bit processor riscv64, ## RISC-V 64-bit processor - wasm32 ## WASM, 32-bit + wasm32, ## WASM, 32-bit + e2k ## MCST Elbrus 2000 OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, @@ -93,5 +94,6 @@ const elif defined(riscv32): CpuPlatform.riscv32 elif defined(riscv64): CpuPlatform.riscv64 elif defined(wasm32): CpuPlatform.wasm32 + elif defined(e2k): CpuPlatform.e2k else: CpuPlatform.none ## the CPU this program will run on. diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index f7aeb72c6..07a221e9c 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -74,6 +74,7 @@ _nim() { '*--cpu=arm64[compile for ARM64 architecture]' \ '*--cpu=avr[compile for AVR architecture]' \ '*--cpu=esp[compile for ESP architecture]' \ + '*--cpu=e2k[compile for Elbrus 2000 architecture]' \ '*--cpu=hppa[compile for HPPA architecture]' \ '*--cpu=i386[compile for i386 architecture]' \ '*--cpu=ia64[compile for ia64 architecture]' \ diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 5f68a6a17..b81ac731f 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -227,6 +227,8 @@ case $ucpu in mycpu="arm" ;; *riscv64|riscv* ) mycpu="riscv64" ;; + *e2k* ) + mycpu="e2k" ;; *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index 69e87b837..c4f2f0e79 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -166,6 +166,9 @@ endif ifeq ($(ucpu),riscv64) mycpu = riscv64 endif +ifeq ($(ucpu),e2k) + mycpu = e2k +endif ifndef mycpu $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif From d6345874236e03d21f4f6acbad34546ce27052b7 Mon Sep 17 00:00:00 2001 From: Abishek PY <43115551+vj-abishek@users.noreply.github.com> Date: Fri, 22 Oct 2021 15:40:32 +0530 Subject: [PATCH 0484/1654] feat: copy to clipboard (#18963) * feat: copy to clipboard * fix: CI failure related issue * fix: CI failure issue * fix: copy to clipboard button bug * feat: copy pragmadots value to clipboard --- doc/nimdoc.css | 32 ++++++++++++- nimdoc/testproject/expected/nimdoc.out.css | 32 ++++++++++++- tools/dochack/dochack.nim | 56 ++++++++++++++++++++++ 3 files changed, 118 insertions(+), 2 deletions(-) diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 4abea9ce0..3353f1cda 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -38,6 +38,10 @@ Modified by Boyd Greenfield and narimiran --program: #6060c0; --option: #508000; --raw-data: #a4255b; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal) } [data-theme="dark"] { @@ -68,6 +72,10 @@ Modified by Boyd Greenfield and narimiran --program: #9090c0; --option: #90b010; --raw-data: #8be9fd; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal); } .theme-switch-wrapper { @@ -572,6 +580,10 @@ span.tok { margin-right: 0.2em; } +.copyToClipBoard { + position: relative; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -590,7 +602,25 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; } + border-radius: 6px; +} + +.copyToClipBoardBtn { + visibility: hidden; + position: absolute; + width: 24px; + border-radius: 4px; + background-image: var(--clipboard-image); + right: 5px; + top: 13px; + background-color: var(--secondary-background); + padding: 11px; + border: 0; +} + +.copyToClipBoard:hover .copyToClipBoardBtn { + visibility: visible; +} .pre-scrollable { max-height: 340px; diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 4abea9ce0..3353f1cda 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -38,6 +38,10 @@ Modified by Boyd Greenfield and narimiran --program: #6060c0; --option: #508000; --raw-data: #a4255b; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: black' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal) } [data-theme="dark"] { @@ -68,6 +72,10 @@ Modified by Boyd Greenfield and narimiran --program: #9090c0; --option: #90b010; --raw-data: #8be9fd; + + --clipboard-image-normal: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' fill='none' viewBox='0 0 24 24' stroke='currentColor'%3E %3Cpath stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2' /%3E %3C/svg%3E"); + --clipboard-image-selected: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='color: lightgray' viewBox='0 0 20 20' fill='currentColor'%3E %3Cpath d='M8 3a1 1 0 011-1h2a1 1 0 110 2H9a1 1 0 01-1-1z' /%3E %3Cpath d='M6 3a2 2 0 00-2 2v11a2 2 0 002 2h8a2 2 0 002-2V5a2 2 0 00-2-2 3 3 0 01-3 3H9a3 3 0 01-3-3z' /%3E %3C/svg%3E"); + --clipboard-image: var(--clipboard-image-normal); } .theme-switch-wrapper { @@ -572,6 +580,10 @@ span.tok { margin-right: 0.2em; } +.copyToClipBoard { + position: relative; +} + pre { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; color: var(--text); @@ -590,7 +602,25 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; } + border-radius: 6px; +} + +.copyToClipBoardBtn { + visibility: hidden; + position: absolute; + width: 24px; + border-radius: 4px; + background-image: var(--clipboard-image); + right: 5px; + top: 13px; + background-color: var(--secondary-background); + padding: 11px; + border: 0; +} + +.copyToClipBoard:hover .copyToClipBoardBtn { + visibility: visible; +} .pre-scrollable { max-height: 340px; diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 4a491cf88..83e81f1c0 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -341,3 +341,59 @@ proc search*() {.exportc.} = if timer != nil: clearTimeout(timer) timer = setTimeout(wrapper, 400) + +proc copyToClipboard*() {.exportc.} = + {.emit: """ + + function updatePreTags() { + + const allPreTags = document.querySelectorAll("pre") + + allPreTags.forEach((e) => { + + const div = document.createElement("div") + div.classList.add("copyToClipBoard") + + const preTag = document.createElement("pre") + preTag.innerHTML = e.innerHTML + + const button = document.createElement("button") + button.value = e.textContent.replace('...', '') + button.classList.add("copyToClipBoardBtn") + + div.appendChild(preTag) + div.appendChild(button) + + e.outerHTML = div.outerHTML + + }) + } + + + function copyTextToClipboard(e) { + const clipBoardContent = e.target.value + navigator.clipboard.writeText(clipBoardContent).then(function() { + e.target.style.setProperty("--clipboard-image", "var(--clipboard-image-selected)") + }, function(err) { + console.error("Could not copy text: ", err); + }); + } + + window.addEventListener("click", (e) => { + if (e.target.classList.contains("copyToClipBoardBtn")) { + copyTextToClipboard(e) + } + }) + + window.addEventListener("mouseover", (e) => { + if (e.target.nodeName === "PRE") { + e.target.nextElementSibling.style.setProperty("--clipboard-image", "var(--clipboard-image-normal)") + } + }) + + window.addEventListener("DOMContentLoaded", updatePreTags) + + """ + .} + +copyToClipboard() \ No newline at end of file From 4d8108046b71b6e40d88cde7496e687e68116877 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 22 Oct 2021 18:15:08 +0800 Subject: [PATCH 0485/1654] remove exception (#18906) --- compiler/semfold.nim | 8 +------- tests/system/tdollars.nim | 7 +++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 1f8efc4de..5ae91d306 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -292,13 +292,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P else: result = newStrNodeT("true", n, g) of mFloatToStr: result = newStrNodeT($getFloat(a), n, g) of mCStrToStr, mCharToStr: - if a.kind == nkBracket: - var s = "" - for b in a.sons: - s.add b.getStrOrChar - result = newStrNodeT(s, n, g) - else: - result = newStrNodeT(getStrOrChar(a), n, g) + result = newStrNodeT(getStrOrChar(a), n, g) of mStrToStr: result = newStrNodeT(getStrOrChar(a), n, g) of mEnumToStr: result = newStrNodeT(ordinalValToString(a, g), n, g) of mArrToSeq: diff --git a/tests/system/tdollars.nim b/tests/system/tdollars.nim index 17d195e76..93fa5cb9e 100644 --- a/tests/system/tdollars.nim +++ b/tests/system/tdollars.nim @@ -187,5 +187,12 @@ proc main()= test2 high(int8), "127" test2 low(int8), "-128" + block: + const + a: array[3, char] = ['N', 'i', 'm'] + aStr = $(a) + + doAssert aStr == """['N', 'i', 'm']""" + static: main() main() From 490c4226a55e8abd7cb504d4e24fcb714bff23f1 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Fri, 22 Oct 2021 17:32:02 +0200 Subject: [PATCH 0486/1654] allow converting static vars to `openArray` (#19035) When assigning constant output to a seq, and then passing that static seq to other functions that take `openArray`, the compiler may end up producing errors, as it does not know how to convert `static[seq[T]]` to `openArray[T]`. By ignoring the `static` wrapper on the type for the purpose of determining data memory location and length, this gets resolved cleanly. Unfortunately, it is relatively tricky to come up with a minimal example, as there are followup problems from the failing conversion, e.g., this may lead to `internal error: inconsistent environment type`, instead of the relevant `openArrayLoc` error message. --- compiler/ccgcalls.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 12f366a53..fe0516f9a 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -214,7 +214,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope = else: var a: TLoc initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a) - case skipTypes(a.t, abstractVar).kind + case skipTypes(a.t, abstractVar+{tyStatic}).kind of tyOpenArray, tyVarargs: if reifiedOpenArray(n): if a.t.kind in {tyVar, tyLent}: From 6621e1a3ead7166fa116bfbb8f0d2acb4e318d81 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Fri, 22 Oct 2021 23:42:54 +0000 Subject: [PATCH 0487/1654] use the correct header for TIOCGWINSZ on Solaris (#19037) --- lib/posix/termios.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index b03ce74f3..f755c720d 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -237,8 +237,11 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow", header: "".} # Get process group ID for session leader for controlling terminal FD. -# Window size ioctl. Should work on on any Unix that xterm has been ported to. -var TIOCGWINSZ*{.importc, header: "".}: culong +# Window size ioctl. Solaris based systems have an uncommen place for this. +when defined(solaris) or defined(sunos): + var TIOCGWINSZ*{.importc, header: "".}: culong +else: + var TIOCGWINSZ*{.importc, header: "".}: culong when defined(nimHasStyleChecks): {.push styleChecks: off.} From 41d99185918923063bd6ae34898c8ac18a941169 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Sun, 24 Oct 2021 05:24:31 +0000 Subject: [PATCH 0488/1654] use `gmake` in the buildscript on Solaris systems (#19036) --- ci/funs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/funs.sh b/ci/funs.sh index 1638931c6..8e7f8faf7 100644 --- a/ci/funs.sh +++ b/ci/funs.sh @@ -76,6 +76,8 @@ _nimBuildCsourcesIfNeeded(){ makeX=gmake elif [ "$unamestr" = 'CROSSOS' ]; then makeX=gmake + elif [ "$unamestr" = 'SunOS' ]; then + makeX=gmake else makeX=make fi From 141b76e36519219915ada9086d1c9b1d0b465659 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 24 Oct 2021 09:51:57 +0000 Subject: [PATCH 0489/1654] Add Zephyr Support (#19003) * Porting Nim to run on Zephyr. Includes changes to `std/net`. Squashed commit of the following: tweaking more memory / malloc things revert back bitmasks tweaking nim to use kernel heap as C malloc doesn't work fixing socket polling on zephyr cleanup getting maximum sockets for process or for rtos'es reorganizing and fixing net for async / system merge netlite changes back into nativesockets merge netlite changes back into nativesockets reverting native sockets back tweaking nim / zephyr network adding option to run 'net-lite' from linux bridging zephyr's max connections fixing net errors fixing compilation with getAddrString fixing compilation with getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... add note regarding incorrect FreeRTOS Sockadd_in fields changing to NIM_STATIC_ASSERT cleaning up the static_assert error messages cleaning up the static_assert error messages setting up static assert ftw! testing compile time asserts reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms finding missing items (issue #18684) fixup posix constants (issue #18684) adding plumbing for zephyr os (issue #18684) adding plumbing for zephyr os (issue #18684) * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * Porting Nim to run on Zephyr. Includes changes to `std/net`. Squashed commit of the following: tweaking more memory / malloc things revert back bitmasks tweaking nim to use kernel heap as C malloc doesn't work fixing socket polling on zephyr cleanup getting maximum sockets for process or for rtos'es reorganizing and fixing net for async / system merge netlite changes back into nativesockets merge netlite changes back into nativesockets reverting native sockets back tweaking nim / zephyr network adding option to run 'net-lite' from linux bridging zephyr's max connections fixing net errors fixing compilation with getAddrString fixing compilation with getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... add note regarding incorrect FreeRTOS Sockadd_in fields changing to NIM_STATIC_ASSERT cleaning up the static_assert error messages cleaning up the static_assert error messages setting up static assert ftw! testing compile time asserts reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms finding missing items (issue #18684) fixup posix constants (issue #18684) adding plumbing for zephyr os (issue #18684) adding plumbing for zephyr os (issue #18684) * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * fixing PR issues * Porting Nim to run on Zephyr. Includes changes to `std/net`. Squashed commit of the following: tweaking more memory / malloc things revert back bitmasks tweaking nim to use kernel heap as C malloc doesn't work fixing socket polling on zephyr cleanup getting maximum sockets for process or for rtos'es reorganizing and fixing net for async / system merge netlite changes back into nativesockets merge netlite changes back into nativesockets reverting native sockets back tweaking nim / zephyr network adding option to run 'net-lite' from linux bridging zephyr's max connections fixing net errors fixing compilation with getAddrString fixing compilation with getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getAddrString experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... getLocalAddr experimenting with a nativesockets_lite ... add note regarding incorrect FreeRTOS Sockadd_in fields changing to NIM_STATIC_ASSERT cleaning up the static_assert error messages cleaning up the static_assert error messages setting up static assert ftw! testing compile time asserts reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms reworking Sockaddr objects to more closely match various platforms finding missing items (issue #18684) fixup posix constants (issue #18684) adding plumbing for zephyr os (issue #18684) adding plumbing for zephyr os (issue #18684) * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * Remerge * fixing constant capitalizations * remove extra debug prints and fix TSa_Family/cint issue * remove extra debug prints and fix TSa_Family/cint issue * fixing PR issues * fix maxDescriptors on zephyr/freertos * move maxDescriptors to selector.nim -- fixes compile issue * change realloc impl on zephyr to match ansi c behavior * change realloc impl on zephyr to match ansi c behavior * force compileOnly mode for tlwip Co-authored-by: Jaremy J. Creechley Co-authored-by: Jaremy Creechley --- compiler/options.nim | 4 +- compiler/platform.nim | 6 +- lib/posix/posix.nim | 6 +- lib/posix/posix_freertos_consts.nim | 2 + lib/posix/posix_other.nim | 94 ++-- lib/posix/posix_other_consts.nim | 12 +- lib/pure/asyncdispatch.nim | 7 +- lib/pure/asyncnet.nim | 14 +- lib/pure/ioselects/ioselectors_epoll.nim | 5 +- lib/pure/ioselects/ioselectors_poll.nim | 5 +- lib/pure/ioselects/ioselectors_select.nim | 2 +- lib/pure/nativesockets.nim | 615 ++++++++++++---------- lib/pure/net.nim | 17 +- lib/pure/selectors.nim | 19 + lib/system/ansi_c.nim | 31 +- lib/system/bitmasks.nim | 1 + lib/system/dyncalls.nim | 2 +- lib/system/io.nim | 9 +- lib/system/mm/malloc.nim | 15 +- tests/stdlib/tlwip.nim | 2 +- 20 files changed, 522 insertions(+), 346 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index d3f868316..2feaa2d43 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -595,7 +595,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osQnx, osAtari, osAix, osHaiku, osVxWorks, osSolaris, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos, - osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos} + osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr} of "linux": result = conf.target.targetOS in {osLinux, osAndroid} of "bsd": @@ -615,6 +615,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = result = conf.target.targetOS == osNintendoSwitch of "freertos", "lwip": result = conf.target.targetOS == osFreeRTOS + of "zephyr": + result = conf.target.targetOS == osZephyr of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian of "cpu8": result = CPU[conf.target.targetCPU].bit == 8 diff --git a/compiler/platform.nim b/compiler/platform.nim index 18d43f006..eb1398677 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -22,7 +22,7 @@ type osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris, osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx, osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks - osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny + osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny type TInfoOSProp* = enum @@ -185,6 +185,10 @@ const objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", props: {ospPosix}), + (name: "Zephyr", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", + objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", + scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", + props: {ospPosix}), (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 45fe00d5d..c2504f994 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -151,8 +151,10 @@ proc htons*(a1: uint16): uint16 {.importc, header: "".} proc ntohl*(a1: uint32): uint32 {.importc, header: "".} proc ntohs*(a1: uint16): uint16 {.importc, header: "".} -proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} -proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} +when not defined(zephyr): + proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} + proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} + proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "".} proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. diff --git a/lib/posix/posix_freertos_consts.nim b/lib/posix/posix_freertos_consts.nim index efe87aacf..ca500534a 100644 --- a/lib/posix/posix_freertos_consts.nim +++ b/lib/posix/posix_freertos_consts.nim @@ -363,6 +363,8 @@ var SEM_FAILED* {.importc: "SEM_FAILED", header: "".}: pointer # # var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "".}: cint +var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "".}: cint + # var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 6584bfab2..688867b98 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -10,7 +10,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(freertos): +when defined(freertos) or defined(zephyr): const hasSpawnH = false # should exist for every Posix system nowadays hasAioH = false @@ -391,9 +391,29 @@ when hasSpawnH: header: "", final, pure.} = object when defined(linux): + const Sockaddr_max_length* = 255 # from sys/un.h const Sockaddr_un_path_length* = 108 +elif defined(zephyr): + when defined(net_ipv6): + const Sockaddr_max_length* = 24 + elif defined(net_raw): + const Sockaddr_max_length* = 20 + else: + const Sockaddr_max_length* = 8 + const Sockaddr_un_path_length* = Sockaddr_max_length + # Zephyr is heavily customizable so it's easy to get to a state + # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} +elif defined(freertos) or defined(lwip): + const Sockaddr_max_length* = 14 + const Sockaddr_un_path_length* = 108 else: + const Sockaddr_max_length* = 255 # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html # this is >=92 const Sockaddr_un_path_length* = 92 @@ -408,48 +428,50 @@ when defined(lwip): pure, final.} = object ## struct sockaddr sa_len*: uint8 ## Address family. sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + s2_len*: uint8 ## Address family. + ss_family*: TSa_Family ## Address family. + s2_data1*: array[2, char] ## Address family. + s2_data2*: array[3, uint32] ## Address family. + when defined(lwip6) or defined(net_ipv6): + s2_data3*: array[3, uint32] ## Address family. +elif defined(zephyr): + type + SockAddr* {.importc: "struct sockaddr", header: "", + pure, final.} = object ## struct sockaddr + sa_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. + data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].} + {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].} else: type SockAddr* {.importc: "struct sockaddr", header: "", pure, final.} = object ## struct sockaddr sa_family*: TSa_Family ## Address family. - sa_data*: array[0..255, char] ## Socket address (variable-length data). + sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data). + + Sockaddr_storage* {.importc: "struct sockaddr_storage", + header: "", + pure, final.} = object ## struct sockaddr_storage + ss_family*: TSa_Family ## Address family. type Sockaddr_un* {.importc: "struct sockaddr_un", header: "", pure, final.} = object ## struct sockaddr_un sun_family*: TSa_Family ## Address family. - sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path + sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path -when defined(lwip): - when not defined(lwip6): - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - s2_len*: uint8 ## Address family. - ss_family*: TSa_Family ## Address family. - s2_data1*: array[2, char] ## Address family. - s2_data2*: array[3, uint32] ## Address family. - else: - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - s2_len*: uint8 ## Address family. - ss_family*: TSa_Family ## Address family. - s2_data1*: array[2, char] ## Address family. - s2_data2*: array[3, uint32] ## Address family. - s2_data3*: array[3, uint32] ## Address family. -else: - type - Sockaddr_storage* {.importc: "struct sockaddr_storage", - header: "", - pure, final.} = object ## struct sockaddr_storage - ss_family*: TSa_Family ## Address family. - type Tif_nameindex* {.importc: "struct if_nameindex", final, pure, header: "".} = object ## struct if_nameindex @@ -494,6 +516,7 @@ type header: "".} = object ## struct in_addr s_addr*: InAddrScalar + # TODO: Fixme for FreeRTOS/LwIP, these are incorrect Sockaddr_in* {.importc: "struct sockaddr_in", pure, final, header: "".} = object ## struct sockaddr_in sin_family*: TSa_Family ## AF_INET. @@ -577,7 +600,12 @@ when not defined(lwip): events*: cshort ## The input event flags (see below). revents*: cshort ## The output event flags (see below). - Tnfds* {.importc: "nfds_t", header: "".} = cint + when defined(zephyr): + type + Tnfds* = distinct cint + else: + type + Tnfds* {.importc: "nfds_t", header: "".} = cint var errno* {.importc, header: "".}: cint ## error variable @@ -625,7 +653,7 @@ elif defined(solaris): # Solaris doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0'i32 -elif defined(freertos) or defined(lwip): +elif defined(zephyr) or defined(freertos) or defined(lwip): # LwIP/FreeRTOS doesn't have MSG_NOSIGNAL const MSG_NOSIGNAL* = 0x20'i32 diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index f43407b40..59ac6f9ed 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -462,6 +462,9 @@ var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "".}: cin # var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint +when defined(zephyr): + # Zephyr specific hardcoded value + var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint @@ -487,10 +490,15 @@ var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "".}: cint var SO_TYPE* {.importc: "SO_TYPE", header: "".}: cint var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "".}: cint var SOCK_RAW* {.importc: "SOCK_RAW", header: "".}: cint -var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "".}: cint +when defined(zephyr): + const SOCK_SEQPACKET* = cint(5) + var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "".}: cint +else: + var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "".}: cint + var SOMAXCONN* {.importc: "SOMAXCONN", header: "".}: cint + var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "".}: cint var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "".}: cint -var SOMAXCONN* {.importc: "SOMAXCONN", header: "".}: cint var MSG_PEEK* {.importc: "MSG_PEEK", header: "".}: cint var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "".}: cint var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "".}: cint diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index d87e2360f..e15fb0851 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1973,15 +1973,18 @@ proc activeDescriptors*(): int {.inline.} = when defined(posix): import posix -when defined(linux) or defined(windows) or defined(macosx) or defined(bsd): +when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(zephyr) or defined(freertos): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is ## supported on the following OSes: Windows, Linux, OSX, BSD. when defined(windows): result = 16_700_000 + elif defined(zephyr) or defined(freertos): + result = FD_MAX else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) - result = int(fdLim.rlim_cur) - 1 + result = int(fdLim.rlim_cur) - 1 \ No newline at end of file diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 5523fa10e..abb4dd075 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -103,6 +103,7 @@ export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) when defineSsl: import openssl @@ -178,11 +179,12 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: AsyncSocket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: AsyncSocket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc newAsyncSocket*(domain, sockType, protocol: cint, buffered = true, @@ -655,7 +657,7 @@ proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = # xxx dedup with std/net s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen -when defined(posix): +when defined(posix) and not useNimNetLite: proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) = ## Binds Unix socket to `path`. diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index b62b4c2db..05d9d19a8 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -73,10 +73,7 @@ type proc newSelector*[T](): Selector[T] = # Retrieve the maximum fd count (for current OS) via getrlimit() - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseOSError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand const numFD = 1024 diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 6b10e19ae..0d8fef78a 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -53,10 +53,7 @@ else: body proc newSelector*[T](): Selector[T] = - var a = RLimit() - if getrlimit(posix.RLIMIT_NOFILE, a) != 0: - raiseIOSelectorsError(osLastError()) - var maxFD = int(a.rlim_max) + var maxFD = maxDescriptors() when hasThreadSupport: result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T]))) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 6f216ac85..88cbaf28c 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -313,7 +313,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos): + when defined(genode) or defined(freertos) or defined(zephyr): tv.tv_sec = Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 13c08dd92..4037d7181 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -21,6 +21,7 @@ when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) when useWinVersion: import winlean @@ -35,9 +36,12 @@ else: export SocketHandle, Sockaddr_in, Addrinfo, INADDR_ANY, SockAddr, SockLen, Sockaddr_in6, Sockaddr_storage, - inet_ntoa, recv, `==`, connect, send, accept, recvfrom, sendto, + recv, `==`, connect, send, accept, recvfrom, sendto, freeAddrInfo +when not useNimNetLite: + export inet_ntoa + export SO_ERROR, SOL_SOCKET, @@ -332,125 +336,6 @@ template htons*(x: uint16): untyped = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) -proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the service name specified by `name` matches the s_name member - ## and the protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyname(name, proto) - else: - var s = posix.getservbyname(name, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getServByPort*(port: Port, proto: string): Servent {.tags: [ReadIOEffect].} = - ## Searches the database from the beginning and finds the first entry for - ## which the port specified by `port` matches the s_port member and the - ## protocol name specified by `proto` matches the s_proto member. - ## - ## On posix this will search through the `/etc/services` file. - when useWinVersion: - var s = winlean.getservbyport(ze(int16(port)).cint, proto) - else: - var s = posix.getservbyport(ze(int16(port)).cint, proto) - if s == nil: raiseOSError(osLastError(), "Service not found.") - result.name = $s.s_name - result.aliases = cstringArrayToSeq(s.s_aliases) - result.port = Port(s.s_port) - result.proto = $s.s_proto - -proc getHostByAddr*(ip: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the hostname of an IP Address. - var myaddr: InAddr - myaddr.s_addr = inet_addr(ip) - - when useWinVersion: - var s = winlean.gethostbyaddr(addr(myaddr), sizeof(myaddr).cuint, - cint(AF_INET)) - if s == nil: raiseOSError(osLastError()) - else: - var s = - when defined(android4): - posix.gethostbyaddr(cast[cstring](addr(myaddr)), sizeof(myaddr).cint, - cint(posix.AF_INET)) - else: - posix.gethostbyaddr(addr(myaddr), sizeof(myaddr).SockLen, - cint(posix.AF_INET)) - if s == nil: - raiseOSError(osLastError(), $hstrerror(h_errno)) - - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostByName*(name: string): Hostent {.tags: [ReadIOEffect].} = - ## This function will lookup the IP address of a hostname. - when useWinVersion: - var s = winlean.gethostbyname(name) - else: - var s = posix.gethostbyname(name) - if s == nil: raiseOSError(osLastError()) - result.name = $s.h_name - result.aliases = cstringArrayToSeq(s.h_aliases) - when useWinVersion: - result.addrtype = Domain(s.h_addrtype) - else: - if s.h_addrtype == posix.AF_INET: - result.addrtype = AF_INET - elif s.h_addrtype == posix.AF_INET6: - result.addrtype = AF_INET6 - else: - raiseOSError(osLastError(), "unknown h_addrtype") - if result.addrtype == AF_INET: - result.addrList = @[] - var i = 0 - while not isNil(s.h_addr_list[i]): - var inaddrPtr = cast[ptr InAddr](s.h_addr_list[i]) - result.addrList.add($inet_ntoa(inaddrPtr[])) - inc(i) - else: - result.addrList = cstringArrayToSeq(s.h_addr_list) - result.length = int(s.h_length) - -proc getHostname*(): string {.tags: [ReadIOEffect].} = - ## Returns the local hostname (not the FQDN) - # https://tools.ietf.org/html/rfc1035#section-2.3.1 - # https://tools.ietf.org/html/rfc2181#section-11 - const size = 256 - result = newString(size) - when useWinVersion: - let success = winlean.gethostname(result, size) - else: - # Posix - let success = posix.gethostname(result, size) - if success != 0.cint: - raiseOSError(osLastError()) - let x = len(cstring(result)) - result.setLen(x) - proc getSockDomain*(socket: SocketHandle): Domain = ## Returns the socket's domain (AF_INET or AF_INET6). var name: Sockaddr_in6 @@ -464,162 +349,354 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -proc getAddrString*(sockAddr: ptr SockAddr): string = - ## Returns the string representation of address within sockAddr - if sockAddr.sa_family.cint == nativeAfInet: - result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr) - elif sockAddr.sa_family.cint == nativeAfInet6: - let addrLen = when not useWinVersion: posix.INET6_ADDRSTRLEN.int - else: 46 # it's actually 46 in both cases - result = newString(addrLen) - let addr6 = addr cast[ptr Sockaddr_in6](sockAddr).sin6_addr - when not useWinVersion: - if posix.inet_ntop(posix.AF_INET6, addr6, addr result[0], - result.len.int32) == nil: - raiseOSError(osLastError()) - if posix.IN6_IS_ADDR_V4MAPPED(addr6) != 0: - result.setSlice("::ffff:".len..= Sockaddr_un_path_length: + raise newException(ValueError, "socket path too long") + copyMem(addr result.sun_path, path.cstring, path.len + 1) + + proc getSockName*(socket: SocketHandle): Port = + ## Returns the socket's associated port number. + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + #name.sin_port = htons(cint16(port)) + #name.sin_addr.s_addr = htonl(INADDR_ANY) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = Port(nativesockets.ntohs(name.sin_port)) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: raiseOSError(osLastError()) - else: - raise newException(IOError, "Unknown socket family in getAddrString") - setLen(strAddress, len(cstring(strAddress))) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") -when defined(posix) and not defined(nimdoc): - proc makeUnixAddr*(path: string): Sockaddr_un = - result.sun_family = AF_UNIX.TSa_Family - if path.len >= Sockaddr_un_path_length: - raise newException(ValueError, "socket path too long") - copyMem(addr result.sun_path, path.cstring, path.len + 1) + proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's peer address and port number. + ## + ## Similar to POSIX's `getpeername`:idx: + case domain + of AF_INET: + var name: Sockaddr_in + when useWinVersion: + name.sin_family = uint16(ord(AF_INET)) + else: + name.sin_family = TSa_Family(posix.AF_INET) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + result = ($inet_ntoa(name.sin_addr), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name: Sockaddr_in6 + when useWinVersion: + name.sin6_family = uint16(ord(AF_INET6)) + else: + name.sin6_family = TSa_Family(posix.AF_INET6) + var namelen = sizeof(name).SockLen + if getpeername(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + # Cannot use INET6_ADDRSTRLEN here, because it's a C define. + result[0] = newString(64) + if inet_ntop(name.sin6_family.cint, + addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: + raiseOSError(osLastError()) + setLen(result[0], result[0].cstring.len) + result[1] = Port(nativesockets.ntohs(name.sin6_port)) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + +when useNimNetLite: -proc getSockName*(socket: SocketHandle): Port = - ## Returns the socket's associated port number. - var name: Sockaddr_in when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - #name.sin_port = htons(cint16(port)) - #name.sin_addr.s_addr = htonl(INADDR_ANY) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = Port(nativesockets.ntohs(name.sin_port)) + const + INET_ADDRSTRLEN = 16 + INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases -proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's local address and port number. - ## - ## Similar to POSIX's `getsockname`:idx:. - case domain - of AF_INET: - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) - else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) - else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getsockname(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: - raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc sockAddrToStr(sa: ptr Sockaddr): string {.noinit.} = + let af_family = sa.sa_family + var nl, v4Slice: cint + var si_addr: ptr InAddr -proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) = - ## Returns the socket's peer address and port number. - ## - ## Similar to POSIX's `getpeername`:idx: - case domain - of AF_INET: - var name: Sockaddr_in - when useWinVersion: - name.sin_family = uint16(ord(AF_INET)) + if af_family == AF_INET.TSa_Family: + nl = INET_ADDRSTRLEN + si_addr = cast[ptr Sockaddr_in](sa).sin_addr.addr() + elif af_family == AF_INET6.TSa_Family: + nl = INET6_ADDRSTRLEN + let si6_addr = cast[ptr Sockaddr_in6](sa).sin6_addr.addr() + si_addr = cast[ptr InAddr](si6_addr) # let's us reuse logic below + when defined(posix) and not defined(nimdoc) and not defined(zephyr): + if posix.IN6_IS_ADDR_V4MAPPED(si6_addr) != 0: + v4Slice = "::ffff:".len() else: - name.sin_family = TSa_Family(posix.AF_INET) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: - raiseOSError(osLastError()) - result = ($inet_ntoa(name.sin_addr), - Port(nativesockets.ntohs(name.sin_port))) - of AF_INET6: - var name: Sockaddr_in6 - when useWinVersion: - name.sin6_family = uint16(ord(AF_INET6)) + when defined(posix) and not defined(nimdoc): + if af_family.cint == nativeAfUnix: + return "unix" + return "" + + result = newString(nl) + let namePtr = result.cstring() + if namePtr == inet_ntop(af_family.cint, si_addr, namePtr, nl): + result.setLen(len(namePtr)) + if v4Slice > 0: result.setSlice(v4Slice.int ..< nl.int) else: - name.sin6_family = TSa_Family(posix.AF_INET6) - var namelen = sizeof(name).SockLen - if getpeername(socket, cast[ptr SockAddr](addr(name)), - addr(namelen)) == -1'i32: + return "" + + proc sockAddrToStr(sa: var Sockaddr_in | var Sockaddr_in6): string = + result = sockAddrToStr(cast[ptr SockAddr](unsafeAddr(sa))) + + proc getAddrString*(sockAddr: ptr SockAddr): string = + result = sockAddrToStr(sockAddr) + if result.len() == 0: raiseOSError(osLastError()) - # Cannot use INET6_ADDRSTRLEN here, because it's a C define. - result[0] = newString(64) - if inet_ntop(name.sin6_family.cint, - addr name.sin6_addr, addr result[0][0], (result[0].len+1).int32).isNil: - raiseOSError(osLastError()) - setLen(result[0], result[0].cstring.len) - result[1] = Port(nativesockets.ntohs(name.sin6_port)) - else: - raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + + proc getAddrString*(sockAddr: ptr SockAddr, strAddress: var string) {.noinit.} = + strAddress = getAddrString(sockAddr) + + proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) = + ## Returns the socket's local address and port number. + ## + ## Similar to POSIX's `getsockname`:idx:. + template sockGetNameOrRaiseError(socket: untyped, name: untyped) = + var namelen = sizeof(socket).SockLen + if getsockname(socket, cast[ptr SockAddr](addr(name)), + addr(namelen)) == -1'i32: + raiseOSError(osLastError()) + + case domain + of AF_INET: + var name = Sockaddr_in(sin_family: TSa_Family(posix.AF_INET)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin_port))) + of AF_INET6: + var name = Sockaddr_in6(sin6_family: TSa_Family(posix.AF_INET6)) + sockGetNameOrRaiseError(socket, name) + result = (sockAddrToStr(name), + Port(nativesockets.ntohs(name.sin6_port))) + else: + raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr") + proc getSockOptInt*(socket: SocketHandle, level, optname: int): int {. tags: [ReadIOEffect].} = @@ -733,14 +810,14 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke ## child processes. ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr_in + var sockAddress: Sockaddr var addrLen = sizeof(sockAddress).SockLen var sock = when (defined(linux) or defined(bsd)) and not defined(nimdoc): - accept4(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen), + accept4(fd, addr(sockAddress), addr(addrLen), if inheritable: 0 else: SOCK_CLOEXEC) else: - accept(fd, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) + accept(fd, addr(sockAddress), addr(addrLen)) when declared(setInheritable) and not (defined(linux) or defined(bsd)): if not setInheritable(sock, inheritable): close sock @@ -748,7 +825,11 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke if sock == osInvalidSocket: return (osInvalidSocket, "") else: - return (sock, $inet_ntoa(sockAddress.sin_addr)) + when useNimNetLite: + var name = sockAddrToStr(addr sockAddress) + return (sock, name) + else: + return (sock, $inet_ntoa(cast[Sockaddr_in](sockAddress).sin_addr)) when defined(windows): var wsa: WSAData diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 09cdc8afb..7b0ff78e7 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -85,12 +85,14 @@ runnableExamples("-r:off"): import std/private/since -import nativesockets, os, strutils, times, sets, options, std/monotimes +import nativesockets +import os, strutils, times, sets, options, std/monotimes import ssl_config export nativesockets.Port, nativesockets.`$`, nativesockets.`==` export Domain, SockType, Protocol const useWinVersion = defined(windows) or defined(nimdoc) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: @@ -1243,11 +1245,12 @@ proc getLocalAddr*(socket: Socket): (string, Port) = ## This is high-level interface for `getsockname`:idx:. getLocalAddr(socket.fd, socket.domain) -proc getPeerAddr*(socket: Socket): (string, Port) = - ## Get the socket's peer address and port number. - ## - ## This is high-level interface for `getpeername`:idx:. - getPeerAddr(socket.fd, socket.domain) +when not useNimNetLite: + proc getPeerAddr*(socket: Socket): (string, Port) = + ## Get the socket's peer address and port number. + ## + ## This is high-level interface for `getpeername`:idx:. + getPeerAddr(socket.fd, socket.domain) proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.tags: [WriteIOEffect].} = @@ -1259,7 +1262,7 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(posix) or defined(nimdoc): +when defined(nimdoc) or (defined(posix) and not useNimNetLite): proc connectUnix*(socket: Socket, path: string) = ## Connects to Unix socket on `path`. ## This only works on Unix-style systems: Mac OS X, BSD and Linux diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 82550e09b..ec441f6da 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -323,6 +323,23 @@ else: # Anything higher is the time to wait in milliseconds. doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) + when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or + defined(zephyr) or defined(freertos): + template maxDescriptors*(): int = + ## Returns the maximum number of active file descriptors for the current + ## process. This involves a system call. For now `maxDescriptors` is + ## supported on the following OSes: Windows, Linux, OSX, BSD. + when defined(windows): + 16_700_000 + elif defined(zephyr) or defined(freertos): + FD_MAX + else: + var fdLim: RLimit + var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) + if res >= 0: + res = int(fdLim.rlim_cur) - 1 + res + when defined(linux) and not defined(emscripten): include ioselects/ioselectors_epoll elif bsdPlatform: @@ -337,5 +354,7 @@ else: include ioselects/ioselectors_select elif defined(freertos) or defined(lwip): include ioselects/ioselectors_select + elif defined(zephyr): + include ioselects/ioselectors_poll else: include ioselects/ioselectors_poll diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index e5f26e070..259d36633 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -142,14 +142,29 @@ proc c_sprintf*(buf, frmt: cstring): cint {. importc: "sprintf", header: "", varargs, noSideEffect.} # we use it only in a way that cannot lead to security issues -proc c_malloc*(size: csize_t): pointer {. - importc: "malloc", header: "".} -proc c_calloc*(nmemb, size: csize_t): pointer {. - importc: "calloc", header: "".} -proc c_free*(p: pointer) {. - importc: "free", header: "".} -proc c_realloc*(p: pointer, newsize: csize_t): pointer {. - importc: "realloc", header: "".} +when defined(zephyr) and not defined(zephyrUseLibcMalloc): + proc c_malloc*(size: csize_t): pointer {. + importc: "k_malloc", header: "".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "k_calloc", header: "".} + proc c_free*(p: pointer) {. + importc: "k_free", header: "".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer = + # Zephyr's kernel malloc doesn't support realloc + result = c_malloc(newSize) + # match the ansi c behavior + if not result.isNil(): + copyMem(result, p, newSize) + c_free(p) +else: + proc c_malloc*(size: csize_t): pointer {. + importc: "malloc", header: "".} + proc c_calloc*(nmemb, size: csize_t): pointer {. + importc: "calloc", header: "".} + proc c_free*(p: pointer) {. + importc: "free", header: "".} + proc c_realloc*(p: pointer, newsize: csize_t): pointer {. + importc: "realloc", header: "".} proc c_fwrite*(buf: pointer, size, n: csize_t, f: CFilePtr): cint {. importc: "fwrite", header: "".} diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim index d7c55a4d9..caf86568a 100644 --- a/lib/system/bitmasks.nim +++ b/lib/system/bitmasks.nim @@ -15,6 +15,7 @@ const PageSize = 1 shl PageShift PageMask = PageSize-1 + MemAlign = # also minimal allocatable memory block when defined(useMalloc): when defined(amd64): 16 diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index c14fcf31e..36c2c5fe1 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -176,7 +176,7 @@ elif defined(genode): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = raiseAssert("nimGetProcAddr not implemented") -elif defined(nintendoswitch) or defined(freertos): +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr): proc nimUnloadLibrary(lib: LibHandle) = cstderr.rawWrite("nimUnLoadLibrary not implemented") cstderr.rawWrite("\n") diff --git a/lib/system/io.nim b/lib/system/io.nim index 59f070f73..6f4accc2a 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -347,7 +347,7 @@ when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(w ## availability with `declared() `. when SupportIoctlInheritCtl: result = c_ioctl(f, if inheritable: FIONCLEX else: FIOCLEX) != -1 - elif defined(freertos): + elif defined(freertos) or defined(zephyr): result = true elif defined(posix): var flags = c_fcntl(f, F_GETFD) @@ -780,8 +780,11 @@ when declared(stdout): var echoLock: SysLock initSysLock echoLock - const stdOutLock = not defined(windows) and not defined(android) and - not defined(nintendoswitch) and not defined(freertos) and + const stdOutLock = not defined(windows) and + not defined(android) and + not defined(nintendoswitch) and + not defined(freertos) and + not defined(zephyr) and hostOS != "any" proc echoBinSafe(args: openArray[string]) {.compilerproc.} = diff --git a/lib/system/mm/malloc.nim b/lib/system/mm/malloc.nim index 2ba5c0fec..d41dce705 100644 --- a/lib/system/mm/malloc.nim +++ b/lib/system/mm/malloc.nim @@ -2,13 +2,22 @@ {.push stackTrace: off.} proc allocImpl(size: Natural): pointer = - c_malloc(size.csize_t) + result = c_malloc(size.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc alloc0Impl(size: Natural): pointer = - c_calloc(size.csize_t, 1) + result = c_calloc(size.csize_t, 1) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc reallocImpl(p: pointer, newSize: Natural): pointer = - c_realloc(p, newSize.csize_t) + result = c_realloc(p, newSize.csize_t) + when defined(zephyr): + if result == nil: + raiseOutOfMem() proc realloc0Impl(p: pointer, oldsize, newSize: Natural): pointer = result = realloc(p, newSize.csize_t) diff --git a/tests/stdlib/tlwip.nim b/tests/stdlib/tlwip.nim index 7f7822236..fc53be592 100644 --- a/tests/stdlib/tlwip.nim +++ b/tests/stdlib/tlwip.nim @@ -1,6 +1,6 @@ discard """ targets: "c" - cmd: "nim $target --os:freertos --gc:arc $options $file" + cmd: "nim $target --compileOnly --os:freertos --gc:arc $options $file" disabled: "bsd" disabled: "windows" action: compile From 582468da1cc266d0dd922ec5c620713bbc45d02a Mon Sep 17 00:00:00 2001 From: Tail Wag Games Date: Mon, 25 Oct 2021 03:13:30 -0500 Subject: [PATCH 0490/1654] deinitializing locks at program exit (#19043) * deinitializing locks at program exit * deinitLock shouldn't be called for js backend I guess... * I suppose this is the best way to detect the ewruntime option * I guess I need these guards here too... * fixing merge conflict --- lib/std/exitprocs.nim | 3 +++ lib/system/io.nim | 56 ++++++++++++++++++++++++------------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c6537f7f8..c76583a8c 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -44,6 +44,9 @@ proc callClosures() {.noconv.} = of kClosure: fun.fun1() of kNoconv: fun.fun2() + when not defined(js) and not defined(nimOwnedEnabled): + deinitLock(gFunsLock) + template fun() = if gFuns.len == 0: addAtExit(callClosures) diff --git a/lib/system/io.nim b/lib/system/io.nim index 6f4accc2a..e7369392a 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -20,19 +20,19 @@ type incompleteStruct.} = object File* = ptr CFile ## The type representing a file handle. - FileMode* = enum ## The file mode when opening a file. - fmRead, ## Open the file for read access only. - fmWrite, ## Open the file for write access only. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWrite, ## Open the file for read and write access. - ## If the file does not exist, it will be - ## created. Existing files will be cleared! - fmReadWriteExisting, ## Open the file for read and write access. - ## If the file does not exist, it will not be - ## created. The existing file will not be cleared. - fmAppend ## Open the file for writing only; append data - ## at the end. + FileMode* = enum ## The file mode when opening a file. + fmRead, ## Open the file for read access only. + fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWrite, ## Open the file for read and write access. + ## If the file does not exist, it will be + ## created. Existing files will be cleared! + fmReadWriteExisting, ## Open the file for read and write access. + ## If the file does not exist, it will not be + ## created. The existing file will not be cleared. + fmAppend ## Open the file for writing only; append data + ## at the end. FileHandle* = cint ## type that represents an OS file handle; this is ## useful for low-level file access @@ -40,7 +40,8 @@ type # text file handling: when not defined(nimscript) and not defined(js): # duplicated between io and ansi_c - const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined(dragonfly)) and not defined(emscripten) + const stdioUsesMacros = (defined(osx) or defined(freebsd) or defined( + dragonfly)) and not defined(emscripten) const stderrName = when stdioUsesMacros: "__stderrp" else: "stderr" const stdoutName = when stdioUsesMacros: "__stdoutp" else: "stdout" const stdinName = when stdioUsesMacros: "__stdinp" else: "stdin" @@ -158,7 +159,7 @@ proc checkErr(f: File) = # shouldn't happen quit(1) -{.push stackTrace:off, profiler:off.} +{.push stackTrace: off, profiler: off.} proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} = ## reads `len` bytes into the buffer pointed to by `buffer`. Returns @@ -167,7 +168,8 @@ proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) if result != len: checkErr(f) -proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. +proc readBytes*(f: File, a: var openArray[int8|uint8], start, + len: Natural): int {. tags: [ReadIOEffect], benign.} = ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than @@ -418,8 +420,8 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], numberOfCharsRead -= 2 # handle Ctrl+Z as EOF for i in 0..", final, pure.} = object ## struct stat filler_1: array[24, char] - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file filler_2: array[144 - 24 - 4, char] proc modeIsDir(m: Mode): bool = @@ -660,7 +662,7 @@ when defined(posix) and not defined(nimscript): Stat {.importc: "struct stat", header: "", final, pure.} = object ## struct stat - st_mode: Mode ## Mode of file + st_mode: Mode ## Mode of file proc modeIsDir(m: Mode): bool {.importc: "S_ISDIR", header: "".} ## Test for a directory. @@ -727,7 +729,8 @@ proc open*(f: var File, filehandle: FileHandle, ## ## The passed file handle will no longer be inheritable. when not defined(nimInheritHandles) and declared(setInheritable): - let oshandle = when defined(windows): FileHandle getOsfhandle(filehandle) else: filehandle + let oshandle = when defined(windows): FileHandle getOsfhandle( + filehandle) else: filehandle if not setInheritable(oshandle, false): return false f = c_fdopen(filehandle, FormatOpen[mode]) @@ -780,6 +783,10 @@ when declared(stdout): var echoLock: SysLock initSysLock echoLock + when not defined(js) and not defined(nimOwnedEnabled): + import std/exitprocs + addExitProc(proc() {.noconv.} = deinitSys echoLock) + const stdOutLock = not defined(windows) and not defined(android) and not defined(nintendoswitch) and @@ -898,7 +905,8 @@ proc readLines*(filename: string, n: Natural): seq[string] = else: sysFatal(IOError, "cannot open: " & filename) -template readLines*(filename: string): seq[string] {.deprecated: "use readLines with two arguments".} = +template readLines*(filename: string): seq[ + string] {.deprecated: "use readLines with two arguments".} = readLines(filename, 1) iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = @@ -915,7 +923,7 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = for line in filename.lines: buffer.add(line.replace("a", "0") & '\n') writeFile(filename, buffer) - var f = open(filename, bufSize=8000) + var f = open(filename, bufSize = 8000) try: var res = newStringOfCap(80) while f.readLine(res): yield res From 3ecb369ca1a12754486429bda4ab0bc60f9b86d7 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Tue, 26 Oct 2021 02:33:26 -0400 Subject: [PATCH 0491/1654] Document file changes that may be needed for add a new platform (#19055) Co-authored-by: quantimnot --- doc/intern.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/doc/intern.rst b/doc/intern.rst index fe0d333fd..e8fb4ac86 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -207,6 +207,42 @@ implementation uses C's `setjmp`:c: function to store all registers on the hardware stack. It may be necessary that the new platform needs to replace this generic code by some assembler code. +Files that may need changed for your platform include: + +* `compiler/platform.nim` + Add os/cpu properties. +* `lib/system.nim` + Add os/cpu to the documentation for `system.hostOS` and `system.hostCPU`. +* `compiler/options.nim` + Add special os/cpu property checks in `isDefined`. +* `compiler/installer.ini` + Add os/cpu to `Project.Platforms` field. +* `lib/system/platforms.nim` + Add os/cpu. +* `lib/pure/include/osseps.nim` + Add os specializations. +* `lib/pure/distros.nim` + Add os, package handler. +* `tools/niminst/makefile.nimf` + Add os/cpu compiler/linker flags. +* `tools/niminst/buildsh.nimf` + Add os/cpu compiler/linker flags. + +If the `--os` or `--cpu` options aren't passed to the compiler, then Nim will +determine the current host os, cpu and endianess from `system.cpuEndian`, +`system.hostOS` and `system.hostCPU`. Those values are derived from +`compiler/platform.nim`. + +In order for the new platform to be bootstrapped from the `csources`, it must: + +* have `compiler/platform.nim` updated +* have `compiler/installer.ini` updated +* have `tools/niminst/buildsh.nimf` updated +* have `tools/niminst/makefile.nimf` updated +* be backported to the Nim version used by the `csources` +* the new `csources` must be pushed +* the new `csources` revision must be updated in `config/build_config.txt` + Runtime type information ======================== From 83a2515af7aeb9a1c12015321243399a0d1f4c95 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 26 Oct 2021 03:27:11 -0600 Subject: [PATCH 0492/1654] Fixed generic distinct conversions for 'var' (#18837) * SameTypeAux now properly traverses generic distincts * Smarter traversal of distincts * Removed redundant check * Fixed nkConv for jsgen * Added test for non distinct nkConv * using skiptypes for distinct now * Fixed genaddr for nkconv --- compiler/jsgen.nim | 140 +++++++++++++---------- compiler/types.nim | 8 +- tests/converter/texplicit_conversion.nim | 6 + tests/distinct/tdistinct.nim | 17 +++ 4 files changed, 104 insertions(+), 67 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index ff4d2839e..7e8c25173 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1288,75 +1288,89 @@ template isIndirect(x: PSym): bool = v.kind notin {skProc, skFunc, skConverter, skMethod, skIterator, skConst, skTemp, skLet}) -proc genAddr(p: PProc, n: PNode, r: var TCompRes) = - case n[0].kind - of nkSym: - let s = n[0].sym - if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3") - case s.kind - of skParam: - r.res = s.loc.r - r.address = nil - r.typ = etyNone - of skVar, skLet, skResult: - r.kind = resExpr - let jsType = mapType(p, n.typ) - if jsType == etyObject: - # make addr() a no-op: - r.typ = etyNone - if isIndirect(s): - r.res = s.loc.r & "[0]" - else: - r.res = s.loc.r - r.address = nil - elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex: - # for ease of code generation, we do not distinguish between - # sfAddrTaken and sfGlobal. - r.typ = etyBaseIndex - r.address = s.loc.r - r.res = rope("0") +proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = + let s = n.sym + if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3") + case s.kind + of skParam: + r.res = s.loc.r + r.address = nil + r.typ = etyNone + of skVar, skLet, skResult: + r.kind = resExpr + let jsType = mapType(p): + if typ.isNil: + n.typ else: - # 'var openArray' for instance produces an 'addr' but this is harmless: - gen(p, n[0], r) - #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n)) - else: internalError(p.config, n.info, $("genAddr: 2", s.kind)) - of nkCheckedFieldExpr: - genCheckedFieldOp(p, n[0], n.typ, r) - of nkDotExpr: - if mapType(p, n.typ) == etyBaseIndex: - genFieldAddr(p, n[0], r) + typ + if jsType == etyObject: + # make addr() a no-op: + r.typ = etyNone + if isIndirect(s): + r.res = s.loc.r & "[0]" + else: + r.res = s.loc.r + r.address = nil + elif {sfGlobal, sfAddrTaken} * s.flags != {} or jsType == etyBaseIndex: + # for ease of code generation, we do not distinguish between + # sfAddrTaken and sfGlobal. + r.typ = etyBaseIndex + r.address = s.loc.r + r.res = rope("0") else: - genFieldAccess(p, n[0], r) - of nkBracketExpr: - var ty = skipTypes(n[0].typ, abstractVarRange) - if ty.kind in MappedToObject: - gen(p, n[0], r) - else: - let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind - case kindOfIndexedExpr - of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs: - genArrayAddr(p, n[0], r) - of tyTuple: - genFieldAddr(p, n[0], r) - else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')') - of nkObjDownConv: - gen(p, n[0], r) - of nkHiddenDeref: - gen(p, n[0], r) - of nkHiddenAddr: - gen(p, n[0], r) - of nkStmtListExpr: - if n.len == 1: gen(p, n[0], r) - else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr") - of nkCallKinds: - if n[0].typ.kind == tyOpenArray: # 'var openArray' for instance produces an 'addr' but this is harmless: - # namely toOpenArray(a, 1, 3) + gen(p, n, r) + #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n)) + else: internalError(p.config, n.info, $("genAddr: 2", s.kind)) + +proc genAddr(p: PProc, n: PNode, r: var TCompRes) = + if n.kind == nkSym: + genSymAddr(p, n, nil, r) + else: + case n[0].kind + of nkSym: + genSymAddr(p, n[0], n.typ, r) + of nkCheckedFieldExpr: + genCheckedFieldOp(p, n[0], n.typ, r) + of nkDotExpr: + if mapType(p, n.typ) == etyBaseIndex: + genFieldAddr(p, n[0], r) + else: + genFieldAccess(p, n[0], r) + of nkBracketExpr: + var ty = skipTypes(n[0].typ, abstractVarRange) + if ty.kind in MappedToObject: + gen(p, n[0], r) + else: + let kindOfIndexedExpr = skipTypes(n[0][0].typ, abstractVarRange).kind + case kindOfIndexedExpr + of tyArray, tyOpenArray, tySequence, tyString, tyCstring, tyVarargs: + genArrayAddr(p, n[0], r) + of tyTuple: + genFieldAddr(p, n[0], r) + of tyGenericBody: + genAddr(p, n[^1], r) + else: internalError(p.config, n[0].info, "expr(nkBracketExpr, " & $kindOfIndexedExpr & ')') + of nkObjDownConv: gen(p, n[0], r) + of nkHiddenDeref: + gen(p, n[0], r) + of nkHiddenAddr: + gen(p, n[0], r) + of nkConv: + genAddr(p, n[0], r) + of nkStmtListExpr: + if n.len == 1: gen(p, n[0], r) + else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr") + of nkCallKinds: + if n[0].typ.kind == tyOpenArray: + # 'var openArray' for instance produces an 'addr' but this is harmless: + # namely toOpenArray(a, 1, 3) + gen(p, n[0], r) + else: + internalError(p.config, n[0].info, "genAddr: " & $n[0].kind) else: internalError(p.config, n[0].info, "genAddr: " & $n[0].kind) - else: - internalError(p.config, n[0].info, "genAddr: " & $n[0].kind) proc attachProc(p: PProc; content: Rope; s: PSym) = p.g.code.add(content) diff --git a/compiler/types.nim b/compiler/types.nim index cf6579317..2c7b91d9f 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1121,15 +1121,15 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = case c.cmp of dcEq: return false of dcEqIgnoreDistinct: - while a.kind == tyDistinct: a = a[0] - while b.kind == tyDistinct: b = b[0] + a = a.skipTypes({tyDistinct, tyGenericInst}) + b = b.skipTypes({tyDistinct, tyGenericInst}) if a.kind != b.kind: return false of dcEqOrDistinctOf: - while a.kind == tyDistinct: a = a[0] + a = a.skipTypes({tyDistinct, tyGenericInst}) if a.kind != b.kind: return false # this is required by tunique_type but makes no sense really: - if x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: + if tyDistinct notin {x.kind, y.kind} and x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: let lhs = x.skipGenericAlias rhs = y.skipGenericAlias diff --git a/tests/converter/texplicit_conversion.nim b/tests/converter/texplicit_conversion.nim index 6b2e96faf..e36d78ad5 100644 --- a/tests/converter/texplicit_conversion.nim +++ b/tests/converter/texplicit_conversion.nim @@ -11,3 +11,9 @@ converter toInt(s: string): int = let x = (int)"234" echo x + +block: # Test for nkconv + proc foo(o: var int) = + assert o == 0 + var a = 0 + foo(int(a)) \ No newline at end of file diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index fd60b4ac0..dd8237854 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -8,6 +8,7 @@ false false false Foo +foo ''' """ @@ -140,6 +141,22 @@ block tRequiresInit: let s = "test" doAssert s == "test" +block: #17322 + type + A[T] = distinct string + + proc foo(a: var A) = + a.string.add "foo" + + type + B = distinct A[int] + + var b: B + foo(A[int](b)) + echo A[int](b).string + b.string.add "bar" + assert b.string == "foobar" + type Foo = distinct string template main() = From 8d5a27518929bd4c54f4beb7e40a5fc382d3dd05 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 26 Oct 2021 03:29:07 -0600 Subject: [PATCH 0493/1654] Fixed distinct composite type class proc borrowing (#18904) * Fixed composite type class proc borrowing * Moved borrow search into transf * added borrow check to symbol flag --- compiler/semcall.nim | 13 +++++++++++-- compiler/seminst.nim | 6 +++++- compiler/transf.nim | 8 ++++++++ tests/borrow/typeclassborrow.nim | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 tests/borrow/typeclassborrow.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a3064788e..36658d472 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -703,7 +703,16 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym = call.add(newIdentNode(fn.name, fn.info)) for i in 1.. Date: Tue, 26 Oct 2021 13:32:52 -0500 Subject: [PATCH 0494/1654] Fix #19052; [backport:1.6.0] (#19053) * Fix #19052; [backport:1.6.0] Adds a compile flag to avoid a getrandom syscall, fixing #19052. This is neccesary when the getrandom syscall is missing, as noted in #19052, particularly in kernel versions < 3.17 when getrandom was introduced. Specifically relevant is this is missing from kernel 3.10, which is the supported kernel throughout RHEL 7 and CentOS 7, which is widely used at many organizations. Without this, versions of nim that include sysrand (i.e. versions >= 1.6.0) will not compile without modification, however with this change a compile flag may be used to fall back using /dev/urandom as done with any unknown Posix OS (preferred here as a fallback since it already supplies a cryptographically secure PRNG and existing code deals with entropy pool init, etc). The change is placed behind a compile flag, as discussed in github ticket #19052 (summed up here): * First, I can't seem to catch that a importc such as SYS_getrandom is declared without using it (the declared proc returns true, but compiler throws an undeclared identifier flag when referencing it). * Second, it seemed preferable to be behaviorally explicit vs implicit when considering this is intended to be a cryptographically secure PRNG. * Third, if I intend to compile on a kernel >= 3.17 while running the binary on at least one system < 3.17, I'll want to be able to target this without relying on a compile time determination if the getrandom syscall is available. * Documenting compile flag for -d:nimNoGetRandom and adding changelog entry Related to #19052 and comments in PR #19053. Also created a new changelog file since none currently exists. Co-authored-by: Timothy Alexander --- changelogs/changelog.md | 31 +++++++++++++++++++++++++++++++ lib/std/sysrand.nim | 7 ++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 changelogs/changelog.md diff --git a/changelogs/changelog.md b/changelogs/changelog.md new file mode 100644 index 000000000..475f568ec --- /dev/null +++ b/changelogs/changelog.md @@ -0,0 +1,31 @@ +# v1.xx.x - yyyy-mm-dd + +## Changes affecting backward compatibility + +## Standard library additions and changes + +### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall + +This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. + +On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. + +When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: +```sh +$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details +$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 +$ sh build.sh # per unix install instructions +$ bin/nim c koch # per unix install instructions +$ ./koch boot -d:release # per unix install instructions +$ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall +``` + +This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular). + +## Language changes + + +## Compiler changes + + +## Tool changes diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 75983e3c6..b35f24a72 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -38,6 +38,11 @@ ## .. _randomFillSync: https://nodejs.org/api/crypto.html#crypto_crypto_randomfillsync_buffer_offset_size ## .. _/dev/urandom: https://en.wikipedia.org/wiki//dev/random ## +## On a Linux target, a call to the `getrandom` syscall can be avoided (e.g. +## for targets running kernel version < 3.17) by passing a compile flag of +## `-d:nimNoGetRandom`. If this flag is passed, sysrand will use `/dev/urandom` +## as with any other POSIX compliant OS. +## runnableExamples: doAssert urandom(0).len == 0 @@ -159,7 +164,7 @@ elif defined(windows): result = randomBytes(addr dest[0], size) -elif defined(linux): +elif defined(linux) and not defined(nimNoGetRandom): # TODO using let, pending bootstrap >= 1.4.0 var SYS_getrandom {.importc: "SYS_getrandom", header: "".}: clong const syscallHeader = """#include From f50bcf82c3e45921961ee4145bac88701fec62b0 Mon Sep 17 00:00:00 2001 From: Netsu <52031254+WeebNetsu@users.noreply.github.com> Date: Tue, 26 Oct 2021 21:42:22 +0200 Subject: [PATCH 0495/1654] Minor update to terminal docs (#19056) * Update terminal.nim - Added some extra docs to cursorUp/Down/Forward/Backward - I was able to use hideCursor and showCursor without adding stdout, removed the parameter - Added docs to terminalHeight()* and terminalWidth()* * Update lib/pure/terminal.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/pure/terminal.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Added back f: file to cursor movement * Removed unnecessary comments Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/terminal.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 25b1a4cdd..c9aafc037 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -170,6 +170,7 @@ when defined(windows): return 0 proc terminalWidth*(): int = + ## Returns the terminal width in columns. var w: int = 0 w = terminalWidthIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -178,6 +179,7 @@ when defined(windows): return 80 proc terminalHeight*(): int = + ## Returns the terminal height in rows. var h: int = 0 h = terminalHeightIoctl([getStdHandle(STD_INPUT_HANDLE), getStdHandle(STD_OUTPUT_HANDLE), @@ -389,6 +391,9 @@ when defined(windows): proc cursorUp*(f: File, count = 1) = ## Moves the cursor up by `count` rows. + runnableExamples("-r:off"): + stdout.cursorUp(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -399,6 +404,9 @@ proc cursorUp*(f: File, count = 1) = proc cursorDown*(f: File, count = 1) = ## Moves the cursor down by `count` rows. + runnableExamples("-r:off"): + stdout.cursorDown(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -409,6 +417,9 @@ proc cursorDown*(f: File, count = 1) = proc cursorForward*(f: File, count = 1) = ## Moves the cursor forward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorForward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) @@ -419,6 +430,9 @@ proc cursorForward*(f: File, count = 1) = proc cursorBackward*(f: File, count = 1) = ## Moves the cursor backward by `count` columns. + runnableExamples("-r:off"): + stdout.cursorBackward(2) + write(stdout, "Hello World!") # anything written at that location will be erased/replaced with this when defined(windows): let h = conHandle(f) var p = getCursorPos(h) From ee65767d44cca79ce2677564151be69a60f6a94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Wed, 27 Oct 2021 17:45:05 +0800 Subject: [PATCH 0496/1654] fix a tiny formating issue in doc/destructors.rst (#19058) --- doc/destructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 195dc8f97..b4575b2e2 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -653,7 +653,7 @@ The ability to override a hook leads to a phase ordering problem: discard -The solution is to define `proc `=destroy`[T](f: var Foo[T])` before +The solution is to define ````proc `=destroy`[T](f: var Foo[T])```` before it is used. The compiler generates implicit hooks for all types in *strategic places* so that an explicitly provided hook that comes too "late" can be detected reliably. These *strategic places* From cd2b093d07303aa7bfcb4a007281f56de3755110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Thu, 28 Oct 2021 15:35:57 +0800 Subject: [PATCH 0497/1654] fix a tiny code snippet formatting issue in `doc/constructors.rst`, again (#19065) --- doc/destructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index b4575b2e2..6e4509666 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -653,7 +653,7 @@ The ability to override a hook leads to a phase ordering problem: discard -The solution is to define ````proc `=destroy`[T](f: var Foo[T])```` before +The solution is to define ``proc `=destroy`[T](f: var Foo[T])`` before it is used. The compiler generates implicit hooks for all types in *strategic places* so that an explicitly provided hook that comes too "late" can be detected reliably. These *strategic places* From c80e2c173686bd12904e5487752dc0ce20cb8bcb Mon Sep 17 00:00:00 2001 From: ynfle <23086821+ynfle@users.noreply.github.com> Date: Thu, 28 Oct 2021 11:02:18 +0300 Subject: [PATCH 0498/1654] Docs: change clipboard cursor type to `pointer` (#19064) --- tools/dochack/dochack.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 83e81f1c0..3a663808d 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -360,6 +360,7 @@ proc copyToClipboard*() {.exportc.} = const button = document.createElement("button") button.value = e.textContent.replace('...', '') button.classList.add("copyToClipBoardBtn") + button.style = "cursor: pointer" div.appendChild(preTag) div.appendChild(button) @@ -396,4 +397,4 @@ proc copyToClipboard*() {.exportc.} = """ .} -copyToClipboard() \ No newline at end of file +copyToClipboard() From 7ba2659f733b97db63b7552415ad048e34d4a11a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 28 Oct 2021 20:20:52 +0300 Subject: [PATCH 0499/1654] docgen: implement doc link resolution in current module (#18642) --- compiler/docgen.nim | 154 ++++++++- compiler/lineinfos.nim | 2 + compiler/typesrenderer.nim | 66 +++- config/nimdoc.cfg | 11 + config/nimdoc.tex.cfg | 10 +- doc/docgen.rst | 170 ++++++++++ doc/nimdoc.css | 4 + lib/packages/docutils/dochelpers.nim | 267 +++++++++++++++ lib/packages/docutils/rst.nim | 319 ++++++++++++++---- lib/packages/docutils/rstast.nim | 3 + lib/packages/docutils/rstgen.nim | 27 +- lib/pure/strutils.nim | 5 + .../expected/index.html | 4 + nimdoc/testproject/expected/nimdoc.out.css | 4 + .../expected/subdir/subdir_b/utils.html | 297 +++++++++++++++- .../expected/subdir/subdir_b/utils.idx | 14 + nimdoc/testproject/expected/testproject.html | 210 +++++++++++- nimdoc/testproject/expected/theindex.html | 52 ++- nimdoc/testproject/subdir/subdir_b/utils.nim | 65 ++++ .../subdir/subdir_b/utils_helpers.nim | 1 + .../subdir/subdir_b/utils_overview.rst | 8 + tests/stdlib/tdochelpers.nim | 155 +++++++++ tests/stdlib/trst.nim | 109 +++++- tests/stdlib/trstgen.nim | 19 +- 24 files changed, 1833 insertions(+), 143 deletions(-) create mode 100644 lib/packages/docutils/dochelpers.nim create mode 100644 nimdoc/testproject/subdir/subdir_b/utils_helpers.nim create mode 100644 nimdoc/testproject/subdir/subdir_b/utils_overview.rst create mode 100644 tests/stdlib/tdochelpers.nim diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 1acfc7489..2639840d1 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -13,7 +13,7 @@ import ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents, wordrecg, syntaxes, renderer, lexer, - packages/docutils/rst, packages/docutils/rstgen, + packages/docutils/[rst, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, pathutils, tables, nimpaths, renderverbatim, osproc @@ -44,8 +44,14 @@ type ## runnableExamples). substitutions: seq[string] ## Variable names in `doc.item`... sortName: string ## The string used for sorting in output + info: rstast.TLineInfo ## place where symbol was defined (for messages) + anchor: string ## e.g. HTML anchor + name: string ## short name of the symbol, not unique + ## (includes backticks ` if present) + detailedName: string ## longer name like `proc search(x: int): int` ModSection = object ## Section like Procs, Types, etc. - secItems: seq[Item] ## Pre-processed items. + secItems: Table[string, seq[Item]] + ## Map basic name -> pre-processed items. finalMarkup: string ## The items, after RST pass 2 and rendering. ModSections = array[TSymKind, ModSection] TocItem = object ## HTML TOC item @@ -91,12 +97,22 @@ type thisDir*: AbsoluteDir exampleGroups: OrderedTable[string, ExampleGroup] wroteSupportFiles*: bool + nimToRstFid: Table[lineinfos.FileIndex, rstast.FileIndex] + ## map Nim FileIndex -> RST one, it's needed because we keep them separate PDoc* = ref TDocumentor ## Alias to type less. proc add(dest: var ItemPre, rst: PRstNode) = dest.add ItemFragment(isRst: true, rst: rst) proc add(dest: var ItemPre, str: string) = dest.add ItemFragment(isRst: false, str: str) +proc addRstFileIndex(d: PDoc, info: lineinfos.TLineInfo): rstast.FileIndex = + let invalid = rstast.FileIndex(-1) + result = d.nimToRstFid.getOrDefault(info.fileIndex, default = invalid) + if result == invalid: + let fname = toFullPath(d.conf, info) + result = addFilename(d.sharedState, fname) + d.nimToRstFid[info.fileIndex] = result + proc cmpDecimalsIgnoreCase(a, b: string): int = ## For sorting with correct handling of cases like 'uint8' and 'uint16'. ## Also handles leading zeros well (however note that leading zeros are @@ -223,6 +239,7 @@ template declareClosures = of meFootnoteMismatch: k = errRstFootnoteMismatch of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX + of mwAmbiguousLink: k = warnRstAmbiguousLink of mwBrokenLink: k = warnRstBrokenLink of mwUnsupportedLanguage: k = warnRstLanguageXNotSupported of mwUnsupportedField: k = warnRstFieldXNotSupported @@ -236,7 +253,7 @@ template declareClosures = result = getCurrentDir() / s if not fileExists(result): result = "" -proc parseRst(text, filename: string, +proc parseRst(text: string, line, column: int, conf: ConfigRef, sharedState: PRstSharedState): PRstNode = declareClosures() @@ -352,7 +369,8 @@ proc getVarIdx(varnames: openArray[string], id: string): int = proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: - result = parseRst(n.comment, toFullPath(d.conf, n.info), + d.sharedState.currFileIdx = addRstFileIndex(d, n.info) + result = parseRst(n.comment, toLinenumber(n.info), toColumn(n.info) + DocColOffset, d.conf, d.sharedState) @@ -885,6 +903,57 @@ proc genSeeSrc(d: PDoc, path: string, line: int): string = "path", path.string, "line", $line, "url", gitUrl, "commit", commit, "devel", develBranch]]) +proc symbolPriority(k: TSymKind): int = + result = case k + of skMacro: -3 + of skTemplate: -2 + of skIterator: -1 + else: 0 # including skProc which have higher priority + # documentation itself has even higher priority 1 + +proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = + ## Converts symbol info (names/types/parameters) in `n` into format + ## `LangSymbol` convenient for ``rst.nim``/``dochelpers.nim``. + result.name = baseName.nimIdentNormalize + result.symKind = k.toHumanStr + if k in routineKinds: + var + paramTypes: seq[string] + renderParamTypes(paramTypes, n[paramsPos], toNormalize=true) + let paramNames = renderParamNames(n[paramsPos], toNormalize=true) + # In some rare cases (system.typeof) parameter type is not set for default: + doAssert paramTypes.len <= paramNames.len + for i in 0 ..< paramNames.len: + if i < paramTypes.len: + result.parameters.add (paramNames[i], paramTypes[i]) + else: + result.parameters.add (paramNames[i], "") + result.parametersProvided = true + + result.outType = renderOutType(n[paramsPos], toNormalize=true) + + if k in {skProc, skFunc, skType, skIterator}: + # Obtain `result.generics` + # Use `n[miscPos]` since n[genericParamsPos] does not contain constraints + var genNode: PNode = nil + if k == skType: + genNode = n[1] # FIXME: what is index 1? + else: + if n[miscPos].kind != nkEmpty: + genNode = n[miscPos][1] # FIXME: what is index 1? + if genNode != nil: + var literal = "" + var r: TSrcGen + initTokRender(r, genNode, {renderNoBody, renderNoComments, + renderNoPragmas, renderNoProcDefs}) + var kind = tkEof + while true: + getNextTok(r, kind, literal) + if kind == tkEof: + break + if kind != tkSpaces: + result.generics.add(literal.nimIdentNormalize) + proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let @@ -915,6 +984,8 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = inc(d.id) let plainNameEsc = esc(d.target, plainName.strip) + detailedName = k.toHumanStr & " " & ( + if k in routineKinds: plainName else: name) uniqueName = if k in routineKinds: plainNameEsc else: name sortName = if k in routineKinds: plainName.strip else: name cleanPlainSymbol = renderPlainSymbolName(nameNode) @@ -923,20 +994,32 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = symbolOrId = d.newUniquePlainSymbol(complexSymbol) symbolOrIdEnc = encodeUrl(symbolOrId, usePlus = false) deprecationMsg = genDeprecationMsg(d, pragmaNode) + rstLangSymbol = toLangSymbol(k, n, cleanPlainSymbol) + + # we generate anchors automatically for subsequent use in doc comments + let lineinfo = rstast.TLineInfo( + line: nameNode.info.line, col: nameNode.info.col, + fileIndex: addRstFileIndex(d, nameNode.info)) + addAnchorNim(d.sharedState, refn = symbolOrId, tooltip = detailedName, + rstLangSymbol, priority = symbolPriority(k), info = lineinfo) nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, renderDocComments, renderSyms}, symbolOrIdEnc) let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) - d.section[k].secItems.add Item( + d.section[k].secItems.mgetOrPut(cleanPlainSymbol, newSeq[Item]()).add Item( descRst: comm, sortName: sortName, + info: lineinfo, + anchor: symbolOrId, + detailedName: detailedName, + name: name, substitutions: @[ - "name", name, "uniqueName", uniqueName, + "uniqueName", uniqueName, "header", result, "itemID", $d.id, "header_plain", plainNameEsc, "itemSym", cleanPlainSymbol, - "itemSymOrID", symbolOrId, "itemSymEnc", plainSymbolEnc, + "itemSymEnc", plainSymbolEnc, "itemSymOrIDEnc", symbolOrIdEnc, "seeSrc", seeSrc, "deprecationMsg", deprecationMsg]) @@ -1184,6 +1267,11 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = if comm.len != 0: d.modDescPre.add(comm) else: discard +proc overloadGroupName(s: string, k: TSymKind): string = + ## Turns a name like `f` into anchor `f-procs-all` + #s & " " & k.toHumanStr & "s all" + s & "-" & k.toHumanStr & "s-all" + proc finishGenerateDoc*(d: var PDoc) = ## Perform 2nd RST pass for resolution of links/footnotes/headings... # copy file map `filenames` to ``rstgen.nim`` for its warnings @@ -1197,6 +1285,21 @@ proc finishGenerateDoc*(d: var PDoc) = break preparePass2(d.sharedState, firstRst) + # add anchors to overload groups before RST resolution + for k in TSymKind: + if k in routineKinds: + for plainName, overloadChoices in d.section[k].secItems: + if overloadChoices.len > 1: + let refn = overloadGroupName(plainName, k) + let tooltip = "$1 ($2 overloads)" % [ + k.toHumanStr & " " & plainName, $overloadChoices.len] + addAnchorNim(d.sharedState, refn, tooltip, + LangSymbol(symKind: k.toHumanStr, name: plainName, + isGroup: true), + priority = symbolPriority(k), + # select index `0` just to have any meaningful warning: + info = overloadChoices[0].info) + # Finalize fragments of ``.nim`` or ``.rst`` file proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = for f in fragments: @@ -1207,14 +1310,33 @@ proc finishGenerateDoc*(d: var PDoc) = of false: result &= f.str proc cmp(x, y: Item): int = cmpDecimalsIgnoreCase(x.sortName, y.sortName) for k in TSymKind: - for item in d.section[k].secItems.sorted(cmp): - var itemDesc: string - renderItemPre(d, item.descRst, itemDesc) - d.section[k].finalMarkup.add( - getConfigVar(d.conf, "doc.item") % ( - item.substitutions & @["desc", itemDesc])) - itemDesc = "" - d.section[k].secItems.setLen 0 + # add symbols to section for each `k`, while optionally wrapping + # overloadable items with the same basic name by ``doc.item2`` + let overloadableNames = toSeq(keys(d.section[k].secItems)) + for plainName in overloadableNames.sorted(cmpDecimalsIgnoreCase): + var overloadChoices = d.section[k].secItems[plainName] + overloadChoices.sort(cmp) + var nameContent = "" + for item in overloadChoices: + var itemDesc: string + renderItemPre(d, item.descRst, itemDesc) + nameContent.add( + getConfigVar(d.conf, "doc.item") % ( + item.substitutions & @[ + "desc", itemDesc, + "name", item.name, + "itemSymOrID", item.anchor])) + if k in routineKinds: + let plainNameEsc1 = esc(d.target, plainName.strip) + let plainNameEsc2 = esc(d.target, plainName.strip, escMode=emUrl) + d.section[k].finalMarkup.add( + getConfigVar(d.conf, "doc.item2") % ( + @["header_plain", plainNameEsc1, + "overloadGroupName", overloadGroupName(plainNameEsc2, k), + "content", nameContent])) + else: + d.section[k].finalMarkup.add(nameContent) + d.section[k].secItems.clear renderItemPre(d, d.modDescPre, d.modDescFinal) d.modDescPre.setLen 0 d.hasToc = d.hasToc or d.sharedState.hasToc @@ -1493,7 +1615,7 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; filename: AbsoluteFile, outExt: string) = var filen = addFileExt(filename, "txt") var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) - let rst = parseRst(readFile(filen.string), filen.string, + let rst = parseRst(readFile(filen.string), line=LineRstInit, column=ColRstInit, conf, d.sharedState) d.modDescPre = @[ItemFragment(isRst: true, rst: rst)] diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 8bd5a0890..e13387be6 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -50,6 +50,7 @@ type warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic", warnRstRedefinitionOfLabel = "RedefinitionOfLabel", warnRstUnknownSubstitutionX = "UnknownSubstitutionX", + warnRstAmbiguousLink = "AmbiguousLink", warnRstBrokenLink = "BrokenLink", warnRstLanguageXNotSupported = "LanguageXNotSupported", warnRstFieldXNotSupported = "FieldXNotSupported", @@ -123,6 +124,7 @@ const warnUnknownMagic: "unknown magic '$1' might crash the compiler", warnRstRedefinitionOfLabel: "redefinition of label '$1'", warnRstUnknownSubstitutionX: "unknown substitution '$1'", + warnRstAmbiguousLink: "ambiguous doc link $1", warnRstBrokenLink: "broken link '$1'", warnRstLanguageXNotSupported: "language '$1' not supported", warnRstFieldXNotSupported: "field '$1' not supported", diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim index 0da05d70d..c81ff283e 100644 --- a/compiler/typesrenderer.nim +++ b/compiler/typesrenderer.nim @@ -11,6 +11,12 @@ import renderer, strutils, ast, types const defaultParamSeparator* = "," +template mayNormalize(s: string): string = + if toNormalize: + s.nimIdentNormalize + else: + s + proc renderPlainSymbolName*(n: PNode): string = ## Returns the first non '*' nkIdent node from the tree. ## @@ -30,24 +36,26 @@ proc renderPlainSymbolName*(n: PNode): string = result = "" #internalError(n.info, "renderPlainSymbolName() with " & $n.kind) -proc renderType(n: PNode): string = +proc renderType(n: PNode, toNormalize: bool): string = ## Returns a string with the node type or the empty string. + ## This proc should be kept in sync with `toLangSymbols` from + ## ``lib/packages/docutils/dochelpers.nim``. case n.kind: - of nkIdent: result = n.ident.s - of nkSym: result = typeToString(n.sym.typ) + of nkIdent: result = mayNormalize(n.ident.s) + of nkSym: result = mayNormalize(typeToString(n.sym.typ)) of nkVarTy: if n.len == 1: - result = renderType(n[0]) + result = renderType(n[0], toNormalize) else: result = "var" of nkRefTy: if n.len == 1: - result = "ref." & renderType(n[0]) + result = "ref." & renderType(n[0], toNormalize) else: result = "ref" of nkPtrTy: if n.len == 1: - result = "ptr." & renderType(n[0]) + result = "ptr." & renderType(n[0], toNormalize) else: result = "ptr" of nkProcTy: @@ -57,36 +65,53 @@ proc renderType(n: PNode): string = assert params.kind == nkFormalParams assert params.len > 0 result = "proc(" - for i in 1..= 3 let typePos = n.len - 2 - let typeStr = renderType(n[typePos]) + let typeStr = renderType(n[typePos], toNormalize) result = typeStr for i in 1..= 2 - result = renderType(n[0]) & '[' - for i in 1.. 1: result.add ", " - result.add(renderType(n[i])) + result.add(renderType(n[i], toNormalize)) else: result = "" -proc renderParamTypes(found: var seq[string], n: PNode) = +proc renderParamNames*(n: PNode, toNormalize=false): seq[string] = + ## Returns parameter names of routine `n`. + doAssert n.kind == nkFormalParams + case n.kind + of nkFormalParams: + for i in 1.. 0 - var typeStr = renderType(n[typePos]) + var typeStr = renderType(n[typePos], toNormalize) if typeStr.len < 1 and n[typePos+1].kind != nkEmpty: # Try with the last node, maybe its a default value. let typ = n[typePos+1].typ @@ -111,7 +136,8 @@ proc renderParamTypes(found: var seq[string], n: PNode) = found.add($n) #internalError(n.info, "renderParamTypes(found,n) with " & $n.kind) -proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string = +proc renderParamTypes*(n: PNode, sep = defaultParamSeparator, + toNormalize=false): string = ## Returns the types contained in `n` joined by `sep`. ## ## This proc expects to be passed as `n` the parameters of any callable. The @@ -120,6 +146,10 @@ proc renderParamTypes*(n: PNode, sep = defaultParamSeparator): string = ## other characters may appear too, like ``[]`` or ``|``. result = "" var found: seq[string] = @[] - renderParamTypes(found, n) + renderParamTypes(found, n, toNormalize) if found.len > 0: result = found.join(sep) + +proc renderOutType*(n: PNode, toNormalize=false): string = + assert n.kind == nkFormalParams + result = renderType(n[0], toNormalize) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 162b2b4a3..c1074f344 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -56,6 +56,17 @@ $seeSrc
    """ +# A wrapper of a few overloaded `doc.item`s with the same basic name +# * $header_plain - see above +# * $overloadGroupName - the anchor for this whole group +# * $content - string containing `doc.item`s themselves +doc.item2 = """ + +
    +$content +
    +""" + # Chunk of HTML emitted for each entry in the HTML table of contents. # See doc.item for available substitution variables. diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 4aff9b379..d7ab652eb 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -19,7 +19,8 @@ doc.section.toc = "" doc.item = """ \vspace{1em} -\phantomsection\addcontentsline{toc}{subsection}{$uniqueName} +\phantomsection\addcontentsline{toc}{subsubsection}{$uniqueName} +\label{$itemSymOrID}\hypertarget{$itemSymOrID}{} \begin{rstdocitem} $header @@ -30,6 +31,13 @@ $desc \end{addmargin} """ +doc.item2 = """ +\phantomsection\addcontentsline{toc}{subsection}{$header_plain} +\label{$overloadGroupName}\hypertarget{$overloadGroupName}{} + +$content +""" + doc.item.toc = "" doc.toc = r"\tableofcontents \newpage" diff --git a/doc/docgen.rst b/doc/docgen.rst index 2074ee06f..51e7102eb 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -229,6 +229,176 @@ Output:: Note that the `jsondoc`:option: command outputs its JSON without pretty-printing it, while `jsondoc0`:option: outputs pretty-printed JSON. + +Referencing Nim symbols: simple documentation links +=================================================== + +You can reference Nim identifiers from Nim documentation comments, currently +only inside their ``.nim`` file (or inside a ``.rst`` file included from +a ``.nim``). The point is that such links will be resolved automatically +by `nim doc`:cmd: (or `nim jsondoc`:cmd: or `nim doc2tex`:cmd:). +This pertains to any exported symbol like `proc`, `const`, `iterator`, etc. +Syntax for referencing is basically a normal RST one: addition of +underscore `_` to a *link text*. +Link text is either one word or a group of words enclosed by backticks `\`` +(for a one word case backticks are usually omitted). +Link text will be displayed *as is* while *link target* will be set to +the anchor [*]_ of Nim symbol that corresponds to link text. + +.. [*] anchors' format is described in `HTML anchor generation`_ section below. + +If you have a constant: + +.. code:: Nim + + const pi* = 3.14 + +then it should be referenced in one of the 2 forms: + +A. non-qualified (no symbol kind specification):: + pi_ +B. qualified (with symbol kind specification):: + `const pi`_ + +For routine kinds there are more options. Consider this definition: + +.. code:: Nim + + proc foo*(a: int, b: float): string + +Generally following syntax is allowed for referencing `foo`: + +* short (without parameters): + + A. non-qualified:: + + foo_ + + B. qualified:: + + `proc foo`_ + +* longer variants (with parameters): + + A. non-qualified: + + 1) specifying parameters names:: + + `foo(a, b)`_ + + 2) specifying parameters types:: + + `foo(int, float)`_ + + 3) specifying both names and types:: + + `foo(a: int, b: float)`_ + + 4) output parameter can also be specified if you wish:: + + `foo(a: int, b: float): string`_ + + B. qualified: all 4 options above are valid. + Particularly you can use the full format:: + + `proc foo(a: int, b: float): string`_ + +.. Tip:: Avoid cluttering your text with extraneous information by using + one of shorter forms:: + + binarySearch_ + `binarySearch(a, key, cmp)`_ + + Brevity is better for reading! If you use a short form and have an + ambiguity problem (see below) then just add some additional info. + +Symbol kind like `proc` can also be specified in the postfix form:: + + `foo proc`_ + `walkDir(d: string) iterator`_ + +.. Warning:: An ambiguity in resolving documentation links may arise because of: + + 1. clash with other RST anchors + * manually setup anchors + * automatically set up, e.g. section names + 2. collision with other Nim symbols: + + * routines with different parameters can exist e.g. for + `proc` and `template`. In this case they are split between their + corresponding sections in output file. Qualified references are + useful in this case -- just disambiguate by referring to these + sections explicitly:: + + See `foo proc`_ and `foo template`_. + + * because in Nim `proc` and `iterator` belong to different namespaces, + so there can be a collision even if parameters are the same. + Use `\`proc foo\`_`:literal: or `\`iterator foo\`_`:literal: then. + + Any ambiguity is always reported with Nim compiler warnings and an anchor + with higher priority is selected. Manual anchors have highest + priority, then go automatic RST anchors; then Nim-generated anchors + (while procs have higher priority than other Nim symbol kinds). + +Generic parameters can also be used. All in all, this long form will be +recognized fine:: + + `proc binarySearch*[T; K](a: openArray[T], key: K, cmp: proc(T, K)): int`_ + +**Limitations**: + +1. The parameters of a nested routine type can be specified only with types + (without parameter names, see form A.2 above). + E.g. for this signature: + + .. code:: Nim + + proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int + ~~ ~~ ~~~~~ + + you cannot use names underlined by `~~` so it must be referenced with + ``cmp: proc(T, K)``. Hence these forms are valid:: + + `binarySearch(a: openArray[T], key: K, cmp: proc(T, K))`_ + `binarySearch(openArray[T], K, proc(T, K))`_ + `binarySearch(a, key, cmp)`_ +2. Default values in routine parameters are not recognized, one needs to + specify the type and/or name instead. E.g. for referencing `proc f(x = 7)` + use one of the mentioned forms:: + + `f(int)`_ or `f(x)`_ or `f(x: int)`_. +3. Generic parameters must be given the same way as in the + definition of referenced symbol. + + * their names should be the same + * parameters list should be given the same way, e.g. without substitutions + between commas (,) and semicolons (;). + +.. Note:: A bit special case is operators + (as their signature is also defined with `\``): + + .. code:: Nim + + func `$`(x: MyType): string + func `[]`*[T](x: openArray[T]): T + + A short form works without additional backticks:: + + `$`_ + `[]`_ + + However for fully-qualified reference copy-pasting backticks (`) into other + backticks will not work in our RST parser (because we use Markdown-like + inline markup rules). You need either to delete backticks or keep + them and escape with backslash \\:: + + no backticks: `func $`_ + escaped: `func \`$\``_ + no backticks: `func [][T](x: openArray[T]): T`_ + escaped: `func \`[]\`[T](x: openArray[T]): T`_ + Related Options =============== diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 3353f1cda..7c819ea30 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -263,6 +263,10 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; color: var(--text); } diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim new file mode 100644 index 000000000..c488c4d99 --- /dev/null +++ b/lib/packages/docutils/dochelpers.nim @@ -0,0 +1,267 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2021 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Integration helpers between ``docgen.nim`` and ``rst.nim``. +## +## Function `toLangSymbol(linkText)`_ produces a signature `docLink` of +## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ +## matches it with `generated`, produced from `PNode` by ``docgen.rst``. + +import rstast + +type + LangSymbol* = object ## symbol signature in Nim + symKind*: string ## "proc", "const", etc + name*: string ## plain symbol name without any parameters + generics*: string ## generic parameters (without brackets) + isGroup*: bool ## is LangSymbol a group with overloads? + # the following fields are valid iff `isGroup` == false + # (always false when parsed by `toLangSymbol` because link like foo_ + # can point to just a single symbol foo, e.g. proc). + parametersProvided*: bool ## to disambiguate `proc f`_ and `proc f()`_ + parameters*: seq[tuple[name: string, `type`: string]] + ## name-type seq, e.g. for proc + outType*: string ## result type, e.g. for proc + +func nimIdentBackticksNormalize*(s: string): string = + ## Normalizes the string `s` as a Nim identifier. + ## + ## Unlike `nimIdentNormalize` removes spaces and backticks. + ## + ## .. Warning:: No checking (e.g. that identifiers cannot start from + ## digits or '_', or that number of backticks is even) is performed. + runnableExamples: + doAssert nimIdentBackticksNormalize("Foo_bar") == "Foobar" + doAssert nimIdentBackticksNormalize("FoO BAr") == "Foobar" + doAssert nimIdentBackticksNormalize("`Foo BAR`") == "Foobar" + doAssert nimIdentBackticksNormalize("` Foo BAR `") == "Foobar" + # not a valid identifier: + doAssert nimIdentBackticksNormalize("`_x_y`") == "_xy" + result = newString(s.len) + var firstChar = true + var j = 0 + for i in 0..len(s) - 1: + if s[i] in {'A'..'Z'}: + if not firstChar: # to lowercase + result[j] = chr(ord(s[i]) + (ord('a') - ord('A'))) + else: + result[j] = s[i] + firstChar = false + inc j + elif s[i] notin {'_', ' ', '`'}: + result[j] = s[i] + inc j + firstChar = false + elif s[i] == '_' and firstChar: + result[j] = '_' + inc j + firstChar = false + else: discard # just omit '`' or ' ' + if j != s.len: setLen(result, j) + +proc toLangSymbol*(linkText: PRstNode): LangSymbol = + ## Parses `linkText` into a more structured form using a state machine. + ## + ## This proc is designed to allow link syntax with operators even + ## without escaped backticks inside:: + ## + ## `proc *`_ + ## `proc []`_ + ## + ## This proc should be kept in sync with the `renderTypes` proc from + ## ``compiler/typesrenderer.nim``. + assert linkText.kind in {rnRef, rnInner} + + const NimDefs = ["proc", "func", "macro", "method", "iterator", + "template", "converter", "const", "type", "var"] + type + State = enum + inBeginning + afterSymKind + beforeSymbolName # auxiliary state to catch situations like `proc []`_ after space + atSymbolName + afterSymbolName + genericsPar + parameterName + parameterType + outType + var state = inBeginning + var curIdent = "" + template flushIdent() = + if curIdent != "": + case state + of inBeginning: doAssert false, "incorrect state inBeginning" + of afterSymKind: result.symKind = curIdent + of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" + of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize + of afterSymbolName: doAssert false, "incorrect state afterSymbolName" + of genericsPar: result.generics = curIdent + of parameterName: result.parameters.add (curIdent, "") + of parameterType: + for a in countdown(result.parameters.len - 1, 0): + if result.parameters[a].`type` == "": + result.parameters[a].`type` = curIdent + of outType: result.outType = curIdent + curIdent = "" + var parens = 0 + let L = linkText.sons.len + template s(i: int): string = linkText.sons[i].text + var i = 0 + template nextState = + case s(i) + of " ": + if state == afterSymKind: + flushIdent + state = beforeSymbolName + of "`": + curIdent.add "`" + inc i + while i < L: # add contents between ` ` as a whole + curIdent.add s(i) + if s(i) == "`": + break + inc i + curIdent = curIdent.nimIdentBackticksNormalize + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + flushIdent + state = afterSymbolName + of "[": + if state notin {inBeginning, afterSymKind, beforeSymbolName}: + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + state = atSymbolName + curIdent.add s(i) + elif state in {atSymbolName, afterSymbolName} and parens == 1: + flushIdent + state = genericsPar + curIdent.add s(i) + else: curIdent.add s(i) + of "]": + if state notin {inBeginning, afterSymKind, beforeSymbolName, atSymbolName}: + dec parens + if state == genericsPar and parens == 0: + curIdent.add s(i) + flushIdent + else: curIdent.add s(i) + of "(": + inc parens + if state in {inBeginning, afterSymKind, beforeSymbolName}: + result.parametersProvided = true + state = atSymbolName + flushIdent + state = parameterName + elif state in {atSymbolName, afterSymbolName, genericsPar} and parens == 1: + result.parametersProvided = true + flushIdent + state = parameterName + else: curIdent.add s(i) + of ")": + dec parens + if state in {parameterName, parameterType} and parens == 0: + flushIdent + state = outType + else: curIdent.add s(i) + of "{": # remove pragmas + while i < L: + if s(i) == "}": + break + inc i + of ",", ";": + if state in {parameterName, parameterType} and parens == 1: + flushIdent + state = parameterName + else: curIdent.add s(i) + of "*": # skip export symbol + if state == atSymbolName: + flushIdent + state = afterSymbolName + elif state == afterSymbolName: + discard + else: curIdent.add "*" + of ":": + if state == outType: discard + elif state == parameterName: + flushIdent + state = parameterType + else: curIdent.add ":" + else: + let isPostfixSymKind = i > 0 and i == L - 1 and + result.symKind == "" and s(i) in NimDefs + if isPostfixSymKind: # for links like `foo proc`_ + result.symKind = s(i) + else: + case state + of inBeginning: + if s(i) in NimDefs: + state = afterSymKind + else: + state = atSymbolName + curIdent.add s(i) + of afterSymKind, beforeSymbolName: + state = atSymbolName + curIdent.add s(i) + of parameterType: + case s(i) + of "ref": curIdent.add "ref." + of "ptr": curIdent.add "ptr." + of "var": discard + else: curIdent.add s(i).nimIdentBackticksNormalize + of atSymbolName: + curIdent.add s(i) + else: + curIdent.add s(i).nimIdentBackticksNormalize + while i < L: + nextState + inc i + if state == afterSymKind: # treat `type`_ as link to symbol `type` + state = atSymbolName + flushIdent + result.isGroup = false + +proc match*(generated: LangSymbol, docLink: LangSymbol): bool = + ## Returns true if `generated` can be a target for `docLink`. + ## If `generated` is an overload group then only `symKind` and `name` + ## are compared for success. + result = true + if docLink.symKind != "": + if generated.symKind == "proc": + result = docLink.symKind in ["proc", "func"] + else: + result = generated.symKind == docLink.symKind + if not result: return + result = generated.name == docLink.name + if not result: return + if generated.isGroup: + # if `()` were added then it's not a reference to the whole group: + return not docLink.parametersProvided + if docLink.generics != "": + result = generated.generics == docLink.generics + if not result: return + if docLink.outType != "": + result = generated.outType == docLink.outType + if not result: return + if docLink.parametersProvided: + result = generated.parameters.len == docLink.parameters.len + if not result: return + var onlyType = false + for i in 0 ..< generated.parameters.len: + let g = generated.parameters[i] + let d = docLink.parameters[i] + if i == 0: + if g.`type` == d.name: + onlyType = true # only types, not names, are provided in `docLink` + if onlyType: + result = g.`type` == d.name: + else: + if d.`type` != "": + result = g.`type` == d.`type` + if not result: return + result = g.name == d.name + if not result: return diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 29234f28b..2e908f4e5 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -114,7 +114,7 @@ ## .. _`extra features`: ## ## Optional additional features, turned on by ``options: RstParseOption`` in -## `rstParse proc <#rstParse,string,string,int,int,bool,RstParseOptions,FindFileHandler,MsgHandler>`_: +## `proc rstParse`_: ## ## * emoji / smiley symbols ## * Markdown tables @@ -196,7 +196,7 @@ ## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html import - os, strutils, rstast, std/enumutils, algorithm, lists, sequtils, + os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, std/private/miscdollars, tables from highlite import SourceLanguage, getSourceLanguage @@ -231,6 +231,7 @@ type meFootnoteMismatch = "mismatch in number of footnotes and their refs: $1", mwRedefinitionOfLabel = "redefinition of label '$1'", mwUnknownSubstitution = "unknown substitution '$1'", + mwAmbiguousLink = "ambiguous doc link $1", mwBrokenLink = "broken link '$1'", mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", @@ -473,12 +474,42 @@ type hasPeers: bool # has headings on the same level of hierarchy? LevelMap = seq[LevelInfo] # Saves for each possible title adornment # style its level in the current document. + SubstitutionKind = enum + rstSubstitution = "substitution", + hyperlinkAlias = "hyperlink alias", + implicitHyperlinkAlias = "implicitly-generated hyperlink alias" Substitution = object + kind*: SubstitutionKind key*: string value*: PRstNode - AnchorSubst = tuple - mainAnchor: string - aliases: seq[string] + info*: TLineInfo # place where the substitution was defined + AnchorRule = enum + arInternalRst, ## For automatically generated RST anchors (from + ## headings, footnotes, inline internal targets): + ## case-insensitive, 1-space-significant (by RST spec) + arNim ## For anchors generated by ``docgen.rst``: Nim-style case + ## sensitivity, etc. (see `proc normalizeNimName`_ for details) + arHyperlink, ## For links with manually set anchors in + ## form `text `_ + RstAnchorKind = enum + manualDirectiveAnchor = "manual directive anchor", + manualInlineAnchor = "manual inline anchor", + footnoteAnchor = "footnote anchor", + headlineAnchor = "implicitly-generated headline anchor" + AnchorSubst = object + mainAnchor: ref string # A reference name that will be inserted directly + # into HTML/Latex. It's declared as `ref` because + # it can be shared between aliases. + info: TLineInfo # where the anchor was defined + priority: int + case kind: range[arInternalRst .. arNim] + of arInternalRst: + anchorType: RstAnchorKind + of arNim: + tooltip: string # displayed tooltip for Nim-generated anchors + langSym: LangSymbol + AnchorSubstTable = Table[string, seq[AnchorSubst]] + # use `seq` to account for duplicate anchors FootnoteType = enum fnManualNumber, # manually numbered footnote like [3] fnAutoNumber, # auto-numbered footnote [#] @@ -505,7 +536,8 @@ type currRoleKind: RstNodeKind # ... and its node kind subs: seq[Substitution] # substitutions refs*: seq[Substitution] # references - anchors*: seq[AnchorSubst] # internal target substitutions + anchors*: AnchorSubstTable + # internal target substitutions lineFootnoteNum: seq[TLineInfo] # footnote line, auto numbers .. [#] lineFootnoteNumRef: seq[TLineInfo] # footnote line, their reference [#]_ currFootnoteNumRef: int # ... their counter for `resolveSubs` @@ -518,7 +550,7 @@ type findFile: FindFileHandler # How to find files. filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) - currFileIdx: FileIndex # current index in `filesnames` + currFileIdx*: FileIndex # current index in `filenames` hasToc*: bool PRstSharedState* = ref RstSharedState @@ -532,6 +564,7 @@ type ## in case of error/warning reporting to ## (relative) line/column of the token. curAnchor*: string # variable to track latest anchor in s.anchors + curAnchorName*: string # corresponding name in human-readable format EParseError* = object of ValueError @@ -590,13 +623,16 @@ proc whichRoleAux(sym: string): RstNodeKind = proc len(filenames: RstFileTable): int = filenames.idxToFilename.len -proc setCurrFilename(s: PRstSharedState, file1: string) = +proc addFilename*(s: PRstSharedState, file1: string): FileIndex = + ## Returns index of filename, adding it if it has not been used before let nextIdx = s.filenames.len.FileIndex - let v = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) - if v == nextIdx: - s.filenames.filenameToIdx[file1] = v + result = getOrDefault(s.filenames.filenameToIdx, file1, default = nextIdx) + if result == nextIdx: + s.filenames.filenameToIdx[file1] = result s.filenames.idxToFilename.add file1 - s.currFileIdx = v + +proc setCurrFilename*(s: PRstSharedState, file1: string) = + s.currFileIdx = addFilename(s, file1) proc getFilename(filenames: RstFileTable, fid: FileIndex): string = doAssert(0 <= fid.int and fid.int < filenames.len, @@ -730,6 +766,8 @@ proc initParser(p: var RstParser, sharedState: PRstSharedState) = p.s = sharedState proc addNodesAux(n: PRstNode, result: var string) = + if n == nil: + return if n.kind == rnLeaf: result.add(n.text) else: @@ -738,6 +776,11 @@ proc addNodesAux(n: PRstNode, result: var string) = proc addNodes(n: PRstNode): string = n.addNodesAux(result) +proc linkName(n: PRstNode): string = + ## Returns a normalized reference name, see: + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#reference-names + n.addNodes.toLowerAscii + proc rstnodeToRefnameAux(n: PRstNode, r: var string, b: var bool) = template special(s) = if b: @@ -804,15 +847,26 @@ proc findSub(s: PRstSharedState, n: PRstNode): int = return i result = -1 +proc lineInfo(p: RstParser, iTok: int): TLineInfo = + result.col = int16(p.col + p.tok[iTok].col) + result.line = uint16(p.line + p.tok[iTok].line) + result.fileIndex = p.s.currFileIdx + +proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) +# TODO: we need this simplification because we don't preserve exact starting +# token of currently parsed element: +proc prevLineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx-1) + proc setSub(p: var RstParser, key: string, value: PRstNode) = var length = p.s.subs.len for i in 0 ..< length: if key == p.s.subs[i].key: p.s.subs[i].value = value return - p.s.subs.add(Substitution(key: key, value: value)) + p.s.subs.add(Substitution(key: key, value: value, info: prevLineInfo(p))) -proc setRef(p: var RstParser, key: string, value: PRstNode) = +proc setRef(p: var RstParser, key: string, value: PRstNode, + refType: SubstitutionKind) = var length = p.s.refs.len for i in 0 ..< length: if key == p.s.refs[i].key: @@ -820,37 +874,111 @@ proc setRef(p: var RstParser, key: string, value: PRstNode) = rstMessage(p, mwRedefinitionOfLabel, key) p.s.refs[i].value = value return - p.s.refs.add(Substitution(key: key, value: value)) + p.s.refs.add(Substitution(kind: refType, key: key, value: value, + info: prevLineInfo(p))) -proc findRef(s: PRstSharedState, key: string): PRstNode = +proc findRef(s: PRstSharedState, key: string): seq[Substitution] = for i in countup(0, high(s.refs)): if key == s.refs[i].key: - return s.refs[i].value + result.add s.refs[i] -proc addAnchor(p: var RstParser, refn: string, reset: bool) = - ## add anchor `refn` to anchor aliases and update last anchor ``curAnchor`` - if p.curAnchor == "": - p.s.anchors.add (refn, @[refn]) +# Ambiguity in links: we don't follow procedure of removing implicit targets +# defined in https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#implicit-hyperlink-targets +# Instead we just give explicit links a higher priority than to implicit ones +# and report ambiguities as warnings. Hopefully it is easy to remove +# ambiguities manually. Nim auto-generated links from ``docgen.nim`` +# have lowest priority: 1 (for procs) and below for other symbol types. + +proc refPriority(k: SubstitutionKind): int = + case k + of rstSubstitution: result = 8 + of hyperlinkAlias: result = 7 + of implicitHyperlinkAlias: result = 2 + +proc internalRefPriority(k: RstAnchorKind): int = + case k + of manualDirectiveAnchor: result = 6 + of manualInlineAnchor: result = 5 + of footnoteAnchor: result = 4 + of headlineAnchor: result = 3 + +proc addAnchorRst(p: var RstParser, name: string, refn: string, reset: bool, + anchorType: RstAnchorKind) = + ## Adds anchor `refn` with an alias `name` and + ## updates the corresponding `curAnchor` / `curAnchorName`. + let prio = internalRefPriority(anchorType) + if p.curAnchorName == "": + var anchRef = new string + anchRef[] = refn + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) else: - p.s.anchors[^1].mainAnchor = refn - p.s.anchors[^1].aliases.add refn + # override previous mainAnchor by `ref` in all aliases + var anchRef = p.s.anchors[p.curAnchorName][0].mainAnchor + anchRef[] = refn + p.s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arInternalRst, mainAnchor: anchRef, priority: prio, + info: prevLineInfo(p), anchorType: anchorType)) if reset: p.curAnchor = "" + p.curAnchorName = "" else: p.curAnchor = refn + p.curAnchorName = name -proc findMainAnchor(s: PRstSharedState, refn: string): string = - for subst in s.anchors: - if subst.mainAnchor == refn: # no need to rename - result = subst.mainAnchor - break - var toLeave = false - for anchor in subst.aliases: - if anchor == refn: # this anchor will be named as mainAnchor - result = subst.mainAnchor - toLeave = true - if toLeave: - break +proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string, + langSym: LangSymbol, priority: int, + info: TLineInfo) = + ## Adds an anchor `refn` (`mainAnchor`), which follows + ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) + var anchRef = new string + anchRef[] = refn + s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arNim, mainAnchor: anchRef, langSym: langSym, + tooltip: tooltip, priority: priority, + info: info)) + +proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, + info: TLineInfo): + seq[AnchorSubst] = + let langSym = toLangSymbol(signature) + let substitutions = s.anchors.getOrDefault(langSym.name, + newSeq[AnchorSubst]()) + if substitutions.len == 0: + return + # map symKind (like "proc") -> found symbols/groups: + var found: Table[string, seq[AnchorSubst]] + for s in substitutions: + if s.kind == arNim: + if match(s.langSym, langSym): + found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s + for symKind, sList in found: + if sList.len == 1: + result.add sList[0] + else: # > 1, there are overloads, potential ambiguity in this `symKind` + if langSym.parametersProvided: + # there are non-group signatures, select only them + for s in sList: + if not s.langSym.isGroup: + result.add s + else: # when there are many overloads a link like foo_ points to all + # of them, so selecting the group + var foundGroup = true + for s in sList: + if s.langSym.isGroup: + result.add s + foundGroup = true + break + doAssert foundGroup, "docgen has not generated the group" + +proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): + seq[AnchorSubst] = + let name = linkText.toLowerAscii + let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) + for s in substitutions: + if s.kind == arInternalRst: + result.add s proc addFootnoteNumManual(p: var RstParser, num: int) = ## add manually-numbered footnote @@ -860,13 +988,6 @@ proc addFootnoteNumManual(p: var RstParser, num: int) = return p.s.footnotes.add((fnManualNumber, num, -1, -1, $num)) -proc lineInfo(p: RstParser, iTok: int): TLineInfo = - result.col = int16(p.col + p.tok[iTok].col) - result.line = uint16(p.line + p.tok[iTok].line) - result.fileIndex = p.s.currFileIdx - -proc lineInfo(p: RstParser): TLineInfo = lineInfo(p, p.idx) - proc addFootnoteNumAuto(p: var RstParser, label: string) = ## add auto-numbered footnote. ## Empty label [#] means it'll be resolved by the occurrence. @@ -989,6 +1110,7 @@ proc newRstNodeA(p: var RstParser, kind: RstNodeKind): PRstNode = if p.curAnchor != "": result.anchor = p.curAnchor p.curAnchor = "" + p.curAnchorName = "" template newLeaf(s: string): PRstNode = newRstLeaf(s) @@ -1255,7 +1377,7 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = else: newKind = rnHyperlink newSons = @[a, b] - setRef(p, rstnodeToRefname(a), b) + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef @@ -1562,7 +1684,8 @@ proc parseInline(p: var RstParser, father: PRstNode) = inc p.idx parseUntil(p, n, "`", false) let refn = rstnodeToRefname(n) - p.s.anchors.add (refn, @[refn]) + addAnchorRst(p, name = linkName(n), refn = refn, reset = true, + anchorType=manualInlineAnchor) father.add(n) elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": inc p.idx @@ -2084,7 +2207,8 @@ proc parseHeadline(p: var RstParser): PRstNode = result.level = getLevel(p, c, hasOverline=false) checkHeadingHierarchy(p, result.level) p.s.hCurLevel = result.level - addAnchor(p, rstnodeToRefname(result), reset=true) + addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true, + anchorType=headlineAnchor) proc parseOverline(p: var RstParser): PRstNode = var c = currentTok(p).symbol[0] @@ -2106,7 +2230,8 @@ proc parseOverline(p: var RstParser): PRstNode = if currentTok(p).kind == tkAdornment: inc p.idx if currentTok(p).kind == tkIndent: inc p.idx - addAnchor(p, rstnodeToRefname(result), reset=true) + addAnchorRst(p, linkName(result), rstnodeToRefname(result), reset=true, + anchorType=headlineAnchor) type IntSeq = seq[int] @@ -2837,7 +2962,7 @@ proc parseFootnote(p: var RstParser): PRstNode = anchor.add $p.s.lineFootnoteSym.len of fnCitation: anchor.add rstnodeToRefname(label) - addAnchor(p, anchor, reset=true) + addAnchorRst(p, anchor, anchor, reset=true, anchorType=footnoteAnchor) result.anchor = anchor if currentTok(p).kind == tkWhite: inc p.idx discard parseBlockContent(p, result, parseSectionWrapper) @@ -2858,13 +2983,23 @@ proc parseDotDot(p: var RstParser): PRstNode = elif match(p, p.idx, " _"): # hyperlink target: inc p.idx, 2 - var a = getReferenceName(p, ":") + var ending = ":" + if currentTok(p).symbol == "`": + inc p.idx + ending = "`" + var a = getReferenceName(p, ending) + if ending == "`": + if currentTok(p).symbol == ":": + inc p.idx + else: + rstMessage(p, meExpected, ":") if currentTok(p).kind == tkWhite: inc p.idx var b = untilEol(p) if len(b) == 0: # set internal anchor - addAnchor(p, rstnodeToRefname(a), reset=false) + addAnchorRst(p, linkName(a), rstnodeToRefname(a), reset=false, + anchorType=manualDirectiveAnchor) else: # external hyperlink - setRef(p, rstnodeToRefname(a), b) + setRef(p, rstnodeToRefname(a), b, refType=hyperlinkAlias) elif match(p, p.idx, " |"): # substitution definitions: inc p.idx, 2 @@ -2892,7 +3027,7 @@ proc rstParsePass1*(fragment: string, sharedState: PRstSharedState): PRstNode = ## Parses an RST `fragment`. ## The result should be further processed by - ## `preparePass2` and `resolveSubs` (which is pass 2). + ## preparePass2_ and resolveSubs_ (which is pass 2). var p: RstParser initParser(p, sharedState) p.line = line @@ -2905,6 +3040,65 @@ proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = countTitles(s, mainNode) orderFootnotes(s) +proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = + # Associate this link alias with its target and change node kind to + # rnHyperlink or rnInternalRef appropriately. + type LinkDef = object + ar: AnchorRule + priority: int + tooltip: string + target: PRstNode + info: TLineInfo + proc cmp(x, y: LinkDef): int = + result = cmp(x.priority, y.priority) + if result == 0: + result = cmp(x.target, y.target) + var foundLinks: seq[LinkDef] + let text = newRstNode(rnInner, n.sons) + let refn = rstnodeToRefname(n) + var hyperlinks = findRef(s, refn) + for y in hyperlinks: + foundLinks.add LinkDef(ar: arHyperlink, priority: refPriority(y.kind), + target: y.value, info: y.info, + tooltip: "(" & $y.kind & ")") + let substRst = findMainAnchorRst(s, text.addNodes, n.info) + for subst in substRst: + foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority, + target: newLeaf(subst.mainAnchor[]), + info: subst.info, + tooltip: "(" & $subst.anchorType & ")") + if roNimFile in s.options: + let substNim = findMainAnchorNim(s, signature=text, n.info) + for subst in substNim: + foundLinks.add LinkDef(ar: arNim, priority: subst.priority, + target: newLeaf(subst.mainAnchor[]), + info: subst.info, tooltip: subst.tooltip) + foundLinks.sort(cmp = cmp, order = Descending) + let linkText = addNodes(n) + if foundLinks.len >= 1: + let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink + elif foundLinks[0].ar == arNim: rnNimdocRef + else: rnInternalRef + result = newRstNode(kind) + result.sons = @[text, foundLinks[0].target] + if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip + if foundLinks.len > 1: # report ambiguous link + var targets = newSeq[string]() + for l in foundLinks: + var t = " " + if s.filenames.len > 1: + t.add getFilename(s.filenames, l.info.fileIndex) + let n = l.info.line + let c = l.info.col + ColRstOffset + t.add "($1, $2): $3" % [$n, $c, l.tooltip] + targets.add t + rstMessage(s.filenames, s.msgHandler, n.info, mwAmbiguousLink, + "`$1`\n clash:\n$2" % [ + linkText, targets.join("\n")]) + else: # nothing found + result = n + rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, linkText) + proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = ## Makes pass 2 of RST parsing. ## Resolves substitutions and anchor aliases, groups footnotes. @@ -2933,21 +3127,7 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = elif s.hTitleCnt == 0: n.level += 1 of rnRef: - let refn = rstnodeToRefname(n) - var y = findRef(s, refn) - if y != nil: - result = newRstNode(rnHyperlink) - let text = newRstNode(rnInner, n.sons) - result.sons = @[text, y] - else: - let anchor = findMainAnchor(s, refn) - if anchor != "": - result = newRstNode(rnInternalRef) - let text = newRstNode(rnInner, n.sons) - result.sons = @[text, # visible text of reference - newLeaf(anchor)] # link itself - else: - rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) + result = resolveLink(s, n) of rnFootnote: var (fnType, num) = getFootnoteType(n.sons[0]) case fnType @@ -2993,9 +3173,10 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = of fnCitation: result.add n.sons[0] refn.add rstnodeToRefname(n) - let anch = findMainAnchor(s, refn) - if anch != "": - result.add newLeaf(anch) # add link + # TODO: correctly report ambiguities + let anchorInfo = findMainAnchorRst(s, refn, n.info) + if anchorInfo.len != 0: + result.add newLeaf(anchorInfo[0].mainAnchor[]) # add link else: rstMessage(s.filenames, s.msgHandler, n.info, mwBrokenLink, refn) result.add newLeaf(refn) # add link diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index e1ed7c099..bc7b9a650 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -43,6 +43,7 @@ type rnFootnoteGroup, # footnote group - exists for a purely stylistic # reason: to display a few footnotes as 1 block rnStandaloneHyperlink, rnHyperlink, rnRef, rnInternalRef, rnFootnoteRef, + rnNimdocRef, # reference to automatically generated Nim symbol rnDirective, # a general directive rnDirArg, # a directive argument (for some directives). # here are directives that are not rnDirective: @@ -104,6 +105,8 @@ type rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: info*: TLineInfo ## To have line/column info for warnings at ## nodes that are post-processed after parsing + of rnNimdocRef: + tooltip*: string else: discard anchor*: string ## anchor, internal link target diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 8eac37307..a556aa2e3 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -60,8 +60,8 @@ type MetaEnum* = enum metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion - EscapeMode = enum # in Latex text inside options [] and URLs is - # escaped slightly differently than in normal text + EscapeMode* = enum # in Latex text inside options [] and URLs is + # escaped slightly differently than in normal text emText, emOption, emUrl # emText is currently used for code also RstGenerator* = object of RootObj @@ -201,7 +201,9 @@ proc addTexChar(dest: var string, c: char, escMode: EscapeMode) = ## All escapes that need to work in text and code blocks (`emText` mode) ## should start from \ (to be compatible with fancyvrb/fvextra). case c - of '_', '$', '&', '#', '%': add(dest, "\\" & c) + of '_', '&', '#', '%': add(dest, "\\" & c) + # commands \label and \pageref don't accept \$ by some reason but OK with $: + of '$': (if escMode == emUrl: add(dest, c) else: add(dest, "\\" & c)) # \~ and \^ have a special meaning unless they are followed by {} of '~', '^': add(dest, "\\" & c & "{}") # Latex loves to substitute ` to opening quote, even in texttt mode! @@ -1180,7 +1182,8 @@ proc renderAdmonition(d: PDoc, n: PRstNode, result: var string) = "$1\n\\end{rstadmonition}\n", result) -proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external: bool) = +proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, + external: bool, nimdoc = false, tooltip="") = var linkStr = "" block: let mode = d.escMode @@ -1189,14 +1192,19 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, external d.escMode = mode var textStr = "" renderRstToOut(d, text, textStr) + let nimDocStr = if nimdoc: " nimdoc" else: "" + var tooltipStr = "" + if tooltip != "": + tooltipStr = """ title="$1"""" % [ esc(d.target, tooltip) ] if external: dispA(d.target, result, - "$1", - "\\href{$2}{$1}", [textStr, linkStr]) + "$1", + "\\href{$2}{$1}", [textStr, linkStr, nimDocStr, tooltipStr]) else: dispA(d.target, result, - "$1", - "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr]) + "$1", + "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", + [textStr, linkStr, nimDocStr, tooltipStr]) proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return @@ -1329,6 +1337,9 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderHyperlink(d, text=n.sons[0], link=n.sons[0], result, external=true) of rnInternalRef: renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false) + of rnNimdocRef: + renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=false, + nimdoc=true, tooltip=n.tooltip) of rnHyperlink: renderHyperlink(d, text=n.sons[0], link=n.sons[1], result, external=true) of rnFootnoteRef: diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 873949ca6..2234a8625 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -273,6 +273,11 @@ func nimIdentNormalize*(s: string): string = ## ## That means to convert to lower case and remove any '_' on all characters ## except first one. + ## + ## .. Warning:: Backticks (`) are not handled: they remain *as is* and + ## spaces are preserved. See `nimIdentBackticksNormalize + ## `_ for + ## an alternative approach. runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len) diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 3499b5326..69dd2eb2e 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -121,6 +121,8 @@ window.addEventListener('DOMContentLoaded', main);

    Procs

    + +
    proc foo() {....raises: [], tags: [].}
    @@ -130,6 +132,8 @@ I do foo
    +
    +
    diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 3353f1cda..7c819ea30 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -263,6 +263,10 @@ a.reference-toplevel { font-weight: bold; } +a.nimdoc { + word-spacing: 0.3em; +} + a.toc-backref { text-decoration: none; color: var(--text); } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 47901ce0d..c64039941 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -106,7 +106,10 @@ window.addEventListener('DOMContentLoaded', main);
  • Types
      -
    • G
      • +
    • SomeType
      • @@ -115,9 +118,54 @@ window.addEventListener('DOMContentLoaded', main);
    • Procs
        -
          fn2 + + + + + + + +
            fn3 @@ -159,6 +207,11 @@ window.addEventListener('DOMContentLoaded', main);
          • fn10(a: int): int
            • +
          +
            someType
            • +
          • + Iterators + +
          • Templates
              @@ -197,7 +261,13 @@ window.addEventListener('DOMContentLoaded', main);
              -

              +

              This is a description of the utils module.

              +

              Links work:

              + +

              This is now a header

              Next header

              And so on

              @@ -209,10 +279,31 @@ window.addEventListener('DOMContentLoaded', main);
              1. Other case value
              2. Second case.
              -

              +

              Ref group fn2 or specific function like fn2() or fn2( int ) or fn2(int, float).

              +

              Ref generics like this: binarySearch or binarySearch(openArray[T], K, proc (T, K)) or proc binarySearch(openArray[T], K, proc (T, K)) or in different style: proc binarysearch(openarray[T], K, proc(T, K)). Can be combined with export symbols and type parameters: binarysearch*[T, K](openArray[T], K, proc (T, K)). With spaces binary search.

              +

              Note that proc can be used in postfix form: binarySearch proc.

              +

              Ref. type like G and type G and G[T] and type G*[T].

              +Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

              Types

              +
              +
              G[T] = object
+  val: T
+
              +
              + + + +
              +
              SomeType = enum
   enumValueA, enumValueB, enumValueC
              @@ -227,6 +318,109 @@ window.addEventListener('DOMContentLoaded', main);

              Procs

              + +
              +
              +
              proc `$`[T](a: G[T]): string
              +
              + + + +
              +
              +
              +
              proc `$`[T](a: ref SomeType): string
              +
              + + + +
              +
              + +
              + +
              +
              +
              func `'big`(a: string): SomeType {....raises: [], tags: [].}
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc `[]`[T](x: G[T]): T
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc `[]=`[T](a: var G[T]; index: int; value: T)
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc binarySearch[T, K](a: openArray[T]; key: K;
+                        cmp: proc (x: T; y: K): int {.closure.}): int
              +
              + + + +
              +
              + +
              + +
              +
              +
              proc f(x: G[int]) {....raises: [], tags: [].}
              +
              + +There is also variant f(G[string]) + +
              +
              +
              +
              proc f(x: G[string]) {....raises: [], tags: [].}
              +
              + +See also f(G[int]). + +
              +
              + +
              + +
              +
              +
              proc fn[T; U, V: SomeFloat]()
              +
              + + + +
              +
              + +
              + +
              proc fn2() {....raises: [], tags: [].}
              @@ -235,6 +429,26 @@ comment
              +
              +
              proc fn2(x: int) {....raises: [], tags: [].}
              +
              + +fn2 comment + +
              +
              +
              +
              proc fn2(x: int; y: float) {....raises: [], tags: [].}
              +
              + + + +
              +
              + +
              + +
              proc fn3(): auto {....raises: [], tags: [].}
              @@ -243,6 +457,10 @@ comment
              + +
              + +
              proc fn4(): auto {....raises: [], tags: [].}
              @@ -251,6 +469,10 @@ comment
              + +
              + +
              proc fn5() {....raises: [], tags: [].}
              @@ -259,6 +481,10 @@ comment
              + +
              + +
              proc fn6() {....raises: [], tags: [].}
              @@ -267,6 +493,10 @@ comment
              + +
              + +
              proc fn7() {....raises: [], tags: [].}
              @@ -275,6 +505,10 @@ comment
              + +
              + +
              proc fn8(): auto {....raises: [], tags: [].}
              @@ -283,6 +517,10 @@ comment
              + +
              + +
              func fn9(a: int): int {....raises: [], tags: [].}
              @@ -291,6 +529,10 @@ comment
              + +
              + +
              func fn10(a: int): int {....raises: [], tags: [].}
              @@ -299,6 +541,22 @@ comment
              + +
              + +
              +
              +
              proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
              +
              + + + +
              +
              + +
              + +
              proc someType(): SomeType {....raises: [], tags: [].}
              @@ -308,10 +566,31 @@ constructor.
              +
              + +
              +
              +

              Iterators

              +
              + +
              +
              +
              iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
              +
              + + + +
              +
              + +
              +

              Templates

              + +
              template aEnum(): untyped
              @@ -320,6 +599,10 @@ constructor.
              + +
              + +
              template bEnum(): untyped
              @@ -328,6 +611,10 @@ constructor.
              + +
              + +
              template fromUtilsGen(): untyped
              @@ -339,6 +626,8 @@ should be shown in utils.html only
              +
              +
              diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index a87393f09..441bd8675 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,9 +1,14 @@ +funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U utils: funWithGenerics[T, U: SomeFloat](a: T; b: U) enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType +G subdir/subdir_b/utils.html#G utils: G someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType fn2 subdir/subdir_b/utils.html#fn2 utils: fn2() +fn2 subdir/subdir_b/utils.html#fn2,int utils: fn2(x: int) +fn2 subdir/subdir_b/utils.html#fn2,int,float utils: fn2(x: int; y: float) +binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) utils: binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int fn3 subdir/subdir_b/utils.html#fn3 utils: fn3(): auto fn4 subdir/subdir_b/utils.html#fn4 utils: fn4(): auto fn5 subdir/subdir_b/utils.html#fn5 utils: fn5() @@ -15,6 +20,15 @@ fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped +f subdir/subdir_b/utils.html#f,G[int] utils: f(x: G[int]) +f subdir/subdir_b/utils.html#f,G[string] utils: f(x: G[string]) +`[]` subdir/subdir_b/utils.html#[],G[T] utils: `[]`[T](x: G[T]): T +`[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T utils: `[]=`[T](a: var G[T]; index: int; value: T) +`$` subdir/subdir_b/utils.html#$,G[T] utils: `$`[T](a: G[T]): string +`$` subdir/subdir_b/utils.html#$,ref.SomeType utils: `$`[T](a: ref SomeType): string +fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] utils: fooBar(a: seq[SomeType]): int +fn subdir/subdir_b/utils.html#fn utils: fn[T; U, V: SomeFloat]() +`'big` subdir/subdir_b/utils.html#'big,string utils: `'big`(a: string): SomeType This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index dbacb57a7..e45492dac 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -559,6 +559,8 @@ This should be visible.

              Procs

              + +
              proc addfBug14485() {....raises: [], tags: [].}
              @@ -579,6 +581,10 @@ Some proc
              + +
              + +
              proc anything() {....raises: [], tags: [].}
              @@ -587,6 +593,10 @@ There is no block quote after blank lines at the beginning.
              + +
              + +
              proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
              @@ -596,6 +606,10 @@ ok1
              + +
              + +
              proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
              @@ -604,6 +618,10 @@ ok1
              + +
              + +
              proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
              @@ -614,6 +632,10 @@ ok1
              + +
              + +
              proc bar[T](a, b: T): T
              @@ -622,6 +644,10 @@ ok1
              + +
              + +
              proc baz() {....raises: [], tags: [].}
              @@ -641,6 +667,10 @@ This is deprecated without message.
              + +
              + +
              proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
              @@ -652,6 +682,10 @@ This is deprecated with a message.
              + +
              + +
              proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
              @@ -661,6 +695,10 @@ This is deprecated with a message.
              + +
              + +
              proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable, ...raises: [], tags: [].}
              @@ -670,6 +708,10 @@ the c printf. etc.
              + +
              + +
              proc fromUtils3() {....raises: [], tags: [].}
              @@ -681,6 +723,10 @@ came form utils but should be shown where
              + +
              + +
              proc isValid[T](x: T): bool
              @@ -689,6 +735,27 @@ came form utils but should be shown where
              + +
              + +
              +
              +
              proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+    ...raises: [], tags: [].}
              +
              + +

              Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

              +

              See also:

              + +
              low(2) # => -9223372036854775808
              + +
              +
              + +
              + +
              proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}
              @@ -704,19 +771,10 @@ came form utils but should be shown where
              -
              -
              proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
-    ...raises: [], tags: [].}
              -
              -

              Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.

              -

              See also:

              - -
              low(2) # => -9223372036854775808
              - -
              + +
              proc p1() {....raises: [], tags: [].}
              @@ -741,6 +799,10 @@ this is a nested doc comment
              + +
              + +
              func someFunc() {....raises: [], tags: [].}
              @@ -749,6 +811,10 @@ My someFunc. Stuff in
              + +
              + +
              proc tripleStrLitTest() {....raises: [], tags: [].}
              @@ -793,6 +859,10 @@ at indent 0
              + +
              + +
              proc z1(): Foo {....raises: [], tags: [].}
              @@ -801,6 +871,10 @@ cz1
              + +
              + +
              proc z2() {....raises: [], tags: [].}
              @@ -811,6 +885,10 @@ cz2
              + +
              + +
              proc z3() {....raises: [], tags: [].}
              @@ -819,6 +897,10 @@ cz3
              + +
              + +
              proc z4() {....raises: [], tags: [].}
              @@ -827,6 +909,10 @@ cz4
              + +
              + +
              proc z5(): int {....raises: [], tags: [].}
              @@ -835,6 +921,10 @@ cz5
              + +
              + +
              proc z6(): int {....raises: [], tags: [].}
              @@ -843,6 +933,10 @@ cz6
              + +
              + +
              proc z7(): int {....raises: [], tags: [].}
              @@ -851,6 +945,10 @@ cz7
              + +
              + +
              proc z8(): int {....raises: [], tags: [].}
              @@ -859,6 +957,10 @@ cz8
              + +
              + +
              proc z9() {....raises: [], tags: [].}
              @@ -869,6 +971,10 @@ cz8
              + +
              + +
              proc z10() {....raises: [], tags: [].}
              @@ -879,6 +985,10 @@ cz8
              + +
              + +
              proc z11() {....raises: [], tags: [].}
              @@ -889,6 +999,10 @@ cz8
              + +
              + +
              proc z12(): int {....raises: [], tags: [].}
              @@ -899,6 +1013,10 @@ cz8
              + +
              + +
              proc z13() {....raises: [], tags: [].}
              @@ -909,6 +1027,10 @@ cz13
              + +
              + +
              proc z17() {....raises: [], tags: [].}
              @@ -920,10 +1042,14 @@ cz17 rest
              +
              +

              Methods

              + +
              method method1(self: Moo) {.base, ...raises: [], tags: [].}
              @@ -932,6 +1058,10 @@ foo1
              + +
              + +
              method method2(self: Moo): int {.base, ...raises: [], tags: [].}
              @@ -940,6 +1070,10 @@ foo2
              + +
              + +
              method method3(self: Moo): int {.base, ...raises: [], tags: [].}
              @@ -949,10 +1083,14 @@ foo3
              +
              +

              Iterators

              + +
              iterator fromUtils1(): int {....raises: [], tags: [].}
              @@ -965,6 +1103,10 @@ foo3
              + +
              + +
              iterator iter1(n: int): int {....raises: [], tags: [].}
              @@ -973,6 +1115,10 @@ foo1
              + +
              + +
              iterator iter2(n: int): int {....raises: [], tags: [].}
              @@ -984,10 +1130,14 @@ foo2
              +
              +

              Macros

              + +
              macro bar(): untyped
              @@ -996,6 +1146,10 @@ foo2
              + +
              + +
              macro z16()
              @@ -1008,6 +1162,10 @@ foo2
              + +
              + +
              macro z18(): int
              @@ -1017,10 +1175,14 @@ cz18
              +
              +

              Templates

              + +
              template foo(a, b: SomeType)
              @@ -1029,6 +1191,10 @@ This does nothing
              + +
              + +
              template fromUtils2()
              @@ -1040,6 +1206,10 @@ ok3
              + +
              + +
              template myfn()
              @@ -1063,6 +1233,10 @@ bar
              + +
              + +
              template testNimDocTrailingExample()
              @@ -1073,6 +1247,10 @@ bar
              + +
              + +
              template z6t(): int
              @@ -1081,6 +1259,10 @@ cz6t
              + +
              + +
              template z14()
              @@ -1091,6 +1273,10 @@ cz14
              + +
              + +
              template z15()
              @@ -1110,6 +1296,8 @@ cz15
              +
              +
              diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index f72f6c37e..da76334d4 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -71,7 +71,25 @@ window.addEventListener('DOMContentLoaded', main);

              Index

              Modules: subdir/subdir_b/utils, testproject.

              API symbols

              -
              A:
              +
              binarySearch:
              buzz:
              +
              f:
              +
              fn:
              fn10:
              • utils: fn10(a: int): int
                • @@ -178,6 +212,10 @@ window.addEventListener('DOMContentLoaded', main);
                fn2:
                fn3:
                +
                fooBar:
                FooBuzz:
                +
                funWithGenerics:
                +
                G:
                isValid:
                • testproject: isValid[T](x: T): bool
                  • diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 37c91dd3c..5d881e620 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -1,5 +1,7 @@ ##[ +.. include:: ./utils_overview.rst + # This is now a header ## Next header @@ -19,13 +21,32 @@ More text. 1. Other case value 2. Second case. +Ref group fn2_ or specific function like `fn2()`_ +or `fn2( int )`_ or `fn2(int, +float)`_. + +Ref generics like this: binarySearch_ or `binarySearch(openArray[T], K, +proc (T, K))`_ or `proc binarySearch(openArray[T], K, proc (T, K))`_ or +in different style: `proc binarysearch(openarray[T], K, proc(T, K))`_. +Can be combined with export symbols and type parameters: +`binarysearch*[T, K](openArray[T], K, proc (T, K))`_. +With spaces `binary search`_. + +Note that `proc` can be used in postfix form: `binarySearch proc`_. + +Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. + ]## +include ./utils_helpers + type SomeType* = enum enumValueA, enumValueB, enumValueC + G*[T] = object + val: T proc someType*(): SomeType = ## constructor. @@ -33,6 +54,14 @@ proc someType*(): SomeType = proc fn2*() = discard ## comment +proc fn2*(x: int) = + ## fn2 comment + discard +proc fn2*(x: int, y: float) = + discard +proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.}): int = + discard proc fn3*(): auto = 1 ## comment proc fn4*(): auto = 2 * 3 + 4 ## comment proc fn5*() ## comment @@ -89,3 +118,39 @@ template fromUtilsGen*(): untyped = ## came form utils but should be shown where `fromUtilsGen` is called runnableExamples: discard """should be shown as examples for fromUtils3 in module calling fromUtilsGen""" + +proc f*(x: G[int]) = + ## There is also variant `f(G[string])`_ + discard +proc f*(x: G[string]) = + ## See also `f(G[int])`_. + discard + +## Ref. `[]`_ is the same as `proc \`[]\`(G[T])`_ because there are no +## overloads. The full form: `proc \`[]\`*[T](x: G[T]): T`_ + +proc `[]`*[T](x: G[T]): T = x.val + +## Ref. `[]=`_ aka `\`[]=\`(G[T], int, T)`_. + +proc `[]=`*[T](a: var G[T], index: int, value: T) = discard + +## Ref. `$`_ aka `proc $`_ or `proc \`$\``_. + +proc `$`*[T](a: G[T]): string = "" + +## Ref. `$(a: ref SomeType)`_. + +proc `$`*[T](a: ref SomeType): string = "" + +## Ref. foo_bar_ aka `iterator foo_bar_`_. + +iterator fooBar*(a: seq[SomeType]): int = discard + +## Ref. `fn[T; U,V: SomeFloat]()`_. + +proc fn*[T; U, V: SomeFloat]() = discard + +## Ref. `'big`_ or `func \`'big\``_ or `\`'big\`(string)`_. + +func `'big`*(a: string): SomeType = discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim new file mode 100644 index 000000000..2c45ffb83 --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_helpers.nim @@ -0,0 +1 @@ +proc funWithGenerics*[T, U: SomeFloat](a: T, b: U) = discard diff --git a/nimdoc/testproject/subdir/subdir_b/utils_overview.rst b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst new file mode 100644 index 000000000..58ce930bf --- /dev/null +++ b/nimdoc/testproject/subdir/subdir_b/utils_overview.rst @@ -0,0 +1,8 @@ +This is a description of the utils module. + +Links work: + +* other module: `iterators `_ (not in this dir, just an example) +* internal: `fn2(x)`_ +* internal included from another module: `funWithGenerics*[T, U: + SomeFloat](a: T, b: U)`_. diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim new file mode 100644 index 000000000..5bcdf32d5 --- /dev/null +++ b/tests/stdlib/tdochelpers.nim @@ -0,0 +1,155 @@ +discard """ + output: ''' + +[Suite] Integration with Nim +''' +""" + +# tests for dochelpers.nim module + +import ../../lib/packages/docutils/[rstast, rst, dochelpers] +import unittest + +proc rstParseTest(text: string): PRstNode = + proc testMsgHandler(filename: string, line, col: int, msgkind: MsgKind, + arg: string) = + doAssert msgkind == mwBrokenLink + let r = rstParse(text, "-input-", LineRstInit, ColRstInit, + {roPreferMarkdown, roSupportMarkdown, roNimFile}, + msgHandler=testMsgHandler) + result = r.node + +suite "Integration with Nim": + test "simple symbol parsing (shortest form)": + let input1 = "g_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "g") + + test "simple symbol parsing (group of words)": + let input1 = "`Y`_".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "", name: "Y") + + # this means not a statement 'type', it's a backticked identifier `type`: + let input2 = "`type`_".rstParseTest + check input2.toLangSymbol == LangSymbol(symKind: "", name: "type") + + let input3 = "`[]`_".rstParseTest + check input3.toLangSymbol == LangSymbol(symKind: "", name: "[]") + + let input4 = "`X Y Z`_".rstParseTest + check input4.toLangSymbol == LangSymbol(symKind: "", name: "Xyz") + + test "simple proc parsing": + let input1 = "proc f".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "proc", name: "f") + + test "another backticked name": + let input1 = """`template \`type\``_""".rstParseTest + check input1.toLangSymbol == LangSymbol(symKind: "template", name: "type") + + test "simple proc parsing with parameters": + let input1 = "`proc f*()`_".rstParseTest + let input2 = "`proc f()`_".rstParseTest + let expected = LangSymbol(symKind: "proc", name: "f", + parametersProvided: true) + check input1.toLangSymbol == expected + check input2.toLangSymbol == expected + + test "symbol parsing with 1 parameter": + let input = "`f(G[int])`_".rstParseTest + let expected = LangSymbol(symKind: "", name: "f", + parameters: @[("G[int]", "")], + parametersProvided: true) + check input.toLangSymbol == expected + + test "more proc parsing": + let input1 = "`proc f[T](x:G[T]):M[T]`_".rstParseTest + let input2 = "`proc f[ T ] ( x: G [T] ): M[T]`_".rstParseTest + let input3 = "`proc f*[T](x: G[T]): M[T]`_".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "f", + generics: "[T]", + parameters: @[("x", "G[T]")], + parametersProvided: true, + outType: "M[T]") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + check(input3.toLangSymbol == expected) + + test "advanced proc parsing with Nim identifier normalization": + let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int)`_""".rstParseTest + let expected = LangSymbol(symKind: "proc", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "the same without proc": + let input = """`binarySearch*[T, K](a: openArray[T]; key: K; + cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest + let expected = LangSymbol(symKind: "", + name: "binarysearch", + generics: "[T,K]", + parameters: @[ + ("a", "openarray[T]"), + ("key", "K"), + ("cmp", "proc(x:T;y:K):int")], + parametersProvided: true, + outType: "") + check(input.toLangSymbol == expected) + + test "operator $ with and without backticks": + let input1 = """`func \`$\`*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let input2 = """`func $*[T](a: \`open Array\`[T]): string`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "$", + generics: "[T]", + parameters: @[("a", "openarray[T]")], + parametersProvided: true, + outType: "string") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "operator [] with and without backticks": + let input1 = """`func \`[]\`[T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let input2 = """`func [][T](a: \`open Array\`[T], idx: int): T`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) + + test "postfix symbol specifier #1": + let input = """`walkDir iterator`_""". + rstParseTest + let expected = LangSymbol(symKind: "iterator", + name: "walkdir") + check(input.toLangSymbol == expected) + + test "postfix symbol specifier #2": + let input1 = """`\`[]\`[T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let input2 = """`[][T](a: \`open Array\`[T], idx: int): T func`_""". + rstParseTest + let expected = LangSymbol(symKind: "func", + name: "[]", + generics: "[T]", + parameters: @[("a", "openarray[T]"), + ("idx", "int")], + parametersProvided: true, + outType: "T") + check(input1.toLangSymbol == expected) + check(input2.toLangSymbol == expected) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index e9f67324c..e4609a671 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -17,9 +17,7 @@ discard """ # tests for rst module -import ../../lib/packages/docutils/rstgen -import ../../lib/packages/docutils/rst -import ../../lib/packages/docutils/rstast +import ../../lib/packages/docutils/[rstgen, rst, rstast] import unittest, strutils import std/private/miscdollars import os @@ -61,6 +59,52 @@ proc toAst(input: string, result = e.msg suite "RST parsing": + test "References are whitespace-neutral and case-insensitive": + # refname is 'lexical-analysis', the same for all the 3 variants: + check(dedent""" + Lexical Analysis + ================ + + Ref. `Lexical Analysis`_ or `Lexical analysis`_ or `lexical analysis`_. + """.toAst == + dedent""" + rnInner + rnHeadline level=1 + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'Analysis' + rnParagraph + rnLeaf 'Ref' + rnLeaf '.' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'Analysis' + rnLeaf 'lexical-analysis' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'Lexical' + rnLeaf ' ' + rnLeaf 'analysis' + rnLeaf 'lexical-analysis' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'lexical' + rnLeaf ' ' + rnLeaf 'analysis' + rnLeaf 'lexical-analysis' + rnLeaf '.' + rnLeaf ' ' + """) + test "option list has priority over definition list": check(dedent""" --defusages @@ -376,9 +420,9 @@ suite "Warnings": let output = input.toAst(warnings=warnings) check(warnings[] == @[ "input(3, 14) Warning: broken link 'citation-som'", - "input(5, 7) Warning: broken link 'a-broken-link'", + "input(5, 7) Warning: broken link 'a broken Link'", "input(7, 15) Warning: unknown substitution 'undefined subst'", - "input(9, 6) Warning: broken link 'shortdotlink'" + "input(9, 6) Warning: broken link 'short.link'" ]) test "With include directive and blank lines at the beginning": @@ -391,7 +435,7 @@ suite "Warnings": let input = ".. include:: other.rst" var warnings = new seq[string] let output = input.toAst(warnings=warnings) - check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenlink'"] + check warnings[] == @["other.rst(5, 6) Warning: broken link 'brokenLink'"] check(output == dedent""" rnInner rnParagraph @@ -404,6 +448,59 @@ suite "Warnings": """) removeFile("other.rst") + test "warnings for ambiguous links (references + anchors)": + # Reference like `x`_ generates a link alias x that may clash with others + let input = dedent""" + Manual reference: `foo <#foo,string,string>`_ + + .. _foo: + + Paragraph. + + Ref foo_ + """ + var warnings = new seq[string] + let output = input.toAst(warnings=warnings) + check(warnings[] == @[ + dedent """ + input(7, 5) Warning: ambiguous doc link `foo` + clash: + (3, 8): (manual directive anchor) + (1, 45): (implicitly-generated hyperlink alias)""" + ]) + # reference should be resolved to the manually set anchor: + check(output == + dedent""" + rnInner + rnParagraph + rnLeaf 'Manual' + rnLeaf ' ' + rnLeaf 'reference' + rnLeaf ':' + rnLeaf ' ' + rnHyperlink + rnInner + rnLeaf 'foo' + rnInner + rnLeaf '#' + rnLeaf 'foo' + rnLeaf ',' + rnLeaf 'string' + rnLeaf ',' + rnLeaf 'string' + rnParagraph anchor='foo' + rnLeaf 'Paragraph' + rnLeaf '.' + rnParagraph + rnLeaf 'Ref' + rnLeaf ' ' + rnInternalRef + rnInner + rnLeaf 'foo' + rnLeaf 'foo' + rnLeaf ' ' + """) + suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 867677404..45cf1a68e 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1054,8 +1054,9 @@ Test1 Paragraph2 ref `internal anchor`_. """ let output9 = input9.toHtml - #doAssert "id=\"internal-anchor\"" in output9 - #doAssert "internal anchor" notin output9 + # _`internal anchor` got erased: + check "href=\"#internal-anchor\"" notin output9 + check "href=\"#citation-another\"" in output9 doAssert output9.count("
                  " & "
                  ") == 1 doAssert output9.count("
                  ") == 3 @@ -1330,12 +1331,12 @@ Test1 """ let output1 = input1.toHtml # "target101" should be erased and changed to "section-xyz": - doAssert "href=\"#target101\"" notin output1 - doAssert "id=\"target101\"" notin output1 - doAssert "href=\"#target102\"" notin output1 - doAssert "id=\"target102\"" notin output1 - doAssert "id=\"section-xyz\"" in output1 - doAssert "href=\"#section-xyz\"" in output1 + check "href=\"#target101\"" notin output1 + check "id=\"target101\"" notin output1 + check "href=\"#target102\"" notin output1 + check "id=\"target102\"" notin output1 + check "id=\"section-xyz\"" in output1 + check "href=\"#section-xyz\"" in output1 let input2 = dedent """ .. _target300: @@ -1405,7 +1406,7 @@ Test1 let output1 = input1.toHtml doAssert "id=\"secdot1\"" in output1 doAssert "id=\"Z2minusothercolonsecplusc-2\"" in output1 - doAssert "id=\"linkdot1-2021\"" in output1 + check "id=\"linkdot1-2021\"" in output1 let ref1 = "sec.1" let ref2 = "2-other:sec+c_2" let ref3 = "link.1_2021" From ee703c5db4ceb8d0228b4429c53d85d5f1ff91ea Mon Sep 17 00:00:00 2001 From: Tomasz Kramkowski Date: Fri, 29 Oct 2021 10:39:41 +0100 Subject: [PATCH 0500/1654] Fix nimIdentNormalize, fixes #19067 (#19068) * Make nimIdentNormalize return "" when passed ""; fixes #19067 Fixes #19067 * Add tests for nimIdentNormalize --- lib/pure/strutils.nim | 5 +++-- tests/stdlib/tstrutils.nim | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 2234a8625..e4a4a5ec1 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -281,8 +281,9 @@ func nimIdentNormalize*(s: string): string = runnableExamples: doAssert nimIdentNormalize("Foo_bar") == "Foobar" result = newString(s.len) - if s.len > 0: - result[0] = s[0] + if s.len == 0: + return + result[0] = s[0] var j = 1 for i in 1..len(s) - 1: if s[i] in {'A'..'Z'}: diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index 234991bdb..8000b6b07 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -841,5 +841,12 @@ bar doAssert s.endsWith('a') == false doAssert s.endsWith('\0') == false + block: # nimIdentNormalize + doAssert nimIdentNormalize("") == "" + doAssert nimIdentNormalize("foo") == "foo" + doAssert nimIdentNormalize("foo_bar") == "foobar" + doAssert nimIdentNormalize("Foo_bar") == "Foobar" + doAssert nimIdentNormalize("_Foo_bar") == "_foobar" + static: main() main() From cfdac6666f5772479724e082fb08c2db694d8d40 Mon Sep 17 00:00:00 2001 From: Tail Wag Games Date: Fri, 29 Oct 2021 06:42:44 -0500 Subject: [PATCH 0501/1654] Freeing critical sections via atexit in system/alloc and system/io (#19062) * adding new system module sysexitprocs and including system exit procedures when registering exit handlers defined in userland * fixing failing tests and adding initialization guard to handle cases where the module's global init logic isn't invoked first as is the case with some gc implementaions * js backend shouldn't try to invoke actual system exit procs * fixing formatting in sysexitprocs.nim * 256 was too much - my max number of plugins in my engine is 64 and I require two hooks per runtime it looks like with tls emulation turned off, so for my purposes 128 should be sufficient * so atExit should be enough here, can get rid of all the extra cruft I had added on top since I didn't realize atExit already provided a stack * done being cute - since newruntime prevents correct cpp codegen for object variants apparently and breaks tests if I try to use std/exitprocs, ddSysExitProc is just going into both modules. Since system doesn't include system/io, polluting system with it doesn't make sense either... at least it is only importc'd when it is required in either module and we don't have to have any weird when defined(nimOwnedEnabled) with a comment explaining why --- lib/std/exitprocs.nim | 3 --- lib/system/alloc.nim | 3 +++ lib/system/io.nim | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index c76583a8c..c6537f7f8 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -44,9 +44,6 @@ proc callClosures() {.noconv.} = of kClosure: fun.fun1() of kNoconv: fun.fun2() - when not defined(js) and not defined(nimOwnedEnabled): - deinitLock(gFunsLock) - template fun() = if gFuns.len == 0: addAtExit(callClosures) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 6d83a2c17..2c6ab4462 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -1054,9 +1054,12 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = it = it.next when hasThreadSupport: + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "".} + var sharedHeap: MemRegion var heapLock: SysLock initSysLock(heapLock) + addSysExitProc(proc() {.noconv.} = deinitSys(heapLock)) proc getFreeMem(): int = #sysAssert(result == countFreeMem()) diff --git a/lib/system/io.nim b/lib/system/io.nim index e7369392a..2ad43acdb 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -777,15 +777,15 @@ proc setStdIoUnbuffered*() {.tags: [], benign.} = when declared(stdout): when defined(windows) and compileOption("threads"): + proc addSysExitProc(quitProc: proc() {.noconv.}) {.importc: "atexit", header: "".} + const insideRLocksModule = false include "system/syslocks" + var echoLock: SysLock initSysLock echoLock - - when not defined(js) and not defined(nimOwnedEnabled): - import std/exitprocs - addExitProc(proc() {.noconv.} = deinitSys echoLock) + addSysExitProc(proc() {.noconv.} = deinitSys(echoLock)) const stdOutLock = not defined(windows) and not defined(android) and From f755e452d2a2c7656f07793b7f01dc654a7c5253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Sat, 30 Oct 2021 01:55:48 +0800 Subject: [PATCH 0502/1654] fix #18971 (#19070) [backport:1.6] since the example code return value from global variable, instead of first argument, the `n.len` is 1 which causes compiler crashes. --- compiler/varpartitions.nim | 2 +- tests/arc/t18971.nim | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/arc/t18971.nim diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 8f422face..709d00fa0 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -479,7 +479,7 @@ proc destMightOwn(c: var Partitions; dest: var VarIndex; n: PNode) = # calls do construct, what we construct must be destroyed, # so dest cannot be a cursor: dest.flags.incl ownsData - elif n.typ.kind in {tyLent, tyVar}: + elif n.typ.kind in {tyLent, tyVar} and n.len > 1: # we know the result is derived from the first argument: var roots: seq[(PSym, int)] allRoots(n[1], roots, RootEscapes) diff --git a/tests/arc/t18971.nim b/tests/arc/t18971.nim new file mode 100644 index 000000000..9b587d956 --- /dev/null +++ b/tests/arc/t18971.nim @@ -0,0 +1,10 @@ +discard """ + cmd: "nim c --gc:arc $file" +""" + +type MyObj = ref object + +var o = MyObj() +proc x: var MyObj = o + +var o2 = x() From 2bda4a30a6b12dd0840dc347e454e54fe26721e7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 30 Oct 2021 10:14:23 +0200 Subject: [PATCH 0503/1654] fixes #19000 (#19032) * fixes #19000 * progress --- lib/system/memalloc.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim index f8ebc8c5f..49766e69d 100644 --- a/lib/system/memalloc.nim +++ b/lib/system/memalloc.nim @@ -116,6 +116,9 @@ when hasAlloc and not defined(js): ## ## See also: ## * `create <#create,typedesc>`_ + static: + when sizeof(T) <= 0: + {.fatal: "createU does not support types T where sizeof(T) == 0".} cast[ptr T](alloc(T.sizeof * size)) template alloc0*(size: Natural): pointer = @@ -141,6 +144,9 @@ when hasAlloc and not defined(js): ## ## The allocated memory belongs to its allocating thread! ## Use `createShared <#createShared,typedesc>`_ to allocate from a shared heap. + static: + when sizeof(T) <= 0: + {.fatal: "create does not support types T where sizeof(T) == 0".} cast[ptr T](alloc0(sizeof(T) * size)) template realloc*(p: pointer, newSize: Natural): pointer = From 2f730afe9e7eb981258aea257deba0240f488dd1 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 31 Oct 2021 13:22:00 +0800 Subject: [PATCH 0504/1654] fix #18410 (Errors initializing an object of RootObj with the C++ backend) [backport] (#18836) * fix #18410 * one line comment * typo * typo * cover cpp --- compiler/cgen.nim | 3 ++- tests/arc/tconst_to_sink.nim | 3 ++- tests/cpp/torc.nim | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 tests/cpp/torc.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 412a428da..b669483cc 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -363,7 +363,8 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc, else: linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfoV1(p.module, t, a.lode.info)]) of frEmbedded: - if optTinyRtti in p.config.globalOptions: + # inheritance in C++ does not allow struct initialization: bug #18410 + if not p.module.compileToCpp and optTinyRtti in p.config.globalOptions: var tmp: TLoc if mode == constructRefObj: let objType = t.skipTypes(abstractInst+{tyRef}) diff --git a/tests/arc/tconst_to_sink.nim b/tests/arc/tconst_to_sink.nim index ddcc46e67..25f659341 100644 --- a/tests/arc/tconst_to_sink.nim +++ b/tests/arc/tconst_to_sink.nim @@ -1,6 +1,7 @@ discard """ output: '''@[(s1: "333", s2: ""), (s1: "abc", s2: "def"), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "3x", s2: ""), (s1: "lastone", s2: "")]''' - cmd: "nim c --gc:arc $file" + matrix: "--gc:arc" + targets: "c cpp" """ # bug #13240 diff --git a/tests/cpp/torc.nim b/tests/cpp/torc.nim new file mode 100644 index 000000000..7fc474f94 --- /dev/null +++ b/tests/cpp/torc.nim @@ -0,0 +1,15 @@ +discard """ + targets: "cpp" + matrix: "--gc:orc" +""" + +import std/options + +# bug #18410 +type + O = object of RootObj + val: pointer + +proc p(): Option[O] = none(O) + +doAssert $p() == "none(O)" \ No newline at end of file From 4c510d5577a0ced6a31b8417fed74acdd604b9a3 Mon Sep 17 00:00:00 2001 From: haxscramper Date: Tue, 2 Nov 2021 10:54:04 +0300 Subject: [PATCH 0505/1654] [FIX] Do not break formatted string line (#19085) [backport] Otherwise, compiler produces broken error message - `$1` is not interpolated `Error: The $1 type doesn't have a default value. The following fields must be initialized: importGraph.` --- compiler/semobjconstr.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 001956ede..30ab76957 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -368,8 +368,7 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = let initResult = semConstructTypeAux(c, constrCtx, {}) assert constrCtx.missingFields.len > 0 localError(c.config, info, - "The $1 type doesn't have a default value. The following fields must " & - "be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) + "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) elif objType.kind == tyDistinct: localError(c.config, info, "The $1 distinct type doesn't have a default value." % typeToString(t)) From 24a75842998709b29ef50ef4cc10d30304efe3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Derek=20=E5=91=86?= <116649+derekdai@users.noreply.github.com> Date: Tue, 2 Nov 2021 15:55:50 +0800 Subject: [PATCH 0506/1654] update numbers of lifetime-tracking hooks in doc/destructors.rst (#19088) --- doc/destructors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index 6e4509666..d6f240dee 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -101,7 +101,7 @@ The memory management for Nim's standard `string` and `seq` types as well as other standard collections is performed via so-called "Lifetime-tracking hooks", which are particular `type bound operators `_. -There are 3 different hooks for each (generic or concrete) object type `T` (`T` can also be a +There are 4 different hooks for each (generic or concrete) object type `T` (`T` can also be a `distinct` type) that are called implicitly by the compiler. (Note: The word "hook" here does not imply any kind of dynamic binding From cc984217a98438de5a36447ea6574eb2c5da4ae4 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Tue, 2 Nov 2021 02:02:53 -0600 Subject: [PATCH 0507/1654] Fix VM's sametype impl to work for generics/typedescs (#19073) * Fix vm's sametype implementation to properly handle generics and typedescs * actually fixed sametype + have test * added comments and removed unsafe code --- compiler/types.nim | 5 ++++ compiler/vm.nim | 3 ++- tests/stdlib/tmacros.nim | 58 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/compiler/types.nim b/compiler/types.nim index 2c7b91d9f..7cf02c240 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1128,6 +1128,11 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = a = a.skipTypes({tyDistinct, tyGenericInst}) if a.kind != b.kind: return false + #[ + The following code should not run in the case either side is an generic alias, + but it's not presently possible to distinguish the genericinsts from aliases of + objects ie `type A[T] = SomeObject` + ]# # this is required by tunique_type but makes no sense really: if tyDistinct notin {x.kind, y.kind} and x.kind == tyGenericInst and IgnoreTupleFields notin c.flags: let diff --git a/compiler/vm.nim b/compiler/vm.nim index 1b443aff1..b64545081 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1044,7 +1044,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = strictSymEquality=true)) of opcSameNodeType: decodeBC(rkInt) - regs[ra].intVal = ord(regs[rb].node.typ.sameTypeOrNil regs[rc].node.typ) + regs[ra].intVal = ord(regs[rb].node.typ.sameTypeOrNil(regs[rc].node.typ, {ExactTypeDescValues, ExactGenericParams})) + # The types should exactly match which is why we pass `{ExactTypeDescValues..ExactGcSafety}`. of opcXor: decodeBC(rkInt) regs[ra].intVal = ord(regs[rb].intVal != regs[rc].intVal) diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 18cfdc75c..64a474743 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -66,3 +66,61 @@ block: # unpackVarargs doAssert call1(toString) == "" doAssert call1(toString, 10) == "10" doAssert call1(toString, 10, 11) == "1011" + +block: # SameType + type + A = int + B = distinct int + C = object + Generic[T, Y] = object + macro isSameType(a, b: typed): untyped = + newLit(sameType(a, b)) + + static: + assert Generic[int, int].isSameType(Generic[int, int]) + assert Generic[A, string].isSameType(Generic[int, string]) + assert not Generic[A, string].isSameType(Generic[B, string]) + assert not Generic[int, string].isSameType(Generic[int, int]) + assert isSameType(int, A) + assert isSameType(10, 20) + assert isSameType("Hello", "world") + assert not isSameType("Hello", cstring"world") + assert not isSameType(int, B) + assert not isSameType(int, Generic[int, int]) + assert not isSameType(C, string) + assert not isSameType(C, int) + + + #[ + # compiler sameType fails for the following, read more in `types.nim`'s `sameTypeAux`. + type + D[T] = C + G[T] = T + static: + assert isSameType(D[int], C) + assert isSameType(D[int], D[float]) + assert isSameType(G[float](1.0), float(1.0)) + assert isSameType(float(1.0), G[float](1.0)) + ]# + + type Tensor[T] = object + data: T + + macro testTensorInt(x: typed): untyped = + let + tensorIntType = getTypeInst(Tensor[int])[1] + xTyp = x.getTypeInst + + newLit(xTyp.sameType(tensorIntType)) + + var + x: Tensor[int] + x1 = Tensor[float]() + x2 = Tensor[A]() + x3 = Tensor[B]() + + static: + assert testTensorInt(x) + assert not testTensorInt(x1) + assert testTensorInt(x2) + assert not testTensorInt(x3) From 2e0db988e7efd05b07f5165cf1357efea9391eb3 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Tue, 2 Nov 2021 04:06:46 -0400 Subject: [PATCH 0508/1654] tests: Fix warnings in tstrscans (#19082) --- tests/stdlib/tstrscans.nim | 60 +++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tests/stdlib/tstrscans.nim b/tests/stdlib/tstrscans.nim index d443c5dda..24a3c02f7 100644 --- a/tests/stdlib/tstrscans.nim +++ b/tests/stdlib/tstrscans.nim @@ -15,7 +15,7 @@ block ParsePasswd: else: break - const etc_passwd = """root:x:0:0:root:/root:/bin/bash + const etcPasswd = """root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/bin/sh bin:x:2:2:bin:/bin:/bin/sh sys:x:3:3:sys:/dev:/bin/sh @@ -23,7 +23,7 @@ nobody:x:65534:65534:nobody:/nonexistent:/bin/sh messagebus:x:103:107::/var/run/dbus:/bin/false """ - const parsed_etc_passwd = @[ + const parsedEtcPasswd = @[ "root:x:0:0:root:/root:/bin/bash", "daemon:x:1:1:daemon:/usr/sbin:/bin/sh", "bin:x:2:2:bin:/bin:/bin/sh", @@ -31,7 +31,7 @@ messagebus:x:103:107::/var/run/dbus:/bin/false "nobody:x:65534:65534:nobody:/nonexistent:/bin/sh", "messagebus:x:103:107::/var/run/dbus:/bin/false", ] - doAssert etc_passwd.parsePasswd == parsed_etc_passwd + doAssert etcPasswd.parsePasswd == parsedEtcPasswd block LastNot: var idx : int @@ -139,27 +139,27 @@ block: break var key, val: string - var intval: int - var floatval: float - doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intval, floatVal) + var intVal: int + var floatVal: float + doAssert scanf("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f", key, val, intVal, floatVal) doAssert key == "abc" doAssert val == "xyz" - doAssert intval == 89 + doAssert intVal == 89 doAssert floatVal == 33.25 - var binval: int - var octval: int - var hexval: int - doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) - doAssert binval == 0b0101 - doAssert octval == 0o1234 - doAssert hexval == 0xabcd + var binVal: int + var octVal: int + var hexVal: int + doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binVal, octVal, hexVal) + doAssert binVal == 0b0101 + doAssert octVal == 0o1234 + doAssert hexVal == 0xabcd - let xx = scanf("$abc", "$$$i", intval) + let xx = scanf("$abc", "$$$i", intVal) doAssert xx == false - let xx2 = scanf("$1234", "$$$i", intval) + let xx2 = scanf("$1234", "$$$i", intVal) doAssert xx2 let yy = scanf(";.--Breakpoint00 [output]", @@ -246,24 +246,32 @@ block: result = 0 while start+result < input.len and input[start+result] in seps: inc result + type + ScanRetType = tuple + success: bool + lo: int + hi: int + ch: char + word: string + var res = 0 for line in input.splitLines: - let (success, lo, hi, c ,w) = scanTuple(line, "$i-$i $c: $w") - if success: + let ret: ScanRetType = scanTuple(line, "$i-$i $c: $w") + if ret.success: inc res doAssert res == 4 - let (_, key, val, intval, floatVal) = scanTuple("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f") + let (_, key, val, intVal, floatVal) = scanTuple("abc:: xyz 89 33.25", "$w$s::$s$w$s$i $f") doAssert key == "abc" doAssert val == "xyz" - doAssert intval == 89 + doAssert intVal == 89 doAssert floatVal == 33.25 - let (_, binVal, octVal, hexVal) = scanTuple("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval) - doAssert binval == 0b0101 - doAssert octval == 0o1234 - doAssert hexval == 0xabcd + let (_, binVal, octVal, hexVal) = scanTuple("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binVal, octVal, hexVal) + doAssert binVal == 0b0101 + doAssert octVal == 0o1234 + doAssert hexVal == 0xabcd var (xx,_) = scanTuple("$abc", "$$$i") doAssert xx == false @@ -272,9 +280,9 @@ block: let (xx2, _) = block: scanTuple("$1234", "$$$i") doAssert xx2 - var (yy, intval2, key2) = scanTuple(";.--Breakpoint00 [output]", + var (yy, intVal2, key2) = scanTuple(";.--Breakpoint00 [output]", "$[someSep]Breakpoint${twoDigits}$[someSep({';','.','-'})] [$+]$.", int) doAssert yy doAssert key2 == "output" - doAssert intVal2 == 13 \ No newline at end of file + doAssert intVal2 == 13 From e01e543fce8eab147d4e7eb5b89bba80c25e9cb3 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 2 Nov 2021 16:10:25 +0800 Subject: [PATCH 0509/1654] bootstrapping Nim compiler with `cpp --gc:orc` (#19087) --- koch.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/koch.nim b/koch.nim index 8dd897028..06515aad7 100644 --- a/koch.nim +++ b/koch.nim @@ -602,9 +602,8 @@ proc runCI(cmd: string) = execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls") when not defined(bsd): - if not doUseCpp: - # the BSDs are overwhelmed already, so only run this test on the other machines: - kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") + # the BSDs are overwhelmed already, so only run this test on the other machines: + kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) From 12d2901e89d394cb1eb1242acc0107a2d20e3b3d Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Tue, 2 Nov 2021 04:12:54 -0400 Subject: [PATCH 0510/1654] libs/impore/re: Add note about the requirement of `matches` to be pre-allocated (#19081) Add few runnableExamples for `findBounds` for clarity. Fixes https://github.com/nim-lang/Nim/issues/18775 --- lib/impure/re.nim | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index d6922f4f1..8e5efbb28 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -152,6 +152,9 @@ proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], ## and the captured ## substrings in the array `matches`. If it does not match, nothing ## is written into `matches` and `(-1,0)` is returned. + ## + ## Note: The memory for `matches` needs to be allocated before this function is + ## called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -171,6 +174,14 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], ## and the captured substrings in the array `matches`. ## If it does not match, nothing ## is written into `matches` and `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[string](1) + let (first, last) = findBounds("Hello World", re"(W\w+)", matches) + doAssert first == 6 + doAssert last == 10 + doAssert matches[0] == "World" result = findBounds(cstring(s), pattern, matches, min(start, MaxReBufSize), min(s.len, MaxReBufSize)) @@ -182,6 +193,8 @@ proc findBounds*(buf: cstring, pattern: Regex, ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -202,6 +215,14 @@ proc findBounds*(s: string, pattern: Regex, ## and the captured substrings in the array `matches`. ## If it does not match, nothing is written into `matches` and ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[tuple[first, last: int]](1) + let (first, last) = findBounds("Hello World", re"(\w+)", matches) + doAssert first == 0 + doAssert last == 4 + doAssert matches[0] == (0, 4) result = findBounds(cstring(s), pattern, matches, min(start, MaxReBufSize), min(s.len, MaxReBufSize)) @@ -255,6 +276,8 @@ proc matchLen*(s: string, pattern: Regex, matches: var openArray[string], ## the same as `match`, but it returns the length of the match, ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED) proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], @@ -262,6 +285,8 @@ proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], ## the same as `match`, but it returns the length of the match, ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED) proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = @@ -292,6 +317,7 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string], ## match, nothing is written into `matches` and `false` is ## returned. ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. runnableExamples: import std/sequtils var matches: array[2, string] @@ -306,6 +332,8 @@ proc match*(buf: cstring, pattern: Regex, matches: var openArray[string], ## match, nothing is written into `matches` and `false` is ## returned. ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchLen(buf, pattern, matches, start, bufSize) != -1 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], @@ -314,6 +342,8 @@ proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], ## substrings in the array `matches`. If it does not match, nothing ## is written into `matches` and `-1` is returned. ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -332,6 +362,8 @@ proc find*(s: string, pattern: Regex, matches: var openArray[string], ## returns the starting position of `pattern` in `s` and the captured ## substrings in the array `matches`. If it does not match, nothing ## is written into `matches` and `-1` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = find(cstring(s), pattern, matches, start, s.len) proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = @@ -433,6 +465,8 @@ proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} = proc contains*(s: string, pattern: Regex, matches: var openArray[string], start = 0): bool {.inline.} = ## same as `find(s, pattern, matches, start) >= 0` + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return find(s, pattern, matches, start) >= 0 proc startsWith*(s: string, prefix: Regex): bool {.inline.} = From efaaa38eefa38d842ef9225b6ac3f60c4c842209 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Tue, 2 Nov 2021 05:34:11 -0400 Subject: [PATCH 0511/1654] docstyle.rst: Do not export it to HTML as a standalone doc (#19083) .. This doc is included in contributing.rst. Fixes https://github.com/nim-lang/Nim/issues/14593. --- tools/kochdocs.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 0c5c8f4a3..42e648fb4 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -111,7 +111,10 @@ proc getRst2html(): seq[string] = for a in walkDirRecFilter("doc"): let path = a.path if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin - ["docs.rst", "nimfix.rst"]: + ["docs.rst", "nimfix.rst", + "docstyle.rst" # docstyle.rst shouldn't be converted to html separately; + # it's included in contributing.rst. + ]: # maybe we should still show nimfix, could help reviving it # `docs` is redundant with `overview`, might as well remove that file? result.add path From a55b90827ed87b61972147e8120c9afa949d17f9 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 2 Nov 2021 23:22:11 +0800 Subject: [PATCH 0512/1654] remove the mention of travis from readme (#19093) --- readme.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7f333d5d4..af07c2a7b 100644 --- a/readme.md +++ b/readme.md @@ -149,7 +149,7 @@ read [this guide][pull-request-instructions]. Ideally, you should make sure that all tests pass before submitting a pull request. However, if you are short on time, you can just run the tests specific to your -changes by only running the corresponding categories of tests. Travis CI verifies +changes by only running the corresponding categories of tests. CI verifies that all tests pass before allowing the pull request to be accepted, so only running specific tests should be harmless. Integration tests should go in ``tests/untestable``. @@ -220,7 +220,6 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest [csources-repo-deprecated]: https://github.com/nim-lang/csources [csources-v1-repo]: https://github.com/nim-lang/csources_v1 -[badge-nim-travisci]: https://img.shields.io/travis/nim-lang/Nim/devel.svg?style=flat-square [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square [badge-nim-gitter]: https://img.shields.io/badge/chat-on_gitter-blue.svg?style=flat-square From 9d51197aa4eb404c3cce8c5989146dea8d3de024 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 2 Nov 2021 16:30:59 +0100 Subject: [PATCH 0513/1654] fixes #19078 [backport] (#19090) --- compiler/condsyms.nim | 2 ++ compiler/pragmas.nim | 4 +++- compiler/wordrecg.nim | 2 +- lib/std/private/digitsutils.nim | 9 ++++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index df355bc26..5a7e78d5f 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -138,3 +138,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasHintAll") defineSymbol("nimHasTrace") defineSymbol("nimHasEffectsOf") + + defineSymbol("nimHasEnforceNoRaises") diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index a6e6131d7..702a9e082 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -31,7 +31,7 @@ const wAsmNoStackFrame, wDiscardable, wNoInit, wCodegenDecl, wGensym, wInject, wRaises, wEffectsOf, wTags, wLocks, wDelegator, wGcSafe, wConstructor, wLiftLocals, wStackTrace, wLineTrace, wNoDestroy, - wRequires, wEnsures} + wRequires, wEnsures, wEnforceNoRaises} converterPragmas* = procPragmas methodPragmas* = procPragmas+{wBase}-{wImportCpp} templatePragmas* = {wDeprecated, wError, wGensym, wInject, wDirty, @@ -1237,6 +1237,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, pragmaProposition(c, it) of wEnsures: pragmaEnsures(c, it) + of wEnforceNoRaises: + sym.flags.incl sfNeverRaises else: invalidPragma(c, it) elif comesFromPush and whichKeyword(ident) != wInvalid: discard "ignore the .push pragma; it doesn't apply" diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index ce06effab..22f6cc71d 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -86,7 +86,7 @@ type wAsmNoStackFrame = "asmNoStackFrame", wImplicitStatic = "implicitStatic", wGlobal = "global", wCodegenDecl = "codegenDecl", wUnchecked = "unchecked", wGuard = "guard", wLocks = "locks", wPartial = "partial", wExplain = "explain", - wLiftLocals = "liftlocals", + wLiftLocals = "liftlocals", wEnforceNoRaises = "enforceNoRaises", wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char", wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default", diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index 9be2ab3ef..588bcaec0 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -78,14 +78,17 @@ func addIntImpl(result: var string, x: uint64) {.inline.} = dec next addChars(result, tmp, next, tmp.len - next) -func addInt*(result: var string, x: uint64) = +when not defined(nimHasEnforceNoRaises): + {.pragma: enforceNoRaises.} + +func addInt*(result: var string, x: uint64) {.enforceNoRaises.} = when nimvm: addIntImpl(result, x) else: when not defined(js): addIntImpl(result, x) else: addChars(result, numToString(x)) -proc addInt*(result: var string; x: int64) = +proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = ## Converts integer to its string representation and appends it to `result`. runnableExamples: var s = "foo" @@ -110,5 +113,5 @@ proc addInt*(result: var string; x: int64) = addChars(result, numToString(x)) else: impl() -proc addInt*(result: var string; x: int) {.inline.} = +proc addInt*(result: var string; x: int) {.inline, enforceNoRaises.} = addInt(result, int64(x)) From b155864967ba08861c39403003095d4822b03b4e Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 2 Nov 2021 23:31:30 +0800 Subject: [PATCH 0514/1654] build testament in package CI (#19092) * build testament in package CI * Update testament/important_packages.nim --- koch.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 06515aad7..f87828cdc 100644 --- a/koch.nim +++ b/koch.nim @@ -559,7 +559,8 @@ proc runCI(cmd: string) = let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_") if getEnv("NIM_TEST_PACKAGES", "0") == "1": - execFold("Test selected Nimble packages", "nim r testament/testament $# pcat nimble-packages" % batchParam) + nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release") + execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam) else: buildTools() From 539bced70df035ddbdbf4c3d9c7fd6074f0b1073 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 3 Nov 2021 01:46:34 -0400 Subject: [PATCH 0515/1654] Add test for issue 15435 (#19079) * Add test for issue 15435 Closes https://github.com/nim-lang/Nim/issues/15435. * Specify bug # in comment Addresses https://github.com/nim-lang/Nim/pull/19079#discussion_r740279547 --- tests/sets/t15435.nim | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/sets/t15435.nim diff --git a/tests/sets/t15435.nim b/tests/sets/t15435.nim new file mode 100644 index 000000000..5ead7e641 --- /dev/null +++ b/tests/sets/t15435.nim @@ -0,0 +1,29 @@ +# bug #15435 +discard """ +errormsg: "type mismatch: got " +nimout: '''t15435.nim(28, 13) Error: type mismatch: got +but expected one of: +proc `<`[T](x, y: set[T]): bool + first type mismatch at position: 2 + required type for y: set[T] + but expression 'x' is of type: set[range 1..5(uint8)] +20 other mismatching symbols have been suppressed; compile with --showAllMismatches:on to see them + +expression: {1'u8, 5} < x''' +""" + + + + + + +## line 20 +var + x: set[range[1u8..5u8]] + +x.incl(1) +x.incl(3) +x.incl(5) + +if {1u8, 5} < x: + echo "successful" From 5fed1c05ce7ea9cbeb2b70bb8583e836c981abd4 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 3 Nov 2021 01:48:30 -0400 Subject: [PATCH 0516/1654] manual: Document the use of `static` as a proc call (#19084) * manual: Document the use of `static` as a proc call Also adds tests. Fixes https://github.com/nim-lang/Nim/issues/16987 . * Update doc/manual.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Use the "bug #NNNN" comment syntax for consistency Ref: https://nim-lang.github.io/Nim/contributing.html#writing-tests-stdlib > Always refer to a GitHub issue using the following exact syntax: bug for tooling. * manual: Undocument usage of foo.static foo.static and foo.static() are not expected to work. Ref: https://github.com/nim-lang/Nim/pull/19084/files#r741203578 Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- doc/manual.rst | 14 ++++++++++++++ tests/system/tstatic_callable.nim | 12 ++++++++++++ tests/system/tstatic_callable_error.nim | 14 ++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 tests/system/tstatic_callable.nim create mode 100644 tests/system/tstatic_callable_error.nim diff --git a/doc/manual.rst b/doc/manual.rst index 4b212fe19..1371dba17 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2975,6 +2975,20 @@ Even some code that has side effects is permitted in a static block: static: echo "echo at compile time" +`static` can also be used like a routine. + +.. code-block:: nim + + proc getNum(a: int): int = a + + # Below calls "echo getNum(123)" at compile time. + static: + echo getNum(123) + + # Below call evaluates the "getNum(123)" at compile time, but its + # result gets used at run time. + echo static(getNum(123)) + There are limitations on what Nim code can be executed at compile time; see `Restrictions on Compile-Time Execution <#restrictions-on-compileminustime-execution>`_ for details. diff --git a/tests/system/tstatic_callable.nim b/tests/system/tstatic_callable.nim new file mode 100644 index 000000000..92d1fca8d --- /dev/null +++ b/tests/system/tstatic_callable.nim @@ -0,0 +1,12 @@ +# bug #16987 + +proc getNum(a: int): int = a + +# Below calls "doAssert getNum(123) == 123" at compile time. +static: + doAssert getNum(123) == 123 + +# Below calls evaluate the "getNum(123)" at compile time, but the +# results of those calls get used at run time. +doAssert (static getNum(123)) == 123 +doAssert (static(getNum(123))) == 123 diff --git a/tests/system/tstatic_callable_error.nim b/tests/system/tstatic_callable_error.nim new file mode 100644 index 000000000..c6f1e3d07 --- /dev/null +++ b/tests/system/tstatic_callable_error.nim @@ -0,0 +1,14 @@ +# bug #16987 + +discard """ +errormsg: "cannot evaluate at compile time: inp" +nimout: ''' +tstatic_callable_error.nim(14, 21) Error: cannot evaluate at compile time: inp''' +""" + + +# line 10 +proc getNum(a: int): int = a + +let inp = 123 +echo (static getNum(inp)) From dbbafd320ce1edb0e51167c8e9f6ba3d865672c3 Mon Sep 17 00:00:00 2001 From: Kaushal Modi Date: Wed, 3 Nov 2021 03:08:50 -0400 Subject: [PATCH 0517/1654] manual: Document that comma propagates the default values of parameters (#19080) * manual: Document that comma propagates the default values of parameters Fixes https://github.com/nim-lang/Nim/issues/15949. * Use the "bug #NNNN" comment syntax for consistency Ref: https://nim-lang.github.io/Nim/contributing.html#writing-tests-stdlib > Always refer to a GitHub issue using the following exact syntax: bug for tooling. --- doc/manual.rst | 10 +++++++++- tests/proc/t15949.nim | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/proc/t15949.nim diff --git a/doc/manual.rst b/doc/manual.rst index 1371dba17..001402036 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3603,9 +3603,17 @@ does not provide a value for the argument. The value will be reevaluated every time the function is called. .. code-block:: nim - # b is optional with 47 as its default value + # b is optional with 47 as its default value. proc foo(a: int, b: int = 47): int +Just as the comma propagates the types from right to left until the +first parameter or until a semicolon is hit, it also propagates the +default value starting from the parameter declared with it. + +.. code-block:: nim + # Both a and b are optional with 47 as their default values. + proc foo(a, b: int = 47): int + Parameters can be declared mutable and so allow the proc to modify those arguments, by using the type modifier `var`. diff --git a/tests/proc/t15949.nim b/tests/proc/t15949.nim new file mode 100644 index 000000000..bc3fddc84 --- /dev/null +++ b/tests/proc/t15949.nim @@ -0,0 +1,20 @@ +# bug #15949 + +discard """ +errormsg: "parameter 'a' requires a type" +nimout: ''' +t15949.nim(20, 14) Error: parameter 'a' requires a type''' +""" + + +# line 10 +proc procGood(a, b = 1): (int, int) = (a, b) + +doAssert procGood() == (1, 1) +doAssert procGood(b = 3) == (1, 3) +doAssert procGood(a = 2) == (2, 1) +doAssert procGood(a = 5, b = 6) == (5, 6) + +# The type (and default value propagation breaks in the below example +# as semicolon is used instead of comma. +proc procBad(a; b = 1): (int, int) = (a, b) From f7cfc46ae17b81d9ccdc84573da00561f24ea99f Mon Sep 17 00:00:00 2001 From: David Krause Date: Wed, 3 Nov 2021 09:14:59 +0100 Subject: [PATCH 0518/1654] Added support for sending "ehlo" and receiving multiline "ehlo" response. (#19077) * Added support for sending "ehlo" and receiving multiline "ehlo" response. Signed-off-by: David Krause * fix typo Signed-off-by: David Krause * send "EHLO" first, if not supported, send "HELO" to smtp server. Signed-off-by: David Krause * fix english Signed-off-by: David Krause * add changelog entry for smtp `ehlo` Signed-off-by: David Krause * recvEhlo must not be exported Signed-off-by: David Krause --- changelog.md | 2 ++ lib/pure/smtp.nim | 33 ++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index fbda70582..1636d125f 100644 --- a/changelog.md +++ b/changelog.md @@ -7,7 +7,9 @@ ## Standard library additions and changes +## `std/smtp` +- Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. ## Language changes diff --git a/lib/pure/smtp.nim b/lib/pure/smtp.nim index 5ba048608..f5196ce1e 100644 --- a/lib/pure/smtp.nim +++ b/lib/pure/smtp.nim @@ -100,7 +100,6 @@ proc debugRecv*(smtp: Smtp | AsyncSmtp): Future[string] {.multisync.} = ## `SMTP extensions`_. ## ## See `checkReply(reply)<#checkReply,AsyncSmtp,string>`_. - result = await smtp.sock.recvLine() if smtp.debug: echo("S:" & result) @@ -171,8 +170,7 @@ proc `$`*(msg: Message): string = result.add("\c\L") result.add(msg.msgBody) -proc newSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): Smtp = +proc newSmtp*(useSsl = false, debug = false, sslContext: SslContext = nil): Smtp = ## Creates a new `Smtp` instance. new result result.debug = debug @@ -186,12 +184,10 @@ proc newSmtp*(useSsl = false, debug = false, else: {.error: "SMTP module compiled without SSL support".} -proc newAsyncSmtp*(useSsl = false, debug = false, - sslContext: SslContext = nil): AsyncSmtp = +proc newAsyncSmtp*(useSsl = false, debug = false, sslContext: SslContext = nil): AsyncSmtp = ## Creates a new `AsyncSmtp` instance. new result result.debug = debug - result.sock = newAsyncSocket() if useSsl: when compiledWithSsl: @@ -220,7 +216,6 @@ proc checkReply*(smtp: Smtp | AsyncSmtp, reply: string) {.multisync.} = ## would need to call when using this module. One exception to ## this is if you are implementing any ## `SMTP extensions`_. - var line = await smtp.debugRecv() if not line.startsWith(reply): await quitExcpt(smtp, "Expected " & reply & " reply, got: " & line) @@ -230,6 +225,22 @@ proc helo*(smtp: Smtp | AsyncSmtp) {.multisync.} = await smtp.debugSend("HELO " & smtp.address & "\c\L") await smtp.checkReply("250") +proc recvEhlo(smtp: Smtp | AsyncSmtp): Future[bool] {.multisync.} = + ## Skips "250-" lines, read until "250 " found. + ## Return `true` if server supports `EHLO`, false otherwise. + while true: + var line = await smtp.sock.recvLine() + if smtp.debug: + echo("S:" & line) + if line.startsWith("250-"): continue + elif line.startsWith("250 "): return true # last line + else: return false + +proc ehlo*(smtp: Smtp | AsyncSmtp): Future[bool] {.multisync.} = + ## Sends EHLO request. + await smtp.debugSend("EHLO " & smtp.address & "\c\L") + return await smtp.recvEhlo() + proc connect*(smtp: Smtp | AsyncSmtp, address: string, port: Port) {.multisync.} = ## Establishes a connection with a SMTP server. @@ -237,7 +248,9 @@ proc connect*(smtp: Smtp | AsyncSmtp, smtp.address = address await smtp.sock.connect(address, port) await smtp.checkReply("220") - await smtp.helo() + let speaksEsmtp = await smtp.ehlo() + if not speaksEsmtp: + await smtp.helo() proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync.} = ## Put the SMTP connection in TLS (Transport Layer Security) mode. @@ -249,7 +262,9 @@ proc startTls*(smtp: Smtp | AsyncSmtp, sslContext: SslContext = nil) {.multisync getSSLContext().wrapConnectedSocket(smtp.sock, handshakeAsClient) else: sslContext.wrapConnectedSocket(smtp.sock, handshakeAsClient) - await smtp.helo() + let speaksEsmtp = await smtp.ehlo() + if not speaksEsmtp: + await smtp.helo() else: {.error: "SMTP module compiled without SSL support".} From b2edc3468cf0857b337bff09cb68649d49803ccf Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Wed, 3 Nov 2021 11:16:24 +0300 Subject: [PATCH 0519/1654] Use testament to check Norm test pass (#19018) * Use testament to check Norm test pass This is what I actually use to test Norm, so it's better to use it. This should not currently pass. This is expected because this is exactly the problem I want to highlight with this PR. My tests do indeed not pass at the moment. * Remove clearNimblePath from testament command. Co-authored-by: flywind --- testament/important_packages.nim | 2 +- testament/specs.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index e037cc609..fc5c808ab 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -115,7 +115,7 @@ pkg "nimterop", "nimble minitest" pkg "nimwc", "nim c nimwc.nim" pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg "norm", "nim c -r tests/sqlite/trows.nim" +pkg "norm", "testament r tests/sqlite/trows.nim" pkg "npeg", "nimble testarc" pkg "numericalnim", "nim c -r tests/test_integrate.nim" pkg "optionsutils" diff --git a/testament/specs.nim b/testament/specs.nim index 6fd0ab22d..c7e332262 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -103,7 +103,7 @@ type proc getCmd*(s: TSpec): string = if s.cmd.len == 0: - result = compilerPrefix & " $target --hints:on -d:testing --clearNimblePath --nimblePath:build/deps/pkgs $options $file" + result = compilerPrefix & " $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file" else: result = s.cmd From f2f15e972645496f901fcfde197048de368453d6 Mon Sep 17 00:00:00 2001 From: Don-Duong Quach Date: Wed, 3 Nov 2021 08:47:31 -0700 Subject: [PATCH 0520/1654] fix for #19020, credit to @ElegantBeef (#19021) --- lib/core/macros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 806ea7e31..6dbb92875 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1611,7 +1611,7 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = let pragmaNode = customPragmaNode(n) for p in pragmaNode: if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp: - if p.len == 2: + if p.len == 2 or (p.len == 3 and p[1].kind == nnkSym and p[1].symKind == nskType): result = p[1] else: let def = p[0].getImpl[3] From fce89cb60a3428c61da3c7e6c2d3e6cdcb14bb3b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 7 Nov 2021 16:38:02 +0100 Subject: [PATCH 0521/1654] fixes another effect inference bug [backport:1.6] (#19100) * fixes another effect inference bug [backport:1.6] --- compiler/sempass2.nim | 3 ++ lib/pure/asyncdispatch.nim | 4 +- lib/pure/json.nim | 10 +++-- lib/system/excpt.nim | 13 ++++--- lib/system/seqs_v2.nim | 2 +- tests/effects/tnestedprocs.nim | 63 +++++++++++++++++++++++++++++++ tests/pragmas/tcompile_pragma.nim | 1 + 7 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 tests/effects/tnestedprocs.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 9a27d14fa..5624d7adb 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -851,6 +851,9 @@ proc trackCall(tracked: PEffects; n: PNode) = elif isIndirectCall(tracked, a): assumeTheWorst(tracked, n, op) gcsafeAndSideeffectCheck() + else: + if strictEffects in tracked.c.features and a.kind == nkSym and a.sym.kind in routineKinds: + propagateEffects(tracked, n, a.sym) else: mergeRaises(tracked, effectList[exceptionEffects], n) mergeTags(tracked, effectList[tagEffects], n) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index e15fb0851..c924d0a3d 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -733,7 +733,7 @@ when defined(windows) or defined(nimdoc): proc acceptAddr*(socket: AsyncFD, flags = {SocketFlag.SafeDisconn}, inheritable = defined(nimInheritHandles)): - owned(Future[tuple[address: string, client: AsyncFD]]) = + owned(Future[tuple[address: string, client: AsyncFD]]) {.gcsafe.} = ## Accepts a new connection. Returns a future containing the client socket ## corresponding to that connection and the remote address of the client. ## The future will complete when the connection is successfully accepted. @@ -800,7 +800,7 @@ when defined(windows) or defined(nimdoc): var ol = newCustom() ol.data = CompletionData(fd: socket, cb: - proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = + proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) {.gcsafe.} = if not retFuture.finished: if errcode == OSErrorCode(-1): completeAccept() diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 85c3393b2..c831bf85d 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -437,7 +437,7 @@ macro `%*`*(x: untyped): untyped = ## `%` for every element. result = toJsonImpl(x) -proc `==`*(a, b: JsonNode): bool = +proc `==`*(a, b: JsonNode): bool {.noSideEffect.} = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -464,12 +464,16 @@ proc `==`*(a, b: JsonNode): bool = if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - if b.fields[key] != val: return false + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: + if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} -proc hash*(n: JsonNode): Hash = +proc hash*(n: JsonNode): Hash {.noSideEffect.} = ## Compute the hash for a JSON node case n.kind of JArray: diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 3e520570f..a71328c14 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -354,7 +354,7 @@ var onUnhandledException*: (proc (errorMsg: string) {. ## The default is to write a stacktrace to `stderr` and then call `quit(1)`. ## Unstable API. -proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = +proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy, gcsafe.} = when hasSomeStackTrace: var buf = newStringOfCap(2000) if e.trace.len == 0: @@ -362,7 +362,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = else: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") @@ -373,7 +374,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = onUnhandledException(buf) else: showErrorMessage2(buf) - `=destroy`(buf) + {.gcsafe.}: + `=destroy`(buf) else: # ugly, but avoids heap allocations :-) template xadd(buf, s, slen) = @@ -387,7 +389,8 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = if e.trace.len != 0: var trace = $e.trace add(buf, trace) - `=destroy`(trace) + {.gcsafe.}: + `=destroy`(trace) add(buf, "Error: unhandled exception: ") add(buf, e.msg) add(buf, " [") @@ -398,7 +401,7 @@ proc reportUnhandledErrorAux(e: ref Exception) {.nodestroy.} = else: showErrorMessage(buf.addr, L) -proc reportUnhandledError(e: ref Exception) {.nodestroy.} = +proc reportUnhandledError(e: ref Exception) {.nodestroy, gcsafe.} = if unhandledExceptionHook != nil: unhandledExceptionHook(e) when hostOS != "any": diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 375eef340..0ac51109e 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -73,7 +73,7 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin q.cap = newCap result = q -proc shrink*[T](x: var seq[T]; newLen: Natural) = +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [].} = when nimvm: setLen(x, newLen) else: diff --git a/tests/effects/tnestedprocs.nim b/tests/effects/tnestedprocs.nim new file mode 100644 index 000000000..125896d44 --- /dev/null +++ b/tests/effects/tnestedprocs.nim @@ -0,0 +1,63 @@ +discard """ + cmd: "nim check --hints:off $file" + nimout: '''tnestedprocs.nim(27, 8) Error: 'inner' can have side effects +> tnestedprocs.nim(29, 13) Hint: 'inner' calls `.sideEffect` 'outer2' +>> tnestedprocs.nim(26, 6) Hint: 'outer2' called by 'inner' + +tnestedprocs.nim(45, 8) Error: 'inner' can have side effects +> tnestedprocs.nim(47, 13) Hint: 'inner' calls `.sideEffect` 'outer6' +>> tnestedprocs.nim(44, 6) Hint: 'outer6' called by 'inner' + +tnestedprocs.nim(58, 41) Error: type mismatch: got but expected 'proc (){.closure, noSideEffect.}' + Pragma mismatch: got '{..}', but expected '{.noSideEffect.}'. +''' + errormsg: "type mismatch: got but expected 'proc (){.closure, noSideEffect.}'" +""" +{.experimental: "strictEffects".} +proc outer {.noSideEffect.} = + proc inner(p: int) = + if p == 0: + outer() + + inner(4) + +outer() + +proc outer2 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + outer2() + + inner(4) + +outer2() + +proc outer3(p: int) {.noSideEffect.} = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + p.outer3() + + inner(4) + +outer3(5) + +proc outer6 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + outer6() + + inner(4) + echo "bad" + +outer6() + + +proc outer4 = + proc inner(p: int) {.noSideEffect.} = + if p == 0: + let x: proc () {.noSideEffect.} = outer4 + x() + + inner(4) + +outer4() diff --git a/tests/pragmas/tcompile_pragma.nim b/tests/pragmas/tcompile_pragma.nim index 4e09a7501..5b99352dd 100644 --- a/tests/pragmas/tcompile_pragma.nim +++ b/tests/pragmas/tcompile_pragma.nim @@ -1,5 +1,6 @@ discard """ output: '''34''' + joinable: false """ {.compile("cfunction.c", "-DNUMBER_HERE=34").} From b423ab138f41c24db3b168fd30f1ecce7850a170 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 8 Nov 2021 07:21:15 +0000 Subject: [PATCH 0522/1654] Remove invalid statements about try in async docs. (#19108) --- lib/pure/asyncdispatch.nim | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index c924d0a3d..20ad60427 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -101,16 +101,9 @@ ## Handling Exceptions ## ------------------- ## -## The most reliable way to handle exceptions is to use `yield` on a future -## then check the future's `failed` property. For example: +## You can handle exceptions in the same way as in ordinary Nim code; +## by using the try statement: ## -## .. code-block:: Nim -## var future = sock.recv(100) -## yield future -## if future.failed: -## # Handle exception -## -## The `async` procedures also offer limited support for the try statement. ## ## .. code-block:: Nim ## try: @@ -119,9 +112,16 @@ ## except: ## # Handle exception ## -## Unfortunately the semantics of the try statement may not always be correct, -## and occasionally the compilation may fail altogether. -## As such it is better to use the former style when possible. +## +## +## An alternative approach to handling exceptions is to use `yield` on a future +## then check the future's `failed` property. For example: +## +## .. code-block:: Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception ## ## ## Discarding futures @@ -1987,4 +1987,4 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: raiseOSError(osLastError()) - result = int(fdLim.rlim_cur) - 1 \ No newline at end of file + result = int(fdLim.rlim_cur) - 1 From b21eb1ed36aa068cc9b0a304742e4c81b5112304 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 8 Nov 2021 15:10:01 +0300 Subject: [PATCH 0523/1654] change os.nim doc links to new style (#19102) --- compiler/docgen.nim | 7 + doc/docgen.rst | 7 + lib/packages/docutils/dochelpers.nim | 19 +- lib/pure/includes/osenv.nim | 42 +- lib/pure/includes/oserr.nim | 21 +- lib/pure/includes/osseps.nim | 6 +- lib/pure/os.nim | 558 +++++++++++++-------------- tests/stdlib/tdochelpers.nim | 6 + 8 files changed, 345 insertions(+), 321 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2639840d1..7ba952083 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -954,6 +954,13 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = if kind != tkSpaces: result.generics.add(literal.nimIdentNormalize) + if k == skType: + case n[2].kind + of nkEnumTy: result.symTypeKind = "enum" + of nkObjectTy: result.symTypeKind = "object" + of nkTupleTy: result.symTypeKind = "tuple" + else: discard + proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let diff --git a/doc/docgen.rst b/doc/docgen.rst index 51e7102eb..48166b2c5 100644 --- a/doc/docgen.rst +++ b/doc/docgen.rst @@ -399,6 +399,13 @@ recognized fine:: no backticks: `func [][T](x: openArray[T]): T`_ escaped: `func \`[]\`[T](x: openArray[T]): T`_ +.. Note:: Types that defined as `enum`, or `object`, or `tuple` can also be + referenced with those names directly (instead of `type`):: + + type CopyFlag = enum + ... + ## Ref. `CopyFlag enum`_ + Related Options =============== diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim index c488c4d99..8f97a7119 100644 --- a/lib/packages/docutils/dochelpers.nim +++ b/lib/packages/docutils/dochelpers.nim @@ -17,7 +17,9 @@ import rstast type LangSymbol* = object ## symbol signature in Nim - symKind*: string ## "proc", "const", etc + symKind*: string ## "proc", "const", "type", etc + symTypeKind*: string ## ""|enum|object|tuple - + ## valid only when `symKind == "type"` name*: string ## plain symbol name without any parameters generics*: string ## generic parameters (without brackets) isGroup*: bool ## is LangSymbol a group with overloads? @@ -79,7 +81,14 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = assert linkText.kind in {rnRef, rnInner} const NimDefs = ["proc", "func", "macro", "method", "iterator", - "template", "converter", "const", "type", "var"] + "template", "converter", "const", "type", "var", + "enum", "object", "tuple"] + template resolveSymKind(x: string) = + if x in ["enum", "object", "tuple"]: + result.symKind = "type" + result.symTypeKind = x + else: + result.symKind = x type State = enum inBeginning @@ -97,7 +106,7 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = if curIdent != "": case state of inBeginning: doAssert false, "incorrect state inBeginning" - of afterSymKind: result.symKind = curIdent + of afterSymKind: resolveSymKind curIdent of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize of afterSymbolName: doAssert false, "incorrect state afterSymbolName" @@ -195,7 +204,7 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = let isPostfixSymKind = i > 0 and i == L - 1 and result.symKind == "" and s(i) in NimDefs if isPostfixSymKind: # for links like `foo proc`_ - result.symKind = s(i) + resolveSymKind s(i) else: case state of inBeginning: @@ -235,6 +244,8 @@ proc match*(generated: LangSymbol, docLink: LangSymbol): bool = result = docLink.symKind in ["proc", "func"] else: result = generated.symKind == docLink.symKind + if result and docLink.symKind == "type" and docLink.symTypeKind != "": + result = generated.symTypeKind == docLink.symTypeKind if not result: return result = generated.name == docLink.name if not result: return diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 00a82327c..1a01ab9bc 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -56,13 +56,13 @@ when not defined(nimscript): ## ## If the variable does not exist, `""` is returned. To distinguish ## whether a variable exists or it's value is just `""`, call - ## `existsEnv(key) proc <#existsEnv,string>`_. + ## `existsEnv(key) proc`_. ## ## See also: - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ runnableExamples: assert getEnv("unknownEnv") == "" assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" @@ -76,10 +76,10 @@ when not defined(nimscript): ## Returns true if it exists, false otherwise. ## ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ runnableExamples: assert not existsEnv("unknownEnv") @@ -90,10 +90,10 @@ when not defined(nimscript): ## If an error occurs, `OSError` is raised. ## ## See also: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `delEnv proc <#delEnv,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ when defined(windows): if key.len == 0 or '=' in key: raise newException(OSError, "invalid key, got: " & $(key, val)) @@ -108,10 +108,10 @@ when not defined(nimscript): ## If an error occurs, `OSError` is raised. ## ## See also:ven - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `envPairs iterator <#envPairs.i>`_ + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ template bail = raiseOSError(osLastError(), key) when defined(windows): #[ @@ -190,10 +190,10 @@ iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = ## in the second its value. ## ## Works in native backends, nodejs and vm, like the following APIs: - ## * `getEnv proc <#getEnv,string,string>`_ - ## * `existsEnv proc <#existsEnv,string>`_ - ## * `putEnv proc <#putEnv,string,string>`_ - ## * `delEnv proc <#delEnv,string>`_ + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ when nimvm: for ai in envPairsImplSeq(): yield ai else: diff --git a/lib/pure/includes/oserr.nim b/lib/pure/includes/oserr.nim index e42771d52..01403fa70 100644 --- a/lib/pure/includes/oserr.nim +++ b/lib/pure/includes/oserr.nim @@ -18,7 +18,7 @@ proc `$`*(err: OSErrorCode): string {.borrow.} proc osErrorMsg*(errorCode: OSErrorCode): string = ## Converts an OS error code into a human readable string. ## - ## The error code can be retrieved using the `osLastError proc <#osLastError>`_. + ## The error code can be retrieved using the `osLastError proc`_. ## ## If conversion fails, or `errorCode` is `0` then `""` will be ## returned. @@ -28,8 +28,8 @@ proc osErrorMsg*(errorCode: OSErrorCode): string = ## message. ## ## See also: - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ - ## * `osLastError proc <#osLastError>`_ + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ runnableExamples: when defined(linux): assert osErrorMsg(OSErrorCode(0)) == "" @@ -63,18 +63,17 @@ proc newOSError*( ## Creates a new `OSError exception `_. ## ## The `errorCode` will determine the - ## message, `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ will be used + ## message, `osErrorMsg proc`_ will be used ## to get this message. ## - ## The error code can be retrieved using the `osLastError proc - ## <#osLastError>`_. + ## The error code can be retrieved using the `osLastError proc`_. ## ## If the error code is `0` or an error message could not be retrieved, ## the message `unknown OS error` will be used. ## ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `osLastError proc <#osLastError>`_ + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ var e: owned(ref OSError); new(e) e.errorCode = errorCode.int32 e.msg = osErrorMsg(errorCode) @@ -90,7 +89,7 @@ proc newOSError*( proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = ## Raises an `OSError exception `_. ## - ## Read the description of the `newOSError proc <#newOSError,OSErrorCode,string>`_ to learn + ## Read the description of the `newOSError proc`_ to learn ## how the exception object is created. raise newOSError(errorCode, additionalInfo) @@ -109,8 +108,8 @@ proc osLastError*(): OSErrorCode {.sideEffect.} = ## immediately after an OS call fails. On POSIX systems this is not a problem. ## ## See also: - ## * `osErrorMsg proc <#osErrorMsg,OSErrorCode>`_ - ## * `raiseOSError proc <#raiseOSError,OSErrorCode,string>`_ + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ when defined(nimscript): discard elif defined(windows): diff --git a/lib/pure/includes/osseps.nim b/lib/pure/includes/osseps.nim index 10c85047b..1ea587e3c 100644 --- a/lib/pure/includes/osseps.nim +++ b/lib/pure/includes/osseps.nim @@ -37,9 +37,9 @@ const when doslikeFileSystem: '/' else: DirSep ## An alternative character used by the operating system to separate - ## pathname components, or the same as `DirSep <#DirSep>`_ if only one separator + ## pathname components, or the same as DirSep_ if only one separator ## character exists. This is set to `'/'` on Windows systems - ## where `DirSep <#DirSep>`_ is a backslash (`'\\'`). + ## where DirSep_ is a backslash (`'\\'`). PathSep* = when defined(macos) or defined(RISCOS): ',' @@ -55,7 +55,7 @@ const defined(PalmOS) or defined(MorphOS): false else: true ## True if the file system is case sensitive, false otherwise. Used by - ## `cmpPaths proc <#cmpPaths,string,string>`_ to compare filenames properly. + ## `cmpPaths proc`_ to compare filenames properly. ExeExt* = when doslikeFileSystem: "exe" diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 5320aa87e..513e09b48 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -21,9 +21,9 @@ runnableExamples: ## **See also:** ## * `osproc module `_ for process communication beyond -## `execShellCmd proc <#execShellCmd,string>`_ +## `execShellCmd proc`_ ## * `parseopt module `_ for command-line parser beyond -## `parseCmdLine proc <#parseCmdLine,string>`_ +## `parseCmdLine proc`_ ## * `uri module `_ ## * `distros module `_ ## * `dynlib module `_ @@ -152,9 +152,9 @@ proc joinPath*(head, tail: string): string {. ## head has one). ## ## See also: - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `splitPath proc`_ ## * `uri.combine proc `_ ## * `uri./ proc `_ runnableExamples: @@ -189,17 +189,17 @@ proc joinPath*(head, tail: string): string {. proc joinPath*(parts: varargs[string]): string {.noSideEffect, rtl, extern: "nos$1OpenArray".} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_, + ## The same as `joinPath(head, tail) proc`_, ## but works with any number of directory parts. ## ## You need to pass at least one element or the proc ## will assert in debug builds and crash on release builds. ## ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ + ## * `joinPath(head, tail) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `splitPath proc`_ runnableExamples: when defined(posix): assert joinPath("a") == "a" @@ -214,13 +214,13 @@ proc joinPath*(parts: varargs[string]): string {.noSideEffect, joinPathImpl(result, state, parts[i]) proc `/`*(head, tail: string): string {.noSideEffect, inline.} = - ## The same as `joinPath(head, tail) proc <#joinPath,string,string>`_. + ## The same as `joinPath(head, tail) proc`_. ## ## See also: - ## * `/../ proc <#/../,string,string>`_ - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `splitPath proc <#splitPath,string>`_ + ## * `/../ proc`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `splitPath proc`_ ## * `uri.combine proc `_ ## * `uri./ proc `_ runnableExamples: @@ -239,11 +239,11 @@ proc splitPath*(path: string): tuple[head, tail: string] {. ## ``head / tail == path`` (except for edge cases like "/usr"). ## ## See also: - ## * `joinPath(head, tail) proc <#joinPath,string,string>`_ - ## * `joinPath(varargs) proc <#joinPath,varargs[string]>`_ - ## * `/ proc <#/,string,string>`_ - ## * `/../ proc <#/../,string,string>`_ - ## * `relativePath proc <#relativePath,string,string>`_ + ## * `joinPath(head, tail) proc`_ + ## * `joinPath(parts: varargs[string]) proc`_ + ## * `/ proc`_ + ## * `/../ proc`_ + ## * `relativePath proc`_ runnableExamples: assert splitPath("usr/local/bin") == ("usr/local", "bin") assert splitPath("usr/local/bin/") == ("usr/local/bin", "") @@ -366,7 +366,7 @@ proc relativePath*(path, base: string, sep = DirSep): string {. rtl, extern: "nos$1".} = ## Converts `path` to a path relative to `base`. ## - ## The `sep` (default: `DirSep <#DirSep>`_) is used for the path normalizations, + ## The `sep` (default: DirSep_) is used for the path normalizations, ## this can be useful to ensure the relative path only contains `'/'` ## so that it can be used for URL constructions. ## @@ -375,9 +375,9 @@ proc relativePath*(path, base: string, sep = DirSep): string {. ## That means an absolute path can be returned. ## ## See also: - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ - ## * `tailDir proc <#tailDir,string>`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ + ## * `tailDir proc`_ runnableExamples: assert relativePath("/Users/me/bar/z.nim", "/Users/other/bad", '/') == "../../me/bar/z.nim" assert relativePath("/Users/me/bar/z.nim", "/Users/other", '/') == "../me/bar/z.nim" @@ -476,14 +476,13 @@ proc parentDir*(path: string): string {. ## ## This is similar to ``splitPath(path).head`` when ``path`` doesn't end ## in a dir separator, but also takes care of path normalizations. - ## The remainder can be obtained with `lastPathPart(path) proc - ## <#lastPathPart,string>`_. + ## The remainder can be obtained with `lastPathPart(path) proc`_. ## ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `tailDir proc <#tailDir,string>`_ - ## * `parentDirs iterator <#parentDirs.i,string>`_ + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `tailDir proc`_ + ## * `parentDirs iterator`_ runnableExamples: assert parentDir("") == "" when defined(posix): @@ -511,9 +510,9 @@ proc tailDir*(path: string): string {. ## Returns the tail part of `path`. ## ## See also: - ## * `relativePath proc <#relativePath,string,string>`_ - ## * `splitPath proc <#splitPath,string>`_ - ## * `parentDir proc <#parentDir,string>`_ + ## * `relativePath proc`_ + ## * `splitPath proc`_ + ## * `parentDir proc`_ runnableExamples: assert tailDir("/bin") == "bin" assert tailDir("bin") == "" @@ -556,7 +555,7 @@ iterator parentDirs*(path: string, fromRoot=false, inclusive=true): string = ## only the directories appearing in the relative path. ## ## See also: - ## * `parentDir proc <#parentDir,string>`_ + ## * `parentDir proc`_ ## runnableExamples: let g = "a/b/c" @@ -599,8 +598,8 @@ proc `/../`*(head, tail: string): string {.noSideEffect.} = ## directory. Then ``head / tail`` is performed instead. ## ## See also: - ## * `/ proc <#/,string,string>`_ - ## * `parentDir proc <#parentDir,string>`_ + ## * `/ proc`_ + ## * `parentDir proc`_ runnableExamples: when defined(posix): assert "a/b/c" /../ "d/e" == "a/b/d/e" @@ -621,11 +620,11 @@ proc searchExtPos*(path: string): int = ## of extension. Returns -1 otherwise. ## ## See also: - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: assert searchExtPos("a/b/c") == -1 assert searchExtPos("c.nim") == 1 @@ -645,7 +644,7 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {. noSideEffect, rtl, extern: "nos$1".} = ## Splits a filename into `(dir, name, extension)` tuple. ## - ## `dir` does not end in `DirSep <#DirSep>`_ unless it's `/`. + ## `dir` does not end in DirSep_ unless it's `/`. ## `extension` includes the leading dot. ## ## If `path` has no extension, `ext` is the empty string. @@ -653,11 +652,11 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {. ## If `path` has no filename component, `name` and `ext` are empty strings. ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: var (dir, name, ext) = splitFile("usr/local/nimc.html") assert dir == "usr/local" @@ -698,15 +697,14 @@ proc extractFilename*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = ## Extracts the filename of a given `path`. ## - ## This is the same as ``name & ext`` from `splitFile(path) proc - ## <#splitFile,string>`_. + ## This is the same as ``name & ext`` from `splitFile(path) proc`_. ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: assert extractFilename("foo/bar/") == "" assert extractFilename("foo/bar") == "bar" @@ -719,15 +717,15 @@ proc extractFilename*(path: string): string {. proc lastPathPart*(path: string): string {. noSideEffect, rtl, extern: "nos$1".} = - ## Like `extractFilename proc <#extractFilename,string>`_, but ignores + ## Like `extractFilename proc`_, but ignores ## trailing dir separator; aka: `baseName`:idx: in some other languages. ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `changeFileExt proc`_ + ## * `addFileExt proc`_ runnableExamples: assert lastPathPart("foo/bar/") == "bar" assert lastPathPart("foo/bar") == "bar" @@ -747,11 +745,11 @@ proc changeFileExt*(filename, ext: string): string {. ## of none such beast.) ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `addFileExt proc <#addFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `addFileExt proc`_ runnableExamples: assert changeFileExt("foo.bar", "baz") == "foo.baz" assert changeFileExt("foo.bar", "") == "foo" @@ -771,11 +769,11 @@ proc addFileExt*(filename, ext: string): string {. ## (Although I know of none such beast.) ## ## See also: - ## * `searchExtPos proc <#searchExtPos,string>`_ - ## * `splitFile proc <#splitFile,string>`_ - ## * `extractFilename proc <#extractFilename,string>`_ - ## * `lastPathPart proc <#lastPathPart,string>`_ - ## * `changeFileExt proc <#changeFileExt,string,string>`_ + ## * `searchExtPos proc`_ + ## * `splitFile proc`_ + ## * `extractFilename proc`_ + ## * `lastPathPart proc`_ + ## * `changeFileExt proc`_ runnableExamples: assert addFileExt("foo.bar", "baz") == "foo.bar" assert addFileExt("foo.bar", "") == "foo.bar" @@ -875,15 +873,15 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = ## Returns the home directory of the current user. ## - ## This proc is wrapped by the `expandTilde proc <#expandTilde,string>`_ + ## This proc is wrapped by the `expandTilde proc`_ ## for the convenience of processing paths coming from user configuration files. ## ## See also: - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert getHomeDir() == expandTilde("~") @@ -903,11 +901,11 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", ## returned string: `\\` on Windows and `/` on all other OSs. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ when defined(windows): result = getEnv("APPDATA") else: @@ -927,9 +925,9 @@ proc getCacheDir*(): string = ## * On other platforms: `getEnv("XDG_CACHE_HOME", getEnv("HOME") / ".cache")` ## ## **See also:** - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ + ## * `getHomeDir proc`_ + ## * `getTempDir proc`_ + ## * `getConfigDir proc`_ # follows https://crates.io/crates/platform-dirs when defined(windows): result = getEnv("LOCALAPPDATA") @@ -985,11 +983,11 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", ## **Note:** This proc does not check whether the returned path exists. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `expandTilde proc <#expandTilde,string>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ const tempDirDefault = "/tmp" when defined(tempDir): const tempDir {.strdefine.}: string = tempDirDefault @@ -1015,17 +1013,17 @@ proc getTempDir*(): string {.rtl, extern: "nos$1", proc expandTilde*(path: string): string {. tags: [ReadEnvEffect, ReadIOEffect].} = ## Expands ``~`` or a path starting with ``~/`` to a full path, replacing - ## ``~`` with `getHomeDir() <#getHomeDir>`_ (otherwise returns ``path`` unmodified). + ## ``~`` with `getHomeDir()`_ (otherwise returns ``path`` unmodified). ## ## Windows: this is still supported despite the Windows platform not having this ## convention; also, both ``~/`` and ``~\`` are handled. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ runnableExamples: assert expandTilde("~" / "appname.cfg") == getHomeDir() / "appname.cfg" assert expandTilde("~/foo/bar") == getHomeDir() / "foo/bar" @@ -1089,9 +1087,8 @@ when defined(windows) or defined(posix) or defined(nintendoswitch): proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = ## Quote ``s``, so it can be safely passed to shell. ## - ## When on Windows, it calls `quoteShellWindows proc - ## <#quoteShellWindows,string>`_. Otherwise, calls `quoteShellPosix proc - ## <#quoteShellPosix,string>`_. + ## When on Windows, it calls `quoteShellWindows proc`_. + ## Otherwise, calls `quoteShellPosix proc`_. when defined(windows): result = quoteShellWindows(s) else: @@ -1158,8 +1155,8 @@ proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", ## Directories, device files, named pipes and sockets return false. ## ## See also: - ## * `dirExists proc <#dirExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ + ## * `dirExists proc`_ + ## * `symlinkExists proc`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, filename) @@ -1177,8 +1174,8 @@ proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] ## is returned. Follows symlinks. ## ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `symlinkExists proc <#symlinkExists,string>`_ + ## * `fileExists proc`_ + ## * `symlinkExists proc`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, dir) @@ -1197,8 +1194,8 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", ## regardless of whether the link points to a directory or file. ## ## See also: - ## * `fileExists proc <#fileExists,string>`_ - ## * `dirExists proc <#dirExists,string>`_ + ## * `fileExists proc`_ + ## * `dirExists proc`_ when defined(windows): when useWinUnicode: wrapUnary(a, getFileAttributesW, link) @@ -1228,7 +1225,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; ## in directories listed in the ``PATH`` environment variable. ## ## Returns `""` if the `exe` cannot be found. `exe` - ## is added the `ExeExts <#ExeExts>`_ file extensions if it has none. + ## is added the `ExeExts`_ file extensions if it has none. ## ## If the system supports symlinks it also resolves them until it ## meets the actual file. This behavior can be disabled if desired @@ -1283,9 +1280,9 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1", ## Returns the `file`'s last modification time. ## ## See also: - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1301,9 +1298,9 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeir ## Returns the `file`'s last read or write access time. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getCreationTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1323,9 +1320,9 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1", noWeirdT ## `here `_ for details. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `fileNewer proc <#fileNewer,string,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `fileNewer proc`_ when defined(posix): var res: Stat if stat(file, res) < 0'i32: raiseOSError(osLastError(), file) @@ -1342,9 +1339,9 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1", noWeirdTarget.} = ## modification time is later than `b`'s. ## ## See also: - ## * `getLastModificationTime proc <#getLastModificationTime,string>`_ - ## * `getLastAccessTime proc <#getLastAccessTime,string>`_ - ## * `getCreationTime proc <#getCreationTime,string>`_ + ## * `getLastModificationTime proc`_ + ## * `getLastAccessTime proc`_ + ## * `getCreationTime proc`_ when defined(posix): # If we don't have access to nanosecond resolution, use '>=' when not StatHasNanoseconds: @@ -1362,10 +1359,10 @@ when not defined(nimscript): ## So the path returned by this proc is determined at run time. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `setCurrentDir proc <#setCurrentDir,string>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `setCurrentDir proc`_ ## * `currentSourcePath template `_ ## * `getProjectPath proc `_ when defined(nodejs): @@ -1421,10 +1418,10 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = ## is raised if `newDir` cannot been set. ## ## See also: - ## * `getHomeDir proc <#getHomeDir>`_ - ## * `getConfigDir proc <#getConfigDir>`_ - ## * `getTempDir proc <#getTempDir>`_ - ## * `getCurrentDir proc <#getCurrentDir>`_ + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `getCurrentDir proc`_ when defined(windows): when useWinUnicode: if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: @@ -1441,8 +1438,8 @@ proc absolutePath*(path: string, root = getCurrentDir()): string = ## If `path` is absolute, return it, ignoring `root`. ## ## See also: - ## * `normalizedPath proc <#normalizedPath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ + ## * `normalizedPath proc`_ + ## * `normalizePath proc`_ runnableExamples: assert absolutePath("a") == getCurrentDir() / "a" @@ -1479,9 +1476,9 @@ proc normalizePath*(path: var string) {.rtl, extern: "nos$1", tags: [].} = ## Triple dot is not handled. ## ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizedPath proc <#normalizedPath,string>`_ for outplace version - ## * `normalizeExe proc <#normalizeExe,string>`_ + ## * `absolutePath proc`_ + ## * `normalizedPath proc`_ for outplace version + ## * `normalizeExe proc`_ runnableExamples: when defined(posix): var a = "a///b//..//c///d" @@ -1522,8 +1519,8 @@ proc normalizedPath*(path: string): string {.rtl, extern: "nos$1", tags: [].} = ## Returns a normalized path for the current OS. ## ## See also: - ## * `absolutePath proc <#absolutePath,string>`_ - ## * `normalizePath proc <#normalizePath,string>`_ for the in-place version + ## * `absolutePath proc`_ + ## * `normalizePath proc`_ for the in-place version runnableExamples: when defined(posix): assert normalizedPath("a///b//..//c///d") == "a/c/d" @@ -1561,7 +1558,7 @@ proc sameFile*(path1, path2: string): bool {.rtl, extern: "nos$1", ## sym-linked paths to the same file or directory. ## ## See also: - ## * `sameFileContent proc <#sameFileContent,string,string>`_ + ## * `sameFileContent proc`_ when defined(windows): var success = true var f1 = openHandle(path1) @@ -1598,9 +1595,9 @@ type FilePermission* = enum ## File access permission, modelled after UNIX. ## ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FileInfo object <#FileInfo>`_ + ## * `getFilePermissions`_ + ## * `setFilePermissions`_ + ## * `FileInfo object`_ fpUserExec, ## execute access for the file owner fpUserWrite, ## write access for the file owner fpUserRead, ## read access for the file owner @@ -1620,8 +1617,8 @@ proc getFilePermissions*(filename: string): set[FilePermission] {. ## permission is available in any case. ## ## See also: - ## * `setFilePermissions proc <#setFilePermissions,string,set[FilePermission]>`_ - ## * `FilePermission enum <#FilePermission>`_ + ## * `setFilePermissions proc`_ + ## * `FilePermission enum`_ when defined(posix): var a: Stat if stat(filename, a) < 0'i32: raiseOSError(osLastError(), filename) @@ -1666,8 +1663,8 @@ proc setFilePermissions*(filename: string, permissions: set[FilePermission], ## ``fpUserWrite`` permission. ## ## See also: - ## * `getFilePermissions <#getFilePermissions,string>`_ - ## * `FilePermission enum <#FilePermission>`_ + ## * `getFilePermissions proc`_ + ## * `FilePermission enum`_ when defined(posix): var p = 0.Mode if fpUserRead in permissions: p = p or S_IRUSR.Mode @@ -1744,8 +1741,8 @@ proc createSymlink*(src, dest: string) {.noWeirdTarget.} = ## of symlinks to root users (administrators) or users with developper mode enabled. ## ## See also: - ## * `createHardlink proc <#createHardlink,string,string>`_ - ## * `expandSymlink proc <#expandSymlink,string>`_ + ## * `createHardlink proc`_ + ## * `expandSymlink proc`_ when defined(windows): const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 @@ -1769,7 +1766,7 @@ proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = ## On Windows this is a noop, `symlinkPath` is simply returned. ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ + ## * `createSymlink proc`_ when defined(windows): result = symlinkPath else: @@ -1823,11 +1820,11 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, ## copy the source file's attributes into dest. ## ## On other platforms you need - ## to use `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions <#setFilePermissions,string,set[FilePermission]>`_ + ## to use `getFilePermissions`_ and + ## `setFilePermissions`_ ## procs ## to copy them by hand (or use the convenience `copyFileWithPermissions - ## proc <#copyFileWithPermissions,string,string>`_), + ## proc`_), ## otherwise `dest` will inherit the default permissions of a newly ## created file for the user. ## @@ -1838,12 +1835,12 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, ## `-d:nimLegacyCopyFile` is used. ## ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `CopyFlag enum`_ + ## * `copyDir proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ doAssert card(copyFlagSymlink * options) == 1, "There should be exactly " & "one cfSymlink* in options" @@ -1908,8 +1905,8 @@ proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}) ## ignored on Windows: symlinks are skipped. ## ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyDir,string,string>`_ + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ if dir.len == 0: # treating "" as "." is error prone raise newException(ValueError, "dest is empty") copyFile(source, dir / source.lastPathPart, options) @@ -1942,10 +1939,10 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE ## On Windows, ignores the read-only attribute. ## ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ result = true when defined(windows): when useWinUnicode: @@ -1974,11 +1971,11 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n ## On Windows, ignores the read-only attribute. ## ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `removeDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `tryRemoveFile proc`_ + ## * `moveFile proc`_ if not tryRemoveFile(file): raiseOSError(osLastError(), file) @@ -2022,11 +2019,11 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", ## Can be used to `rename files`:idx:. ## ## See also: - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ + ## * `moveDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeFile proc`_ + ## * `tryRemoveFile proc`_ if not tryMoveFSObject(source, dest, isDir = false): when defined(windows): @@ -2135,10 +2132,10 @@ iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdT ## notation is supported. ## ## See also: - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ runnableExamples: import std/sequtils let paths = toSeq(walkPattern("lib/pure/*")) # works on Windows too @@ -2154,10 +2151,10 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTar ## notation is supported. ## ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ runnableExamples: import std/sequtils assert "lib/pure/os.nim".unixToNativePath in toSeq(walkFiles("lib/pure/*.nim")) # works on Windows too @@ -2171,10 +2168,10 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarg ## notation is supported. ## ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDir iterator`_ + ## * `walkDirRec iterator`_ runnableExamples: import std/sequtils let paths = toSeq(walkDirs("lib/pure/*")) # works on Windows too @@ -2235,15 +2232,15 @@ type PathComponent* = enum ## Enumeration specifying a path component. ## ## See also: - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - ## * `FileInfo object <#FileInfo>`_ + ## * `walkDirRec iterator`_ + ## * `FileInfo object`_ pcFile, ## path refers to a file pcLinkToFile, ## path refers to a symbolic link to a file pcDir, ## path refers to a directory pcLinkToDir ## path refers to a symbolic link to a directory proc getCurrentCompilerExe*(): string {.compileTime.} = discard - ## This is `getAppFilename() <#getAppFilename>`_ at compile time. + ## This is `getAppFilename()`_ at compile time. ## ## Can be used to retrieve the currently executing ## Nim compiler from a Nim or nimscript program, or the nimble binary @@ -2291,10 +2288,10 @@ iterator walkDir*(dir: string; relative = false, checkDir = false): assert collect(for k in walkDir("dirA"): k.path).join(" ") == "dirA/dirB dirA/dirC dirA/fileA2.txt dirA/fileA1.txt" ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDirRec iterator`_ when nimvm: for k, v in items(staticWalkDir(dir, relative)): @@ -2401,10 +2398,10 @@ iterator walkDirRec*(dir: string, ## ## ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDir iterator <#walkDir.i,string>`_ + ## * `walkPattern iterator`_ + ## * `walkFiles iterator`_ + ## * `walkDirs iterator`_ + ## * `walkDir iterator`_ var stack = @[""] var checkDir = checkDir @@ -2444,13 +2441,13 @@ proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ ## existed in the first place, unless `checkDir` = true. ## ## See also: - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ for kind, path in walkDir(dir, checkDir = checkDir): case kind of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) @@ -2512,11 +2509,11 @@ proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", ## Returns `true` if the directory already exists, and `false` otherwise. ## ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `removeDir proc`_ + ## * `createDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ result = not rawCreateDir(dir) if result: # path already exists - need to check that it is indeed a directory @@ -2534,11 +2531,11 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", ## most usages this does not indicate an error. ## ## See also: - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `moveDir proc`_ var omitNext = false when doslikeFileSystem: omitNext = isAbsolute(dir) @@ -2568,17 +2565,17 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", ## ## On other platforms created files and directories will inherit the ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## Use `copyDirWithPermissions proc`_ ## to preserve attributes recursively on these platforms. ## ## See also: - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `copyDirWithPermissions proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ + ## * `moveDir proc`_ createDir(dest) for kind, path in walkDir(source): var noSource = splitPath(path).tail @@ -2596,12 +2593,12 @@ proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWei ## If this fails, `OSError` is raised. ## ## See also: - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ + ## * `moveFile proc`_ + ## * `copyDir proc`_ + ## * `copyDirWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ if not tryMoveFSObject(source, dest, isDir = true): # Fallback to copy & del copyDir(source, dest) @@ -2615,7 +2612,7 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## root users (administrators). ## ## See also: - ## * `createSymlink proc <#createSymlink,string,string>`_ + ## * `createSymlink proc`_ when defined(windows): when useWinUnicode: var wSrc = newWideCString(src) @@ -2638,13 +2635,12 @@ proc copyFileWithPermissions*(source, dest: string, ## if `source` is a symlink, copies the file symlink points to. `options` is ## ignored on Windows: symlinks are skipped. ## - ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, - ## `getFilePermissions <#getFilePermissions,string>`_ and - ## `setFilePermissions<#setFilePermissions,string,set[FilePermission]>`_ + ## This is a wrapper proc around `copyFile`_, + ## `getFilePermissions`_ and `setFilePermissions`_ ## procs on non-Windows platforms. ## - ## On Windows this proc is just a wrapper for `copyFile proc - ## <#copyFile,string,string>`_ since that proc already copies attributes. + ## On Windows this proc is just a wrapper for `copyFile proc`_ since + ## that proc already copies attributes. ## ## On non-Windows systems permissions are copied after the file itself has ## been copied, which won't happen atomically and could lead to a race @@ -2653,13 +2649,13 @@ proc copyFileWithPermissions*(source, dest: string, ## `OSError`. ## ## See also: - ## * `CopyFlag enum <#CopyFlag>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `CopyFlag enum`_ + ## * `copyFile proc`_ + ## * `copyDir proc`_ + ## * `tryRemoveFile proc`_ + ## * `removeFile proc`_ + ## * `moveFile proc`_ + ## * `copyDirWithPermissions proc`_ copyFile(source, dest, options) when not defined(windows): try: @@ -2678,13 +2674,12 @@ proc copyDirWithPermissions*(source, dest: string, ## On non-Windows OSes, symlinks are copied as symlinks. On Windows, symlinks ## are skipped. ## - ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir - ## <#copyDir,string,string>`_ and `copyFileWithPermissions - ## <#copyFileWithPermissions,string,string>`_ procs + ## If this fails, `OSError` is raised. This is a wrapper proc around + ## `copyDir`_ and `copyFileWithPermissions`_ procs ## on non-Windows platforms. ## - ## On Windows this proc is just a wrapper for `copyDir proc - ## <#copyDir,string,string>`_ since that proc already copies attributes. + ## On Windows this proc is just a wrapper for `copyDir proc`_ since + ## that proc already copies attributes. ## ## On non-Windows systems permissions are copied after the file or directory ## itself has been copied, which won't happen atomically and could lead to a @@ -2693,13 +2688,13 @@ proc copyDirWithPermissions*(source, dest: string, ## `OSError`. ## ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ - ## * `removeDir proc <#removeDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `createDir proc <#createDir,string>`_ + ## * `copyDir proc`_ + ## * `copyFile proc`_ + ## * `copyFileWithPermissions proc`_ + ## * `removeDir proc`_ + ## * `moveDir proc`_ + ## * `existsOrCreateDir proc`_ + ## * `createDir proc`_ createDir(dest) when not defined(windows): try: @@ -2769,9 +2764,9 @@ proc parseCmdLine*(c: string): seq[string] {. ## ## See also: ## * `parseopt module `_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ result = @[] var i = 0 @@ -2838,9 +2833,8 @@ when defined(nimdoc): ## ## Unlike `argc`:idx: in C, if your binary was called without parameters this ## will return zero. - ## You can query each individual parameter with `paramStr proc <#paramStr,int>`_ - ## or retrieve all of them in one go with `commandLineParams proc - ## <#commandLineParams>`_. + ## You can query each individual parameter with `paramStr proc`_ + ## or retrieve all of them in one go with `commandLineParams proc`_. ## ## **Availability**: When generating a dynamic library (see `--app:lib`) on ## Posix this proc is not defined. @@ -2848,9 +2842,9 @@ when defined(nimdoc): ## ## See also: ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `commandLineParams proc <#commandLineParams>`_ + ## * `parseCmdLine proc`_ + ## * `paramStr proc`_ + ## * `commandLineParams proc`_ ## ## **Examples:** ## @@ -2865,13 +2859,13 @@ when defined(nimdoc): ## ## `i` should be in the range `1..paramCount()`, the `IndexDefect` ## exception will be raised for invalid values. Instead of iterating - ## over `paramCount() <#paramCount>`_ with this proc you can - ## call the convenience `commandLineParams() <#commandLineParams>`_. + ## over `paramCount()`_ with this proc you can + ## call the convenience `commandLineParams()`_. ## ## Similarly to `argv`:idx: in C, ## it is possible to call `paramStr(0)` but this will return OS specific ## contents (usually the name of the invoked executable). You should avoid - ## this and call `getAppFilename() <#getAppFilename>`_ instead. + ## this and call `getAppFilename()`_ instead. ## ## **Availability**: When generating a dynamic library (see `--app:lib`) on ## Posix this proc is not defined. @@ -2879,10 +2873,10 @@ when defined(nimdoc): ## ## See also: ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `commandLineParams proc <#commandLineParams>`_ - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `commandLineParams proc`_ + ## * `getAppFilename proc`_ ## ## **Examples:** ## @@ -2971,7 +2965,7 @@ when declared(paramCount) or defined(nimdoc): ## Convenience proc which returns the command line parameters. ## ## This returns **only** the parameters. If you want to get the application - ## executable filename, call `getAppFilename() <#getAppFilename>`_. + ## executable filename, call `getAppFilename()`_. ## ## **Availability**: On Posix there is no portable way to get the command ## line from a DLL and thus the proc isn't defined in this environment. You @@ -2980,10 +2974,10 @@ when declared(paramCount) or defined(nimdoc): ## ## See also: ## * `parseopt module `_ - ## * `parseCmdLine proc <#parseCmdLine,string>`_ - ## * `paramCount proc <#paramCount>`_ - ## * `paramStr proc <#paramStr,int>`_ - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `parseCmdLine proc`_ + ## * `paramCount proc`_ + ## * `paramStr proc`_ + ## * `getAppFilename proc`_ ## ## **Examples:** ## @@ -3133,8 +3127,8 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW ## This proc will resolve symlinks. ## ## See also: - ## * `getAppDir proc <#getAppDir>`_ - ## * `getCurrentCompilerExe proc <#getCurrentCompilerExe>`_ + ## * `getAppDir proc`_ + ## * `getCurrentCompilerExe proc`_ # Linux: /proc//exe # Solaris: @@ -3198,7 +3192,7 @@ proc getAppDir*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noWeirdT ## Returns the directory of the application's executable. ## ## See also: - ## * `getAppFilename proc <#getAppFilename>`_ + ## * `getAppFilename proc`_ result = splitFile(getAppFilename()).dir proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect], noWeirdTarget.} = @@ -3242,9 +3236,9 @@ type ## Contains information associated with a file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ id*: tuple[device: DeviceId, file: FileId] ## Device and file id. kind*: PathComponent ## Kind of file object - directory, symlink, etc. size*: BiggestInt ## Size of file. @@ -3334,8 +3328,8 @@ proc getFileInfo*(handle: FileHandle): FileInfo {.noWeirdTarget.} = ## is invalid, `OSError` is raised. ## ## See also: - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(file) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ # Done: ID, Kind, Size, Permissions, Link Count when defined(windows): @@ -3356,8 +3350,8 @@ proc getFileInfo*(file: File): FileInfo {.noWeirdTarget.} = ## Retrieves file information for the file object. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(path) proc <#getFileInfo,string>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(path, followSymlink) proc`_ if file.isNil: raise newException(IOError, "File is nil") result = getFileInfo(file.getFileHandle()) @@ -3366,7 +3360,7 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## Retrieves file information for the file object pointed to by `path`. ## ## Due to intrinsic differences between operating systems, the information - ## contained by the returned `FileInfo object <#FileInfo>`_ will be slightly + ## contained by the returned `FileInfo object`_ will be slightly ## different across platforms, and in some cases, incomplete or inaccurate. ## ## When `followSymlink` is true (default), symlinks are followed and the @@ -3378,8 +3372,8 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget. ## file information, `OSError` is raised. ## ## See also: - ## * `getFileInfo(handle) proc <#getFileInfo,FileHandle>`_ - ## * `getFileInfo(file) proc <#getFileInfo,File>`_ + ## * `getFileInfo(handle) proc`_ + ## * `getFileInfo(file) proc`_ when defined(windows): var handle = openHandle(path, followSymlink) @@ -3406,7 +3400,7 @@ proc sameFileContent*(path1, path2: string): bool {.rtl, extern: "nos$1", ## binary content. ## ## See also: - ## * `sameFile proc <#sameFile,string,string>`_ + ## * `sameFile proc`_ var a, b: File if not open(a, path1): return false diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim index 5bcdf32d5..9d440a1d6 100644 --- a/tests/stdlib/tdochelpers.nim +++ b/tests/stdlib/tdochelpers.nim @@ -153,3 +153,9 @@ suite "Integration with Nim": outType: "T") check(input1.toLangSymbol == expected) check(input2.toLangSymbol == expected) + + test "type of type": + check ("`CopyFlag enum`_".rstParseTest.toLangSymbol == + LangSymbol(symKind: "type", + symTypeKind: "enum", + name: "Copyflag")) From 83a9c3ba31d180cd5e31026d8b7603bf7adea18c Mon Sep 17 00:00:00 2001 From: Zoom Date: Mon, 8 Nov 2021 18:47:20 +0000 Subject: [PATCH 0524/1654] Update readme.md: matrix links (#19109) Added links to the main #nim-lang room on Matrix and to the Nim matrix space. Added Matrix badge. --- readme.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readme.md b/readme.md index af07c2a7b..7d45556db 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( [![Join the IRC chat][badge-nim-irc]][nim-irc] [![Join the Discord server][badge-nim-discord]][nim-discord] [![Join the Gitter chat][badge-nim-gitter]][nim-gitter] +[![Join the Matrix room](https://img.shields.io/matrix/nim-lang:matrix.org?color=blue&style=flat&logo=matrix)](https://matrix.to/#/#nim-lang:matrix.org) [![Get help][badge-nim-forum-gethelp]][nim-forum] [![View Nim posts on Stack Overflow][badge-nim-stackoverflow]][nim-stackoverflow-newest] [![Follow @nim_lang on Twitter][badge-nim-twitter]][nim-twitter] @@ -22,6 +23,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( channels there are bridged to IRC. * [Gitter][nim-gitter] - an additional place to discuss Nim in real-time. There is a bridge between Gitter and the IRC channel. +* [Matrix][nim-matrix] - the main room to discuss Nim in real-time. [Matrix space][nim-matrix-space] contains a list of rooms, most of them are bridged to IRC. * [Telegram][nim-telegram] - an additional place to discuss Nim in real-time. There is the official Telegram channel. Not bridged to IRC. * [Stack Overflow][nim-stackoverflow] - a popular Q/A site for programming related @@ -213,6 +215,8 @@ Copyright © 2006-2021 Andreas Rumpf, all rights reserved. [nim-stackoverflow-newest]: https://stackoverflow.com/questions/tagged/nim-lang?sort=newest&pageSize=15 [nim-discord]: https://discord.gg/nim [nim-gitter]: https://gitter.im/nim-lang/Nim +[nim-matrix]: https://matrix.to/#/#nim-lang:matrix.org +[nim-matrix-space]: https://matrix.to/#/#nim:envs.net [nim-telegram]: https://t.me/nim_lang [nim-bountysource]: https://www.bountysource.com/teams/nim [nim-bitcoin]: https://blockchain.info/address/1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ From b7c66ce860cfbadfa8ed60784fc387a0818e2e5d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Nov 2021 12:43:16 +0100 Subject: [PATCH 0525/1654] fixes #19013 [backport:1.6] (#19111) * fixes #19013 [backport:1.6] * added test case --- compiler/ast.nim | 8 ++++++++ compiler/isolation_check.nim | 17 ++++++++++++++++- compiler/varpartitions.nim | 7 ------- tests/isolate/tisolate2.nim | 22 ++++++++++++++++++++++ 4 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 tests/isolate/tisolate2.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index abd2ff01e..9086860b9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -2101,3 +2101,11 @@ proc skipAddr*(n: PNode): PNode {.inline.} = proc isNewStyleConcept*(n: PNode): bool {.inline.} = assert n.kind == nkTypeClassTy result = n[0].kind == nkEmpty + +const + nodesToIgnoreSet* = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit, + nkTypeSection, nkProcDef, nkConverterDef, + nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, + nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, + nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, + nkTypeOfExpr, nkMixinStmt, nkBindStmt} diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 777e7f6ce..a8c5a3651 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -77,6 +77,17 @@ proc canAlias*(arg, ret: PType): bool = var marker = initIntSet() result = canAlias(arg, ret, marker) +proc containsVariable(n: PNode): bool = + case n.kind + of nodesToIgnoreSet: + result = false + of nkSym: + result = n.sym.kind in {skForVar, skParam, skVar, skLet, skConst, skResult, skTemp} + else: + for ch in n: + if containsVariable(ch): return true + result = false + proc checkIsolate*(n: PNode): bool = if types.containsTyRef(n.typ): # XXX Maybe require that 'n.typ' is acyclic. This is not much @@ -96,7 +107,11 @@ proc checkIsolate*(n: PNode): bool = else: let argType = n[i].typ if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType): - if argType.canAlias(n.typ): + if argType.canAlias(n.typ) or containsVariable(n[i]): + # bug #19013: Alias information is not enough, we need to check for potential + # "overlaps". I claim the problem can only happen by reading again from a location + # that materialized which is only possible if a variable that contains a `ref` + # is involved. return false result = true of nkIfStmt, nkIfExpr: diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 709d00fa0..721de900e 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -647,13 +647,6 @@ proc deps(c: var Partitions; dest, src: PNode) = when explainCursors: echo "D not a cursor ", d.sym, " reassignedTo ", c.s[srcid].reassignedTo c.s[vid].flags.incl preventCursor -const - nodesToIgnoreSet = {nkNone..pred(nkSym), succ(nkSym)..nkNilLit, - nkTypeSection, nkProcDef, nkConverterDef, - nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo, - nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt, - nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, - nkTypeOfExpr, nkMixinStmt, nkBindStmt} proc potentialMutationViaArg(c: var Partitions; n: PNode; callee: PType) = if constParameters in c.goals and tfNoSideEffect in callee.flags: diff --git a/tests/isolate/tisolate2.nim b/tests/isolate/tisolate2.nim new file mode 100644 index 000000000..9bf92d82e --- /dev/null +++ b/tests/isolate/tisolate2.nim @@ -0,0 +1,22 @@ +discard """ + errormsg: "expression cannot be isolated: a_to_b(a)" + line: 22 +""" + +# bug #19013 +import std/isolation + +type Z = ref object + i: int + +type A = object + z: Z + +type B = object + z: Z + +func a_to_b(a: A): B = + result = B(z: a.z) + +let a = A(z: Z(i: 3)) +let b = isolate(a_to_b(a)) From 6ff61766daec10d4cf78488b38bbe05d034e3a1c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Nov 2021 18:59:59 +0100 Subject: [PATCH 0526/1654] fixes #19011 [backport:1.6] (#19114) --- compiler/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 5e9a7424a..aedf62d54 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1877,7 +1877,7 @@ proc parseEnum(p: var Parser): PNode = var symPragma = a var pragma: PNode - if p.tok.tokType == tkCurlyDotLe: + if (p.tok.indent < 0 or p.tok.indent >= p.currInd) and p.tok.tokType == tkCurlyDotLe: pragma = optPragmas(p) symPragma = newNodeP(nkPragmaExpr, p) symPragma.add(a) From b0f4a9b845e4d483547f1a3818e3caf53cd90964 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:00:33 +0100 Subject: [PATCH 0527/1654] Add deprecation pragmas in lib/deprecated/pure (#19113) --- lib/deprecated/pure/LockFreeHash.nim | 6 ++++-- lib/deprecated/pure/events.nim | 2 ++ lib/deprecated/pure/ospaths.nim | 5 +++-- lib/deprecated/pure/securehash.nim | 5 ++--- lib/deprecated/pure/sharedstrings.nim | 2 ++ 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/deprecated/pure/LockFreeHash.nim b/lib/deprecated/pure/LockFreeHash.nim index 97e1ef6ad..69e12226f 100644 --- a/lib/deprecated/pure/LockFreeHash.nim +++ b/lib/deprecated/pure/LockFreeHash.nim @@ -1,10 +1,12 @@ #nim c -t:-march=i686 --cpu:amd64 --threads:on -d:release lockfreehash.nim -import math, hashes - #------------------------------------------------------------------------------ ## Memory Utility Functions +{.deprecated.} + +import math, hashes + proc newHeap*[T](): ptr T = result = cast[ptr T](alloc0(sizeof(T))) diff --git a/lib/deprecated/pure/events.nim b/lib/deprecated/pure/events.nim index 218cb45d5..85fc349c2 100644 --- a/lib/deprecated/pure/events.nim +++ b/lib/deprecated/pure/events.nim @@ -36,6 +36,8 @@ ## myobj.SomeEvent.addHandler(handleevent) ## ee.emit(myobj.SomeEvent, genericargs) +{.deprecated.} + type EventArgs* = object of RootObj ## Base object for event arguments that are passed to callback functions. EventHandler* = tuple[name: string, handlers: seq[proc(e: EventArgs) {.closure.}]] ## An eventhandler for an event. diff --git a/lib/deprecated/pure/ospaths.nim b/lib/deprecated/pure/ospaths.nim index 6c7fe4fb3..e253a3e5a 100644 --- a/lib/deprecated/pure/ospaths.nim +++ b/lib/deprecated/pure/ospaths.nim @@ -7,8 +7,9 @@ # distribution, for details about the copyright. # -## This module is deprecated, `import os` instead. -{.deprecated: "import os.nim instead".} +## This module is deprecated, `import std/os` instead. + +{.deprecated: "use `std/os` instead".} import os export ReadEnvEffect, WriteEnvEffect, ReadDirEffect, WriteDirEffect, OSErrorCode, diff --git a/lib/deprecated/pure/securehash.nim b/lib/deprecated/pure/securehash.nim index 2f4530d88..ea7930f81 100644 --- a/lib/deprecated/pure/securehash.nim +++ b/lib/deprecated/pure/securehash.nim @@ -1,6 +1,5 @@ - - ## This module is a deprecated alias for the `sha1` module. -{.deprecated.} + +{.deprecated: "use `std/sha1` instead".} include "../std/sha1" diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim index 4d4ae151b..99963e1f2 100644 --- a/lib/deprecated/pure/sharedstrings.nim +++ b/lib/deprecated/pure/sharedstrings.nim @@ -9,6 +9,8 @@ ## Shared string support for Nim. +{.deprecated.} + type UncheckedCharArray = UncheckedArray[char] From 46fb85542518c8594c24d168b025cc8a64b8fb3d Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Tue, 9 Nov 2021 19:00:43 +0100 Subject: [PATCH 0528/1654] Deprecate `std/sharedlist` and `std/sharedtables` (#19112) --- changelogs/changelog.md | 2 ++ changelogs/changelog_1_4_0.md | 2 +- doc/lib.rst | 6 ------ lib/pure/collections/lists.nim | 1 - lib/pure/collections/sharedlist.nim | 2 ++ lib/pure/collections/sharedtables.nim | 2 ++ lib/pure/collections/tables.nim | 1 - 7 files changed, 7 insertions(+), 9 deletions(-) diff --git a/changelogs/changelog.md b/changelogs/changelog.md index 475f568ec..d793c4313 100644 --- a/changelogs/changelog.md +++ b/changelogs/changelog.md @@ -4,6 +4,8 @@ ## Standard library additions and changes +- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433). + ### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. diff --git a/changelogs/changelog_1_4_0.md b/changelogs/changelog_1_4_0.md index 091048ad1..77f66ffde 100644 --- a/changelogs/changelog_1_4_0.md +++ b/changelogs/changelog_1_4_0.md @@ -282,7 +282,7 @@ - Removed the deprecated `asyncdispatch.newAsyncNativeSocket`. - Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`. -- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour. +- Removed `sharedlist.initSharedList`, was deprecated and produces undefined behaviour. - There is a new experimental feature called "strictFuncs" which makes the definition of `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs) diff --git a/doc/lib.rst b/doc/lib.rst index 54728d681..881d857d3 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -134,12 +134,6 @@ Collections * `sets `_ Nim hash set support. -* `sharedlist `_ - Nim shared linked list support. Contains a shared singly-linked list. - -* `sharedtables `_ - Nim shared hash table support. Contains shared tables. - * `tables `_ Nim hash table support. Contains tables, ordered tables, and count tables. diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index af53bb68b..8b255fb60 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -56,7 +56,6 @@ runnableExamples: ## # See also ## * `deques module `_ for double-ended queues -## * `sharedlist module `_ for shared singly-linked lists import std/private/since diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim index 73e147e05..e61883220 100644 --- a/lib/pure/collections/sharedlist.nim +++ b/lib/pure/collections/sharedlist.nim @@ -11,6 +11,8 @@ ## ## Unstable API. +{.deprecated.} + {.push stackTrace: off.} import diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index a85674df9..816ab49ab 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -14,6 +14,8 @@ ## ## Unstable API. +{.deprecated.} + import hashes, math, locks diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 2695f3693..00f71ef1d 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -191,7 +191,6 @@ runnableExamples: ## ## * `json module`_ for table-like structure which allows ## heterogeneous members -## * `sharedtables module`_ for shared hash table support ## * `strtabs module`_ for efficient hash tables ## mapping from strings to strings ## * `hashes module`_ for helper functions for hashing From 997ccc588955dab5cdbfd4c0fc067d91ccc1a7c8 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 9 Nov 2021 21:01:47 +0300 Subject: [PATCH 0529/1654] fix nimindexterm in rst2tex/doc2tex [backport] (#19106) * fix nimindexterm (rst2tex/doc2tex) [backport] * Add support for indexing in rst --- config/nimdoc.tex.cfg | 12 +++++++++++- lib/packages/docutils/rstgen.nim | 2 +- tools/kochdocs.nim | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index d7ab652eb..457f452dd 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -63,6 +63,11 @@ doc.file = """ % % Compile it by: xelatex (up to 3 times to get labels generated) % ------- +% For example: +% xelatex file.tex +% xelatex file.tex +% makeindex file +% xelatex file.tex % \documentclass[a4paper,11pt]{article} \usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry} @@ -105,7 +110,9 @@ doc.file = """ \usepackage{parskip} % paragraphs delimited by vertical space, no indent \usepackage{graphicx} -\newcommand{\nimindexterm}[2]{#2\label{#1}} +\usepackage{makeidx} +\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}} +\makeindex \usepackage{dingbat} % for \carriagereturn, etc \usepackage{fvextra} % for code blocks (works better than original fancyvrb) @@ -249,5 +256,8 @@ doc.file = """ \maketitle $content + +\printindex + \end{document} """ diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index a556aa2e3..4cbd05379 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -408,7 +408,7 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) - dispA(d.target, result, "$2", "\\nimindexterm{$2}{$1}", + dispA(d.target, result, "$2", "\\nimindexterm{$1}{$2}", [id, term]) type diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 42e648fb4..70ee5ad07 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -301,6 +301,12 @@ proc nim2pdf(src: string, dst: string, nimArgs: string) = # `>` should work on windows, if not, we can use `execCmdEx` let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) moveFile(texFile.changeFileExt("pdf"), dst) proc buildPdfDoc*(nimArgs, destPath: string) = From 15157d06c3e0bba0c372a094917a13b916e15b54 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 9 Nov 2021 22:48:04 +0100 Subject: [PATCH 0530/1654] fixes #14470 [backport:1.2] (#19115) --- compiler/ccgtypes.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 23e1a5867..e7e099659 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -220,7 +220,8 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil: result = true + if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3: + result = true else: case mapType(conf, rettype, skResult) of ctArray: From a78ee8ae8453aabde04fa0103e02378efe870827 Mon Sep 17 00:00:00 2001 From: treeform Date: Tue, 9 Nov 2021 23:52:34 -0800 Subject: [PATCH 0531/1654] Call {.cursor.} a pragma. (#19116) * Call {.cursor.} a pragma. Its hard to find .curser annotation while googling because all other things like it are called pragmas. See https://nim-lang.org/docs/manual.html#pragmas Also the . in front of the name makes it hard to find and search for. Can we just call it cursor pragma? * Small fix for comment. --- doc/destructors.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/destructors.rst b/doc/destructors.rst index d6f240dee..83a50230b 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -555,14 +555,14 @@ for expressions of type `lent T` or of type `var T`. echo t[0] # accessor does not copy the element! -The .cursor annotation -====================== +The cursor pragma +================= Under the `--gc:arc|orc`:option: modes Nim's `ref` type is implemented via the same runtime "hooks" and thus via reference counting. This means that cyclic structures cannot be freed immediately (`--gc:orc`:option: ships with a cycle collector). -With the `.cursor` annotation one can break up cycles declaratively: +With the `cursor` pragma one can break up cycles declaratively: .. code-block:: nim @@ -575,7 +575,7 @@ But please notice that this is not C++'s weak_ptr, it means the right field is n involved in the reference counting, it is a raw pointer without runtime checks. Automatic reference counting also has the disadvantage that it introduces overhead -when iterating over linked structures. The `.cursor` annotation can also be used +when iterating over linked structures. The `cursor` pragma can also be used to avoid this overhead: .. code-block:: nim @@ -586,10 +586,10 @@ to avoid this overhead: it = it.next -In fact, `.cursor` more generally prevents object construction/destruction pairs +In fact, `cursor` more generally prevents object construction/destruction pairs and so can also be useful in other contexts. The alternative solution would be to use raw pointers (`ptr`) instead which is more cumbersome and also more dangerous -for Nim's evolution: Later on, the compiler can try to prove `.cursor` annotations +for Nim's evolution: Later on, the compiler can try to prove `cursor` pragmas to be safe, but for `ptr` the compiler has to remain silent about possible problems. @@ -597,7 +597,7 @@ problems. Cursor inference / copy elision =============================== -The current implementation also performs `.cursor` inference. Cursor inference is +The current implementation also performs `cursor` inference. Cursor inference is a form of copy elision. To see how and when we can do that, think about this question: In `dest = src` when @@ -612,7 +612,7 @@ indirections: .. code-block:: nim proc main(tab: Table[string, string]) = - let v = tab["key"] # inferred as .cursor because 'tab' is not mutated. + let v = tab["key"] # inferred as cursor because 'tab' is not mutated. # no copy into 'v', no destruction of 'v'. use(v) useItAgain(v) From 77b696c2c92b5f478526290c5e184a4c41060f7b Mon Sep 17 00:00:00 2001 From: Ryan Oldenburg Date: Wed, 10 Nov 2021 23:48:22 -0600 Subject: [PATCH 0532/1654] Remove tlsEmulation enabled from Windows + GCC config (#19119) [backport:1.6] This flag has a very significant performance impact on programs compiled with --threads:on. It is also apparently not needed anymore for standard circumstances. Can we remove the config? See https://github.com/nim-lang/Nim/issues/18146#issuecomment-876802676 for discussion and perf impact. [backport:1.6] --- config/nim.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 3b964d124..86c6e2141 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -154,9 +154,6 @@ nimblepath="$home/.nimble/pkgs/" # Configuration for the GNU C/C++ compiler: @if windows: #gcc.path = r"$nim\dist\mingw\bin" - @if gcc or tcc: - tlsEmulation:on - @end @end gcc.maxerrorsimpl = "-fmax-errors=3" From 036d894e6a13cf9792c3250e34252a225c73e468 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Thu, 11 Nov 2021 07:41:21 +0000 Subject: [PATCH 0533/1654] Add security tip for setCookie (#19117) * Add security tip for setCookie * Update lib/pure/cookies.nim Co-authored-by: Dominik Picheta * Update lib/pure/cookies.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Andreas Rumpf Co-authored-by: Dominik Picheta Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/cookies.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 25d701eb4..132f64637 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -50,6 +50,9 @@ proc setCookie*(key, value: string, domain = "", path = "", maxAge = none(int), sameSite = SameSite.Default): string = ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` + ## + + ## .. tip: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value From 7313b70a6af3fec6f367dd72bdd4f7651bf60d21 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 11 Nov 2021 15:53:27 +0800 Subject: [PATCH 0534/1654] correct cookie docs (#19122) --- lib/pure/cookies.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 132f64637..7bf34f75b 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -51,8 +51,7 @@ proc setCookie*(key, value: string, domain = "", path = "", ## Creates a command in the format of ## `Set-Cookie: key=value; Domain=...; ...` ## - - ## .. tip: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. + ## .. tip:: Cookies can be vulnerable. Consider setting `secure=true`, `httpOnly=true` and `sameSite=Strict`. result = "" if not noName: result.add("Set-Cookie: ") result.add key & "=" & value From cde83d90aa888ccc83bd78e6b6fb54d0ab0b2c26 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 11 Nov 2021 19:07:08 +0100 Subject: [PATCH 0535/1654] refactoring: orc can use getThreadId() (#19123) * refactoring: orc can use getThreadId() * progress --- lib/system/orc.nim | 7 +- lib/system/threadids.nim | 103 ++++++++++++++++++++++++++++++ lib/system/threadlocalstorage.nim | 3 - lib/system/threads.nim | 93 +-------------------------- 4 files changed, 110 insertions(+), 96 deletions(-) create mode 100644 lib/system/threadids.nim diff --git a/lib/system/orc.nim b/lib/system/orc.nim index 6273e1a21..f9f5cd81f 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -79,10 +79,12 @@ proc trace(s: Cell; desc: PNimTypeV2; j: var GcEnv) {.inline.} = var p = s +! sizeof(RefHeader) cast[TraceProc](desc.traceImpl)(p, addr(j)) +include threadids + when logOrc: proc writeCell(msg: cstring; s: Cell; desc: PNimTypeV2) = - cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld\n", - msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color) + cfprintf(cstderr, "%s %s %ld root index: %ld; RC: %ld; color: %ld; thread: %ld\n", + msg, desc.name, s.refId, s.rootIdx, s.rc shr rcShift, s.color, getThreadId()) proc free(s: Cell; desc: PNimTypeV2) {.inline.} = when traceCollector: @@ -419,6 +421,7 @@ proc registerCycle(s: Cell; desc: PNimTypeV2) = proc GC_runOrc* = ## Forces a cycle collection pass. collectCycles() + orcAssert roots.len == 0, "roots not empty!" proc GC_enableOrc*() = ## Enables the cycle collector subsystem of `--gc:orc`. This is a `--gc:orc` diff --git a/lib/system/threadids.nim b/lib/system/threadids.nim new file mode 100644 index 000000000..3a6eadcbb --- /dev/null +++ b/lib/system/threadids.nim @@ -0,0 +1,103 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2020 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# we need to cache current threadId to not perform syscall all the time +var threadId {.threadvar.}: int + +when defined(windows): + proc getCurrentThreadId(): int32 {. + stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getCurrentThreadId()) + result = threadId + +elif defined(linux): + proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "".} + when defined(amd64): + const NR_gettid = clong(186) + else: + var NR_gettid {.importc: "__NR_gettid", header: "".}: clong + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(NR_gettid)) + result = threadId + +elif defined(dragonfly): + proc lwp_gettid(): int32 {.importc, header: "unistd.h".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_gettid()) + result = threadId + +elif defined(openbsd): + proc getthrid(): int32 {.importc: "getthrid", header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(getthrid()) + result = threadId + +elif defined(netbsd): + proc lwp_self(): int32 {.importc: "_lwp_self", header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(lwp_self()) + result = threadId + +elif defined(freebsd): + proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "".} + var SYS_thr_self {.importc:"SYS_thr_self", header:"".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + var tid = 0.cint + if threadId == 0: + discard syscall(SYS_thr_self, addr tid) + threadId = tid + result = threadId + +elif defined(macosx): + proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "".} + var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"".}: cint + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(syscall(SYS_thread_selfid)) + result = threadId + +elif defined(solaris): + type thread_t {.importc: "thread_t", header: "".} = distinct int + proc thr_self(): thread_t {.importc, header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(thr_self()) + result = threadId + +elif defined(haiku): + type thr_id {.importc: "thread_id", header: "".} = distinct int32 + proc find_thread(name: cstring): thr_id {.importc, header: "".} + + proc getThreadId*(): int = + ## Gets the ID of the currently running thread. + if threadId == 0: + threadId = int(find_thread(nil)) + result = threadId diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index 30962ef77..e772f270c 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -29,9 +29,6 @@ when defined(windows): proc terminateThread(hThread: SysThread, dwExitCode: int32): int32 {. stdcall, dynlib: "kernel32", importc: "TerminateThread".} - proc getCurrentThreadId(): int32 {. - stdcall, dynlib: "kernel32", importc: "GetCurrentThreadId".} - type ThreadVarSlot = distinct int32 diff --git a/lib/system/threads.nim b/lib/system/threads.nim index a9006b0c5..a8a413bcb 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -343,94 +343,5 @@ else: proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = createThread[void](t, tp) -# we need to cache current threadId to not perform syscall all the time -var threadId {.threadvar.}: int - -when defined(windows): - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(getCurrentThreadId()) - result = threadId - -elif defined(linux): - proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "".} - when defined(amd64): - const NR_gettid = clong(186) - else: - var NR_gettid {.importc: "__NR_gettid", header: "".}: clong - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(NR_gettid)) - result = threadId - -elif defined(dragonfly): - proc lwp_gettid(): int32 {.importc, header: "unistd.h".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_gettid()) - result = threadId - -elif defined(openbsd): - proc getthrid(): int32 {.importc: "getthrid", header: "".} - - proc getThreadId*(): int = - ## get the ID of the currently running thread. - if threadId == 0: - threadId = int(getthrid()) - result = threadId - -elif defined(netbsd): - proc lwp_self(): int32 {.importc: "_lwp_self", header: "".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(lwp_self()) - result = threadId - -elif defined(freebsd): - proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "".} - var SYS_thr_self {.importc:"SYS_thr_self", header:"".}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - var tid = 0.cint - if threadId == 0: - discard syscall(SYS_thr_self, addr tid) - threadId = tid - result = threadId - -elif defined(macosx): - proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "".} - var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"".}: cint - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(syscall(SYS_thread_selfid)) - result = threadId - -elif defined(solaris): - type thread_t {.importc: "thread_t", header: "".} = distinct int - proc thr_self(): thread_t {.importc, header: "".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(thr_self()) - result = threadId - -elif defined(haiku): - type thr_id {.importc: "thread_id", header: "".} = distinct int32 - proc find_thread(name: cstring): thr_id {.importc, header: "".} - - proc getThreadId*(): int = - ## Gets the ID of the currently running thread. - if threadId == 0: - threadId = int(find_thread(nil)) - result = threadId +when not defined(gcOrc): + include threadids From 528ef6c218d90653f0f814c3dd256caa2cc389dc Mon Sep 17 00:00:00 2001 From: orthoplex <39635932+orthoplex@users.noreply.github.com> Date: Thu, 11 Nov 2021 20:01:51 +0100 Subject: [PATCH 0536/1654] fixed colorNames sorting mistake (#19125) [backport] --- lib/pure/colors.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 4a3f47e92..2193f69a6 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -353,8 +353,8 @@ const ("lightcoral", colLightCoral), ("lightcyan", colLightCyan), ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgrey", colLightGrey), ("lightgreen", colLightGreen), + ("lightgrey", colLightGrey), ("lightpink", colLightPink), ("lightsalmon", colLightSalmon), ("lightseagreen", colLightSeaGreen), From 3aaa12dbe5796cc55c6161e4d30a70a7f75cb3b5 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 12 Nov 2021 14:59:48 +0800 Subject: [PATCH 0537/1654] update manual (#19130) [backport] --- doc/manual.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 001402036..328035b6d 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -6721,11 +6721,11 @@ statement, as seen in stack backtraces: if not cond: # change run-time line information of the 'raise' statement: {.line: instantiationInfo().}: - raise newException(EAssertionFailed, msg) + raise newException(AssertionDefect, msg) If the `line` pragma is used with a parameter, the parameter needs be a `tuple[filename: string, line: int]`. If it is used without a parameter, -`system.InstantiationInfo()` is used. +`system.instantiationInfo()` is used. linearScanEnd pragma From f8b71f7075b68aefc3955388ad5ec3178aee7658 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 12 Nov 2021 18:00:19 +1100 Subject: [PATCH 0538/1654] Fix type mismatch with future logging (#19131) --- lib/pure/asyncfutures.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 6906d03fd..4782f0a73 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -225,7 +225,7 @@ proc complete*[T](future: FutureVar[T], val: T) = fut.finished = true fut.value = val fut.callbacks.call() - when isFutureLoggingEnabled: logFutureFinish(future) + when isFutureLoggingEnabled: logFutureFinish(fut) proc fail*[T](future: Future[T], error: ref Exception) = ## Completes `future` with `error`. From c6fc3b2eae5aac68480a787b7fa3b81e3748bc2f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 Nov 2021 11:19:24 +0100 Subject: [PATCH 0539/1654] fixes #19051 [backport:1.6] (#19133) --- compiler/sizealignoffsetimpl.nim | 19 +++++++++++++++++-- tests/ccgbugs/t5296.nim | 9 ++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index 44ba32579..c5cd3ba64 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -196,6 +196,11 @@ proc computeUnionObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode; packed: bo accum.offset = szUnknownSize proc computeSizeAlign(conf: ConfigRef; typ: PType) = + template setSize(typ, s) = + typ.size = s + typ.align = s + typ.paddingAtEnd = 0 + ## computes and sets ``size`` and ``align`` members of ``typ`` assert typ != nil let hasSize = typ.size != szUncomputedSize @@ -258,14 +263,14 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) = of tyArray: computeSizeAlign(conf, typ[1]) - let elemSize = typ[1].size + let elemSize = typ[1].size let len = lengthOrd(conf, typ[0]) if elemSize < 0: typ.size = elemSize typ.align = int16(elemSize) elif len < 0: typ.size = szUnknownSize - typ.align = szUnknownSize + typ.align = szUnknownSize else: typ.size = toInt64Checked(len * int32(elemSize), szTooBigSize) typ.align = typ[1].align @@ -445,6 +450,16 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) = typ.size = szUnknownSize typ.align = szUnknownSize typ.paddingAtEnd = szUnknownSize + of tyInt, tyUInt: + setSize typ, conf.target.intSize.int16 + of tyBool, tyChar, tyUInt8, tyInt8: + setSize typ, 1 + of tyInt16, tyUInt16: + setSize typ, 2 + of tyInt32, tyUInt32: + setSize typ, 4 + of tyInt64, tyUInt64: + setSize typ, 8 else: typ.size = szUnknownSize typ.align = szUnknownSize diff --git a/tests/ccgbugs/t5296.nim b/tests/ccgbugs/t5296.nim index 990b4bee2..8fbed35c4 100644 --- a/tests/ccgbugs/t5296.nim +++ b/tests/ccgbugs/t5296.nim @@ -1,6 +1,7 @@ discard """ cmd: "nim c -d:release $file" -output: 1 +output: '''1 +-1''' """ proc bug() : void = @@ -12,3 +13,9 @@ proc bug() : void = echo x bug() + +# bug #19051 +type GInt[T] = int + +var a = 1 +echo -a From 0a1049881eaa08cb7d10194d9b446d8a9ca52122 Mon Sep 17 00:00:00 2001 From: Clay Sweetser Date: Fri, 12 Nov 2021 22:51:43 -0500 Subject: [PATCH 0540/1654] Merge file size fields correctly on Windows (#19141) * Merge file size fields correctly on Windows Merge file size fields correctly on Windows - Merge the two 32-bit file size fields from `BY_HANDLE_FILE_INFORMATION` correctly in `rawToFormalFileInfo`. - Fixes #19135 * Update os.nim --- lib/pure/os.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 513e09b48..9b08fe2e1 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -3255,7 +3255,11 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(windows): - template merge(a, b): untyped = a or (b shl 32) + template merge(a, b): untyped = + int64( + (uint64(cast[uint32](a))) or + (uint64(cast[uint32](b)) shl 32) + ) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) formalInfo.size = merge(rawInfo.nFileSizeLow, rawInfo.nFileSizeHigh) From ade85ee91fee849f4b036ed8e42e31b978144522 Mon Sep 17 00:00:00 2001 From: Christoph Krybus Date: Sat, 13 Nov 2021 04:51:58 +0100 Subject: [PATCH 0541/1654] Fix punycode.decode function (#19136) * Refactor: rename proc to func * Fix punycode.decode function This function could only properly decode punycodes containing a single encoded unicode character. As soon as there was more than one punycode character group to decode it produced invalid output - the number of characters was correct, but their position was not. * Update tpunycode.nim Co-authored-by: Clay Sweetser --- lib/pure/punycode.nim | 31 +++--- tests/stdlib/tpunycode.nim | 202 ++++++++++++++++++++++++++++++++++++- 2 files changed, 217 insertions(+), 16 deletions(-) diff --git a/lib/pure/punycode.nim b/lib/pure/punycode.nim index dfeeea35e..49e46496e 100644 --- a/lib/pure/punycode.nim +++ b/lib/pure/punycode.nim @@ -28,7 +28,7 @@ const type PunyError* = object of ValueError -proc decodeDigit(x: char): int {.raises: [PunyError].} = +func decodeDigit(x: char): int {.raises: [PunyError].} = if '0' <= x and x <= '9': result = ord(x) - (ord('0') - 26) elif 'A' <= x and x <= 'Z': @@ -38,7 +38,7 @@ proc decodeDigit(x: char): int {.raises: [PunyError].} = else: raise newException(PunyError, "Bad input") -proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = +func encodeDigit(digit: int): Rune {.raises: [PunyError].} = if 0 <= digit and digit < 26: result = Rune(digit + ord('a')) elif 26 <= digit and digit < 36: @@ -46,10 +46,10 @@ proc encodeDigit(digit: int): Rune {.raises: [PunyError].} = else: raise newException(PunyError, "internal error in punycode encoding") -proc isBasic(c: char): bool = ord(c) < 0x80 -proc isBasic(r: Rune): bool = int(r) < 0x80 +func isBasic(c: char): bool = ord(c) < 0x80 +func isBasic(r: Rune): bool = int(r) < 0x80 -proc adapt(delta, numPoints: int, first: bool): int = +func adapt(delta, numPoints: int, first: bool): int = var d = if first: delta div Damp else: delta div 2 d += d div numPoints var k = 0 @@ -58,7 +58,7 @@ proc adapt(delta, numPoints: int, first: bool): int = k += Base result = k + (Base - TMin + 1) * d div (d + Skew) -proc encode*(prefix, s: string): string {.raises: [PunyError].} = +func encode*(prefix, s: string): string {.raises: [PunyError].} = ## Encode a string that may contain Unicode. ## Prepend `prefix` to the result result = prefix @@ -114,18 +114,18 @@ proc encode*(prefix, s: string): string {.raises: [PunyError].} = inc d inc n -proc encode*(s: string): string {.raises: [PunyError].} = +func encode*(s: string): string {.raises: [PunyError].} = ## Encode a string that may contain Unicode. Prefix is empty. result = encode("", s) -proc decode*(encoded: string): string {.raises: [PunyError].} = +func decode*(encoded: string): string {.raises: [PunyError].} = ## Decode a Punycode-encoded string var n = InitialN i = 0 bias = InitialBias var d = rfind(encoded, Delimiter) - result = "" + var output: seq[Rune] if d > 0: # found Delimiter @@ -133,7 +133,7 @@ proc decode*(encoded: string): string {.raises: [PunyError].} = var c = encoded[j] # char if not c.isBasic: raise newException(PunyError, "Encoded contains a non-basic char") - result.add(c) # add the character + output.add(Rune(c)) # add the character inc d else: d = 0 # set to first index @@ -161,16 +161,17 @@ proc decode*(encoded: string): string {.raises: [PunyError].} = break w *= Base - t k += Base - bias = adapt(i - oldi, runeLen(result) + 1, oldi == 0) + bias = adapt(i - oldi, len(output) + 1, oldi == 0) - if i div (runeLen(result) + 1) > high(int32) - n: + if i div (len(output) + 1) > high(int32) - n: raise newException(PunyError, "Value too large") - n += i div (runeLen(result) + 1) - i = i mod (runeLen(result) + 1) - insert(result, $Rune(n), i) + n += i div (len(output) + 1) + i = i mod (len(output) + 1) + insert(output, Rune(n), i) inc i + result = $output runnableExamples: static: diff --git a/tests/stdlib/tpunycode.nim b/tests/stdlib/tpunycode.nim index 596c5ef63..bb2b4beb4 100644 --- a/tests/stdlib/tpunycode.nim +++ b/tests/stdlib/tpunycode.nim @@ -1,5 +1,205 @@ -import punycode +import punycode, std/unicode doAssert(decode(encode("", "bücher")) == "bücher") doAssert(decode(encode("münchen")) == "münchen") doAssert encode("xn--", "münchen") == "xn--mnchen-3ya" + +# source: https://datatracker.ietf.org/doc/html/rfc3492.html#section-7 +let punycode_testcases = [ + # (A) Arabic (Egyptian): + ( + input: + "\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644" & + "\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F", + output: + "egbpdaj6bu4bxfgehfvwxn" + ), + + # (B) Chinese (simplified): + ( + input: + "\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587", + output: + "ihqwcrb4cv8a8dqg056pqjye" + ), + + # (C) Chinese (traditional): + ( + input: + "\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587", + output: + "ihqwctvzc91f659drss3x8bo0yb" + ), + + # (D) Czech: Proprostnemluvesky + ( + input: + "\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074" & + "\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D" & + "\u0065\u0073\u006B\u0079", + output: + "Proprostnemluvesky-uyb24dma41a" + ), + + # (E) Hebrew: + ( + input: + "\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8" & + "\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2" & + "\u05D1\u05E8\u05D9\u05EA", + output: + "4dbcagdahymbxekheh6e0a7fei0b" + ), + + # (F) Hindi (Devanagari): + ( + input: + "\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D" & + "\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939" & + "\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947" & + "\u0939\u0948\u0902", + output: + "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd" + ), + + # (G) Japanese (kanji and hiragana): + ( + input: + "\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092" & + "\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B", + output: + "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa" + ), + + # (H) Korean (Hangul syllables): + ( + input: + "\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774" & + "\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74" & + "\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C", + output: + "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5jpsd879ccm6fea98c" + ), + + # (I) Russian (Cyrillic): + ( + input: + "\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E" & + "\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440" & + "\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A" & + "\u0438", + output: + "b1abfaaepdrnnbgefbaDotcwatmq2g4l" + ), + + # (J) Spanish: PorqunopuedensimplementehablarenEspaol + ( + input: + "\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070" & + "\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070" & + "\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061" & + "\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070" & + "\u0061\u00F1\u006F\u006C", + output: + "PorqunopuedensimplementehablarenEspaol-fmd56a" + ), + + # (K) Vietnamese: + # Tisaohkhngthch\ + # nitingVit + ( + input: + "\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B" & + "\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068" & + "\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067" & + "\u0056\u0069\u1EC7\u0074", + output: + "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g" + ), + + # (L) 3B + ( + input: + "\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F", + output: + "3B-ww4c5e180e575a65lsy2b" + ), + + # (M) -with-SUPER-MONKEYS + ( + input: + "\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074" & + "\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D" & + "\u004F\u004E\u004B\u0045\u0059\u0053", + output: + "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n" + ), + + # (N) Hello-Another-Way- + ( + input: + "\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F" & + "\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D" & + "\u305D\u308C\u305E\u308C\u306E\u5834\u6240", + output: + "Hello-Another-Way--fc4qua05auwb3674vfr0b" + ), + + # (O) 2 + ( + input: + "\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032", + output: + "2-u9tlzr9756bt3uc0v" + ), + + # (P) MajiKoi5 + ( + input: + "\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059" & + "\u308B\u0035\u79D2\u524D", + output: + "MajiKoi5-783gue6qz075azm5e" + ), + + # (Q) de + ( + input: + "\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0", + output: + "de-jg4avhby1noc0d" + ), + + # (R) + ( + input: + "\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067", + output: + "d9juau41awczczp" + ), + + # (S) -> $1.00 <- + ( + input: + "\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020" & + "\u003C\u002D", + output: + "-> $1.00 <--" + ) +] + +block encodeTests: + for (uni, puny) in punycode_testcases: + # Need to convert both strings to lower case, since + # some of the extended encodings use upper case, but our + # code produces only lower case. Converting just puny to + # lower is also insufficient, since some of the input characters + # are upper case. + doAssert encode(uni).toLower() == puny.toLower() + +block decodeTests: + for (uni, puny) in punycode_testcases: + doAssert decode(puny) == uni + +block decodeInvalidTest: + doAssertRaises(PunyError): discard decode("xn--w&") From 270a5a372ddcb8fa97eacdf926dd90bfe189298f Mon Sep 17 00:00:00 2001 From: Anuken Date: Sat, 13 Nov 2021 19:09:15 -0500 Subject: [PATCH 0542/1654] Fix undeclared 'SYS_getrandom' on emscripten (#19144) --- lib/std/sysrand.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index b35f24a72..6b354e5a3 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -164,7 +164,7 @@ elif defined(windows): result = randomBytes(addr dest[0], size) -elif defined(linux) and not defined(nimNoGetRandom): +elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): # TODO using let, pending bootstrap >= 1.4.0 var SYS_getrandom {.importc: "SYS_getrandom", header: "".}: clong const syscallHeader = """#include From bec490016548e73f01317736ff8863f9ce6aab31 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 14 Nov 2021 19:46:21 +0800 Subject: [PATCH 0543/1654] wrong spaces (3 => 2) (#19145) --- lib/system/threads.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index a8a413bcb..0e5198035 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -190,7 +190,7 @@ when defined(windows): threadProcWrapperBody(closure) # implicitly return 0 elif defined(genode): - proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = + proc threadProcWrapper[TArg](closure: pointer) {.noconv.} = threadProcWrapperBody(closure) else: proc threadProcWrapper[TArg](closure: pointer): pointer {.noconv.} = From 6976d18519d102c9cc857a84a4b10b009c1e1a38 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 14 Nov 2021 03:49:30 -0800 Subject: [PATCH 0544/1654] Implement zephyr urandom and monotime (#19142) * implement urandom for Zephyr * add monotime on zephyr Co-authored-by: Jaremy Creechley --- lib/std/monotimes.nim | 7 +++++++ lib/std/sysrand.nim | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/std/monotimes.nim b/lib/std/monotimes.nim index bd078d7c0..5c67a5d4c 100644 --- a/lib/std/monotimes.nim +++ b/lib/std/monotimes.nim @@ -76,6 +76,10 @@ when defined(js): elif defined(posix) and not defined(osx): import posix +when defined(zephyr): + proc k_uptime_ticks(): int64 {.importc: "k_uptime_ticks", header: "".} + proc k_ticks_to_ns_floor64(ticks: int64): int64 {.importc: "k_ticks_to_ns_floor64", header: "".} + elif defined(windows): proc QueryPerformanceCounter(res: var uint64) {. importc: "QueryPerformanceCounter", stdcall, dynlib: "kernel32".} @@ -98,6 +102,9 @@ proc getMonoTime*(): MonoTime {.tags: [TimeEffect].} = mach_timebase_info(machAbsoluteTimeFreq) result = MonoTime(ticks: ticks * machAbsoluteTimeFreq.numer div machAbsoluteTimeFreq.denom) + elif defined(zephyr): + let ticks = k_ticks_to_ns_floor64(k_uptime_ticks()) + result = MonoTime(ticks: ticks) elif defined(posix): var ts: Timespec discard clock_gettime(CLOCK_MONOTONIC, ts) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 6b354e5a3..1b7b2c024 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -63,7 +63,7 @@ when defined(posix): import posix const - batchImplOS = defined(freebsd) or defined(openbsd) or (defined(macosx) and not defined(ios)) + batchImplOS = defined(freebsd) or defined(openbsd) or defined(zephyr) or (defined(macosx) and not defined(ios)) batchSize {.used.} = 256 when batchImplOS: @@ -207,6 +207,16 @@ elif defined(openbsd): proc getRandomImpl(p: pointer, size: int): int {.inline.} = result = getentropy(p, cint(size)).int +elif defined(zephyr): + proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "".} + # Fill the destination buffer with cryptographically secure + # random data values + # + + proc getRandomImpl(p: pointer, size: int): int {.inline.} = + # 0 if success, -EIO if entropy reseed error + result = sys_csrand_get(p, csize_t(size)).int + elif defined(freebsd): type cssize_t {.importc: "ssize_t", header: "".} = int From 309ec7167e6ea17112901533539be0a1884c8b65 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 16 Nov 2021 15:49:04 +0100 Subject: [PATCH 0545/1654] fixes .raises inference for newSeq builtin under --gc:orc [backport] (#19158) --- lib/system/seqs_v2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 0ac51109e..0c487b31d 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -73,7 +73,7 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin q.cap = newCap result = q -proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [].} = +proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = when nimvm: setLen(x, newLen) else: From 92d6fb86c66ceda154972314ba695cea4aac9301 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 16 Nov 2021 11:30:07 -0800 Subject: [PATCH 0546/1654] Enable customizing PageShift to set PageSize for embedded targets (#19129) * Enable customizing PageSize (via PageShift). This enables adjusting PageSize for embedded targets without abusing cpu16. * copy nimPageXYZ settings for mmpaptest * add docs for Nim manual * add docs for Nim manual * docs tweaks Co-authored-by: Jaremy Creechley --- doc/nimc.rst | 24 ++++++++++++++++++++++++ lib/system/bitmasks.nim | 10 +++++++--- tests/mmaptest.nim | 7 +++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/doc/nimc.rst b/doc/nimc.rst index 98b754729..d35dad6c2 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -656,6 +656,30 @@ is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc` define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently only supported with `--gc:arc` or `--gc:orc`. (Since version 1.6) +nimPage256 / nimPage512 / nimPage1k +=================================== + +Adjust the page size for Nim's GC allocator. This enables using +`nimAllocPagesViaMalloc` on devices with less RAM. The default +page size requires too much RAM to work. + +Recommended settings: + +- < 32 kB of RAM use `nimPage256` + +- < 512 kB of RAM use `nimPage512` + +- < 2 MB of RAM use `nimPage1k` + +Initial testing hasn't shown much difference between 512B or 1kB page sizes +in terms of performance or latency. Using `nimPages256` will limit the +total amount of allocatable RAM. + +nimMemAlignTiny +=============== + +Sets `MemAlign` to `4` bytes which reduces the memory alignment +to better match some embedded devices. Nim for realtime systems ======================== diff --git a/lib/system/bitmasks.nim b/lib/system/bitmasks.nim index caf86568a..9cb57bc45 100644 --- a/lib/system/bitmasks.nim +++ b/lib/system/bitmasks.nim @@ -10,14 +10,18 @@ # Page size of the system; in most cases 4096 bytes. For exotic OS or # CPU this needs to be changed: const - PageShift = when defined(cpu16): 8 else: 12 # \ - # my tests showed no improvements for using larger page sizes. + PageShift = when defined(nimPage256) or defined(cpu16): 8 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + PageSize = 1 shl PageShift PageMask = PageSize-1 MemAlign = # also minimal allocatable memory block - when defined(useMalloc): + when defined(nimMemAlignTiny): 4 + elif defined(useMalloc): when defined(amd64): 16 else: 8 else: 16 diff --git a/tests/mmaptest.nim b/tests/mmaptest.nim index 7a93cdd45..33010606f 100644 --- a/tests/mmaptest.nim +++ b/tests/mmaptest.nim @@ -19,8 +19,11 @@ proc `+!!`(p: pointer, size: int): pointer {.inline.} = result = cast[pointer](cast[int](p) + size) const - PageShift = when defined(cpu16): 8 else: 12 # \ - # my tests showed no improvements for using larger page sizes. + PageShift = when defined(nimPage256) or defined(cpu16): 8 + elif defined(nimPage512): 9 + elif defined(nimPage1k): 10 + else: 12 # \ # my tests showed no improvements for using larger page sizes. + PageSize = 1 shl PageShift var p = osAllocPages(3 * PageSize) From fe46c8b5f13f854e4109183d2199b922f939548c Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 17 Nov 2021 17:15:54 +0800 Subject: [PATCH 0547/1654] fix marshal bugs in VM (#19161) [backport:1.6] --- compiler/vmmarshal.nim | 4 ++-- tests/ccgbugs/t8967.nim | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/compiler/vmmarshal.nim b/compiler/vmmarshal.nim index d48d2408c..d28f0325b 100644 --- a/compiler/vmmarshal.nim +++ b/compiler/vmmarshal.nim @@ -92,7 +92,8 @@ proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet; if a[i].kind == nkRange: var x = copyNode(a[i][0]) storeAny(s, t.lastSon, x, stored, conf) - while x.intVal+1 <= a[i][1].intVal: + inc x.intVal + while x.intVal <= a[i][1].intVal: s.add(", ") storeAny(s, t.lastSon, x, stored, conf) inc x.intVal @@ -231,7 +232,6 @@ proc loadAny(p: var JsonParser, t: PType, result = newNode(nkCurly) while p.kind != jsonArrayEnd and p.kind != jsonEof: result.add loadAny(p, t.lastSon, tab, cache, conf, idgen) - next(p) if p.kind == jsonArrayEnd: next(p) else: raiseParseErr(p, "']' end of array expected") of tyPtr, tyRef: diff --git a/tests/ccgbugs/t8967.nim b/tests/ccgbugs/t8967.nim index e342b7eae..0301a2e4f 100644 --- a/tests/ccgbugs/t8967.nim +++ b/tests/ccgbugs/t8967.nim @@ -4,7 +4,11 @@ discard """ import marshal -let orig: set[char] = {'A'..'Z'} -let m = $$orig -let old = to[set[char]](m) -doAssert orig - old == {} +template main() = + let orig: set[char] = {'A'..'Z'} + let m = $$orig + let old = to[set[char]](m) + doAssert orig - old == {} + +static: main() +main() From a8611c665f26adbf7f6dd2438359b4c5d930bf86 Mon Sep 17 00:00:00 2001 From: OxFred <88140942+OxFred@users.noreply.github.com> Date: Wed, 17 Nov 2021 13:16:12 +0100 Subject: [PATCH 0548/1654] Update dependency (#19151) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index fc5c808ab..e11799e73 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -55,7 +55,7 @@ pkg "chronos", "nim c -r -d:release tests/testall", allowFailure = true # pendin pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" -pkg "comprehension", "nimble test", "https://github.com/alehander42/comprehension" +pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehension" pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" From eb5358dcdb1b650db658def14cc8c60b4bb7c203 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Fri, 19 Nov 2021 02:19:01 -0500 Subject: [PATCH 0549/1654] Testament now checks OS, CPU, endianess, bitsizes 8-64 in discard statements (#19137) * Testament checks host/cpu in compiler/platform for discard spec statement * Remove duplicated value checks, add comments, add all cpu bit sizes and endianesses Co-authored-by: quantimnot --- testament/specs.nim | 70 ++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/testament/specs.nim b/testament/specs.nim index c7e332262..42400f22d 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -9,6 +9,7 @@ import sequtils, parseutils, strutils, os, streams, parsecfg, tables, hashes, sets +import compiler/platform type TestamentData* = ref object # better to group globals under 1 object; could group the other ones here too @@ -329,42 +330,71 @@ proc parseSpec*(filename: string): TSpec = # Valgrind only supports OSX <= 17.x result.useValgrind = disabled of "disabled": - case e.value.normalize + let value = e.value.normalize + case value of "y", "yes", "true", "1", "on": result.err = reDisabled of "n", "no", "false", "0", "off": discard - of "win", "windows": + # These values are defined in `compiler/options.isDefined` + of "win": when defined(windows): result.err = reDisabled of "linux": when defined(linux): result.err = reDisabled of "bsd": when defined(bsd): result.err = reDisabled - of "osx", "macosx": # xxx remove `macosx` alias? + of "osx": when defined(osx): result.err = reDisabled - of "unix": - when defined(unix): result.err = reDisabled - of "posix": + of "unix", "posix": when defined(posix): result.err = reDisabled + of "freebsd": + when defined(freebsd): result.err = reDisabled + of "littleendian": + when defined(littleendian): result.err = reDisabled + of "bigendian": + when defined(bigendian): result.err = reDisabled + of "cpu8", "8bit": + when defined(cpu8): result.err = reDisabled + of "cpu16", "16bit": + when defined(cpu16): result.err = reDisabled + of "cpu32", "32bit": + when defined(cpu32): result.err = reDisabled + of "cpu64", "64bit": + when defined(cpu64): result.err = reDisabled + # These values are for CI environments of "travis": # deprecated if isTravis: result.err = reDisabled of "appveyor": # deprecated if isAppVeyor: result.err = reDisabled of "azure": if isAzure: result.err = reDisabled - of "32bit": - if sizeof(int) == 4: - result.err = reDisabled - of "freebsd": - when defined(freebsd): result.err = reDisabled - of "arm64": - when defined(arm64): result.err = reDisabled - of "i386": - when defined(i386): result.err = reDisabled - of "openbsd": - when defined(openbsd): result.err = reDisabled - of "netbsd": - when defined(netbsd): result.err = reDisabled else: - result.parseErrors.addLine "cannot interpret as a bool: ", e.value + # Check whether the value exists as an OS or CPU that is + # defined in `compiler/platform`. + block checkHost: + for os in platform.OS: + # Check if the value exists as OS. + if value == os.name.normalize: + # The value exists; is it the same as the current host? + if value == hostOS.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + for cpu in platform.CPU: + # Check if the value exists as CPU. + if value == cpu.name.normalize: + # The value exists; is it the same as the current host? + if value == hostCPU.normalize: + # The value exists and is the same as the current host, + # so disable the test. + result.err = reDisabled + # The value was defined, so there is no need to check further + # values or raise an error. + break checkHost + # The value doesn't exist as an OS, CPU, or any previous value + # defined in this case statement, so raise an error. + result.parseErrors.addLine "cannot interpret as a bool: ", e.value of "cmd": if e.value.startsWith("nim "): result.cmd = compilerPrefix & e.value[3..^1] From 1b143f5e79c940ba7f70e0512f36b5c61a6bc24d Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Mon, 22 Nov 2021 12:41:35 +0300 Subject: [PATCH 0550/1654] Accept nnkTypeSection from typedef macro pragmas (#19168) --- changelog.md | 29 ++++++++++++++ compiler/semstmts.nim | 18 ++++++--- doc/manual.rst | 1 + tests/pragmas/ttypedef_macro.nim | 66 ++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 tests/pragmas/ttypedef_macro.nim diff --git a/changelog.md b/changelog.md index 1636d125f..8e70513e0 100644 --- a/changelog.md +++ b/changelog.md @@ -13,7 +13,36 @@ ## Language changes +- Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, + allowing multiple type definitions to be injected in place of the original type definition. + ```nim + import macros + + macro multiply(amount: static int, s: untyped): untyped = + let name = $s[0].basename + result = newNimNode(nnkTypeSection) + for i in 1 .. amount: + result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) + + type + Foo = object + Bar {.multiply: 3.} = object + x, y, z: int + Baz = object + + # becomes + + type + Foo = object + Bar1 = object + x, y, z: int + Bar2 = object + x, y, z: int + Bar3 = object + x, y, z: int + Baz = object + ``` ## Compiler changes diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 4607e857a..36d76608e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1112,7 +1112,12 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = if name.kind == nkPragmaExpr: let rewritten = applyTypeSectionPragmas(c, name[1], typeDef) if rewritten != nil: - typeSection[i] = rewritten + case rewritten.kind + of nkTypeDef: + typeSection[i] = rewritten + of nkTypeSection: + typeSection.sons[i .. i] = rewritten.sons + else: illFormedAst(rewritten, c.config) typeDefLeftSidePass(c, typeSection, i) return pragma(c, s, name[1], typePragmas) @@ -1143,16 +1148,19 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) = proc typeSectionLeftSidePass(c: PContext, n: PNode) = # process the symbols on the left side for the whole type section, before # we even look at the type definitions on the right - for i in 0.. Date: Tue, 23 Nov 2021 00:13:03 -0800 Subject: [PATCH 0551/1654] Implement threads on Zephyr (#19156) * pthreads setup for zephyr - enable tweak stack size - update lib/system/threads.nim - Fix int/uint in casting pointer. * add documentation and tweak flag names * add documentation and tweak flag names * fix configuration flag names * fix configuration flag names * cleanup Co-authored-by: Jaremy Creechley --- doc/nimc.rst | 9 ++++++++ lib/system/threadlocalstorage.nim | 2 ++ lib/system/threads.nim | 34 +++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/doc/nimc.rst b/doc/nimc.rst index d35dad6c2..6be4c204e 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -681,6 +681,15 @@ nimMemAlignTiny Sets `MemAlign` to `4` bytes which reduces the memory alignment to better match some embedded devices. +Thread stack size +================= + +Nim's thread API provides a simple wrapper around more advanced +RTOS task features. Customizing the stack size and stack guard size can +be done by setting `-d:nimThreadStackSize=16384` or `-d:nimThreadStackGuard=32`. + +Currently only Zephyr and FreeRTOS support these configurations. + Nim for realtime systems ======================== diff --git a/lib/system/threadlocalstorage.nim b/lib/system/threadlocalstorage.nim index e772f270c..cea9abb42 100644 --- a/lib/system/threadlocalstorage.nim +++ b/lib/system/threadlocalstorage.nim @@ -153,6 +153,8 @@ else: proc pthread_attr_init(a1: var Pthread_attr): cint {. importc, header: pthreadh.} + proc pthread_attr_setstack*(a1: ptr Pthread_attr, a2: pointer, a3: int): cint {. + importc, header: pthreadh.} proc pthread_attr_setstacksize(a1: var Pthread_attr, a2: int): cint {. importc, header: pthreadh.} proc pthread_attr_destroy(a1: var Pthread_attr): cint {. diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 0e5198035..ac727be0a 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -47,14 +47,23 @@ when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} -const - StackGuardSize = 4096 - ThreadStackMask = - when defined(genode): - 1024*64*sizeof(int)-1 - else: - 1024*256*sizeof(int)-1 - ThreadStackSize = ThreadStackMask+1 - StackGuardSize +when defined(zephyr) or defined(freertos): + const + nimThreadStackSize {.intdefine.} = 8192 + nimThreadStackGuard {.intdefine.} = 128 + + StackGuardSize = nimThreadStackGuard + ThreadStackSize = nimThreadStackSize - nimThreadStackGuard +else: + const + StackGuardSize = 4096 + ThreadStackMask = + when defined(genode): + 1024*64*sizeof(int)-1 + else: + 1024*256*sizeof(int)-1 + + ThreadStackSize = ThreadStackMask+1 - StackGuardSize #const globalsSlot = ThreadVarSlot(0) #sysAssert checkSlot.int == globalsSlot.int @@ -321,7 +330,14 @@ else: when hasSharedHeap: t.core.stackSize = ThreadStackSize var a {.noinit.}: Pthread_attr doAssert pthread_attr_init(a) == 0 - let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + when defined(zephyr): + var + rawstk = allocShared0(ThreadStackSize + StackGuardSize) + stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) + let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + else: + let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) + when not defined(ios): # This fails on iOS doAssert(setstacksizeResult == 0) From 040d23e799e94aa3e0c6b254f1dc708b2eada96b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 23 Nov 2021 15:02:03 +0300 Subject: [PATCH 0552/1654] implement RST & Markdown quote blocks (#19147) * implement RST & Markdown quote blocks * compile with nim 1.0 * Fix indentation --- config/nimdoc.tex.cfg | 10 +- doc/nimdoc.css | 5 + lib/packages/docutils/rst.nim | 210 +++++++++++- lib/packages/docutils/rstast.nim | 9 +- lib/packages/docutils/rstgen.nim | 29 +- nimdoc/testproject/expected/nimdoc.out.css | 5 + tests/stdlib/trst.nim | 361 +++++++++++++++++++++ 7 files changed, 617 insertions(+), 12 deletions(-) diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 457f452dd..3912d1279 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -139,9 +139,17 @@ doc.file = """ \usepackage[most]{tcolorbox} % boxes around admonitions, code blocks, doc.item \newtcolorbox{rstadmonition}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, + left=3mm, right=0mm, top=1mm, bottom=1mm, before upper=\indent, parbox=false, #1} +\newtcolorbox{rstquote}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + parbox=false, + borderline west={0.3em}{0pt}{lightgray}, + borderline north={0.05em}{0pt}{lightgray}, + borderline east={0.05em}{0pt}{lightgray}, + borderline south={0.05em}{0pt}{lightgray}} + \definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} \newtcolorbox{rstprebox}[1][]{blanker, breakable, diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 7c819ea30..0014cf196 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -567,6 +567,11 @@ blockquote { border-left: 5px solid #bbc; } +blockquote.markdown-quote { + font-size: 0.9rem; /* use rem to avoid recursion */ + font-style: normal; +} + .pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 2e908f4e5..4c28894fb 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -53,6 +53,8 @@ ## + field lists ## + option lists ## + indented literal blocks +## + quoted literal blocks +## + line blocks ## + simple tables ## + directives (see official documentation in `RST directives list`_): ## - ``image``, ``figure`` for including images and videos @@ -121,6 +123,7 @@ ## * Markdown code blocks ## * Markdown links ## * Markdown headlines +## * Markdown block quotes ## * using ``1`` as auto-enumerator in enumerated lists like RST ``#`` ## (auto-enumerator ``1`` can not be used with ``#`` in the same list) ## @@ -145,7 +148,7 @@ ## 2) Compatibility mode which is RST rules. ## ## .. Note:: in both modes the parser interpretes text between single -## backticks (code) identically: +## backticks (code) identically: ## backslash does not escape; the only exception: ``\`` folowed by ` ## does escape so that we can always input a single backtick ` in ## inline code. However that makes impossible to input code with @@ -156,13 +159,35 @@ ## ``\`` -- GOOD ## So single backticks can always be input: `\`` will turn to ` code ## +## .. Attention:: +## We don't support some obviously poor design choices of Markdown (or RST). +## +## - no support for the rule of 2 spaces causing a line break in Markdown +## (use RST "line blocks" syntax for making line breaks) +## +## - interpretation of Markdown block quotes is also slightly different, +## e.g. case +## +## :: +## +## >>> foo +## > bar +## >>baz +## +## is a single 3rd-level quote `foo bar baz` in original Markdown, while +## in Nim we naturally see it as 3rd-level quote `foo` + 1st level `bar` + +## 2nd level `baz`: +## +## >>> foo +## > bar +## >>baz +## ## Limitations ## ----------- ## ## * no Unicode support in character width calculations ## * body elements ## - no roman numerals in enumerated lists -## - no quoted literal blocks ## - no doctest blocks ## - no grid tables ## - some directives are missing (check official `RST directives list`_): @@ -472,6 +497,10 @@ type line: int # the last line of this style occurrence # (for error message) hasPeers: bool # has headings on the same level of hierarchy? + LiteralBlockKind = enum # RST-style literal blocks after `::` + lbNone, + lbIndentedLiteralBlock, + lbQuotedLiteralBlock LevelMap = seq[LevelInfo] # Saves for each possible title adornment # style its level in the current document. SubstitutionKind = enum @@ -1953,6 +1982,44 @@ proc parseLiteralBlock(p: var RstParser): PRstNode = inc p.idx result.add(n) +proc parseQuotedLiteralBlock(p: var RstParser): PRstNode = + result = newRstNodeA(p, rnLiteralBlock) + var n = newLeaf("") + if currentTok(p).kind == tkIndent: + var indent = currInd(p) + while currentTok(p).kind == tkIndent: inc p.idx # skip blank lines + var quoteSym = currentTok(p).symbol[0] + while true: + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if currentTok(p).ival < indent: + break + elif currentTok(p).ival == indent: + if nextTok(p).kind == tkPunct and nextTok(p).symbol[0] == quoteSym: + n.text.add("\n") + inc p.idx + elif nextTok(p).kind == tkIndent: + break + else: + rstMessage(p, mwRstStyle, "no newline after quoted literal block") + break + else: + rstMessage(p, mwRstStyle, + "unexpected indentation in quoted literal block") + break + else: + n.text.add(currentTok(p).symbol) + inc p.idx + result.add(n) + +proc parseRstLiteralBlock(p: var RstParser, kind: LiteralBlockKind): PRstNode = + if kind == lbIndentedLiteralBlock: + result = parseLiteralBlock(p) + else: + result = parseQuotedLiteralBlock(p) + proc getLevel(p: var RstParser, c: char, hasOverline: bool): int = ## Returns (preliminary) heading level corresponding to `c` and ## `hasOverline`. If level does not exist, add it first. @@ -2023,6 +2090,33 @@ proc isLineBlock(p: RstParser): bool = p.tok[j].col > currentTok(p).col or p.tok[j].symbol == "\n" +proc isMarkdownBlockQuote(p: RstParser): bool = + result = currentTok(p).symbol[0] == '>' + +proc whichRstLiteralBlock(p: RstParser): LiteralBlockKind = + ## Checks that the following tokens are either Indented Literal Block or + ## Quoted Literal Block (which is not quite the same as Markdown quote block). + ## https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#quoted-literal-blocks + if currentTok(p).symbol == "::" and nextTok(p).kind == tkIndent: + if currInd(p) > nextTok(p).ival: + result = lbNone + if currInd(p) < nextTok(p).ival: + result = lbIndentedLiteralBlock + elif currInd(p) == nextTok(p).ival: + var i = p.idx + 1 + while p.tok[i].kind == tkIndent: inc i + const validQuotingCharacters = { + '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', + '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', + '_', '`', '{', '|', '}', '~'} + if p.tok[i].kind in {tkPunct, tkAdornment} and + p.tok[i].symbol[0] in validQuotingCharacters: + result = lbQuotedLiteralBlock + else: + result = lbNone + else: + result = lbNone + proc predNL(p: RstParser): bool = result = true if p.idx > 0: @@ -2078,6 +2172,8 @@ proc whichSection(p: RstParser): RstNodeKind = elif match(p, p.idx + 1, " a"): result = rnTable elif currentTok(p).symbol == "|" and isLineBlock(p): result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote elif match(p, p.idx + 1, "i") and isAdornmentHeadline(p, p.idx): result = rnOverline else: @@ -2090,6 +2186,8 @@ proc whichSection(p: RstParser): RstNodeKind = result = rnMarkdownTable elif currentTok(p).symbol == "|" and isLineBlock(p): result = rnLineBlock + elif roSupportMarkdown in p.s.options and isMarkdownBlockQuote(p): + result = rnMarkdownBlockQuote elif match(p, tokenAfterNewline(p), "aI") and isAdornmentHeadline(p, tokenAfterNewline(p)): result = rnHeadline @@ -2143,6 +2241,102 @@ proc parseLineBlock(p: var RstParser): PRstNode = else: break +proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} + +proc getQuoteSymbol(p: RstParser, idx: int): tuple[sym: string, depth: int, tokens: int] = + result = ("", 0, 0) + var i = idx + result.sym &= p.tok[i].symbol + result.depth += p.tok[i].symbol.len + inc result.tokens + inc i + while p.tok[i].kind == tkWhite and i+1 < p.tok.len and + p.tok[i+1].kind == tkPunct and p.tok[i+1].symbol[0] == '>': + result.sym &= p.tok[i].symbol + result.sym &= p.tok[i+1].symbol + result.depth += p.tok[i+1].symbol.len + inc result.tokens, 2 + inc i, 2 + +proc parseMarkdownQuoteSegment(p: var RstParser, curSym: string, col: int): + PRstNode = + ## We define *segment* as a group of lines that starts with exactly the + ## same quote symbol. If the following lines don't contain any `>` (*lazy* + ## continuation) they considered as continuation of the current segment. + var q: RstParser # to delete `>` at a start of line and then parse normally + initParser(q, p.s) + q.col = p.col + q.line = p.line + var minCol = int.high # minimum colum num in the segment + while true: # move tokens of segment from `p` to `q` skipping `curSym` + case currentTok(p).kind + of tkEof: + break + of tkIndent: + if nextTok(p).kind in {tkIndent, tkEof}: + break + else: + if nextTok(p).symbol[0] == '>': + var (quoteSym, _, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + if quoteSym == curSym: # the segment continues + var iTok = tokenAfterNewline(p, p.idx+1) + if p.tok[iTok].kind notin {tkEof, tkIndent} and + p.tok[iTok].symbol[0] != '>': + rstMessage(p, mwRstStyle, + "two or more quoted lines are followed by unquoted line " & + $(curLine(p) + 1)) + break + q.tok.add currentTok(p) + var ival = currentTok(p).ival + quoteSym.len + inc p.idx, (1 + quoteTokens) # skip newline and > > > + if currentTok(p).kind == tkWhite: + ival += currentTok(p).symbol.len + inc p.idx + # fix up previous `tkIndent`s to ival (as if >>> were not there) + var j = q.tok.len - 1 + while j >= 0 and q.tok[j].kind == tkIndent: + q.tok[j].ival = ival + dec j + else: # next segment started + break + elif currentTok(p).ival < col: + break + else: # the segment continues, a case like: + # > beginning + # continuation + q.tok.add currentTok(p) + inc p.idx + else: + if currentTok(p).col < minCol: minCol = currentTok(p).col + q.tok.add currentTok(p) + inc p.idx + q.indentStack = @[minCol] + # if initial indentation `minCol` is > 0 then final newlines + # should be omitted so that parseDoc could advance to the end of tokens: + var j = q.tok.len - 1 + while q.tok[j].kind == tkIndent: dec j + q.tok.setLen (j+1) + q.tok.add Token(kind: tkEof, line: currentTok(p).line) + result = parseDoc(q) + +proc parseMarkdownBlockQuote(p: var RstParser): PRstNode = + var (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx) + let col = currentTok(p).col + result = newRstNodeA(p, rnMarkdownBlockQuote) + inc p.idx, quoteTokens # skip first > + while true: + var item = newRstNode(rnMarkdownBlockQuoteItem) + item.quotationDepth = quotationDepth + if currentTok(p).kind == tkWhite: inc p.idx + item.add parseMarkdownQuoteSegment(p, curSym, col) + result.add(item) + if currentTok(p).kind == tkIndent and currentTok(p).ival == col and + nextTok(p).kind != tkEof and nextTok(p).symbol[0] == '>': + (curSym, quotationDepth, quoteTokens) = getQuoteSymbol(p, p.idx + 1) + inc p.idx, (1 + quoteTokens) # skip newline and > > > + else: + break + proc parseParagraph(p: var RstParser, result: PRstNode) = while true: case currentTok(p).kind @@ -2158,16 +2352,17 @@ proc parseParagraph(p: var RstParser, result: PRstNode) = result.add newLeaf(" ") of rnLineBlock: result.addIfNotNil(parseLineBlock(p)) + of rnMarkdownBlockQuote: + result.addIfNotNil(parseMarkdownBlockQuote(p)) else: break else: break of tkPunct: - if currentTok(p).symbol == "::" and - nextTok(p).kind == tkIndent and - currInd(p) < nextTok(p).ival: + if (let literalBlockKind = whichRstLiteralBlock(p); + literalBlockKind != lbNone): result.add newLeaf(":") inc p.idx # skip '::' - result.add(parseLiteralBlock(p)) + result.add(parseRstLiteralBlock(p, literalBlockKind)) break else: parseInline(p, result) @@ -2257,8 +2452,6 @@ proc getColumns(p: var RstParser, cols: var IntSeq) = # last column has no limit: cols[L - 1] = 32000 -proc parseDoc(p: var RstParser): PRstNode {.gcsafe.} - proc parseSimpleTable(p: var RstParser): PRstNode = var cols: IntSeq @@ -2585,6 +2778,7 @@ proc parseSection(p: var RstParser, result: PRstNode) = a = parseLiteralBlock(p) of rnBulletList: a = parseBulletList(p) of rnLineBlock: a = parseLineBlock(p) + of rnMarkdownBlockQuote: a = parseMarkdownBlockQuote(p) of rnDirective: a = parseDotDot(p) of rnEnumList: a = parseEnumList(p) of rnLeaf: rstMessage(p, meNewSectionExpected, "(syntax error)") diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index bc7b9a650..1d5da5e1c 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -32,7 +32,10 @@ type rnFieldName, # consisting of a field name ... rnFieldBody, # ... and a field body rnOptionList, rnOptionListItem, rnOptionGroup, rnOption, rnOptionString, - rnOptionArgument, rnDescription, rnLiteralBlock, rnQuotedLiteralBlock, + rnOptionArgument, rnDescription, rnLiteralBlock, + rnMarkdownBlockQuote, # a quote starting from punctuation like >>> + rnMarkdownBlockQuoteItem, # a quotation block, quote lines starting with + # the same number of chars rnLineBlock, # the | thingie rnLineBlockItem, # a son of rnLineBlock - one line inside it. # When `RstNode` lineIndent="\n" the line's empty @@ -101,6 +104,8 @@ type of rnFootnote, rnCitation, rnOptionListItem: order*: int ## footnote order (for auto-symbol footnotes and ## auto-numbered ones without a label) + of rnMarkdownBlockQuoteItem: + quotationDepth*: int ## number of characters in line prefix of rnRef, rnSubstitutionReferences, rnInterpretedText, rnField, rnInlineCode, rnCodeBlock, rnFootnoteRef: info*: TLineInfo ## To have line/column info for warnings at @@ -409,6 +414,8 @@ proc treeRepr*(node: PRstNode, indent=0): string = result.add " level=" & $node.level of rnFootnote, rnCitation, rnOptionListItem: result.add (if node.order == 0: "" else: " order=" & $node.order) + of rnMarkdownBlockQuoteItem: + result.add " quotationDepth=" & $node.quotationDepth else: discard result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 4cbd05379..dc6ca25c1 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -89,6 +89,7 @@ type onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int; content: string) escMode*: EscapeMode + curQuotationDepth: int PDoc = var RstGenerator ## Alias to type less. @@ -167,6 +168,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget, g.currentSection = "" g.id = 0 g.escMode = emText + g.curQuotationDepth = 0 let fileParts = filename.splitFile if fileParts.ext == ".nim": g.currentSection = "Module " & fileParts.name @@ -1268,8 +1270,31 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnLiteralBlock: renderAux(d, n, "$1\n", "\n\n$2\\begin{rstpre}\n$1\n\\end{rstpre}\n\n", result) - of rnQuotedLiteralBlock: - doAssert false, "renderRstToOut" + of rnMarkdownBlockQuote: + d.curQuotationDepth = 1 + var tmp = "" + renderAux(d, n, "$1", "$1", tmp) + let itemEnding = + if d.target == outHtml: "" else: "\\end{rstquote}" + tmp.add itemEnding.repeat(d.curQuotationDepth - 1) + dispA(d.target, result, + "$1\n", + "\n\\begin{rstquote}\n$2\n$1\\end{rstquote}\n", [tmp, n.anchor.idS]) + of rnMarkdownBlockQuoteItem: + let addQuotationDepth = n.quotationDepth - d.curQuotationDepth + var itemPrefix: string # start or ending (quotation grey bar on the left) + if addQuotationDepth >= 0: + let s = + if d.target == outHtml: "
                  " + else: "\\begin{rstquote}" + itemPrefix = s.repeat(addQuotationDepth) + else: + let s = + if d.target == outHtml: "
                  " + else: "\\end{rstquote}" + itemPrefix = s.repeat(-addQuotationDepth) + renderAux(d, n, itemPrefix & "

                  $1

                  ", itemPrefix & "\n$1", result) + d.curQuotationDepth = n.quotationDepth of rnLineBlock: if n.sons.len == 1 and n.sons[0].lineIndent == "\n": # whole line block is one empty line, no need to add extra spacing diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 7c819ea30..0014cf196 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -567,6 +567,11 @@ blockquote { border-left: 5px solid #bbc; } +blockquote.markdown-quote { + font-size: 0.9rem; /* use rem to avoid recursion */ + font-style: normal; +} + .pre, span.tok { font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; font-weight: 500; diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index e4609a671..0a9eb8038 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -105,6 +105,367 @@ suite "RST parsing": rnLeaf ' ' """) + test "RST quoted literal blocks": + let expected = + dedent""" + rnInner + rnLeaf 'Paragraph' + rnLeaf ':' + rnLiteralBlock + rnLeaf '>x' + """ + + check(dedent""" + Paragraph:: + + >x""".toAst == expected) + + check(dedent""" + Paragraph:: + + >x""".toAst == expected) + + test "RST quoted literal blocks, :: at a separate line": + let expected = + dedent""" + rnInner + rnInner + rnLeaf 'Paragraph' + rnLiteralBlock + rnLeaf '>x + >>y' + """ + + check(dedent""" + Paragraph + + :: + + >x + >>y""".toAst == expected) + + check(dedent""" + Paragraph + + :: + + >x + >>y""".toAst == expected) + + test "Markdown quoted blocks": + check(dedent""" + Paragraph. + >x""".toAst == + dedent""" + rnInner + rnLeaf 'Paragraph' + rnLeaf '.' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'x' + """) + + # bug #17987 + check(dedent""" + foo https://github.com/nim-lang/Nim/issues/8258 + + > bar""".toAst == + dedent""" + rnInner + rnInner + rnLeaf 'foo' + rnLeaf ' ' + rnStandaloneHyperlink + rnLeaf 'https://github.com/nim-lang/Nim/issues/8258' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'bar' + """) + + let expected = dedent""" + rnInner + rnLeaf 'Paragraph' + rnLeaf '.' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'x1' + rnLeaf ' ' + rnLeaf 'x2' + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'y1' + rnLeaf ' ' + rnLeaf 'y2' + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'z' + """ + + check(dedent""" + Paragraph. + >x1 x2 + >>y1 y2 + >z""".toAst == expected) + + check(dedent""" + Paragraph. + > x1 x2 + >> y1 y2 + > z""".toAst == expected) + + check(dedent""" + >x + >y + >z""".toAst == + dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'x' + rnLeaf ' ' + rnLeaf 'y' + rnLeaf ' ' + rnLeaf 'z' + """) + + check(dedent""" + > z + > > >y + """.toAst == + dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'z' + rnMarkdownBlockQuoteItem quotationDepth=3 + rnLeaf 'y' + """) + + test "Markdown quoted blocks: lazy": + let expected = dedent""" + rnInner + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'x' + rnLeaf ' ' + rnLeaf 'continuation1' + rnLeaf ' ' + rnLeaf 'continuation2' + rnParagraph + rnLeaf 'newParagraph' + """ + check(dedent""" + >>x + continuation1 + continuation2 + + newParagraph""".toAst == expected) + + check(dedent""" + >> x + continuation1 + continuation2 + + newParagraph""".toAst == expected) + + # however mixing more than 1 non-lazy line and lazy one(s) splits quote + # in our parser, which appeared the easiest way to handle such cases: + var warnings = new seq[string] + check(dedent""" + >> x + >> continuation1 + continuation2 + + newParagraph""".toAst(warnings=warnings) == + dedent""" + rnInner + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=2 + rnLeaf 'x' + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'continuation1' + rnLeaf ' ' + rnLeaf 'continuation2' + rnParagraph + rnLeaf 'newParagraph' + """) + check(warnings[] == @[ + "input(2, 1) Warning: RST style: two or more quoted lines " & + "are followed by unquoted line 3"]) + + test "Markdown quoted blocks: not lazy": + # here is where we deviate from CommonMark specification: 'bar' below is + # not considered as continuation of 2-level '>> foo' quote. + check(dedent""" + >>> foo + > bar + >> baz + """.toAst() == + dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=3 + rnLeaf 'foo' + rnMarkdownBlockQuoteItem quotationDepth=1 + rnLeaf 'bar' + rnMarkdownBlockQuoteItem quotationDepth=2 + rnLeaf 'baz' + """) + + + test "Markdown quoted blocks: inline markup works": + check(dedent""" + > hi **bold** text + """.toAst == dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'hi' + rnLeaf ' ' + rnStrongEmphasis + rnLeaf 'bold' + rnLeaf ' ' + rnLeaf 'text' + """) + + test "Markdown quoted blocks: blank line separator": + let expected = dedent""" + rnInner + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'x' + rnLeaf ' ' + rnLeaf 'y' + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnLeaf 'z' + rnLeaf ' ' + rnLeaf 't' + """ + check(dedent""" + >x + >y + + > z + > t""".toAst == expected) + + check(dedent""" + >x + y + + > z + t""".toAst == expected) + + test "Markdown quoted blocks: nested body blocks/elements work #1": + let expected = dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnBulletList + rnBulletItem + rnInner + rnLeaf 'x' + rnBulletItem + rnInner + rnLeaf 'y' + """ + + check(dedent""" + > - x + - y + """.toAst == expected) + + # TODO: if bug #17340 point 28 is resolved then this may work: + # check(dedent""" + # > - x + # - y + # """.toAst == expected) + + check(dedent""" + > - x + > - y + """.toAst == expected) + + check(dedent""" + > + > - x + > + > - y + > + """.toAst == expected) + + test "Markdown quoted blocks: nested body blocks/elements work #2": + let expected = dedent""" + rnAdmonition adType=note + [nil] + [nil] + rnDefList + rnDefItem + rnDefName + rnLeaf 'deflist' + rnLeaf ':' + rnDefBody + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=2 + rnInner + rnLeaf 'quote' + rnLeaf ' ' + rnLeaf 'continuation' + """ + + check(dedent""" + .. Note:: deflist: + >> quote + continuation + """.toAst == expected) + + check(dedent""" + .. Note:: + deflist: + >> quote + continuation + """.toAst == expected) + + check(dedent""" + .. Note:: + deflist: + >> quote + >> continuation + """.toAst == expected) + + # spaces are not significant between `>`: + check(dedent""" + .. Note:: + deflist: + > > quote + > > continuation + """.toAst == expected) + + test "Markdown quoted blocks: de-indent handled well": + check(dedent""" + > + > - x + > - y + > + > Paragraph. + """.toAst == dedent""" + rnMarkdownBlockQuote + rnMarkdownBlockQuoteItem quotationDepth=1 + rnInner + rnBlockQuote + rnBulletList + rnBulletItem + rnInner + rnLeaf 'x' + rnBulletItem + rnInner + rnLeaf 'y' + rnParagraph + rnLeaf 'Paragraph' + rnLeaf '.' + """) + test "option list has priority over definition list": check(dedent""" --defusages From 5933aece9b5f7320e8feb2ef610b59d347c74d27 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Tue, 23 Nov 2021 18:30:17 +0300 Subject: [PATCH 0553/1654] `caseStmtMacros` no longer experimental, experimental manual refactor (#19173) * `caseStmtMacros` no longer experimental, experimental manual refactor * Update doc/manual.rst * apply review suggestions * apply review Co-authored-by: Andreas Rumpf --- changelog.md | 2 + compiler/options.nim | 2 +- compiler/semstmts.nim | 8 +- doc/manual.rst | 409 ++++++++--- doc/manual_experimental.rst | 1180 +++++++++++++------------------ lib/core/macros.nim | 7 +- tests/macros/tcasestmtmacro.nim | 2 - 7 files changed, 814 insertions(+), 796 deletions(-) diff --git a/changelog.md b/changelog.md index 8e70513e0..b645bc5c7 100644 --- a/changelog.md +++ b/changelog.md @@ -43,6 +43,8 @@ x, y, z: int Baz = object ``` +- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. ## Compiler changes diff --git a/compiler/options.nim b/compiler/options.nim index 2feaa2d43..d8e5057f9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -195,7 +195,7 @@ type notnil, dynamicBindSym, forLoopMacros, # not experimental anymore; remains here for backwards compatibility - caseStmtMacros, + caseStmtMacros, # ditto codeReordering, compiletimeFFI, ## This requires building nim with `-d:nimHasLibFFI` diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 36d76608e..f302dd4c3 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -987,10 +987,10 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode = else: popCaseContext(c) closeScope(c) - if caseStmtMacros in c.features: - result = handleCaseStmtMacro(c, n, flags) - if result != nil: - return result + #if caseStmtMacros in c.features: + result = handleCaseStmtMacro(c, n, flags) + if result != nil: + return result localError(c.config, n[0].info, errSelectorMustBeOfCertainTypes) return for i in 1..`_. +See also `custom numeric literals <#custom-numeric-literals>`_. -Numeric Literals +Numeric literals ---------------- Numeric literals have the form:: @@ -625,7 +625,7 @@ Hence: 0b10000000'u8 == 0x80'u8 == 128, but, 0b10000000'i8 == 0x80'i8 == -1 instead of causing an overflow error. -Custom Numeric Literals +Custom numeric literals ~~~~~~~~~~~~~~~~~~~~~~~ If the suffix is not predefined, then the suffix is assumed to be a call @@ -700,26 +700,6 @@ contain a dot: `{..}` are the three tokens `{`:tok:, `..`:tok:, `}`:tok: and not the two tokens `{.`:tok:, `.}`:tok:. -Unicode Operators ------------------ - -Under the `--experimental:unicodeOperators` switch these Unicode operators are -also parsed as operators:: - - ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ # same priority as * (multiplication) - ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ # same priority as + (addition) - - -If enabled, Unicode operators can be combined with non-Unicode operator -symbols. The usual precedence extensions then apply, for example, `⊠=` is an -assignment like operator just like `*=` is. - -No Unicode normalization step is performed. - -**Note**: Due to parser limitations one **cannot** enable this feature via a -pragma `{.experimental: "unicodeOperators".}` reliably. - - Syntax ====== @@ -1323,46 +1303,6 @@ as `MyEnum.value`: To implement bit fields with enums see `Bit fields <#set-type-bit-fields>`_ -Overloadable enum field names ------------------------------ - -To be enabled via `{.experimental: "overloadableEnums".}`. - -Enum field names are overloadable much like routines. When an overloaded -enum field is used, it produces a closed sym choice construct, here -written as `(E|E)`. -During overload resolution the right `E` is picked, if possible. -For (array/object...) constructors the right `E` is picked, comparable to -how `[byte(1), 2, 3]` works, one needs to use `[T.E, E2, E3]`. Ambiguous -enum fields produce a static error: - -.. code-block:: nim - :test: "nim c $1" - - {.experimental: "overloadableEnums".} - - type - E1 = enum - value1, - value2 - E2 = enum - value1, - value2 = 4 - - const - Lookuptable = [ - E1.value1: "1", - value2: "2" - ] - - proc p(e: E1) = - # disambiguation in 'case' statements: - case e - of value1: echo "A" - of value2: echo "B" - - p value2 - String type ----------- @@ -1684,11 +1624,6 @@ must match the order of the tuple's definition. Different tuple-types are *equivalent* if they specify the same fields of the same type in the same order. The *names* of the fields also have to be the same. -The assignment operator for tuples copies each component. -The default assignment operator for objects copies each component. Overloading -of the assignment operator is described `here -`_. - .. code-block:: nim type @@ -1765,6 +1700,10 @@ introduce new object roots apart from `system.RootObj`. Student = ref object of Person # Error: inheritance only works with non-final objects id: int +The assignment operator for tuples and objects copies each component. +The methods to override this copying behavior are described `here +`_. + Object construction ------------------- @@ -2685,6 +2624,7 @@ Varargs matching See `Varargs <#types-varargs>`_. + iterable -------- @@ -2725,6 +2665,24 @@ available. Let `p` be an overloaded symbol. These contexts are: As usual, ambiguous matches produce a compile-time error. +Named argument overloading +-------------------------- + +Routines with the same type signature can be called individually if +a parameter has different names between them. + +.. code-block:: Nim + proc foo(x: int) = + echo "Using x: ", x + proc foo(y: int) = + echo "Using y: ", y + + foo(x = 2) # Using x: 2 + foo(y = 2) # Using y: 2 + +Not supplying the parameter name in such cases results in an +ambiguity error. + Statements and expressions ========================== @@ -3200,7 +3158,7 @@ Return statement Example: .. code-block:: nim - return 40+2 + return 40 + 2 The `return` statement ends the execution of the current procedure. It is only allowed in procedures. If there is an `expr`, this is syntactic @@ -3836,8 +3794,8 @@ behavior inside loop bodies. See `closureScope `_ and `capture `_ for details on how to change this behavior. -Anonymous Procs ---------------- +Anonymous procedures +-------------------- Unnamed procedures can be used as lambda expressions to pass into other procedures: @@ -3845,8 +3803,8 @@ procedures: .. code-block:: nim var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] - cities.sort(proc (x,y: string): int = - cmp(x.len, y.len)) + cities.sort(proc (x, y: string): int = + cmp(x.len, y.len)) Procs as expressions can appear both as nested procs and inside top-level @@ -3854,6 +3812,42 @@ executable code. The `sugar `_ module contains the `=>` macro which enables a more succinct syntax for anonymous procedures resembling lambdas as they are in languages like JavaScript, C#, etc. +Do notation +----------- + +As a special convenience notation that keeps most elements of a +regular proc expression, the `do` keyword can be used to pass +anonymous procedures to routines: + +.. code-block:: nim + var cities = @["Frankfurt", "Tokyo", "New York", "Kyiv"] + + sort(cities) do (x, y: string) -> int: + cmp(x.len, y.len) + + # Less parentheses using the method plus command syntax: + cities = cities.map do (x: string) -> string: + "City of " & x + +`do` is written after the parentheses enclosing the regular proc params. +The proc expression represented by the `do` block is appended to the routine +call as the last argument. In calls using the command syntax, the `do` block +will bind to the immediately preceding expression rather than the command call. + +`do` with a parameter list corresponds to an anonymous `proc`, however +`do` without parameters is treated as a normal statement list. This allows +macros to receive both indented statement lists as an argument in inline +calls, as well as a direct mirror of Nim's routine syntax. + +.. code-block:: nim + # Passing a statement list to an inline macro: + macroResults.add quote do: + if not `ex`: + echo `info`, ": Check failed: ", `expString` + + # Processing a routine definition in a macro: + rpc(router, "add") do (a, b: int) -> int: + result = a + b Func ---- @@ -5133,7 +5127,7 @@ code: deletedKeys: seq[bool] -Type Classes +Type classes ------------ A type class is a special pseudo-type that can be used to match against @@ -5863,7 +5857,15 @@ twice: While macros enable advanced compile-time code transformations, they cannot change Nim's syntax. -Debug Example +**Style note:** For code readability, it is best to use the least powerful +programming construct that remains expressive. So the "check list" is: + +(1) Use an ordinary proc/iterator, if possible. +(2) Else: Use a generic proc/iterator, if possible. +(3) Else: Use a template, if possible. +(4) Else: Use a macro. + +Debug example ------------- The following example implements a powerful `debug` command that accepts a @@ -5921,7 +5923,7 @@ constructor expression. This is why `debug` iterates over all of `args`'s children. -BindSym +bindSym ------- The above `debug` macro relies on the fact that `write`, `writeLine` and @@ -5970,43 +5972,38 @@ However, the symbols `write`, `writeLine` and `stdout` are already bound and are not looked up again. As the example shows, `bindSym` does work with overloaded symbols implicitly. -Case-Of Macro -------------- +Note that the symbol names passed to `bindSym` have to be constant. The +experimental feature `dynamicBindSym` (`experimental manual +`_) +allows this value to be computed dynamically. -In Nim, it is possible to have a macro with the syntax of a *case-of* -expression just with the difference that all *of-branches* are passed to -and processed by the macro implementation. It is then up the macro -implementation to transform the *of-branches* into a valid Nim -statement. The following example should show how this feature could be -used for a lexical analyzer. +Post-statement blocks +--------------------- + +Macros can receive `of`, `elif`, `else`, `except`, `finally` and `do` +blocks (including their different forms such as `do` with routine parameters) +as arguments if called in statement form. .. code-block:: nim - import std/macros + macro performWithUndo(task, undo: untyped) = ... - macro case_token(args: varargs[untyped]): untyped = - echo args.treeRepr - # creates a lexical analyzer from regular expressions - # ... (implementation is an exercise for the reader ;-) - discard - - case_token: # this colon tells the parser it is a macro statement - of r"[A-Za-z_]+[A-Za-z_0-9]*": - return tkIdentifier - of r"0-9+": - return tkInteger - of r"[\+\-\*\?]+": - return tkOperator + performWithUndo do: + # multiple-line block of code + # to perform the task + do: + # code to undo it + + let num = 12 + # a single colon may be used if there is no initial block + match (num mod 3, num mod 5): + of (0, 0): + echo "FizzBuzz" + of (0, _): + echo "Fizz" + of (_, 0): + echo "Buzz" else: - return tkUnknown - - -**Style note**: For code readability, it is best to use the least powerful -programming construct that still suffices. So the "check list" is: - -(1) Use an ordinary proc/iterator, if possible. -(2) Else: Use a generic proc/iterator, if possible. -(3) Else: Use a template, if possible. -(4) Else: Use a macro. + echo num For loop macro @@ -6072,6 +6069,48 @@ Another example: echo a, " ", b +Case statement macros +--------------------- + +Macros named `` `case` `` can provide implementations of `case` statements +for certain types. The following is an example of such an implementation +for tuples, leveraging the existing equality operator for tuples +(as provided in `system.==`): + +.. code-block:: nim + :test: "nim c $1" + import std/macros + + macro `case`(n: tuple): untyped = + result = newTree(nnkIfStmt) + let selector = n[0] + for i in 1 ..< n.len: + let it = n[i] + case it.kind + of nnkElse, nnkElifBranch, nnkElifExpr, nnkElseExpr: + result.add it + of nnkOfBranch: + for j in 0..it.len-2: + let cond = newCall("==", selector, it[j]) + result.add newTree(nnkElifBranch, cond, it[^1]) + else: + error "custom 'case' for tuple cannot handle this node", it + + case ("foo", 78) + of ("foo", 78): echo "yes" + of ("bar", 88): echo "no" + else: discard + +`case` macros are subject to overload resolution. The type of the +`case` statement's selector expression is matched against the type +of the first argument of the `case` macro. Then the complete `case` +statement is passed in place of the argument and the macro is evaluated. + +In other words, the macro needs to transform the full `case` statement +but only the statement's selector expression is used to determine which +macro to call. + + Special Types ============= @@ -6959,7 +6998,8 @@ experimental pragma The `experimental` pragma enables experimental language features. Depending on the concrete feature, this means that the feature is either considered too unstable for an otherwise stable release or that the future of the feature -is uncertain (it may be removed at any time). +is uncertain (it may be removed at any time). See the +`experimental manual `_ for more details. Example: @@ -7060,6 +7100,21 @@ alignment requirement of the type are ignored. This pragma has no effect on the JS backend. +Noalias pragma +============== + +Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables +and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that +the underlying pointer is pointing to a unique location in memory, no other aliases to +this location exist. It is *unchecked* that this alias restriction is followed. If the +restriction is violated, the backend optimizer is free to miscompile the code. +This is an **unsafe** language feature. + +Ideally in later versions of the language, the restriction will be enforced at +compile time. (This is also why the name `noalias` was choosen instead of a more +verbose name like `unsafeAssumeNoAlias`.) + + Volatile pragma --------------- The `volatile` pragma is for variables only. It declares the variable as @@ -7641,7 +7696,7 @@ Example: {.pragma: rtl, importc, dynlib: "client.dll", cdecl.} proc p*(a, b: int): int {.rtl.} = - result = a+b + result = a + b In the example, a new pragma named `rtl` is introduced that either imports a symbol from a dynamic library or exports the symbol for dynamic library @@ -8032,3 +8087,137 @@ Threads and exceptions The interaction between threads and exceptions is simple: A *handled* exception in one thread cannot affect any other thread. However, an *unhandled* exception in one thread terminates the whole *process*. + + +Guards and locks +================ + +Nim provides common low level concurrency mechanisms like locks, atomic +intrinsics or condition variables. + +Nim significantly improves on the safety of these features via additional +pragmas: + +1) A `guard`:idx: annotation is introduced to prevent data races. +2) Every access of a guarded memory location needs to happen in an + appropriate `locks`:idx: statement. + + +Guards and locks sections +------------------------- + +Protecting global variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Object fields and global variables can be annotated via a `guard` pragma: + +.. code-block:: nim + + var glock: TLock + var gdata {.guard: glock.}: int + +The compiler then ensures that every access of `gdata` is within a `locks` +section: + +.. code-block:: nim + + proc invalid = + # invalid: unguarded access: + echo gdata + + proc valid = + # valid access: + {.locks: [glock].}: + echo gdata + +Top level accesses to `gdata` are always allowed so that it can be initialized +conveniently. It is *assumed* (but not enforced) that every top level statement +is executed before any concurrent action happens. + +The `locks` section deliberately looks ugly because it has no runtime +semantics and should not be used directly! It should only be used in templates +that also implement some form of locking at runtime: + +.. code-block:: nim + + template lock(a: TLock; body: untyped) = + pthread_mutex_lock(a) + {.locks: [a].}: + try: + body + finally: + pthread_mutex_unlock(a) + + +The guard does not need to be of any particular type. It is flexible enough to +model low level lockfree mechanisms: + +.. code-block:: nim + + var dummyLock {.compileTime.}: int + var atomicCounter {.guard: dummyLock.}: int + + template atomicRead(x): untyped = + {.locks: [dummyLock].}: + memoryReadBarrier() + x + + echo atomicRead(atomicCounter) + + +The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]` +in order to support *multi lock* statements. Why these are essential is +explained in the `lock levels <#guards-and-locks-lock-levels>`_ section. + + +Protecting general locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `guard` annotation can also be used to protect fields within an object. +The guard then needs to be another field within the same object or a +global variable. + +Since objects can reside on the heap or on the stack, this greatly enhances +the expressivity of the language: + +.. code-block:: nim + + type + ProtectedCounter = object + v {.guard: L.}: int + L: TLock + + proc incCounters(counters: var openArray[ProtectedCounter]) = + for i in 0..counters.high: + lock counters[i].L: + inc counters[i].v + +The access to field `x.v` is allowed since its guard `x.L` is active. +After template expansion, this amounts to: + +.. code-block:: nim + + proc incCounters(counters: var openArray[ProtectedCounter]) = + for i in 0..counters.high: + pthread_mutex_lock(counters[i].L) + {.locks: [counters[i].L].}: + try: + inc counters[i].v + finally: + pthread_mutex_unlock(counters[i].L) + +There is an analysis that checks that `counters[i].L` is the lock that +corresponds to the protected location `counters[i].v`. This analysis is called +`path analysis`:idx: because it deals with paths to locations +like `obj.field[i].fieldB[j]`. + +The path analysis is **currently unsound**, but that doesn't make it useless. +Two paths are considered equivalent if they are syntactically the same. + +This means the following compiles (for now) even though it really should not: + +.. code-block:: nim + + {.locks: [a[i].L].}: + inc i + access a[i].v diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 458512292..4ed8439df 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -19,47 +19,8 @@ Some of these are not covered by the `.experimental` pragma or one may want to use Nim libraries using these features without using them oneself. -**Note**: Unless otherwise indicated, these features are not to be removed, -but refined and overhauled. - - -Package level objects -===================== - -Every Nim module resides in a (nimble) package. An object type can be attached -to the package it resides in. If that is done, the type can be referenced from -other modules as an `incomplete`:idx: object type. This feature allows to -break up recursive type dependencies across module boundaries. Incomplete -object types are always passed `byref` and can only be used in pointer like -contexts (`var/ref/ptr IncompleteObject`) in general since the compiler does -not yet know the size of the object. To complete an incomplete object -the `package` pragma has to be used. `package` implies `byref`. - -As long as a type `T` is incomplete, neither `sizeof(T)` nor runtime -type information for `T` is available. - - -Example: - -.. code-block:: nim - - # module A (in an arbitrary package) - type - Pack.SomeObject = object ## declare as incomplete object of package 'Pack' - Triple = object - a, b, c: ref SomeObject ## pointers to incomplete objects are allowed - - ## Incomplete objects can be used as parameters: - proc myproc(x: SomeObject) = discard - - -.. code-block:: nim - - # module B (in package "Pack") - type - SomeObject* {.package.} = object ## Use 'package' to complete the object - s, t: string - x, y: int +.. note:: Unless otherwise indicated, these features are not to be removed, + but refined and overhauled. Void type @@ -105,23 +66,106 @@ The `void` type is only valid for parameters and return types; other symbols cannot have the type `void`. -Automatic dereferencing -======================= +Unicode Operators +================= -Automatic dereferencing is performed for the first argument of a routine call. -This feature has to be enabled via `{.experimental: "implicitDeref".}`: +Under the `--experimental:unicodeOperators`:option: switch, +these Unicode operators are also parsed as operators:: + + ∙ ∘ × ★ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ # same priority as * (multiplication) + ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ # same priority as + (addition) + + +If enabled, Unicode operators can be combined with non-Unicode operator +symbols. The usual precedence extensions then apply, for example, `⊠=` is an +assignment like operator just like `*=` is. + +No Unicode normalization step is performed. + +.. note:: Due to parser limitations one **cannot** enable this feature via a + pragma `{.experimental: "unicodeOperators".}` reliably. + + +Overloadable enum value names +============================= + +Enabled via `{.experimental: "overloadableEnums".}`. + +Enum value names are overloadable, much like routines. If both of the enums +`T` and `U` have a member named `foo`, then the identifier `foo` corresponds +to a choice between `T.foo` and `U.foo`. During overload resolution, +the correct type of `foo` is decided from the context. If the type of `foo` is +ambiguous, a static error will be produced. + +.. code-block:: nim + :test: "nim c $1" + + {.experimental: "overloadableEnums".} + + type + E1 = enum + value1, + value2 + E2 = enum + value1, + value2 = 4 + + const + Lookuptable = [ + E1.value1: "1", + # no need to qualify value2, known to be E1.value2 + value2: "2" + ] + + proc p(e: E1) = + # disambiguation in 'case' statements: + case e + of value1: echo "A" + of value2: echo "B" + + p value2 + + +Package level objects +===================== + +Every Nim module resides in a (nimble) package. An object type can be attached +to the package it resides in. If that is done, the type can be referenced from +other modules as an `incomplete`:idx: object type. This feature allows to +break up recursive type dependencies across module boundaries. Incomplete +object types are always passed `byref` and can only be used in pointer like +contexts (`var/ref/ptr IncompleteObject`) in general, since the compiler does +not yet know the size of the object. To complete an incomplete object, +the `package` pragma has to be used. `package` implies `byref`. + +As long as a type `T` is incomplete, no runtime type information for `T` is +available. + + +Example: .. code-block:: nim - {.experimental: "implicitDeref".} + # module A (in an arbitrary package) + type + Pack.SomeObject = object # declare as incomplete object of package 'Pack' + Triple = object + a, b, c: ref SomeObject # pointers to incomplete objects are allowed - proc depth(x: NodeObj): int = ... + # Incomplete objects can be used as parameters: + proc myproc(x: SomeObject) = discard - var - n: Node - new(n) - echo n.depth - # no need to write n[].depth either + +.. code-block:: nim + + # module B (in package "Pack") + type + SomeObject* {.package.} = object # Use 'package' to complete the object + s, t: string + x, y: int + +This feature will likely be superseded in the future by support for +recursive module dependencies. Code reordering @@ -230,65 +274,30 @@ scope. Therefore, the following will *fail to compile:* a() - -Named argument overloading -========================== - -Routines with the same type signature can be called differently if a parameter -has different names. This does not need an `experimental` switch, but is an -unstable feature. - -.. code-block:: Nim - - proc foo(x: int) = - echo "Using x: ", x - proc foo(y: int) = - echo "Using y: ", y - - foo(x = 2) - # Using x: 2 - foo(y = 2) - # Using y: 2 +This feature will likely be replaced with a better solution to remove +the need for forward declarations. -Do notation -=========== +Automatic dereferencing +======================= -As a special more convenient notation, proc expressions involved in procedure -calls can use the `do` keyword: +Automatic dereferencing is performed for the first argument of a routine call. +This feature has to be enabled via `{.experimental: "implicitDeref".}`: .. code-block:: nim - sort(cities) do (x,y: string) -> int: - cmp(x.len, y.len) + {.experimental: "implicitDeref".} - # Less parenthesis using the method plus command syntax: - cities = cities.map do (x:string) -> string: - "City of " & x + type + NodeObj = object + # ... + Node = ref NodeObj - # In macros, the do notation is often used for quasi-quoting - macroResults.add quote do: - if not `ex`: - echo `info`, ": Check failed: ", `expString` + proc depth(x: NodeObj): int = ... -`do` is written after the parentheses enclosing the regular proc params. -The proc expression represented by the do block is appended to them. -In calls using the command syntax, the do block will bind to the immediately -preceding expression, transforming it in a call. - -`do` with parentheses is an anonymous `proc`; however a `do` without -parentheses is just a block of code. The `do` notation can be used to -pass multiple blocks to a macro: - -.. code-block:: nim - - macro performWithUndo(task, undo: untyped) = ... - - performWithUndo do: - # multiple-line block of code - # to perform the task - do: - # code to undo it + let n = Node() + echo n.depth + # no need to write n[].depth Special Operators @@ -297,8 +306,8 @@ Special Operators dot operators ------------- -**Note**: Dot operators are still experimental and so need to be enabled -via `{.experimental: "dotOperators".}`. +.. note:: Dot operators are still experimental and so need to be enabled + via `{.experimental: "dotOperators".}`. Nim offers a special family of dot operators that can be used to intercept and rewrite proc call and field access attempts, referring @@ -418,6 +427,314 @@ here. .. include:: manual_experimental_strictnotnil.rst +Aliasing restrictions in parameter passing +========================================== + +.. note:: The aliasing restrictions are currently not enforced by the + implementation and need to be fleshed out further. + +"Aliasing" here means that the underlying storage locations overlap in memory +at runtime. An "output parameter" is a parameter of type `var T`, +an input parameter is any parameter that is not of type `var`. + +1. Two output parameters should never be aliased. +2. An input and an output parameter should not be aliased. +3. An output parameter should never be aliased with a global or thread local + variable referenced by the called proc. +4. An input parameter should not be aliased with a global or thread local + variable updated by the called proc. + +One problem with rules 3 and 4 is that they affect specific global or thread +local variables, but Nim's effect tracking only tracks "uses no global variable" +via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule: + +5. A global or thread local variable (or a location derived from such a location) + can only passed to a parameter of a `.noSideEffect` proc. + + +Strict funcs +============ + +Since version 1.4, a stricter definition of "side effect" is available. +In addition to the existing rule that a side effect is calling a function +with side effects, the following rule is also enforced: + +Any mutation to an object does count as a side effect if that object is reachable +via a parameter that is not declared as a `var` parameter. + +For example: + +.. code-block:: nim + + {.experimental: "strictFuncs".} + + type + Node = ref object + le, ri: Node + data: string + + func len(n: Node): int = + # valid: len does not have side effects + var it = n + while it != nil: + inc result + it = it.ri + + func mut(n: Node) = + let m = n # is the statement that connected the mutation to the parameter + m.data = "yeah" # the mutation is here + # Error: 'mut' can have side effects + # an object reachable from 'n' is potentially mutated + + +The algorithm behind this analysis is described in +the `view types section <#view-types-algorithm>`_. + + +View types +========== + +.. tip:: `--experimental:views`:option: is more effective + with `--experimental:strictFuncs`:option:. + +A view type is a type that is or contains one of the following types: + +- `lent T` (view into `T`) +- `openArray[T]` (pair of (pointer to array of `T`, size)) + +For example: + +.. code-block:: nim + + type + View1 = openArray[byte] + View2 = lent string + View3 = Table[openArray[char], int] + + +Exceptions to this rule are types constructed via `ptr` or `proc`. +For example, the following types are **not** view types: + +.. code-block:: nim + + type + NotView1 = proc (x: openArray[int]) + NotView2 = ptr openArray[char] + NotView3 = ptr array[4, lent int] + + +The mutability aspect of a view type is not part of the type but part +of the locations it's derived from. More on this later. + +A *view* is a symbol (a let, var, const, etc.) that has a view type. + +Since version 1.4, Nim allows view types to be used as local variables. +This feature needs to be enabled via `{.experimental: "views".}`. + +A local variable of a view type *borrows* from the locations and +it is statically enforced that the view does not outlive the location +it was borrowed from. + +For example: + +.. code-block:: nim + + {.experimental: "views".} + + proc take(a: openArray[int]) = + echo a.len + + proc main(s: seq[int]) = + var x: openArray[int] = s # 'x' is a view into 's' + # it is checked that 'x' does not outlive 's' and + # that 's' is not mutated. + for i in 0 .. high(x): + echo x[i] + take(x) + + take(x.toOpenArray(0, 1)) # slicing remains possible + let y = x # create a view from a view + take y + # it is checked that 'y' does not outlive 'x' and + # that 'x' is not mutated as long as 'y' lives. + + + main(@[11, 22, 33]) + + +A local variable of a view type can borrow from a location +derived from a parameter, another local variable, a global `const` or `let` +symbol or a thread-local `var` or `let`. + +Let `p` the proc that is analysed for the correctness of the borrow operation. + +Let `source` be one of: + +- A formal parameter of `p`. Note that this does not cover parameters of + inner procs. +- The `result` symbol of `p`. +- A local `var` or `let` or `const` of `p`. Note that this does + not cover locals of inner procs. +- A thread-local `var` or `let`. +- A global `let` or `const`. +- A constant array/seq/object/tuple constructor. + + +Path expressions +---------------- + +A location derived from `source` is then defined as a path expression that +has `source` as the owner. A path expression `e` is defined recursively: + +- `source` itself is a path expression. +- Container access like `e[i]` is a path expression. +- Tuple access `e[0]` is a path expression. +- Object field access `e.field` is a path expression. +- `system.toOpenArray(e, ...)` is a path expression. +- Pointer dereference `e[]` is a path expression. +- An address `addr e`, `unsafeAddr e` is a path expression. +- A type conversion `T(e)` is a path expression. +- A cast expression `cast[T](e)` is a path expression. +- `f(e, ...)` is a path expression if `f`'s return type is a view type. + Because the view can only have been borrowed from `e`, we then know + that the owner of `f(e, ...)` is `e`. + + +If a view type is used as a return type, the location must borrow from a location +that is derived from the first parameter that is passed to the proc. +See `the manual `_ +for details about how this is done for `var T`. + +A mutable view can borrow from a mutable location, an immutable view can borrow +from both a mutable or an immutable location. + +If a view borrows from a mutable location, the view can be used to update the +location. Otherwise it cannot be used for mutations. + +The *duration* of a borrow is the span of commands beginning from the assignment +to the view and ending with the last usage of the view. + +For the duration of the borrow operation, no mutations to the borrowed locations +may be performed except via the view that borrowed from the +location. The borrowed location is said to be *sealed* during the borrow. + +.. code-block:: nim + + {.experimental: "views".} + + type + Obj = object + field: string + + proc dangerous(s: var seq[Obj]) = + let v: lent Obj = s[0] # seal 's' + s.setLen 0 # prevented at compile-time because 's' is sealed. + echo v.field + + +The scope of the view does not matter: + +.. code-block:: nim + + proc valid(s: var seq[Obj]) = + let v: lent Obj = s[0] # begin of borrow + echo v.field # end of borrow + s.setLen 0 # valid because 'v' isn't used afterwards + + +The analysis requires as much precision about mutations as is reasonably obtainable, +so it is more effective with the experimental `strict funcs <#strict-funcs>`_ +feature. In other words `--experimental:views`:option: works better +with `--experimental:strictFuncs`:option:. + +The analysis is currently control flow insensitive: + +.. code-block:: nim + + proc invalid(s: var seq[Obj]) = + let v: lent Obj = s[0] + if false: + s.setLen 0 + echo v.field + +In this example, the compiler assumes that `s.setLen 0` invalidates the +borrow operation of `v` even though a human being can easily see that it +will never do that at runtime. + + +Start of a borrow +----------------- + +A borrow starts with one of the following: + +- The assignment of a non-view-type to a view-type. +- The assignment of a location that is derived from a local parameter + to a view-type. + + +End of a borrow +--------------- + +A borrow operation ends with the last usage of the view variable. + + +Reborrows +--------- + +A view `v` can borrow from multiple different locations. However, the borrow +is always the full span of `v`'s lifetime and every location that is borrowed +from is sealed during `v`'s lifetime. + + +Algorithm +--------- + +The following section is an outline of the algorithm that the current implementation +uses. The algorithm performs two traversals over the AST of the procedure or global +section of code that uses a view variable. No fixpoint iterations are performed, the +complexity of the analysis is O(N) where N is the number of nodes of the AST. + +The first pass over the AST computes the lifetime of each local variable based on +a notion of an "abstract time", in the implementation it's a simple integer that is +incremented for every visited node. + +In the second pass, information about the underlying object "graphs" is computed. +Let `v` be a parameter or a local variable. Let `G(v)` be the graph +that `v` belongs to. A graph is defined by the set of variables that belong +to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only +be part of a single graph. + +Assignments like `a = b` "connect" two variables, both variables end up in the +same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is +much more complex than that and can involve multiple assignment targets +and sources:: + + f(x, y) = g(a, b) + +connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`. +A type based alias analysis rules out some of these combinations, for example +a `string` value cannot possibly be connected to a `seq[int]`. + +A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated. +After the second pass a set of disjoint graphs was computed. + +For strict functions it is then enforced that there is no graph that is both mutated +and has an element that is an immutable parameter (that is a parameter that is not +of type `var T`). + +For borrow checking, a different set of checks is performed. Let `v` be the view +and `b` the location that is borrowed from. + +- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of + a parameter is the complete proc body. +- If `v` is used for a mutation, `b` must be a mutable location too. +- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if + `v` is a mutable view). +- If `v` is `result` then `b` has to be a location derived from the first + formal parameter or from a constant location. +- A view cannot be used for a read or a write access before it was assigned to. + + Concepts ======== @@ -441,10 +758,10 @@ Concepts are written in the following form: for value in s: value is T -The concept is a match if: +The concept matches if: -a) all of the expressions within the body can be compiled for the tested type -b) all statically evaluable boolean expressions in the body must be true +a) all expressions within the body can be compiled for the tested type +b) all statically evaluable boolean expressions in the body are true The identifiers following the `concept` keyword represent instances of the currently matched type. You can apply any of the standard type modifiers such @@ -874,15 +1191,17 @@ There are 4 operations that are bound to a type: 3. Destruction 4. Deep copying for communication between threads -These operations can be *overridden* instead of *overloaded*. This means the -implementation is automatically lifted to structured types. For instance if type -`T` has an overridden assignment operator `=` this operator is also used -for assignments of the type `seq[T]`. Since these operations are bound to a -type they have to be bound to a nominal type for reasons of simplicity of -implementation: This means an overridden `deepCopy` for `ref T` is really -bound to `T` and not to `ref T`. This also means that one cannot override -`deepCopy` for both `ptr T` and `ref T` at the same time; instead a -helper distinct or object type has to be used for one pointer type. +These operations can be *overridden* instead of *overloaded*. This means that +the implementation is automatically lifted to structured types. For instance, +if the type `T` has an overridden assignment operator `=`, this operator is +also used for assignments of the type `seq[T]`. + +Since these operations are bound to a type, they have to be bound to a +nominal type for reasons of simplicity of implementation; this means an +overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`. +This also means that one cannot override `deepCopy` for both `ptr T` and +`ref T` at the same time, instead a distinct or object helper type has to be +used for one pointer type. Assignments, moves and destruction are specified in the `destructors `_ document. @@ -902,60 +1221,29 @@ The signature has to be: proc `=deepCopy`(x: T): T -This mechanism will be used by most data structures that support shared memory -like channels to implement thread safe automatic memory management. +This mechanism will be used by most data structures that support shared memory, +like channels, to implement thread safe automatic memory management. The builtin `deepCopy` can even clone closures and their environments. See the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. -Case statement macros -===================== +Dynamic arguments for bindSym +============================= -Macros named `case` can rewrite `case` statements for certain types in order to -implement `pattern matching`:idx:. The following example implements a -simplistic form of pattern matching for tuples, leveraging the existing -equality operator for tuples (as provided in `system.==`): +This experimental feature allows the symbol name argument of `macros.bindSym` +to be computed dynamically. .. code-block:: nim - :test: "nim c $1" + {.experimental: "dynamicBindSym".} - {.experimental: "caseStmtMacros".} + import macros - import std/macros + macro callOp(opName, arg1, arg2): untyped = + result = newCall(bindSym($opName), arg1, arg2) - macro `case`(n: tuple): untyped = - result = newTree(nnkIfStmt) - let selector = n[0] - for i in 1 ..< n.len: - let it = n[i] - case it.kind - of nnkElse, nnkElifBranch, nnkElifExpr, nnkElseExpr: - result.add it - of nnkOfBranch: - for j in 0..it.len-2: - let cond = newCall("==", selector, it[j]) - result.add newTree(nnkElifBranch, cond, it[^1]) - else: - error "custom 'case' for tuple cannot handle this node", it - - case ("foo", 78) - of ("foo", 78): echo "yes" - of ("bar", 88): echo "no" - else: discard - - -Currently case statement macros must be enabled explicitly -via `{.experimental: "caseStmtMacros".}`. - -`case` macros are subject to overload resolution. The type of the -`case` statement's selector expression is matched against the type -of the first argument of the `case` macro. Then the complete `case` -statement is passed in place of the argument and the macro is evaluated. - -In other words, the macro needs to transform the full `case` statement -but only the statement's selector expression is used to determine which -macro to call. + echo callOp("+", 1, 2) + echo callOp("-", 5, 4) Term rewriting macros @@ -968,30 +1256,30 @@ compilation pipeline with user defined optimizations: .. code-block:: nim - template optMul{`*`(a, 2)}(a: int): int = a+a + template optMul{`*`(a, 2)}(a: int): int = a + a let x = 3 echo x * 2 The compiler now rewrites `x * 2` as `x + x`. The code inside the -curlies is the pattern to match against. The operators `*`, `**`, +curly brackets is the pattern to match against. The operators `*`, `**`, `|`, `~` have a special meaning in patterns if they are written in infix notation, so to match verbatim against `*` the ordinary function call syntax needs to be used. -Term rewriting macro are applied recursively, up to a limit. This means that +Term rewriting macros are applied recursively, up to a limit. This means that if the result of a term rewriting macro is eligible for another rewriting, the compiler will try to perform it, and so on, until no more optimizations are applicable. To avoid putting the compiler into an infinite loop, there is a hard limit on how many times a single term rewriting macro can be applied. Once this limit has been passed, the term rewriting macro will be ignored. -Unfortunately optimizations are hard to get right and even the tiny example +Unfortunately optimizations are hard to get right and even this tiny example is **wrong**: .. code-block:: nim - template optMul{`*`(a, 2)}(a: int): int = a+a + template optMul{`*`(a, 2)}(a: int): int = a + a proc f(): int = echo "side effect!" @@ -1004,7 +1292,7 @@ Fortunately Nim supports side effect analysis: .. code-block:: nim - template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a + template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a + a proc f(): int = echo "side effect!" @@ -1022,13 +1310,13 @@ blindly: .. code-block:: nim - template mulIsCommutative{`*`(a, b)}(a, b: int): int = b*a + template mulIsCommutative{`*`(a, b)}(a, b: int): int = b * a What optimizers really need to do is a *canonicalization*: .. code-block:: nim - template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a + template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b * a The `int{lit}` parameter pattern matches against an expression of type `int`, but only if it's a literal. @@ -1143,7 +1431,7 @@ constant folding, so the following does not work: The reason is that the compiler already transformed the 1 into "1" for the `echo` statement. However, a term rewriting macro should not change the -semantics anyway. In fact they can be deactivated with the `--patterns:off`:option: +semantics anyway. In fact, they can be deactivated with the `--patterns:off`:option: command line option or temporarily with the `patterns` pragma. @@ -1155,7 +1443,7 @@ notation: .. code-block:: nim - template t{(0|1|2){x}}(x: untyped): untyped = x+1 + template t{(0|1|2){x}}(x: untyped): untyped = x + 1 let a = 1 # outputs 2: echo a @@ -1164,7 +1452,7 @@ notation: The `~` operator ~~~~~~~~~~~~~~~~~~ -The `~` operator is the **not** operator in patterns: +The `~` operator is the 'not' operator in patterns: .. code-block:: nim @@ -1257,14 +1545,14 @@ an `nnkArgList` node containing:: Sym "x" Sym "-" -(Which is the reverse polish notation of `x + y * z - x`.) +(This is the reverse polish notation of `x + y * z - x`.) Parameters ---------- Parameters in a pattern are type checked in the matching process. If a -parameter is of the type `varargs` it is treated specially and it can match +parameter is of the type `varargs`, it is treated specially and can match 0 or more arguments in the AST to be matched against: .. code-block:: nim @@ -1276,6 +1564,27 @@ parameter is of the type `varargs` it is treated specially and it can match w(f, x, y) +noRewrite pragma +---------------- + +Term rewriting macros and templates are currently greedy and +they will rewrite as long as there is a match. +There was no way to ensure some rewrite happens only once, +e.g. when rewriting term to same term plus extra content. + +`noRewrite` pragma can actually prevent further rewriting on marked code, +e.g. with given example `echo("ab")` will be rewritten just once: + +.. code-block:: nim + + template pwnEcho{echo(x)}(x: untyped) = + {.noRewrite.}: echo("pwned!") + + echo "ab" + +`noRewrite` pragma can be useful to control term-rewriting macros recursion. + + Example: Partial evaluation --------------------------- @@ -1319,7 +1628,7 @@ AST based overloading ===================== Parameter constraints can also be used for ordinary routine parameters; these -constraints affect ordinary overloading resolution then: +constraints then affect ordinary overloading resolution: .. code-block:: nim @@ -1356,22 +1665,25 @@ module to work. Somewhat confusingly, `spawn` is also used in the `parallel` statement with slightly different semantics. `spawn` always takes a call expression of -the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void` -then `spawn`'s return type is also `void` otherwise it is `FlowVar[T]`. +the form `f(a, ...)`. Let `T` be `f`'s return type. If `T` is `void`, +then `spawn`'s return type is also `void`, otherwise it is `FlowVar[T]`. -Within a `parallel` section sometimes the `FlowVar[T]` is eliminated +Within a `parallel` section, the `FlowVar[T]` is sometimes eliminated to `T`. This happens when `T` does not contain any GC'ed memory. The compiler can ensure the location in `location = spawn f(...)` is not read prematurely within a `parallel` section and so there is no need for the overhead of an indirection via `FlowVar[T]` to ensure correctness. -**Note**: Currently exceptions are not propagated between `spawn`'ed tasks! +.. note:: Currently exceptions are not propagated between `spawn`'ed tasks! + +This feature is likely to be removed in the future as external packages +can have better solutions. Spawn statement --------------- -`spawn`:idx: can be used to pass a task to the thread pool: +The `spawn`:idx: statement can be used to pass a task to the thread pool: .. code-block:: nim @@ -1393,10 +1705,10 @@ that `spawn` takes is restricted: * `f`'s parameters may not be of type `var`. This means one has to use raw `ptr`'s for data passing reminding the programmer to be careful. -* `ref` parameters are deeply copied which is a subtle semantic change and - can cause performance problems but ensures memory safety. This deep copy - is performed via `system.deepCopy` and so can be overridden. -* For *safe* data exchange between `f` and the caller a global `TChannel` +* `ref` parameters are deeply copied, which is a subtle semantic change and + can cause performance problems, but ensures memory safety. This deep copy + is performed via `system.deepCopy`, so it can be overridden. +* For *safe* data exchange between `f` and the caller, a global `Channel` needs to be used. However, since spawn can return a result, often no further communication is required. @@ -1420,10 +1732,10 @@ wait on multiple flow variables at the same time: responses.del(index) discard blockUntilAny(responses) -Data flow variables ensure that no data races -are possible. Due to technical limitations not every type `T` is possible in -a data flow variable: `T` has to be of the type `ref`, `string`, `seq` -or of a type that doesn't contain a type that is garbage collected. This +Data flow variables ensure that no data races are possible. Due to +technical limitations, not every type `T` can be used in +a data flow variable: `T` has to be a `ref`, `string`, `seq` +or of a type that doesn't contain any GC'd type. This restriction is not hard to work-around in practice. @@ -1436,14 +1748,14 @@ Example: .. code-block:: nim :test: "nim c --threads:on $1" - # Compute PI in an inefficient way + # Compute pi in an inefficient way import std/[strutils, math, threadpool] {.experimental: "parallel".} proc term(k: float): float = 4 * math.pow(-1, k) / (2*k + 1) proc pi(n: int): float = - var ch = newSeq[float](n+1) + var ch = newSeq[float](n + 1) parallel: for k in 0..ch.high: ch[k] = spawn term(float(k)) @@ -1454,16 +1766,16 @@ Example: The parallel statement is the preferred mechanism to introduce parallelism in a -Nim program. A subset of the Nim language is valid within a `parallel` +Nim program. Only a subset of the Nim language is valid within a `parallel` section. This subset is checked during semantic analysis to be free of data races. A sophisticated `disjoint checker`:idx: ensures that no data races are -possible even though shared memory is extensively supported! +possible, even though shared memory is extensively supported! The subset is in fact the full language with the following restrictions / changes: * `spawn` within a `parallel` section has special semantics. -* Every location of the form `a[i]` and `a[i..j]` and `dest` where +* Every location of the form `a[i]`, `a[i..j]` and `dest` where `dest` is part of the pattern `dest = spawn f(...)` has to be provably disjoint. This is called the *disjoint check*. * Every other complex location `loc` that is used in a spawned @@ -1477,152 +1789,16 @@ restrictions / changes: yet performed for ordinary slices outside of a `parallel` section. -Guards and locks -================ - -Apart from `spawn` and `parallel` Nim also provides all the common low level -concurrency mechanisms like locks, atomic intrinsics or condition variables. - -Nim significantly improves on the safety of these features via additional -pragmas: - -1) A `guard`:idx: annotation is introduced to prevent data races. -2) Every access of a guarded memory location needs to happen in an - appropriate `locks`:idx: statement. -3) Locks and routines can be annotated with `lock levels`:idx: to allow - potential deadlocks to be detected during semantic analysis. - - -Guards and the locks section ----------------------------- - -Protecting global variables -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Object fields and global variables can be annotated via a `guard` pragma: - -.. code-block:: nim - - var glock: TLock - var gdata {.guard: glock.}: int - -The compiler then ensures that every access of `gdata` is within a `locks` -section: - -.. code-block:: nim - - proc invalid = - # invalid: unguarded access: - echo gdata - - proc valid = - # valid access: - {.locks: [glock].}: - echo gdata - -Top level accesses to `gdata` are always allowed so that it can be initialized -conveniently. It is *assumed* (but not enforced) that every top level statement -is executed before any concurrent action happens. - -The `locks` section deliberately looks ugly because it has no runtime -semantics and should not be used directly! It should only be used in templates -that also implement some form of locking at runtime: - -.. code-block:: nim - - template lock(a: TLock; body: untyped) = - pthread_mutex_lock(a) - {.locks: [a].}: - try: - body - finally: - pthread_mutex_unlock(a) - - -The guard does not need to be of any particular type. It is flexible enough to -model low level lockfree mechanisms: - -.. code-block:: nim - - var dummyLock {.compileTime.}: int - var atomicCounter {.guard: dummyLock.}: int - - template atomicRead(x): untyped = - {.locks: [dummyLock].}: - memoryReadBarrier() - x - - echo atomicRead(atomicCounter) - - -The `locks` pragma takes a list of lock expressions `locks: [a, b, ...]` -in order to support *multi lock* statements. Why these are essential is -explained in the `lock levels <#guards-and-locks-lock-levels>`_ section. - - -Protecting general locations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The `guard` annotation can also be used to protect fields within an object. -The guard then needs to be another field within the same object or a -global variable. - -Since objects can reside on the heap or on the stack this greatly enhances the -expressivity of the language: - -.. code-block:: nim - - type - ProtectedCounter = object - v {.guard: L.}: int - L: TLock - - proc incCounters(counters: var openArray[ProtectedCounter]) = - for i in 0..counters.high: - lock counters[i].L: - inc counters[i].v - -The access to field `x.v` is allowed since its guard `x.L` is active. -After template expansion, this amounts to: - -.. code-block:: nim - - proc incCounters(counters: var openArray[ProtectedCounter]) = - for i in 0..counters.high: - pthread_mutex_lock(counters[i].L) - {.locks: [counters[i].L].}: - try: - inc counters[i].v - finally: - pthread_mutex_unlock(counters[i].L) - -There is an analysis that checks that `counters[i].L` is the lock that -corresponds to the protected location `counters[i].v`. This analysis is called -`path analysis`:idx: because it deals with paths to locations -like `obj.field[i].fieldB[j]`. - -The path analysis is **currently unsound**, but that doesn't make it useless. -Two paths are considered equivalent if they are syntactically the same. - -This means the following compiles (for now) even though it really should not: - -.. code-block:: nim - - {.locks: [a[i].L].}: - inc i - access a[i].v - - Lock levels ------------ +=========== Lock levels are used to enforce a global locking order in order to detect potential deadlocks during semantic analysis. A lock level is an constant integer in the range 0..1_000. Lock level 0 means that no lock is acquired at all. -If a section of code holds a lock of level `M` than it can also acquire any +If a section of code holds a lock of level `M`, it can also acquire any lock of level `N < M`. Another lock of level `M` cannot be acquired. Locks of the same level can only be acquired *at the same time* within a single `locks` section: @@ -1685,7 +1861,7 @@ This is essential so that procs can be called within a `locks` section: p() -As usual `locks` is an inferred effect and there is a subtype +As usual, `locks` is an inferred effect and there is a subtype relation: `proc () {.locks: N.}` is a subtype of `proc () {.locks: M.}` iff (M <= N). @@ -1709,346 +1885,4 @@ having unknown lock level as well: if g.memberProc != nil: g.memberProc() - -noRewrite pragma ----------------- - -Term rewriting macros and templates are currently greedy and -they will rewrite as long as there is a match. -There was no way to ensure some rewrite happens only once, -e.g. when rewriting term to same term plus extra content. - -`noRewrite` pragma can actually prevent further rewriting on marked code, -e.g. with given example `echo("ab")` will be rewritten just once: - -.. code-block:: nim - - template pwnEcho{echo(x)}(x: untyped) = - {.noRewrite.}: echo("pwned!") - - echo "ab" - -`noRewrite` pragma can be useful to control term-rewriting macros recursion. - - -Aliasing restrictions in parameter passing -========================================== - -**Note**: The aliasing restrictions are currently not enforced by the -implementation and need to be fleshed out further. - -"Aliasing" here means that the underlying storage locations overlap in memory -at runtime. An "output parameter" is a parameter of type `var T`, -an input parameter is any parameter that is not of type `var`. - -1. Two output parameters should never be aliased. -2. An input and an output parameter should not be aliased. -3. An output parameter should never be aliased with a global or thread local - variable referenced by the called proc. -4. An input parameter should not be aliased with a global or thread local - variable updated by the called proc. - -One problem with rules 3 and 4 is that they affect specific global or thread -local variables, but Nim's effect tracking only tracks "uses no global variable" -via `.noSideEffect`. The rules 3 and 4 can also be approximated by a different rule: - -5. A global or thread local variable (or a location derived from such a location) - can only passed to a parameter of a `.noSideEffect` proc. - - -Noalias annotation -================== - -Since version 1.4 of the Nim compiler, there is a `.noalias` annotation for variables -and parameters. It is mapped directly to C/C++'s `restrict`:c: keyword and means that -the underlying pointer is pointing to a unique location in memory, no other aliases to -this location exist. It is *unchecked* that this alias restriction is followed, if the -restriction is violated, the backend optimizer is free to miscompile the code. -This is an **unsafe** language feature. - -Ideally in later versions of the language, the restriction will be enforced at -compile time. (Which is also why the name `noalias` was choosen instead of a more -verbose name like `unsafeAssumeNoAlias`.) - - -Strict funcs -============ - -Since version 1.4 a stricter definition of "side effect" is available. In addition -to the existing rule that a side effect is calling a function with side effects -the following rule is also enforced: - -Any mutation to an object does count as a side effect if that object is reachable -via a parameter that is not declared as a `var` parameter. - -For example: - -.. code-block:: nim - - {.experimental: "strictFuncs".} - - type - Node = ref object - le, ri: Node - data: string - - func len(n: Node): int = - # valid: len does not have side effects - var it = n - while it != nil: - inc result - it = it.ri - - func mut(n: Node) = - let m = n # is the statement that connected the mutation to the parameter - m.data = "yeah" # the mutation is here - # Error: 'mut' can have side effects - # an object reachable from 'n' is potentially mutated - - -The algorithm behind this analysis is described in -the `view types section <#view-types-algorithm>`_. - - -View types -========== - -**Note**: `--experimental:views`:option: is more effective -with `--experimental:strictFuncs`:option:. - -A view type is a type that is or contains one of the following types: - -- `lent T` (view into `T`) -- `openArray[T]` (pair of (pointer to array of `T`, size)) - -For example: - -.. code-block:: nim - - type - View1 = openArray[byte] - View2 = lent string - View3 = Table[openArray[char], int] - - -Exceptions to this rule are types constructed via `ptr` or `proc`. -For example, the following types are **not** view types: - -.. code-block:: nim - - type - NotView1 = proc (x: openArray[int]) - NotView2 = ptr openArray[char] - NotView3 = ptr array[4, lent int] - - -The mutability aspect of a view type is not part of the type but part -of the locations it's derived from. More on this later. - -A *view* is a symbol (a let, var, const, etc.) that has a view type. - -Since version 1.4 Nim allows view types to be used as local variables. -This feature needs to be enabled via `{.experimental: "views".}`. - -A local variable of a view type *borrows* from the locations and -it is statically enforced that the view does not outlive the location -it was borrowed from. - -For example: - -.. code-block:: nim - - {.experimental: "views".} - - proc take(a: openArray[int]) = - echo a.len - - proc main(s: seq[int]) = - var x: openArray[int] = s # 'x' is a view into 's' - # it is checked that 'x' does not outlive 's' and - # that 's' is not mutated. - for i in 0 .. high(x): - echo x[i] - take(x) - - take(x.toOpenArray(0, 1)) # slicing remains possible - let y = x # create a view from a view - take y - # it is checked that 'y' does not outlive 'x' and - # that 'x' is not mutated as long as 'y' lives. - - - main(@[11, 22, 33]) - - -A local variable of a view type can borrow from a location -derived from a parameter, another local variable, a global `const` or `let` -symbol or a thread-local `var` or `let`. - -Let `p` the proc that is analysed for the correctness of the borrow operation. - -Let `source` be one of: - -- A formal parameter of `p`. Note that this does not cover parameters of - inner procs. -- The `result` symbol of `p`. -- A local `var` or `let` or `const` of `p`. Note that this does - not cover locals of inner procs. -- A thread-local `var` or `let`. -- A global `let` or `const`. -- A constant array/seq/object/tuple constructor. - - -Path expressions ----------------- - -A location derived from `source` is then defined as a path expression that -has `source` as the owner. A path expression `e` is defined recursively: - -- `source` itself is a path expression. -- Container access like `e[i]` is a path expression. -- Tuple access `e[0]` is a path expression. -- Object field access `e.field` is a path expression. -- `system.toOpenArray(e, ...)` is a path expression. -- Pointer dereference `e[]` is a path expression. -- An address `addr e`, `unsafeAddr e` is a path expression. -- A type conversion `T(e)` is a path expression. -- A cast expression `cast[T](e)` is a path expression. -- `f(e, ...)` is a path expression if `f`'s return type is a view type. - Because the view can only have been borrowed from `e`, we then know - that owner of `f(e, ...)` is `e`. - - -If a view type is used as a return type, the location must borrow from a location -that is derived from the first parameter that is passed to the proc. -See https://nim-lang.org/docs/manual.html#procedures-var-return-type for -details about how this is done for `var T`. - -A mutable view can borrow from a mutable location, an immutable view can borrow -from both a mutable or an immutable location. - -If a view borrows from a mutable location, the view can be used to update the -location. Otherwise it cannot be used for mutations. - -The *duration* of a borrow is the span of commands beginning from the assignment -to the view and ending with the last usage of the view. - -For the duration of the borrow operation, no mutations to the borrowed locations -may be performed except via the view that borrowed from the -location. The borrowed location is said to be *sealed* during the borrow. - -.. code-block:: nim - - {.experimental: "views".} - - type - Obj = object - field: string - - proc dangerous(s: var seq[Obj]) = - let v: lent Obj = s[0] # seal 's' - s.setLen 0 # prevented at compile-time because 's' is sealed. - echo v.field - - -The scope of the view does not matter: - -.. code-block:: nim - - proc valid(s: var seq[Obj]) = - let v: lent Obj = s[0] # begin of borrow - echo v.field # end of borrow - s.setLen 0 # valid because 'v' isn't used afterwards - - -The analysis requires as much precision about mutations as is reasonably obtainable, -so it is more effective with the experimental `strict funcs <#strict-funcs>`_ -feature. In other words `--experimental:views`:option: works better -with `--experimental:strictFuncs`:option:. - -The analysis is currently control flow insensitive: - -.. code-block:: nim - - proc invalid(s: var seq[Obj]) = - let v: lent Obj = s[0] - if false: - s.setLen 0 - echo v.field - -In this example, the compiler assumes that `s.setLen 0` invalidates the -borrow operation of `v` even though a human being can easily see that it -will never do that at runtime. - - -Start of a borrow ------------------ - -A borrow starts with one of the following: - -- The assignment of a non-view-type to a view-type. -- The assignment of a location that is derived from a local parameter - to a view-type. - - -End of a borrow ---------------- - -A borrow operation ends with the last usage of the view variable. - - -Reborrows ---------- - -A view `v` can borrow from multiple different locations. However, the borrow -is always the full span of `v`'s lifetime and every location that is borrowed -from is sealed during `v`'s lifetime. - - -Algorithm ---------- - -The following section is an outline of the algorithm that the current implementation -uses. The algorithm performs two traversals over the AST of the procedure or global -section of code that uses a view variable. No fixpoint iterations are performed, the -complexity of the analysis is O(N) where N is the number of nodes of the AST. - -The first pass over the AST computes the lifetime of each local variable based on -a notion of an "abstract time", in the implementation it's a simple integer that is -incremented for every visited node. - -In the second pass information about the underlying object "graphs" is computed. -Let `v` be a parameter or a local variable. Let `G(v)` be the graph -that `v` belongs to. A graph is defined by the set of variables that belong -to the graph. Initially for all `v`: `G(v) = {v}`. Every variable can only -be part of a single graph. - -Assignments like `a = b` "connect" two variables, both variables end up in the -same graph `{a, b} = G(a) = G(b)`. Unfortunately, the pattern to look for is -much more complex than that and can involve multiple assignment targets -and sources:: - - f(x, y) = g(a, b) - -connects `x` and `y` to `a` and `b`: `G(x) = G(y) = G(a) = G(b) = {x, y, a, b}`. -A type based alias analysis rules out some of these combinations, for example -a `string` value cannot possibly be connected to a `seq[int]`. - -A pattern like `v[] = value` or `v.field = value` marks `G(v)` as mutated. -After the second pass a set of disjoint graphs was computed. - -For strict functions it is then enforced that there is no graph that is both mutated -and has an element that is an immutable parameter (that is a parameter that is not -of type `var T`). - -For borrow checking a different set of checks is performed. Let `v` be the view -and `b` the location that is borrowed from. - -- The lifetime of `v` must not exceed `b`'s lifetime. Note: The lifetime of - a parameter is the complete proc body. -- If `v` is used for a mutation, `b` must be a mutable location too. -- During `v`'s lifetime, `G(b)` can only be modified by `v` (and only if - `v` is a mutable view). -- If `v` is `result` then `b` has to be a location derived from the first - formal parameter or from a constant location. -- A view cannot be used for a read or a write access before it was assigned to. +This feature may be removed in the future due to its practical difficulties. diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 6dbb92875..c7d56c0ed 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -465,12 +465,7 @@ proc bindSym*(ident: string | NimNode, rule: BindSymRule = brClosed): NimNode {. ## If `rule == brForceOpen` always an `nnkOpenSymChoice` tree is ## returned even if the symbol is not ambiguous. ## - ## Experimental feature: - ## use {.experimental: "dynamicBindSym".} to activate it. - ## If called from template / regular code, `ident` and `rule` must be - ## constant expression / literal value. - ## If called from macros / compile time procs / static blocks, - ## `ident` and `rule` can be VM computed value. + ## See the `manual `_ for more details. proc genSym*(kind: NimSymKind = nskLet; ident = ""): NimNode {. magic: "NGenSym", noSideEffect.} diff --git a/tests/macros/tcasestmtmacro.nim b/tests/macros/tcasestmtmacro.nim index 26519f637..32019a92a 100644 --- a/tests/macros/tcasestmtmacro.nim +++ b/tests/macros/tcasestmtmacro.nim @@ -4,8 +4,6 @@ yes ''' """ -{.experimental: "caseStmtMacros".} - import macros macro `case`(n: tuple): untyped = From 2859069dbe1c943494cb6be299b200afa748cf52 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 09:08:07 +0300 Subject: [PATCH 0554/1654] fix #12274 (#19180) --- compiler/parser.nim | 2 +- tests/parser/t12274.nim | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/parser/t12274.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index aedf62d54..0ef6d1f54 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1146,7 +1146,7 @@ proc isExprStart(p: Parser): bool = of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor, tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCustomLit, tkVar, tkRef, tkPtr, - tkTuple, tkObject, tkWhen, tkCase, tkOut: + tkTuple, tkObject, tkWhen, tkCase, tkOut, tkTry, tkBlock: result = true else: result = false diff --git a/tests/parser/t12274.nim b/tests/parser/t12274.nim new file mode 100644 index 000000000..40c85f158 --- /dev/null +++ b/tests/parser/t12274.nim @@ -0,0 +1,9 @@ +var s: seq[int] +s.add block: + let i = 1 + i +s.add try: + 2 +except: + 3 +doAssert s == @[1, 2] From ff39f6e26062651a56d6ac4c24fe5a799c177c50 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 10:20:15 +0300 Subject: [PATCH 0555/1654] make JS trunc polyfill opt-in, closes #16144 (#19183) --- changelog.md | 8 +++++++- lib/system/jssys.nim | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index b645bc5c7..f7f92ad3c 100644 --- a/changelog.md +++ b/changelog.md @@ -3,7 +3,13 @@ ## Changes affecting backward compatibility - +- The `Math.trunc` polyfill for targeting Internet Explorer was + previously emitted for every JavaScript output file except if + the symbol `nodejs` was defined via `-d:nodejs`. Now, it is only + emitted if the symbol `nimJsMathTruncPolyfill` is defined. If you are + targeting Internet Explorer, you may choose to enable this option + or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). Nim uses + `Math.trunc` for the division and modulo operators for integers. ## Standard library additions and changes diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index cd4f378be..250bd069d 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -763,7 +763,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start: int): int # Workaround for IE, IE up to version 11 lacks 'Math.trunc'. We produce # 'Math.trunc' for Nim's ``div`` and ``mod`` operators: -const jsMathTrunc = """ +when defined(nimJsMathTruncPolyfill): + {.emit: """ if (!Math.trunc) { Math.trunc = function(v) { v = +v; @@ -771,5 +772,4 @@ if (!Math.trunc) { return (v - v % 1) || (v < 0 ? -0 : v === 0 ? v : 0); }; } -""" -when not defined(nodejs): {.emit: jsMathTrunc .} +""".} From c7c6b13a325958c6b55cbec2eb812eb06074101b Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 14:22:40 +0300 Subject: [PATCH 0556/1654] parseExpr/parseStmt accept filename, fixes #13540 (#19182) --- changelog.md | 3 +++ compiler/vm.nim | 9 ++++----- compiler/vmgen.nim | 4 ++-- lib/core/macros.nim | 14 ++++++++------ tests/macros/mparsefile.nim | 4 ++++ tests/macros/tparsefile.nim | 11 +++++++++++ 6 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 tests/macros/mparsefile.nim create mode 100644 tests/macros/tparsefile.nim diff --git a/changelog.md b/changelog.md index f7f92ad3c..63c3eff40 100644 --- a/changelog.md +++ b/changelog.md @@ -13,6 +13,9 @@ ## Standard library additions and changes +- `macros.parseExpr` and `macros.parseStmt` now accept an optional + filename argument for more informative errors. + ## `std/smtp` - Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. diff --git a/compiler/vm.nim b/compiler/vm.nim index b64545081..d58a49f9e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1748,11 +1748,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = elif instr.opcode == opcNHint: message(c.config, info, hintUser, a.strVal) of opcParseExprToAst: - decodeB(rkNode) - # c.debug[pc].line.int - countLines(regs[rb].strVal) ? + decodeBC(rkNode) var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, - toFullPath(c.config, c.debug[pc]), c.debug[pc].line.int, + regs[rc].node.strVal, 0, proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) @@ -1764,10 +1763,10 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = else: regs[ra].node = ast[0] of opcParseStmtToAst: - decodeB(rkNode) + decodeBC(rkNode) var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, - toFullPath(c.config, c.debug[pc]), c.debug[pc].line.int, + regs[rc].node.strVal, 0, proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bb095d3ee..9a823dff4 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1225,9 +1225,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = unused(c, n, dest) genBinaryStmtVar(c, n, opcAddSeqElem) of mParseExprToAst: - genUnaryABC(c, n, dest, opcParseExprToAst) + genBinaryABC(c, n, dest, opcParseExprToAst) of mParseStmtToAst: - genUnaryABC(c, n, dest, opcParseStmtToAst) + genBinaryABC(c, n, dest, opcParseStmtToAst) of mTypeTrait: let tmp = c.genx(n[1]) if dest < 0: dest = c.getTemp(n.typ) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index c7d56c0ed..444783b5a 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -511,27 +511,29 @@ proc lineInfo*(arg: NimNode): string = ## Return line info in the form `filepath(line, column)`. $arg.lineInfoObj -proc internalParseExpr(s: string): NimNode {. +proc internalParseExpr(s, filename: string): NimNode {. magic: "ParseExprToAst", noSideEffect.} -proc internalParseStmt(s: string): NimNode {. +proc internalParseStmt(s, filename: string): NimNode {. magic: "ParseStmtToAst", noSideEffect.} proc internalErrorFlag*(): string {.magic: "NError", noSideEffect.} ## Some builtins set an error flag. This is then turned into a proper ## exception. **Note**: Ordinary application code should not call this. -proc parseExpr*(s: string): NimNode {.noSideEffect.} = +proc parseExpr*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. ## Expects a single expression. Raises `ValueError` for parsing errors. - result = internalParseExpr(s) + ## A filename can be given for more informative errors. + result = internalParseExpr(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc parseStmt*(s: string): NimNode {.noSideEffect.} = +proc parseStmt*(s: string; filename: string = ""): NimNode {.noSideEffect.} = ## Compiles the passed string to its AST representation. ## Expects one or more statements. Raises `ValueError` for parsing errors. - result = internalParseStmt(s) + ## A filename can be given for more informative errors. + result = internalParseStmt(s, filename) let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) diff --git a/tests/macros/mparsefile.nim b/tests/macros/mparsefile.nim new file mode 100644 index 000000000..8ac99d568 --- /dev/null +++ b/tests/macros/mparsefile.nim @@ -0,0 +1,4 @@ +let a = 1 +let b = 2 +let c = +let d = 4 diff --git a/tests/macros/tparsefile.nim b/tests/macros/tparsefile.nim new file mode 100644 index 000000000..a41223f80 --- /dev/null +++ b/tests/macros/tparsefile.nim @@ -0,0 +1,11 @@ +import macros + +static: + let fn = "mparsefile.nim" + var raised = false + try: + discard parseStmt(staticRead(fn), filename = fn) + except ValueError as e: + raised = true + doAssert e.msg == "mparsefile.nim(4, 1) Error: invalid indentation" + doAssert raised From a0073d2d4c18f030eccef98a130f7f1f2ad9d67a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 24 Nov 2021 15:49:32 +0100 Subject: [PATCH 0557/1654] renamed 'gc' switch to 'mm'; [backport:1.6] (#19187) * renamed 'gc' switch to 'mm'; [backport:1.6] * better docs --- changelog.md | 5 +- compiler/commands.nim | 4 +- doc/advopt.txt | 7 +-- doc/backends.rst | 38 +------------- doc/destructors.rst | 12 ++--- doc/docs.rst | 4 +- doc/mm.rst | 95 ++++++++++++++++++++++++++++++++++ doc/nimc.rst | 22 ++++---- doc/{gc.rst => refc.rst} | 109 +++++++++++---------------------------- koch.nim | 2 +- tools/kochdocs.nim | 2 +- 11 files changed, 157 insertions(+), 143 deletions(-) create mode 100644 doc/mm.rst rename doc/{gc.rst => refc.rst} (62%) diff --git a/changelog.md b/changelog.md index 63c3eff40..258cf73e4 100644 --- a/changelog.md +++ b/changelog.md @@ -53,7 +53,7 @@ Baz = object ``` - [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, - meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. ## Compiler changes @@ -63,5 +63,6 @@ ## Tool changes - +- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the + reality better. (Nim moved away from all techniques based on "tracing".) diff --git a/compiler/commands.nim b/compiler/commands.nim index a8caad916..d5c5f24e4 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -248,7 +248,7 @@ template deprecatedAlias(oldName, newName: string) = proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool = case switch.normalize - of "gc": + of "gc", "mm": case arg.normalize of "boehm": result = conf.selectedGC == gcBoehm of "refc": result = conf.selectedGC == gcRefc @@ -596,7 +596,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info) of "project": processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info) - of "gc": + of "gc", "mm": if conf.backend == backendJs: return # for: bug #16033 expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: diff --git a/doc/advopt.txt b/doc/advopt.txt index 063d018d1..79e784fde 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -122,8 +122,9 @@ Advanced options: --skipUserCfg:on|off do not read the user's configuration file --skipParentCfg:on|off do not read the parent dirs' configuration files --skipProjCfg:on|off do not read the project's configuration file - --gc:refc|arc|orc|markAndSweep|boehm|go|none|regions - select the GC to use; default is 'refc' + --mm:orc|arc|refc|markAndSweep|boehm|go|none|regions + select which memory management to use; default is 'refc' + recommended is 'orc' --exceptions:setjmp|cpp|goto|quirky select the exception handling implementation --index:on|off turn index file generation on|off @@ -163,4 +164,4 @@ Advanced options: --profileVM:on|off turn compile time VM profiler on|off --sinkInference:on|off turn sink parameter inference on|off (default: on) --panics:on|off turn panics into process terminations (default: off) - --deepcopy:on|off enable 'system.deepCopy' for ``--gc:arc|orc`` + --deepcopy:on|off enable 'system.deepCopy' for ``--mm:arc|orc`` diff --git a/doc/backends.rst b/doc/backends.rst index 377e899b0..d55503d10 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -105,7 +105,7 @@ file. However, you can also run the code with `nodejs`:idx: If you experience errors saying that `globalThis` is not defined, be sure to run a recent version of Node.js (at least 12.0). - + Interfacing =========== @@ -387,14 +387,8 @@ A similar thing happens with C code invoking Nim code which returns a proc gimme(): cstring {.exportc.} = result = "Hey there C code! " & $rand(100) -Since Nim's garbage collector is not aware of the C code, once the +Since Nim's reference counting mechanism is not aware of the C code, once the `gimme` proc has finished it can reclaim the memory of the `cstring`. -However, from a practical standpoint, the C code invoking the `gimme` -function directly will be able to use it since Nim's garbage collector has -not had a chance to run *yet*. This gives you enough time to make a copy for -the C side of the program, as calling any further Nim procs *might* trigger -garbage collection making the previously returned string garbage. Or maybe you -are `yourself triggering the collection `_. Custom data types @@ -414,31 +408,3 @@ you can clean it up. And of course, once cleaned you should avoid accessing it from Nim (or C for that matter). Typically C data structures have their own `malloc_structure`:c: and `free_structure`:c: specific functions, so wrapping these for the Nim side should be enough. - - -Thread coordination -------------------- - -When the `NimMain()` function is called Nim initializes the garbage -collector to the current thread, which is usually the main thread of your -application. If your C code later spawns a different thread and calls Nim -code, the garbage collector will fail to work properly and you will crash. - -As long as you don't use the threadvar emulation Nim uses native thread -variables, of which you get a fresh version whenever you create a thread. You -can then attach a GC to this thread via - -.. code-block:: nim - - system.setupForeignThreadGc() - -It is **not** safe to disable the garbage collector and enable it after the -call from your background thread even if the code you are calling is short -lived. - -Before the thread exits, you should tear down the thread's GC to prevent memory -leaks by calling - -.. code-block:: nim - - system.tearDownForeignThreadGc() diff --git a/doc/destructors.rst b/doc/destructors.rst index 83a50230b..c98e94536 100644 --- a/doc/destructors.rst +++ b/doc/destructors.rst @@ -43,7 +43,7 @@ written as: dealloc(x.data) proc `=trace`[T](x: var myseq[T]; env: pointer) = - # `=trace` allows the cycle collector `--gc:orc` + # `=trace` allows the cycle collector `--mm:orc` # to understand how to trace the object graph. if x.data != nil: for i in 0..`_ | Description of some tools that come with the standard distribution. -- | `GC `_ - | Additional documentation about Nim's multi-paradigm memory management strategies +- | `Memory management `_ + | Additional documentation about Nim's memory management strategies | and how to operate them in a realtime setting. - | `Source code filters `_ diff --git a/doc/mm.rst b/doc/mm.rst new file mode 100644 index 000000000..b6941a901 --- /dev/null +++ b/doc/mm.rst @@ -0,0 +1,95 @@ +======================= +Nim's Memory Management +======================= + +.. default-role:: code +.. include:: rstcommon.rst + +:Author: Andreas Rumpf +:Version: |nimversion| + +.. + + + "The road to hell is paved with good intentions." + + +Multi-paradigm Memory Management Strategies +=========================================== + +.. default-role:: option + +Nim offers multiple different memory management strategies. +To choose the memory management strategy use the `--mm:` switch. + +**The recommended switch for newly written Nim code is `--mm:orc`.** + + +ARC/ORC +------- + +`--mm:orc` is a memory management mode primarily based on reference counting. Cycles +in the object graph are handled by a "cycle collector" which is based on "trial deletion". +Since algorithms based on "tracing" are not used, the runtime behavior is oblivious to +the involved heap sizes. + +The reference counting operations (= "RC ops") do not use atomic instructions and do not have to -- +instead entire subgraphs are *moved* between threads. The Nim compiler also aggressively +optimizes away RC ops and exploits `move semantics `_. + +Nim performs a fair share of optimizations for ARC/ORC; you can inspect what it did +to your time critical function via `--expandArc:functionName`. + +`--mm:arc` uses the same mechanism as `--mm:orc`, but it leaves out the cycle collector. +Both ARC and ORC offer deterministic performance for `hard realtime`:idx: systems, but +ARC can be easier to reason about for people coming from Ada/C++/C -- roughly speaking +the memory for a variable is freed when it goes "out of scope". + +We generally advise you to use the `acyclic` annotation in order to optimize away the +cycle collector's overhead +but `--mm:orc` also produces more machine code than `--mm:arc`, so if you're on a target +where code size matters and you know that your code does not produce cycles, you can +use `--mm:arc`. Notice that the default `async`:idx: implementation produces cycles +and leaks memory with `--mm:arc`, in other words, for `async` you need to use `--mm:orc`. + + + +Other MM modes +-------------- + +.. note:: The default `refc` GC is incremental, thread-local and not "stop-the-world". + +--mm:refc This is the default memory management strategy. It's a + deferred reference counting based garbage collector + with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local. + `This document `_ contains further information. +--mm:markAndSweep Simple Mark-And-Sweep based garbage collector. + Heaps are thread-local. +--mm:boehm Boehm based garbage collector, it offers a shared heap. +--mm:go Go's garbage collector, useful for interoperability with Go. + Offers a shared heap. + +--mm:none No memory management strategy nor a garbage collector. Allocated memory is + simply never freed. You should use `--mm:arc` instead. + +Here is a comparison of the different memory management modes: + +================== ======== ================= ============== =================== +Memory Management Heap Reference Cycles Stop-The-World Command line switch +================== ======== ================= ============== =================== +ORC Shared Cycle Collector No `--mm:orc` +ARC Shared Leak No `--mm:arc` +RefC Local Cycle Collector No `--mm:refc` +Mark & Sweep Local Cycle Collector No `--mm:markAndSweep` +Boehm Shared Cycle Collector Yes `--mm:boehm` +Go Shared Cycle Collector Yes `--mm:go` +None Manual Manual Manual `--mm:none` +================== ======== ================= ============== =================== + +.. default-role:: code +.. include:: rstcommon.rst + +JavaScript's garbage collector is used for the `JavaScript and NodeJS +`_ compilation targets. +The `NimScript `_ target uses the memory management strategy built into +the Nim compiler. diff --git a/doc/nimc.rst b/doc/nimc.rst index 6be4c204e..eb066e748 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -408,13 +408,13 @@ to your usual `nim c`:cmd: or `nim cpp`:cmd: command and set the `passC`:option: and `passL`:option: command line switches to something like: .. code-block:: cmd - nim c ... --d:nimAllocPagesViaMalloc --gc:orc --passC="-I$DEVKITPRO/libnx/include" ... + nim c ... --d:nimAllocPagesViaMalloc --mm:orc --passC="-I$DEVKITPRO/libnx/include" ... --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" or setup a ``nim.cfg`` file like so:: #nim.cfg - --gc:orc + --mm:orc --d:nimAllocPagesViaMalloc --passC="-I$DEVKITPRO/libnx/include" --passL="-specs=$DEVKITPRO/libnx/switch.specs -L$DEVKITPRO/libnx/lib -lnx" @@ -485,10 +485,10 @@ Define Effect `useMalloc` Makes Nim use C's `malloc`:idx: instead of Nim's own memory manager, albeit prefixing each allocation with its size to support clearing memory on reallocation. - This only works with `--gc:none`:option:, - `--gc:arc`:option: and `--gc:orc`:option:. + This only works with `--mm:none`:option:, + `--mm:arc`:option: and `--mm:orc`:option:. `useRealtimeGC` Enables support of Nim's GC for *soft* realtime - systems. See the documentation of the `gc `_ + systems. See the documentation of the `mm `_ for further information. `logGC` Enable GC logging to stdout. `nodejs` The JS target is actually ``node.js``. @@ -614,9 +614,9 @@ A good start is to use the `any` operating target together with the .. code:: cmd - nim c --os:any --gc:arc -d:useMalloc [...] x.nim + nim c --os:any --mm:arc -d:useMalloc [...] x.nim -- `--gc:arc`:option: will enable the reference counting memory management instead +- `--mm:arc`:option: will enable the reference counting memory management instead of the default garbage collector. This enables Nim to use heap memory which is required for strings and seqs, for example. @@ -654,7 +654,7 @@ devices. This allocator gets blocks/pages of memory via a currently undocumented `osalloc` API which usually uses POSIX's `mmap` call. On many environments `mmap` is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc` define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently -only supported with `--gc:arc` or `--gc:orc`. (Since version 1.6) +only supported with `--mm:arc` or `--mm:orc`. (Since version 1.6) nimPage256 / nimPage512 / nimPage1k =================================== @@ -681,19 +681,19 @@ nimMemAlignTiny Sets `MemAlign` to `4` bytes which reduces the memory alignment to better match some embedded devices. -Thread stack size +Thread stack size ================= Nim's thread API provides a simple wrapper around more advanced RTOS task features. Customizing the stack size and stack guard size can be done by setting `-d:nimThreadStackSize=16384` or `-d:nimThreadStackGuard=32`. -Currently only Zephyr and FreeRTOS support these configurations. +Currently only Zephyr and FreeRTOS support these configurations. Nim for realtime systems ======================== -See the documentation of Nim's soft realtime `GC `_ for further +See the `--mm:arc` or `--mm:orc` memory management settings in `MM `_ for further information. diff --git a/doc/gc.rst b/doc/refc.rst similarity index 62% rename from doc/gc.rst rename to doc/refc.rst index 5de02f73d..766097f23 100644 --- a/doc/gc.rst +++ b/doc/refc.rst @@ -1,81 +1,3 @@ -======================= -Nim's Memory Management -======================= - -.. default-role:: code -.. include:: rstcommon.rst - -:Author: Andreas Rumpf -:Version: |nimversion| - -.. - - - "The road to hell is paved with good intentions." - - -Introduction -============ - -A memory-management algorithm optimal for every use-case cannot exist. -Nim provides multiple paradigms for needs ranging from large multi-threaded -applications, to games, hard-realtime systems and small microcontrollers. - -This document describes how the management strategies work; -How to tune the garbage collectors for your needs, like (soft) `realtime systems`:idx:, -and how the memory management strategies other than garbage collectors work. - -.. note:: the default GC is incremental, thread-local and not "stop-the-world" - -Multi-paradigm Memory Management Strategies -=========================================== - -.. default-role:: option - -To choose the memory management strategy use the `--gc:` switch. - ---gc:refc This is the default GC. It's a - deferred reference counting based garbage collector - with a simple Mark&Sweep backup GC in order to collect cycles. Heaps are thread-local. ---gc:markAndSweep Simple Mark-And-Sweep based garbage collector. - Heaps are thread-local. ---gc:boehm Boehm based garbage collector, it offers a shared heap. ---gc:go Go's garbage collector, useful for interoperability with Go. - Offers a shared heap. ---gc:arc Plain reference counting with - `move semantic optimizations `_, offers a shared heap. - It offers deterministic performance for `hard realtime`:idx: systems. Reference cycles - cause memory leaks, beware. - ---gc:orc Same as `--gc:arc` but adds a cycle collector based on "trial deletion". - Unfortunately, that makes its performance profile hard to reason about so it is less - useful for hard real-time systems. - ---gc:none No memory management strategy nor a garbage collector. Allocated memory is - simply never freed. You should use `--gc:arc` instead. - - -================== ======== ================= ============== =================== -Memory Management Heap Reference Cycles Stop-The-World Command line switch -================== ======== ================= ============== =================== -RefC Local Cycle Collector No `--gc:refc` -Mark & Sweep Local Cycle Collector No `--gc:markAndSweep` -ARC Shared Leak No `--gc:arc` -ORC Shared Cycle Collector No `--gc:orc` -Boehm Shared Cycle Collector Yes `--gc:boehm` -Go Shared Cycle Collector Yes `--gc:go` -None Manual Manual Manual `--gc:none` -================== ======== ================= ============== =================== - -.. default-role:: code -.. include:: rstcommon.rst - -JavaScript's garbage collector is used for the `JavaScript and NodeJS -`_ compilation targets. -The `NimScript `_ target uses the memory management strategy built into -the Nim compiler. - - Tweaking the refc GC ==================== @@ -164,6 +86,35 @@ that up to 100 objects are traversed and freed before it checks again. Thus highly specialized environments or for older hardware. +Thread coordination +------------------- + +When the `NimMain()` function is called Nim initializes the garbage +collector to the current thread, which is usually the main thread of your +application. If your C code later spawns a different thread and calls Nim +code, the garbage collector will fail to work properly and you will crash. + +As long as you don't use the threadvar emulation Nim uses native thread +variables, of which you get a fresh version whenever you create a thread. You +can then attach a GC to this thread via + +.. code-block:: nim + + system.setupForeignThreadGc() + +It is **not** safe to disable the garbage collector and enable it after the +call from your background thread even if the code you are calling is short +lived. + +Before the thread exits, you should tear down the thread's GC to prevent memory +leaks by calling + +.. code-block:: nim + + system.tearDownForeignThreadGc() + + + Keeping track of memory ======================= @@ -178,7 +129,7 @@ Other useful procs from `system `_ you can use to keep track of mem * `GC_getStatistics()` Garbage collector statistics as a human-readable string. These numbers are usually only for the running thread, not for the whole heap, -with the exception of `--gc:boehm`:option: and `--gc:go`:option:. +with the exception of `--mm:boehm`:option: and `--mm:go`:option:. In addition to `GC_ref` and `GC_unref` you can avoid the garbage collector by manually allocating memory with procs like `alloc`, `alloc0`, `allocShared`, `allocShared0` or `allocCStringArray`. diff --git a/koch.nim b/koch.nim index f87828cdc..295b1584b 100644 --- a/koch.nim +++ b/koch.nim @@ -604,7 +604,7 @@ proc runCI(cmd: string) = when not defined(bsd): # the BSDs are overwhelmed already, so only run this test on the other machines: - kochExecFold("Boot Nim ORC", "boot -d:release --gc:orc --lib:lib") + kochExecFold("Boot Nim ORC", "boot -d:release --mm:orc --lib:lib") proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 70ee5ad07..4fbea8c77 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -129,7 +129,7 @@ tut2.rst tut3.rst nimc.rst niminst.rst -gc.rst +mm.rst """.splitWhitespace().mapIt("doc" / it) doc0 = """ From a59ad2006291eafcd6790f3011b5cd355b219a5b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 24 Nov 2021 18:16:20 +0300 Subject: [PATCH 0558/1654] fix inline syntax highlighting in system.nim (#19184) --- lib/system_overview.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system_overview.rst b/lib/system_overview.rst index df22697e0..0e4ffaf17 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -1,4 +1,5 @@ .. default-role:: code +.. include:: ../doc/rstcommon.rst The System module imports several separate modules, and their documentation is in separate files: From f91867aa31f3ae2c4f28d24547d62603daaa6d0c Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Wed, 24 Nov 2021 18:34:42 +0300 Subject: [PATCH 0559/1654] accept object type node from macros (#19179) --- compiler/semexprs.nim | 2 +- tests/macros/ttypenodes.nim | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/macros/ttypenodes.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c11cecfa9..0df332689 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2856,7 +2856,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkBind: message(c.config, n.info, warnDeprecated, "bind is deprecated") result = semExpr(c, n[0], flags) - of nkTypeOfExpr, nkTupleTy, nkTupleClassTy, nkRefTy..nkEnumTy, nkStaticTy: + of nkTypeOfExpr..nkTupleClassTy, nkStaticTy, nkRefTy..nkEnumTy: if c.matchedConcept != nil and n.len == 1: let modifier = n.modifierTypeKindOfNode if modifier != tyNone: diff --git a/tests/macros/ttypenodes.nim b/tests/macros/ttypenodes.nim new file mode 100644 index 000000000..233ea9780 --- /dev/null +++ b/tests/macros/ttypenodes.nim @@ -0,0 +1,16 @@ +import macros + +macro makeEnum(): untyped = + newTree(nnkEnumTy, newEmptyNode(), ident"a", ident"b", ident"c") + +macro makeObject(): untyped = + newTree(nnkObjectTy, newEmptyNode(), newEmptyNode(), newTree(nnkRecList, + newTree(nnkIdentDefs, ident"x", ident"y", ident"int", newEmptyNode()))) + +type + Foo = makeEnum() + Bar = makeObject() + +doAssert {a, b, c} is set[Foo] +let bar = Bar(x: 3, y: 4) +doAssert (bar.x, bar.y) == (3, 4) From 0d0c249074d6a1041de16108dc247396efef5513 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 24 Nov 2021 08:37:14 -0800 Subject: [PATCH 0560/1654] swap port to correct port order (#19177) Co-authored-by: Jaremy Creechley --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 7b0ff78e7..06024e46d 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -478,14 +478,14 @@ proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, sl = sizeof(Sockaddr_in).SockLen let s = cast[ptr Sockaddr_in](addr sa) s.sin_family = typeof(s.sin_family)(toInt(AF_INET)) - s.sin_port = port + s.sin_port = htons(port) copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr)) of IpAddressFamily.IPv6: sl = sizeof(Sockaddr_in6).SockLen let s = cast[ptr Sockaddr_in6](addr sa) s.sin6_family = typeof(s.sin6_family)(toInt(AF_INET6)) - s.sin6_port = port + s.sin6_port = htons(port) copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr)) From 373c909300c9ecef159666ac196e360fbd7a3ebc Mon Sep 17 00:00:00 2001 From: Iced Quinn <70673392+IcedQuinn@users.noreply.github.com> Date: Sat, 4 Dec 2021 00:42:03 -0600 Subject: [PATCH 0561/1654] feat: TLS-ALPN wrappers for OpenSSL (#19202) Co-authored-by: Iced Quinn --- lib/wrappers/openssl.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index c756fa938..00ec160cb 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -847,3 +847,17 @@ when not defined(nimDisableCertificateValidation) and not defined(windows): let cert = d2i_X509(certbytes) let encoded = cert.i2d_X509() assert encoded == certbytes + +# Application Layer Protocol Negociation extension (TLS-ALPN, RFC7301) +# Available in at least OpenSSL 1.1.1 and later, not sure if earlier +# --Iced Quinn + +proc SSL_CTX_set_alpn_protos*(ctx: SslCtx; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_set_alpn_protos*(ssl: SslPtr; protos: cstring; protos_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_alpn_select_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_alpn_selected*(ssl: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_protos_advertised_cb*(ctx: SslCtx; cb: proc(ssl: SslPtr; out_proto: ptr cstring; outlen: ptr cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_CTX_set_next_proto_select_cb*(ctx: SslCtx; cb: proc(s: SslPtr; out_proto: cstring; outlen: cstring; in_proto: cstring; inlen: cuint; arg: pointer): cint {.cdecl.}; arg: pointer) {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_select_next_proto*(out_proto: ptr cstring; outlen: cstring; server: cstring; server_len: cuint; client: cstring; client_len: cuint): cint {.cdecl, dynlib: DLLSSLName, importc.} +proc SSL_get0_next_proto_negotiated*(s: SslPtr; data: ptr cstring; len: ptr cuint) {.cdecl, dynlib: DLLSSLName, importc.} + From 23c117a950a3ce0e3c85d4f89409075329648230 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:42:34 +0100 Subject: [PATCH 0562/1654] misc bugfixes [backport:1.2] (#19203) --- compiler/semobjconstr.nim | 6 +++--- doc/backends.rst | 23 +++++++++-------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 30ab76957..6d11d321e 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -366,9 +366,9 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = if objType.kind == tyObject: var constrCtx = initConstrContext(objType, newNodeI(nkObjConstr, info)) let initResult = semConstructTypeAux(c, constrCtx, {}) - assert constrCtx.missingFields.len > 0 - localError(c.config, info, - "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) + if constrCtx.missingFields.len > 0: + localError(c.config, info, + "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) elif objType.kind == tyDistinct: localError(c.config, info, "The $1 distinct type doesn't have a default value." % typeToString(t)) diff --git a/doc/backends.rst b/doc/backends.rst index d55503d10..3a3359fca 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -52,7 +52,7 @@ The commands to compile to either C, C++ or Objective-C are: The most significant difference between these commands is that if you look into the ``nimcache`` directory you will find ``.c``, ``.cpp`` or ``.m`` files, other than that all of them will produce a native binary for your -project. This allows you to take the generated code and place it directly +project. This allows you to take the generated code and place it directly into a project using any of these languages. Here are some typical command- line invocations: @@ -123,7 +123,7 @@ Nim code can interface with the backend through the `Foreign function interface `_ mainly through the `importc pragma `_. The `importc` pragma is the *generic* way of making backend symbols available -in Nim and is available in all the target backends (JavaScript too). The C++ +in Nim and is available in all the target backends (JavaScript too). The C++ or Objective-C backends have their respective `ImportCpp `_ and `ImportObjC `_ @@ -246,11 +246,6 @@ Also, C code requires you to specify a forward declaration for functions or the compiler will assume certain types for the return value and parameters which will likely make your program crash at runtime. -The Nim compiler can generate a C interface header through the `--header`:option: -command-line switch. The generated header will contain all the exported -symbols and the `NimMain` proc which you need to call before any other -Nim code. - Nim invocation example from C ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -269,9 +264,10 @@ Create a ``maths.c`` file with the following content: .. code-block:: c - #include "fib.h" #include + extern int fib(int a); + int main(void) { NimMain(); @@ -286,13 +282,12 @@ program: .. code:: cmd - nim c --noMain --noLinking --header:fib.h fib.nim + nim c --noMain --noLinking fib.nim gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c The first command runs the Nim compiler with three special options to avoid -generating a `main()`:c: function in the generated files, avoid linking the -object files into a final binary, and explicitly generate a header file for C -integration. All the generated files are placed into the ``nimcache`` +generating a `main()`:c: function in the generated files and to avoid linking the +object files into a final binary. All the generated files are placed into the ``nimcache`` directory. That's why the next command compiles the ``maths.c`` source plus all the ``.c`` files from ``nimcache``. In addition to this path, you also have to tell the C compiler where to find Nim's ``nimbase.h`` header file. @@ -302,12 +297,12 @@ also ask the Nim compiler to generate a statically linked library: .. code:: cmd - nim c --app:staticLib --noMain --header fib.nim + nim c --app:staticLib --noMain fib.nim gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c The Nim compiler will handle linking the source files generated in the ``nimcache`` directory into the ``libfib.nim.a`` static library, which you can -then link into your C program. Note that these commands are generic and will +then link into your C program. Note that these commands are generic and will vary for each system. For instance, on Linux systems you will likely need to use `-ldl`:option: too to link in required dlopen functionality. From d584dd5b99f4c1e32477db2dfb791d52b0139df1 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:42:58 +0100 Subject: [PATCH 0563/1654] fixes #19015 [backport:1.6] (#19204) --- changelog.md | 3 +++ compiler/options.nim | 3 ++- compiler/sigmatch.nim | 3 ++- tests/misc/trfc405.nim | 3 +++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 258cf73e4..a8f1920e9 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,9 @@ or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). Nim uses `Math.trunc` for the division and modulo operators for integers. +- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via + `experimental:flexibleOptionalParams`. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/compiler/options.nim b/compiler/options.nim index d8e5057f9..383b6fa2a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -207,7 +207,8 @@ type strictNotNil, overloadableEnums, strictEffects, - unicodeOperators + unicodeOperators, + flexibleOptionalParams LegacyFeature* = enum allowSemcheckedAstModification, diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 6bc6eefe0..93be92676 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2461,7 +2461,8 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int if m.callee.n[f].kind != nkSym: internalError(c.config, n[a].info, "matches") noMatch() - if a >= firstArgBlock: f = max(f, m.callee.n.len - (n.len - a)) + if flexibleOptionalParams in c.features and a >= firstArgBlock: + f = max(f, m.callee.n.len - (n.len - a)) formal = m.callee.n[f].sym m.firstMismatch.kind = kTypeMismatch if containsOrIncl(marker, formal.position) and container.isNil: diff --git a/tests/misc/trfc405.nim b/tests/misc/trfc405.nim index adc39b1df..8c967eb77 100644 --- a/tests/misc/trfc405.nim +++ b/tests/misc/trfc405.nim @@ -1,3 +1,6 @@ + +{.experimental: "flexibleOptionalParams".} + # https://github.com/nim-lang/RFCs/issues/405 template main = From f90620fb325d5b5b924774ce84384cb4098ed18e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:43:20 +0100 Subject: [PATCH 0564/1654] fixes #19198 [backport:1.6] (#19209) * fixes #19198 [backport:1.6] * added a test case --- compiler/vm.nim | 6 ++++-- compiler/vmgen.nim | 28 ++++++++++++++++------------ tests/vm/tvmmisc.nim | 23 +++++++++++++++++++++-- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index d58a49f9e..3343eb781 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -434,8 +434,10 @@ proc opConv(c: PCtx; dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): of tyFloat..tyFloat64: dest.intVal = int(src.floatVal) else: - let srcDist = (sizeof(src.intVal) - styp.size) * 8 - let destDist = (sizeof(dest.intVal) - desttyp.size) * 8 + let srcSize = getSize(c.config, styp) + let destSize = getSize(c.config, desttyp) + let srcDist = (sizeof(src.intVal) - srcSize) * 8 + let destDist = (sizeof(dest.intVal) - destSize) * 8 var value = cast[BiggestUInt](src.intVal) value = (value shl srcDist) shr srcDist value = (value shl destDist) shr destDist diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 9a823dff4..c0c4bac19 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -749,18 +749,20 @@ proc genNarrow(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) # uint is uint64 in the VM, we we only need to mask the result for # other unsigned types: - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) - elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8): - c.gABC(n, opcNarrowS, dest, TRegister(t.size*8)) + let size = getSize(c.config, t) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) + elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and size < 8): + c.gABC(n, opcNarrowS, dest, TRegister(size*8)) proc genNarrowU(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) # uint is uint64 in the VM, we we only need to mask the result for # other unsigned types: + let size = getSize(c.config, t) if t.kind in {tyUInt8..tyUInt32, tyInt8..tyInt32} or - (t.kind in {tyUInt, tyInt} and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) + (t.kind in {tyUInt, tyInt} and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) proc genBinaryABCnarrow(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = genBinaryABC(c, n, dest, opc) @@ -1088,10 +1090,11 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = genBinaryABC(c, n, dest, opcShlInt) # genNarrowU modified let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) - elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and t.size < 8): - c.gABC(n, opcSignExtend, dest, TRegister(t.size*8)) + let size = getSize(c.config, t) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) + elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and size < 8): + c.gABC(n, opcSignExtend, dest, TRegister(size*8)) of mAshrI: genBinaryABC(c, n, dest, opcAshrInt) of mBitandI: genBinaryABC(c, n, dest, opcBitandInt) of mBitorI: genBinaryABC(c, n, dest, opcBitorInt) @@ -1125,8 +1128,9 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = genUnaryABC(c, n, dest, opcBitnotInt) #genNarrowU modified, do not narrow signed types let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and t.size < 8): - c.gABC(n, opcNarrowU, dest, TRegister(t.size*8)) + let size = getSize(c.config, t) + if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + c.gABC(n, opcNarrowU, dest, TRegister(size*8)) of mCharToStr, mBoolToStr, mIntToStr, mInt64ToStr, mFloatToStr, mCStrToStr, mStrToStr, mEnumToStr: genConv(c, n, n[1], dest) of mEqStr, mEqCString: genBinaryABC(c, n, dest, opcEqStr) diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index f6fbde0f9..dfe086d10 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -290,10 +290,10 @@ block: # bug #10815 const a = P() doAssert $a == "" - + when defined osx: # xxx bug https://github.com/nim-lang/Nim/issues/10815#issuecomment-476380734 block: - type CharSet {.union.} = object + type CharSet {.union.} = object cs: set[char] vs: array[4, uint64] const a = Charset(cs: {'a'..'z'}) @@ -553,3 +553,22 @@ block: # bug #8015 doAssert $viaProc.table[0] == "(kind: Fixed, cost: 999)" doAssert viaProc.table[1].handler() == 100 doAssert viaProc.table[2].handler() == 200 + + +# bug #19198 + +block: + type + Foo[n: static int] = int + +block: + static: + let x = int 1 + echo x.type # Foo + +block: + static: + let x = int 1 + let y = x + 1 + # Error: unhandled exception: value out of range: -8 notin 0 .. 65535 [RangeDefect] + echo y From 1cbdc1573af5db8bf555b0b6454424188c9f345c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 4 Dec 2021 07:44:26 +0100 Subject: [PATCH 0565/1654] fixes #19159 [backport:1.6] (#19210) --- compiler/sempass2.nim | 7 ++++--- tests/effects/tstrict_effects3.nim | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 5624d7adb..d6cc4180b 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -852,13 +852,14 @@ proc trackCall(tracked: PEffects; n: PNode) = assumeTheWorst(tracked, n, op) gcsafeAndSideeffectCheck() else: - if strictEffects in tracked.c.features and a.kind == nkSym and a.sym.kind in routineKinds: + if strictEffects in tracked.c.features and a.kind == nkSym and + a.sym.kind in routineKinds: propagateEffects(tracked, n, a.sym) else: mergeRaises(tracked, effectList[exceptionEffects], n) mergeTags(tracked, effectList[tagEffects], n) gcsafeAndSideeffectCheck() - if a.kind != nkSym or a.sym.magic notin {mNBindSym, mFinished}: + if a.kind != nkSym or a.sym.magic notin {mNBindSym, mFinished, mExpandToAst, mQuoteAst}: for i in 1.. Date: Sat, 4 Dec 2021 07:54:11 +0100 Subject: [PATCH 0566/1654] Improve documentation around func and method (#19207) * Improve documentation around func and method * Update doc/tut1.rst Co-authored-by: Danil Yarantsev * Update doc/tut1.rst Co-authored-by: Danil Yarantsev * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update doc/tut1.rst Co-authored-by: Zoom * Update doc/tut1.rst Co-authored-by: Zoom * Update doc/tut1.rst Co-authored-by: Zoom * Update doc/tut1.rst Co-authored-by: Zoom * Rewrite of Zooms suggestion * Update doc/tut1.rst Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Danil Yarantsev Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Zoom --- doc/tut1.rst | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index f492f12b0..2856625b7 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -595,8 +595,10 @@ Procedures To define new commands like `echo `_ and `readLine `_ in the examples, the concept of a -*procedure* is needed. (Some languages call them *methods* or *functions*.) -In Nim new procedures are defined with the `proc` keyword: +*procedure* is needed. You might be used to them being called *methods* or +*functions* in other languages, but Nim +`differentiates these concepts `_. In +Nim, new procedures are defined with the `proc` keyword: .. code-block:: nim :test: "nim c $1" @@ -874,6 +876,31 @@ The example also shows that a proc's body can consist of a single expression whose value is then returned implicitly. +Funcs and methods +----------------- + +As mentioned in the introduction, Nim differentiates between procedures, +functions, and methods, defined by the `proc`, `func`, and `method` keywords +respectively. In some ways, Nim is a bit more pedantic in its definitions than +other languages. + +Functions are closer to the concept of a pure mathematical +function, which might be familiar to you if you've ever done functional +programming. Essentially they are procedures with additional limitations set on +them: they can't access global state (except `const`) and can't produce +side-effects. The `func` keyword is basically an alias for `proc` tagged +with `{.noSideEffects.}`. Functions can still change their mutable arguments +however, which are those marked as `var`, along with any `ref` objects. + +Unlike procedures, methods are dynamically dispatched. This sounds a bit +complicated, but it is a concept closely related to inheritance and object oriented +programming. If you overload a procedure (two procedures with the same name but +of different types or with different sets of arguments are said to be overloaded), the procedure to use is determined +at compile-time. Methods, on the other hand, depend on objects that inherit from +the `RootObj`. This is something that is covered in much greater depth in +the `second part of the tutorial`_. + + Iterators ========= From 1ef945668d458163cb7116a8c936ff92b63d679d Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sun, 5 Dec 2021 08:44:22 +0300 Subject: [PATCH 0567/1654] treat do with pragmas but no parens as proc (#19191) fixes #19188 --- compiler/parser.nim | 12 +++++++++--- doc/manual.rst | 8 ++++---- tests/parser/tdo.nim | 19 +++++++++++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 0ef6d1f54..9ff847ef6 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1110,11 +1110,14 @@ proc optPragmas(p: var Parser): PNode = proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = #| doBlock = 'do' paramListArrow pragma? colcom stmt - let params = parseParamList(p, retColon=false) + var params = parseParamList(p, retColon=false) let pragmas = optPragmas(p) colcom(p, result) result = parseStmt(p) - if params.kind != nkEmpty: + if params.kind != nkEmpty or pragmas.kind != nkEmpty: + if params.kind == nkEmpty: + params = newNodeP(nkFormalParams, p) + params.add(p.emptyNode) # return type result = newProcNode(nkDo, info, body = result, params = params, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) @@ -1381,7 +1384,10 @@ proc postExprBlocks(p: var Parser, x: PNode): PNode = if stmtList[0].kind == nkStmtList: stmtList = stmtList[0] stmtList.flags.incl nfBlockArg - if openingParams.kind != nkEmpty: + if openingParams.kind != nkEmpty or openingPragmas.kind != nkEmpty: + if openingParams.kind == nkEmpty: + openingParams = newNodeP(nkFormalParams, p) + openingParams.add(p.emptyNode) # return type result.add newProcNode(nkDo, stmtList.info, body = stmtList, params = openingParams, name = p.emptyNode, pattern = p.emptyNode, diff --git a/doc/manual.rst b/doc/manual.rst index 4fcc516ec..6705e673e 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3834,10 +3834,10 @@ The proc expression represented by the `do` block is appended to the routine call as the last argument. In calls using the command syntax, the `do` block will bind to the immediately preceding expression rather than the command call. -`do` with a parameter list corresponds to an anonymous `proc`, however -`do` without parameters is treated as a normal statement list. This allows -macros to receive both indented statement lists as an argument in inline -calls, as well as a direct mirror of Nim's routine syntax. +`do` with a parameter list or pragma list corresponds to an anonymous `proc`, +however `do` without parameters or pragmas is treated as a normal statement +list. This allows macros to receive both indented statement lists as an +argument in inline calls, as well as a direct mirror of Nim's routine syntax. .. code-block:: nim # Passing a statement list to an inline macro: diff --git a/tests/parser/tdo.nim b/tests/parser/tdo.nim index 7bd1f7411..382d03398 100644 --- a/tests/parser/tdo.nim +++ b/tests/parser/tdo.nim @@ -1,8 +1,12 @@ discard """ - output: '''true + output: ''' true true -true inner B''' +true +true inner B +running with pragma +ran with pragma +''' """ template withValue(a, b, c, d, e: untyped) = @@ -77,3 +81,14 @@ proc main2 = echo "true inner B" main2() + +proc withPragma(foo: int, bar: proc() {.raises: [].}) = + echo "running with pragma" + bar() + +withPragma(3) do {.raises: [].}: + echo "ran with pragma" + +doAssert not (compiles do: + withPragma(3) do {.raises: [].}: + raise newException(Exception)) From e5475768ba5b540489fb47bdb6fdd338d75175e1 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 5 Dec 2021 14:17:52 +0800 Subject: [PATCH 0568/1654] feature_request should be labelled automatically (#19215) --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index b9f7caad5..6a9afe071 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Do you want to suggest a new feature? Use this template. title: '' -labels: '' +labels: ["Feature"] assignees: '' --- From faacd63bf69561fe915f3651e6b917bb2c0e5bc2 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 6 Dec 2021 17:04:54 +0800 Subject: [PATCH 0569/1654] the title of PR should contain a brief description (#19211) In addition to an issue number, the title of a PR should contain a brief description; then users can follow the progress more easily from the title. --- doc/contributing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index 07fa01714..d3b2770f3 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -10,6 +10,9 @@ Contributing Contributing happens via "Pull requests" (PR) on github. Every PR needs to be reviewed before it can be merged and the Continuous Integration should be green. +The title of a PR should contain a brief description. If it fixes an issue, +in addition to the number of the issue, the title should also contain a description +of the issue. The PR has to be approved by two core developers or by Araq. From 7e3da693f5bd8879d91ccadc469a3ec80d77a34f Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 7 Dec 2021 01:58:01 +0800 Subject: [PATCH 0570/1654] [format minor] remove unnecessary spaces (#19216) --- lib/system/atomics.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index f69cb753d..75731764b 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -73,7 +73,7 @@ when someGcc and hasThreadSupport: proc atomicCompareExchangeN*[T: AtomType](p, expected: ptr T, desired: T, weak: bool, success_memmodel: AtomMemModel, failure_memmodel: AtomMemModel): bool {. - importc: "__atomic_compare_exchange_n ", nodecl.} + importc: "__atomic_compare_exchange_n", nodecl.} ## This proc implements an atomic compare and exchange operation. This compares the ## contents at p with the contents at expected and if equal, writes desired at p. ## If they are not equal, the current contents at p is written into expected. @@ -100,13 +100,13 @@ when someGcc and hasThreadSupport: proc atomicSubFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_sub_fetch", nodecl.} proc atomicOrFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_or_fetch ", nodecl.} + importc: "__atomic_or_fetch", nodecl.} proc atomicAndFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_and_fetch", nodecl.} proc atomicXorFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. importc: "__atomic_xor_fetch", nodecl.} proc atomicNandFetch*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. - importc: "__atomic_nand_fetch ", nodecl.} + importc: "__atomic_nand_fetch", nodecl.} ## Perform the operation return the old value, all memory models are valid proc atomicFetchAdd*[T: AtomType](p: ptr T, val: T, mem: AtomMemModel): T {. @@ -177,8 +177,8 @@ elif someVcc and hasThreadSupport: proc `==`(x1, x2: AtomMemModel): bool {.borrow.} - proc readBarrier() {.importc: "_ReadBarrier", header: "".} - proc writeBarrier() {.importc: "_WriteBarrier", header: "".} + proc readBarrier() {.importc: "_ReadBarrier", header: "".} + proc writeBarrier() {.importc: "_WriteBarrier", header: "".} proc fence*() {.importc: "_ReadWriteBarrier", header: "".} template barrier(mem: AtomMemModel) = From 0213c7313b2edf8e8ec9d81764685744e874ef84 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Mon, 6 Dec 2021 20:38:23 +0100 Subject: [PATCH 0571/1654] allow `HSlice` bounded by constants of distinct types (#19219) [backport:1.2] When creating heterogenous slices of distinct types, the compiler does not initialize the internal type's `size` before accessing it. This then leads to this crash message: ``` compiler/int128.nim(594, 11) `false` masking only implemented for 1, 2, 4 and 8 bytes [AssertionError] ``` This patch initializes the `size` properly, fixing the problem. --- compiler/semfold.nim | 16 ++++++++-------- tests/slice/tdistinct.nim | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 tests/slice/tdistinct.nim diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 5ae91d306..d0473b5ce 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -133,7 +133,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P of mCard: result = newIntNodeT(toInt128(nimsets.cardSet(g.config, a)), n, idgen, g) of mBitnotI: if n.typ.isUnsigned: - result = newIntNodeT(bitnot(getInt(a)).maskBytes(int(n.typ.size)), n, idgen, g) + result = newIntNodeT(bitnot(getInt(a)).maskBytes(int(getSize(g.config, n.typ))), n, idgen, g) else: result = newIntNodeT(bitnot(getInt(a)), n, idgen, g) of mLengthArray: result = newIntNodeT(lengthOrd(g.config, a.typ), n, idgen, g) @@ -248,23 +248,23 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P of mBitorI, mOr: result = newIntNodeT(bitor(getInt(a), getInt(b)), n, idgen, g) of mBitxorI, mXor: result = newIntNodeT(bitxor(getInt(a), getInt(b)), n, idgen, g) of mAddU: - let val = maskBytes(getInt(a) + getInt(b), int(n.typ.size)) + let val = maskBytes(getInt(a) + getInt(b), int(getSize(g.config, n.typ))) result = newIntNodeT(val, n, idgen, g) of mSubU: - let val = maskBytes(getInt(a) - getInt(b), int(n.typ.size)) + let val = maskBytes(getInt(a) - getInt(b), int(getSize(g.config, n.typ))) result = newIntNodeT(val, n, idgen, g) # echo "subU: ", val, " n: ", n, " result: ", val of mMulU: - let val = maskBytes(getInt(a) * getInt(b), int(n.typ.size)) + let val = maskBytes(getInt(a) * getInt(b), int(getSize(g.config, n.typ))) result = newIntNodeT(val, n, idgen, g) of mModU: - let argA = maskBytes(getInt(a), int(a.typ.size)) - let argB = maskBytes(getInt(b), int(a.typ.size)) + let argA = maskBytes(getInt(a), int(getSize(g.config, a.typ))) + let argB = maskBytes(getInt(b), int(getSize(g.config, a.typ))) if argB != Zero: result = newIntNodeT(argA mod argB, n, idgen, g) of mDivU: - let argA = maskBytes(getInt(a), int(a.typ.size)) - let argB = maskBytes(getInt(b), int(a.typ.size)) + let argA = maskBytes(getInt(a), int(getSize(g.config, a.typ))) + let argB = maskBytes(getInt(b), int(getSize(g.config, a.typ))) if argB != Zero: result = newIntNodeT(argA div argB, n, idgen, g) of mLeSet: result = newIntNodeT(toInt128(ord(containsSets(g.config, a, b))), n, idgen, g) diff --git a/tests/slice/tdistinct.nim b/tests/slice/tdistinct.nim new file mode 100644 index 000000000..d99b529a7 --- /dev/null +++ b/tests/slice/tdistinct.nim @@ -0,0 +1,2 @@ +type Foo = distinct uint64 +const slice = 0 ..< 42.Foo From ebd88725f9d899440408cc635b79041942bc4fd0 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 7 Dec 2021 05:41:42 +0800 Subject: [PATCH 0572/1654] remove `sysspawn` which is dead code (#19218) --- lib/system/sysspawn.nim | 187 ---------------------------------------- 1 file changed, 187 deletions(-) delete mode 100644 lib/system/sysspawn.nim diff --git a/lib/system/sysspawn.nim b/lib/system/sysspawn.nim deleted file mode 100644 index c9b9a68f0..000000000 --- a/lib/system/sysspawn.nim +++ /dev/null @@ -1,187 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Implements Nim's 'spawn'. - -when not declared(NimString): - {.error: "You must not import this module explicitly".} - -{.push stackTrace:off.} - -# We declare our own condition variables here to get rid of the dummy lock -# on Windows: - -type - CondVar = object - c: SysCond - stupidLock: SysLock - counter: int - -proc createCondVar(): CondVar = - initSysCond(result.c) - initSysLock(result.stupidLock) - #acquireSys(result.stupidLock) - -proc destroyCondVar(c: var CondVar) {.inline.} = - deinitSysCond(c.c) - -proc await(cv: var CondVar) = - acquireSys(cv.stupidLock) - while cv.counter <= 0: - waitSysCond(cv.c, cv.stupidLock) - dec cv.counter - releaseSys(cv.stupidLock) - -proc signal(cv: var CondVar) = - acquireSys(cv.stupidLock) - inc cv.counter - releaseSys(cv.stupidLock) - signalSysCond(cv.c) - -type - FastCondVar = object - event, slowPath: bool - slow: CondVar - -proc createFastCondVar(): FastCondVar = - initSysCond(result.slow.c) - initSysLock(result.slow.stupidLock) - #acquireSys(result.slow.stupidLock) - result.event = false - result.slowPath = false - -proc await(cv: var FastCondVar) = - #for i in 0 .. 50: - # if cas(addr cv.event, true, false): - # # this is a HIT: Triggers > 95% in my tests. - # return - # cpuRelax() - #cv.slowPath = true - # XXX For some reason this crashes some test programs - await(cv.slow) - cv.event = false - -proc signal(cv: var FastCondVar) = - cv.event = true - #if cas(addr cv.slowPath, true, false): - signal(cv.slow) - -type - Barrier* {.compilerproc.} = object - counter: int - cv: CondVar - -proc barrierEnter*(b: ptr Barrier) {.compilerproc.} = - atomicInc b.counter - -proc barrierLeave*(b: ptr Barrier) {.compilerproc.} = - atomicDec b.counter - if b.counter <= 0: signal(b.cv) - -proc openBarrier*(b: ptr Barrier) {.compilerproc.} = - b.counter = 0 - b.cv = createCondVar() - -proc closeBarrier*(b: ptr Barrier) {.compilerproc.} = - await(b.cv) - destroyCondVar(b.cv) - -{.pop.} - -# ---------------------------------------------------------------------------- - -type - WorkerProc = proc (thread, args: pointer) {.nimcall, gcsafe.} - Worker = object - taskArrived: CondVar - taskStarted: FastCondVar #\ - # task data: - f: WorkerProc - data: pointer - ready: bool # put it here for correct alignment! - -proc nimArgsPassingDone(p: pointer) {.compilerproc.} = - let w = cast[ptr Worker](p) - signal(w.taskStarted) - -var gSomeReady = createFastCondVar() - -proc slave(w: ptr Worker) {.thread.} = - while true: - w.ready = true # If we instead signal "workerReady" we need the scheduler - # to notice this. The scheduler could then optimize the - # layout of the worker threads (e.g. keep the list sorted) - # so that no search for a "ready" thread is necessary. - # This might be implemented later, but is more tricky than - # it looks because 'spawn' itself can run concurrently. - signal(gSomeReady) - await(w.taskArrived) - assert(not w.ready) - # shield against spurious wakeups: - if w.data != nil: - w.f(w, w.data) - w.data = nil - -const NumThreads = 4 - -var - workers: array[NumThreads, Thread[ptr Worker]] - workersData: array[NumThreads, Worker] - -proc setup() = - for i in 0 ..< NumThreads: - workersData[i].taskArrived = createCondVar() - workersData[i].taskStarted = createFastCondVar() - createThread(workers[i], slave, addr(workersData[i])) - -proc preferSpawn*(): bool = - ## Use this proc to determine quickly if a 'spawn' or a direct call is - ## preferable. If it returns 'true' a 'spawn' may make sense. In general - ## it is not necessary to call this directly; use 'spawnX' instead. - result = gSomeReady.event - -proc spawn*(call: typed) {.magic: "Spawn".} - ## always spawns a new task, so that the 'call' is never executed on - ## the calling thread. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - -template spawnX*(call: typed) = - ## spawns a new task if a CPU core is ready, otherwise executes the - ## call in the calling thread. Usually it is advised to - ## use 'spawn' in order to not block the producer for an unknown - ## amount of time. 'call' has to be proc call 'p(...)' where 'p' - ## is gcsafe and has 'void' as the return type. - if preferSpawn(): spawn call - else: call - -proc nimSpawn(fn: WorkerProc; data: pointer) {.compilerproc.} = - # implementation of 'spawn' that is used by the code generator. - while true: - for i in 0.. high(workers): - let w = addr(workersData[i]) - if cas(addr w.ready, true, false): - w.data = data - w.f = fn - signal(w.taskArrived) - await(w.taskStarted) - return - await(gSomeReady) - -proc sync*() = - ## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate - ## waiting, you have to use an explicit barrier. - while true: - var allReady = true - for i in 0 .. high(workers): - if not allReady: break - allReady = allReady and workersData[i].ready - if allReady: break - await(gSomeReady) - -setup() From 93c8427fca0bdc0f32c8f7c011c661808ea1e6dc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 7 Dec 2021 17:59:43 +0100 Subject: [PATCH 0573/1654] re-enable chronos testing once again [backport:1.2] (#19222) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index e11799e73..7e15e7c65 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -51,7 +51,7 @@ pkg "cascade" pkg "cello" pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim" -pkg "chronos", "nim c -r -d:release tests/testall", allowFailure = true # pending https://github.com/nim-lang/Nim/issues/17130 +pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" From cd592ed85ba3bf440d7b24a4bc5b117d1a5f8725 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Tue, 7 Dec 2021 18:01:22 +0100 Subject: [PATCH 0574/1654] fix #19193 (#19195) [backport:1.2] --- compiler/cgen.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b669483cc..c862d4d30 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -443,8 +443,14 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}: linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)]) elif not isComplexValueType(typ): - linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), - getTypeDesc(p.module, typ, mapTypeChooser(loc))]) + if containsGarbageCollectedRef(loc.t): + var nilLoc: TLoc + initLoc(nilLoc, locTemp, loc.lode, OnStack) + nilLoc.r = rope("NIM_NIL") + genRefAssign(p, loc, nilLoc) + else: + linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), + getTypeDesc(p.module, typ, mapTypeChooser(loc))]) else: if not isTemp or containsGarbageCollectedRef(loc.t): # don't use nimZeroMem for temporary values for performance if we can From 7806ec525eec83d250141a32cfa197949ea1a42b Mon Sep 17 00:00:00 2001 From: rockcavera Date: Wed, 8 Dec 2021 02:48:34 -0300 Subject: [PATCH 0575/1654] Making TCC work again on Windows --cpu:amd64 - fix #16326 (#19221) * fix #16326 * removing comments --- lib/system/io.nim | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/system/io.nim b/lib/system/io.nim index 2ad43acdb..661d1a9ba 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -109,8 +109,22 @@ when defined(windows): else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "_fseeki64", header: "", tags: [].} - proc c_ftell(f: File): int64 {. - importc: "_ftelli64", header: "", tags: [].} + when defined(tcc): + proc c_fsetpos(f: File, pos: var int64): int32 {. + importc: "fsetpos", header: "", tags: [].} + proc c_fgetpos(f: File, pos: var int64): int32 {. + importc: "fgetpos", header: "", tags: [].} + proc c_telli64(f: cint): int64 {. + importc: "_telli64", header: "", tags: [].} + proc c_ftell(f: File): int64 = + # Taken from https://pt.osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.4-trunk/mingwrt/mingwex/stdio/ftelli64.c + result = -1'i64 + var pos: int64 + if c_fgetpos(f, pos) == 0 and c_fsetpos(f, pos) == 0: + result = c_telli64(c_fileno(f)) + else: + proc c_ftell(f: File): int64 {. + importc: "_ftelli64", header: "", tags: [].} else: proc c_fseek(f: File, offset: int64, whence: cint): cint {. importc: "fseeko", header: "", tags: [].} From 099285494124dedd7d83a848e213f895f54f9e81 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 8 Dec 2021 15:40:35 +0800 Subject: [PATCH 0576/1654] Atomic inc/dec should use ATOMIC_SEQ_CST (#19212) --- lib/system/atomics.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index 75731764b..8d29c287d 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -217,7 +217,7 @@ else: proc atomicInc*(memLoc: var int, x: int = 1): int = when someGcc and hasThreadSupport: - result = atomicAddFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, x) inc(result, x) @@ -228,9 +228,9 @@ proc atomicInc*(memLoc: var int, x: int = 1): int = proc atomicDec*(memLoc: var int, x: int = 1): int = when someGcc and hasThreadSupport: when declared(atomicSubFetch): - result = atomicSubFetch(memLoc.addr, x, ATOMIC_RELAXED) + result = atomicSubFetch(memLoc.addr, x, ATOMIC_SEQ_CST) else: - result = atomicAddFetch(memLoc.addr, -x, ATOMIC_RELAXED) + result = atomicAddFetch(memLoc.addr, -x, ATOMIC_SEQ_CST) elif someVcc and hasThreadSupport: result = addAndFetch(memLoc.addr, -x) dec(result, x) From 502ac4ed5e539146d31920d50020ae35668f4755 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Dec 2021 13:56:51 +0100 Subject: [PATCH 0577/1654] fixes a converter handling regression that caused private converters to leak into client modules; fixes #19213; [backport:1.6] (#19229) --- compiler/importer.nim | 6 ++++-- tests/converter/mdontleak.nim | 3 +++ tests/converter/tdontleak.nim | 10 ++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 tests/converter/mdontleak.nim create mode 100644 tests/converter/tdontleak.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index af392f849..acca2c644 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -185,11 +185,13 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} = for it in mitems c.graph.ifaces[fromMod.position].converters: if filter: loadPackedSym(c.graph, it) - addConverter(c, it) + if sfExported in it.sym.flags: + addConverter(c, it) for it in mitems c.graph.ifaces[fromMod.position].patterns: if filter: loadPackedSym(c.graph, it) - addPattern(c, it) + if sfExported in it.sym.flags: + addPattern(c, it) for it in mitems c.graph.ifaces[fromMod.position].pureEnums: if filter: loadPackedSym(c.graph, it) diff --git a/tests/converter/mdontleak.nim b/tests/converter/mdontleak.nim new file mode 100644 index 000000000..e55c3f87c --- /dev/null +++ b/tests/converter/mdontleak.nim @@ -0,0 +1,3 @@ + +converter toBool(x: uint32): bool = x != 0 +# Note: This convertes is not exported! diff --git a/tests/converter/tdontleak.nim b/tests/converter/tdontleak.nim new file mode 100644 index 000000000..4965fa90a --- /dev/null +++ b/tests/converter/tdontleak.nim @@ -0,0 +1,10 @@ +discard """ + output: '''5''' +joinable: false +""" + +import mdontleak +# bug #19213 + +let a = 5'u32 +echo a From 742e9d65ad6b56387dc6bf9a2be1b95c510fd0c4 Mon Sep 17 00:00:00 2001 From: wenghongquan <93646063+wenghongquan@users.noreply.github.com> Date: Thu, 9 Dec 2021 21:00:33 +0800 Subject: [PATCH 0578/1654] Add support for LoongArch (#19223) * Add support for LoongArch * Update compiler/installer.ini Co-authored-by: Andreas Rumpf --- compiler/platform.nim | 5 +++-- lib/system.nim | 2 +- lib/system/platforms.nim | 4 +++- tools/nim.zsh-completion | 1 + tools/niminst/buildsh.nimf | 2 ++ tools/niminst/makefile.nimf | 3 +++ 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/compiler/platform.nim b/compiler/platform.nim index eb1398677..4e6054d5c 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -202,7 +202,7 @@ type cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430, cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32, - cpuE2k + cpuE2k, cpuLoongArch64 type TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness, @@ -238,7 +238,8 @@ const (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), - (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] + (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), + (name: "loongarch64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] type Target* = object diff --git a/lib/system.nim b/lib/system.nim index 012aefe6a..c424cbc1b 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1126,7 +1126,7 @@ const ## Possible values: ## `"i386"`, `"alpha"`, `"powerpc"`, `"powerpc64"`, `"powerpc64el"`, ## `"sparc"`, `"amd64"`, `"mips"`, `"mipsel"`, `"arm"`, `"arm64"`, - ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`. + ## `"mips64"`, `"mips64el"`, `"riscv32"`, `"riscv64"`, '"loongarch64"'. seqShallowFlag = low(int) strlitFlag = 1 shl (sizeof(int)*8 - 2) # later versions of the codegen \ diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index 5bca8cb1c..b4c9f1f06 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -36,7 +36,8 @@ type riscv32, ## RISC-V 32-bit processor riscv64, ## RISC-V 64-bit processor wasm32, ## WASM, 32-bit - e2k ## MCST Elbrus 2000 + e2k, ## MCST Elbrus 2000 + loongarch64 ## LoongArch 64-bit processor OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, @@ -95,5 +96,6 @@ const elif defined(riscv64): CpuPlatform.riscv64 elif defined(wasm32): CpuPlatform.wasm32 elif defined(e2k): CpuPlatform.e2k + elif defined(loongarch64): CpuPlatform.loongarch64 else: CpuPlatform.none ## the CPU this program will run on. diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index 07a221e9c..45477d860 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -95,6 +95,7 @@ _nim() { '*--cpu=sparc64[compile for SPARC64 architecture]' \ '*--cpu=vm[compile for Nim VM]' \ '*--cpu=wasm32[compile to WASM 32]' \ + '*--cpu=loongarch64[compile for LoongArch64 architecture]' \ '*--gc=refc[use reference counting garbage collection]' \ '*--gc=arc[use ARC garbage collection]' \ '*--gc=orc[use ORC garbage collection]' \ diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index b81ac731f..464c70abf 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -229,6 +229,8 @@ case $ucpu in mycpu="riscv64" ;; *e2k* ) mycpu="e2k" ;; + *loongarch64* ) + mycpu="loongarch64" ;; *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index c4f2f0e79..a1f3fa977 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -169,6 +169,9 @@ endif ifeq ($(ucpu),e2k) mycpu = e2k endif +ifeq ($(ucpu),loongarch64) + mycpu = loongarch64 +endif ifndef mycpu $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif From 99f8793502e65cd43f6e301557d428758948c3ca Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 9 Dec 2021 22:37:59 +0800 Subject: [PATCH 0579/1654] remove `std/sharedstrings` (#19228) * remove std/sharedstrings it has been broken since 0.18.0 * rephrase the changelog entry --- changelog.md | 2 + lib/deprecated/pure/sharedstrings.nim | 154 -------------------------- testament/testament.nim | 1 - 3 files changed, 2 insertions(+), 155 deletions(-) delete mode 100644 lib/deprecated/pure/sharedstrings.nim diff --git a/changelog.md b/changelog.md index a8f1920e9..ebad54fe4 100644 --- a/changelog.md +++ b/changelog.md @@ -14,6 +14,8 @@ - Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via `experimental:flexibleOptionalParams`. +- `std/sharedstrings` module is removed. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/lib/deprecated/pure/sharedstrings.nim b/lib/deprecated/pure/sharedstrings.nim deleted file mode 100644 index 99963e1f2..000000000 --- a/lib/deprecated/pure/sharedstrings.nim +++ /dev/null @@ -1,154 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Shared string support for Nim. - -{.deprecated.} - -type - UncheckedCharArray = UncheckedArray[char] - -type - Buffer = ptr object - refcount: int - capacity, realLen: int - data: UncheckedCharArray - - SharedString* = object ## A string that can be shared. Slicing is O(1). - buffer: Buffer - first, len: int - -proc decRef(b: Buffer) {.inline.} = - if atomicDec(b.refcount) <= 0: - deallocShared(b) - -proc incRef(b: Buffer) {.inline.} = - atomicInc(b.refcount) - -{.experimental.} - -proc `=destroy`*(s: SharedString) = - #echo "destroyed" - if not s.buffer.isNil: - decRef(s.buffer) - -when false: - proc `=copy`*(dest: var SharedString; src: SharedString) = - incRef(src.buffer) - if not dest.buffer.isNil: - decRef(dest.buffer) - dest.buffer = src.buffer - dest.first = src.first - dest.len = src.len - -proc len*(s: SharedString): int = s.len - -proc `[]`*(s: SharedString; i: Natural): char = - if i < s.len: result = s.buffer.data[i+s.first] - else: raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) - -proc `[]=`*(s: var SharedString; i: Natural; value: char) = - if i < s.len: s.buffer.data[i+s.first] = value - else: raise newException(IndexDefect, formatErrorIndexBound(i, s.len-1)) - -proc `[]`*(s: SharedString; ab: HSlice[int, int]): SharedString = - #incRef(src.buffer) - if ab.a < s.len: - result.buffer = s.buffer - result.first = ab.a - result.len = min(s.len, ab.b - ab.a + 1) - # else: produce empty string ;-) - -proc newBuffer(cap, len: int): Buffer = - assert cap >= len - result = cast[Buffer](allocShared0(sizeof(int)*3 + cap)) - result.refcount = 0 - result.capacity = cap - result.realLen = len - -proc newSharedString*(len: Natural): SharedString = - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - result.first = 0 - result.len = len - -proc newSharedString*(s: string): SharedString = - let len = s.len - if len != 0: - # optimization: Don't have an underlying buffer when 'len == 0' - result.buffer = newBuffer(len, len) - copyMem(addr result.buffer.data[0], cstring(s), s.len) - result.first = 0 - result.len = len - -when declared(atomicLoadN): - template load(x): untyped = atomicLoadN(addr x, ATOMIC_SEQ_CST) -else: - # XXX Fixme - template load(x): untyped = x - -proc add*(s: var SharedString; t: cstring; len: Natural) = - if len == 0: return - let newLen = s.len + len - if s.buffer.isNil: - s.buffer = newBuffer(len, len) - copyMem(addr s.buffer.data[0], t, len) - s.len = len - elif newLen >= s.buffer.capacity or s.first != 0 or - s.len != s.buffer.realLen or load(s.buffer.refcount) > 1: - let oldBuf = s.buffer - s.buffer = newBuffer(max(s.buffer.capacity * 3 div 2, newLen), newLen) - copyMem(addr s.buffer.data[0], addr oldBuf.data[s.first], s.len) - copyMem(addr s.buffer.data[s.len], t, len) - decRef(oldBuf) - else: - copyMem(addr s.buffer.data[s.len], t, len) - s.buffer.realLen += len - s.len += len - -proc add*(s: var SharedString; t: string) = - s.add(t.cstring, t.len) - -proc rawData*(s: var SharedString): pointer = - if s.buffer.isNil: result = nil - else: result = addr s.buffer.data[s.first] - -proc add*(s: var SharedString; t: SharedString) = - if t.buffer.isNil: return - s.add(cast[cstring](addr s.buffer.data[s.first]), t.len) - -proc `$`*(s: SharedString): string = - result = newString(s.len) - if s.len > 0: - copyMem(addr result[0], addr s.buffer.data[s.first], s.len) - -proc `==`*(s: SharedString; t: string): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - cstring(t), t.len) - -proc `==`*(s, t: SharedString): bool = - if s.buffer.isNil: result = t.len == 0 - else: result = t.len == s.len and equalMem(addr s.buffer.data[s.first], - addr t.buffer.data[t.first], t.len) - -iterator items*(s: SharedString): char = - let buf = s.buffer.data - let x = s.first - if buf != nil: - for i in 0.. Date: Fri, 10 Dec 2021 01:49:31 +0800 Subject: [PATCH 0580/1654] add comments to spawn and pinnedSpawn (#19230) `spawn` uses `nimSpawn3` internally and `pinnedSpawn` uses `nimSpawn4` internally. I comment it in order to help contributors get the gist of its functionality. --- lib/pure/concurrency/threadpool.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index f8a5f9ba8..9334b4833 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -448,19 +448,21 @@ proc preferSpawn*(): bool = ## <#spawnX.t>`_ instead. result = gSomeReady.counter > 0 -proc spawn*(call: sink typed) {.magic: "Spawn".} +proc spawn*(call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task, so that the `call` is never executed on ## the calling thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn3` internally" -proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} +proc pinnedSpawn*(id: ThreadId; call: sink typed) {.magic: "Spawn".} = ## Always spawns a new task on the worker thread with `id`, so that ## the `call` is **always** executed on the thread. ## ## `call` has to be a proc call `p(...)` where `p` is gcsafe and has a ## return type that is either `void` or compatible with `FlowVar[T]`. + discard "It uses `nimSpawn4` internally" template spawnX*(call) = ## Spawns a new task if a CPU core is ready, otherwise executes the From 32d4bf352585a4fc4a6fa1bd24b270af087b0372 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Dec 2021 22:23:16 +0100 Subject: [PATCH 0581/1654] fixes an old ARC bug: the produced copy/sink operations don't copy the hidden type field for objects with enabled inheritance; fixes #19205 [backport:1.6] (#19232) --- compiler/ast.nim | 2 +- compiler/ccgexprs.nim | 8 ++++++++ compiler/ccgtypes.nim | 2 +- compiler/liftdestructors.nim | 10 ++++++++++ compiler/types.nim | 3 +++ tests/arc/tarcmisc.nim | 15 +++++++++++++++ 6 files changed, 38 insertions(+), 2 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 9086860b9..4286f940f 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -673,7 +673,7 @@ type mSwap, mIsNil, mArrToSeq, mNewString, mNewStringOfCap, mParseBiggestFloat, mMove, mWasMoved, mDestroy, mTrace, - mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mReset, + mDefault, mUnown, mFinished, mIsolate, mAccessEnv, mAccessTypeField, mReset, mArray, mOpenArray, mRange, mSet, mSeq, mVarargs, mRef, mPtr, mVar, mDistinct, mVoid, mTuple, mOrdinal, mIterableType, diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index fc09fefc3..1f1035115 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1741,6 +1741,13 @@ proc genGetTypeInfoV2(p: BProc, e: PNode, d: var TLoc) = # use the dynamic type stored at offset 0: putIntoDest(p, d, e, rdMType(p, a, nilCheck)) +proc genAccessTypeField(p: BProc; e: PNode; d: var TLoc) = + var a: TLoc + initLocExpr(p, e[1], a) + var nilCheck = Rope(nil) + # use the dynamic type stored at offset 0: + putIntoDest(p, d, e, rdMType(p, a, nilCheck)) + template genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = var a: TLoc initLocExpr(p, n[1], a) @@ -2449,6 +2456,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mMove: genMove(p, e, d) of mDestroy: genDestroy(p, e) of mAccessEnv: unaryExpr(p, e, d, "$1.ClE_0") + of mAccessTypeField: genAccessTypeField(p, e, d) of mSlice: genSlice(p, e, d) of mTrace: discard "no code to generate" else: diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index e7e099659..740ce5f19 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -583,7 +583,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, if typ.kind == tyObject: if typ[0] == nil: - if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags: + if lacksMTypeField(typ): appcg(m, result, " {$n", []) else: if optTinyRtti in m.config.globalOptions: diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index cc7e36cdb..51b4ddfb0 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -941,6 +941,12 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp incl result.flags, sfFromGeneric incl result.flags, sfGeneratedOp +proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) = + let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x) + let yy = genBuiltin(c, mAccessTypeField, "accessTypeField", y) + xx.typ = getSysType(c.g, c.info, tyPointer) + yy.typ = xx.typ + body.add newAsgnStmt(xx, yy) proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; info: TLineInfo; idgen: IdGenerator): PSym = @@ -980,6 +986,10 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; fillStrOp(a, typ, result.ast[bodyPos], d, src) else: fillBody(a, typ, result.ast[bodyPos], d, src) + if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy} and not lacksMTypeField(typ): + # bug #19205: Do not forget to also copy the hidden type field: + genTypeFieldCopy(a, typ, result.ast[bodyPos], d, src) + if not a.canRaise: incl result.flags, sfNeverRaises completePartialOp(g, idgen.module, typ, kind, result) diff --git a/compiler/types.nim b/compiler/types.nim index 7cf02c240..007a61356 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1705,3 +1705,6 @@ proc isCharArrayPtr*(t: PType; allowPointerToChar: bool): bool = result = allowPointerToChar else: discard + +proc lacksMTypeField*(typ: PType): bool {.inline.} = + (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index d259e1197..45d1514cd 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -30,6 +30,7 @@ ok true copying 123 +42 closed destroying variable: 20 destroying variable: 10 @@ -482,3 +483,17 @@ method testMethod(self: BrokenObject) {.base.} = let mikasa = BrokenObject() mikasa.testMethod() + +# bug #19205 +type + InputSectionBase* = object of RootObj + relocations*: seq[int] # traced reference. string has a similar SIGSEGV. + InputSection* = object of InputSectionBase + +proc fooz(sec: var InputSectionBase) = + if sec of InputSection: # this line SIGSEGV. + echo 42 + +var sec = create(InputSection) +sec[] = InputSection(relocations: newSeq[int]()) +fooz sec[] From 69aabdab800077e9aaa08344494c83138a02f57c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Fri, 10 Dec 2021 06:31:29 +0100 Subject: [PATCH 0582/1654] nimRawSetjmp: support Windows (#19197) * nimRawSetjmp: support Windows Using `_setjmp()` directly is required to avoid some rare (but very annoying) exception-related stack corruption leading to segfaults on Windows, with Mingw-w64 and SEH. More details: https://github.com/status-im/nimbus-eth2/issues/3121 Also add "nimBuiltinSetjmp" - mostly for benchmarking. * fix for Apple's Clang++ --- compiler/ccgstmts.nim | 13 +++- doc/nimc.rst | 14 ++++ lib/system/ansi_c.nim | 46 +++++++++-- tests/exception/texceptions.nim | 6 +- tests/exception/texceptions2.nim | 130 +++++++++++++++++++++++++++++++ 5 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 tests/exception/texceptions2.nim diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 559849f0d..7fecc1475 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1356,8 +1356,19 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimSigSetjmp"): linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint]) + elif isDefined(p.config, "nimBuiltinSetjmp"): + linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimRawSetjmp"): - linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) + if isDefined(p.config, "mswindows"): + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) + else: + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) else: linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) lineCg(p, cpsStmts, "if ($1.status == 0) {$n", [safePoint]) diff --git a/doc/nimc.rst b/doc/nimc.rst index eb066e748..0ab1501ff 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -165,6 +165,20 @@ ignored too. `--define:FOO`:option: and `--define:foo`:option: are identical. Compile-time symbols starting with the `nim` prefix are reserved for the implementation and should not be used elsewhere. +========================== ============================================ +Name Description +========================== ============================================ +nimStdSetjmp Use the standard `setjmp()/longjmp()` library + functions for setjmp-based exceptions. This is + the default on most platforms. +nimSigSetjmp Use `sigsetjmp()/siglongjmp()` for setjmp-based exceptions. +nimRawSetjmp Use `_setjmp()/_longjmp()` on POSIX and `_setjmp()/longjmp()` + on Windows, for setjmp-based exceptions. It's the default on + BSDs and BSD-like platforms, where it's significantly faster + than the standard functions. +nimBuiltinSetjmp Use `__builtin_setjmp()/__builtin_longjmp()` for setjmp-based + exceptions. This will not work if an exception is being thrown + and caught inside the same procedure. Useful for benchmarking. Configuration files ------------------- diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 259d36633..23fb9fdef 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -31,7 +31,10 @@ proc c_abort*() {. importc: "abort", header: "", noSideEffect, noreturn.} -when defined(linux) and defined(amd64): +when defined(nimBuiltinSetjmp): + type + C_JmpBuf* = array[5, pointer] +elif defined(linux) and defined(amd64): type C_JmpBuf* {.importc: "jmp_buf", header: "", bycopy.} = object abi: array[200 div sizeof(clong), clong] @@ -92,18 +95,47 @@ when defined(macosx): elif defined(haiku): const SIGBUS* = cint(30) -when defined(nimSigSetjmp) and not defined(nimStdSetjmp): +# "nimRawSetjmp" is defined by default for certain platforms, so we need the +# "nimStdSetjmp" escape hatch with it. +when defined(nimSigSetjmp): proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "siglongjmp".} - template c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_sigsetjmp(jmpb: C_JmpBuf, savemask: cint): cint {. header: "", importc: "sigsetjmp".} c_sigsetjmp(jmpb, 0) +elif defined(nimBuiltinSetjmp): + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) = + # Apple's Clang++ has trouble converting array names to pointers, so we need + # to be very explicit here. + proc c_builtin_longjmp(jmpb: ptr pointer, retval: cint) {. + importc: "__builtin_longjmp", nodecl.} + # The second parameter needs to be 1 and sometimes the C/C++ compiler checks it. + c_builtin_longjmp(unsafeAddr jmpb[0], 1) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_builtin_setjmp(jmpb: ptr pointer): cint {. + importc: "__builtin_setjmp", nodecl.} + c_builtin_setjmp(unsafeAddr jmpb[0]) elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. - header: "", importc: "_longjmp".} - proc c_setjmp*(jmpb: C_JmpBuf): cint {. - header: "", importc: "_setjmp".} + when defined(windows): + # No `_longjmp()` on Windows. + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "longjmp".} + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) + else: + proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. + header: "", importc: "_longjmp".} + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "", importc: "_setjmp".} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "longjmp".} diff --git a/tests/exception/texceptions.nim b/tests/exception/texceptions.nim index adee5d1d5..62d24c934 100644 --- a/tests/exception/texceptions.nim +++ b/tests/exception/texceptions.nim @@ -1,4 +1,6 @@ discard """ + disabled: "windows" # no sigsetjmp() there + matrix: "-d:nimStdSetjmp; -d:nimSigSetjmp; -d:nimRawSetjmp; -d:nimBuiltinSetjmp" output: ''' BEFORE @@ -17,7 +19,7 @@ FINALLY echo "" -proc no_expcetion = +proc no_exception = try: echo "BEFORE" @@ -28,7 +30,7 @@ proc no_expcetion = finally: echo "FINALLY" -try: no_expcetion() +try: no_exception() except: echo "RECOVER" echo "" diff --git a/tests/exception/texceptions2.nim b/tests/exception/texceptions2.nim new file mode 100644 index 000000000..97fd856a0 --- /dev/null +++ b/tests/exception/texceptions2.nim @@ -0,0 +1,130 @@ +discard """ + disabled: "posix" # already covered by texceptions.nim + matrix: "-d:nimStdSetjmp; -d:nimRawSetjmp; -d:nimBuiltinSetjmp" + output: ''' + +BEFORE +FINALLY + +BEFORE +EXCEPT +FINALLY +RECOVER + +BEFORE +EXCEPT: IOError: hi +FINALLY +''' +""" + +echo "" + +proc no_exception = + try: + echo "BEFORE" + + except: + echo "EXCEPT" + raise + + finally: + echo "FINALLY" + +try: no_exception() +except: echo "RECOVER" + +echo "" + +proc reraise_in_except = + try: + echo "BEFORE" + raise newException(IOError, "") + + except IOError: + echo "EXCEPT" + raise + + finally: + echo "FINALLY" + +try: reraise_in_except() +except: echo "RECOVER" + +echo "" + +proc return_in_except = + try: + echo "BEFORE" + raise newException(IOError, "hi") + + except: + echo "EXCEPT: ", getCurrentException().name, ": ", getCurrentExceptionMsg() + return + + finally: + echo "FINALLY" + +try: return_in_except() +except: echo "RECOVER" + +block: #10417 + proc moo() {.noreturn.} = discard + + let bar = + try: + 1 + except: + moo() + + doAssert(bar == 1) + +# Make sure the VM handles the exceptions correctly +block: + proc fun1(): seq[int] = + try: + try: + raise newException(ValueError, "xx") + except: + doAssert("xx" == getCurrentExceptionMsg()) + raise newException(KeyError, "yy") + except: + doAssert("yy" == getCurrentExceptionMsg()) + result.add(1212) + try: + try: + raise newException(AssertionDefect, "a") + finally: + result.add(42) + except AssertionDefect: + result.add(99) + finally: + result.add(10) + result.add(4) + result.add(0) + try: + result.add(1) + except KeyError: + result.add(-1) + except ValueError: + result.add(-1) + except IndexDefect: + result.add(2) + except: + result.add(3) + + try: + try: + result.add(1) + return + except: + result.add(-1) + finally: + result.add(2) + except KeyError: + doAssert(false) + finally: + result.add(3) + + let x1 = fun1() + const x2 = fun1() + doAssert(x1 == x2) From d39147219a41f54b45b585a103bfb04cce71c52c Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 9 Dec 2021 22:38:39 -0800 Subject: [PATCH 0583/1654] Revert "swap port to correct port order (#19177)" (#19234) This reverts commit 0d0c249074d6a1041de16108dc247396efef5513. --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 06024e46d..7b0ff78e7 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -478,14 +478,14 @@ proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, sl = sizeof(Sockaddr_in).SockLen let s = cast[ptr Sockaddr_in](addr sa) s.sin_family = typeof(s.sin_family)(toInt(AF_INET)) - s.sin_port = htons(port) + s.sin_port = port copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr)) of IpAddressFamily.IPv6: sl = sizeof(Sockaddr_in6).SockLen let s = cast[ptr Sockaddr_in6](addr sa) s.sin6_family = typeof(s.sin6_family)(toInt(AF_INET6)) - s.sin6_port = htons(port) + s.sin6_port = port copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr)) From c98954233981cb30807bd8be8b805663d44964d1 Mon Sep 17 00:00:00 2001 From: MichalMarsalek Date: Fri, 10 Dec 2021 07:39:12 +0100 Subject: [PATCH 0584/1654] move toDeque to after addLast (#19233) [backport:1.0] Changes the order of procs definitions in order to avoid calling an undefined proc. --- lib/pure/collections/deques.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index ecdf199d2..c4eb5331e 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -87,20 +87,6 @@ proc initDeque*[T](initialSize: int = defaultInitialSize): Deque[T] = ## * `toDeque proc <#toDeque,openArray[T]>`_ result.initImpl(initialSize) -proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = - ## Creates a new deque that contains the elements of `x` (in the same order). - ## - ## **See also:** - ## * `initDeque proc <#initDeque,int>`_ - runnableExamples: - let a = toDeque([7, 8, 9]) - assert len(a) == 3 - assert $a == "[7, 8, 9]" - - result.initImpl(x.len) - for item in items(x): - result.addLast(item) - proc len*[T](deq: Deque[T]): int {.inline.} = ## Returns the number of elements of `deq`. result = deq.count @@ -303,6 +289,20 @@ proc addLast*[T](deq: var Deque[T], item: sink T) = deq.data[deq.tail] = item deq.tail = (deq.tail + 1) and deq.mask +proc toDeque*[T](x: openArray[T]): Deque[T] {.since: (1, 3).} = + ## Creates a new deque that contains the elements of `x` (in the same order). + ## + ## **See also:** + ## * `initDeque proc <#initDeque,int>`_ + runnableExamples: + let a = toDeque([7, 8, 9]) + assert len(a) == 3 + assert $a == "[7, 8, 9]" + + result.initImpl(x.len) + for item in items(x): + result.addLast(item) + proc peekFirst*[T](deq: Deque[T]): lent T {.inline.} = ## Returns the first element of `deq`, but does not remove it from the deque. ## From 9338aa24977e84a33b9a7802eaff0777fcf4d9c3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Dec 2021 09:24:20 +0100 Subject: [PATCH 0585/1654] fixes a possible 'javascript:' protocol exploit [backport:1.0] (#19134) * fixes a possible 'javascript:' protocol exploit [backport:1.0] * add tests * Update tests/stdlib/trstgen.nim * add the same logic for hyperlinks * move the logic into a proc Co-authored-by: narimiran --- lib/packages/docutils/rstgen.nim | 15 +++++++++++++- tests/stdlib/trstgen.nim | 35 ++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index dc6ca25c1..9018087f7 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, std/strbasics + algorithm, parseutils, std/strbasics, strscans import ../../std/private/since @@ -827,6 +827,16 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) +proc safeProtocol(linkStr: var string) = + var protocol = "" + if scanf(linkStr, "$w:", protocol): + # if it has a protocol at all, ensure that it's not 'javascript:' or worse: + if cmpIgnoreCase(protocol, "http") == 0 or cmpIgnoreCase(protocol, "https") == 0 or + cmpIgnoreCase(protocol, "ftp") == 0: + discard "it's fine" + else: + linkStr = "" + proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) = dispA(d.target, result, "
                • $2
                  • \n", @@ -891,6 +901,8 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) = # support for `:target:` links for images: var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl) + safeProtocol(target) + if target.len > 0: # `htmlOut` needs to be of the following format for link to work for images: # @@ -1192,6 +1204,7 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, d.escMode = emUrl renderRstToOut(d, link, linkStr) d.escMode = mode + safeProtocol(linkStr) var textStr = "" renderRstToOut(d, text, textStr) let nimDocStr = if nimdoc: " nimdoc" else: "" diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 45cf1a68e..2be65b35b 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -398,7 +398,7 @@ Some chapter Level2 ------ - + Level3 ~~~~~~ @@ -407,7 +407,7 @@ Some chapter More ~~~~ - + Another ------- @@ -683,7 +683,7 @@ Test1 test "RST line blocks": let input2 = dedent""" Paragraph1 - + | Paragraph2""" @@ -704,7 +704,7 @@ Test1 # check that '| ' with a few spaces is still parsed as new line let input4 = dedent""" | xxx - | + | | zzz""" let output4 = input4.toHtml @@ -1549,3 +1549,30 @@ suite "RST/Code highlight": check strip(rstToHtml(pythonCode, {}, newStringTable(modeCaseSensitive))) == strip(expected) + + +suite "invalid targets": + test "invalid image target": + let input1 = dedent """.. image:: /images/myimage.jpg + :target: https://bar.com + :alt: Alt text for the image""" + let output1 = input1.toHtml + check output1 == """Alt text for the image""" + + let input2 = dedent """.. image:: /images/myimage.jpg + :target: javascript://bar.com + :alt: Alt text for the image""" + let output2 = input2.toHtml + check output2 == """Alt text for the image""" + + let input3 = dedent """.. image:: /images/myimage.jpg + :target: bar.com + :alt: Alt text for the image""" + let output3 = input3.toHtml + check output3 == """Alt text for the image""" + + test "invalid links": + check("(([Nim](https://nim-lang.org/)))".toHtml == + """((Nim))""") + check("(([Nim](javascript://nim-lang.org/)))".toHtml == + """((Nim))""") From 7ff43d07b2b983271d666b1c654b1426cbbbf236 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Dec 2021 14:28:59 +0100 Subject: [PATCH 0586/1654] added --nimMainPrefix switch; fixes #15955; refs #16945 [backport:1.6] (#19235) --- changelog.md | 2 ++ compiler/cgen.nim | 37 ++++++++++++++++++++----------------- compiler/commands.nim | 1 + compiler/options.nim | 1 + doc/advopt.txt | 2 ++ doc/backends.rst | 3 +++ doc/nimc.rst | 7 +++++++ 7 files changed, 36 insertions(+), 17 deletions(-) diff --git a/changelog.md b/changelog.md index ebad54fe4..df8051a16 100644 --- a/changelog.md +++ b/changelog.md @@ -71,3 +71,5 @@ - The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the reality better. (Nim moved away from all techniques based on "tracing".) +- There is a new switch `--nimMainPrefix:prefix` to influence the `NimMain` that the + compiler produces. This is particularly useful for generating static libraries. diff --git a/compiler/cgen.nim b/compiler/cgen.nim index c862d4d30..032c22ec0 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -154,6 +154,11 @@ macro ropecg(m: BModule, frmt: static[FormatStr], args: untyped): Rope = inc(i) result.add newCall(formatValue, resVar, args[num]) inc(num) + of '^': + flushStrLit() + inc(i) + result.add newCall(formatValue, resVar, args[^1]) + inc(num) of '0'..'9': var j = 0 while true: @@ -1366,7 +1371,7 @@ proc genMainProc(m: BModule) = "}$N$N" MainProcs = - "\tNimMain();$N" + "\t$^NimMain();$N" MainProcsWithResult = MainProcs & ("\treturn $1nim_program_result;$N") @@ -1376,7 +1381,7 @@ proc genMainProc(m: BModule) = "}$N$N" NimMainProc = - "N_CDECL(void, NimMain)(void) {$N" & + "N_CDECL(void, $5NimMain)(void) {$N" & "\tvoid (*volatile inner)(void);$N" & "$4" & "\tinner = NimMainInner;$N" & @@ -1456,28 +1461,27 @@ proc genMainProc(m: BModule) = if optGenGuiApp in m.config.globalOptions: const nimMain = WinNimMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) else: const nimMain = WinNimDllMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) elif m.config.target.targetOS == osGenode: const nimMain = GenodeNimMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) elif optGenDynLib in m.config.globalOptions: const nimMain = PosixNimDllMain appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) elif m.config.target.targetOS == osStandalone: const nimMain = NimMainBody appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) else: const nimMain = NimMainBody appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode]) - + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) if optNoMain notin m.config.globalOptions: if m.config.cppCustomNamespace.len > 0: @@ -1487,23 +1491,22 @@ proc genMainProc(m: BModule) = m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: if optGenGuiApp in m.config.globalOptions: const otherMain = WinCMain - appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""]) + appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix]) else: const otherMain = WinCDllMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif m.config.target.targetOS == osGenode: const otherMain = ComponentConstruct - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif optGenDynLib in m.config.globalOptions: const otherMain = PosixCDllMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) elif m.config.target.targetOS == osStandalone: const otherMain = StandaloneCMain - appcg(m, m.s[cfsProcs], otherMain, []) + appcg(m, m.s[cfsProcs], otherMain, [m.config.nimMainPrefix]) else: const otherMain = PosixCMain - appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: ""]) - + appcg(m, m.s[cfsProcs], otherMain, [if m.hcrOn: "*" else: "", m.config.nimMainPrefix]) if m.config.cppCustomNamespace.len > 0: m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) @@ -1885,7 +1888,7 @@ proc writeHeader(m: BModule) = if optGenDynLib in m.config.globalOptions: result.add("N_LIB_IMPORT ") - result.addf("N_CDECL(void, NimMain)(void);$n", []) + result.addf("N_CDECL(void, $1NimMain)(void);$n", [rope m.config.nimMainPrefix]) if m.config.cppCustomNamespace.len > 0: result.add closeNamespaceNim() result.addf("#endif /* $1 */$n", [guard]) if not writeRope(result, m.filename): diff --git a/compiler/commands.nim b/compiler/commands.nim index d5c5f24e4..c4df46bc2 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1052,6 +1052,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -) handleStdinInput(conf) of "nilseqs", "nilchecks", "mainmodule", "m", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch) + of "nimmainprefix": conf.nimMainPrefix = arg else: if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg) else: invalidCmdLineOption(conf, pass, switch, info) diff --git a/compiler/options.nim b/compiler/options.nim index 383b6fa2a..9e957332f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -390,6 +390,7 @@ type structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string; severity: Severity) {.closure, gcsafe.} cppCustomNamespace*: string + nimMainPrefix*: string vmProfileData*: ProfileData proc parseNimVersion*(a: string): NimVer = diff --git a/doc/advopt.txt b/doc/advopt.txt index 79e784fde..e27c75ada 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -135,6 +135,8 @@ Advanced options: --cppCompileToNamespace:namespace use the provided namespace for the generated C++ code, if no namespace is provided "Nim" will be used + --nimMainPrefix:prefix use `{prefix}NimMain` instead of `NimMain` in the produced + C/C++ code --expandMacro:MACRO dump every generated AST from MACRO --expandArc:PROCNAME show how PROCNAME looks like after diverse optimizations before the final backend phase (mostly ARC/ORC specific) diff --git a/doc/backends.rst b/doc/backends.rst index 3a3359fca..65dd8a6f2 100644 --- a/doc/backends.rst +++ b/doc/backends.rst @@ -246,6 +246,9 @@ Also, C code requires you to specify a forward declaration for functions or the compiler will assume certain types for the return value and parameters which will likely make your program crash at runtime. +The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch. +Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`. + Nim invocation example from C ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/nimc.rst b/doc/nimc.rst index 0ab1501ff..6b4f3c9a1 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -385,6 +385,10 @@ of your program. NimMain() # initialize garbage collector memory, types and stack +The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch. +Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`. + + Cross-compilation for iOS ========================= @@ -413,6 +417,9 @@ of your program. Note: XCode's "make clean" gets confused about the generated nim.c files, so you need to clean those files manually to do a clean build. +The name `NimMain` can be influenced via the `--nimMainPrefix:prefix` switch. +Use `--nimMainPrefix:MyLib` and the function to call is named `MyLibNimMain`. + Cross-compilation for Nintendo Switch ===================================== From 908fc2a22e7336670173bcd06ab30f440ca6d321 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Dec 2021 17:13:48 +0100 Subject: [PATCH 0587/1654] let Nim support Nimble 0.14 with lock-file support [backport:1.6] (#19236) --- changelog.md | 4 ++++ config/nim.cfg | 2 ++ 2 files changed, 6 insertions(+) diff --git a/changelog.md b/changelog.md index df8051a16..6f47ef8a4 100644 --- a/changelog.md +++ b/changelog.md @@ -71,5 +71,9 @@ - The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the reality better. (Nim moved away from all techniques based on "tracing".) +- Nim now supports Nimble version 0.14 which added support for lock-files. This is done by + a simple configuration change setting that you can do yourself too. In `$nim/config/nim.cfg` + replace `pkgs` by `pkgs2`. + - There is a new switch `--nimMainPrefix:prefix` to influence the `NimMain` that the compiler produces. This is particularly useful for generating static libraries. diff --git a/config/nim.cfg b/config/nim.cfg index 86c6e2141..59d7f48d2 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -44,10 +44,12 @@ path="$lib/core" path="$lib/pure" @if not windows: + nimblepath="/opt/nimble/pkgs2/" nimblepath="/opt/nimble/pkgs/" @else: # TODO: @end +nimblepath="$home/.nimble/pkgs2/" nimblepath="$home/.nimble/pkgs/" # Syncronize with compiler/commands.specialDefine From 1a92edeb894a10a13f60905a79304bd26e1c144e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Sat, 11 Dec 2021 05:15:44 +0100 Subject: [PATCH 0588/1654] nimc.rst: fix table markup (#19239) --- doc/nimc.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/nimc.rst b/doc/nimc.rst index 6b4f3c9a1..aa6650491 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -179,6 +179,8 @@ nimRawSetjmp Use `_setjmp()/_longjmp()` on POSIX and `_setjm nimBuiltinSetjmp Use `__builtin_setjmp()/__builtin_longjmp()` for setjmp-based exceptions. This will not work if an exception is being thrown and caught inside the same procedure. Useful for benchmarking. +========================== ============================================ + Configuration files ------------------- From a3ef5df680e55d9bf68027fcb0ec6358b4279d09 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 11 Dec 2021 08:24:23 +0000 Subject: [PATCH 0589/1654] Update uri.nim (#19148) [backport:1.0] --- lib/pure/uri.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index a828298c2..5de333b8f 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -216,6 +216,8 @@ func parseAuthority(authority: string, result: var Uri) = result.isIpv6 = true of ']': inIPv6 = false + of '\0': + break else: if inPort: result.port.add(authority[i]) From 4b5cecd902cc4126ff9d6cda9edb78a13a421239 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Sun, 12 Dec 2021 13:39:56 -0800 Subject: [PATCH 0590/1654] Various std net improvements (#19132) * Variant of that works with raw IpAddresses. - Add doc tests for new net proc's. - Aadd recvFrom impl - Add recvFrom impl -- tweak handling data var - Update lib/pure/net.nim Co-authored-by: Dominik Picheta - cleaning up sendTo args - remove extra connect test - cleaning up sendTo args - fix inet_ntop test - fix test failing - byte len * fix test failing - byte len * debugging odd windows build failure * debugging odd windows build failure * more experiments to figure out the windows failure * try manual assigment on InAddr Co-authored-by: Jaremy Creechley --- lib/posix/posix.nim | 4 +- lib/pure/net.nim | 61 +++++++++++++++++++++++----- tests/stdlib/tnet_ll.nim | 88 ++++++++++++++++++++-------------------- 3 files changed, 97 insertions(+), 56 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index c2504f994..57dd6e2c7 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -155,9 +155,9 @@ when not defined(zephyr): proc inet_addr*(a1: cstring): InAddrT {.importc, header: "".} proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "".} -proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {. +proc inet_ntop*(a1: cint, a2: pointer | ptr InAddr | ptr In6Addr, a3: cstring, a4: int32): cstring {. importc:"(char *)$1", header: "".} -proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {. +proc inet_pton*(a1: cint, a2: cstring, a3: pointer | ptr InAddr | ptr In6Addr): cint {. importc, header: "".} var diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 7b0ff78e7..2d1bb0b33 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -65,6 +65,11 @@ runnableExamples("-r:off"): let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) socket.sendTo("192.168.0.1", Port(27960), "status\n") +runnableExamples("-r:off"): + let socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + let ip = parseIpAddress("192.168.0.1") + doAssert socket.sendTo(ip, Port(27960), "status\c\l") == 8 + ## Creating a server ## ----------------- ## @@ -1607,11 +1612,12 @@ proc recvLine*(socket: Socket, timeout = -1, result = "" readLine(socket, result, timeout, flags, maxLength) -proc recvFrom*(socket: Socket, data: var string, length: int, - address: var string, port: var Port, flags = 0'i32): int {. +proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: int, + address: var T, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = ## Receives data from `socket`. This function should normally be used with - ## connection-less sockets (UDP sockets). + ## connection-less sockets (UDP sockets). The source address of the data + ## packet is stored in the `address` argument as either a string or an IpAddress. ## ## If an error occurs an OSError exception will be raised. Otherwise the return ## value will be the length of data received. @@ -1620,31 +1626,37 @@ proc recvFrom*(socket: Socket, data: var string, length: int, ## so when `socket` is buffered the non-buffered implementation will be ## used. Therefore if `socket` contains something in its buffer this ## function will make no effort to return it. - template adaptRecvFromToDomain(domain: Domain) = + template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = var addrLen = sizeof(sockAddress).SockLen result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) if result != -1: data.setLen(result) - address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) - when domain == AF_INET6: - port = ntohs(sockAddress.sin6_port).Port + + when typeof(address) is string: + address = getAddrString(cast[ptr SockAddr](addr(sockAddress))) + when domain == AF_INET6: + port = ntohs(sockAddress.sin6_port).Port + else: + port = ntohs(sockAddress.sin_port).Port else: - port = ntohs(sockAddress.sin_port).Port + data.setLen(result) + sockAddress.fromSockAddr(addrLen, address, port) else: raiseOSError(osLastError()) assert(socket.protocol != IPPROTO_TCP, "Cannot `recvFrom` on a TCP socket") # TODO: Buffered sockets data.setLen(length) + case socket.domain of AF_INET6: var sockAddress: Sockaddr_in6 - adaptRecvFromToDomain(AF_INET6) + adaptRecvFromToDomain(sockAddress, AF_INET6) of AF_INET: var sockAddress: Sockaddr_in - adaptRecvFromToDomain(AF_INET) + adaptRecvFromToDomain(sockAddress, AF_INET) else: raise newException(ValueError, "Unknown socket address family") @@ -1707,7 +1719,8 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, tags: [WriteIOEffect].} = ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified - ## this function will try each IP of that hostname. + ## this function will try each IP of that hostname. This function + ## should normally be used with connection-less sockets (UDP sockets). ## ## If an error occurs an OSError exception will be raised. ## @@ -1741,12 +1754,38 @@ proc sendTo*(socket: Socket, address: string, port: Port, ## This proc sends `data` to the specified `address`, ## which may be an IP address or a hostname, if a hostname is specified ## this function will try each IP of that hostname. + ## + ## Generally for use with connection-less (UDP) sockets. ## ## If an error occurs an OSError exception will be raised. ## ## This is the high-level version of the above `sendTo` function. socket.sendTo(address, port, cstring(data), data.len, socket.domain) +proc sendTo*(socket: Socket, address: IpAddress, port: Port, + data: string, flags = 0'i32): int {. + discardable, tags: [WriteIOEffect].} = + ## This proc sends `data` to the specified `IpAddress` and returns + ## the number of bytes written. + ## + ## Generally for use with connection-less (UDP) sockets. + ## + ## If an error occurs an OSError exception will be raised. + ## + ## This is the high-level version of the above `sendTo` function. + assert(socket.protocol != IPPROTO_TCP, "Cannot `sendTo` on a TCP socket") + assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") + + var sa: Sockaddr_storage + var sl: Socklen + toSockAddr(address, port, sa, sl) + result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, + cast[ptr SockAddr](addr sa), sl) + + if result == -1'i32: + let osError = osLastError() + raiseOSError(osError) + proc isSsl*(socket: Socket): bool = ## Determines whether `socket` is a SSL socket. diff --git a/tests/stdlib/tnet_ll.nim b/tests/stdlib/tnet_ll.nim index affa21947..248dbb79b 100644 --- a/tests/stdlib/tnet_ll.nim +++ b/tests/stdlib/tnet_ll.nim @@ -1,43 +1,45 @@ -discard """ - action: run - output: ''' - -[Suite] inet_ntop tests -''' -""" - -when defined(windows): - import winlean -elif defined(posix): - import posix -else: - {.error: "Unsupported OS".} - -import unittest, strutils - -suite "inet_ntop tests": - - setup: - when defined(windows): - var wsa: WSAData - discard wsaStartup(0x101'i16, wsa.addr) - - test "IP V4": - var ip4 = 0x10111213 - var buff: array[0..255, char] - let r = inet_ntop(AF_INET, ip4.addr, buff[0].addr, buff.sizeof.int32) - let res = if r == nil: "" else: $r - check: res == "19.18.17.16" - - - test "IP V6": - when defined(windows): - let ipv6Support = (getVersion() and 0xff) > 0x5 - else: - let ipv6Support = true - - var ip6 = [0x1000'u16, 0x1001, 0x2000, 0x2001, 0x3000, 0x3001, 0x4000, 0x4001] - var buff: array[0..255, char] - let r = inet_ntop(AF_INET6, ip6[0].addr, buff[0].addr, buff.sizeof.int32) - let res = if r == nil: "" else: $r - check: not ipv6Support or res == "10:110:20:120:30:130:40:140" +discard """ + action: run + output: ''' + +[Suite] inet_ntop tests +''' +""" + +when defined(windows): + import winlean +elif defined(posix): + import posix +else: + {.error: "Unsupported OS".} + +import unittest, strutils + +suite "inet_ntop tests": + + setup: + when defined(windows): + var wsa: WSAData + discard wsaStartup(0x101'i16, wsa.addr) + + test "IP V4": + # regular + var ip4 = InAddr() + ip4.s_addr = 0x10111213'u32 + + var buff: array[0..255, char] + let r = inet_ntop(AF_INET, cast[pointer](ip4.s_addr.addr), buff[0].addr, buff.len.int32) + let res = if r == nil: "" else: $r + check: res == "19.18.17.16" + + test "IP V6": + when defined(windows): + let ipv6Support = (getVersion() and 0xff) > 0x5 + else: + let ipv6Support = true + + var ip6 = [0x1000'u16, 0x1001, 0x2000, 0x2001, 0x3000, 0x3001, 0x4000, 0x4001] + var buff: array[0..255, char] + let r = inet_ntop(AF_INET6, cast[pointer](ip6[0].addr), buff[0].addr, buff.len.int32) + let res = if r == nil: "" else: $r + check: not ipv6Support or res == "10:110:20:120:30:130:40:140" From 0ff4b2ba7ee74ed2758b0eb5992e4ccab7433952 Mon Sep 17 00:00:00 2001 From: Carlo Capocasa Date: Mon, 13 Dec 2021 07:29:22 +0100 Subject: [PATCH 0591/1654] fix bug #14468 zero-width split (#19248) --- lib/impure/re.nim | 11 +++++++---- tests/stdlib/tre.nim | 6 ++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index 8e5efbb28..4eacf0091 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -556,19 +556,22 @@ iterator split*(s: string, sep: Regex; maxsplit = -1): string = @["", "this", "is", "an", "example", ""] var last = 0 var splits = maxsplit - var x: int + var x = -1 + if len(s) == 0: + last = 1 + if matchLen(s, sep, 0) == 0: + x = 0 while last <= len(s): var first = last var sepLen = 1 + if x == 0: + inc(last) while last < len(s): x = matchLen(s, sep, last) if x >= 0: sepLen = x break inc(last) - if x == 0: - if last >= len(s): break - inc last if splits == 0: last = len(s) yield substr(s, first, last-1) if splits == 0: break diff --git a/tests/stdlib/tre.nim b/tests/stdlib/tre.nim index 2857c6c9e..9f27f7db2 100644 --- a/tests/stdlib/tre.nim +++ b/tests/stdlib/tre.nim @@ -108,4 +108,10 @@ proc testAll() = doAssert replace("foo", re"", "-") == "-f-o-o-" doAssert replace("ooo", re"o", "-") == "---" + block: # bug #14468 + accum = @[] + for word in split("this is an example", re"\b"): + accum.add(word) + doAssert(accum == @["this", " ", "is", " ", "an", " ", "example"]) + testAll() From 78b86b79425f37683ce522dbdd567f52ea26aa35 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Tue, 14 Dec 2021 16:54:01 +0800 Subject: [PATCH 0592/1654] basicopt.txt: Unify the format (#19251) --- doc/basicopt.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/basicopt.txt b/doc/basicopt.txt index a2db1c92e..390fc6fd2 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -27,11 +27,11 @@ Options: -a, --assertions:on|off turn assertions on|off --opt:none|speed|size optimize not at all or for speed|size Note: use -d:release for a release build! - --debugger:native Use native debugger (gdb) + --debugger:native use native debugger (gdb) --app:console|gui|lib|staticlib generate a console app|GUI app|DLL|static library -r, --run run the compiled program with given arguments - --eval:cmd evaluates nim code directly; e.g.: `nim --eval:"echo 1"` + --eval:cmd evaluate nim code directly; e.g.: `nim --eval:"echo 1"` defaults to `e` (nimscript) but customizable: `nim r --eval:'for a in stdin.lines: echo a'` --fullhelp show all command line switches From c55930f2e644fe04526eb4878e7e106229812fe4 Mon Sep 17 00:00:00 2001 From: Nick Wilburn Date: Tue, 14 Dec 2021 06:22:10 -0600 Subject: [PATCH 0593/1654] fix: fixes bug in CVerifyPeerUseEnvVars (#19247) Previously CVerifyPeerUseEnvVars was not being passed into scanSslCertificates, which meant that we weren't scanning additional certificate locations given via the SSL_CERT_FILE and SSL_CERT_DIR environment variables --- lib/pure/net.nim | 3 ++- tests/stdlib/thttpclient_ssl.nim | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 2d1bb0b33..ced6b2fb2 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -680,7 +680,8 @@ when defineSsl: # Scan for certs in known locations. For CVerifyPeerUseEnvVars also scan # the SSL_CERT_FILE and SSL_CERT_DIR env vars var found = false - for fn in scanSSLCertificates(): + let useEnvVars = (if verifyMode == CVerifyPeerUseEnvVars: true else: false) + for fn in scanSSLCertificates(useEnvVars = useEnvVars): if newCTX.SSL_CTX_load_verify_locations(fn, nil) == VerifySuccess: found = true break diff --git a/tests/stdlib/thttpclient_ssl.nim b/tests/stdlib/thttpclient_ssl.nim index 1c531eae9..3acdacfe3 100644 --- a/tests/stdlib/thttpclient_ssl.nim +++ b/tests/stdlib/thttpclient_ssl.nim @@ -129,3 +129,19 @@ when not defined(windows): msg.contains("certificate verify failed")): echo "CVerifyPeer exception: " & msg check(false) + + test "HttpClient with CVerifyPeerUseEnvVars": + const port = 12346.Port + let t = spawn runServer(port) + sleep(100) + + putEnv("SSL_CERT_FILE", getCurrentDir() / certFile) + var client = newHttpClient(sslContext=newContext(verifyMode=CVerifyPeerUseEnvVars)) + try: + log "client: connect" + discard client.getContent("https://127.0.0.1:12346") + except: + let msg = getCurrentExceptionMsg() + log "client: exception: " & msg + log "getContent should not have raised an exception" + fail() From 3e31f55aed4cd9fdcc289687fd0e31edfc0af805 Mon Sep 17 00:00:00 2001 From: PMunch Date: Tue, 14 Dec 2021 14:28:42 +0100 Subject: [PATCH 0594/1654] Fix #19253 (#19254) This fixes 19253 by marking the bodyStream as completed when no content was sent. --- lib/pure/httpclient.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 33da6ef6c..67ba8cf2b 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -869,6 +869,9 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, client.parseBodyFut.addCallback do(): if client.parseBodyFut.failed: client.bodyStream.fail(client.parseBodyFut.error) + else: + when client is AsyncHttpClient: + result.bodyStream.complete() proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = From c17baaefbcff5c207a4e95242fa0790e64ca6c8c Mon Sep 17 00:00:00 2001 From: Miran Date: Tue, 14 Dec 2021 18:16:49 +0100 Subject: [PATCH 0595/1654] [backport:1.0] json: limit recursion depth (#19252) * json: limit recursion depth * do not run this check for JS backend --- lib/pure/json.nim | 12 +++++++++--- tests/stdlib/tjson.nim | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index c831bf85d..922cd4e2f 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -202,6 +202,8 @@ type of JArray: elems*: seq[JsonNode] +const DepthLimit = 1000 + proc newJString*(s: string): JsonNode = ## Creates a new `JString JsonNode`. result = JsonNode(kind: JString, str: s) @@ -849,7 +851,7 @@ iterator mpairs*(node: var JsonNode): tuple[key: string, val: var JsonNode] = for key, val in mpairs(node.fields): yield (key, val) -proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = +proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool, depth = 0): JsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: @@ -885,6 +887,8 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = result = newJNull() discard getTok(p) of tkCurlyLe: + if depth > DepthLimit: + raiseParseErr(p, "}") result = newJObject() discard getTok(p) while p.tok != tkCurlyRi: @@ -893,16 +897,18 @@ proc parseJson(p: var JsonParser; rawIntegers, rawFloats: bool): JsonNode = var key = p.a discard getTok(p) eat(p, tkColon) - var val = parseJson(p, rawIntegers, rawFloats) + var val = parseJson(p, rawIntegers, rawFloats, depth+1) result[key] = val if p.tok != tkComma: break discard getTok(p) eat(p, tkCurlyRi) of tkBracketLe: + if depth > DepthLimit: + raiseParseErr(p, "]") result = newJArray() discard getTok(p) while p.tok != tkBracketRi: - result.add(parseJson(p, rawIntegers, rawFloats)) + result.add(parseJson(p, rawIntegers, rawFloats, depth+1)) if p.tok != tkComma: break discard getTok(p) eat(p, tkBracketRi) diff --git a/tests/stdlib/tjson.nim b/tests/stdlib/tjson.nim index 289ef9d05..336558ff3 100644 --- a/tests/stdlib/tjson.nim +++ b/tests/stdlib/tjson.nim @@ -345,3 +345,35 @@ block: doAssert c == "18446744073709552000" else: doAssert c == "18446744073709551615" + +block: + let a = """ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ + [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ +""" + + when not defined(js): + try: + discard parseJson(a) + except JsonParsingError: + doAssert getCurrentExceptionMsg().contains("] expected") From 5d2bab7558b54813b212a26fbcdb51556ff37b05 Mon Sep 17 00:00:00 2001 From: Sven Keller Date: Thu, 16 Dec 2021 08:58:32 +0100 Subject: [PATCH 0596/1654] suggestion to respect typedarray type (#19257) * suggestion to respect typedarray * Update jssys.nim Co-authored-by: Sven Keller --- lib/system/jssys.nim | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 250bd069d..6608a2927 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -587,7 +587,33 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = else: asm "`result` = (`dest` === null || `dest` === undefined) ? {} : `dest`;" nimCopyAux(result, src, ti.node) - of tySequence, tyArrayConstr, tyOpenArray, tyArray: + of tyArrayConstr, tyArray: + # In order to prevent a type change (TypedArray -> Array) and to have better copying performance, + # arrays constructors are considered separately + asm """ + if(ArrayBuffer.isView(`src`)) { + if(`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new `src`.constructor(`src`); + } else { + `dest`.set(`src`, 0); + } + `result` = `dest`; + } else { + if (`src` === null) { + `result` = null; + } + else { + if (`dest` === null || `dest` === undefined || `dest`.length != `src`.length) { + `dest` = new Array(`src`.length); + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } + } + } + """ + of tySequence, tyOpenArray: asm """ if (`src` === null) { `result` = null; From 609388e33fb9a19d65b5dee8711d411916981802 Mon Sep 17 00:00:00 2001 From: snxx Date: Fri, 17 Dec 2021 13:58:05 +0600 Subject: [PATCH 0597/1654] Add icon for Nim (#19258) * Add Nim's file icon * Add installer icon * Add uninstaller icon --- tools/niminst/nim-file.ico | Bin 0 -> 3311 bytes tools/niminst/setup.ico | Bin 0 -> 2551 bytes tools/niminst/uninstall.ico | Bin 0 -> 2226 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tools/niminst/nim-file.ico create mode 100644 tools/niminst/setup.ico create mode 100644 tools/niminst/uninstall.ico diff --git a/tools/niminst/nim-file.ico b/tools/niminst/nim-file.ico new file mode 100644 index 0000000000000000000000000000000000000000..0685fedcb0b7851d874bbe9606912b243e712e68 GIT binary patch literal 3311 zcmcImc{r47AAV=dVJZ$K8ctcWv{wMn{(~UoFwoaF+h5(Ea8~eT?Oosm0PHyi+L~7b#)y-_?!!MI9h;na7f`h} zXFk}JSzA3vZA#{%w7YC|(+9um$fjy{M#buOB%}P^+uxT{Ug&jKK!`AQP*HcO$LWs*MNm@IPit%3;KnR(&oYeK@_I zSh=1>ik^&tLGXN2Er&&)q$j97f^z{}5D_GRg#9^fTj56fJ=Xm$V4MOJJ9{EJ>l952 z3tBRglp{ME8tL1hKslps`^bHLZTz1k2+%9Hp3wP`Cp~{e)7m9?C|N9^OCBDc6J5vR zK~BV!x;7|ibOHheL~Lik=$3ZSIX+Sp3E-_n6B2^j&k8D|_pz+%sB7xRaUlSCte5;j z6b8t8fy)0D!_|d4L{*21>MqF}(D>%hm$L#?%^gICwrKDocpZ#ReTSeO4d z2(g9#g@!CF9Nhml>R?p}OwEY+3=gnB9^@FAL{HCTDH-guS>Iyjm@}Sx)+(8$g<98X zsN`)Mr%a@A`%xW^LWBTVPK|1r;iLMVw~0?HPZsuPNW*MN<=nEcz8*546B#wA5;%k> z+QJ~RRXYv$8Xa4@$V*gR9$xF@zA^iEi@JbkhxM(HC$|ffhLiTLoxiS*ySo^bdGUB8 z0FkAh#O$RNX?Ao_C@bc5O?6X&Yi+LV;#CjVM=AUZj`XXRp2npLv+3uak&WjW2$ZRhOjr|2RixqFRg_i$slK! zTJ$cZR&xg#t&QY;VPP`qr_lWTuI^Cw@|hzU4a$hE(zT!M-9lUd){UgI6n^A47O!aV zv}5J*0K4L7O@yXbq;da(0(;lDK=VZ@GD8H4UL-~i5l%YWkJ~VwPbj~ilK@@~H1WzN zJlbhc7V)Jzq%^5I4-6!m)&+@qBdhZ-V6fAMl%5|xdw9VT65;XXb8b4Y7NaPX`_r`f z)y@dwOwTc2Fztfr6r77?`rBl`35>l*pd2feYU%XO8|RZ)P^0zt&);pyWc{yL~qxG~pw& z!lmze=F8WV)SRyDdbRS|jiLOHqzEBk!HjO4Yp9gxuWj3*?xqJDli0H*eiJlYAJdX+Arakc=^FnZu-H z!Bbmm{;kgrsU+a-OqUPbq;ag z4%{0@t)?`u1m-qA06$z*jCYbw2lGO$$=<7A8ODd65UBu>kk8^F7Fz=30`jVEUQDGV z*1k$^Fff!lGe9?gy|cguMFMc7^M^hCA*?($_UEwb)lAZI*WxF{k7e++TAtL!uyL92(JM*mdVd<5i3=-OyiP$htOvEZHTx*xEkz9(Uz_$E?7#*bdqVpU_6w0riY?K$P8bD>cD z`?Thj?Kcw=m%Uo^*(%Qbz5@gNUCk4%`k9*z{6T%fdzQZi^)ap_TAAEoq|0*w6VETQ zt^Vk|VlLc>$(eaz60c^?L*io-0uVxJuXpb)8z z`YVQmHv-JCqGCO{BYrq;W3yuX{j;A$3pNoouX70yqh~6HVBMbCR`$B&tuwtfsEV8P z%i<2a@1-aNK`;?&N@vjW0+0FLD25L?hkRc4;5Wb1`(>5$7hwRriwF8#Ny+PGAKOh0 z`J7F?2XBIP9+|7E9@cqtcH`?ET0n+5-7i<#yk_#F*)y=YYsrC3O9Hm`u>WTx@XuZY zh8O`IcDL58x& zNTR4C`wRl7#+CN2(EO_Zn{uXOOjrS>UGa;2KcUA zbwo=3gM8mkXn@_}a0t{^*KKMl=bi}*3y8(DXyj(Plig?Bb#k)yXXxotG0E` z@e{Lc8=7tGzkV6DPUk~&rYnNoC;UGe_%DdNS!bQS&N+MU@B7X=>l^?8aoT}_09jy$ z9{`SX{6uR@xZq*Q!yH!-VQPHsVD+H$9pVhmA*Egbz+Z|m{>?6Y?AxS^|C6m>+Gx`k z_0L^%wWw{2@4XsdYJSV+h@$!*q5PLrJra+JXdltOo#4-ZvpNRiMiM}Z2e!XG=db85 zprWoIkF+aS6_@z4+_23BHr>&jViv%~&Mfx-U?3XIdApF>y2kux{u6 zSvars>y=p}Qb5EzQzKZWYO88(U9U7Cs&*jm{|ih4a`EybSKIM*nc9`xr*4YpQv3uP zfJeuRCL`eA8ZIDeK&nPZL4S$(f0X5mUPuBm)O;~q@oFREs4royN9=ulaXM~Ml5Q`e zt7S}93p5?qNE;V+!Q-W!K}gwGZ+WTYKJfyDoe`7fXPoVif9VW65=NN6kkKlv!!2)N31nU4}y_GNYOB_%GCZ&*2UsyEwBp+wCmJ^VKN-NH_vh2Ppb2-y_Mj_%SY<#i z$fkB7v*QJMNIvAo)|Q}+EgI=K{!kS!>PCzRlTMw8M@ONT#aqL7dPQvI-Pe4kFFXKH z60wn&qOF-%i-1bc|^kzKuITh7&e=u#AiWro#r0*K&aGN5yIcy69>CAFla|R%G9wZFP za05)uxW~}l=F!}Fj+$8kQZ;-WVH8^d>lDcO9iAAu)HeLbX?*i=n*iXOWC&+?WypY( z?Ug$956=A{O5!B`Bb!htFB8iF{GUJ{m2u$`v$gZ}V8betHr{)(z+Fkss-SqNWYxzs z;A@q@&e_<^>M%tZm{aC}D{12Xp)}W=qd5x@i5__J#S->xJsP3aqNa*9AXo#3HNaV< z=>U3K?1?A%Y#aPED2UG*fGnXvY2+;+p; zh*4GX)^C`tuUK}G&TfP?KyT-(NVM&V^n!KvfNWtKj{8e9b4oq^$VYD_Cqbjbj$A+Q z+M=hEiJx8&^KrK0l+7dN9UVU)ndMYyMx}{V5}8@`Ay=2BYevo)PHr}nN3bdibG@nB+nz|wb)16u*-POZ7(FTadnF4) z>z0a$W9rKN76hBHDc-887OgOB>+P|WvMmU0+YpASay``K8GTuA|3f1LLR%hJFQ5(3 zvwy$gXa3SBnNnlf9#6|%m{_yBsq%2hA`yo}g=J9f{Cp$#G8;2G$LVQertoM;lOqdZFtxuG6saLOIfUh2&s2Ce>o@^QfIPZ^7(p zcWjM!Zc0QT`%8{M;!U=e(iU}AI3}MX?)BgeQxoIwdQ=!~HI+uSNLE>_-Qy^V8jzXq zVn#P87;G=H*Oj(AHnrPdFhXH_@KLTjTzQ0sI3XH?(u-V=<<#!GI1OkxbL!DHGU__H zFfxgq#k9wbU;Tnd(#@M*7n~VFaYV=uexsG>jdhjA-digE&1#=#vpRSo{hd|DE8ThH zwe-r4vFmR~Z{)6hR0TwzxC~9IvFPUh(G42Md%BP6(6nJLk_6(XgH7SIx+1Owfi$)qzOSAs|(PiH0fs|pft*#7L5$qg2-7&~z z9b;wHswd*-Fy}r;S0MSXB-Jq(&%R((mMDR{bNb@}l+1l^EGTbrwS0}=(72u4L2+!* zfeI28Y|zSyOHc3yoMY~bM{a$>xeU3e`5Gy<+(m0CM&D|#oDU+OISmF**`P~xB@GaL z2DU`}pqaSPx5yWXa}Z{9$=4iqmTLb#WEfC3>i2&eF7jqR=$Ph%1|ERC{Hl49R^yiX zV?6Grll(IZ8SjAl6%Z|Q8Ip&_GGXTK9chw31L*=ko1K-~ znCY=szQ`<@$xzs@F3|nm9y8am!U8`Ebm){dkbViXRTSEW?rrl-khJCUkz7GndY`Y= zBx4*LJZG27%zH{#8yz;7mk}CL{lu&tZXjXNCm9i06pqR6 zD<1+sVj3jlv~45Dg#ne!sJ$tQ45aP^{}qDffB!;a;;2zT$`kD)?{GUo^-!<$LY=;d wL%Sh-oNRoYf7Z{m|A-R+ow1$5#O@i^ICKpvQHMNZ-`BJO;KT7(uw57M@WEOFY8<42S`WP!n3 z*&ln-C)|k!FC}Y#KifKdj$t>U#GEDGf9I~09&~lbuYtW$Sl39)~C^FGZ5o^>%}TKsq`5l1Z+@3QpwM zzcKhjCg_VdDjf44z%%7Vp6Sbb!!G^~)b~)J9k@x)Uj14-y2iLI2HY-L_#+ihD_#e` zDU=3TRy1(X1V2}t>eC)v;*wdpu(B5tw;(w(23LyXjw<|TIBiYaciKN823k1$&|jQ0 z+|A)phY1a>hAaqSnQCL^R+u$zB=nbe^*bdt-=!L&jM4s_aHLggzvMH>uUVYt>^_;! zpEXh2{blpW;)26Ps9$=8RdaacgD7rT2_<*Qjdx^AQsApzRnf zE?BR&I@8%T>nqQEsW2FDU@=&wsKC=%L?=`*lrmb<;PKf*Uoq38Qio+eoCL*vHzRa5yP*?X8Ei zZDUn~KFlj-XVi4@OQq#_Uy;X(4_IF`gWW`sJbB9hd-FThzWm5;2fOAZxn&t+!YbW` zw;$+3Z``oM&D|;w6}DDb#|S7~%W2f0t`SfJ^|+Jj=hTF+*aLjvz9kp(X_+Lm%&EznGb?ZqQX zgG_}0)G6N6*xPUbUwkLGSTl9qsGAm@+07QWLlPkGE<{A;chhV_Qn#|_0n7hx3DJky zbX1AIQ?~4e{aetJHcrzjUXO~};5K=65$B(KpZVL=&ly7f;b=e1=K!pdY5kYd>iw*3 z?I25M1)AVObU;R;d{qwT=cYxC%-wAHE)tcOsm54b_?eTJ0gq2HhRFpN?iO*4HRVf{ zP6VxYQ*=Nfm7Plo(}Q+38zO4>H4hWQoo3AIt|K&eZs^{-*x9RU<9pm^OL9AkgfmMx)dyX}&^I0)G$(X5h5Iz-|U>pHG2_ccrHayazRp}&W@I+s}5FOzk{PC9X z+{DTGkv!7+om&N_2b8cvPt56<%dOA3rwud?i2)sI_%(eKw%r|U<)Heul!K2 z<}6EpDjs_{&qFAE1zdQS6OH6R=;4QwALS&7fxwloR}vOpVjVX9JC3))oMzX`9(P%S z9#QG3On-eZb?TOtx_5Jvu>JpqkkCSs%JnVTHfg=x) z`I$;Li9hYO c(!hC-V4HpC^wmC3csCFYjFlt0+R`unUtQjos{jB1 literal 0 HcmV?d00001 From 5ddaf3e1fbff4c0519088612f9ec3b10d62d2316 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 17 Dec 2021 05:01:33 -0300 Subject: [PATCH 0598/1654] fix #19244 - solves the problem of the InAddr object constructor in Windows. (#19259) * Update winlean.nim * Update tnet_ll.nim --- lib/windows/winlean.nim | 2 +- tests/stdlib/tnet_ll.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index b8f3886d7..6e626e4ba 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -478,7 +478,7 @@ type PSockAddr = ptr SockAddr - InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object + InAddr* {.importc: "IN_ADDR", header: "winsock2.h", union.} = object s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", diff --git a/tests/stdlib/tnet_ll.nim b/tests/stdlib/tnet_ll.nim index 248dbb79b..2d340cea8 100644 --- a/tests/stdlib/tnet_ll.nim +++ b/tests/stdlib/tnet_ll.nim @@ -26,6 +26,7 @@ suite "inet_ntop tests": # regular var ip4 = InAddr() ip4.s_addr = 0x10111213'u32 + check: ip4.s_addr == 0x10111213'u32 var buff: array[0..255, char] let r = inet_ntop(AF_INET, cast[pointer](ip4.s_addr.addr), buff[0].addr, buff.len.int32) @@ -43,3 +44,8 @@ suite "inet_ntop tests": let r = inet_ntop(AF_INET6, cast[pointer](ip6[0].addr), buff[0].addr, buff.len.int32) let res = if r == nil: "" else: $r check: not ipv6Support or res == "10:110:20:120:30:130:40:140" + + test "InAddr": + # issue 19244 + var ip4 = InAddr(s_addr: 0x10111213'u32) + check: ip4.s_addr == 0x10111213'u32 From 3d9b6cfebce898dede2f63d0288050a106a7f5e4 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 17 Dec 2021 18:16:59 +0800 Subject: [PATCH 0599/1654] [formatter] async.nim uses two spaces (#19264) according to https://nim-lang.org/docs/nep1.html#introduction-spacing-and-whitespace-conventions, two spaces should be preferred in stdlib. --- lib/pure/async.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/async.nim b/lib/pure/async.nim index 97b29f81d..482ab32c6 100644 --- a/lib/pure/async.nim +++ b/lib/pure/async.nim @@ -1,6 +1,6 @@ when defined(js): - import asyncjs - export asyncjs + import asyncjs + export asyncjs else: - import asyncmacro, asyncfutures - export asyncmacro, asyncfutures + import asyncmacro, asyncfutures + export asyncmacro, asyncfutures From 610516e02756dd1447fc807077bacf86eb75acdd Mon Sep 17 00:00:00 2001 From: Don-Duong Quach Date: Fri, 17 Dec 2021 11:58:31 -0800 Subject: [PATCH 0600/1654] Fixed typo in manual.rst unsafeAssign->uncheckedAssign. Fixes part 1 of #19266 (#19267) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index 6705e673e..d9f5d4ea6 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1838,7 +1838,7 @@ A small example: cast uncheckedAssign -------------------- -Some restrictions for case objects can be disabled via a `{.cast(unsafeAssign).}` section: +Some restrictions for case objects can be disabled via a `{.cast(uncheckedAssign).}` section: .. code-block:: nim :test: "nim c $1" From b812431f831748c9013a12074cc6a7a838b580cd Mon Sep 17 00:00:00 2001 From: xioren <40043405+xioren@users.noreply.github.com> Date: Sun, 19 Dec 2021 05:01:17 -0800 Subject: [PATCH 0601/1654] use uppercase "type" for Proxy-Authorization header (#19273) Some servers will reject authorization requests with a lowercase "basic" type. Changing to "Basic" seems to solve these issues. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Proxy-Authorization --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 67ba8cf2b..7686d15bd 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -523,7 +523,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade # Proxy auth header. if not proxy.isNil and proxy.auth != "": let auth = base64.encode(proxy.auth) - add(result, "Proxy-Authorization: basic " & auth & httpNewLine) + add(result, "Proxy-Authorization: Basic " & auth & httpNewLine) for key, val in headers: add(result, key & ": " & val & httpNewLine) From ea475a4e98bc1cfc861f870baf6f2cc4c979ac7a Mon Sep 17 00:00:00 2001 From: MichalMarsalek Date: Mon, 20 Dec 2021 10:40:32 +0100 Subject: [PATCH 0602/1654] Update colors.nim (#19274) * Update colors.nim Added `lightgray` alias to `lightgrey` and `...grey`aliases for the rest of the gray colors. Added color `rebeccapurple`. Fixed the incorrect values for the `PaleVioletRed` and `MediumPurple` colors. This module should now be matching the CSS colors. I used the seq[tuple] syntax for defining the names. * Document colors changes. --- changelog.md | 2 + lib/pure/colors.nim | 307 +++++++++++++++++++++++--------------------- 2 files changed, 164 insertions(+), 145 deletions(-) diff --git a/changelog.md b/changelog.md index 6f47ef8a4..52159b865 100644 --- a/changelog.md +++ b/changelog.md @@ -15,11 +15,13 @@ `experimental:flexibleOptionalParams`. - `std/sharedstrings` module is removed. +- Constants `colors.colPaleVioletRed` and `colors.colMediumPurple` changed to match the CSS color standard. ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional filename argument for more informative errors. +- Module `colors` expanded with missing colors from the CSS color standard. ## `std/smtp` diff --git a/lib/pure/colors.nim b/lib/pure/colors.nim index 2193f69a6..8cda2fab8 100644 --- a/lib/pure/colors.nim +++ b/lib/pure/colors.nim @@ -6,7 +6,8 @@ # distribution, for details about the copyright. # -## This module implements color handling for Nim. +## This module implements color handling for Nim, +## namely color mixing and parsing the CSS color names. import strutils from algorithm import binarySearch @@ -170,6 +171,7 @@ const colDarkGoldenRod* = Color(0xB8860B) colDarkGray* = Color(0xA9A9A9) colDarkGreen* = Color(0x006400) + colDarkGrey* = Color(0xA9A9A9) colDarkKhaki* = Color(0xBDB76B) colDarkMagenta* = Color(0x8B008B) colDarkOliveGreen* = Color(0x556B2F) @@ -180,11 +182,13 @@ const colDarkSeaGreen* = Color(0x8FBC8F) colDarkSlateBlue* = Color(0x483D8B) colDarkSlateGray* = Color(0x2F4F4F) + colDarkSlateGrey* = Color(0x2F4F4F) colDarkTurquoise* = Color(0x00CED1) colDarkViolet* = Color(0x9400D3) colDeepPink* = Color(0xFF1493) colDeepSkyBlue* = Color(0x00BFFF) colDimGray* = Color(0x696969) + colDimGrey* = Color(0x696969) colDodgerBlue* = Color(0x1E90FF) colFireBrick* = Color(0xB22222) colFloralWhite* = Color(0xFFFAF0) @@ -196,6 +200,7 @@ const colGoldenRod* = Color(0xDAA520) colGray* = Color(0x808080) colGreen* = Color(0x008000) + colGrey* = Color(0x808080) colGreenYellow* = Color(0xADFF2F) colHoneyDew* = Color(0xF0FFF0) colHotPink* = Color(0xFF69B4) @@ -211,13 +216,15 @@ const colLightCoral* = Color(0xF08080) colLightCyan* = Color(0xE0FFFF) colLightGoldenRodYellow* = Color(0xFAFAD2) - colLightGrey* = Color(0xD3D3D3) + colLightGray* = Color(0xD3D3D3) colLightGreen* = Color(0x90EE90) + colLightGrey* = Color(0xD3D3D3) colLightPink* = Color(0xFFB6C1) colLightSalmon* = Color(0xFFA07A) colLightSeaGreen* = Color(0x20B2AA) colLightSkyBlue* = Color(0x87CEFA) colLightSlateGray* = Color(0x778899) + colLightSlateGrey* = Color(0x778899) colLightSteelBlue* = Color(0xB0C4DE) colLightYellow* = Color(0xFFFFE0) colLime* = Color(0x00FF00) @@ -228,7 +235,7 @@ const colMediumAquaMarine* = Color(0x66CDAA) colMediumBlue* = Color(0x0000CD) colMediumOrchid* = Color(0xBA55D3) - colMediumPurple* = Color(0x9370D8) + colMediumPurple* = Color(0x9370DB) colMediumSeaGreen* = Color(0x3CB371) colMediumSlateBlue* = Color(0x7B68EE) colMediumSpringGreen* = Color(0x00FA9A) @@ -249,7 +256,7 @@ const colPaleGoldenRod* = Color(0xEEE8AA) colPaleGreen* = Color(0x98FB98) colPaleTurquoise* = Color(0xAFEEEE) - colPaleVioletRed* = Color(0xD87093) + colPaleVioletRed* = Color(0xDB7093) colPapayaWhip* = Color(0xFFEFD5) colPeachPuff* = Color(0xFFDAB9) colPeru* = Color(0xCD853F) @@ -257,6 +264,7 @@ const colPlum* = Color(0xDDA0DD) colPowderBlue* = Color(0xB0E0E6) colPurple* = Color(0x800080) + colRebeccaPurple* = Color(0x663399) colRed* = Color(0xFF0000) colRosyBrown* = Color(0xBC8F8F) colRoyalBlue* = Color(0x4169E1) @@ -270,6 +278,7 @@ const colSkyBlue* = Color(0x87CEEB) colSlateBlue* = Color(0x6A5ACD) colSlateGray* = Color(0x708090) + colSlateGrey* = Color(0x708090) colSnow* = Color(0xFFFAFA) colSpringGreen* = Color(0x00FF7F) colSteelBlue* = Color(0x4682B4) @@ -285,147 +294,155 @@ const colYellow* = Color(0xFFFF00) colYellowGreen* = Color(0x9ACD32) - colorNames = [ - ("aliceblue", colAliceBlue), - ("antiquewhite", colAntiqueWhite), - ("aqua", colAqua), - ("aquamarine", colAquamarine), - ("azure", colAzure), - ("beige", colBeige), - ("bisque", colBisque), - ("black", colBlack), - ("blanchedalmond", colBlanchedAlmond), - ("blue", colBlue), - ("blueviolet", colBlueViolet), - ("brown", colBrown), - ("burlywood", colBurlyWood), - ("cadetblue", colCadetBlue), - ("chartreuse", colChartreuse), - ("chocolate", colChocolate), - ("coral", colCoral), - ("cornflowerblue", colCornflowerBlue), - ("cornsilk", colCornsilk), - ("crimson", colCrimson), - ("cyan", colCyan), - ("darkblue", colDarkBlue), - ("darkcyan", colDarkCyan), - ("darkgoldenrod", colDarkGoldenRod), - ("darkgray", colDarkGray), - ("darkgreen", colDarkGreen), - ("darkkhaki", colDarkKhaki), - ("darkmagenta", colDarkMagenta), - ("darkolivegreen", colDarkOliveGreen), - ("darkorange", colDarkorange), - ("darkorchid", colDarkOrchid), - ("darkred", colDarkRed), - ("darksalmon", colDarkSalmon), - ("darkseagreen", colDarkSeaGreen), - ("darkslateblue", colDarkSlateBlue), - ("darkslategray", colDarkSlateGray), - ("darkturquoise", colDarkTurquoise), - ("darkviolet", colDarkViolet), - ("deeppink", colDeepPink), - ("deepskyblue", colDeepSkyBlue), - ("dimgray", colDimGray), - ("dodgerblue", colDodgerBlue), - ("firebrick", colFireBrick), - ("floralwhite", colFloralWhite), - ("forestgreen", colForestGreen), - ("fuchsia", colFuchsia), - ("gainsboro", colGainsboro), - ("ghostwhite", colGhostWhite), - ("gold", colGold), - ("goldenrod", colGoldenRod), - ("gray", colGray), - ("green", colGreen), - ("greenyellow", colGreenYellow), - ("honeydew", colHoneyDew), - ("hotpink", colHotPink), - ("indianred", colIndianRed), - ("indigo", colIndigo), - ("ivory", colIvory), - ("khaki", colKhaki), - ("lavender", colLavender), - ("lavenderblush", colLavenderBlush), - ("lawngreen", colLawnGreen), - ("lemonchiffon", colLemonChiffon), - ("lightblue", colLightBlue), - ("lightcoral", colLightCoral), - ("lightcyan", colLightCyan), - ("lightgoldenrodyellow", colLightGoldenRodYellow), - ("lightgreen", colLightGreen), - ("lightgrey", colLightGrey), - ("lightpink", colLightPink), - ("lightsalmon", colLightSalmon), - ("lightseagreen", colLightSeaGreen), - ("lightskyblue", colLightSkyBlue), - ("lightslategray", colLightSlateGray), - ("lightsteelblue", colLightSteelBlue), - ("lightyellow", colLightYellow), - ("lime", colLime), - ("limegreen", colLimeGreen), - ("linen", colLinen), - ("magenta", colMagenta), - ("maroon", colMaroon), - ("mediumaquamarine", colMediumAquaMarine), - ("mediumblue", colMediumBlue), - ("mediumorchid", colMediumOrchid), - ("mediumpurple", colMediumPurple), - ("mediumseagreen", colMediumSeaGreen), - ("mediumslateblue", colMediumSlateBlue), - ("mediumspringgreen", colMediumSpringGreen), - ("mediumturquoise", colMediumTurquoise), - ("mediumvioletred", colMediumVioletRed), - ("midnightblue", colMidnightBlue), - ("mintcream", colMintCream), - ("mistyrose", colMistyRose), - ("moccasin", colMoccasin), - ("navajowhite", colNavajoWhite), - ("navy", colNavy), - ("oldlace", colOldLace), - ("olive", colOlive), - ("olivedrab", colOliveDrab), - ("orange", colOrange), - ("orangered", colOrangeRed), - ("orchid", colOrchid), - ("palegoldenrod", colPaleGoldenRod), - ("palegreen", colPaleGreen), - ("paleturquoise", colPaleTurquoise), - ("palevioletred", colPaleVioletRed), - ("papayawhip", colPapayaWhip), - ("peachpuff", colPeachPuff), - ("peru", colPeru), - ("pink", colPink), - ("plum", colPlum), - ("powderblue", colPowderBlue), - ("purple", colPurple), - ("red", colRed), - ("rosybrown", colRosyBrown), - ("royalblue", colRoyalBlue), - ("saddlebrown", colSaddleBrown), - ("salmon", colSalmon), - ("sandybrown", colSandyBrown), - ("seagreen", colSeaGreen), - ("seashell", colSeaShell), - ("sienna", colSienna), - ("silver", colSilver), - ("skyblue", colSkyBlue), - ("slateblue", colSlateBlue), - ("slategray", colSlateGray), - ("snow", colSnow), - ("springgreen", colSpringGreen), - ("steelblue", colSteelBlue), - ("tan", colTan), - ("teal", colTeal), - ("thistle", colThistle), - ("tomato", colTomato), - ("turquoise", colTurquoise), - ("violet", colViolet), - ("wheat", colWheat), - ("white", colWhite), - ("whitesmoke", colWhiteSmoke), - ("yellow", colYellow), - ("yellowgreen", colYellowGreen)] + colorNames = { + "aliceblue": colAliceBlue, + "antiquewhite": colAntiqueWhite, + "aqua": colAqua, + "aquamarine": colAquamarine, + "azure": colAzure, + "beige": colBeige, + "bisque": colBisque, + "black": colBlack, + "blanchedalmond": colBlanchedAlmond, + "blue": colBlue, + "blueviolet": colBlueViolet, + "brown": colBrown, + "burlywood": colBurlyWood, + "cadetblue": colCadetBlue, + "chartreuse": colChartreuse, + "chocolate": colChocolate, + "coral": colCoral, + "cornflowerblue": colCornflowerBlue, + "cornsilk": colCornsilk, + "crimson": colCrimson, + "cyan": colCyan, + "darkblue": colDarkBlue, + "darkcyan": colDarkCyan, + "darkgoldenrod": colDarkGoldenRod, + "darkgray": colDarkGray, + "darkgreen": colDarkGreen, + "darkgrey": colDarkGrey, + "darkkhaki": colDarkKhaki, + "darkmagenta": colDarkMagenta, + "darkolivegreen": colDarkOliveGreen, + "darkorange": colDarkorange, + "darkorchid": colDarkOrchid, + "darkred": colDarkRed, + "darksalmon": colDarkSalmon, + "darkseagreen": colDarkSeaGreen, + "darkslateblue": colDarkSlateBlue, + "darkslategray": colDarkSlateGray, + "darkslategrey": colDarkSlateGrey, + "darkturquoise": colDarkTurquoise, + "darkviolet": colDarkViolet, + "deeppink": colDeepPink, + "deepskyblue": colDeepSkyBlue, + "dimgray": colDimGray, + "dimgrey": colDimGrey, + "dodgerblue": colDodgerBlue, + "firebrick": colFireBrick, + "floralwhite": colFloralWhite, + "forestgreen": colForestGreen, + "fuchsia": colFuchsia, + "gainsboro": colGainsboro, + "ghostwhite": colGhostWhite, + "gold": colGold, + "goldenrod": colGoldenRod, + "gray": colGray, + "green": colGreen, + "grey": colGrey, + "greenyellow": colGreenYellow, + "honeydew": colHoneyDew, + "hotpink": colHotPink, + "indianred": colIndianRed, + "indigo": colIndigo, + "ivory": colIvory, + "khaki": colKhaki, + "lavender": colLavender, + "lavenderblush": colLavenderBlush, + "lawngreen": colLawnGreen, + "lemonchiffon": colLemonChiffon, + "lightblue": colLightBlue, + "lightcoral": colLightCoral, + "lightcyan": colLightCyan, + "lightgoldenrodyellow": colLightGoldenRodYellow, + "lightgray": colLightGray, + "lightgreen": colLightGreen, + "lightgrey": colLightGrey, + "lightpink": colLightPink, + "lightsalmon": colLightSalmon, + "lightseagreen": colLightSeaGreen, + "lightskyblue": colLightSkyBlue, + "lightslategray": colLightSlateGray, + "lightslategrey": colLightSlateGrey, + "lightsteelblue": colLightSteelBlue, + "lightyellow": colLightYellow, + "lime": colLime, + "limegreen": colLimeGreen, + "linen": colLinen, + "magenta": colMagenta, + "maroon": colMaroon, + "mediumaquamarine": colMediumAquaMarine, + "mediumblue": colMediumBlue, + "mediumorchid": colMediumOrchid, + "mediumpurple": colMediumPurple, + "mediumseagreen": colMediumSeaGreen, + "mediumslateblue": colMediumSlateBlue, + "mediumspringgreen": colMediumSpringGreen, + "mediumturquoise": colMediumTurquoise, + "mediumvioletred": colMediumVioletRed, + "midnightblue": colMidnightBlue, + "mintcream": colMintCream, + "mistyrose": colMistyRose, + "moccasin": colMoccasin, + "navajowhite": colNavajoWhite, + "navy": colNavy, + "oldlace": colOldLace, + "olive": colOlive, + "olivedrab": colOliveDrab, + "orange": colOrange, + "orangered": colOrangeRed, + "orchid": colOrchid, + "palegoldenrod": colPaleGoldenRod, + "palegreen": colPaleGreen, + "paleturquoise": colPaleTurquoise, + "palevioletred": colPaleVioletRed, + "papayawhip": colPapayaWhip, + "peachpuff": colPeachPuff, + "peru": colPeru, + "pink": colPink, + "plum": colPlum, + "powderblue": colPowderBlue, + "purple": colPurple, + "rebeccapurple": colRebeccaPurple, + "red": colRed, + "rosybrown": colRosyBrown, + "royalblue": colRoyalBlue, + "saddlebrown": colSaddleBrown, + "salmon": colSalmon, + "sandybrown": colSandyBrown, + "seagreen": colSeaGreen, + "seashell": colSeaShell, + "sienna": colSienna, + "silver": colSilver, + "skyblue": colSkyBlue, + "slateblue": colSlateBlue, + "slategray": colSlateGray, + "slategrey": colSlateGrey, + "snow": colSnow, + "springgreen": colSpringGreen, + "steelblue": colSteelBlue, + "tan": colTan, + "teal": colTeal, + "thistle": colThistle, + "tomato": colTomato, + "turquoise": colTurquoise, + "violet": colViolet, + "wheat": colWheat, + "white": colWhite, + "whitesmoke": colWhiteSmoke, + "yellow": colYellow, + "yellowgreen": colYellowGreen} proc `$`*(c: Color): string = ## Converts a color into its textual representation. From 4da7dbffc504db9159285ca59e2cdfe9b103afb8 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Dec 2021 03:29:03 +1100 Subject: [PATCH 0603/1654] Extract runnables that specify `doccmd` (#19275) [backport:1.6] --- lib/core/macros.nim | 4 ++-- tests/stdlib/tmacros.nim | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 444783b5a..ed7e85c64 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1715,8 +1715,8 @@ proc extractDocCommentsAndRunnables*(n: NimNode): NimNode = case ni.kind of nnkCommentStmt: result.add ni - of nnkCall: - if ni[0].kind == nnkIdent and ni[0].strVal == "runnableExamples": + of nnkCall, nnkCommand: + if ni[0].kind == nnkIdent and ni[0].eqIdent "runnableExamples": result.add ni else: break else: break diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 64a474743..299eac49b 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -124,3 +124,23 @@ block: # SameType assert not testTensorInt(x1) assert testTensorInt(x2) assert not testTensorInt(x3) + +block: # extractDocCommentsAndRunnables + macro checkRunnables(prc: untyped) = + let runnables = prc.body.extractDocCommentsAndRunnables() + doAssert runnables[0][0].eqIdent("runnableExamples") + + macro checkComments(comment: static[string], prc: untyped) = + let comments = prc.body.extractDocCommentsAndRunnables() + doAssert comments[0].strVal == comment + + proc a() {.checkRunnables.} = + runnableExamples: discard + discard + + proc b() {.checkRunnables.} = + runnableExamples "-d:ssl": discard + discard + + proc c() {.checkComments("Hello world").} = + ## Hello world From 19898e1225862560b216b765187fc0b98dca7608 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 20 Dec 2021 23:10:15 +0300 Subject: [PATCH 0604/1654] Fix group reference (with capital letters (#19196) in group name) --- compiler/docgen.nim | 3 +- lib/packages/docutils/rst.nim | 2 +- .../expected/subdir/subdir_b/utils.html | 28 +++++++++++++++++++ .../expected/subdir/subdir_b/utils.idx | 2 ++ nimdoc/testproject/expected/theindex.html | 6 ++++ nimdoc/testproject/subdir/subdir_b/utils.nim | 6 ++++ 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 7ba952083..90e0f54b3 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1301,7 +1301,8 @@ proc finishGenerateDoc*(d: var PDoc) = let tooltip = "$1 ($2 overloads)" % [ k.toHumanStr & " " & plainName, $overloadChoices.len] addAnchorNim(d.sharedState, refn, tooltip, - LangSymbol(symKind: k.toHumanStr, name: plainName, + LangSymbol(symKind: k.toHumanStr, + name: nimIdentBackticksNormalize(plainName), isGroup: true), priority = symbolPriority(k), # select index `0` just to have any meaningful warning: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 4c28894fb..a04eb696f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -993,7 +993,7 @@ proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, result.add s else: # when there are many overloads a link like foo_ points to all # of them, so selecting the group - var foundGroup = true + var foundGroup = false for s in sList: if s.langSym.isGroup: result.add s diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index c64039941..574ac5b9b 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -207,6 +207,13 @@ window.addEventListener('DOMContentLoaded', main);
                • fn10(a: int): int
                  • +
                +
                +
              + +
              + +
              +
              +
              func fN11() {....raises: [], tags: [].}
              +
              + + + +
              +
              +
              +
              func fN11(x: int) {....raises: [], tags: [].}
              +
              + + +
              diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index 441bd8675..5c8fee527 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -17,6 +17,8 @@ fn7 subdir/subdir_b/utils.html#fn7 utils: fn7() fn8 subdir/subdir_b/utils.html#fn8 utils: fn8(): auto fn9 subdir/subdir_b/utils.html#fn9,int utils: fn9(a: int): int fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int +fN11 subdir/subdir_b/utils.html#fN11 utils: fN11() +fN11 subdir/subdir_b/utils.html#fN11,int utils: fN11(x: int) aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index da76334d4..88d4b86aa 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -209,6 +209,12 @@ window.addEventListener('DOMContentLoaded', main);
            • utils: fn10(a: int): int
            +
            fN11:
            fn2:
            • utils: fn2()
              • diff --git a/nimdoc/testproject/subdir/subdir_b/utils.nim b/nimdoc/testproject/subdir/subdir_b/utils.nim index 5d881e620..e201a3d38 100644 --- a/nimdoc/testproject/subdir/subdir_b/utils.nim +++ b/nimdoc/testproject/subdir/subdir_b/utils.nim @@ -36,6 +36,7 @@ Note that `proc` can be used in postfix form: `binarySearch proc`_. Ref. type like G_ and `type G`_ and `G[T]`_ and `type G*[T]`_. +Group ref. with capital letters works: fN11_ or fn11_ ]## include ./utils_helpers @@ -77,6 +78,11 @@ proc fn8*(): auto = func fn9*(a: int): int = 42 ## comment func fn10*(a: int): int = a ## comment +# Note capital letter N will be handled correctly in +# group references like fN11_ or fn11_: +func fN11*() = discard +func fN11*(x: int) = discard + # bug #9235 template aEnum*(): untyped = From 7a5314c571ec2c370f63d5fab0afd336e04ec161 Mon Sep 17 00:00:00 2001 From: pkubaj Date: Thu, 23 Dec 2021 02:52:33 +0100 Subject: [PATCH 0605/1654] Fix build on FreeBSD/powerpc (#19282) It's currently misdetected as powerpc64. --- tools/niminst/buildsh.nimf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 464c70abf..5f1db9cd3 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -192,8 +192,10 @@ case $ucpu in mycpu="powerpc64" ;; *power*|*ppc* ) if [ "$myos" = "freebsd" ] ; then - COMP_FLAGS="$COMP_FLAGS -m64" - LINK_FLAGS="$LINK_FLAGS -m64" + if [ "$ucpu" != "powerpc" ] ; then + COMP_FLAGS="$COMP_FLAGS -m64" + LINK_FLAGS="$LINK_FLAGS -m64" + fi mycpu=`uname -p` case $mycpu in powerpc64le) From 81d32cf7e53b244776c4196098be154c1223d726 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 23 Dec 2021 07:03:40 +0100 Subject: [PATCH 0606/1654] Revert "Update uri.nim (#19148) [backport:1.0]" (#19280) This reverts commit a3ef5df680e55d9bf68027fcb0ec6358b4279d09. --- lib/pure/uri.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 5de333b8f..a828298c2 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -216,8 +216,6 @@ func parseAuthority(authority: string, result: var Uri) = result.isIpv6 = true of ']': inIPv6 = false - of '\0': - break else: if inPort: result.port.add(authority[i]) From fa96e56ad06f0562b5d608cc896a9b17c9512958 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Wed, 22 Dec 2021 23:12:56 -0700 Subject: [PATCH 0607/1654] Fixed object field access of static objects in generics (#19283) [backport] --- compiler/semexprs.nim | 2 +- tests/system/tstatic.nim | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 0df332689..c593efe55 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1414,7 +1414,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass: ty = ty.lastSon - ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink}) + ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink, tyStatic}) while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias}) var check: PNode = nil if ty.kind == tyObject: diff --git a/tests/system/tstatic.nim b/tests/system/tstatic.nim index 1f1b9dbe3..6e2893e2b 100644 --- a/tests/system/tstatic.nim +++ b/tests/system/tstatic.nim @@ -44,6 +44,14 @@ template main() = proc parseInt(f: static[bool]): int {.used.} = discard doAssert "123".parseInt == 123 + block: + type + MyType = object + field: float32 + AType[T: static MyType] = distinct range[0f32 .. T.field] + var a: AType[MyType(field: 5f32)] + proc n(S: static Slice[int]): range[S.a..S.b] = discard + assert typeof(n 1..2) is range[1..2] static: main() From fdbec969d8601af0f666426b523e8ebba644ab56 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Sat, 25 Dec 2021 18:31:35 +0900 Subject: [PATCH 0608/1654] Fix #19107 (#19286) [backport] --- lib/pure/strformat.nim | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 746f01ace..40a33951c 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -574,6 +574,9 @@ template formatValue(result: var string; value: cstring; specifier: string) = result.add value proc strformatImpl(f: string; openChar, closeChar: char): NimNode = + template missingCloseChar = + error("invalid format string: missing closing character '" & closeChar & "'") + if openChar == ':' or closeChar == ':': error "openChar and closeChar must not be ':'" var i = 0 @@ -618,6 +621,8 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = let start = i inc i i += f.skipWhitespace(i) + if i == f.len: + missingCloseChar if f[i] == closeChar or f[i] == ':': result.add newCall(bindSym"add", res, newLit(subexpr & f[start ..< i])) else: @@ -627,6 +632,9 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = subexpr.add f[i] inc i + if i == f.len: + missingCloseChar + var x: NimNode try: x = parseExpr(subexpr) @@ -639,10 +647,10 @@ proc strformatImpl(f: string; openChar, closeChar: char): NimNode = while i < f.len and f[i] != closeChar: options.add f[i] inc i + if i == f.len: + missingCloseChar if f[i] == closeChar: inc i - else: - doAssert false, "invalid format string: missing '}'" result.add newCall(formatSym, res, x, newLit(options)) elif f[i] == closeChar: if i Date: Mon, 27 Dec 2021 17:28:19 +0100 Subject: [PATCH 0609/1654] fixes grammar typos [backport] (#19289) --- compiler/parser.nim | 6 +++--- doc/grammar.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 9ff847ef6..7bc2c78e0 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -582,10 +582,10 @@ proc parsePar(p: var Parser): PNode = #| | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' #| | 'when' | 'var' | 'mixin' #| par = '(' optInd - #| ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';' - #| | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';' + #| ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';' + #| | ';' (ifExpr / complexOrSimpleStmt) ^+ ';' #| | pragmaStmt - #| | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? ) + #| | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? ) #| | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) #| optPar ')' # diff --git a/doc/grammar.txt b/doc/grammar.txt index 8f86dd98c..bf6d381de 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -37,10 +37,10 @@ parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' | 'when' | 'var' | 'mixin' par = '(' optInd - ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';' - | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';' + ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';' + | ';' (ifExpr / complexOrSimpleStmt) ^+ ';' | pragmaStmt - | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? ) + | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? ) | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) optPar ')' literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT From 77ad8b81e414088c3116bf33f96efacd27eeb471 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Wed, 29 Dec 2021 03:26:54 -0300 Subject: [PATCH 0610/1654] fix 19292 (#19293) --- lib/std/private/win_setenv.nim | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 067e656a3..91c3f15b0 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -23,20 +23,21 @@ check errno_t vs cint when not defined(windows): discard else: + type wchar_t {.importc: "wchar_t".} = int16 + proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} # same as winlean.setEnvironmentVariableA proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} var errno {.importc, header: "".}: cint - type wchar_t {.importc: "wchar_t".} = int16 var gWenviron {.importc:"_wenviron".}: ptr ptr wchar_t # xxx `ptr UncheckedArray[WideCString]` did not work - proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: WideCString, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} + proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: ptr wchar_t, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} # xxx cint vs errno_t? proc setEnvImpl*(name: cstring, value: cstring, overwrite: cint): cint = @@ -77,13 +78,15 @@ else: if gWenviron != nil: # var buf: array[MAX_ENV + 1, WideCString] var buf: array[MAX_ENV + 1, Utf16Char] - let buf2 = cast[WideCString](buf[0].addr) + let buf2 = cast[ptr wchar_t](buf[0].addr) var len: csize_t if mbstowcs_s(len.addr, buf2, buf.len.csize_t, name, MAX_ENV) != 0: errno = EINVAL return -1 - c_wgetenv(buf2)[0] = '\0'.Utf16Char - c_wgetenv(buf2)[1] = '='.Utf16Char + let ptrToEnv = cast[WideCString](c_wgetenv(buf2)) + ptrToEnv[0] = '\0'.Utf16Char + let ptrToEnv2 = cast[WideCString](c_wgetenv(buf2)) + ptrToEnv2[1] = '='.Utf16Char # And now, we have to update the outer environment to have a proper empty value. if setEnvironmentVariableA(name, value) == 0: From a9223d4cdd2e6c23506b666a0958cd2ae3bf5dfd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 29 Dec 2021 12:18:32 +0100 Subject: [PATCH 0611/1654] nep1: make header reflect reality (#19294) --- doc/nep1.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/nep1.rst b/doc/nep1.rst index fc69bab69..8eb31332f 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -1,6 +1,6 @@ -========================================================== -Nim Enhancement Proposal #1 - Standard Library Style Guide -========================================================== +============================ +Standard Library Style Guide +============================ :Author: Clay Sweetser, Dominik Picheta :Version: |nimversion| From 23d64be860958fb7149baed44f6e493394950d69 Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Thu, 30 Dec 2021 20:10:36 +0000 Subject: [PATCH 0612/1654] enable `maxDescriptors` on Illumos/Solaris (#19295) --- lib/pure/asyncdispatch.nim | 4 ++-- lib/pure/selectors.nim | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 20ad60427..5e2a8d98e 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1974,11 +1974,11 @@ when defined(posix): import posix when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is - ## supported on the following OSes: Windows, Linux, OSX, BSD. + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. when defined(windows): result = 16_700_000 elif defined(zephyr) or defined(freertos): diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index ec441f6da..313702370 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -324,11 +324,11 @@ else: doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos): template maxDescriptors*(): int = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is - ## supported on the following OSes: Windows, Linux, OSX, BSD. + ## supported on the following OSes: Windows, Linux, OSX, BSD, Solaris. when defined(windows): 16_700_000 elif defined(zephyr) or defined(freertos): From dc5c88ca79e871aefa76c7fefae4282c639a3283 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 30 Dec 2021 17:52:48 -0300 Subject: [PATCH 0613/1654] Fix #19297 - fixing broken list after adding empty list (#19299) * Update lists.nim * Update tlists.nim * removed check `if b.tail != nil` The tail of the list being null it is still possible to retrieve its end by going through all nodes from the head. So checking for null from `b.tail` is unnecessary. However, setting `a.tail = b.tail` only if `a.head != nil`, so you don't break a good list with an already broken one. --- lib/pure/collections/lists.nim | 11 ++++++----- tests/stdlib/tlists.nim | 12 ++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 8b255fb60..de5550d6e 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -530,11 +530,12 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} = ci assert s == [0, 1, 0, 1, 0, 1] - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if b.head != nil: + if a.head == nil: + a.head = b.head + else: + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 54fa8b24f..2f6d5ee6a 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -233,6 +233,18 @@ template main = doAssert l.toSeq == [1] doAssert l.remove(l.head) == true doAssert l.toSeq == [] + + block issue19297: # add (appends a shallow copy) + var a: SinglyLinkedList[int] + var b: SinglyLinkedList[int] + + doAssert a.toSeq == @[] + a.add(1) + doAssert a.toSeq == @[1] + a.add(b) + doAssert a.toSeq == @[1] + a.add(2) + doAssert a.toSeq == @[1, 2] static: main() main() From ac37eed5a28042d50e23d9ae145f2d46b8dc5058 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 31 Dec 2021 09:21:30 +0100 Subject: [PATCH 0614/1654] fixes #16617 [backport] (#19300) --- compiler/typeallowed.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index d00aa538f..78cd74e56 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -57,6 +57,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, of tyVar, tyLent: if kind in {skProc, skFunc, skConst} and (views notin c.features): result = t + elif taIsOpenArray in flags: + result = t elif t.kind == tyLent and ((kind != skResult and views notin c.features) or kind == skParam): # lent can't be used as parameters. result = t From faaf271928acaa19b53c40d9c5b37e6441cb96c1 Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sat, 1 Jan 2022 15:33:16 +0300 Subject: [PATCH 0615/1654] Update JS and nimscript import tests (#19306) * add new modules, except experimental ones * remove deprecated modules mersenne and sharedlist * better describe why some modules fail and some modules don't --- tests/js/tstdlib_imports.nim | 43 +++++++++++++++++-------------- tests/test_nimscript.nims | 50 +++++++++++++++++++++--------------- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/tests/js/tstdlib_imports.nim b/tests/js/tstdlib_imports.nim index 6fe3772d3..9986fa059 100644 --- a/tests/js/tstdlib_imports.nim +++ b/tests/js/tstdlib_imports.nim @@ -7,46 +7,48 @@ discard """ import std/[ # Core: bitops, typetraits, lenientops, macros, volatile, typeinfo, - # fails: endians, rlocks - # works but shouldn't: cpuinfo, locks + # fails due to FFI: rlocks + # fails due to cstring cast/copyMem: endians + # works but uses FFI: cpuinfo, locks # Algorithms: - algorithm, sequtils, + algorithm, enumutils, sequtils, setutils, # Collections: critbits, deques, heapqueue, intsets, lists, options, sets, - sharedlist, tables, - # fails: sharedtables + tables, packedsets, # Strings: cstrutils, editdistance, wordwrap, parseutils, ropes, pegs, punycode, strformat, strmisc, strscans, strtabs, strutils, unicode, unidecode, - # fails: encodings + # fails due to FFI: encodings # Time handling: monotimes, times, # Generic operator system services: os, streams, - # fails: distros, dynlib, marshal, memfiles, osproc, terminal + # fails intentionally: dynlib, marshal, memfiles + # fails due to FFI: osproc, terminal + # fails due to osproc import: distros # Math libraries: - complex, math, mersenne, random, rationals, stats, sums, - # works but shouldn't: fenv + complex, math, random, rationals, stats, sums, sysrand, + # works but uses FFI: fenv # Internet protocols: cookies, httpcore, mimetypes, uri, - # fails: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, + # fails due to FFI: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, # asyncnet, cgi, httpclient, nativesockets, net, selectors, smtp - # works but shouldn't test: asyncstreams, asyncfutures + # works but no need to test: asyncstreams, asyncfutures # Threading: - # fails: threadpool + # fails due to FFI: threadpool # Parsers: htmlparser, json, lexbase, parsecfg, parsecsv, parsesql, parsexml, - parseopt, + parseopt, jsonutils, # XML processing: xmltree, xmlparser, @@ -56,16 +58,19 @@ import std/[ # Hashing: base64, hashes, - # fails: md5, oids, sha1 + # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + # fails due to cstring cast/endians import: oids + # fails due to copyMem/endians import: sha1 # Miscellaneous: - colors, logging, sugar, unittest, varints, - # fails: browsers, coro - # works but shouldn't: segfaults + colors, logging, sugar, unittest, varints, enumerate, with, + # fails due to FFI: browsers, coro + # works but uses FFI: segfaults # Modules for JS backend: - asyncjs, dom, jsconsole, jscore, jsffi, + asyncjs, dom, jsconsole, jscore, jsffi, jsbigints, # Unlisted in lib.html: - decls, compilesettings, with, wrapnils + decls, compilesettings, wrapnils, exitprocs, effecttraits, + genasts, importutils, isolation, jsfetch, jsformdata, jsheaders ] diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index ecb409677..761ddafdf 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -8,47 +8,49 @@ from stdtest/specialpaths import buildDir import std/[ # Core: bitops, typetraits, lenientops, macros, volatile, - # fails: typeinfo, endians - # works but shouldn't: cpuinfo, rlocks, locks + # fails due to FFI: typeinfo + # fails due to cstring cast/copyMem: endians + # works but uses FFI: cpuinfo, rlocks, locks # Algorithms: - algorithm, sequtils, + algorithm, enumutils, sequtils, setutils, # Collections: critbits, deques, heapqueue, intsets, lists, options, sets, - sharedlist, tables, - # fails: sharedtables + tables, packedsets, # Strings: editdistance, wordwrap, parseutils, ropes, pegs, punycode, strformat, strmisc, strscans, strtabs, - strutils, unicode, unidecode, - # works but shouldn't: cstrutils, encodings + strutils, unicode, unidecode, cstrutils, + # works but uses FFI: encodings # Time handling: - # fails: monotimes, times + # fails due to FFI: monotimes, times # but times.getTime() implemented for VM # Generic operator system services: - os, streams, - # fails: distros, dynlib, marshal, memfiles, osproc, terminal + os, streams, distros, + # fails due to FFI: memfiles, osproc, terminal + # works but uses FFI: dynlib + # intentionally fails: marshal # Math libraries: - complex, math, mersenne, random, rationals, stats, sums, - # works but shouldn't: fenv + complex, math, random, rationals, stats, sums, + # works but uses FFI: fenv, sysrand # Internet protocols: httpcore, mimetypes, uri, - # fails: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, + # fails due to FFI: asyncdispatch, asyncfile, asyncftpclient, asynchttpserver, # asyncnet, cgi, cookies, httpclient, nativesockets, net, selectors, smtp - # works but shouldn't test: asyncstreams, asyncfutures + # works but no need to test: asyncstreams, asyncfutures # Threading: - # fails: threadpool + # fails due to FFI: threadpool # Parsers: htmlparser, json, lexbase, parsecfg, parsecsv, parsesql, parsexml, - parseopt, + parseopt, jsonutils, # XML processing: xmltree, xmlparser, @@ -58,17 +60,23 @@ import std/[ # Hashing: base64, hashes, - # fails: md5, oids, sha1 + # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + # fails due to cstring cast/times import/endians import: oids + # fails due to copyMem/endians import: sha1 # Miscellaneous: - colors, sugar, varints, - # fails: browsers, coro, logging (times), segfaults, unittest (uses methods) + colors, sugar, varints, enumerate, with, + # fails due to FFI: browsers, coro, segfaults + # fails due to times import/methods: logging + # fails due to methods: unittest # Modules for JS backend: - # fails: asyncjs, dom, jsconsole, jscore, jsffi, + # fails intentionally: asyncjs, dom, jsconsole, jscore, jsffi, jsbigints, + # jsfetch, jsformdata, jsheaders # Unlisted in lib.html: - decls, compilesettings, with, wrapnils + decls, compilesettings, wrapnils, effecttraits, genasts, + importutils, isolation ] # non-std imports From ef634cc2516ce81a05486f726e423ac458c4467a Mon Sep 17 00:00:00 2001 From: rockcavera Date: Sat, 1 Jan 2022 09:33:29 -0300 Subject: [PATCH 0616/1654] Update manual.rst (#19301) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index d9f5d4ea6..8788e53a3 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5018,7 +5018,7 @@ be used: See also: -- `Shared heap memory management `_. +- `Shared heap memory management `_. From 61d682e733005534a16af72fc9b0c0e74d2f9ebf Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sun, 2 Jan 2022 09:29:55 +0300 Subject: [PATCH 0617/1654] Allow full commands and blocks in type sections (#19181) * allow full commands and blocks in type sections * update grammar * fix changelog [skip ci] * more tests * even more tests --- changelog.md | 3 + compiler/parser.nim | 28 ++- doc/grammar.txt | 5 +- tests/parser/ttypecommandcomma.nim | 9 + tests/parser/ttypecommandindent1.nim | 9 + tests/parser/ttypecommandindent2.nim | 11 + tests/parser/ttypecommandindent3.nim | 11 + tests/parser/ttypesectioncalls.nim | 328 +++++++++++++++++++++++++++ 8 files changed, 393 insertions(+), 11 deletions(-) create mode 100644 tests/parser/ttypecommandcomma.nim create mode 100644 tests/parser/ttypecommandindent1.nim create mode 100644 tests/parser/ttypecommandindent2.nim create mode 100644 tests/parser/ttypecommandindent3.nim create mode 100644 tests/parser/ttypesectioncalls.nim diff --git a/changelog.md b/changelog.md index 52159b865..89aaff664 100644 --- a/changelog.md +++ b/changelog.md @@ -61,6 +61,9 @@ ``` - [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed + for the right-hand side of type definitions in type sections. Previously + they would error with "invalid indentation". ## Compiler changes diff --git a/compiler/parser.nim b/compiler/parser.nim index 7bc2c78e0..0fb60440c 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -811,7 +811,7 @@ proc primarySuffix(p: var Parser, r: PNode, #| | DOTLIKEOP optInd symbol generalizedLit? #| | '[' optInd exprColonEqExprList optPar ']' #| | '{' optInd exprColonEqExprList optPar '}' - #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax + #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr (comma expr)* # command syntax result = r # progress guaranteed @@ -821,14 +821,14 @@ proc primarySuffix(p: var Parser, r: PNode, of tkParLe: # progress guaranteed if p.tok.strongSpaceA > 0: - # inside type sections, expressions such as `ref (int, bar)` - # are parsed as a nkCommand with a single tuple argument (nkPar) + result = commandExpr(p, result, mode) + # type sections allow full command syntax if mode == pmTypeDef: - result = newNodeP(nkCommand, p) - result.add r - result.add primary(p, pmNormal) - else: - result = commandExpr(p, result, mode) + var isFirstParam = false + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, result) + result.add(commandParam(p, isFirstParam, mode)) break result = namedParams(p, result, nkCall, tkParRi) if result.len > 1 and result[1].kind == nkExprColonExpr: @@ -869,9 +869,18 @@ proc primarySuffix(p: var Parser, r: PNode, # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet # solution, but pragmas.nim can't handle that result = commandExpr(p, result, mode) + if mode == pmTypeDef: + var isFirstParam = false + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, result) + result.add(commandParam(p, isFirstParam, mode)) break else: break + # type sections allow post-expr blocks + if mode == pmTypeDef: + result = postExprBlocks(p, result) proc parseOperators(p: var Parser, headNode: PNode, limit: int, mode: PrimaryMode): PNode = @@ -1342,7 +1351,8 @@ proc parseTypeDesc(p: var Parser): PNode = result = binaryNot(p, result) proc parseTypeDefAux(p: var Parser): PNode = - #| typeDefAux = simpleExpr ('not' expr)? + #| typeDefAux = simpleExpr ('not' expr + #| | postExprBlocks)? result = simpleExpr(p, pmTypeDef) result = binaryNot(p, result) diff --git a/doc/grammar.txt b/doc/grammar.txt index bf6d381de..1e41b9eeb 100644 --- a/doc/grammar.txt +++ b/doc/grammar.txt @@ -60,7 +60,7 @@ primarySuffix = '(' (exprColonEqExpr comma?)* ')' | DOTLIKEOP optInd symbol generalizedLit? | '[' optInd exprColonEqExprList optPar ']' | '{' optInd exprColonEqExprList optPar '}' - | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax + | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr (comma expr)* # command syntax pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}') identVis = symbol OPR? # postfix position identVisDot = symbol '.' optInd symbol OPR? @@ -93,7 +93,8 @@ primary = operatorB primary primarySuffix* | ('var' | 'out' | 'ref' | 'ptr' | 'distinct') primary / prefixOperator* identOrLiteral primarySuffix* typeDesc = simpleExpr ('not' expr)? -typeDefAux = simpleExpr ('not' expr)? +typeDefAux = simpleExpr ('not' expr + | postExprBlocks)? postExprBlocks = ':' stmt? ( IND{=} doBlock | IND{=} 'of' exprList ':' stmt | IND{=} 'elif' expr ':' stmt diff --git a/tests/parser/ttypecommandcomma.nim b/tests/parser/ttypecommandcomma.nim new file mode 100644 index 000000000..7ca59a799 --- /dev/null +++ b/tests/parser/ttypecommandcomma.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "invalid indentation" + line: 8 + column: 19 +""" + +type + Foo = call(1, 2), 3: + 4 \ No newline at end of file diff --git a/tests/parser/ttypecommandindent1.nim b/tests/parser/ttypecommandindent1.nim new file mode 100644 index 000000000..9aa274e2a --- /dev/null +++ b/tests/parser/ttypecommandindent1.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "invalid indentation" + line: 9 + column: 3 +""" + +type + Foo = call x, y, z: + abc diff --git a/tests/parser/ttypecommandindent2.nim b/tests/parser/ttypecommandindent2.nim new file mode 100644 index 000000000..0883df90f --- /dev/null +++ b/tests/parser/ttypecommandindent2.nim @@ -0,0 +1,11 @@ +discard """ + errormsg: "invalid indentation" + line: 10 + column: 5 +""" + +type + Foo = call x, y, z: + abc + do: + def diff --git a/tests/parser/ttypecommandindent3.nim b/tests/parser/ttypecommandindent3.nim new file mode 100644 index 000000000..f9fdcc1e4 --- /dev/null +++ b/tests/parser/ttypecommandindent3.nim @@ -0,0 +1,11 @@ +discard """ + errormsg: "expression expected, but found 'keyword do'" + line: 10 + column: 1 +""" + +type + Foo = call x, y, z: + abc +do: + def diff --git a/tests/parser/ttypesectioncalls.nim b/tests/parser/ttypesectioncalls.nim new file mode 100644 index 000000000..003444fc5 --- /dev/null +++ b/tests/parser/ttypesectioncalls.nim @@ -0,0 +1,328 @@ +discard """ +nimout: ''' +StmtList + TypeSection + TypeDef + Ident "A" + Empty + Call + Ident "call" + IntLit 1 + TypeSection + TypeDef + Ident "B" + Empty + Command + Ident "call" + IntLit 2 + TypeDef + Ident "C" + Empty + Call + Ident "call" + StmtList + IntLit 3 + TypeDef + Ident "D" + Empty + Call + Ident "call" + StmtList + IntLit 4 + TypeSection + TypeDef + Ident "E" + Empty + Call + Ident "call" + IntLit 5 + IntLit 6 + TypeDef + Ident "F" + Empty + Command + Ident "call" + IntLit 7 + IntLit 8 + TypeDef + Ident "G" + Empty + Call + Ident "call" + IntLit 9 + StmtList + IntLit 10 + TypeDef + Ident "H" + Empty + Call + Ident "call" + IntLit 11 + StmtList + IntLit 12 + TypeDef + Ident "I" + Empty + Command + Ident "call" + IntLit 13 + StmtList + IntLit 14 + TypeDef + Ident "J" + Empty + Command + Ident "call" + IntLit 15 + StmtList + IntLit 16 + TypeSection + TypeDef + Ident "K" + Empty + Call + Ident "call" + IntLit 17 + IntLit 18 + IntLit 19 + TypeDef + Ident "L" + Empty + Command + Ident "call" + IntLit 20 + IntLit 21 + IntLit 22 + TypeDef + Ident "M" + Empty + Call + Ident "call" + IntLit 23 + IntLit 24 + StmtList + IntLit 25 + TypeDef + Ident "N" + Empty + Command + Ident "call" + IntLit 26 + IntLit 27 + StmtList + IntLit 28 + TypeDef + Ident "O" + Empty + Command + Ident "call" + IntLit 29 + IntLit 30 + StmtList + IntLit 31 + TypeSection + TypeDef + Ident "P" + Empty + Command + Ident "call" + TupleConstr + IntLit 32 + IntLit 33 + Infix + Ident "+" + Infix + Ident "*" + IntLit 34 + IntLit 35 + IntLit 36 + StmtList + IntLit 37 + TypeDef + Ident "R" + Empty + Command + Ident "call" + Infix + Ident "@" + TupleConstr + IntLit 38 + IntLit 39 + Infix + Ident "shl" + IntLit 40 + IntLit 41 + Infix + Ident "-" + Infix + Ident "*" + IntLit 42 + IntLit 43 + IntLit 44 + StmtList + IntLit 45 + TypeDef + Ident "S" + Empty + Command + Ident "call" + IntLit 46 + StmtList + IntLit 47 + StmtList + IntLit 48 + TypeDef + Ident "T" + Empty + Call + Ident "call" + StmtList + IntLit 49 + StmtList + IntLit 50 + StmtList + IntLit 51 +a: IntLit 1 +a: IntLit 2 +a: StmtList + IntLit 3 +a: StmtList + IntLit 4 +a: IntLit 5 +b: IntLit 6 +a: IntLit 7 +b: IntLit 8 +a: IntLit 9 +b: StmtList + IntLit 10 +a: IntLit 11 +b: StmtList + IntLit 12 +a: IntLit 13 +b: StmtList + IntLit 14 +a: IntLit 15 +b: StmtList + IntLit 16 +a: IntLit 17 +b: IntLit 18 +c: IntLit 19 +a: IntLit 20 +b: IntLit 21 +c: IntLit 22 +a: IntLit 23 +b: IntLit 24 +c: StmtList + IntLit 25 +a: IntLit 26 +b: IntLit 27 +c: StmtList + IntLit 28 +a: IntLit 29 +b: IntLit 30 +c: StmtList + IntLit 31 +a: TupleConstr + IntLit 32 + IntLit 33 +b: Infix + Ident "+" + Infix + Ident "*" + IntLit 34 + IntLit 35 + IntLit 36 +c: StmtList + IntLit 37 +a: Infix + Ident "@" + TupleConstr + IntLit 38 + IntLit 39 + Infix + Ident "shl" + IntLit 40 + IntLit 41 +b: Infix + Ident "-" + Infix + Ident "*" + IntLit 42 + IntLit 43 + IntLit 44 +c: StmtList + IntLit 45 +a: IntLit 46 +b: StmtList + IntLit 47 +c: StmtList + IntLit 48 +a: StmtList + IntLit 49 +b: StmtList + IntLit 50 +c: StmtList + IntLit 51 +''' +""" +import macros + +macro call(a): untyped = + echo "a: ", a.treeRepr + result = ident"int" +macro call(a, b): untyped = + echo "a: ", a.treeRepr + echo "b: ", b.treeRepr + result = ident"int" +macro call(a, b, c): untyped = + echo "a: ", a.treeRepr + echo "b: ", b.treeRepr + echo "c: ", c.treeRepr + result = ident"int" + +macro sections(x): untyped = + echo x.treeRepr + result = newStmtList(x) + for ts in x: + for td in ts: + let t = td[0] + result.add quote do: + doAssert `t` is int + +sections: + type A = call(1) + type + B = call 2 + C = call: 3 + D = call(): 4 + type + E = call(5, 6) + F = call 7, 8 + G = call(9): 10 + H = call(11): + 12 + I = call 13: 14 + J = call 15: + 16 + type + K = call(17, 18, 19) + L = call 20, 21, 22 + M = call(23, 24): 25 + N = call 26, 27: 28 + O = call 29, 30: + 31 + type + P = call (32, 33), 34 * 35 + 36: + 37 + R = call (38, 39) @ 40 shl 41, 42 * 43 - 44: + 45 + S = call 46: + 47 + do: + 48 + T = call: + 49 + do: + 50 + do: + 51 From 53e1d57419431050a54dcdf539a2a02bf6d28c5b Mon Sep 17 00:00:00 2001 From: BarrOff <58253563+BarrOff@users.noreply.github.com> Date: Sun, 2 Jan 2022 06:37:13 +0000 Subject: [PATCH 0618/1654] add compile time option for POSIX sigwait on Illumos/Solaris (#19296) * add compile time option for POSIX sigwait on Illumos/Solaris * fix link to documentation of `sigwait` on Illumos/Solaris --- lib/posix/posix.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 57dd6e2c7..912e395a7 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -772,6 +772,13 @@ else: proc sigtimedwait*(a1: var Sigset, a2: var SigInfo, a3: var Timespec): cint {.importc, header: "".} +when defined(sunos) or defined(solaris): + # The following compile time flag is needed on Illumos/Solaris to use the POSIX + # `sigwait` implementation. See the documentation here: + # https://docs.oracle.com/cd/E19455-01/806-5257/6je9h033k/index.html + # https://www.illumos.org/man/2/sigwait + {.passc: "-D_POSIX_PTHREAD_SEMANTICS".} + proc sigwait*(a1: var Sigset, a2: var cint): cint {. importc, header: "".} proc sigwaitinfo*(a1: var Sigset, a2: var SigInfo): cint {. From bbd5086bc319dc22971a06ef24d02dc49ff308f3 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 3 Jan 2022 16:02:58 +0800 Subject: [PATCH 0619/1654] [docs] clarify the raised exception (#19308) * [docs] clarify the raised exception Lest developers wanna know what the exception is. * Apply suggestions from @konsumlamm Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- lib/pure/uri.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index a828298c2..7583dbd1e 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -151,10 +151,10 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add(encodeUrl(val, usePlus)) iterator decodeQuery*(data: string): tuple[key, value: string] = - ## Reads and decodes query string `data` and yields the `(key, value)` pairs - ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, an - ## error is raised when there is an unencoded `=` character in a decoded - ## value, which was the behavior in Nim < 1.5.1 + ## Reads and decodes the query string `data` and yields the `(key, value)` pairs + ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, + ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded + ## value, which was the behavior in Nim < 1.5.1. runnableExamples: import std/sequtils assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] From e49d52eb6174b2721293968e7811928c303c3776 Mon Sep 17 00:00:00 2001 From: Carlo Capocasa Date: Mon, 3 Jan 2022 09:11:23 +0100 Subject: [PATCH 0620/1654] Add Week-Of-Year Implementation to Times Module (#17223) * initial * more tests * Apply suggestions from code review idiomatize Co-authored-by: Timothee Cour * test iron age dates * add examples * fix typo * consistent param mention * add since pragrams * add changelog * Update lib/pure/times.nim Co-authored-by: Timothee Cour * fix examples * fix negative years * add getWeeksInYear tests * add back fix dropped by rebase * week-year tuple api * add changelog * fix doc tags * add docstrings * fix typos Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: Timothee Cour Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 6 +++ lib/pure/times.nim | 82 +++++++++++++++++++++++++++++++++++ tests/stdlib/ttimes.nim | 95 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+) diff --git a/changelog.md b/changelog.md index 89aaff664..1b97fd99a 100644 --- a/changelog.md +++ b/changelog.md @@ -27,6 +27,12 @@ - Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. +- Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. +- Added `IsoYear`, a distinct int type to prevent bugs from confusing the week-based year and the regular year. +- Added `initDateTime` in `times` to create a datetime from a weekday, and ISO 8601 week number and week-based year. +- Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. +- Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 113f73d2a..7851bf158 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -288,6 +288,13 @@ type YeardayRange* = range[0..365] NanosecondRange* = range[0..999_999_999] + IsoWeekRange* = range[1 .. 53] + ## An ISO 8601 calendar week number. + IsoYear* = distinct int + ## An ISO 8601 calendar year number. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + Time* = object ## Represents a point in time. seconds: int64 nanosecond: NanosecondRange @@ -529,6 +536,54 @@ proc getDaysInYear*(year: int): int = doAssert getDaysInYear(2001) == 365 result = 365 + (if isLeapYear(year): 1 else: 0) +proc `==`*(a, b: IsoYear): bool {.borrow.} +proc `$`*(p: IsoYear): string {.borrow.} + +proc getWeeksInIsoYear*(y: IsoYear): IsoWeekRange {.since: (1, 5).} = + ## Returns the number of weeks in the specified ISO 8601 week-based year, which can be + ## either 53 or 52. + runnableExamples: + assert getWeeksInIsoYear(IsoYear(2000)) == 52 + assert getWeeksInIsoYear(IsoYear(2001)) == 53 + + var y = int(y) + + # support negative years + y = if y < 0: 400 + y mod 400 else: y + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let p = (y + (y div 4) - (y div 100) + (y div 400)) mod 7 + let y1 = y - 1 + let p1 = (y1 + (y1 div 4) - (y1 div 100) + (y1 div 400)) mod 7 + if p == 4 or p1 == 3: 53 else: 52 + +proc getIsoWeekAndYear*(dt: DateTime): + tuple[isoweek: IsoWeekRange, isoyear: IsoYear] {.since: (1, 5).} = + ## Returns the ISO 8601 week and year. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert getIsoWeekAndYear(initDateTime(21, mApr, 2018, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2018.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) + assert w == 01.IsoWeekRange + assert y == 2020.IsoYear + assert getIsoWeekAndYear(initDateTime(13, mSep, 2020, 00, 00, 00)) == (isoweek: 37.IsoWeekRange, isoyear: 2020.IsoYear) + block: + let (w, y) = getIsoWeekAndYear(initDateTime(2, mJan, 2021, 00, 00, 00)) + assert w.int > 52 + assert w.int < 54 + assert y.int mod 100 == 20 + + # source: https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + var w = (dt.yearday.int - dt.weekday.int + 10) div 7 + if w < 1: + (isoweek: getWeeksInIsoYear(IsoYear(dt.year - 1)), isoyear: IsoYear(dt.year - 1)) + elif (w > getWeeksInIsoYear(IsoYear(dt.year))): + (isoweek: IsoWeekRange(1), isoyear: IsoYear(dt.year + 1)) + else: + (isoweek: IsoWeekRange(w), isoyear: IsoYear(dt.year)) + proc stringifyUnit(value: int | int64, unit: TimeUnit): string = ## Stringify time unit with it's name, lowercased let strUnit = $unit @@ -2578,6 +2633,33 @@ proc `+=`*(t: var Time, b: TimeInterval) = proc `-=`*(t: var Time, b: TimeInterval) = t = t - b +# +# Day of year +# + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + nanosecond: NanosecondRange, + zone: Timezone = local()): DateTime {.since: (1, 5).} = + ## Create a new `DateTime <#DateTime>`_ from a weekday and an ISO 8601 week number and year + ## in the specified timezone. + ## + ## .. warning:: The ISO week-based year can correspond to the following or previous year from 29 December to January 3. + runnableExamples: + assert initDateTime(21, mApr, 2018, 00, 00, 00) == initDateTime(dSat, 16, 2018.IsoYear, 00, 00, 00) + assert initDateTime(30, mDec, 2019, 00, 00, 00) == initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) + assert initDateTime(13, mSep, 2020, 00, 00, 00) == initDateTime(dSun, 37, 2020.IsoYear, 00, 00, 00) + assert initDateTime(2, mJan, 2021, 00, 00, 00) == initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) + + # source https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm + let d = isoweek * 7 + weekday.int - initDateTime(4, mJan, isoyear.int, 00, 00, 00).weekday.int - 4 + initDateTime(1, mJan, isoyear.int, hour, minute, second, nanosecond, zone) + initTimeInterval(days=d) + +proc initDateTime*(weekday: WeekDay, isoweek: IsoWeekRange, isoyear: IsoYear, + hour: HourRange, minute: MinuteRange, second: SecondRange, + zone: Timezone = local()): DateTime {.since: (1, 5).} = + initDateTime(weekday, isoweek, isoyear, hour, minute, second, 0, zone) + # # Other # diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index 66157b91c..2cf945786 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -646,3 +646,98 @@ block: # ttimes doAssert initDuration(milliseconds = 500).inMilliseconds == 500 doAssert initDuration(milliseconds = -500).inMilliseconds == -500 doAssert initDuration(nanoseconds = -999999999).inMilliseconds == -999 + + block: # getIsoWeekAndYear + doAssert getIsoWeekAndYear(initDateTime(04, mNov, 2019, 00, 00, 00)) == (isoweek: 45.IsoWeekRange, isoyear: 2019.IsoYear) + doAssert initDateTime(dMon, 45, 2019.IsoYear, 00, 00, 00) == initDateTime(04, mNov, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(28, mDec, 2019, 00, 00, 00)) == (isoweek: 52.IsoWeekRange, isoyear: 2019.IsoYear) + doAssert initDateTime(dSat, 52, 2019.IsoYear, 00, 00, 00) == initDateTime(28, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(29, mDec, 2019, 00, 00, 00)) == (isoweek: 52.IsoWeekRange, isoyear: 2019.IsoYear) + doAssert initDateTime(dSun, 52, 2019.IsoYear, 00, 00, 00) == initDateTime(29, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(30, mDec, 2019, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(30, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(31, mDec, 2019, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dTue, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(31, mDec, 2019, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mJan, 2020, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dWed, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(01, mJan, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(02, mJan, 2020, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 01, 2020.IsoYear, 00, 00, 00) == initDateTime(02, mJan, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(05, mApr, 2020, 00, 00, 00)) == (isoweek: 14.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 14, 2020.IsoYear, 00, 00, 00) == initDateTime(05, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(06, mApr, 2020, 00, 00, 00)) == (isoweek: 15.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 15, 2020.IsoYear, 00, 00, 00) == initDateTime(06, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(10, mApr, 2020, 00, 00, 00)) == (isoweek: 15.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dFri, 15, 2020.IsoYear, 00, 00, 00) == initDateTime(10, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(12, mApr, 2020, 00, 00, 00)) == (isoweek: 15.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 15, 2020.IsoYear, 00, 00, 00) == initDateTime(12, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(13, mApr, 2020, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 16, 2020.IsoYear, 00, 00, 00) == initDateTime(13, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(15, mApr, 2020, 00, 00, 00)) == (isoweek: 16.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 16, 2020.IsoYear, 00, 00, 00) == initDateTime(16, mApr, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(17, mJul, 2020, 00, 00, 00)) == (isoweek: 29.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dFri, 29, 2020.IsoYear, 00, 00, 00) == initDateTime(17, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(19, mJul, 2020, 00, 00, 00)) == (isoweek: 29.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 29, 2020.IsoYear, 00, 00, 00) == initDateTime(19, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(20, mJul, 2020, 00, 00, 00)) == (isoweek: 30.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dMon, 30, 2020.IsoYear, 00, 00, 00) == initDateTime(20, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(23, mJul, 2020, 00, 00, 00)) == (isoweek: 30.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 30, 2020.IsoYear, 00, 00, 00) == initDateTime(23, mJul, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(31, mDec, 2020, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dThu, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(31, mDec, 2020, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mJan, 2021, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dFri, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(01, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(02, mJan, 2021, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSat, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(02, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(03, mJan, 2021, 00, 00, 00)) == (isoweek: 53.IsoWeekRange, isoyear: 2020.IsoYear) + doAssert initDateTime(dSun, 53, 2020.IsoYear, 00, 00, 00) == initDateTime(03, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(04, mJan, 2021, 00, 00, 00)) == (isoweek: 01.IsoWeekRange, isoyear: 2021.IsoYear) + doAssert initDateTime(dMon, 01, 2021.IsoYear, 00, 00, 00) == initDateTime(04, mJan, 2021, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mFeb, 2021, 00, 00, 00)) == (isoweek: 05.IsoWeekRange, isoyear: 2021.IsoYear) + doAssert initDateTime(dMon, 05, 2021.IsoYear, 00, 00, 00) == initDateTime(01, mFeb, 2021, 00, 00, 00) + + doAssert getIsoWeekAndYear(initDateTime(01, mFeb, 2021, 01, 02, 03, 400_000_000, staticTz(hours=1))) == (isoweek: 05.IsoWeekRange, isoyear: 2021.IsoYear) + doAssert initDateTime(dMon, 05, 2021.IsoYear, 01, 02, 03, 400_000_000, staticTz(hours=1)) == initDateTime(01, mFeb, 2021, 01, 02, 03, 400_000_000, staticTz(hours=1)) + + doAssert getIsoWeekAndYear(initDateTime(01, mApr, +0001, 00, 00, 00)) == (isoweek: 13.IsoWeekRange, isoyear: 0001.IsoYear) + doAssert initDateTime(dSun, 13, 0001.IsoYear, 00, 00, 00) == initDateTime(01, mApr, 0001, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, +0000, 00, 00, 00)) == (isoweek: 13.IsoWeekRange, isoyear: 0000.IsoYear) + doAssert initDateTime(dSat, 13, 0000.IsoYear, 00, 00, 00) == initDateTime(01, mApr, 0000, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, -0001, 00, 00, 00)) == (isoweek: 13.IsoWeekRange, isoyear: (-0001).IsoYear) + doAssert initDateTime(dThu, 13, (-0001).IsoYear, 00, 00, 00) == initDateTime(01, mApr, -0001, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, -0002, 00, 00, 00)) == (isoweek: 14.IsoWeekRange, isoyear: (-0002).IsoYear) + doAssert initDateTime(dWed, 14, (-0002).IsoYear, 00, 00, 00) == initDateTime(01, mApr, -0002, 00, 00, 00) + doAssert getIsoWeekAndYear(initDateTime(01, mApr, -0753, 00, 00, 00)) == (isoweek: 14.IsoWeekRange, isoyear: (-0753).IsoYear) + doAssert initDateTime(dMon, 14, (-0753).IsoYear, 00, 00, 00) == initDateTime(01, mApr, -0753, 00, 00, 00) + + block: # getWeeksInIsoYear + doAssert getWeeksInIsoYear((-0014).IsoYear) == 52 + doAssert getWeeksInIsoYear((-0013).IsoYear) == 53 + doAssert getWeeksInIsoYear((-0012).IsoYear) == 52 + + doAssert getWeeksInIsoYear((-0009).IsoYear) == 52 + doAssert getWeeksInIsoYear((-0008).IsoYear) == 53 + doAssert getWeeksInIsoYear((-0007).IsoYear) == 52 + + doAssert getWeeksInIsoYear((-0003).IsoYear) == 52 + doAssert getWeeksInIsoYear((-0002).IsoYear) == 53 + doAssert getWeeksInIsoYear((-0001).IsoYear) == 52 + + doAssert getWeeksInIsoYear(0003.IsoYear) == 52 + doAssert getWeeksInIsoYear(0004.IsoYear) == 53 + doAssert getWeeksInIsoYear(0005.IsoYear) == 52 + + doAssert getWeeksInIsoYear(1653.IsoYear) == 52 + doAssert getWeeksInIsoYear(1654.IsoYear) == 53 + doAssert getWeeksInIsoYear(1655.IsoYear) == 52 + + doAssert getWeeksInIsoYear(1997.IsoYear) == 52 + doAssert getWeeksInIsoYear(1998.IsoYear) == 53 + doAssert getWeeksInIsoYear(1999.IsoYear) == 52 + + doAssert getWeeksInIsoYear(2008.IsoYear) == 52 + doAssert getWeeksInIsoYear(2009.IsoYear) == 53 + doAssert getWeeksInIsoYear(2010.IsoYear) == 52 + + doAssert getWeeksInIsoYear(2014.IsoYear) == 52 + doAssert getWeeksInIsoYear(2015.IsoYear) == 53 + doAssert getWeeksInIsoYear(2016.IsoYear) == 52 From bd4f76a2e3b9282548e62ba3b620b09d6c564373 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Mon, 3 Jan 2022 17:30:26 +0800 Subject: [PATCH 0621/1654] docs: Fix typo in tut1.rst (#19309) --- doc/tut1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 2856625b7..d8e0ec8e4 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -913,7 +913,7 @@ Let's return to the simple counting example: echo i Can a `countup `_ proc be written that -supports this loop? Lets try: +supports this loop? Let's try: .. code-block:: nim proc countup(a, b: int): int = From 19bcb43a0e21dd222ffd3fad8c5fb2d8e85d59ea Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 3 Jan 2022 20:40:43 +0800 Subject: [PATCH 0622/1654] enable multiple packages (arraymancer, fidget ...) (#19311) The cause of arraymancer failure has been tracked here: https://github.com/mratsim/Arraymancer/issues/505 And it was fixed by https://github.com/mratsim/Arraymancer/pull/542 --- testament/important_packages.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 7e15e7c65..381aa27ac 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -35,7 +35,7 @@ proc pkg(name: string; cmd = "nimble test"; url = "", useHead = true, allowFailu pkg "alea", allowFailure = true pkg "argparse" -pkg "arraymancer", "nim c tests/tests_cpu.nim", allowFailure = true +pkg "arraymancer", "nim c tests/tests_cpu.nim" pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true pkg "asyncthreadpool" pkg "awk" @@ -63,7 +63,7 @@ pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" -pkg "fidget", allowFailure = true +pkg "fidget" pkg "fragments", "nim c -r fragments/dsl.nim" pkg "fusion" pkg "gara" @@ -91,7 +91,7 @@ pkg "memo" pkg "msgpack4nim", "nim c -r tests/test_spec.nim" pkg "nake", "nim c nakefile.nim" pkg "neo", "nim c -d:blas=openblas tests/all.nim" -pkg "nesm", "nimble tests", allowFailure = true # notice plural 'tests' +pkg "nesm", "nimble tests" # notice plural 'tests' pkg "netty" pkg "nico", allowFailure = true pkg "nicy", "nim c -r src/nicy.nim" From 526a32e1697489b73e54c1b984ba5413842f1252 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Mon, 3 Jan 2022 16:14:08 -0300 Subject: [PATCH 0623/1654] Fix #19314 - fixing broken `DoublyLinkedList` after adding empty `DoublyLinkedList` (#19315) [backport] * Update lists.nim * Update tlists.nim --- lib/pure/collections/lists.nim | 12 ++++++------ tests/stdlib/tlists.nim | 12 ++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index de5550d6e..f7627ae67 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -675,12 +675,12 @@ proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} = assert s == [0, 1, 0, 1, 0, 1] if b.head != nil: - b.head.prev = a.tail - if a.tail != nil: - a.tail.next = b.head - a.tail = b.tail - if a.head == nil: - a.head = b.head + if a.head == nil: + a.head = b.head + else: + b.head.prev = a.tail + a.tail.next = b.head + a.tail = b.tail if a.addr != b.addr: b.head = nil b.tail = nil diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 2f6d5ee6a..14cbf2f9d 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -245,6 +245,18 @@ template main = doAssert a.toSeq == @[1] a.add(2) doAssert a.toSeq == @[1, 2] + + block issue19314: # add (appends a shallow copy) + var a: DoublyLinkedList[int] + var b: DoublyLinkedList[int] + + doAssert a.toSeq == @[] + a.add(1) + doAssert a.toSeq == @[1] + a.add(b) + doAssert a.toSeq == @[1] + a.add(2) + doAssert a.toSeq == @[1, 2] static: main() main() From fcf1df6e56934b1b5021a0898b9152b86e0cb5f7 Mon Sep 17 00:00:00 2001 From: Smarcy Date: Mon, 3 Jan 2022 21:04:15 +0100 Subject: [PATCH 0624/1654] fixed typos (#19316) --- nimpretty/nimpretty.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimpretty/nimpretty.nim b/nimpretty/nimpretty.nim index a7940349d..e3b4c9a3a 100644 --- a/nimpretty/nimpretty.nim +++ b/nimpretty/nimpretty.nim @@ -69,8 +69,8 @@ proc main = var backup = false # when `on`, create a backup file of input in case - # `prettyPrint` could over-write it (note that the backup may happen even - # if input is not actually over-written, when nimpretty is a noop). + # `prettyPrint` could overwrite it (note that the backup may happen even + # if input is not actually overwritten, when nimpretty is a noop). # --backup was un-documented (rely on git instead). var opt = PrettyOptions(indWidth: 0, maxLineLen: 80) From 35cae73aa5c39b950386febfa5182bab920ddfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Tue, 4 Jan 2022 07:56:10 +0100 Subject: [PATCH 0625/1654] devel: style fix (#19318) this allows "--styleCheck:usages --styleCheck:error" --- lib/pure/nativesockets.nim | 2 +- lib/pure/net.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 4037d7181..eb0514a8f 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -810,7 +810,7 @@ proc accept*(fd: SocketHandle, inheritable = defined(nimInheritHandles)): (Socke ## child processes. ## ## Returns (osInvalidSocket, "") if an error occurred. - var sockAddress: Sockaddr + var sockAddress: SockAddr var addrLen = sizeof(sockAddress).SockLen var sock = when (defined(linux) or defined(bsd)) and not defined(nimdoc): diff --git a/lib/pure/net.nim b/lib/pure/net.nim index ced6b2fb2..8dc92e9d6 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1778,7 +1778,7 @@ proc sendTo*(socket: Socket, address: IpAddress, port: Port, assert(not socket.isClosed, "Cannot `sendTo` on a closed socket") var sa: Sockaddr_storage - var sl: Socklen + var sl: SockLen toSockAddr(address, port, sa, sl) result = sendto(socket.fd, cstring(data), data.len().cint, flags.cint, cast[ptr SockAddr](addr sa), sl) From 39a27783fc0d43e9828bb031cc811042226407b9 Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Tue, 4 Jan 2022 17:42:21 +0800 Subject: [PATCH 0626/1654] docs: Fix typo in tut1.rst (#19324) --- doc/tut1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index d8e0ec8e4..3107aef45 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1052,7 +1052,7 @@ errors. Lossless `Automatic type conversion`:idx: is performed in expressions where different kinds of integer types are used. However, if the type conversion -would cause loss of information, the ``RangeDefect``:idx: is raised (if the error +would cause loss of information, the `RangeDefect`:idx: is raised (if the error cannot be detected at compile time). From 0bcd7062c6e7a4131d166dacea41d661b7a09748 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 4 Jan 2022 18:10:46 +0800 Subject: [PATCH 0627/1654] correct the comments (#19322) --expandArc ``` var a b a = matrix(5, 5, 1.0) b = matrix(5, 5, 2.0) `=sink`(b, - let blitTmp = b wasMoved(b) blitTmp + a) `=destroy`(b) `=destroy`(a) ``` --- compiler/optimizer.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/optimizer.nim b/compiler/optimizer.nim index 744c82ab5..f484fdbf5 100644 --- a/compiler/optimizer.nim +++ b/compiler/optimizer.nim @@ -127,11 +127,11 @@ proc analyse(c: var Con; b: var BasicBlock; n: PNode) = for i in 0 ..< n.len: analyse(c, b, n[i]) else: - #[ Test tmatrix.test3: + #[ Test destructor/tmatrix.test3: Prevent this from being elided. We should probably find a better solution... - `=sink`(b, - ( + `=sink`(b, - let blitTmp = b; wasMoved(b); blitTmp + a) From 1869826668e2e3a0fd69cc1b69fb12b07993e417 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 4 Jan 2022 18:49:54 +0800 Subject: [PATCH 0628/1654] add std/private/win_getsysteminfo; refactor the usage of `GetSystemInfo` (#19310) * add std/private/win_getsysteminfo * import at the top level * wrappers follow nep1 too * follow review comment --- lib/pure/concurrency/cpuinfo.nim | 24 ++++++------------------ lib/pure/reservedmem.nim | 19 ++----------------- lib/std/private/win_getsysteminfo.nim | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 35 deletions(-) create mode 100644 lib/std/private/win_getsysteminfo.nim diff --git a/lib/pure/concurrency/cpuinfo.nim b/lib/pure/concurrency/cpuinfo.nim index ee43b8e11..16d32002d 100644 --- a/lib/pure/concurrency/cpuinfo.nim +++ b/lib/pure/concurrency/cpuinfo.nim @@ -18,6 +18,9 @@ include "system/inclrtl" when defined(posix) and not (defined(macosx) or defined(bsd)): import posix +when defined(windows): + import std/private/win_getsysteminfo + when defined(freebsd) or defined(macosx): {.emit: "#include ".} @@ -54,25 +57,10 @@ proc countProcessors*(): int {.rtl, extern: "ncpi$1".} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. when defined(windows): - type - SYSTEM_INFO {.final, pure.} = object - u1: int32 - dwPageSize: int32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr int32 - dwNumberOfProcessors: int32 - dwProcessorType: int32 - dwAllocationGranularity: int32 - wProcessorLevel: int16 - wProcessorRevision: int16 - - proc GetSystemInfo(lpSystemInfo: var SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} - var - si: SYSTEM_INFO - GetSystemInfo(si) - result = si.dwNumberOfProcessors + si: SystemInfo + getSystemInfo(addr si) + result = int(si.dwNumberOfProcessors) elif defined(macosx) or defined(bsd): var mib: array[0..3, cint] diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim index 232a2b383..99fbe1429 100644 --- a/lib/pure/reservedmem.nim +++ b/lib/pure/reservedmem.nim @@ -42,25 +42,10 @@ type when defined(windows): import winlean - - type - SYSTEM_INFO {.final, pure.} = object - u1: uint32 - dwPageSize: uint32 - lpMinimumApplicationAddress: pointer - lpMaximumApplicationAddress: pointer - dwActiveProcessorMask: ptr uint32 - dwNumberOfProcessors: uint32 - dwProcessorType: uint32 - dwAllocationGranularity: uint32 - wProcessorLevel: uint16 - wProcessorRevision: uint16 - - proc getSystemInfo(lpSystemInfo: ptr SYSTEM_INFO) {.stdcall, - dynlib: "kernel32", importc: "GetSystemInfo".} + import std/private/win_getsysteminfo proc getAllocationGranularity: uint = - var sysInfo: SYSTEM_INFO + var sysInfo: SystemInfo getSystemInfo(addr sysInfo) return uint(sysInfo.dwAllocationGranularity) diff --git a/lib/std/private/win_getsysteminfo.nim b/lib/std/private/win_getsysteminfo.nim new file mode 100644 index 000000000..b98478231 --- /dev/null +++ b/lib/std/private/win_getsysteminfo.nim @@ -0,0 +1,15 @@ +type + SystemInfo* = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors*: uint32 + dwProcessorType: uint32 + dwAllocationGranularity*: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + +proc getSystemInfo*(lpSystemInfo: ptr SystemInfo) {.stdcall, + dynlib: "kernel32", importc: "GetSystemInfo".} From 9df195ef581879cf9e5c3c3154755bd1dba677c6 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 4 Jan 2022 20:29:50 +0800 Subject: [PATCH 0629/1654] style usages part one (openarray => openArray) (#19321) * style usages (openArray) * revert doc changes --- tests/arc/tmovebug.nim | 4 ++-- tests/arc/tmovebugcopy.nim | 4 ++-- tests/array/tarray.nim | 2 +- tests/async/tioselectors.nim | 4 ++-- tests/closure/tclosure.nim | 4 ++-- tests/compiles/trecursive_generic_in_compiles.nim | 2 +- tests/concepts/tconcepts_issues.nim | 8 ++++---- tests/converter/tconverter_with_varargs.nim | 2 +- tests/cpp/tcovariancerules.nim | 8 ++++---- tests/destructor/tmove_objconstr.nim | 2 +- tests/errmsgs/t17460.nim | 2 +- tests/errmsgs/tproper_stacktrace.nim | 2 +- tests/errmsgs/tsigmatch.nim | 4 ++-- tests/generics/tgenerics_issues.nim | 2 +- tests/generics/tgenerics_various.nim | 4 ++-- tests/iter/titer13.nim | 2 +- tests/iter/titer_issues.nim | 4 ++-- tests/iter/tmoditer.nim | 2 +- tests/iter/tpermutations.nim | 2 +- tests/js/t9410.nim | 4 ++-- tests/js/tbasics.nim | 2 +- tests/js/tbyvar.nim | 4 ++-- tests/lent/tbasic_lent_check.nim | 2 +- tests/magics/tmagics.nim | 2 +- tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim | 2 +- .../named_argument_bug/tri_engine/gfx/gl/primitive.nim | 2 +- tests/misc/t9039.nim | 2 +- tests/misc/theaproots.nim | 2 +- tests/misc/tsimplesort.nim | 2 +- tests/openarray/topena1.nim | 2 +- tests/openarray/topenarrayrepr.nim | 4 ++-- tests/openarray/topenlen.nim | 2 +- tests/openarray/tptrarrayderef.nim | 6 +++--- tests/overload/toverload_various.nim | 4 ++-- tests/showoff/tdrdobbs_examples.nim | 4 ++-- tests/stdlib/tdochelpers.nim | 4 ++-- tests/stdlib/tparseipv6.nim | 4 ++-- tests/stdlib/trepr.nim | 2 +- tests/system/tostring.nim | 2 +- tests/system/tsystem_misc.nim | 4 ++-- tests/template/otests.nim | 4 ++-- tests/template/sunset.nimf | 2 +- tests/template/template_issues.nim | 2 +- tests/tuples/tuple_with_nil.nim | 4 ++-- tests/typerel/tcommontype.nim | 2 +- tests/typerel/texplicitcmp.nim | 2 +- tests/typerel/tstr_as_openarray.nim | 6 +++--- tests/views/tviews1.nim | 2 +- tests/vm/tyaytypedesc.nim | 2 +- 49 files changed, 76 insertions(+), 76 deletions(-) diff --git a/tests/arc/tmovebug.nim b/tests/arc/tmovebug.nim index 7977d330a..002bd6796 100644 --- a/tests/arc/tmovebug.nim +++ b/tests/arc/tmovebug.nim @@ -292,7 +292,7 @@ when false: # bug #13456 -iterator combinations[T](s: openarray[T], k: int): seq[T] = +iterator combinations[T](s: openArray[T], k: int): seq[T] = let n = len(s) assert k >= 0 and k <= n var pos = newSeq[int](k) @@ -455,7 +455,7 @@ initFoo7(2) # bug #14902 -iterator zip[T](s: openarray[T]): (T, T) = +iterator zip[T](s: openArray[T]): (T, T) = var i = 0 while i < 10: yield (s[i mod 2], s[i mod 2 + 1]) diff --git a/tests/arc/tmovebugcopy.nim b/tests/arc/tmovebugcopy.nim index 7c5228147..ec4315777 100644 --- a/tests/arc/tmovebugcopy.nim +++ b/tests/arc/tmovebugcopy.nim @@ -253,7 +253,7 @@ when false: # bug #13456 -iterator combinations[T](s: openarray[T], k: int): seq[T] = +iterator combinations[T](s: openArray[T], k: int): seq[T] = let n = len(s) assert k >= 0 and k <= n var pos = newSeq[int](k) @@ -416,7 +416,7 @@ initFoo7(2) # bug #14902 -iterator zip[T](s: openarray[T]): (T, T) = +iterator zip[T](s: openArray[T]): (T, T) = var i = 0 while i < 10: yield (s[i mod 2], s[i mod 2 + 1]) diff --git a/tests/array/tarray.nim b/tests/array/tarray.nim index d10011ef2..e9f330e3b 100644 --- a/tests/array/tarray.nim +++ b/tests/array/tarray.nim @@ -44,7 +44,7 @@ block tarray: arr: TMyarray - proc sum(a: openarray[int]): int = + proc sum(a: openArray[int]): int = result = 0 var i = 0 while i < len(a): diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index f7487525a..fdb8d9597 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -288,8 +288,8 @@ when not defined(windows): events: set[Event] proc vnode_test(): bool = - proc validate(test: openarray[ReadyKey], - check: openarray[valType]): bool = + proc validate(test: openArray[ReadyKey], + check: openArray[valType]): bool = result = false if len(test) == len(check): for checkItem in check: diff --git a/tests/closure/tclosure.nim b/tests/closure/tclosure.nim index 546d7026d..8ae6c44bb 100644 --- a/tests/closure/tclosure.nim +++ b/tests/closure/tclosure.nim @@ -38,10 +38,10 @@ joinable: false block tclosure: - proc map(n: var openarray[int], fn: proc (x: int): int {.closure}) = + proc map(n: var openArray[int], fn: proc (x: int): int {.closure}) = for i in 0..n.len-1: n[i] = fn(n[i]) - proc each(n: openarray[int], fn: proc(x: int) {.closure.}) = + proc each(n: openArray[int], fn: proc(x: int) {.closure.}) = for i in 0..n.len-1: fn(n[i]) diff --git a/tests/compiles/trecursive_generic_in_compiles.nim b/tests/compiles/trecursive_generic_in_compiles.nim index 88d3faeed..ab31ad0f5 100644 --- a/tests/compiles/trecursive_generic_in_compiles.nim +++ b/tests/compiles/trecursive_generic_in_compiles.nim @@ -50,7 +50,7 @@ proc `==`*[T](xs, ys: List[T]): bool = proc asList*[T](xs: varargs[T]): List[T] = ## Creates list from varargs - proc initListImpl(i: int, xs: openarray[T]): List[T] = + proc initListImpl(i: int, xs: openArray[T]): List[T] = if i > high(xs): Nil[T]() else: diff --git a/tests/concepts/tconcepts_issues.nim b/tests/concepts/tconcepts_issues.nim index c76049bdd..1a0dd8063 100644 --- a/tests/concepts/tconcepts_issues.nim +++ b/tests/concepts/tconcepts_issues.nim @@ -324,15 +324,15 @@ block t6691: block t6782: type Reader = concept c - c.read(openarray[byte], int, int) is int + c.read(openArray[byte], int, int) is int Rdr = concept c - c.rd(openarray[byte], int, int) is int + c.rd(openArray[byte], int, int) is int type TestFile = object - proc read(r: TestFile, dest: openarray[byte], offset: int, limit: int): int = + proc read(r: TestFile, dest: openArray[byte], offset: int, limit: int): int = result = 0 - proc rd(r: TestFile, dest: openarray[byte], offset: int, limit: int): int = + proc rd(r: TestFile, dest: openArray[byte], offset: int, limit: int): int = result = 0 doAssert TestFile is Reader diff --git a/tests/converter/tconverter_with_varargs.nim b/tests/converter/tconverter_with_varargs.nim index 6d7e31e85..fae83221b 100644 --- a/tests/converter/tconverter_with_varargs.nim +++ b/tests/converter/tconverter_with_varargs.nim @@ -8,7 +8,7 @@ type converter to_py*(i: int) : PPyRef = nil when false: - proc to_tuple*(vals: openarray[PPyRef]): PPyRef = + proc to_tuple*(vals: openArray[PPyRef]): PPyRef = discard proc abc(args: varargs[PPyRef]) = diff --git a/tests/cpp/tcovariancerules.nim b/tests/cpp/tcovariancerules.nim index 5ac7d8bac..9d49f2cbd 100644 --- a/tests/cpp/tcovariancerules.nim +++ b/tests/cpp/tcovariancerules.nim @@ -79,16 +79,16 @@ proc wantsCovariantSeq2(s: seq[AnimalRef]) = proc wantsCovariantSeq3(s: seq[RefAlias[Animal]]) = for a in s: echo a.x -proc wantsCovariantOpenArray(s: openarray[ref Animal]) = +proc wantsCovariantOpenArray(s: openArray[ref Animal]) = for a in s: echo a.x -proc modifiesCovariantOpenArray(s: var openarray[ref Animal]) = +proc modifiesCovariantOpenArray(s: var openArray[ref Animal]) = for a in s: echo a.x -proc modifiesDerivedOpenArray(s: var openarray[ref Dog]) = +proc modifiesDerivedOpenArray(s: var openArray[ref Dog]) = for a in s: echo a.x -proc wantsNonCovariantOpenArray(s: openarray[Animal]) = +proc wantsNonCovariantOpenArray(s: openArray[Animal]) = for a in s: echo a.x proc wantsCovariantArray(s: array[2, ref Animal]) = diff --git a/tests/destructor/tmove_objconstr.nim b/tests/destructor/tmove_objconstr.nim index 5faaabb8b..9eee4bfdb 100644 --- a/tests/destructor/tmove_objconstr.nim +++ b/tests/destructor/tmove_objconstr.nim @@ -178,7 +178,7 @@ proc myfuncLoop(x: int): MySeqNonCopyable = discard myfuncLoop(3) #------------------------------------------------------------ -# Move into table via openarray +# Move into table via openArray #------------------------------------------------------------ type diff --git a/tests/errmsgs/t17460.nim b/tests/errmsgs/t17460.nim index 8192cc4c8..e377bc48a 100644 --- a/tests/errmsgs/t17460.nim +++ b/tests/errmsgs/t17460.nim @@ -3,7 +3,7 @@ discard """ errormsg: "wrong number of variables" """ -iterator xclusters*[T](a: openarray[T]; s: static[int]): array[s, T] {.inline.} = +iterator xclusters*[T](a: openArray[T]; s: static[int]): array[s, T] {.inline.} = var result: array[s, T] # iterators have no default result variable var i = 0 while i < len(a): diff --git a/tests/errmsgs/tproper_stacktrace.nim b/tests/errmsgs/tproper_stacktrace.nim index c7dfbaf2a..8617984fb 100644 --- a/tests/errmsgs/tproper_stacktrace.nim +++ b/tests/errmsgs/tproper_stacktrace.nim @@ -6,7 +6,7 @@ import strscans, strutils proc raiseTestException*() = raise newException(Exception, "test") -proc matchStackTrace(actualEntries: openarray[StackTraceEntry], expected: string) = +proc matchStackTrace(actualEntries: openArray[StackTraceEntry], expected: string) = var expectedEntries = newSeq[StackTraceEntry]() var i = 0 diff --git a/tests/errmsgs/tsigmatch.nim b/tests/errmsgs/tsigmatch.nim index 023b7d518..8f32ef9e6 100644 --- a/tests/errmsgs/tsigmatch.nim +++ b/tests/errmsgs/tsigmatch.nim @@ -132,12 +132,12 @@ block: echo foo(fun) block: - # bug #10285 Function signature don't match when inside seq/array/openarray + # bug #10285 Function signature don't match when inside seq/array/openArray # Note: the error message now shows `closure` which helps debugging the issue # out why it doesn't match proc takesFunc(f: proc (x: int) {.gcsafe, locks: 0.}) = echo "takes single Func" - proc takesFuncs(fs: openarray[proc (x: int) {.gcsafe, locks: 0.}]) = + proc takesFuncs(fs: openArray[proc (x: int) {.gcsafe, locks: 0.}]) = echo "takes multiple Func" takesFunc(proc (x: int) {.gcsafe, locks: 0.} = echo x) # works takesFuncs([proc (x: int) {.gcsafe, locks: 0.} = echo x]) # fails diff --git a/tests/generics/tgenerics_issues.nim b/tests/generics/tgenerics_issues.nim index 9e70efcd2..092926c01 100644 --- a/tests/generics/tgenerics_issues.nim +++ b/tests/generics/tgenerics_issues.nim @@ -603,7 +603,7 @@ block t7854: block t5864: - proc defaultStatic(s: openarray, N: static[int] = 1): int = N + proc defaultStatic(s: openArray, N: static[int] = 1): int = N proc defaultGeneric[T](a: T = 2): int = a let a = [1, 2, 3, 4].defaultStatic() diff --git a/tests/generics/tgenerics_various.nim b/tests/generics/tgenerics_various.nim index 285108cd3..6c76502e1 100644 --- a/tests/generics/tgenerics_various.nim +++ b/tests/generics/tgenerics_various.nim @@ -230,8 +230,8 @@ block tvarargs_vs_generics: echo "direct" proc withDirectType[T](arg: T) = echo "generic" - proc withOpenArray(args: openarray[string]) = - echo "openarray" + proc withOpenArray(args: openArray[string]) = + echo "openArray" proc withOpenArray[T](arg: T) = echo "generic" proc withVarargs(args: varargs[string]) = diff --git a/tests/iter/titer13.nim b/tests/iter/titer13.nim index 0d4a399c5..086c40ca4 100644 --- a/tests/iter/titer13.nim +++ b/tests/iter/titer13.nim @@ -72,7 +72,7 @@ block: echo a block t5859: - proc flatIterator[T](s: openarray[T]): auto {.noSideEffect.}= + proc flatIterator[T](s: openArray[T]): auto {.noSideEffect.}= result = iterator(): auto = when (T is not seq|array): for item in s: diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 4a76b2fb0..65f66ad26 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -135,7 +135,7 @@ block t3837_chained: block t3221_complex: - iterator permutations[T](ys: openarray[T]): seq[T] = + iterator permutations[T](ys: openArray[T]): seq[T] = var d = 1 c = newSeq[int](ys.len) @@ -228,7 +228,7 @@ block t2023_objiter: block: # bug #13739 - iterator myIter(arg: openarray[int]): int = + iterator myIter(arg: openArray[int]): int = var tmp = 0 let len = arg.len while tmp < len: diff --git a/tests/iter/tmoditer.nim b/tests/iter/tmoditer.nim index b92a416fb..99e5b642d 100644 --- a/tests/iter/tmoditer.nim +++ b/tests/iter/tmoditer.nim @@ -43,7 +43,7 @@ proc `=copy`(dst: var NonCopyable, src: NonCopyable) {.error.} proc `=sink`(dst: var NonCopyable, src: NonCopyable) = dst.x = src.x -iterator lentItems[T](a: openarray[T]): lent T = +iterator lentItems[T](a: openArray[T]): lent T = for i in 0..a.high: yield a[i] diff --git a/tests/iter/tpermutations.nim b/tests/iter/tpermutations.nim index c5067ba31..30a66460f 100644 --- a/tests/iter/tpermutations.nim +++ b/tests/iter/tpermutations.nim @@ -12,7 +12,7 @@ perm: 6778800.0 det: 0.0 import sequtils, sugar -iterator permutations*[T](ys: openarray[T]): tuple[perm: seq[T], sign: int] = +iterator permutations*[T](ys: openArray[T]): tuple[perm: seq[T], sign: int] = var d = 1 c = newSeq[int](ys.len) diff --git a/tests/js/t9410.nim b/tests/js/t9410.nim index 78c329a24..042520dc5 100644 --- a/tests/js/t9410.nim +++ b/tests/js/t9410.nim @@ -447,10 +447,10 @@ template tests = doAssert seqOfSeqs == @[@[10, 2], @[30, 4]] when false: - block: # openarray + block: # openArray # Error: internal error: genAddr: nkStmtListExpr var calls = 0 - proc getvarint(x: var openarray[int]): var int = + proc getvarint(x: var openArray[int]): var int = calls += 1 if true: x[1] diff --git a/tests/js/tbasics.nim b/tests/js/tbasics.nim index b297bb037..ef84f8bb5 100644 --- a/tests/js/tbasics.nim +++ b/tests/js/tbasics.nim @@ -54,7 +54,7 @@ for x in someGlobal: doAssert(x == 0) proc tdefault = var x = default(int) doAssert(x == 0) - proc inner(v: openarray[string]) = + proc inner(v: openArray[string]) = doAssert(v.len == 0) inner(default(seq[string])) diff --git a/tests/js/tbyvar.nim b/tests/js/tbyvar.nim index a4c60b0b3..93724a2f1 100644 --- a/tests/js/tbyvar.nim +++ b/tests/js/tbyvar.nim @@ -39,9 +39,9 @@ proc main = main() -# Test: pass var seq to var openarray +# Test: pass var seq to var openArray var s = @[2, 1] -proc foo(a: var openarray[int]) = a[0] = 123 +proc foo(a: var openArray[int]) = a[0] = 123 proc bar(s: var seq[int], a: int) = doAssert(a == 5) diff --git a/tests/lent/tbasic_lent_check.nim b/tests/lent/tbasic_lent_check.nim index 049fa276a..e33da06f8 100644 --- a/tests/lent/tbasic_lent_check.nim +++ b/tests/lent/tbasic_lent_check.nim @@ -39,7 +39,7 @@ template main2 = # bug #15958 p[] = 20.0 doAssert byLent(p)[] == 20.0 - proc byLent2[T](a: openarray[T]): lent T = a[0] + proc byLent2[T](a: openArray[T]): lent T = a[0] doAssert byLent2(a) == 11 doAssert byLent2(a).unsafeAddr == a[0].unsafeAddr doAssert byLent2(b) == 21 diff --git a/tests/magics/tmagics.nim b/tests/magics/tmagics.nim index 70926a0d7..0e412940c 100644 --- a/tests/magics/tmagics.nim +++ b/tests/magics/tmagics.nim @@ -18,7 +18,7 @@ block tlowhigh: for i in low(a) .. high(a): a[i] = 0 - proc sum(a: openarray[int]): int = + proc sum(a: openArray[int]): int = result = 0 for i in low(a)..high(a): inc(result, a[i]) diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim index 7d787c07b..29e23f9d0 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/gl.nim @@ -53,7 +53,7 @@ else: const glRealType* = cGLfloat -proc setUniformV4*[T](loc: GLint, vecs: var openarray[TV4[T]]) = +proc setUniformV4*[T](loc: GLint, vecs: var openArray[TV4[T]]) = glUniform4fv(loc, vecs.len.GLsizei, cast[ptr GLfloat](vecs[0].addr)) proc setUniformV4*[T](loc: GLint, vec: TV4[T]) = diff --git a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim index 3bfaf1cbc..accc2d96b 100644 --- a/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim +++ b/tests/manyloc/named_argument_bug/tri_engine/gfx/gl/primitive.nim @@ -47,7 +47,7 @@ proc newVert*(rect: rect.TRect): seq[TVert] = proc newVertAttrib(i: GLuint, size: GLint, stride: GLsizei, offset: GLvoid): TVertAttrib = TVertAttrib(i: i, size: size, stride: stride, offset: offset) -proc genBuf*[T](vboTarget, objUsage: GLenum, data: var openarray[T]): GLuint = +proc genBuf*[T](vboTarget, objUsage: GLenum, data: var openArray[T]): GLuint = result = 0.GLuint ?glGenBuffers(1, result.addr) ?glBindBuffer(vboTarget, result) diff --git a/tests/misc/t9039.nim b/tests/misc/t9039.nim index ba636d1db..3271cd34e 100644 --- a/tests/misc/t9039.nim +++ b/tests/misc/t9039.nim @@ -15,7 +15,7 @@ but expression 'nesting + 1' is of type: int # line 15 func default(T: typedesc[array]): T = discard doAssert default(array[3, int]) == [0, 0, 0] -func shapeBad*[T: not char](s: openarray[T], rank: static[int], nesting = 0, parent_shape = default(array[rank, int])): array[rank, int] = +func shapeBad*[T: not char](s: openArray[T], rank: static[int], nesting = 0, parent_shape = default(array[rank, int])): array[rank, int] = result = parent_shape result[nesting] = s.len when (T is seq|array): diff --git a/tests/misc/theaproots.nim b/tests/misc/theaproots.nim index 1ea3c86b9..2dd345254 100644 --- a/tests/misc/theaproots.nim +++ b/tests/misc/theaproots.nim @@ -28,7 +28,7 @@ proc acc(x: var Foo): var ref Bar = proc test(maybeFoo: var Foo, maybeSeq: var seq[ref Bar], - bars: var openarray[ref Bar], + bars: var openArray[ref Bar], maybeTup: var Tup) = var bb: ref Bar maybeFoo.rmaybe = bb diff --git a/tests/misc/tsimplesort.nim b/tests/misc/tsimplesort.nim index e4a8e0b37..4793c63df 100644 --- a/tests/misc/tsimplesort.nim +++ b/tests/misc/tsimplesort.nim @@ -112,7 +112,7 @@ proc initTable*[A, B](initialSize=64): TTable[A, B] = result.counter = 0 newSeq(result.data, initialSize) -proc toTable*[A, B](pairs: openarray[tuple[key: A, +proc toTable*[A, B](pairs: openArray[tuple[key: A, val: B]]): TTable[A, B] = ## creates a new hash table that contains the given `pairs`. result = initTable[A, B](nextPowerOfTwo(pairs.len+10)) diff --git a/tests/openarray/topena1.nim b/tests/openarray/topena1.nim index ed3a0cedb..380c57f2a 100644 --- a/tests/openarray/topena1.nim +++ b/tests/openarray/topena1.nim @@ -6,4 +6,4 @@ discard """ # Tests a special bug var - x: ref openarray[string] #ERROR_MSG invalid type + x: ref openArray[string] #ERROR_MSG invalid type diff --git a/tests/openarray/topenarrayrepr.nim b/tests/openarray/topenarrayrepr.nim index 3784d4bbb..fc40d88c3 100644 --- a/tests/openarray/topenarrayrepr.nim +++ b/tests/openarray/topenarrayrepr.nim @@ -2,12 +2,12 @@ discard """ output: "5 - [1]" """ type - TProc = proc (n: int, m: openarray[int64]) {.nimcall.} + TProc = proc (n: int, m: openArray[int64]) {.nimcall.} proc Foo(x: int, P: TProc) = P(x, [ 1'i64 ]) -proc Bar(n: int, m: openarray[int64]) = +proc Bar(n: int, m: openArray[int64]) = echo($n & " - " & repr(m)) Foo(5, Bar) #OUT 5 - [1] diff --git a/tests/openarray/topenlen.nim b/tests/openarray/topenlen.nim index 83d58ac5c..164241b85 100644 --- a/tests/openarray/topenlen.nim +++ b/tests/openarray/topenlen.nim @@ -5,7 +5,7 @@ discard """ proc choose(b: openArray[string]): string = return b[0] -proc p(a, b: openarray[string]): int = +proc p(a, b: openArray[string]): int = result = a.len + b.len - 1 for j in 0 .. a.len: inc(result) discard choose(a) diff --git a/tests/openarray/tptrarrayderef.nim b/tests/openarray/tptrarrayderef.nim index 5e77430d1..1b7ef0df0 100644 --- a/tests/openarray/tptrarrayderef.nim +++ b/tests/openarray/tptrarrayderef.nim @@ -15,14 +15,14 @@ var raa = [11,12,13] #bug #3586 -proc mutate[T](arr:openarray[T], brr: openArray[T]) = +proc mutate[T](arr:openArray[T], brr: openArray[T]) = for i in 0..arr.len-1: doAssert(arr[i] == brr[i]) mutate(arr, arr) #bug #2240 -proc f(a: openarray[int], b: openArray[int]) = +proc f(a: openArray[int], b: openArray[int]) = for i in 0..a.len-1: doAssert(a[i] == b[i]) @@ -37,7 +37,7 @@ ra[2] = 13 f(ra[], raa) #bug #2240b -proc fillBuffer(buf: var openarray[char]) = +proc fillBuffer(buf: var openArray[char]) = for i in 0..buf.len-1: buf[i] = chr(i) diff --git a/tests/overload/toverload_various.nim b/tests/overload/toverload_various.nim index 55a8d17e6..0741fce60 100644 --- a/tests/overload/toverload_various.nim +++ b/tests/overload/toverload_various.nim @@ -264,8 +264,8 @@ proc init*[T](hmctx: HMAC[T], key: ptr byte, ulen: uint) = const sizeBlock = hmctx.sizeBlock echo sizeBlock -proc hmac*[A, B](HashType: typedesc, key: openarray[A], - data: openarray[B]) = +proc hmac*[A, B](HashType: typedesc, key: openArray[A], + data: openArray[B]) = var ctx: HMAC[HashType] ctx.init(nil, 0) diff --git a/tests/showoff/tdrdobbs_examples.nim b/tests/showoff/tdrdobbs_examples.nim index 0e10c6dd8..c61e177dc 100644 --- a/tests/showoff/tdrdobbs_examples.nim +++ b/tests/showoff/tdrdobbs_examples.nim @@ -40,12 +40,12 @@ const msb3999 = mostSignificantBit(3999) echo msb3999, " ", mostSignificantBit(0), " ", square(44) -proc filter[T](a: openarray[T], predicate: proc (x: T): bool): seq[T] = +proc filter[T](a: openArray[T], predicate: proc (x: T): bool): seq[T] = result = @[] # @[] constructs the empty seq for x in a: if predicate(x): result.add(x) -proc map[T, S](a: openarray[T], fn: proc (x: T): S): seq[S] = +proc map[T, S](a: openArray[T], fn: proc (x: T): S): seq[S] = newSeq(result, a.len) for i in 0 ..< a.len: result[i] = fn(a[i]) diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim index 9d440a1d6..8dcb158ca 100644 --- a/tests/stdlib/tdochelpers.nim +++ b/tests/stdlib/tdochelpers.nim @@ -76,7 +76,7 @@ suite "Integration with Nim": check(input3.toLangSymbol == expected) test "advanced proc parsing with Nim identifier normalization": - let input = """`proc binarySearch*[T, K](a: openArray[T]; key: K; + let input = """`proc binarySearch*[T, K](a: openarray[T]; key: K; cmp: proc (x: T; y: K): int)`_""".rstParseTest let expected = LangSymbol(symKind: "proc", name: "binarysearch", @@ -90,7 +90,7 @@ suite "Integration with Nim": check(input.toLangSymbol == expected) test "the same without proc": - let input = """`binarySearch*[T, K](a: openArray[T]; key: K; + let input = """`binarySearch*[T, K](a: openarray[T]; key: K; cmp: proc (x: T; y: K): int {.closure.})`_""".rstParseTest let expected = LangSymbol(symKind: "", name: "binarysearch", diff --git a/tests/stdlib/tparseipv6.nim b/tests/stdlib/tparseipv6.nim index dd9abc511..9b9c464c7 100644 --- a/tests/stdlib/tparseipv6.nim +++ b/tests/stdlib/tparseipv6.nim @@ -210,12 +210,12 @@ const "::a:b:85:e4d9:252.9.229.056", ] -proc ok(pos: openarray[string]) = +proc ok(pos: openArray[string]) = for p in pos: if not isIpAddress(p): echo "failure ", p -proc notok(neg: openarray[string]) = +proc notok(neg: openArray[string]) = for n in neg: if isIpAddress(n): echo "failure ", n diff --git a/tests/stdlib/trepr.nim b/tests/stdlib/trepr.nim index 83ae7b119..82c991805 100644 --- a/tests/stdlib/trepr.nim +++ b/tests/stdlib/trepr.nim @@ -64,7 +64,7 @@ template main() = doAssert repr(arr) == "[1, 2, 3]" block: # bug #7878 - proc reprOpenarray(variable: var openarray[int]): string = repr(variable) + proc reprOpenarray(variable: var openArray[int]): string = repr(variable) when defined(js): discard # BUG: doesn't work else: doAssert reprOpenarray(arr) == "[1, 2, 3]" diff --git a/tests/system/tostring.nim b/tests/system/tostring.nim index fa82acc3b..bdbaa2ce6 100644 --- a/tests/system/tostring.nim +++ b/tests/system/tostring.nim @@ -68,7 +68,7 @@ doAssert yy == "" proc bar(arg: cstring) = doAssert arg[0] == '\0' -proc baz(arg: openarray[char]) = +proc baz(arg: openArray[char]) = doAssert arg.len == 0 proc stringCompare() = diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim index 13b8c18a9..7f5914725 100644 --- a/tests/system/tsystem_misc.nim +++ b/tests/system/tsystem_misc.nim @@ -131,12 +131,12 @@ let str = "0123456789" foo(toOpenArrayByte(str, 0, str.high)) -template boundedOpenArray[T](x: seq[T], first, last: int): openarray[T] = +template boundedOpenArray[T](x: seq[T], first, last: int): openArray[T] = toOpenarray(x, max(0, first), min(x.high, last)) # bug #9281 -proc foo[T](x: openarray[T]) = +proc foo[T](x: openArray[T]) = echo x.len let a = @[1, 2, 3] diff --git a/tests/template/otests.nim b/tests/template/otests.nim index 120146343..9cb9d7fcb 100644 --- a/tests/template/otests.nim +++ b/tests/template/otests.nim @@ -163,7 +163,7 @@ when true: #embeddingTest """ # Expression template - proc test_expression(nums: openarray[int] = []): string = + proc test_expression(nums: openArray[int] = []): string = var i = 2 tmpli html""" $(no_substitution()) @@ -171,7 +171,7 @@ when true: #embeddingTest
              Age: $($nums[i] & "!!")
              """ - proc test_statements(nums: openarray[int] = []): string = + proc test_statements(nums: openArray[int] = []): string = tmpli html""" $(test_expression(nums)) $if true { diff --git a/tests/template/sunset.nimf b/tests/template/sunset.nimf index 465b12a5e..bd57bb8e1 100644 --- a/tests/template/sunset.nimf +++ b/tests/template/sunset.nimf @@ -1,6 +1,6 @@ #? stdtmpl #proc sunsetTemplate*(current, ticker, content: string, -# tabs: openarray[array[0..1, string]]): string = +# tabs: openArray[array[0..1, string]]): string = # result = "" diff --git a/tests/template/template_issues.nim b/tests/template/template_issues.nim index 502c6d1ee..1fed694ef 100644 --- a/tests/template/template_issues.nim +++ b/tests/template/template_issues.nim @@ -173,7 +173,7 @@ block t2585: st echo "a ", $fb - proc render(rdat: var RenderData; passes: var openarray[RenderPass]; proj: Mat2; + proc render(rdat: var RenderData; passes: var openArray[RenderPass]; proj: Mat2; indexType = 1) = for i in 0 ..< len(passes): echo "blah ", repr(passes[i]) diff --git a/tests/tuples/tuple_with_nil.nim b/tests/tuples/tuple_with_nil.nim index 7136c8cab..9cad6eccd 100644 --- a/tests/tuples/tuple_with_nil.nim +++ b/tests/tuples/tuple_with_nil.nim @@ -477,7 +477,7 @@ proc writeformat(o: var Writer; b: bool; fmt: Format) = else: raise newException(FormatError, "Boolean values must of one of the following types: s,b,o,x,X,d,n") -proc writeformat(o: var Writer; ary: openarray[system.any]; fmt: Format) = +proc writeformat(o: var Writer; ary: openArray[system.any]; fmt: Format) = ## Write array `ary` according to format `fmt` using output object ## `o` and output function `add`. if ary.len == 0: return @@ -657,7 +657,7 @@ proc literal[T](x: T): NimNode {.compiletime, nosideeffect.} = result = newLit(x) proc generatefmt(fmtstr: string; - args: var openarray[tuple[arg:NimNode, cnt:int]]; + args: var openArray[tuple[arg:NimNode, cnt:int]]; arg: var int;): seq[tuple[val, fmt:NimNode]] {.compiletime.} = ## fmtstr ## the format string diff --git a/tests/typerel/tcommontype.nim b/tests/typerel/tcommontype.nim index 8215ebd5e..488047fd1 100644 --- a/tests/typerel/tcommontype.nim +++ b/tests/typerel/tcommontype.nim @@ -12,7 +12,7 @@ type proc newDog():PDog = new(result) proc newCat():PCat = new(result) -proc test(a:openarray[PAnimal])= +proc test(a:openArray[PAnimal])= echo("dummy") #test(newDog(),newCat()) #does not work diff --git a/tests/typerel/texplicitcmp.nim b/tests/typerel/texplicitcmp.nim index b11aa2f4e..57f1e81b6 100644 --- a/tests/typerel/texplicitcmp.nim +++ b/tests/typerel/texplicitcmp.nim @@ -8,7 +8,7 @@ discard """ import json, tables, algorithm -proc outp(a: openarray[int]) = +proc outp(a: openArray[int]) = stdout.write "[" for i in a: stdout.write($i & " ") stdout.write "]\n" diff --git a/tests/typerel/tstr_as_openarray.nim b/tests/typerel/tstr_as_openarray.nim index fc28d6c93..14cd21479 100644 --- a/tests/typerel/tstr_as_openarray.nim +++ b/tests/typerel/tstr_as_openarray.nim @@ -3,18 +3,18 @@ discard """ """ var s = "HI" -proc x (zz: openarray[char]) = +proc x (zz: openArray[char]) = discard x s -proc z [T] (zz: openarray[T]) = +proc z [T] (zz: openArray[T]) = discard z s z([s,s,s]) -proc y [T] (arg: var openarray[T]) = +proc y [T] (arg: var openArray[T]) = arg[0] = 'X' y s doAssert s == "XI" diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index ced487ce8..249058eb6 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -44,7 +44,7 @@ echo foo(x) type F = object - oa: openarray[int] + oa: openArray[int] let s1 = @[1,3,4,5,6] var test = F(oa: toOpenArray(s1, 0, 2)) diff --git a/tests/vm/tyaytypedesc.nim b/tests/vm/tyaytypedesc.nim index a3ad9b707..aab94d643 100644 --- a/tests/vm/tyaytypedesc.nim +++ b/tests/vm/tyaytypedesc.nim @@ -10,7 +10,7 @@ type NodeType* = enum type TokenType* = enum ttWhitespace -proc enumTable*[A, B, C](a: openarray[tuple[key: A, val: B]], ret: typedesc[C]): C = +proc enumTable*[A, B, C](a: openArray[tuple[key: A, val: B]], ret: typedesc[C]): C = for item in a: result[item.key] = item.val From 5ec8b609421a2e74e7d5f8d00293194faf4437d4 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 6 Jan 2022 08:19:32 -0300 Subject: [PATCH 0630/1654] Update net.nim (#19327) [backport] --- lib/pure/net.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 8dc92e9d6..c1d4c6228 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1628,7 +1628,7 @@ proc recvFrom*[T: string | IpAddress](socket: Socket, data: var string, length: ## used. Therefore if `socket` contains something in its buffer this ## function will make no effort to return it. template adaptRecvFromToDomain(sockAddress: untyped, domain: Domain) = - var addrLen = sizeof(sockAddress).SockLen + var addrLen = SockLen(sizeof(sockAddress)) result = recvfrom(socket.fd, cstring(data), length.cint, flags.cint, cast[ptr SockAddr](addr(sockAddress)), addr(addrLen)) From c7d5b8c83ddf204c6e0cd7b6937d33318320ed91 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 7 Jan 2022 04:26:55 -0300 Subject: [PATCH 0631/1654] Fix #19038 - making the Nim compiler work again on Windows XP (#19331) * Update osenv.nim * Update win_setenv.nim * Update lib/pure/includes/osenv.nim * Update lib/pure/includes/osenv.nim * fixing cstring Co-authored-by: Andreas Rumpf --- lib/pure/includes/osenv.nim | 5 +++-- lib/std/private/win_setenv.nim | 41 +++++++++++++++++----------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/pure/includes/osenv.nim b/lib/pure/includes/osenv.nim index 1a01ab9bc..4a776eb78 100644 --- a/lib/pure/includes/osenv.nim +++ b/lib/pure/includes/osenv.nim @@ -45,7 +45,7 @@ when not defined(nimscript): proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "".} when defined(windows): - proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "".} from std/private/win_setenv import setEnvImpl else: proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} @@ -121,7 +121,8 @@ when not defined(nimscript): ]# if key.len == 0 or '=' in key: raise newException(OSError, "invalid key, got: " & key) - if c_putenv_s(key, "") != 0'i32: bail + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail else: if c_unsetenv(key) != 0'i32: bail diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 91c3f15b0..0dfe0ed46 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -30,44 +30,45 @@ else: # same as winlean.setEnvironmentVariableA proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} - proc c_putenv_s(envname: cstring, envval: cstring): cint {.importc: "_putenv_s", header: "".} + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "".} proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} var errno {.importc, header: "".}: cint - var gWenviron {.importc:"_wenviron".}: ptr ptr wchar_t + var gWenviron {.importc: "_wenviron".}: ptr ptr wchar_t # xxx `ptr UncheckedArray[WideCString]` did not work - proc mbstowcs_s(pReturnValue: ptr csize_t, wcstr: ptr wchar_t, sizeInWords: csize_t, mbstr: cstring, count: csize_t): cint {.importc: "mbstowcs_s", header: "".} + proc mbstowcs(wcstr: ptr wchar_t, mbstr: cstring, count: csize_t): csize_t {.importc: "mbstowcs", header: "".} # xxx cint vs errno_t? - proc setEnvImpl*(name: cstring, value: cstring, overwrite: cint): cint = + proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = const EINVAL = cint(22) - const MAX_ENV = 32767 - # xxx get it from: `var MAX_ENV {.importc: "_MAX_ENV", header:"".}: cint` - if overwrite == 0 and c_getenv(name) != nil: return 0 - if value[0] != '\0': - let e = c_putenv_s(name, value) + if overwrite == 0 and c_getenv(cstring(name)) != nil: return 0 + if value != "": + let envstring = name & "=" & value + let e = c_putenv(cstring(envstring)) if e != 0: - errno = e + errno = EINVAL return -1 return 0 #[ - We are trying to set the value to an empty string, but `_putenv_s` deletes + We are trying to set the value to an empty string, but `_putenv` deletes entries if the value is an empty string, and just calling SetEnvironmentVariableA doesn't update `_environ`, so we have to do these terrible things. ]# - if c_putenv_s(name, " ") != 0: + let envstring = name & "= " + if c_putenv(cstring(envstring)) != 0: errno = EINVAL return -1 # Here lies the documentation we blatently ignore to make this work. - var s = c_getenv(name) + var s = c_getenv(cstring(name)) s[0] = '\0' #[ This would result in a double null termination, which normally signifies the end of the environment variable list, so we stick a completely empty environment variable into the list instead. ]# + s = c_getenv(cstring(name)) s[1] = '=' #[ If gWenviron is null, the wide environment has not been initialized @@ -77,19 +78,19 @@ else: ]# if gWenviron != nil: # var buf: array[MAX_ENV + 1, WideCString] - var buf: array[MAX_ENV + 1, Utf16Char] + let requiredSize = mbstowcs(nil, cstring(name), 0).int + var buf = newSeq[Utf16Char](requiredSize + 1) let buf2 = cast[ptr wchar_t](buf[0].addr) - var len: csize_t - if mbstowcs_s(len.addr, buf2, buf.len.csize_t, name, MAX_ENV) != 0: + if mbstowcs(buf2, cstring(name), csize_t(requiredSize + 1)) == csize_t(high(uint)): errno = EINVAL return -1 - let ptrToEnv = cast[WideCString](c_wgetenv(buf2)) + var ptrToEnv = cast[WideCString](c_wgetenv(buf2)) ptrToEnv[0] = '\0'.Utf16Char - let ptrToEnv2 = cast[WideCString](c_wgetenv(buf2)) - ptrToEnv2[1] = '='.Utf16Char + ptrToEnv = cast[WideCString](c_wgetenv(buf2)) + ptrToEnv[1] = '='.Utf16Char # And now, we have to update the outer environment to have a proper empty value. - if setEnvironmentVariableA(name, value) == 0: + if setEnvironmentVariableA(cstring(name), cstring(value)) == 0: errno = EINVAL return -1 return 0 From 4306d8fb36d3810b8d38a665f98dc89ce107740e Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 8 Jan 2022 12:06:28 +0800 Subject: [PATCH 0632/1654] disable fragments (#19341) The original repo has moved to Rust, while the package path stay unchanged. And it causes troubles to https://github.com/nim-lang/Nim/pull/19338 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 381aa27ac..853a58698 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -64,7 +64,7 @@ pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" pkg "fidget" -pkg "fragments", "nim c -r fragments/dsl.nim" +pkg "fragments", "nim c -r fragments/dsl.nim", allowFailure = true # pending https://github.com/nim-lang/packages/issues/2115 pkg "fusion" pkg "gara" pkg "glob" From 58656aa5bba572672f093499280b69bb0f0d4c06 Mon Sep 17 00:00:00 2001 From: Zachary Marquez Date: Sat, 8 Jan 2022 03:22:56 -0600 Subject: [PATCH 0633/1654] fix nim-lang#19343 (#19344) [backport] Ensure HttpClient onProgress is called once per second Ensure that reported speed is accurate --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 7686d15bd..5b6e439db 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -673,7 +673,7 @@ proc reportProgress(client: HttpClient | AsyncHttpClient, progress: BiggestInt) {.multisync.} = client.contentProgress += progress client.oneSecondProgress += progress - if (getMonoTime() - client.lastProgressReport).inSeconds > 1: + if (getMonoTime() - client.lastProgressReport).inSeconds >= 1: if not client.onProgressChanged.isNil: await client.onProgressChanged(client.contentTotal, client.contentProgress, From ae92eac06031ed57a755c01efdbb9cc5b72e5dbf Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 9 Jan 2022 04:34:22 +0800 Subject: [PATCH 0634/1654] stylecheck usages part two: stdlib cleanup (#19338) * stylecheck usages part two: stdlib cleanup typeinfo.nim: importCompilerProc => importcompilerproc nre.nim: newLineFlags => newlineFlags system.nim: JSRoot => JsRoot ref #19319 * prefer importCompilerProc --- compiler/wordrecg.nim | 2 +- lib/impure/nre.nim | 2 +- lib/pure/os.nim | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 22f6cc71d..c819f4306 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -41,7 +41,7 @@ type wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor", wDelegator = "delegator", wOverride = "override", wImportCpp = "importcpp", wCppNonPod = "cppNonPod", - wImportObjC = "importobjc", wImportCompilerProc = "importcompilerproc", + wImportObjC = "importobjc", wImportCompilerProc = "importCompilerProc", wImportc = "importc", wImportJs = "importjs", wExportc = "exportc", wExportCpp = "exportcpp", wExportNims = "exportnims", wIncompleteStruct = "incompleteStruct", # deprecated diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index f346d84cb..03cbf5220 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -281,7 +281,7 @@ proc matchesCrLf(pattern: Regex): bool = let newlineFlags = flags and (pcre.NEWLINE_CRLF or pcre.NEWLINE_ANY or pcre.NEWLINE_ANYCRLF) - if newLineFlags > 0u32: + if newlineFlags > 0u32: return true # get flags from build config diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 9b08fe2e1..81b432d7f 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2888,7 +2888,7 @@ when defined(nimdoc): elif defined(nimscript): discard elif defined(nodejs): - type Argv = object of JSRoot + type Argv = object of JsRoot let argv {.importjs: "process.argv".} : Argv proc len(argv: Argv): int {.importjs: "#.length".} proc `[]`(argv: Argv, i: int): cstring {.importjs: "#[#]".} From b098546da051ed026e85a9f398e74c396c79a8ad Mon Sep 17 00:00:00 2001 From: Smarcy Date: Sun, 9 Jan 2022 14:02:08 +0100 Subject: [PATCH 0635/1654] added filemode docs (#19346) --- lib/system/io.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/system/io.nim b/lib/system/io.nim index 661d1a9ba..04a43328d 100644 --- a/lib/system/io.nim +++ b/lib/system/io.nim @@ -22,6 +22,8 @@ type FileMode* = enum ## The file mode when opening a file. fmRead, ## Open the file for read access only. + ## If the file does not exist, it will not + ## be created. fmWrite, ## Open the file for write access only. ## If the file does not exist, it will be ## created. Existing files will be cleared! @@ -32,7 +34,8 @@ type ## If the file does not exist, it will not be ## created. The existing file will not be cleared. fmAppend ## Open the file for writing only; append data - ## at the end. + ## at the end. If the file does not exist, it + ## will be created. FileHandle* = cint ## type that represents an OS file handle; this is ## useful for low-level file access From 955040f0f1d4f9b2eab68c385fab9d4cfbef8ea5 Mon Sep 17 00:00:00 2001 From: gecko Date: Mon, 10 Jan 2022 09:27:59 +0000 Subject: [PATCH 0636/1654] Fix `remove` on last node of singly-linked list [backport:1.6] (#19353) --- changelog.md | 1 + lib/pure/collections/lists.nim | 2 ++ tests/stdlib/tlists.nim | 13 +++++++++++++ 3 files changed, 16 insertions(+) diff --git a/changelog.md b/changelog.md index 1b97fd99a..d07f80623 100644 --- a/changelog.md +++ b/changelog.md @@ -22,6 +22,7 @@ - `macros.parseExpr` and `macros.parseStmt` now accept an optional filename argument for more informative errors. - Module `colors` expanded with missing colors from the CSS color standard. +- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). ## `std/smtp` diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index f7627ae67..d1de0ea67 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -739,6 +739,8 @@ proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.disc if prev.next == nil: return false prev.next = n.next + if L.tail == n: + L.tail = prev # update tail if we removed the last node true proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) = diff --git a/tests/stdlib/tlists.nim b/tests/stdlib/tlists.nim index 14cbf2f9d..00c5b1a27 100644 --- a/tests/stdlib/tlists.nim +++ b/tests/stdlib/tlists.nim @@ -258,5 +258,18 @@ template main = a.add(2) doAssert a.toSeq == @[1, 2] + block RemoveLastNodeFromSinglyLinkedList: + var list = initSinglyLinkedList[string]() + let n1 = newSinglyLinkedNode("sonic") + let n2 = newSinglyLinkedNode("the") + let n3 = newSinglyLinkedNode("tiger") + let n4 = newSinglyLinkedNode("hedgehog") + list.add(n1) + list.add(n2) + list.add(n3) + list.remove(n3) + list.add(n4) + doAssert list.toSeq == @["sonic", "the", "hedgehog"] + static: main() main() From 92e5573b20f30f4114a4eb1f35eddc7064bfcd71 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 10 Jan 2022 17:29:04 +0800 Subject: [PATCH 0637/1654] fix stylecheck error with asyncdispatch (#19350) * stylecheck usages part two: stdlib cleanup typeinfo.nim: importCompilerProc => importcompilerproc nre.nim: newLineFlags => newlineFlags system.nim: JSRoot => JsRoot ref #19319 * prefer importCompilerProc * fix stylecheck error with asyncdispatch it is a partial regression since #12842 * add tests * don't use echo in tests --- compiler/semtempl.nim | 3 ++- tests/stylecheck/taccept.nim | 17 +++++++++++++++++ tests/stylecheck/treject.nim | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/stylecheck/taccept.nim create mode 100644 tests/stylecheck/treject.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 49ce55a64..b921fda2c 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -270,7 +270,8 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = # Issue #12832 when defined(nimsuggest): suggestSym(c.graph, n.info, s, c.graph.usageSym, false) - if {optStyleHint, optStyleError} * c.config.globalOptions != {}: + # field access (dot expr) will be handled by builtinFieldAccess + if not isField and {optStyleHint, optStyleError} * c.config.globalOptions != {}: styleCheckUse(c.config, n.info, s) proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode = diff --git a/tests/stylecheck/taccept.nim b/tests/stylecheck/taccept.nim new file mode 100644 index 000000000..6fb55f835 --- /dev/null +++ b/tests/stylecheck/taccept.nim @@ -0,0 +1,17 @@ +discard """ + matrix: "--styleCheck:error --styleCheck:usages" +""" + +import asyncdispatch + +type + Name = object + id: int + +template hello = + var iD = "string" + var name: Name + doAssert name.id == 0 + doAssert iD == "string" + +hello() diff --git a/tests/stylecheck/treject.nim b/tests/stylecheck/treject.nim new file mode 100644 index 000000000..b9d97a58d --- /dev/null +++ b/tests/stylecheck/treject.nim @@ -0,0 +1,17 @@ +discard """ + action: reject + nimout: '''treject.nim(14, 13) Error: 'iD' should be: 'id' [field declared in treject.nim(9, 5)]''' + matrix: "--styleCheck:error --styleCheck:usages" +""" + +type + Name = object + id: int + +template hello = + var iD = "string" + var name: Name + echo name.iD + echo iD + +hello() \ No newline at end of file From bcbeff36321aa203cd258fd02c82d9276badf31f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 10 Jan 2022 18:59:44 +0800 Subject: [PATCH 0638/1654] remove spaces between an identifier and a star (#19355) It makes search easier by searching `+`* instead of `+` which filter lots of unexported versions. Follow https://github.com/nim-lang/Nim/pull/18681 --- lib/js/jsffi.nim | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 937e3727b..ac963eb89 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -163,28 +163,28 @@ macro jsFromAst*(n: untyped): untyped = proc `&`*(a, b: cstring): cstring {.importjs: "(# + #)".} ## Concatenation operator for JavaScript strings. -proc `+` *(x, y: JsObject): JsObject {.importjs: "(# + #)".} -proc `-` *(x, y: JsObject): JsObject {.importjs: "(# - #)".} -proc `*` *(x, y: JsObject): JsObject {.importjs: "(# * #)".} -proc `/` *(x, y: JsObject): JsObject {.importjs: "(# / #)".} -proc `%` *(x, y: JsObject): JsObject {.importjs: "(# % #)".} -proc `+=` *(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} -proc `-=` *(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} -proc `*=` *(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} -proc `/=` *(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} -proc `%=` *(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} -proc `++` *(x: JsObject): JsObject {.importjs: "(++#)".} -proc `--` *(x: JsObject): JsObject {.importjs: "(--#)".} -proc `>` *(x, y: JsObject): JsObject {.importjs: "(# > #)".} -proc `<` *(x, y: JsObject): JsObject {.importjs: "(# < #)".} -proc `>=` *(x, y: JsObject): JsObject {.importjs: "(# >= #)".} -proc `<=` *(x, y: JsObject): JsObject {.importjs: "(# <= #)".} -proc `**` *(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} +proc `+`*(x, y: JsObject): JsObject {.importjs: "(# + #)".} +proc `-`*(x, y: JsObject): JsObject {.importjs: "(# - #)".} +proc `*`*(x, y: JsObject): JsObject {.importjs: "(# * #)".} +proc `/`*(x, y: JsObject): JsObject {.importjs: "(# / #)".} +proc `%`*(x, y: JsObject): JsObject {.importjs: "(# % #)".} +proc `+=`*(x, y: JsObject): JsObject {.importjs: "(# += #)", discardable.} +proc `-=`*(x, y: JsObject): JsObject {.importjs: "(# -= #)", discardable.} +proc `*=`*(x, y: JsObject): JsObject {.importjs: "(# *= #)", discardable.} +proc `/=`*(x, y: JsObject): JsObject {.importjs: "(# /= #)", discardable.} +proc `%=`*(x, y: JsObject): JsObject {.importjs: "(# %= #)", discardable.} +proc `++`*(x: JsObject): JsObject {.importjs: "(++#)".} +proc `--`*(x: JsObject): JsObject {.importjs: "(--#)".} +proc `>`*(x, y: JsObject): JsObject {.importjs: "(# > #)".} +proc `<`*(x, y: JsObject): JsObject {.importjs: "(# < #)".} +proc `>=`*(x, y: JsObject): JsObject {.importjs: "(# >= #)".} +proc `<=`*(x, y: JsObject): JsObject {.importjs: "(# <= #)".} +proc `**`*(x, y: JsObject): JsObject {.importjs: "((#) ** #)".} # (#) needed, refs https://github.com/nim-lang/Nim/pull/16409#issuecomment-760550812 proc `and`*(x, y: JsObject): JsObject {.importjs: "(# && #)".} -proc `or` *(x, y: JsObject): JsObject {.importjs: "(# || #)".} +proc `or`*(x, y: JsObject): JsObject {.importjs: "(# || #)".} proc `not`*(x: JsObject): JsObject {.importjs: "(!#)".} -proc `in` *(x, y: JsObject): JsObject {.importjs: "(# in #)".} +proc `in`*(x, y: JsObject): JsObject {.importjs: "(# in #)".} proc `[]`*(obj: JsObject, field: cstring): JsObject {.importjs: getImpl.} ## Returns the value of a property of name `field` from a JsObject `obj`. From 9888a29c3de09c73202b2a955e09e2a7b5e56ea4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 11 Jan 2022 10:16:51 +0100 Subject: [PATCH 0639/1654] bitsets.nim: cleanup (#19361) --- compiler/bitsets.nim | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/compiler/bitsets.nim b/compiler/bitsets.nim index 2e8aa1db6..e0d94409e 100644 --- a/compiler/bitsets.nim +++ b/compiler/bitsets.nim @@ -23,53 +23,40 @@ const template modElemSize(arg: untyped): untyped = arg and 7 template divElemSize(arg: untyped): untyped = arg shr 3 -proc bitSetInit*(b: var TBitSet, length: int) -proc bitSetUnion*(x: var TBitSet, y: TBitSet) -proc bitSetDiff*(x: var TBitSet, y: TBitSet) -proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) -proc bitSetIntersect*(x: var TBitSet, y: TBitSet) -proc bitSetIncl*(x: var TBitSet, elem: BiggestInt) -proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) -proc bitSetIn*(x: TBitSet, e: BiggestInt): bool -proc bitSetEquals*(x, y: TBitSet): bool -proc bitSetContains*(x, y: TBitSet): bool -proc bitSetCard*(x: TBitSet): BiggestInt -# implementation - -proc bitSetIn(x: TBitSet, e: BiggestInt): bool = +proc bitSetIn*(x: TBitSet, e: BiggestInt): bool = result = (x[int(e.divElemSize)] and (One shl e.modElemSize)) != Zero -proc bitSetIncl(x: var TBitSet, elem: BiggestInt) = +proc bitSetIncl*(x: var TBitSet, elem: BiggestInt) = assert(elem >= 0) x[int(elem.divElemSize)] = x[int(elem.divElemSize)] or (One shl elem.modElemSize) -proc bitSetExcl(x: var TBitSet, elem: BiggestInt) = +proc bitSetExcl*(x: var TBitSet, elem: BiggestInt) = x[int(elem.divElemSize)] = x[int(elem.divElemSize)] and not(One shl elem.modElemSize) -proc bitSetInit(b: var TBitSet, length: int) = +proc bitSetInit*(b: var TBitSet, length: int) = newSeq(b, length) -proc bitSetUnion(x: var TBitSet, y: TBitSet) = +proc bitSetUnion*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] or y[i] -proc bitSetDiff(x: var TBitSet, y: TBitSet) = +proc bitSetDiff*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] and not y[i] -proc bitSetSymDiff(x: var TBitSet, y: TBitSet) = +proc bitSetSymDiff*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] xor y[i] -proc bitSetIntersect(x: var TBitSet, y: TBitSet) = +proc bitSetIntersect*(x: var TBitSet, y: TBitSet) = for i in 0..high(x): x[i] = x[i] and y[i] -proc bitSetEquals(x, y: TBitSet): bool = +proc bitSetEquals*(x, y: TBitSet): bool = for i in 0..high(x): if x[i] != y[i]: return false result = true -proc bitSetContains(x, y: TBitSet): bool = +proc bitSetContains*(x, y: TBitSet): bool = for i in 0..high(x): if (x[i] and not y[i]) != Zero: return false @@ -96,6 +83,6 @@ const populationCount: array[uint8, uint8] = block: arr -proc bitSetCard(x: TBitSet): BiggestInt = +proc bitSetCard*(x: TBitSet): BiggestInt = for it in x: result.inc int(populationCount[it]) From 9a32905d89d23e423bca2d0ce643b9331a6741c0 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 13 Jan 2022 15:02:41 +0800 Subject: [PATCH 0640/1654] nimscript ignore the threads option (#19370) because nimscript doesn't support threads and causes troubles when the threads option is on --- compiler/commands.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index c4df46bc2..8e79a4477 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -717,7 +717,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info) of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info) of "threads": - if conf.backend == backendJs: discard + if conf.backend == backendJs or conf.cmd == cmdNimscript: discard else: processOnOffSwitchG(conf, {optThreads}, arg, pass, info) #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe) of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info) From ada815e45957e5675a9ac978b85b8467d68c5945 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 13 Jan 2022 15:03:14 +0800 Subject: [PATCH 0641/1654] make rst thread safe (#19369) split for the convenience of review --- lib/packages/docutils/rst.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index a04eb696f..791953767 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -2822,7 +2822,7 @@ type DirFlag = enum hasArg, hasOptions, argIsFile, argIsWord DirFlags = set[DirFlag] - SectionParser = proc (p: var RstParser): PRstNode {.nimcall.} + SectionParser = proc (p: var RstParser): PRstNode {.nimcall, gcsafe.} proc parseDirective(p: var RstParser, k: RstNodeKind, flags: DirFlags): PRstNode = ## Parses arguments and options for a directive block. @@ -2869,7 +2869,7 @@ proc indFollows(p: RstParser): bool = result = currentTok(p).kind == tkIndent and currentTok(p).ival > currInd(p) proc parseBlockContent(p: var RstParser, father: var PRstNode, - contentParser: SectionParser): bool = + contentParser: SectionParser): bool {.gcsafe.} = ## parse the final content part of explicit markup blocks (directives, ## footnotes, etc). Returns true if succeeded. if currentTok(p).kind != tkIndent or indFollows(p): @@ -3123,7 +3123,7 @@ proc prefix(ftnType: FootnoteType): string = of fnAutoSymbol: result = "footnotesym-" of fnCitation: result = "citation-" -proc parseFootnote(p: var RstParser): PRstNode = +proc parseFootnote(p: var RstParser): PRstNode {.gcsafe.} = ## Parses footnotes and citations, always returns 2 sons: ## ## 1) footnote label, always containing rnInner with 1 or more sons From a9135095f02587430b179a33ae03910d0d4201cd Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 14 Jan 2022 00:40:27 +1100 Subject: [PATCH 0642/1654] docs: Mention `import foo {.all.}` syntax (#19377) Mention the `import foo {.all.}` syntax in the manual, with a caveat about private imports. Also link to the experimental importutils module. Co-authored-by: adigitoleo --- doc/manual_experimental.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 4ed8439df..0f82b9950 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -168,6 +168,17 @@ This feature will likely be superseded in the future by support for recursive module dependencies. +Importing private symbols +========================= + +In some situations, it may be useful to import all symbols (public or private) +from a module. The syntax `import foo {.all.}` can be used to import all +symbols from the module `foo`. Note that importing private symbols is +generally not recommended. + +See also the experimental `importutils `_ module. + + Code reordering =============== From 40a9c33eff099a37be236826f84eb2064d0ec247 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 13 Jan 2022 21:43:35 +0800 Subject: [PATCH 0643/1654] update copyright year (#19381) --- compiler/options.nim | 2 +- copying.txt | 2 +- readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 9e957332f..5cfe58119 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -19,7 +19,7 @@ const useEffectSystem* = true useWriteTracking* = false hasFFI* = defined(nimHasLibFFI) - copyrightYear* = "2021" + copyrightYear* = "2022" nimEnableCovariance* = defined(nimEnableCovariance) diff --git a/copying.txt b/copying.txt index a498a9525..60b6a0217 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ ===================================================== Nim -- a Compiler for Nim. https://nim-lang.org/ -Copyright (C) 2006-2021 Andreas Rumpf. All rights reserved. +Copyright (C) 2006-2022 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index 7d45556db..6bc7a2385 100644 --- a/readme.md +++ b/readme.md @@ -203,7 +203,7 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2021 Andreas Rumpf, all rights reserved. +Copyright © 2006-2022 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org From 5853303be09464832ce0f189513c90c9dd15f4a0 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 14 Jan 2022 00:45:47 +1100 Subject: [PATCH 0644/1654] docs: Fix broken cross references to `rfind` in strutils (#19382) [backport] Fixes three broken cross references to `rfind` in strutils. Breakage due to signature changes of the `rfind` methods. Co-authored-by: adigitoleo --- lib/pure/strutils.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index e4a4a5ec1..5e505ec3a 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1865,7 +1865,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, ## Use `s[start..last].rfind` for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,char,Natural>`_ + ## * `rfind func<#rfind,string,char,Natural,int>`_ ## * `replace func<#replace,string,char,char>`_ let last = if last == 0: s.high else: last when nimvm: @@ -1893,7 +1893,7 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. ## Use `s[start..last].find` for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,set[char],Natural>`_ + ## * `rfind func<#rfind,string,set[char],Natural,int>`_ ## * `multiReplace func<#multiReplace,string,varargs[]>`_ let last = if last == 0: s.high else: last for i in int(start)..last: @@ -1910,7 +1910,7 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, ## Use `s[start..last].find` for a `start`-origin index. ## ## See also: - ## * `rfind func<#rfind,string,string,Natural>`_ + ## * `rfind func<#rfind,string,string,Natural,int>`_ ## * `replace func<#replace,string,string,string>`_ if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) From 9b9ae8a487c6fbf77c8c72196e2b74f3371382b2 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 14 Jan 2022 00:39:17 +0800 Subject: [PATCH 0645/1654] nrvo shouldn't touch bycopy object[backport:1.2] (#19385) fix nim-lang#19342 --- compiler/ccgtypes.nim | 2 +- tests/objects/m19342.c | 12 ++++++++++++ tests/objects/t19342.nim | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests/objects/m19342.c create mode 100644 tests/objects/t19342.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 740ce5f19..df1874b26 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -220,7 +220,7 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3: + if rettype == nil or (tfByCopy notin rettype.flags and getSize(conf, rettype) > conf.target.floatSize*3): result = true else: case mapType(conf, rettype, skResult) diff --git a/tests/objects/m19342.c b/tests/objects/m19342.c new file mode 100644 index 000000000..113f65309 --- /dev/null +++ b/tests/objects/m19342.c @@ -0,0 +1,12 @@ +struct Node +{ + int data[25]; +}; + + +struct Node hello(int name) { + struct Node x = {999, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7 ,8, 9, + 1, 2, 3, 4, 5}; + return x; +} \ No newline at end of file diff --git a/tests/objects/t19342.nim b/tests/objects/t19342.nim new file mode 100644 index 000000000..d40d15a4b --- /dev/null +++ b/tests/objects/t19342.nim @@ -0,0 +1,18 @@ +discard """ + targets: "c cpp" +""" + +{.compile: "m19342.c".} + +# bug #19342 +type + Node* {.bycopy.} = object + data: array[25, cint] + +proc myproc(name: cint): Node {.importc: "hello", cdecl.} + +proc parse = + let node = myproc(10) + doAssert node.data[0] == 999 + +parse() From a93f6e7acc9e02deb40a864d345e4c715346a98c Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Thu, 13 Jan 2022 09:39:55 -0700 Subject: [PATCH 0646/1654] Generic parameters now can constrain statics in type definitions (#19362) * Parameters now can constrain static in type definitions resolved regression with generic procedures * Update compiler/sigmatch.nim Co-authored-by: Andreas Rumpf --- compiler/sigmatch.nim | 14 +++++++-- tests/generics/tstatic_constrained.nim | 42 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 tests/generics/tstatic_constrained.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 93be92676..4d8e03fe9 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1739,11 +1739,22 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, let prev = PType(idTableGet(c.bindings, f)) if prev == nil: if aOrig.kind == tyStatic: - if f.base.kind != tyNone: + if f.base.kind notin {tyNone, tyGenericParam}: result = typeRel(c, f.base, a, flags) if result != isNone and f.n != nil: if not exprStructuralEquivalent(f.n, aOrig.n): result = isNone + elif f.base.kind == tyGenericParam: + # Handling things like `type A[T; Y: static T] = object` + if f.base.len > 0: # There is a constraint, handle it + result = typeRel(c, f.base.lastSon, a, flags) + else: + # No constraint + if tfGenericTypeParam in f.flags: + result = isGeneric + else: + # for things like `proc fun[T](a: static[T])` + result = typeRel(c, f.base, a, flags) else: result = isGeneric if result != isNone: put(c, f, aOrig) @@ -1993,7 +2004,6 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, arg = argSemantized a = a c = m.c - if tfHasStatic in fMaybeStatic.flags: # XXX: When implicit statics are the default # this will be done earlier - we just have to diff --git a/tests/generics/tstatic_constrained.nim b/tests/generics/tstatic_constrained.nim new file mode 100644 index 000000000..07318d1bd --- /dev/null +++ b/tests/generics/tstatic_constrained.nim @@ -0,0 +1,42 @@ +discard """ + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout:''' +tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(41, 29) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(41, 29) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +got: +but expected: +tstatic_constrained.nim(42, 32) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(42, 32) Error: expression '' has no type (or is ambiguous) +''' +""" + +type + MyType[T; X: static T] = object + data: T + MyOtherType[T: float or string, Y: static T] = object + +func f[T,X](a: MyType[T,X]): MyType[T,X] = + when T is string: + MyType[T,X](data: a.data & X) + else: + MyType[T,X](data: a.data + X) + +discard MyType[int, 2](data: 1) +discard MyType[string, "Helelello"](data: "Hmmm") +discard MyType[int, 2](data: 1).f() +discard MyType[string, "Helelello"](data: "Hmmm").f() +discard MyOtherType[float, 1.3]() +discard MyOtherType[string, "Hello"]() +discard MyOtherType[int, 10]() +discard MyOtherType[byte, 10u8]() \ No newline at end of file From 7bdfeb78190ca9ff6a0cf702c6ec202f379bff2f Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Sat, 15 Jan 2022 01:24:23 -0700 Subject: [PATCH 0647/1654] Fixed concept constraints for static types (#19391) --- compiler/sigmatch.nim | 9 +++ tests/generics/tstatic_constrained.nim | 87 ++++++++++++++++++-------- 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 4d8e03fe9..e1195d04a 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1634,6 +1634,15 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, bindConcreteTypeToUserTypeClass(matched, a) if doBind: put(c, f, matched) result = isGeneric + elif a.len > 0 and a.lastSon == f: + # Needed for checking `Y` == `Addable` in the following + #[ + type + Addable = concept a, type A + a + a is A + MyType[T: Addable; Y: static T] = object + ]# + result = isGeneric else: result = isNone diff --git a/tests/generics/tstatic_constrained.nim b/tests/generics/tstatic_constrained.nim index 07318d1bd..3c9201548 100644 --- a/tests/generics/tstatic_constrained.nim +++ b/tests/generics/tstatic_constrained.nim @@ -2,41 +2,78 @@ discard """ cmd: "nim check --hints:off --warnings:off $file" action: "reject" nimout:''' -tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(44, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(41, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(44, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(41, 29) Error: object constructor needs an object type [proxy] -tstatic_constrained.nim(41, 29) Error: expression '' has no type (or is ambiguous) -tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(44, 31) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(44, 31) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(45, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(42, 20) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(27, 3)] +tstatic_constrained.nim(45, 22) Error: cannot instantiate MyOtherType [type declared in tstatic_constrained.nim(30, 5)] got: but expected: -tstatic_constrained.nim(42, 32) Error: object constructor needs an object type [proxy] -tstatic_constrained.nim(42, 32) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(45, 34) Error: object constructor needs an object type [proxy] +tstatic_constrained.nim(45, 34) Error: expression '' has no type (or is ambiguous) +tstatic_constrained.nim(77, 14) Error: cannot instantiate MyType [type declared in tstatic_constrained.nim(71, 5)] +got: +but expected: ''' """ +block: + type + MyType[T; X: static T] = object + data: T + MyOtherType[T: float or string, Y: static T] = object -type - MyType[T; X: static T] = object - data: T - MyOtherType[T: float or string, Y: static T] = object + func f[T,X](a: MyType[T,X]): MyType[T,X] = + when T is string: + MyType[T,X](data: a.data & X) + else: + MyType[T,X](data: a.data + X) -func f[T,X](a: MyType[T,X]): MyType[T,X] = - when T is string: - MyType[T,X](data: a.data & X) - else: - MyType[T,X](data: a.data + X) + discard MyType[int, 2](data: 1) + discard MyType[string, "Helelello"](data: "Hmmm") + discard MyType[int, 2](data: 1).f() + discard MyType[string, "Helelello"](data: "Hmmm").f() + discard MyOtherType[float, 1.3]() + discard MyOtherType[string, "Hello"]() + discard MyOtherType[int, 10]() + discard MyOtherType[byte, 10u8]() -discard MyType[int, 2](data: 1) -discard MyType[string, "Helelello"](data: "Hmmm") -discard MyType[int, 2](data: 1).f() -discard MyType[string, "Helelello"](data: "Hmmm").f() -discard MyOtherType[float, 1.3]() -discard MyOtherType[string, "Hello"]() -discard MyOtherType[int, 10]() -discard MyOtherType[byte, 10u8]() \ No newline at end of file +block: + type + Moduloable = concept m, type M + m mod m is M + Addable = concept a, type A + a + a is A + Modulo[T: Moduloable; Mod: static T] = distinct T + ModuloAdd[T: Moduloable or Addable; Mod: static T] = distinct T + ModuAddable = Addable or Moduloable + ModdAddClass[T: ModuAddable; Mod: static T] = distinct T + + proc toMod[T](val: T, modVal: static T): Modulo[T, modVal] = + mixin `mod` + Modulo[T, modVal](val mod modVal) + var + a = 3231.toMod(10) + b = 5483.toMod(10) + discard ModuloAdd[int, 3](0) + discard ModdAddClass[int, 3](0) + +block: + type + MyConstraint = int or string + MyOtherConstraint[T] = object + MyType[T: MyConstraint; Y: static T] = object + MyOtherType[T: MyOtherConstraint; Y: static T] = object + + var + a: MyType[int, 10] + b: MyType[string, "hello"] + c: MyType[float, 10d] + d: MyOtherType[MyOtherConstraint[float],MyOtherConstraint[float]()] + e: MyOtherType[MyOtherConstraint[int], MyOtherConstraint[int]()] \ No newline at end of file From 342b74ef70e6a71c09ba536d20e23a7e887d2fb8 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 15 Jan 2022 18:25:09 +0800 Subject: [PATCH 0648/1654] move type operation section and remove deepcopy document (#19389) ref #19173; because deepcopy is not fit for ORC/ARC which was used for spawn and spawn will be removed from compiler --- doc/manual.rst | 12 +++++++++ doc/manual_experimental.rst | 54 ++++++++++--------------------------- 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 8788e53a3..007b14b2d 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3902,6 +3902,18 @@ the operator is in scope (including if it is private). Type bound operators are: `=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`. +These operations can be *overridden* instead of *overloaded*. This means that +the implementation is automatically lifted to structured types. For instance, +if the type `T` has an overridden assignment operator `=`, this operator is +also used for assignments of the type `seq[T]`. + +Since these operations are bound to a type, they have to be bound to a +nominal type for reasons of simplicity of implementation; this means an +overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`. +This also means that one cannot override `deepCopy` for both `ptr T` and +`ref T` at the same time, instead a distinct or object helper type has to be +used for one pointer type. + For more details on some of those procs, see `Lifetime-tracking hooks `_. diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 0f82b9950..10958c29a 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -1192,51 +1192,25 @@ object inheritance syntax involving the `of` keyword: the `vtptr` magic produced types bound to `ptr` types. -Type bound operations -===================== +.. + deepCopy + -------- + `=deepCopy` is a builtin that is invoked whenever data is passed to + a `spawn`'ed proc to ensure memory safety. The programmer can override its + behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the + language may weaken this restriction.) -There are 4 operations that are bound to a type: + The signature has to be: -1. Assignment -2. Moves -3. Destruction -4. Deep copying for communication between threads + .. code-block:: nim -These operations can be *overridden* instead of *overloaded*. This means that -the implementation is automatically lifted to structured types. For instance, -if the type `T` has an overridden assignment operator `=`, this operator is -also used for assignments of the type `seq[T]`. + proc `=deepCopy`(x: T): T -Since these operations are bound to a type, they have to be bound to a -nominal type for reasons of simplicity of implementation; this means an -overridden `deepCopy` for `ref T` is really bound to `T` and not to `ref T`. -This also means that one cannot override `deepCopy` for both `ptr T` and -`ref T` at the same time, instead a distinct or object helper type has to be -used for one pointer type. + This mechanism will be used by most data structures that support shared memory, + like channels, to implement thread safe automatic memory management. -Assignments, moves and destruction are specified in -the `destructors `_ document. - - -deepCopy --------- - -`=deepCopy` is a builtin that is invoked whenever data is passed to -a `spawn`'ed proc to ensure memory safety. The programmer can override its -behaviour for a specific `ref` or `ptr` type `T`. (Later versions of the -language may weaken this restriction.) - -The signature has to be: - -.. code-block:: nim - - proc `=deepCopy`(x: T): T - -This mechanism will be used by most data structures that support shared memory, -like channels, to implement thread safe automatic memory management. - -The builtin `deepCopy` can even clone closures and their environments. See -the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. + The builtin `deepCopy` can even clone closures and their environments. See + the documentation of `spawn <#parallel-amp-spawn-spawn-statement>`_ for details. Dynamic arguments for bindSym From a95399143fdbd518f4d5fe33487b656c4cde7d6d Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Sat, 15 Jan 2022 15:19:05 +0300 Subject: [PATCH 0649/1654] Use openarray of bytes in md5 (#19307) * Use openarray of bytes in md5 * fix CI * cleanup * use noSideEffect for bootstrapping * fix CI again * actually fix CI by checking if it works * this is getting ridiculous * put old md5 version in compiler, remove vmop --- changelog.md | 1 + compiler/md5_old.nim | 297 +++++++++++++++++++++++++++++++++++ compiler/modulegraphs.nim | 2 +- compiler/sighashes.nim | 2 +- compiler/vmops.nim | 6 - lib/pure/md5.nim | 131 +++++++++++---- tests/js/tstdlib_imports.nim | 3 +- tests/stdlib/tmd5.nim | 19 ++- tests/test_nimscript.nims | 3 +- 9 files changed, 413 insertions(+), 51 deletions(-) create mode 100644 compiler/md5_old.nim diff --git a/changelog.md b/changelog.md index d07f80623..4d27be3a0 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,7 @@ filename argument for more informative errors. - Module `colors` expanded with missing colors from the CSS color standard. - Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). +- `md5` now works at compile time and in JavaScript. ## `std/smtp` diff --git a/compiler/md5_old.nim b/compiler/md5_old.nim new file mode 100644 index 000000000..da4a51198 --- /dev/null +++ b/compiler/md5_old.nim @@ -0,0 +1,297 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2010 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# `std/md5` without VM and JavaScript support, to circumvent a bug with +# openarrays on Nim < 1.4. + +when defined(nimHasStyleChecks): + {.push styleChecks: off.} + +type + MD5State = array[0..3, uint32] + MD5Block = array[0..15, uint32] + MD5CBits = array[0..7, uint8] + MD5Digest* = array[0..15, uint8] + ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. + MD5Buffer = array[0..63, uint8] + MD5Context* {.final.} = object + state: MD5State + count: array[0..1, uint32] + buffer: MD5Buffer + +const + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] + +proc F(x, y, z: uint32): uint32 {.inline.} = + result = (x and y) or ((not x) and z) + +proc G(x, y, z: uint32): uint32 {.inline.} = + result = (x and z) or (y and (not z)) + +proc H(x, y, z: uint32): uint32 {.inline.} = + result = x xor y xor z + +proc I(x, y, z: uint32): uint32 {.inline.} = + result = y xor (x or (not z)) + +proc rot(x: var uint32, n: uint8) {.inline.} = + x = (x shl n) or (x shr (32'u32 - n)) + +proc FF(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + F(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc GG(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + G(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc HH(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + H(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = + a = a + I(b, c, d) + x + ac + rot(a, s) + a = a + b + +proc encode(dest: var MD5Block, src: openArray[uint8]) = + var j = 0 + for i in 0..high(dest): + dest[i] = uint32(ord(src[j])) or + uint32(ord(src[j+1])) shl 8 or + uint32(ord(src[j+2])) shl 16 or + uint32(ord(src[j+3])) shl 24 + inc(j, 4) + +proc decode(dest: var openArray[uint8], src: openArray[uint32]) = + var i = 0 + for j in 0..high(src): + dest[i] = uint8(src[j] and 0xff'u32) + dest[i+1] = uint8(src[j] shr 8 and 0xff'u32) + dest[i+2] = uint8(src[j] shr 16 and 0xff'u32) + dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) + inc(i, 4) + +template slice(s: string | cstring, a, b): openArray[uint8] = + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + s.toOpenArray(a, b) + +proc transform(buffer: openArray[uint8], state: var MD5State) = + var + myBlock: MD5Block + encode(myBlock, buffer) + var a = state[0] + var b = state[1] + var c = state[2] + var d = state[3] + FF(a, b, c, d, myBlock[0], 7'u8, 0xD76AA478'u32) + FF(d, a, b, c, myBlock[1], 12'u8, 0xE8C7B756'u32) + FF(c, d, a, b, myBlock[2], 17'u8, 0x242070DB'u32) + FF(b, c, d, a, myBlock[3], 22'u8, 0xC1BDCEEE'u32) + FF(a, b, c, d, myBlock[4], 7'u8, 0xF57C0FAF'u32) + FF(d, a, b, c, myBlock[5], 12'u8, 0x4787C62A'u32) + FF(c, d, a, b, myBlock[6], 17'u8, 0xA8304613'u32) + FF(b, c, d, a, myBlock[7], 22'u8, 0xFD469501'u32) + FF(a, b, c, d, myBlock[8], 7'u8, 0x698098D8'u32) + FF(d, a, b, c, myBlock[9], 12'u8, 0x8B44F7AF'u32) + FF(c, d, a, b, myBlock[10], 17'u8, 0xFFFF5BB1'u32) + FF(b, c, d, a, myBlock[11], 22'u8, 0x895CD7BE'u32) + FF(a, b, c, d, myBlock[12], 7'u8, 0x6B901122'u32) + FF(d, a, b, c, myBlock[13], 12'u8, 0xFD987193'u32) + FF(c, d, a, b, myBlock[14], 17'u8, 0xA679438E'u32) + FF(b, c, d, a, myBlock[15], 22'u8, 0x49B40821'u32) + GG(a, b, c, d, myBlock[1], 5'u8, 0xF61E2562'u32) + GG(d, a, b, c, myBlock[6], 9'u8, 0xC040B340'u32) + GG(c, d, a, b, myBlock[11], 14'u8, 0x265E5A51'u32) + GG(b, c, d, a, myBlock[0], 20'u8, 0xE9B6C7AA'u32) + GG(a, b, c, d, myBlock[5], 5'u8, 0xD62F105D'u32) + GG(d, a, b, c, myBlock[10], 9'u8, 0x02441453'u32) + GG(c, d, a, b, myBlock[15], 14'u8, 0xD8A1E681'u32) + GG(b, c, d, a, myBlock[4], 20'u8, 0xE7D3FBC8'u32) + GG(a, b, c, d, myBlock[9], 5'u8, 0x21E1CDE6'u32) + GG(d, a, b, c, myBlock[14], 9'u8, 0xC33707D6'u32) + GG(c, d, a, b, myBlock[3], 14'u8, 0xF4D50D87'u32) + GG(b, c, d, a, myBlock[8], 20'u8, 0x455A14ED'u32) + GG(a, b, c, d, myBlock[13], 5'u8, 0xA9E3E905'u32) + GG(d, a, b, c, myBlock[2], 9'u8, 0xFCEFA3F8'u32) + GG(c, d, a, b, myBlock[7], 14'u8, 0x676F02D9'u32) + GG(b, c, d, a, myBlock[12], 20'u8, 0x8D2A4C8A'u32) + HH(a, b, c, d, myBlock[5], 4'u8, 0xFFFA3942'u32) + HH(d, a, b, c, myBlock[8], 11'u8, 0x8771F681'u32) + HH(c, d, a, b, myBlock[11], 16'u8, 0x6D9D6122'u32) + HH(b, c, d, a, myBlock[14], 23'u8, 0xFDE5380C'u32) + HH(a, b, c, d, myBlock[1], 4'u8, 0xA4BEEA44'u32) + HH(d, a, b, c, myBlock[4], 11'u8, 0x4BDECFA9'u32) + HH(c, d, a, b, myBlock[7], 16'u8, 0xF6BB4B60'u32) + HH(b, c, d, a, myBlock[10], 23'u8, 0xBEBFBC70'u32) + HH(a, b, c, d, myBlock[13], 4'u8, 0x289B7EC6'u32) + HH(d, a, b, c, myBlock[0], 11'u8, 0xEAA127FA'u32) + HH(c, d, a, b, myBlock[3], 16'u8, 0xD4EF3085'u32) + HH(b, c, d, a, myBlock[6], 23'u8, 0x04881D05'u32) + HH(a, b, c, d, myBlock[9], 4'u8, 0xD9D4D039'u32) + HH(d, a, b, c, myBlock[12], 11'u8, 0xE6DB99E5'u32) + HH(c, d, a, b, myBlock[15], 16'u8, 0x1FA27CF8'u32) + HH(b, c, d, a, myBlock[2], 23'u8, 0xC4AC5665'u32) + II(a, b, c, d, myBlock[0], 6'u8, 0xF4292244'u32) + II(d, a, b, c, myBlock[7], 10'u8, 0x432AFF97'u32) + II(c, d, a, b, myBlock[14], 15'u8, 0xAB9423A7'u32) + II(b, c, d, a, myBlock[5], 21'u8, 0xFC93A039'u32) + II(a, b, c, d, myBlock[12], 6'u8, 0x655B59C3'u32) + II(d, a, b, c, myBlock[3], 10'u8, 0x8F0CCC92'u32) + II(c, d, a, b, myBlock[10], 15'u8, 0xFFEFF47D'u32) + II(b, c, d, a, myBlock[1], 21'u8, 0x85845DD1'u32) + II(a, b, c, d, myBlock[8], 6'u8, 0x6FA87E4F'u32) + II(d, a, b, c, myBlock[15], 10'u8, 0xFE2CE6E0'u32) + II(c, d, a, b, myBlock[6], 15'u8, 0xA3014314'u32) + II(b, c, d, a, myBlock[13], 21'u8, 0x4E0811A1'u32) + II(a, b, c, d, myBlock[4], 6'u8, 0xF7537E82'u32) + II(d, a, b, c, myBlock[11], 10'u8, 0xBD3AF235'u32) + II(c, d, a, b, myBlock[2], 15'u8, 0x2AD7D2BB'u32) + II(b, c, d, a, myBlock[9], 21'u8, 0xEB86D391'u32) + state[0] = state[0] + a + state[1] = state[1] + b + state[2] = state[2] + c + state[3] = state[3] + d + +proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], + tags: [], gcsafe.} +proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} + +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + + +proc toMD5*(s: string): MD5Digest = + ## Computes the `MD5Digest` value for a string `s`. + ## + ## **See also:** + ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation + ## of the `MD5Digest` + ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string + runnableExamples: + assert $toMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" + + var c: MD5Context + md5Init(c) + md5Update(c, s.slice(0, s.len - 1)) + md5Final(c, result) + +proc `$`*(d: MD5Digest): string = + ## Converts a `MD5Digest` value into its string representation. + const digits = "0123456789abcdef" + result = "" + for i in 0..15: + add(result, digits[(d[i].int shr 4) and 0xF]) + add(result, digits[d[i].int and 0xF]) + +proc getMD5*(s: string): string = + ## Computes an MD5 value of `s` and returns its string representation. + ## + ## **See also:** + ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string + runnableExamples: + assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" + + var + c: MD5Context + d: MD5Digest + md5Init(c) + md5Update(c, s.slice(0, s.len - 1)) + md5Final(c, d) + result = $d + +proc `==`*(D1, D2: MD5Digest): bool = + ## Checks if two `MD5Digest` values are identical. + for i in 0..15: + if D1[i] != D2[i]: return false + return true + + +proc clearBuffer(c: var MD5Context) {.inline.} = + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + +proc md5Init*(c: var MD5Context) = + ## Initializes an `MD5Context`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + c.state[0] = 0x67452301'u32 + c.state[1] = 0xEFCDAB89'u32 + c.state[2] = 0x98BADCFE'u32 + c.state[3] = 0x10325476'u32 + c.count[0] = 0'u32 + c.count[1] = 0'u32 + clearBuffer(c) + +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + +proc md5Update*(c: var MD5Context, input: openArray[uint8]) = + ## Updates the `MD5Context` with the `input` data. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + var Index = int((c.count[0] shr 3) and 0x3F) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) + var PartLen = 64 - Index + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) + var i = PartLen + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) + inc(i, 64) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) + +proc md5Final*(c: var MD5Context, digest: var MD5Digest) = + ## Finishes the `MD5Context` and stores the result in `digest`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + var + Bits: MD5CBits + PadLen: int + decode(Bits, c.count) + var Index = int((c.count[0] shr 3) and 0x3F) + if Index < 56: PadLen = 56 - Index + else: PadLen = 120 - Index + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) + decode(digest, c.state) + clearBuffer(c) + + +when defined(nimHasStyleChecks): + {.pop.} #{.push styleChecks: off.} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 90b130e92..28e0ddb82 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import intsets, tables, hashes, md5 +import intsets, tables, hashes, md5_old import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index d02728627..3d36d2533 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -9,7 +9,7 @@ ## Computes hash values for routine (proc, method etc) signatures. -import ast, tables, ropes, md5, modulegraphs +import ast, tables, ropes, md5_old, modulegraphs from hashes import Hash import types diff --git a/compiler/vmops.nim b/compiler/vmops.nim index b9801234d..22a632ee9 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -25,7 +25,6 @@ when declared(math.signbit): from std/os import getEnv, existsEnv, delEnv, putEnv, envPairs, dirExists, fileExists, walkDir, getAppFilename, raiseOSError, osLastError -from std/md5 import getMD5 from std/times import cpuTime from std/hashes import hash from std/osproc import nil @@ -53,9 +52,6 @@ template ioop(op) {.dirty.} = template macrosop(op) {.dirty.} = registerCallback(c, "stdlib.macros." & astToStr(op), `op Wrapper`) -template md5op(op) {.dirty.} = - registerCallback(c, "stdlib.md5." & astToStr(op), `op Wrapper`) - template wrap1f_math(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = doAssert a.numArgs == 1 @@ -212,8 +208,6 @@ proc registerAdditionalOps*(c: PCtx) = of 2: setResult(a, round(getFloat(a, 0), getInt(a, 1).int)) else: doAssert false, $n - wrap1s(getMD5, md5op) - proc `mod Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, `mod`(getFloat(a, 0), getFloat(a, 1))) registerCallback(c, "stdlib.math.mod", `mod Wrapper`) diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 11c324548..06c5608a4 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -9,7 +9,7 @@ ## Module for computing [MD5 checksums](https://en.wikipedia.org/wiki/MD5). ## -## **Note:** The procs in this module can be used at compile time. +## This module also works at compile time and in JavaScript. ## ## See also ## ======== @@ -34,15 +34,16 @@ type buffer: MD5Buffer const - padding: cstring = "\x80\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0\0\0\0\0" & - "\0\0\0\0" + padding: array[0..63, uint8] = [ + 0x80'u8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 + ] proc F(x, y, z: uint32): uint32 {.inline.} = result = (x and y) or ((not x) and z) @@ -79,7 +80,7 @@ proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = rot(a, s) a = a + b -proc encode(dest: var MD5Block, src: cstring) = +proc encode(dest: var MD5Block, src: openArray[uint8]) = var j = 0 for i in 0..high(dest): dest[i] = uint32(ord(src[j])) or @@ -97,10 +98,48 @@ proc decode(dest: var openArray[uint8], src: openArray[uint32]) = dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) inc(i, 4) -proc transform(buffer: pointer, state: var MD5State) = +template slice(s: string, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + var s2 = newSeq[uint8](s.len) + for i in 0 ..< s2.len: + s2[i] = uint8(s[i]) + s2 + else: + s.toOpenArrayByte(a, b) + +template slice(s: cstring, a, b): openArray[uint8] = + when nimvm: + # toOpenArray is not implemented in VM + slice($s, a, b) + else: + when defined(js): + # toOpenArrayByte for cstring is not implemented in JS + slice($s, a, b) + else: + s.toOpenArrayByte(a, b) + +template slice(s: openArray[uint8], a, b): openArray[uint8] = + when nimvm: + s[a .. b] + else: + s.toOpenArray(a, b) + +const useMem = declared(copyMem) + +template memOrNot(withMem, withoutMem): untyped = + when nimvm: + withoutMem + else: + when useMem: + withMem + else: + withoutMem + +proc transform(buffer: openArray[uint8], state: var MD5State) = var myBlock: MD5Block - encode(myBlock, cast[cstring](buffer)) + encode(myBlock, buffer) var a = state[0] var b = state[1] var c = state[2] @@ -175,10 +214,18 @@ proc transform(buffer: pointer, state: var MD5State) = state[3] = state[3] + d proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], +proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], tags: [], gcsafe.} proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} +proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], + tags: [], gcsafe.} = + ## Updates the `MD5Context` with the `input` data of length `len`. + ## + ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this + ## function explicitly. + md5Update(c, input.slice(0, len - 1)) + proc toMD5*(s: string): MD5Digest = ## Computes the `MD5Digest` value for a string `s`. @@ -192,7 +239,7 @@ proc toMD5*(s: string): MD5Digest = var c: MD5Context md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, result) proc `$`*(d: MD5Digest): string = @@ -215,7 +262,7 @@ proc getMD5*(s: string): string = c: MD5Context d: MD5Digest md5Init(c) - md5Update(c, cstring(s), len(s)) + md5Update(c, s.slice(0, s.len - 1)) md5Final(c, d) result = $d @@ -226,6 +273,12 @@ proc `==`*(D1, D2: MD5Digest): bool = return true +proc clearBuffer(c: var MD5Context) {.inline.} = + memOrNot: + zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + do: + reset(c.buffer) + proc md5Init*(c: var MD5Context) = ## Initializes an `MD5Context`. ## @@ -237,29 +290,39 @@ proc md5Init*(c: var MD5Context) = c.state[3] = 0x10325476'u32 c.count[0] = 0'u32 c.count[1] = 0'u32 - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) + clearBuffer(c) -proc md5Update*(c: var MD5Context, input: cstring, len: int) = - ## Updates the `MD5Context` with the `input` data of length `len`. +proc writeBuffer(c: var MD5Context, index: int, + input: openArray[uint8], inputIndex, len: int) {.inline.} = + memOrNot: + copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) + do: + # cannot use system.`[]=` for arrays and openarrays as + # it can raise RangeDefect which gets tracked + for i in 0..`_, there's no need to call this ## function explicitly. - var input = input var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(len) shl 3) - if c.count[0] < (uint32(len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(len) shr 29) + c.count[0] = c.count[0] + (uint32(input.len) shl 3) + if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 + c.count[1] = c.count[1] + (uint32(input.len) shr 29) var PartLen = 64 - Index - if len >= PartLen: - copyMem(addr(c.buffer[Index]), input, PartLen) - transform(addr(c.buffer), c.state) + if input.len >= PartLen: + writeBuffer(c, Index, input, 0, PartLen) + transform(c.buffer, c.state) var i = PartLen - while i + 63 < len: - transform(addr(input[i]), c.state) + while i + 63 < input.len: + transform(input.slice(i, i + 63), c.state) inc(i, 64) - copyMem(addr(c.buffer[0]), addr(input[i]), len-i) - else: - copyMem(addr(c.buffer[Index]), addr(input[0]), len) + if i < input.len: + writeBuffer(c, 0, input, i, input.len - i) + elif input.len > 0: + writeBuffer(c, Index, input, 0, input.len) proc md5Final*(c: var MD5Context, digest: var MD5Digest) = ## Finishes the `MD5Context` and stores the result in `digest`. @@ -273,10 +336,10 @@ proc md5Final*(c: var MD5Context, digest: var MD5Digest) = var Index = int((c.count[0] shr 3) and 0x3F) if Index < 56: PadLen = 56 - Index else: PadLen = 120 - Index - md5Update(c, padding, PadLen) - md5Update(c, cast[cstring](addr(Bits)), 8) + md5Update(c, padding.slice(0, PadLen - 1)) + md5Update(c, Bits) decode(digest, c.state) - zeroMem(addr(c), sizeof(MD5Context)) + clearBuffer(c) when defined(nimHasStyleChecks): diff --git a/tests/js/tstdlib_imports.nim b/tests/js/tstdlib_imports.nim index 9986fa059..30aca7561 100644 --- a/tests/js/tstdlib_imports.nim +++ b/tests/js/tstdlib_imports.nim @@ -57,8 +57,7 @@ import std/[ htmlgen, # Hashing: - base64, hashes, - # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + base64, hashes, md5, # fails due to cstring cast/endians import: oids # fails due to copyMem/endians import: sha1 diff --git a/tests/stdlib/tmd5.nim b/tests/stdlib/tmd5.nim index 88a7b8d37..4017ac677 100644 --- a/tests/stdlib/tmd5.nim +++ b/tests/stdlib/tmd5.nim @@ -1,7 +1,16 @@ +discard """ + targets: "c cpp js" +""" + import md5 -doAssert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "a3cca2b2aa1e3b5b3b5aad99a8529074") -doAssert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == - "7e716d0e702df0505fc72e2b89467910") -doAssert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") +proc main() {.raises: [].} = + doAssert(getMD5("Franz jagt im komplett verwahrlosten Taxi quer durch Bayern") == + "a3cca2b2aa1e3b5b3b5aad99a8529074") + doAssert(getMD5("Frank jagt im komplett verwahrlosten Taxi quer durch Bayern") == + "7e716d0e702df0505fc72e2b89467910") + doAssert($toMD5("") == "d41d8cd98f00b204e9800998ecf8427e") + +main() + +static: main() diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 761ddafdf..22a566307 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -59,8 +59,7 @@ import std/[ htmlgen, # Hashing: - base64, hashes, - # fails due to cstring cast/zeroMem/copyMem/moveMem: md5 + base64, hashes, md5, # fails due to cstring cast/times import/endians import: oids # fails due to copyMem/endians import: sha1 From d102b2f54c41f19a0545f28109d0a93550b5d886 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 16 Jan 2022 18:08:38 +0800 Subject: [PATCH 0650/1654] deprecate unsafeAddr; extend addr (#19373) * deprecate unsafeAddr; extend addr addr is now available for all addressable locations, unsafeAddr is deprecated and become an alias for addr * follow @Vindaar's advice * change the signature of addr * unsafeAddr => addr (stdlib) * Update changelog.md * unsafeAddr => addr (tests) * Revert "unsafeAddr => addr (stdlib)" This reverts commit ab83c99c507048a8396e636bf22d55fdd84d7d1c. * doc changes; thanks to @konsumlamm Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 3 +++ compiler/isolation_check.nim | 2 +- compiler/semmagic.nim | 9 ++----- doc/manual.rst | 15 ++++------- doc/manual_experimental.rst | 2 +- lib/system.nim | 26 +++++++++++++------ tests/arc/tcustomtrace.nim | 2 +- tests/arc/thard_alignment.nim | 2 +- tests/async/tioselectors.nim | 2 +- tests/ccgbugs/tnoalias.nim | 2 +- tests/compiles/t8630.nim | 2 +- tests/concepts/tconcepts_issues.nim | 4 +-- tests/destructor/t12037.nim | 6 ++--- tests/destructor/tarray_indexing.nim | 2 +- tests/destructor/tcustomseqs.nim | 2 +- tests/destructor/tcustomstrings.nim | 2 +- .../destructor/tuse_result_prevents_sinks.nim | 2 +- tests/errmsgs/t10594.nim | 2 +- tests/errmsgs/tnon_concrete_cast.nim | 2 +- tests/js/tfieldchecks.nim | 2 +- tests/lent/tbasic_lent_check.nim | 10 +++---- tests/misc/taddr.nim | 2 +- tests/overload/t8829.nim | 4 +-- tests/overload/tstatic_with_converter.nim | 2 +- tests/stdlib/tdecls.nim | 14 +++++----- tests/stdlib/tmemory.nim | 2 +- tests/stdlib/tsqlitebindatas.nim | 4 +-- tests/strictnotnil/tnilcheck.nim | 2 +- tests/system/tostring.nim | 2 +- tests/types/tlent_var.nim | 2 +- 30 files changed, 69 insertions(+), 66 deletions(-) diff --git a/changelog.md b/changelog.md index 4d27be3a0..9b74d6d9f 100644 --- a/changelog.md +++ b/changelog.md @@ -17,6 +17,9 @@ - `std/sharedstrings` module is removed. - Constants `colors.colPaleVioletRed` and `colors.colMediumPurple` changed to match the CSS color standard. +- `addr` is now available for all addressable locations, `unsafeAddr` is deprecated and +becomes an alias for `addr`. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index a8c5a3651..68a212794 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -71,7 +71,7 @@ proc isValueOnlyType(t: PType): bool = proc canAlias*(arg, ret: PType): bool = if isValueOnlyType(arg): - # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe + # can alias only with addr(arg.x) and we don't care if it is not safe result = false else: var marker = initIntSet() diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 874098294..c7fc75620 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -14,13 +14,8 @@ proc semAddrArg(c: PContext; n: PNode; isUnsafeAddr = false): PNode = let x = semExprWithType(c, n) if x.kind == nkSym: x.sym.flags.incl(sfAddrTaken) - if isAssignable(c, x, isUnsafeAddr) notin {arLValue, arLocalLValue}: - # Do not suggest the use of unsafeAddr if this expression already is a - # unsafeAddr - if isUnsafeAddr: - localError(c.config, n.info, errExprHasNoAddress) - else: - localError(c.config, n.info, errExprHasNoAddress & "; maybe use 'unsafeAddr'") + if isAssignable(c, x, true) notin {arLValue, arLocalLValue}: + localError(c.config, n.info, errExprHasNoAddress) result = x proc semTypeOf(c: PContext; n: PNode): PNode = diff --git a/doc/manual.rst b/doc/manual.rst index 007b14b2d..4eabd0225 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -3502,8 +3502,9 @@ location is `T`, the `addr` operator result is of the type `ptr T`. An address is always an untraced reference. Taking the address of an object that resides on the stack is **unsafe**, as the pointer may live longer than the object on the stack and can thus reference a non-existing object. One can get -the address of variables, but one can't use it on variables declared through -`let` statements: +the address of variables. For easier interoperability with other compiled languages +such as C, retrieving the address of a `let` variable, a parameter, +or a `for` loop variable can be accomplished too: .. code-block:: nim @@ -3515,24 +3516,18 @@ the address of variables, but one can't use it on variables declared through # --> ref 0x7fff6b71b670 --> 0x10bb81050"Hello" echo cast[ptr string](t3)[] # --> Hello - # The following line doesn't compile: + # The following line also works echo repr(addr(t1)) - # Error: expression has no address - The unsafeAddr operator ----------------------- -For easier interoperability with other compiled languages such as C, retrieving -the address of a `let` variable, a parameter, or a `for` loop variable can -be accomplished by using the `unsafeAddr` operation: +The `unsafeAddr` operator is a deprecated alias for the `addr` operator: .. code-block:: nim - let myArray = [1, 2, 3] foreignProcThatTakesAnAddr(unsafeAddr myArray) - Procedures ========== diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 10958c29a..407924a13 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -603,7 +603,7 @@ has `source` as the owner. A path expression `e` is defined recursively: - Object field access `e.field` is a path expression. - `system.toOpenArray(e, ...)` is a path expression. - Pointer dereference `e[]` is a path expression. -- An address `addr e`, `unsafeAddr e` is a path expression. +- An address `addr e` is a path expression. - A type conversion `T(e)` is a path expression. - A cast expression `cast[T](e)` is a path expression. - `f(e, ...)` is a path expression if `f`'s return type is a view type. diff --git a/lib/system.nim b/lib/system.nim index c424cbc1b..3b3e28344 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -190,8 +190,16 @@ when defined(nimHasDeclaredMagic): else: proc declaredInScope*(x: untyped): bool {.magic: "DefinedInScope", noSideEffect, compileTime.} -proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = +proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## Builtin `addr` operator for taking the address of a memory location. + ## + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. + ## ## Cannot be overloaded. ## ## See also: @@ -205,15 +213,17 @@ proc `addr`*[T](x: var T): ptr T {.magic: "Addr", noSideEffect.} = ## echo p[] # b discard -proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = +proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect, + deprecated: "'unsafeAddr' is a deprecated alias for 'addr'".} = ## Builtin `addr` operator for taking the address of a memory - ## location. This works even for `let` variables or parameters - ## for better interop with C and so it is considered even more - ## unsafe than the ordinary `addr <#addr,T>`_. + ## location. ## - ## **Note**: When you use it to write a wrapper for a C library, you should - ## always check that the original library does never write to data behind the - ## pointer that is returned from this procedure. + ## .. note:: This works for `let` variables or parameters + ## for better interop with C. When you use it to write a wrapper + ## for a C library and take the address of `let` variables or parameters, + ## you should always check that the original library + ## does never write to data behind the pointer that is returned from + ## this procedure. ## ## Cannot be overloaded. discard diff --git a/tests/arc/tcustomtrace.nim b/tests/arc/tcustomtrace.nim index 3977194d9..5e0ecfb24 100644 --- a/tests/arc/tcustomtrace.nim +++ b/tests/arc/tcustomtrace.nim @@ -130,7 +130,7 @@ proc createSeq*[T](elems: varargs[T]): myseq[T] = result.data = cast[type(result.data)](alloc0(result.cap * sizeof(T))) inc allocCount when supportsCopyMem(T): - copyMem(result.data, unsafeAddr(elems[0]), result.cap * sizeof(T)) + copyMem(result.data, addr(elems[0]), result.cap * sizeof(T)) else: for i in 0.. 0 doAssert cmpMem(b.addr, a.addr, sizeof(SomeHash)) < 0 - doAssert cmpMem(a.addr, c.unsafeAddr, sizeof(SomeHash)) == 0 + doAssert cmpMem(a.addr, c.addr, sizeof(SomeHash)) == 0 diff --git a/tests/stdlib/tsqlitebindatas.nim b/tests/stdlib/tsqlitebindatas.nim index 754c80ae1..b2c3247fa 100644 --- a/tests/stdlib/tsqlitebindatas.nim +++ b/tests/stdlib/tsqlitebindatas.nim @@ -27,7 +27,7 @@ block tsqlitebindatas: ## db_sqlite binary data db.exec(createTableStr) var dbuf = newSeq[byte](orig.len*sizeof(float64)) - copyMem(unsafeAddr(dbuf[0]), unsafeAddr(orig[0]), dbuf.len) + copyMem(addr(dbuf[0]), addr(orig[0]), dbuf.len) var insertStmt = db.prepare("INSERT INTO test (id, name, data) VALUES (?, ?, ?)") insertStmt.bindParams(1, origName, dbuf) @@ -42,7 +42,7 @@ block tsqlitebindatas: ## db_sqlite binary data var dataTest = db.getValue(sql"SELECT data FROM test WHERE id = ?", 1) let seqSize = int(dataTest.len*sizeof(byte)/sizeof(float64)) var res: seq[float64] = newSeq[float64](seqSize) - copyMem(unsafeAddr(res[0]), addr(dataTest[0]), dataTest.len) + copyMem(addr(res[0]), addr(dataTest[0]), dataTest.len) doAssert res.len == orig.len doAssert res == orig diff --git a/tests/strictnotnil/tnilcheck.nim b/tests/strictnotnil/tnilcheck.nim index 5b9292522..c2d009b70 100644 --- a/tests/strictnotnil/tnilcheck.nim +++ b/tests/strictnotnil/tnilcheck.nim @@ -194,7 +194,7 @@ proc testAliasChanging(a: Nilable) = # # proc testPtrAlias(a: Nilable) = # # # pointer to a: hm. # # # alias to a? -# # var ptrA = a.unsafeAddr # {0, 1} +# # var ptrA = a.addr # {0, 1} # # if not a.isNil: # {0, 1} # # ptrA[] = nil # {0, 1} 0: MaybeNil 1: MaybeNil # # echo a.a #[ tt.Warning diff --git a/tests/system/tostring.nim b/tests/system/tostring.nim index bdbaa2ce6..cae20865e 100644 --- a/tests/system/tostring.nim +++ b/tests/system/tostring.nim @@ -47,7 +47,7 @@ import strutils let arr = ['H','e','l','l','o',' ','W','o','r','l','d','!','\0'] doAssert $arr == "['H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', '!', '\\x00']" -doAssert $cstring(unsafeAddr arr) == "Hello World!" +doAssert $cstring(addr arr) == "Hello World!" proc takes(c: cstring) = doAssert c == cstring"" diff --git a/tests/types/tlent_var.nim b/tests/types/tlent_var.nim index 491f6fde8..73b5bef9b 100644 --- a/tests/types/tlent_var.nim +++ b/tests/types/tlent_var.nim @@ -15,7 +15,7 @@ proc test_var(x: var MyObj): var int = var x = MyObj(a: 5) doAssert: test_var(x).addr == x.a.addr -doAssert: test_lent(x).unsafeAddr == x.a.addr +doAssert: test_lent(x).addr == x.a.addr proc varProc(x: var int) = x = 100 From 15e3813d96ad5646476b629f3a20ece89dff064d Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 04:10:35 +0800 Subject: [PATCH 0651/1654] add mm to compilesettings; deprecate gc (#19394) --- compiler/vmops.nim | 1 + lib/std/compilesettings.nim | 3 ++- tests/vm/tcompilesetting.nim | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 22a632ee9..018e7b9c6 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -135,6 +135,7 @@ when defined(nimHasInvariant): of backend: result = $conf.backend of libPath: result = conf.libpath.string of gc: result = $conf.selectedGC + of mm: result = $conf.selectedGC proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = template copySeq(field: untyped): untyped = diff --git a/lib/std/compilesettings.nim b/lib/std/compilesettings.nim index 5bb0bd673..6d8bd22f4 100644 --- a/lib/std/compilesettings.nim +++ b/lib/std/compilesettings.nim @@ -32,7 +32,8 @@ type backend ## the backend (eg: c|cpp|objc|js); both `nim doc --backend:js` ## and `nim js` would imply backend=js libPath ## the absolute path to the stdlib library, i.e. nim's `--lib`, since 1.5.1 - gc ## gc selected + gc {.deprecated.} ## gc selected + mm ## memory management selected MultipleValueSetting* {.pure.} = enum ## \ ## settings resulting in a seq of string values diff --git a/tests/vm/tcompilesetting.nim b/tests/vm/tcompilesetting.nim index 5bf559c74..d6c08e70f 100644 --- a/tests/vm/tcompilesetting.nim +++ b/tests/vm/tcompilesetting.nim @@ -1,5 +1,5 @@ discard """ -cmd: "nim c --nimcache:build/myNimCache --nimblePath:myNimblePath $file" +cmd: "nim c --nimcache:build/myNimCache --nimblePath:myNimblePath --gc:arc $file" joinable: false """ @@ -12,6 +12,7 @@ template main = doAssert "myNimblePath" in nimblePaths.querySettingSeq[0] doAssert querySetting(backend) == "c" doAssert fileExists(libPath.querySetting / "system.nim") + doAssert querySetting(mm) == "arc" static: main() main() From 2c5b367001cc53859048d9adf450a5db3a949b92 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 11:54:27 +0800 Subject: [PATCH 0652/1654] improve changelog a bit (#19400) --- changelog.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 9b74d6d9f..16b526f85 100644 --- a/changelog.md +++ b/changelog.md @@ -28,9 +28,7 @@ becomes an alias for `addr`. - Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). - `md5` now works at compile time and in JavaScript. -## `std/smtp` - -- Sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. +- `std/smtp` sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. - Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. - Added `IsoYear`, a distinct int type to prevent bugs from confusing the week-based year and the regular year. From 4f6b59de9698d0c29a2170ecc23c8d009b3181b7 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 18:14:13 +0800 Subject: [PATCH 0653/1654] mangle names in nimbase.h using cppDefine (#19395) [backport] mangle names in nimbase.h fix comments --- config/config.nims | 5 +++++ tests/ccgbugs/tmangle.nim | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 tests/ccgbugs/tmangle.nim diff --git a/config/config.nims b/config/config.nims index 893d6960f..88c344642 100644 --- a/config/config.nims +++ b/config/config.nims @@ -3,6 +3,11 @@ cppDefine "errno" cppDefine "unix" +# mangle the macro names in nimbase.h +cppDefine "NAN_INFINITY" +cppDefine "INF" +cppDefine "NAN" + when defined(nimStrictMode): # xxx add more flags here, and use `-d:nimStrictMode` in more contexts in CI. diff --git a/tests/ccgbugs/tmangle.nim b/tests/ccgbugs/tmangle.nim new file mode 100644 index 000000000..0050cef92 --- /dev/null +++ b/tests/ccgbugs/tmangle.nim @@ -0,0 +1,16 @@ +block: + proc hello() = + let NAN_INFINITY = 12 + doAssert NAN_INFINITY == 12 + let INF = "2.0" + doAssert INF == "2.0" + let NAN = 2.3 + doAssert NAN == 2.3 + + hello() + +block: + proc hello(NAN: float) = + doAssert NAN == 2.0 + + hello(2.0) From dc8ac66873d5189f2e5cf5dbf56a12ab64a38140 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 17 Jan 2022 04:02:31 -0800 Subject: [PATCH 0654/1654] fix no net compilation on zephyr (#19399) Co-authored-by: Jaremy J. Creechley --- lib/posix/posix_other.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 688867b98..941e13192 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -399,16 +399,20 @@ elif defined(zephyr): const Sockaddr_max_length* = 24 elif defined(net_raw): const Sockaddr_max_length* = 20 - else: + elif defined(net_ipv4): const Sockaddr_max_length* = 8 + else: + const Sockaddr_max_length* = 255 # just for compilation purposes + const Sockaddr_un_path_length* = Sockaddr_max_length # Zephyr is heavily customizable so it's easy to get to a state # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. - {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", - Sockaddr_max_length, - ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", - " Check that Nim and Zephyr IPv4/IPv6 settings match.", - " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} + when defined(net_ipv4) or defined(net_ipv6) or defined(net_raw): + {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ", + Sockaddr_max_length, + ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!", + " Check that Nim and Zephyr IPv4/IPv6 settings match.", + " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].} elif defined(freertos) or defined(lwip): const Sockaddr_max_length* = 14 const Sockaddr_un_path_length* = 108 From 07c7a8a526cf4fd12c96192cda7a750fbea3382f Mon Sep 17 00:00:00 2001 From: hlaaftana <10591326+hlaaftana@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:03:40 +0300 Subject: [PATCH 0655/1654] Optimize lent in JS [backport:1.6] (#19393) * Optimize lent in JS [backport:1.6] * addr on lent doesn't work anymore, don't use it * use unsafeAddr in test again for older versions --- compiler/jsgen.nim | 51 ++++++++++++++++++++++---------- tests/js/tlent.nim | 33 +++++++++++++++++++++ tests/lent/tbasic_lent_check.nim | 20 ++++++++----- 3 files changed, 81 insertions(+), 23 deletions(-) create mode 100644 tests/js/tlent.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 7e8c25173..0a7036b46 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -178,7 +178,7 @@ const proc mapType(typ: PType): TJSTypeKind = let t = skipTypes(typ, abstractInst) case t.kind - of tyVar, tyRef, tyPtr, tyLent: + of tyVar, tyRef, tyPtr: if skipTypes(t.lastSon, abstractInst).kind in MappedToObject: result = etyObject else: @@ -186,7 +186,8 @@ proc mapType(typ: PType): TJSTypeKind = of tyPointer: # treat a tyPointer like a typed pointer to an array of bytes result = etyBaseIndex - of tyRange, tyDistinct, tyOrdinal, tyProxy: + of tyRange, tyDistinct, tyOrdinal, tyProxy, tyLent: + # tyLent is no-op as JS has pass-by-reference semantics result = mapType(t[0]) of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt of tyBool: result = etyBool @@ -1060,14 +1061,14 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = xtyp = etySeq case xtyp of etySeq: - if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) else: useMagic(p, "nimCopy") lineF(p, "$1 = nimCopy(null, $2, $3);$n", [a.rdLoc, b.res, genTypeInfo(p, y.typ)]) of etyObject: - if x.typ.kind in {tyVar} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: + if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) else: useMagic(p, "nimCopy") @@ -1092,10 +1093,18 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) = lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res]) lineF(p, "$1 = $2;$n", [a.address, b.res]) lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) + elif a.typ == etyBaseIndex: + # array indexing may not map to var type + if b.address != nil: + lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res]) + else: + lineF(p, "$1 = $2;$n", [a.address, b.res]) else: internalError(p.config, x.info, $("genAsgn", b.typ, a.typ)) - else: + elif b.address != nil: lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res]) + else: + lineF(p, "$1 = $2;$n", [a.address, b.res]) else: lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc]) @@ -1288,7 +1297,7 @@ template isIndirect(x: PSym): bool = v.kind notin {skProc, skFunc, skConverter, skMethod, skIterator, skConst, skTemp, skLet}) -proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = +proc genSymAddr(p: PProc, n: PNode, typ: PType, r: var TCompRes) = let s = n.sym if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3") case s.kind @@ -1456,13 +1465,17 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) = else: if s.loc.r == nil: internalError(p.config, n.info, "symbol has no generated name: " & s.name.s) - r.res = s.loc.r + if mapType(p, s.typ) == etyBaseIndex: + r.address = s.loc.r + r.res = s.loc.r & "_Idx" + else: + r.res = s.loc.r r.kind = resVal proc genDeref(p: PProc, n: PNode, r: var TCompRes) = let it = n[0] let t = mapType(p, it.typ) - if t == etyObject: + if t == etyObject or it.typ.kind == tyLent: gen(p, it, r) else: var a: TCompRes @@ -1703,7 +1716,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = result = putToSeq("0", indirect) of tyFloat..tyFloat128: result = putToSeq("0.0", indirect) - of tyRange, tyGenericInst, tyAlias, tySink, tyOwned: + of tyRange, tyGenericInst, tyAlias, tySink, tyOwned, tyLent: result = createVar(p, lastSon(typ), indirect) of tySet: result = putToSeq("{}", indirect) @@ -1745,7 +1758,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope = createObjInitList(p, t, initIntSet(), initList) result = ("({$1})") % [initList] if indirect: result = "[$1]" % [result] - of tyVar, tyPtr, tyLent, tyRef, tyPointer: + of tyVar, tyPtr, tyRef, tyPointer: if mapType(p, t) == etyBaseIndex: result = putToSeq("[null, 0]", indirect) else: @@ -2394,16 +2407,17 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = if prc.typ[0] != nil and sfPure notin prc.flags: resultSym = prc.ast[resultPos].sym let mname = mangleName(p.module, resultSym) - if not isIndirect(resultSym) and + let returnAddress = not isIndirect(resultSym) and resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and - mapType(p, resultSym.typ) == etyBaseIndex: + mapType(p, resultSym.typ) == etyBaseIndex + if returnAddress: resultAsgn = p.indentLine(("var $# = null;$n") % [mname]) resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname]) else: let resVar = createVar(p, resultSym.typ, isIndirect(resultSym)) resultAsgn = p.indentLine(("var $# = $#;$n") % [mname, resVar]) gen(p, prc.ast[resultPos], a) - if mapType(p, resultSym.typ) == etyBaseIndex: + if returnAddress: returnStmt = "return [$#, $#];$n" % [a.address, a.res] else: returnStmt = "return $#;$n" % [a.res] @@ -2579,8 +2593,15 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = of nkObjConstr: genObjConstr(p, n, r) of nkHiddenStdConv, nkHiddenSubConv, nkConv: genConv(p, n, r) of nkAddr, nkHiddenAddr: - genAddr(p, n, r) - of nkDerefExpr, nkHiddenDeref: genDeref(p, n, r) + if n.typ.kind in {tyLent}: + gen(p, n[0], r) + else: + genAddr(p, n, r) + of nkDerefExpr, nkHiddenDeref: + if n.typ.kind in {tyLent}: + gen(p, n[0], r) + else: + genDeref(p, n, r) of nkBracketExpr: genArrayAccess(p, n, r) of nkDotExpr: genFieldAccess(p, n, r) of nkCheckedFieldExpr: genCheckedFieldOp(p, n, nil, r) diff --git a/tests/js/tlent.nim b/tests/js/tlent.nim new file mode 100644 index 000000000..2546e5b1d --- /dev/null +++ b/tests/js/tlent.nim @@ -0,0 +1,33 @@ +discard """ + output: ''' +hmm +100 +hmm +100 +''' +""" + +# #16800 + +type A = object + b: int +var t = A(b: 100) +block: + proc getValues: lent int = + echo "hmm" + result = t.b + echo getValues() +block: + proc getValues: lent int = + echo "hmm" + t.b + echo getValues() + +when false: # still an issue, #16908 + template main = + iterator fn[T](a:T): lent T = yield a + let a = @[10] + for b in fn(a): echo b + + static: main() + main() diff --git a/tests/lent/tbasic_lent_check.nim b/tests/lent/tbasic_lent_check.nim index 0787e36d9..92d731451 100644 --- a/tests/lent/tbasic_lent_check.nim +++ b/tests/lent/tbasic_lent_check.nim @@ -17,22 +17,26 @@ proc main = main() template main2 = # bug #15958 + when defined(js): + proc sameAddress[T](a, b: T): bool {.importjs: "(# === #)".} + else: + template sameAddress(a, b): bool = a.unsafeAddr == b.unsafeAddr proc byLent[T](a: T): lent T = a let a = [11,12] let b = @[21,23] let ss = {1, 2, 3, 5} doAssert byLent(a) == [11,12] - doAssert byLent(a).addr == a.addr + doAssert sameAddress(byLent(a), a) doAssert byLent(b) == @[21,23] - when not defined(js): # pending bug #16073 - doAssert byLent(b).addr == b.addr + # pending bug #16073 + doAssert sameAddress(byLent(b), b) doAssert byLent(ss) == {1, 2, 3, 5} - doAssert byLent(ss).addr == ss.addr + doAssert sameAddress(byLent(ss), ss) let r = new(float) r[] = 10.0 - when not defined(js): # pending bug #16073 - doAssert byLent(r)[] == 10.0 + # bug #16073 + doAssert byLent(r)[] == 10.0 when not defined(js): # pending bug https://github.com/timotheecour/Nim/issues/372 let p = create(float) @@ -41,9 +45,9 @@ template main2 = # bug #15958 proc byLent2[T](a: openArray[T]): lent T = a[0] doAssert byLent2(a) == 11 - doAssert byLent2(a).addr == a[0].addr + doAssert sameAddress(byLent2(a), a[0]) doAssert byLent2(b) == 21 - doAssert byLent2(b).addr == b[0].addr + doAssert sameAddress(byLent2(b), b[0]) proc byLent3[T](a: varargs[T]): lent T = a[1] let From bc3bee54698b108977766b6b2daf8a33bd00d1ef Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 17 Jan 2022 20:06:31 +0800 Subject: [PATCH 0656/1654] Added `std/oserrors` for OS error reporting (#19390) * Added 'std/oserrors' for OS error reporting * add a simple test * New code should not support -d:useWinAnsi anymore thanks to @Araq --- changelog.md | 2 + lib/std/oserrors.nim | 118 +++++++++++++++++++++++++++++++++++++ tests/stdlib/toserrors.nim | 9 +++ 3 files changed, 129 insertions(+) create mode 100644 lib/std/oserrors.nim create mode 100644 tests/stdlib/toserrors.nim diff --git a/changelog.md b/changelog.md index 16b526f85..82f9daa1e 100644 --- a/changelog.md +++ b/changelog.md @@ -74,6 +74,8 @@ becomes an alias for `addr`. for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". +- Added `std/oserrors` for OS error reporting. + ## Compiler changes - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim new file mode 100644 index 000000000..9c2649eab --- /dev/null +++ b/lib/std/oserrors.nim @@ -0,0 +1,118 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/oserrors` module implements OS error reporting. + +type + OSErrorCode* = distinct int32 ## Specifies an OS Error Code. + +when not defined(nimscript): + when defined(windows): + import winlean + else: + var errno {.importc, header: "".}: cint + + proc c_strerror(errnum: cint): cstring {. + importc: "strerror", header: "".} + +proc `==`*(err1, err2: OSErrorCode): bool {.borrow.} +proc `$`*(err: OSErrorCode): string {.borrow.} + +proc osErrorMsg*(errorCode: OSErrorCode): string = + ## Converts an OS error code into a human readable string. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If conversion fails, or `errorCode` is `0` then `""` will be + ## returned. + ## + ## See also: + ## * `raiseOSError proc`_ + ## * `osLastError proc`_ + runnableExamples: + when defined(linux): + assert osErrorMsg(OSErrorCode(0)) == "" + assert osErrorMsg(OSErrorCode(1)) == "Operation not permitted" + assert osErrorMsg(OSErrorCode(2)) == "No such file or directory" + + result = "" + when defined(nimscript): + discard + elif defined(windows): + if errorCode != OSErrorCode(0'i32): + var msgbuf: WideCString + if formatMessageW(0x00000100 or 0x00001000 or 0x00000200, + nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32: + result = $msgbuf + if msgbuf != nil: localFree(cast[pointer](msgbuf)) + else: + if errorCode != OSErrorCode(0'i32): + result = $c_strerror(errorCode.int32) + +proc newOSError*( + errorCode: OSErrorCode, additionalInfo = "" +): owned(ref OSError) {.noinline.} = + ## Creates a new `OSError exception `_. + ## + ## The `errorCode` will determine the + ## message, `osErrorMsg proc`_ will be used + ## to get this message. + ## + ## The error code can be retrieved using the `osLastError proc`_. + ## + ## If the error code is `0` or an error message could not be retrieved, + ## the message `unknown OS error` will be used. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `osLastError proc`_ + var e: owned(ref OSError); new(e) + e.errorCode = errorCode.int32 + e.msg = osErrorMsg(errorCode) + if additionalInfo.len > 0: + if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' + e.msg.add "Additional info: " + e.msg.add additionalInfo + # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. + if e.msg == "": + e.msg = "unknown OS error" + return e + +proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = + ## Raises an `OSError exception `_. + ## + ## Read the description of the `newOSError proc`_ to learn + ## how the exception object is created. + raise newOSError(errorCode, additionalInfo) + +{.push stackTrace:off.} +proc osLastError*(): OSErrorCode {.sideEffect.} = + ## Retrieves the last operating system error code. + ## + ## This procedure is useful in the event when an OS call fails. In that case + ## this procedure will return the error code describing the reason why the + ## OS call failed. The `OSErrorMsg` procedure can then be used to convert + ## this code into a string. + ## + ## .. warning:: The behaviour of this procedure varies between Windows and POSIX systems. + ## On Windows some OS calls can reset the error code to `0` causing this + ## procedure to return `0`. It is therefore advised to call this procedure + ## immediately after an OS call fails. On POSIX systems this is not a problem. + ## + ## See also: + ## * `osErrorMsg proc`_ + ## * `raiseOSError proc`_ + when defined(nimscript): + discard + elif defined(windows): + result = cast[OSErrorCode](getLastError()) + else: + result = OSErrorCode(errno) +{.pop.} diff --git a/tests/stdlib/toserrors.nim b/tests/stdlib/toserrors.nim new file mode 100644 index 000000000..e907dfe63 --- /dev/null +++ b/tests/stdlib/toserrors.nim @@ -0,0 +1,9 @@ +discard """ + action: compile +""" + +import std/oserrors + +let x1 = osLastError() +raiseOSError(x1) +echo osErrorMsg(x1) From 5d303762f156a276cb37162229e6b82b01e08253 Mon Sep 17 00:00:00 2001 From: Hamid Bluri Date: Tue, 18 Jan 2022 12:55:39 -0800 Subject: [PATCH 0657/1654] update deprecated example (#19415) `toNimIdent` proc is deprecated, so I replaced it with `ident` proc --- lib/core/macros.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ed7e85c64..8ab893d21 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1373,7 +1373,7 @@ template findChild*(n: NimNode; cond: untyped): NimNode {.dirty.} = ## ## .. code-block:: nim ## var res = findChild(n, it.kind == nnkPostfix and - ## it.basename.ident == toNimIdent"foo") + ## it.basename.ident == ident"foo") block: var res: NimNode for it in n.children: From d7869a80092354468594f7d5c5dfdd91f564fec7 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 18 Jan 2022 23:58:18 +0300 Subject: [PATCH 0658/1654] Improve Zshell completion (#19354) --- tools/nim.zsh-completion | 258 +++++++++++++++++---------------------- 1 file changed, 114 insertions(+), 144 deletions(-) diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index 45477d860..1c3670fd9 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -1,150 +1,120 @@ #compdef nim +# Installation note: +# Please name this file as _nim (with underscore!) and copy it to a +# completions directory, either: +# - system wide one, like /usr/share/zsh/functions/Completion/Unix/ on Linux +# - or to a user directory like ~/scripts. Then you also need to add +# that directory in your ~/.zshrc to `fpath` array like so: +# fpath=( ~/scripts "${fpath[@]}" ) + _nim() { - _arguments -C \ - ':command:(( - {compile,c}\:compile\ project\ with\ default\ code\ generator\ C - doc\:generate\ the\ documentation\ for\ inputfile - {compileToC,cc}\:compile\ project\ with\ C\ code\ generator - {compileToCpp,cpp}\:compile\ project\ to\ C++\ code - {compileToOC,objc}\:compile\ project\ to\ Objective\ C\ code - js\:compile\ project\ to\ Javascript - e\:run\ a\ Nimscript\ file - rst2html\:convert\ a\ reStructuredText\ file\ to\ HTML - rst2tex\:convert\ a\ reStructuredText\ file\ to\ TeX - jsondoc\:extract\ the\ documentation\ to\ a\ json\ file - buildIndex\:build\ an\ index\ for\ the\ whole\ documentation - genDepend\:generate\ a\ DOT\ file\ containing\ the\ module\ dependency\ graph - dump\:dump\ all\ defined\ conditionals\ and\ search\ paths - check\:checks\ the\ project\ for\ syntax\ and\ semantic - ))' \ - '*-r[run the application]' \ - '*--run[run the application]' \ - '*-p=[add path to search paths]' \ - '*--path=[add path to search paths]' \ - '*-d=[define a conditional symbol]' \ - '*--define=[define a conditional symbol]' \ - '*-u=[undefine a conditional symbol]' \ - '*--undef=[undefine a conditional symbol]' \ - '*-f[force rebuilding of all modules]' \ - '*--forceBuild[force rebuilding of all modules]' \ - '*--stackTrace=on[turn stack tracing on]' \ - '*--stackTrace=off[turn stack tracing off]' \ - '*--lineTrace=on[turn line tracing on]' \ - '*--lineTrace=off[turn line tracing off]' \ - '*--threads=on[turn support for multi-threading on]' \ - '*--threads=off[turn support for multi-threading off]' \ - '*-x=on[turn all runtime checks on]' \ - '*-x=off[turn all runtime checks off]' \ - '*--checks=on[turn all runtime checks on]' \ - '*--checks=off[turn all runtime checks off]' \ - '*--objChecks=on[turn obj conversion checks on]' \ - '*--objChecks=off[turn obj conversion checks off]' \ - '*--fieldChecks=on[turn case variant field checks on]' \ - '*--fieldChecks=off[turn case variant field checks off]' \ - '*--rangeChecks=on[turn range checks on]' \ - '*--rangeChecks=off[turn range checks off]' \ - '*--boundChecks=on[turn bound checks on]' \ - '*--boundChecks=off[turn bound checks off]' \ - '*--overflowChecks=on[turn int over-/underflow checks on]' \ - '*--overflowChecks=off[turn int over-/underflow checks off]' \ - '*-a[turn assertions on]' \ - '*-a[turn assertions off]' \ - '*--assertions=on[turn assertions on]' \ - '*--assertions=off[turn assertions off]' \ - '*--floatChecks=on[turn all floating point (NaN/Inf) checks on]' \ - '*--floatChecks=off[turn all floating point (NaN/Inf) checks off]' \ - '*--nanChecks=on[turn NaN checks on]' \ - '*--nanChecks=off[turn NaN checks off]' \ - '*--infChecks=on[turn Inf checks on]' \ - '*--infChecks=off[turn Inf checks off]' \ - '*--nilChecks=on[turn nil checks on]' \ - '*--nilChecks=off[turn nil checks off]' \ - '*--opt=none[do not optimize]' \ - '*--opt=speed[optimize for speed|size - use -d:release for a release build]' \ - '*--opt=size[optimize for size]' \ - '*--debugger:native[use native debugger (gdb)]' \ - '*--app=console[generate a console app]' \ - '*--app=gui[generate a GUI app]' \ - '*--app=lib[generate a dynamic library]' \ - '*--app=staticlib[generate a static library]' \ - '*--cpu=alpha[compile for Alpha architecture]' \ - '*--cpu=amd64[compile for x86_64 architecture]' \ - '*--cpu=arm[compile for ARM architecture]' \ - '*--cpu=arm64[compile for ARM64 architecture]' \ - '*--cpu=avr[compile for AVR architecture]' \ - '*--cpu=esp[compile for ESP architecture]' \ - '*--cpu=e2k[compile for Elbrus 2000 architecture]' \ - '*--cpu=hppa[compile for HPPA architecture]' \ - '*--cpu=i386[compile for i386 architecture]' \ - '*--cpu=ia64[compile for ia64 architecture]' \ - '*--cpu=js[compile to JavaScript]' \ - '*--cpu=m68k[compile for m68k architecture]' \ - '*--cpu=mips[compile for MIPS architecture]' \ - '*--cpu=mipsel[compile for MIPS EL architecture]' \ - '*--cpu=mips64[compile for MIPS64 architecture]' \ - '*--cpu=mips64el[compile for MIPS64 EL architecture]' \ - '*--cpu=msp430[compile for msp430 architecture]' \ - '*--cpu=nimvm[compile for Nim VM]' \ - '*--cpu=powerpc[compile for PowerPC architecture]' \ - '*--cpu=powerpc64[compile for PowerPC64 architecture]' \ - '*--cpu=powerpc64el[compile for PowerPC64 EL architecture]' \ - '*--cpu=riscv32[compile for RISC-V 32 architecture]' \ - '*--cpu=riscv64[compile for RISC-V 64 architecture]' \ - '*--cpu=sparc[compile for SPARC architecture]' \ - '*--cpu=sparc64[compile for SPARC64 architecture]' \ - '*--cpu=vm[compile for Nim VM]' \ - '*--cpu=wasm32[compile to WASM 32]' \ - '*--cpu=loongarch64[compile for LoongArch64 architecture]' \ - '*--gc=refc[use reference counting garbage collection]' \ - '*--gc=arc[use ARC garbage collection]' \ - '*--gc=orc[use ORC garbage collection]' \ - '*--gc=markAndSweep[use mark-and-sweep garbage collection]' \ - '*--gc=boehm[use Boehm garbage collection]' \ - '*--gc=go[use Go garbage collection]' \ - '*--gc=regions[use region-based memory management]' \ - '*--gc=none[disable garbage collection]' \ - '*--os=Standalone[generate a stand-alone executable]' \ - '*--os=AIX[compile for AIX]' \ - '*--os=Amiga[compile for Amiga OS]' \ - '*--os=Android[compile for Android]' \ - '*--os=Any[compile for any OS]' \ - '*--os=Atari[compile for Atari]' \ - '*--os=DOS[compile for DOS]' \ - '*--os=DragonFly[compile for DragonFly]' \ - '*--os=FreeBSD[compile for FreeBSD]' \ - '*--os=FreeRTOS[compile for FreeRTOS]' \ - '*--os=Genode[compile for Genode]' \ - '*--os=Haiku[compile for Haiku]' \ - '*--os=iOS[compile for iOS]' \ - '*--os=Irix[compile for Irix]' \ - '*--os=Linux[compile for Linux]' \ - '*--os=MacOS[compile for MacOS]' \ - '*--os=MacOSX[compile for MacOSX]' \ - '*--os=MorphOS[compile for MorphOS]' \ - '*--os=NetBSD[compile for NetBSD]' \ - '*--os=Netware[compile for Netware]' \ - '*--os=NimVM[compile for NimVM]' \ - '*--os=NintendoSwitch[compile for NintendoSwitch]' \ - '*--os=OS2[compile for OS2]' \ - '*--os=OpenBSD[compile for OpenBSD]' \ - '*--os=PalmOS[compile for PalmOS]' \ - '*--os=QNX[compile for QNX]' \ - '*--os=SkyOS[compile for SkyOS]' \ - '*--os=Solaris[compile for Solaris]' \ - '*--os=VxWorks[compile for VxWorks]' \ - '*--os=Windows[compile for Windows]' \ - '*--os=JS[generate javascript]' \ - '*--panics=off[turn panics into process terminations: off by default]' \ - '*--panics=on[turn panics into process terminations]' \ - '*--verbosity=0[set verbosity to 0]' \ - '*--verbosity=1[set verbosity to 1 (default)]' \ - '*--verbosity=2[set verbosity to 2]' \ - '*--verbosity=3[set verbosity to 3]' \ - '*--hints=on[print compilation hints]' \ - '*--hints=off[disable compilation hints]' \ - '*--hints=list[print compilation hints list]' \ - ':filename:_files -g"*.nim"' + local -a nimCommands=( + {compile,c}:'compile project with default code generator C' + {compileToC,cc}:'compile project with C code generator' + {compileToCpp,cpp}:'compile project to C++ code' + {compileToOC,objc}:'compile project to Objective C code' + 'js:compile project to Javascript' + 'e:run a Nimscript file' + 'doc:generate the HTML documentation for inputfile' + 'rst2html:convert a reStructuredText file to HTML' + 'doc2tex:generate the documentation for inputfile to LaTeX' + 'rst2tex:convert a reStructuredText file to TeX' + 'jsondoc:extract the documentation to a json file' + 'buildIndex:build an index for the whole documentation' + 'genDepend:generate a DOT file containing the module dependency graph' + 'dump:dump all defined conditionals and search paths' + 'check:checks the project for syntax and semantic' + {--help,-h}:'basic help' + '--fullhelp:show all switches' + {-v,--version}:'show version' + ) + + _arguments '*:: :->anyState' && return 0 + + if (( CURRENT == 1 )); then + _describe -t commands "Nim commands" nimCommands -V1 + return + fi + + local -a conditionalSymbols=( + release danger mingw androidNDK useNimRtl useMalloc noSignalHandler ssl + debug leanCompiler gcDestructors) + local -a sharedOpts=( + {--define\\:-,-d\\:-}'[define a conditional symbol]:x:($conditionalSymbols)' + {--undef\\:-,-u\\:-}'[undefine a conditional symbol]:x:($conditionalSymbols)' + {--path\\:-,-p\\:-}'[add path to search paths]:x:_files' + '--verbosity\:-[set verbosity level (default\: 1)]:x:(0 1 2 3)' + '--hints\:-[print compilation hints? (or `list`)]:x:(on off list)' + ) + local runOpts=( + {--run,-r}'[run the application]' + ) + local docOpts=( + '--index\:-[enable index .idx files?]:x:(on off)' + '--project\:-[output any dependency for doc?]:x:(on off)' + '--docInternal\:-[generate module-private documentation?]:x:(on off)' + ) + local -a codeOpts=( + {--forceBuild,-f}'[force rebuilding of all modules]' + '--stackTrace\:-[enable stack tracing?]:x:(on off)' + '--lineTrace\:-[enable line tracing?]:x:(on off)' + '--threads\:-[enable support for multi-threading?]:x:(on off)' + {--checks\\:-,-x\\:-}'[enable/disable all runtime checks?]:x:(on off)' + '--objChecks\:-[enable obj conversion checks]:x:(on off)' + '--fieldChecks\:-[enable case variant field checks?]:x:(on off)' + '--rangeChecks\:-[enable range checks?]:x:(on off)' + '--boundChecks\:-[enable bound checks?]:x:(on off)' + '--overflowChecks\:-[enable integer over-/underflow checks?]:x:(on off)' + {--assertions\\:-,-a\\:-}'[enable assertions?]:x:(on off)' + '--floatChecks\:-[enable floating point (NaN/Inf) checks?]:x:(on off)' + '--nanChecks\:-[enable NaN checks?]:x:(on off)' + '--infChecks\:-[enable Inf checks?]:x:(on off)' + '--nilChecks\:-[enable nil checks?]:x:(on off)' + '--expandArc\:-[show how given proc looks before final backend pass]' + '--expandMacro\:-[dump every generated AST from given macro]' + ) + local -a nativeOpts=( + '--opt\:-[optimization mode]:x:(none speed size)' + '--debugger\:native[use native debugger (gdb)]' + '--app\:-[generate this type of app (lib=dynamic)]:x:(console gui lib staticlib)' + '--cpu\:-[target architecture]:x:(alpha amd64 arm arm64 avr e2k esp hppa i386 ia64 js loongarch64 m68k mips mipsel mips64 mips64el msp430 nimvm powerpc powerpc64 powerpc64el riscv32 riscv64 sparc sparc64 vm wasm32)' + '--gc\:-[memory management algorithm to use (default\: refc)]:x:(refc arc orc markAndSweep boehm go regions none)' + '--os\:-[operating system to compile for]:x:(AIX Amiga Android Any Atari DOS DragonFly FreeBSD FreeRTOS Genode Haiku iOS Irix JS Linux MacOS MacOSX MorphOS NetBSD Netware NimVM NintendoSwitch OS2 OpenBSD PalmOS Standalone QNX SkyOS Solaris VxWorks Windows)' + '--panics\:-[turn panics into process termination (default\: off)]:x:(off on)' + ) + + case "$words[1]" in + compile|c|compileToC|cpp|compileToCpp|compileToOC|objc) + _arguments $codeOpts $runOpts $sharedOpts $nativeOpts \ + '*:filename:_files -g"*.nim"' + ;; + js) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nim"' + ;; + e) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nims"' + ;; + doc|doc2tex|jsondoc) + _arguments $runOpts $sharedOpts '*:filename:_files -g"*.nim"' + ;; + rst2html|rst2tex) + _arguments $runOpts $sharedOpts $docOpts '*:filename:_files -g"*.rst"' + ;; + buildIndex|genDepend|check) + _arguments $sharedOpts '*:filename:_files -g"*.nim"' + ;; + dump) + _arguments $sharedOpts + ;; + *) + _arguments '*:filename:_files -g"*"' + ;; + esac + + return 1 } _nim "$@" From aac54b9c7ffb75c9aab446574171b33bf23a29ed Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 19 Jan 2022 05:02:35 +0800 Subject: [PATCH 0659/1654] fix stricteffects (nimsuggest/sexp) (#19405) * fix stricteffects (nimsuggest/sexp) * Update tstrict_effects3.nim * Update tests/effects/tstrict_effects3.nim --- nimsuggest/sexp.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimsuggest/sexp.nim b/nimsuggest/sexp.nim index cee538b6e..31f4988e1 100644 --- a/nimsuggest/sexp.nim +++ b/nimsuggest/sexp.nim @@ -409,7 +409,7 @@ macro convertSexp*(x: untyped): untyped = ## `%` for every element. result = toSexp(x) -proc `==`* (a,b: SexpNode): bool = +proc `==`* (a, b: SexpNode): bool {.noSideEffect.} = ## Check two nodes for equality if a.isNil: if b.isNil: return true From b3c178c2024f679a83275f63ca9e21a8b5ba73d7 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 19 Jan 2022 15:39:22 +0800 Subject: [PATCH 0660/1654] suppress deprecated warnings (#19408) * suppress deprecated warnings once bump version to 1.7.3 enable deprecated messages * deprecate later --- lib/system.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 3b3e28344..02da048e4 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -213,8 +213,7 @@ proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## echo p[] # b discard -proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect, - deprecated: "'unsafeAddr' is a deprecated alias for 'addr'".} = +proc unsafeAddr*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## Builtin `addr` operator for taking the address of a memory ## location. ## From 23c4bbedcbf12a7ab87baefe589b472837d89dbf Mon Sep 17 00:00:00 2001 From: nblaxall <43897322+nblaxall@users.noreply.github.com> Date: Thu, 20 Jan 2022 00:25:21 +1300 Subject: [PATCH 0661/1654] add an example to setControlCHook (#19416) * add an example to setControlCHook * [skip CI] format example for setControlCHook Co-authored-by: Nathan Blaxall --- lib/system.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 02da048e4..09de9fcde 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2319,6 +2319,15 @@ when notJSnotNims: proc setControlCHook*(hook: proc () {.noconv.}) ## Allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. + ## Example: + ## + ## .. code-block:: Nim + ## proc ctrlc() {.noconv.} = + ## echo "Ctrl+C fired!" + ## # do clean up stuff + ## quit() + ## + ## setControlCHook(ctrlc) when not defined(noSignalHandler) and not defined(useNimRtl): proc unsetControlCHook*() From 5d34e81f2363e1de97916ac60b028b9a8b287a4b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 19 Jan 2022 19:37:30 +0800 Subject: [PATCH 0662/1654] fix term rewriting with sideeffect (#19410) * fix term rewriting with sideeffect fix #6217 * add tests * Update tests/template/template_various.nim --- compiler/parampatterns.nim | 7 +++++-- tests/template/t6217.nim | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/template/t6217.nim diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index eb99004ab..bfbc72675 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -143,8 +143,11 @@ proc checkForSideEffects*(n: PNode): TSideEffectAnalysis = let s = op.sym if sfSideEffect in s.flags: return seSideEffect - # assume no side effect: - result = seNoSideEffect + elif tfNoSideEffect in op.typ.flags: + result = seNoSideEffect + else: + # assume side effect: + result = seSideEffect elif tfNoSideEffect in op.typ.flags: # indirect call without side effects: result = seNoSideEffect diff --git a/tests/template/t6217.nim b/tests/template/t6217.nim new file mode 100644 index 000000000..b27b61881 --- /dev/null +++ b/tests/template/t6217.nim @@ -0,0 +1,19 @@ +discard """ + output: ''' +start +side effect! +end +''' +""" + +# bug #6217 + +template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a + +proc f(): int = + echo "side effect!" + result = 55 + +echo "start" +doAssert f() * 2 == 110 +echo "end" From 08261cb9e33445553144219023900b4ced0b0f55 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:38:14 +0300 Subject: [PATCH 0663/1654] Don't reject types directly on AST (#19407) Instead of rejecting type expressions based on node kind, evaluate the expression as a type. This is already the behavior for call results, and it has its own error for non-types, which is the same error you would normally get with 2 words swapped. --- compiler/semtypes.nim | 6 ++++-- tests/types/tnontype.nim | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/types/tnontype.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index d03fa88a8..b4f385fe6 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1993,8 +1993,10 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of nkStmtListType: result = semStmtListType(c, n, prev) of nkBlockType: result = semBlockType(c, n, prev) else: - localError(c.config, n.info, "type expected, but got: " & renderTree(n)) - result = newOrPrevType(tyError, prev, c) + result = semTypeExpr(c, n, prev) + when false: + localError(c.config, n.info, "type expected, but got: " & renderTree(n)) + result = newOrPrevType(tyError, prev, c) n.typ = result dec c.inTypeContext diff --git a/tests/types/tnontype.nim b/tests/types/tnontype.nim new file mode 100644 index 000000000..4e2bafb32 --- /dev/null +++ b/tests/types/tnontype.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "expected type, but got: 3" +""" + +type + Foo = (block: + int) + + Bar = 3 From ce44cf03cc4a78741c423b2b3963b48b6d9e6755 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 19 Jan 2022 09:41:11 -0700 Subject: [PATCH 0664/1654] Add noQuit option (#19419) [backport:1.6] * Add noQuit option * Add nim prefix in case of conflicts Co-authored-by: flywind Co-authored-by: flywind --- lib/system.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 09de9fcde..63cdc3ca8 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1198,8 +1198,8 @@ proc align(address, alignment: int): int = else: result = (address + (alignment - 1)) and not (alignment - 1) -when defined(nimdoc): - proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} +when defined(nimNoQuit): + proc quit*(errorcode: int = QuitSuccess) = discard "ignoring quit" ## Stops the program immediately with an exit code. ## ## Before stopping the program the "exit procedures" are called in the @@ -1223,6 +1223,9 @@ when defined(nimdoc): ## It does *not* call the garbage collector to free all the memory, ## unless an `addExitProc` proc calls `GC_fullCollect <#GC_fullCollect>`_. +elif defined(nimdoc): + proc quit*(errorcode: int = QuitSuccess) {.magic: "Exit", noreturn.} + elif defined(genode): include genode/env From 4a38092ac1f8368cb1ebb0245c99c701963662f8 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 20 Jan 2022 20:55:19 +0800 Subject: [PATCH 0665/1654] Added `std/envvars` for env vars handling (#19421) --- changelog.md | 4 +- lib/std/envvars.nim | 212 ++++++++++++++++++++++++++++++++++++++ tests/stdlib/tenvvars.nim | 59 +++++++++++ 3 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 lib/std/envvars.nim create mode 100644 tests/stdlib/tenvvars.nim diff --git a/changelog.md b/changelog.md index 82f9daa1e..bbbaf6e97 100644 --- a/changelog.md +++ b/changelog.md @@ -36,6 +36,8 @@ becomes an alias for `addr`. - Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. +- Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, @@ -74,8 +76,6 @@ becomes an alias for `addr`. for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". -- Added `std/oserrors` for OS error reporting. - ## Compiler changes - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim new file mode 100644 index 000000000..5b135cbd3 --- /dev/null +++ b/lib/std/envvars.nim @@ -0,0 +1,212 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + + +## The `std/envvars` module implements environment variables handling. +import std/oserrors + +type + ReadEnvEffect* = object of ReadIOEffect ## Effect that denotes a read + ## from an environment variable. + WriteEnvEffect* = object of WriteIOEffect ## Effect that denotes a write + ## to an environment variable. + + +when not defined(nimscript): + when defined(nodejs): + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + var ret = default.cstring + let key2 = key.cstring + {.emit: "const value = process.env[`key2`];".} + {.emit: "if (value !== undefined) { `ret` = value };".} + result = $ret + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + var key2 = key.cstring + var ret: bool + {.emit: "`ret` = `key2` in process.env;".} + result = ret + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + var val2 = val.cstring + {.emit: "process.env[`key2`] = `val2`;".} + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + var key2 = key.cstring + {.emit: "delete process.env[`key2`];".} + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + var num: int + var keys: RootObj + {.emit: "`keys` = Object.keys(process.env); `num` = `keys`.length;".} + for i in 0..".} + when defined(windows): + proc c_putenv(envstring: cstring): cint {.importc: "_putenv", header: "".} + from std/private/win_setenv import setEnvImpl + import winlean + else: + proc c_setenv(envname: cstring, envval: cstring, overwrite: cint): cint {.importc: "setenv", header: "".} + proc c_unsetenv(env: cstring): cint {.importc: "unsetenv", header: "".} + + proc getEnv*(key: string, default = ""): string {.tags: [ReadEnvEffect].} = + ## Returns the value of the `environment variable`:idx: named `key`. + ## + ## If the variable does not exist, `""` is returned. To distinguish + ## whether a variable exists or it's value is just `""`, call + ## `existsEnv(key) proc`_. + ## + ## See also: + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert getEnv("unknownEnv") == "" + assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" + + let env = c_getenv(key) + if env == nil: return default + result = $env + + proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = + ## Checks whether the environment variable named `key` exists. + ## Returns true if it exists, false otherwise. + ## + ## See also: + ## * `getEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + runnableExamples: + assert not existsEnv("unknownEnv") + + return c_getenv(key) != nil + + proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = + ## Sets the value of the `environment variable`:idx: named `key` to `val`. + ## If an error occurs, `OSError` is raised. + ## + ## See also: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `delEnv proc`_ + ## * `envPairs iterator`_ + when defined(windows): + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & $(key, val)) + if setEnvImpl(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + else: + if c_setenv(key, val, 1'i32) != 0'i32: + raiseOSError(osLastError(), $(key, val)) + + proc delEnv*(key: string) {.tags: [WriteEnvEffect].} = + ## Deletes the `environment variable`:idx: named `key`. + ## If an error occurs, `OSError` is raised. + ## + ## See also:ven + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `envPairs iterator`_ + template bail = raiseOSError(osLastError(), key) + when defined(windows): + #[ + # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 + > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string + note that nil is not legal + ]# + if key.len == 0 or '=' in key: + raise newException(OSError, "invalid key, got: " & key) + let envToDel = key & "=" + if c_putenv(cstring envToDel) != 0'i32: bail + else: + if c_unsetenv(key) != 0'i32: bail + + when defined(windows): + when defined(cpp): + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", + header: "".} + else: + proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.importc: "wcschr", + header: "".} + elif defined(macosx) and not defined(ios) and not defined(emscripten): + # From the manual: + # Shared libraries and bundles don't have direct access to environ, + # which is only available to the loader ld(1) when a complete program + # is being linked. + # The environment routines can still be used, but if direct access to + # environ is needed, the _NSGetEnviron() routine, defined in + # , can be used to retrieve the address of environ + # at runtime. + proc NSGetEnviron(): ptr cstringArray {.importc: "_NSGetEnviron", + header: "".} + elif defined(haiku): + var gEnv {.importc: "environ", header: "".}: cstringArray + else: + var gEnv {.importc: "environ".}: cstringArray + + iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + when defined(windows): + block: + template impl(get_fun, typ, size, zero, free_fun) = + let env = get_fun() + var e = env + if e == nil: break + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[typ](cast[ByteAddress](eend)+size) + if typeof(zero)(eend[1]) == zero: break + discard free_fun(env) + impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + else: + var i = 0 + when defined(macosx) and not defined(ios) and not defined(emscripten): + var gEnv = NSGetEnviron()[] + while gEnv[i] != nil: + let kv = $gEnv[i] + inc(i) + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + +proc envPairsImplSeq(): seq[tuple[key, value: string]] = discard # vmops + +iterator envPairs*(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = + ## Iterate over all `environments variables`:idx:. + ## + ## In the first component of the tuple is the name of the current variable stored, + ## in the second its value. + ## + ## Works in native backends, nodejs and vm, like the following APIs: + ## * `getEnv proc`_ + ## * `existsEnv proc`_ + ## * `putEnv proc`_ + ## * `delEnv proc`_ + when nimvm: + for ai in envPairsImplSeq(): yield ai + else: + when defined(nimscript): discard + else: + for ai in envPairsImpl(): yield ai diff --git a/tests/stdlib/tenvvars.nim b/tests/stdlib/tenvvars.nim new file mode 100644 index 000000000..406aa3230 --- /dev/null +++ b/tests/stdlib/tenvvars.nim @@ -0,0 +1,59 @@ +discard """ + matrix: "--threads:on" + joinable: false + targets: "c js cpp" +""" + +import std/envvars +from std/sequtils import toSeq +import stdtest/testutils + +template main = + block: # delEnv, existsEnv, getEnv, envPairs + for val in ["val", ""]: # ensures empty val works too + const key = "NIM_TESTS_TOSENV_KEY" + doAssert not existsEnv(key) + + putEnv(key, "tempval") + doAssert existsEnv(key) + doAssert getEnv(key) == "tempval" + + putEnv(key, val) # change a key that already exists + doAssert existsEnv(key) + doAssert getEnv(key) == val + + doAssert (key, val) in toSeq(envPairs()) + delEnv(key) + doAssert (key, val) notin toSeq(envPairs()) + doAssert not existsEnv(key) + delEnv(key) # deleting an already deleted env var + doAssert not existsEnv(key) + + block: + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "") == "" + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", " ") == " " + doAssert getEnv("NIM_TESTS_TOSENV_NONEXISTENT", "defval") == "defval" + + whenVMorJs: discard # xxx improve + do: + doAssertRaises(OSError, putEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE", "NEW_DUMMY_VALUE")) + doAssertRaises(OSError, putEnv("", "NEW_DUMMY_VALUE")) + doAssert not existsEnv("") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT=DUMMY_VALUE") + doAssert not existsEnv("NIM_TESTS_TOSENV_PUT") + +main() + +when not defined(js) and not defined(nimscript): + block: # bug #18533 + proc c_getenv(env: cstring): cstring {.importc: "getenv", header: "".} + var thr: Thread[void] + proc threadFunc {.thread.} = putEnv("foo", "fooVal2") + + putEnv("foo", "fooVal1") + doAssert getEnv("foo") == "fooVal1" + createThread(thr, threadFunc) + joinThreads(thr) + doAssert getEnv("foo") == $c_getenv("foo") + + doAssertRaises(OSError): delEnv("foo=bar") From 851e515bbae6edf6b28cf6723b2c7a45acce1869 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 20 Jan 2022 04:58:59 -0800 Subject: [PATCH 0666/1654] Resolve cross file resolution errors in atomics (#19422) [backport:1.6] * Resolve call undeclared routine testAndSet * Fix undeclared field atomicType --- lib/pure/concurrency/atomics.nim | 17 +++++++---------- tests/stdlib/concurrency/atomicSample.nim | 9 +++++++++ tests/stdlib/concurrency/tatomic_import.nim | 11 +++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 tests/stdlib/concurrency/atomicSample.nim create mode 100644 tests/stdlib/concurrency/tatomic_import.nim diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index b252789fb..c0a122bf9 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -293,19 +293,16 @@ else: AtomicInt32 {.importc: "_Atomic NI32".} = int32 AtomicInt64 {.importc: "_Atomic NI64".} = int64 - template atomicType*(T: typedesc[Trivial]): untyped = - # Maps the size of a trivial type to it's internal atomic type - when sizeof(T) == 1: AtomicInt8 - elif sizeof(T) == 2: AtomicInt16 - elif sizeof(T) == 4: AtomicInt32 - elif sizeof(T) == 8: AtomicInt64 - type AtomicFlag* {.importc: "atomic_flag", size: 1.} = object Atomic*[T] = object when T is Trivial: - value: T.atomicType + # Maps the size of a trivial type to it's internal atomic type + when sizeof(T) == 1: value: AtomicInt8 + elif sizeof(T) == 2: value: AtomicInt16 + elif sizeof(T) == 4: value: AtomicInt32 + elif sizeof(T) == 8: value: AtomicInt64 else: nonAtomicValue: T guard: AtomicFlag @@ -364,11 +361,11 @@ else: cast[T](atomic_fetch_xor_explicit(addr(location.value), cast[nonAtomicType(T)](value), order)) template withLock[T: not Trivial](location: var Atomic[T]; order: MemoryOrder; body: untyped): untyped = - while location.guard.testAndSet(moAcquire): discard + while testAndSet(location.guard, moAcquire): discard try: body finally: - location.guard.clear(moRelease) + clear(location.guard, moRelease) proc load*[T: not Trivial](location: var Atomic[T]; order: MemoryOrder = moSequentiallyConsistent): T {.inline.} = withLock(location, order): diff --git a/tests/stdlib/concurrency/atomicSample.nim b/tests/stdlib/concurrency/atomicSample.nim new file mode 100644 index 000000000..d56d867df --- /dev/null +++ b/tests/stdlib/concurrency/atomicSample.nim @@ -0,0 +1,9 @@ +import atomics + +type + AtomicWithGeneric*[T] = object + value: Atomic[T] + +proc initAtomicWithGeneric*[T](value: T): AtomicWithGeneric[T] = + result.value.store(value) + diff --git a/tests/stdlib/concurrency/tatomic_import.nim b/tests/stdlib/concurrency/tatomic_import.nim new file mode 100644 index 000000000..e8faaae20 --- /dev/null +++ b/tests/stdlib/concurrency/tatomic_import.nim @@ -0,0 +1,11 @@ +import atomicSample + +block crossFileObjectContainingAGenericWithAComplexObject: + discard initAtomicWithGeneric[string]("foo") + +block crossFileObjectContainingAGenericWithAnInteger: + discard initAtomicWithGeneric[int](1) + discard initAtomicWithGeneric[int8](1) + discard initAtomicWithGeneric[int16](1) + discard initAtomicWithGeneric[int32](1) + discard initAtomicWithGeneric[int64](1) From 927fa890ec91cf13d7e3aa87267b994c2812bdc8 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 20 Jan 2022 20:59:36 +0800 Subject: [PATCH 0667/1654] enable weave (#19363) [backport:1.6] * enable weave * workaround CI --- compiler/cgen.nim | 9 +++++++-- testament/important_packages.nim | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 032c22ec0..246b8c0e9 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -48,8 +48,13 @@ proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) proc findPendingModule(m: BModule, s: PSym): BModule = - let ms = s.itemId.module #getModule(s) - result = m.g.modules[ms] + # TODO fixme + if m.config.symbolFiles == v2Sf: + let ms = s.itemId.module #getModule(s) + result = m.g.modules[ms] + else: + var ms = getModule(s) + result = m.g.modules[ms.position] proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) = result.k = k diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 853a58698..5f6e19b52 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -157,7 +157,7 @@ pkg "tiny_sqlite" pkg "unicodedb", "nim c -d:release -r tests/tests.nim" pkg "unicodeplus", "nim c -d:release -r tests/tests.nim" pkg "unpack" -pkg "weave", "nimble test_gc_arc", allowFailure = true +pkg "weave", "nimble test_gc_arc" pkg "websocket", "nim c websocket.nim" pkg "winim", "nim c winim.nim" pkg "with" From 1563cb2f6e37f07c303d095dabde74955be1e523 Mon Sep 17 00:00:00 2001 From: Regis Caillaud <35006197+Clonkk@users.noreply.github.com> Date: Thu, 20 Jan 2022 20:50:36 +0100 Subject: [PATCH 0668/1654] Fix #11923 (#19427) * Apply commit https://github.com/nim-lang/Nim/commit/5da931fe811717a45f2dd272ea6281979c3e8f0b that was never merged (was part of a bigger PR). Should fix issue #11932 * add a generic object for custom pragma --- lib/core/macros.nim | 7 ++++++- tests/pragmas/tcustom_pragma.nim | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 8ab893d21..28106493f 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1523,7 +1523,12 @@ proc customPragmaNode(n: NimNode): NimNode = if n.kind in {nnkDotExpr, nnkCheckedFieldExpr}: let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) - var typDef = getImpl(if typInst.kind == nnkVarTy: typInst[0] else: typInst) + var typDef = getImpl( + if typInst.kind == nnkVarTy or + typInst.kind == nnkBracketExpr: + typInst[0] + else: typInst + ) while typDef != nil: typDef.expectKind(nnkTypeDef) let typ = typDef[2] diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index b197a7c55..1c3709b26 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -17,11 +17,21 @@ block: MyObj = object myField1, myField2 {.myAttr: "hi".}: int + MyGenericObj[T] = object + myField1, myField2 {.myAttr: "hi".}: int + + var o: MyObj static: doAssert o.myField2.hasCustomPragma(myAttr) doAssert(not o.myField1.hasCustomPragma(myAttr)) + var ogen: MyGenericObj[int] + static: + doAssert ogen.myField2.hasCustomPragma(myAttr) + doAssert(not ogen.myField1.hasCustomPragma(myAttr)) + + import custom_pragma block: # A bit more advanced case type From 2bd1aa186e09565b2103394bd281478fa1b10ef1 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:57:50 +0300 Subject: [PATCH 0669/1654] New/better macro pragmas, mark some as experimental (#19406) * New/better macro pragmas, make some experimental fix #15920, close #18212, close #14781, close #6696, close https://github.com/nim-lang/RFCs/issues/220 Variable macro pragmas have been changed to only take a unary section node. They can now also be applied in sections with multiple variables, as well as `const` sections. They also accept arguments. Templates now support macro pragmas, mirroring other routine types. Type and variable macro pragmas have been made experimental. Symbols without parentheses instatiating nullary macros or templates has also been documented in the experimental manual. A check for a redefinition error based on the left hand side of variable definitions when using variable macro pragmas was disabled. This nerfs `byaddr` specifically, however this has been documented as a consequence of the experimental features `byaddr` uses. Given how simple these changes are I'm worried if I'm missing something. * accomodate compiler boot * allow weird pragmas * add test for #10994 * remove some control flow, try remove some logic --- compiler/semstmts.nim | 280 +++++++++++++++++++------------- compiler/semtempl.nim | 5 + doc/manual.rst | 28 ++-- doc/manual_experimental.rst | 72 +++++++- lib/std/decls.nim | 39 +++-- tests/pragmas/tpragmas_misc.nim | 24 ++- tests/pragmas/tvar_macro.nim | 128 +++++++++++++++ tests/stdlib/tdecls.nim | 64 ++------ 8 files changed, 435 insertions(+), 205 deletions(-) create mode 100644 tests/pragmas/tvar_macro.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index f302dd4c3..33e304ab2 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -341,8 +341,12 @@ proc checkNilable(c: PContext; v: PSym) = #include liftdestructors -proc addToVarSection(c: PContext; result: PNode; orig, identDefs: PNode) = - let value = identDefs[^1] +proc addToVarSection(c: PContext; result: var PNode; n: PNode) = + if result.kind != nkStmtList: + result = makeStmtList(result) + result.add n + +proc addToVarSection(c: PContext; result: var PNode; orig, identDefs: PNode) = if result.kind == nkStmtList: let o = copyNode(orig) o.add identDefs @@ -445,55 +449,115 @@ proc setVarType(c: PContext; v: PSym, typ: PType) = "; new type is: " & typeToString(typ, preferDesc)) v.typ = typ -proc semLowerLetVarCustomPragma(c: PContext, a: PNode, n: PNode): PNode = - var b = a[0] - if b.kind == nkPragmaExpr: - if b[1].len != 1: - # we could in future support pragmas w args e.g.: `var foo {.bar:"goo".} = expr` - return nil - let nodePragma = b[1][0] - # see: `singlePragma` +proc isPossibleMacroPragma(c: PContext, it: PNode, key: PNode): bool = + # make sure it's not a normal pragma, and calls an identifier + # considerQuotedIdent below will fail on non-identifiers + result = whichPragma(it) == wInvalid and key.kind in nkIdentKinds + if result: + # make sure it's not a user pragma + let ident = considerQuotedIdent(c, key) + result = strTableGet(c.userPragmas, ident) == nil + if result: + # make sure it's not a custom pragma + var amb = false + let sym = searchInScopes(c, ident, amb) + result = sym == nil or sfCustomPragma notin sym.flags - var amb = false - var sym: PSym = nil - case nodePragma.kind - of nkIdent, nkAccQuoted: - let ident = considerQuotedIdent(c, nodePragma) - var userPragma = strTableGet(c.userPragmas, ident) - if userPragma != nil: return nil - let w = nodePragma.whichPragma - if n.kind == nkVarSection and w in varPragmas or - n.kind == nkLetSection and w in letPragmas or - n.kind == nkConstSection and w in constPragmas: - return nil - sym = searchInScopes(c, ident, amb) - # XXX what if amb is true? - # CHECKME: should that test also apply to `nkSym` case? - if sym == nil or sfCustomPragma in sym.flags: return nil - of nkSym: - sym = nodePragma.sym - else: - return nil - # skip if not in scope; skip `template myAttr() {.pragma.}` +proc copyExcept(n: PNode, i: int): PNode = + result = copyNode(n) + for j in 0..= 1: it[0] else: it + + when false: + let lhs = b[0] + let clash = strTableGet(c.currentScope.symbols, lhs.ident) + if clash != nil: + # refs https://github.com/nim-lang/Nim/issues/8275 + wrongRedefinition(c, lhs.info, lhs.ident.s, clash.info) - result = newTree(nkCall) - result.add nodePragma - result.add lhs - if a[1].kind != nkEmpty: - result.add a[1] - else: - result.add newNodeIT(nkNilLit, a.info, c.graph.sysTypes[tyNil]) - result.add a[2] - result.info = a.info - let ret = newNodeI(nkStmtList, a.info) - ret.add result - result = semExprNoType(c, ret) + if isPossibleMacroPragma(c, it, key): + # we transform ``var p {.m, rest.}`` into ``m(do: var p {.rest.})`` and + # let the semantic checker deal with it: + var x = newNodeI(nkCall, key.info) + x.add(key) + + if it.kind in nkPragmaCallKinds and it.len > 1: + # pass pragma arguments to the macro too: + for i in 1..= 1: it[0] else: it - if whichPragma(it) != wInvalid: - # Not a custom pragma - continue - else: - let ident = considerQuotedIdent(c, key) - if strTableGet(c.userPragmas, ident) != nil: - continue # User defined pragma + if isPossibleMacroPragma(c, it, key): + # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and + # let the semantic checker deal with it: + var x = newNodeI(nkCall, key.info) + x.add(key) + + if it.kind in nkPragmaCallKinds and it.len > 1: + # pass pragma arguments to the macro too: + for i in 1.. 1: - # pass pragma arguments to the macro too: - for i in 1..` instead. Foreign function interface diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.rst index 407924a13..3089755cb 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.rst @@ -404,6 +404,76 @@ to use this operator. doAssert (a.b)(c) == `()`(a.b, c) +Extended macro pragmas +====================== + +Macro pragmas as described in `the manual `_ +can also be applied to type, variable and constant declarations. + +For types: + +.. code-block:: nim + type + MyObject {.schema: "schema.protobuf".} = object + +This is translated to a call to the `schema` macro with a `nnkTypeDef` +AST node capturing the left-hand side, remaining pragmas and the right-hand +side of the definition. The macro can return either a type section or +another `nnkTypeDef` node, both of which will replace the original row +in the type section. + +In the future, this `nnkTypeDef` argument may be replaced with a unary +type section node containing the type definition, or some other node that may +be more convenient to work with. The ability to return nodes other than type +definitions may also be supported, however currently this is not convenient +when dealing with mutual type recursion. For now, macros can return an unused +type definition where the right-hand node is of kind `nnkStmtListType`. +Declarations in this node will be attached to the same scope as +the parent scope of the type section. + +------ + +For variables and constants, it is largely the same, except a unary node with +the same kind as the section containing a single definition is passed to macros, +and macros can return any expression. + +.. code-block:: nim + var + a = ... + b {.importc, foo, nodecl.} = ... + c = ... + +Assuming `foo` is a macro or a template, this is roughly equivalent to: + +.. code-block:: nim + var a = ... + foo: + var b {.importc, nodecl.} = ... + var c = ... + + +Symbols as template/macro calls +=============================== + +Templates and macros that take no arguments can be called as lone symbols, +i.e. without parentheses. This is useful for repeated uses of complex +expressions that cannot conveniently be represented as runtime values. + +.. code-block:: nim + type Foo = object + bar: int + + var foo = Foo(bar: 10) + template bar: untyped = foo.bar + assert bar == 10 + bar = 15 + assert bar == 15 + +In the future, this may require more specific information on template or macro +signatures to be used. Specializations for some applications of this may also +be introduced to guarantee consistency and circumvent bugs. + + Not nil annotation ================== @@ -613,7 +683,7 @@ has `source` as the owner. A path expression `e` is defined recursively: If a view type is used as a return type, the location must borrow from a location that is derived from the first parameter that is passed to the proc. -See `the manual `_ +See `the manual `_ for details about how this is done for `var T`. A mutable view can borrow from a mutable location, an immutable view can borrow diff --git a/lib/std/decls.nim b/lib/std/decls.nim index dd7d19da7..7b907f5e1 100644 --- a/lib/std/decls.nim +++ b/lib/std/decls.nim @@ -1,19 +1,32 @@ # see `semLowerLetVarCustomPragma` for compiler support that enables these # lowerings -template byaddr*(lhs, typ, ex) = - ## Allows a syntax for lvalue reference, exact analog to - ## `auto& a = ex;` in C++ +import macros + +macro byaddr*(sect) = + ## Allows a syntax for l-value references, being an exact analog to + ## `auto& a = ex;` in C++. + ## + ## Warning: This makes use of 2 experimental features, namely nullary + ## templates instantiated as symbols and variable macro pragmas. + ## For this reason, its behavior is not stable. The current implementation + ## allows redefinition, but this is not an intended consequence. runnableExamples: - var s = @[10,11,12] + var s = @[10, 11, 12] var a {.byaddr.} = s[0] - a+=100 - doAssert s == @[110,11,12] - doAssert a is int + a += 100 + assert s == @[110, 11, 12] + assert a is int var b {.byaddr.}: int = s[0] - doAssert a.addr == b.addr - when typ is typeof(nil): - let tmp = addr(ex) - else: - let tmp: ptr typ = addr(ex) - template lhs: untyped = tmp[] + assert a.addr == b.addr + expectLen sect, 1 + let def = sect[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = addr(`ex`) + template `lhs`: untyped = tmp[] + result.copyLineInfo(def) diff --git a/tests/pragmas/tpragmas_misc.nim b/tests/pragmas/tpragmas_misc.nim index 8cab74053..6dc2e6b80 100644 --- a/tests/pragmas/tpragmas_misc.nim +++ b/tests/pragmas/tpragmas_misc.nim @@ -13,8 +13,8 @@ block: block: # (partial fix) bug #15920 block: # var template pragmas don't work in templates - template foo(lhs, typ, expr) = - let lhs = expr + template foo(expr) = + expr proc fun1()= let a {.foo.} = 1 template fun2()= @@ -24,23 +24,22 @@ block: # (partial fix) bug #15920 template foo2() = discard # distractor (template or other symbol kind) block: - template foo2(lhs, typ, expr) = - let lhs = expr + template foo2(expr) = + expr proc fun1()= let a {.foo2.} = 1 template fun2()= let a {.foo2.} = 1 fun1() # ok - when false: # bug: Error: invalid pragma: foo2 - fun2() + fun2() # bug: Error: invalid pragma: foo2 - block: # proc template pragmas don't work in templates + block: # template pragmas don't work for templates, #18212 # adapted from $nim/lib/std/private/since.nim # case without overload template since3(version: (int, int), body: untyped) {.dirty.} = when (NimMajor, NimMinor) >= version: body - when false: # bug + when true: # bug template fun3(): int {.since3: (1, 3).} = 12 block: # ditto, w @@ -51,7 +50,7 @@ block: # (partial fix) bug #15920 template since2(version: (int, int, int), body: untyped) {.dirty.} = when (NimMajor, NimMinor, NimPatch) >= version: body - when false: # bug + when true: # bug template fun3(): int {.since2: (1, 3).} = 12 when true: # D20210801T100514:here @@ -62,3 +61,10 @@ when true: # D20210801T100514:here discard ret fn() static: discard genSym() + +block: # issue #10994 + macro foo(x): untyped = x + template bar {.pragma.} + + proc a {.bar.} = discard # works + proc b {.bar, foo.} = discard # doesn't diff --git a/tests/pragmas/tvar_macro.nim b/tests/pragmas/tvar_macro.nim new file mode 100644 index 000000000..d6a4ff983 --- /dev/null +++ b/tests/pragmas/tvar_macro.nim @@ -0,0 +1,128 @@ +import macros + +block: # test usage + macro modify(sec) = + result = copy sec + result[0][0] = ident(repr(result[0][0]) & "Modified") + + block: + let foo {.modify.} = 3 + doAssert fooModified == 3 + + block: # in section + let + a = 1 + b {.modify.} = 2 + c = 3 + doAssert (a, bModified, c) == (1, 2, 3) + +block: # with single argument + macro appendToName(name: static string, sec) = + result = sec + result[0][0] = ident(repr(result[0][0]) & name) + + block: + let foo {.appendToName: "Bar".} = 3 + doAssert fooBar == 3 + + block: + let + a = 1 + b {.appendToName("").} = 2 + c = 3 + doAssert (a, b, c) == (1, 2, 3) + +macro appendToNameAndAdd(name: static string, incr: static int, sec) = + result = sec + result[0][0] = ident(repr(result[0][0]) & name) + result[0][2] = infix(result[0][2], "+", newLit(incr)) + +block: # with multiple arguments + block: + let foo {.appendToNameAndAdd("Bar", 5).} = 3 + doAssert fooBar == 8 + + block: + let + a = 1 + b {.appendToNameAndAdd("", 15).} = 2 + c = 3 + doAssert (a, b, c) == (1, 17, 3) + +block: # in other kinds of sections + block: + const + a = 1 + b {.appendToNameAndAdd("", 15).} = 2 + c = 3 + doAssert (a, b, c) == (1, 17, 3) + doAssert static(b) == b + + block: + var + a = 1 + b {.appendToNameAndAdd("", 15).} = 2 + c = 3 + doAssert (a, b, c) == (1, 17, 3) + b += a + c += b + doAssert (a, b, c) == (1, 18, 21) + +block: # with other pragmas + macro appendToNameAndAdd(name: static string, incr, sec) = + result = sec + result[0][0][0] = ident(repr(result[0][0][0]) & name) + result[0][0][1].add(ident"deprecated") + result[0][2] = infix(result[0][2], "+", incr) + + var + a = 1 + foo {.exportc: "exportedFooBar", appendToNameAndAdd("Bar", {'0'..'9'}), used.} = {'a'..'z', 'A'..'Z'} + b = 2 + + doAssert (a, b) == (1, 2) + + let importedFooBar {.importc: "exportedFooBar", nodecl.}: set[char] + + doAssert importedFooBar == fooBar #[tt.Warning + ^ fooBar is deprecated + ]# + + +block: # with stropping + macro `cast`(def) = + let def = def[0] + let + lhs = def[0] + typ = def[1] + ex = def[2] + addrTyp = if typ.kind == nnkEmpty: typ else: newTree(nnkPtrTy, typ) + result = quote do: + let tmp: `addrTyp` = unsafeAddr(`ex`) + template `lhs`: untyped = tmp[] + + macro assign(def) = + result = getAst(`cast`(def)) + + block: + let s = @["foo", "bar"] + let a {.`assign`.} = s[0] + doAssert a == "foo" + doAssert a[0].addr == s[0][0].addr + + block: + let + s = @["foo", "bar"] + a {.`cast`.} = s[0] + doAssert a == "foo" + doAssert a[0].addr == s[0][0].addr + +block: # bug #15920 + macro foo(def) = + result = def + proc fun1()= + let a {.foo.} = 1 + template fun2()= + let a {.foo.} = 1 + fun1() # ok + fun2() # BUG diff --git a/tests/stdlib/tdecls.nim b/tests/stdlib/tdecls.nim index c0d6f8a08..4e7407045 100644 --- a/tests/stdlib/tdecls.nim +++ b/tests/stdlib/tdecls.nim @@ -13,15 +13,18 @@ template fun() = var b {.byaddr.}: int = s[0] doAssert a.addr == b.addr - doAssert not compiles(block: - # redeclaration not allowed - var foo = 0 - var foo {.byaddr.} = s[0]) + when false: + # template specific redeclaration issue + # see https://github.com/nim-lang/Nim/issues/8275 + doAssert not compiles(block: + # redeclaration not allowed + var foo = 0 + var foo {.byaddr.} = s[0]) - doAssert not compiles(block: - # ditto - var foo {.byaddr.} = s[0] - var foo {.byaddr.} = s[0]) + doAssert not compiles(block: + # ditto + var foo {.byaddr.} = s[0] + var foo {.byaddr.} = s[0]) block: var b {.byaddr.} = s[1] # redeclaration ok in sub scope @@ -44,48 +47,3 @@ fun2() static: fun2() when false: # pending bug #13887 static: fun() - -## We can define custom pragmas in user code -template byUnsafeAddr(lhs, typ, expr) = - when typ is type(nil): - let tmp = addr(expr) - else: - let tmp: ptr typ = addr(expr) - template lhs: untyped = tmp[] - -block: - let s = @["foo", "bar"] - let a {.byUnsafeAddr.} = s[0] - doAssert a == "foo" - doAssert a[0].addr == s[0][0].addr - -block: # nkAccQuoted - # shows using a keyword, which requires nkAccQuoted - template `cast`(lhs, typ, expr) = - when typ is type(nil): - let tmp = addr(expr) - else: - let tmp: ptr typ = addr(expr) - template lhs: untyped = tmp[] - - block: - let s = @["foo", "bar"] - let a {.`byUnsafeAddr`.} = s[0] - doAssert a == "foo" - doAssert a[0].addr == s[0][0].addr - - block: - let s = @["foo", "bar"] - let a {.`cast`.} = s[0] - doAssert a == "foo" - doAssert a[0].addr == s[0][0].addr - -block: # bug #15920 - template foo(lhs, typ, expr) = - let lhs = expr - proc fun1()= - let a {.foo.} = 1 - template fun2()= - let a {.foo.} = 1 - fun1() # ok - fun2() # BUG From 15f54de5c4ee2f70e1304201ebc8c8634b731c9a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 22 Jan 2022 20:33:55 +0100 Subject: [PATCH 0670/1654] RST: allow empty number-lines directives just like it was done for a decade; all my documents rely on this feature [backport (#19431) --- lib/packages/docutils/rstgen.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 9018087f7..d2180cb91 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -931,7 +931,8 @@ proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int = let nChars = parseInt(value, number) if nChars == 0: if value.len == 0: - err("field $1 requires an argument" % [fieldName]) + # use a good default value: + result = 1 else: err("field $1 requires an integer, but '$2' was given" % [fieldName, value]) From f7c4fb0014f320824d8f05e2e4d9776c09f51711 Mon Sep 17 00:00:00 2001 From: ehmry Date: Mon, 24 Jan 2022 20:57:37 +0100 Subject: [PATCH 0671/1654] os: faster getFileSize (#19438) Use "stat" rather than "open", "seek", and "close" system calls. The Windows implementation remains the same. --- lib/pure/os.nim | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 81b432d7f..07040d611 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -3216,11 +3216,10 @@ proc getFileSize*(file: string): BiggestInt {.rtl, extern: "nos$1", result = rdFileSize(a) findClose(resA) else: - var f: File - if open(f, file): - result = getFileSize(f) - close(f) - else: raiseOSError(osLastError(), file) + var rawInfo: Stat + if stat(file, rawInfo) < 0'i32: + raiseOSError(osLastError(), file) + rawInfo.st_size when defined(windows) or weirdTarget: type From 6ea622552345fc9a6c451615079d2a0ebbc6c4ca Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 25 Jan 2022 08:08:22 +0100 Subject: [PATCH 0672/1654] bugfix: varargs count as open arrays (#19447) --- compiler/typeallowed.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 78cd74e56..3d6ea0edb 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -233,7 +233,7 @@ proc classifyViewTypeAux(marker: var IntSet, t: PType): ViewTypeKind = case t.kind of tyVar: result = mutableView - of tyLent, tyOpenArray: + of tyLent, tyOpenArray, tyVarargs: result = immutableView of tyGenericInst, tyDistinct, tyAlias, tyInferred, tySink, tyOwned, tyUncheckedArray, tySequence, tyArray, tyRef, tyStatic: From 4b723c0f5350c5bfb9722dace7a40c66a4630a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Granstr=C3=B6m?= <5092565+HugoGranstrom@users.noreply.github.com> Date: Tue, 25 Jan 2022 10:31:24 +0100 Subject: [PATCH 0673/1654] change run command for numericalnim (#19448) Now it makes runs the custom `nimCI` task that installs the external dependencies --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 5f6e19b52..40a3aef36 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -117,7 +117,7 @@ pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" pkg "norm", "testament r tests/sqlite/trows.nim" pkg "npeg", "nimble testarc" -pkg "numericalnim", "nim c -r tests/test_integrate.nim" +pkg "numericalnim", "nimble nimCI" pkg "optionsutils" pkg "ormin", "nim c -o:orminn ormin.nim" pkg "parsetoml" From 6319b00ef7f93fa8d7d44c981f850262d8e0ed6a Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 27 Jan 2022 22:23:34 +0800 Subject: [PATCH 0674/1654] support set other GCs after `mm = orc` or `mm = arc` is set in the global config (#19455) * support set other GCs after arc/orc in global config fix #15535 * set before --- compiler/commands.nim | 54 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 8e79a4477..a1fa55e74 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -497,6 +497,32 @@ proc specialDefine(conf: ConfigRef, key: string; pass: TCmdLinePass) = optOverflowCheck, optAssert, optStackTrace, optLineTrace, optLineDir} conf.globalOptions.excl {optCDebug} +proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) = + if isOrc: + conf.selectedGC = gcOrc + defineSymbol(conf.symbols, "gcorc") + else: + conf.selectedGC = gcArc + defineSymbol(conf.symbols, "gcarc") + + defineSymbol(conf.symbols, "gcdestructors") + incl conf.globalOptions, optSeqDestructors + incl conf.globalOptions, optTinyRtti + if pass in {passCmd2, passPP}: + defineSymbol(conf.symbols, "nimSeqsV2") + defineSymbol(conf.symbols, "nimV2") + if conf.exc == excNone and conf.backend != backendCpp: + conf.exc = excGoto + +proc unregisterArcOrc(conf: ConfigRef) = + undefSymbol(conf.symbols, "gcdestructors") + undefSymbol(conf.symbols, "gcarc") + undefSymbol(conf.symbols, "gcorc") + undefSymbol(conf.symbols, "nimSeqsV2") + undefSymbol(conf.symbols, "nimV2") + excl conf.globalOptions, optSeqDestructors + excl conf.globalOptions, optTinyRtti + proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = var @@ -602,36 +628,21 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if pass in {passCmd2, passPP}: case arg.normalize of "boehm": + unregisterArcOrc(conf) conf.selectedGC = gcBoehm defineSymbol(conf.symbols, "boehmgc") incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS of "refc": + unregisterArcOrc(conf) conf.selectedGC = gcRefc of "markandsweep": + unregisterArcOrc(conf) conf.selectedGC = gcMarkAndSweep defineSymbol(conf.symbols, "gcmarkandsweep") of "destructors", "arc": - conf.selectedGC = gcArc - defineSymbol(conf.symbols, "gcdestructors") - defineSymbol(conf.symbols, "gcarc") - incl conf.globalOptions, optSeqDestructors - incl conf.globalOptions, optTinyRtti - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - defineSymbol(conf.symbols, "nimV2") - if conf.exc == excNone and conf.backend != backendCpp: - conf.exc = excGoto + registerArcOrc(pass, conf, false) of "orc": - conf.selectedGC = gcOrc - defineSymbol(conf.symbols, "gcdestructors") - defineSymbol(conf.symbols, "gcorc") - incl conf.globalOptions, optSeqDestructors - incl conf.globalOptions, optTinyRtti - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - defineSymbol(conf.symbols, "nimV2") - if conf.exc == excNone and conf.backend != backendCpp: - conf.exc = excGoto + registerArcOrc(pass, conf, true) of "hooks": conf.selectedGC = gcHooks defineSymbol(conf.symbols, "gchooks") @@ -640,12 +651,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if pass in {passCmd2, passPP}: defineSymbol(conf.symbols, "nimSeqsV2") of "go": + unregisterArcOrc(conf) conf.selectedGC = gcGo defineSymbol(conf.symbols, "gogc") of "none": + unregisterArcOrc(conf) conf.selectedGC = gcNone defineSymbol(conf.symbols, "nogc") of "stack", "regions": + unregisterArcOrc(conf) conf.selectedGC = gcRegions defineSymbol(conf.symbols, "gcregions") of "v2": warningOptionNoop(arg) From 0c3892c3c707e75520d6c8b5a65a0eaf8c15fcd4 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 28 Jan 2022 16:53:42 +0800 Subject: [PATCH 0675/1654] nvro don't touch cdecl types [backport: 1.6] (#19461) * nvro don't touch cdecl types; fix #19342 again --- compiler/ast.nim | 2 +- compiler/ccgcalls.nim | 6 +++--- compiler/ccgstmts.nim | 21 ++++++++++++++------- compiler/ccgtypes.nim | 18 ++++++++++++------ compiler/cgen.nim | 4 ++-- compiler/semstmts.nim | 1 + tests/objects/t19342_2.nim | 18 ++++++++++++++++++ 7 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 tests/objects/t19342_2.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 4286f940f..ad5ec12f5 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -501,7 +501,7 @@ type nfHasComment # node has a comment TNodeFlags* = set[TNodeFlag] - TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 43) + TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 45) tfVarargs, # procedure has C styled varargs # tyArray type represeting a varargs list tfNoSideEffect, # procedure type does not allow side effects diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index fe0516f9a..197728ccf 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -76,7 +76,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc, # getUniqueType() is too expensive here: var typ = skipTypes(ri[0].typ, abstractInst) if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): + if isInvalidReturnType(p.config, typ): if params != nil: pl.add(~", ") # beware of 'result = p(result)'. We may need to allocate a temporary: if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): @@ -439,7 +439,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = let rawProc = getClosureType(p.module, typ, clHalf) let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0]) if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): + if isInvalidReturnType(p.config, typ): if ri.len > 1: pl.add(~", ") # beware of 'result = p(result)'. We may need to allocate a temporary: if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): @@ -737,7 +737,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = pl.add(~": ") pl.add(genArg(p, ri[i], param, ri)) if typ[0] != nil: - if isInvalidReturnType(p.config, typ[0]): + if isInvalidReturnType(p.config, typ): if ri.len > 1: pl.add(~" ") # beware of 'result = p(result)'. We always allocate a temporary: if d.k in {locTemp, locNone}: diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 7fecc1475..58a6dd015 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -32,13 +32,20 @@ proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) = "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc]) proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} = - if n.kind == nkEmpty: return false - if isInvalidReturnType(conf, n.typ): - # var v = f() - # is transformed into: var v; f(addr v) - # where 'f' **does not** initialize the result! - return false - result = true + if n.kind == nkEmpty: + result = false + elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc: + if isInvalidReturnType(conf, n[0].typ, true): + # var v = f() + # is transformed into: var v; f(addr v) + # where 'f' **does not** initialize the result! + result = false + else: + result = true + elif isInvalidReturnType(conf, n.typ, false): + result = false + else: + result = true proc inExceptBlockLen(p: BProc): int = for x in p.nestedTryStmts: diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index df1874b26..f010b25c9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -215,12 +215,18 @@ proc isObjLackingTypeField(typ: PType): bool {.inline.} = result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and (typ[0] == nil) or isPureObject(typ)) -proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool = +proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool = # Arrays and sets cannot be returned by a C procedure, because C is # such a poor programming language. # We exclude records with refs too. This enhances efficiency and # is necessary for proper code generation of assignments. - if rettype == nil or (tfByCopy notin rettype.flags and getSize(conf, rettype) > conf.target.floatSize*3): + var rettype = typ + var isAllowedCall = true + if isProc: + rettype = rettype[0] + isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall} + if rettype == nil or (isAllowedCall and + getSize(conf, rettype) > conf.target.floatSize*3): result = true else: case mapType(conf, rettype, skResult) @@ -257,11 +263,11 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) = # see `testCodegenABICheck` for example error message it generates -proc fillResult(conf: ConfigRef; param: PNode) = +proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) = fillLoc(param.sym.loc, locParam, param, ~"Result", OnStack) let t = param.sym.typ - if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, t): + if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype): incl(param.sym.loc.flags, lfIndirect) param.sym.loc.storage = OnUnknown @@ -426,7 +432,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, check: var IntSet, declareEnvironment=true; weakDep=false) = params = nil - if t[0] == nil or isInvalidReturnType(m.config, t[0]): + if t[0] == nil or isInvalidReturnType(m.config, t): rettype = ~"void" else: rettype = getTypeDescAux(m, t[0], check, skResult) @@ -461,7 +467,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, params.addf(", NI $1Len_$2", [param.loc.r, j.rope]) inc(j) arr = arr[0].skipTypes({tySink}) - if t[0] != nil and isInvalidReturnType(m.config, t[0]): + if t[0] != nil and isInvalidReturnType(m.config, t): var arr = t[0] if params != nil: params.add(", ") if mapReturnType(m.config, t[0]) != ctArray: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 246b8c0e9..10b4b55d0 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1039,7 +1039,7 @@ proc genProcAux(m: BModule, prc: PSym) = internalError(m.config, prc.info, "proc has no result symbol") let resNode = prc.ast[resultPos] let res = resNode.sym # get result symbol - if not isInvalidReturnType(m.config, prc.typ[0]): + if not isInvalidReturnType(m.config, prc.typ): if sfNoInit in prc.flags: incl(res.flags, sfNoInit) if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(procBody); val != nil): var decl = localVarDecl(p, resNode) @@ -1053,7 +1053,7 @@ proc genProcAux(m: BModule, prc: PSym) = initLocalVar(p, res, immediateAsgn=false) returnStmt = ropecg(p.module, "\treturn $1;$n", [rdLoc(res.loc)]) else: - fillResult(p.config, resNode) + fillResult(p.config, resNode, prc.typ) assignParam(p, res, prc.typ[0]) # We simplify 'unsureAsgn(result, nil); unsureAsgn(result, x)' # to 'unsureAsgn(result, x)' diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 33e304ab2..c2b385c80 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2137,6 +2137,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, incl(s.flags, sfWasForwarded) elif sfBorrow in s.flags: semBorrow(c, n, s) sideEffectsCheck(c, s) + closeScope(c) # close scope for parameters # c.currentScope = oldScope popOwner(c) diff --git a/tests/objects/t19342_2.nim b/tests/objects/t19342_2.nim new file mode 100644 index 000000000..6f6d0f2b3 --- /dev/null +++ b/tests/objects/t19342_2.nim @@ -0,0 +1,18 @@ +discard """ + targets: "c cpp" +""" + +{.compile: "m19342.c".} + +# bug #19342 +type + Node* {.byRef.} = object + data: array[25, cint] + +proc myproc(name: cint): Node {.importc: "hello", cdecl.} + +proc parse = + let node = myproc(10) + doAssert node.data[0] == 999 + +parse() \ No newline at end of file From 520881af9add94b26ef7d0630352ad425bb84490 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 29 Jan 2022 10:08:38 +0800 Subject: [PATCH 0676/1654] update outdated link (#19465) Ref https://github.com/nim-lang/Nim/issues/19463 --- doc/intern.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/intern.rst b/doc/intern.rst index e8fb4ac86..a8846ae02 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -166,7 +166,7 @@ current commit: You can also bisect using custom options to build the compiler, for example if you don't need a debug version of the compiler (which runs slower), you can replace -`./koch temp`:cmd: by explicit compilation command, see `Rebuilding the compiler`_. +`./koch temp`:cmd: by explicit compilation command, see `Bootstrapping the compiler`_. Runtimes From cb894c7094fb49014f85815a9dafc38b5dda743e Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 29 Jan 2022 13:03:01 +0000 Subject: [PATCH 0677/1654] Merge pull request from GHSA-ggrq-h43f-3w7m This fixes a CVE (currently https://github.com/nim-lang/Nim/security/advisories/GHSA-ggrq-h43f-3w7m) --- compiler/docgen.nim | 2 +- lib/packages/docutils/rst.nim | 22 ++++++++++++++--- tests/stdlib/trstgen.nim | 45 +++++++++++++++++++++++++++++------ 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 90e0f54b3..235922349 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -282,7 +282,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, result.cache = cache result.outDir = conf.outDir.string result.isPureRst = isPureRst - var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown} + var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled} if not isPureRst: options.incl roNimFile result.sharedState = newRstSharedState( options, filename.string, diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 791953767..93c2231dc 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -235,6 +235,12 @@ type ## to Markdown) -- implies `roSupportMarkdown` roNimFile ## set for Nim files where default interpreted ## text role should be :nim: + roSandboxDisabled ## this option enables certain options + ## (e.g. raw, include) + ## which are disabled by default as they can + ## enable users to read arbitrary data and + ## perform XSS if the parser is used in a web + ## app. RstParseOptions* = set[RstParseOption] @@ -260,7 +266,8 @@ type mwBrokenLink = "broken link '$1'", mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", - mwRstStyle = "RST style: $1" + mwRstStyle = "RST style: $1", + meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.closure, gcsafe.} ## what to do in case of an error @@ -315,6 +322,7 @@ const ":geek:": "icon_e_geek", ":ugeek:": "icon_e_ugeek" } + SandboxDirAllowlist = ["image", "code", "code-block"] type TokType = enum @@ -2987,10 +2995,14 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode = ## ## As an extension this proc will process the ``file`` extension field and if ## present will replace the code block with the contents of the referenced - ## file. + ## file. This behaviour is disabled in sandboxed mode and can be re-enabled + ## with the `roSandboxDisabled` flag. result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock) var filename = strip(getFieldValue(result, "file")) if filename != "": + if roSandboxDisabled notin p.s.options: + let tok = p.tok[p.idx-2] + rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col) var path = p.findRelativeFile(filename) if path == "": rstMessage(p, meCannotOpenFile, filename) var n = newRstNode(rnLiteralBlock) @@ -3086,6 +3098,11 @@ proc dirRaw(p: var RstParser): PRstNode = proc selectDir(p: var RstParser, d: string): PRstNode = result = nil + let tok = p.tok[p.idx-2] # report on directive in ".. directive::" + if roSandboxDisabled notin p.s.options: + if d notin SandboxDirAllowlist: + rstMessage(p, meSandboxedDirective, d, tok.line, tok.col) + case d of "admonition", "attention", "caution": result = dirAdmonition(p, d) of "code": result = dirCodeBlock(p) @@ -3112,7 +3129,6 @@ proc selectDir(p: var RstParser, d: string): PRstNode = of "title": result = dirTitle(p) of "warning": result = dirAdmonition(p, d) else: - let tok = p.tok[p.idx-2] # report on directive in ".. directive::" rstMessage(p, meInvalidDirective, d, tok.line, tok.col) proc prefix(ftnType: FootnoteType): string = diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 2be65b35b..995c0a151 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -47,6 +47,10 @@ proc optionListLabel(opt: string): string = opt & "              " +const + NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} + + suite "YAML syntax highlighting": test "Basics": let input = """.. code-block:: yaml @@ -631,7 +635,9 @@ let x = 1 let p3 = """

              Par3 """ & id"value3" & ".

              " let p4 = """

              Par4 value4.

              """ let expected = p1 & p2 & "\n" & p3 & "\n" & p4 - check(input.toHtml == expected) + check( + input.toHtml(NoSandboxOpts) == expected + ) test "role directive": let input = dedent""" @@ -642,7 +648,10 @@ let x = 1 :language: brainhelp """ var warnings = new seq[string] - let output = input.toHtml(warnings=warnings) + let output = input.toHtml( + NoSandboxOpts, + warnings=warnings + ) check(warnings[].len == 1 and "language 'brainhelp' not supported" in warnings[0]) test "RST comments": @@ -1187,7 +1196,9 @@ Test1 .. tip:: endOf tip .. warning:: endOf warning """ - let output0 = input0.toHtml + let output0 = input0.toHtml( + NoSandboxOpts + ) for a in ["admonition", "attention", "caution", "danger", "error", "hint", "important", "note", "tip", "warning" ]: doAssert "endOf " & a & "" in output0 @@ -1198,7 +1209,9 @@ Test1 Test paragraph. """ - let output1 = input1.toHtml + let output1 = input1.toHtml( + NoSandboxOpts + ) doAssert "endOfError" in output1 doAssert "

              Test paragraph.

              " in output1 doAssert "class=\"admonition admonition-error\"" in output1 @@ -1210,7 +1223,9 @@ Test1 Test paragraph. """ - let output2 = input2.toHtml + let output2 = input2.toHtml( + NoSandboxOpts + ) doAssert "endOfError Test2p." in output2 doAssert "

              Test paragraph.

              " in output2 doAssert "class=\"admonition admonition-error\"" in output2 @@ -1218,7 +1233,9 @@ Test1 let input3 = dedent """ .. note:: endOfNote """ - let output3 = input3.toHtml + let output3 = input3.toHtml( + NoSandboxOpts + ) doAssert "endOfNote" in output3 doAssert "class=\"admonition admonition-info\"" in output3 @@ -1303,7 +1320,9 @@ Test1 That was a transition. """ - let output1 = input1.toHtml + let output1 = input1.toHtml( + NoSandboxOpts + ) doAssert "

              Nim))""") check("(([Nim](javascript://nim-lang.org/)))".toHtml == """((Nim))""") + +suite "local file inclusion": + test "cannot include files in sandboxed mode": + var error = new string + discard ".. include:: ./readme.md".toHtml(error=error) + check(error[] == "input(1, 11) Error: disabled directive: 'include'") + + test "code-block file directive is disabled": + var error = new string + discard ".. code-block:: nim\n :file: ./readme.md".toHtml(error=error) + check(error[] == "input(2, 20) Error: disabled directive: 'file'") + From 33cd883e27f7ba7564cbeb59d2d38fe02bb507bf Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 29 Jan 2022 21:06:54 +0800 Subject: [PATCH 0678/1654] [add testcase] NRVO does not occur with init procedures (#19462) * [add testcase] NRVO does not occur with init procedures close #19094 * Update tests/ccgbugs2/tcodegen.nim --- tests/ccgbugs2/tcodegen.nim | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/ccgbugs2/tcodegen.nim diff --git a/tests/ccgbugs2/tcodegen.nim b/tests/ccgbugs2/tcodegen.nim new file mode 100644 index 000000000..84cd76e2f --- /dev/null +++ b/tests/ccgbugs2/tcodegen.nim @@ -0,0 +1,28 @@ +discard """ + targets: "c cpp" +""" + +# bug #19094 +type + X = object + filler: array[2048, int] + innerAddress: uint + +proc initX(): X = + result.innerAddress = cast[uint](result.addr) + +proc initXInPlace(x: var X) = + x.innerAddress = cast[uint](x.addr) + +block: # NRVO1 + var x = initX() + let innerAddress = x.innerAddress + let outerAddress = cast[uint](x.addr) + doAssert(innerAddress == outerAddress) # [OK] + +block: # NRVO2 + var x: X + initXInPlace(x) + let innerAddress = x.innerAddress + let outerAddress = cast[uint](x.addr) + doAssert(innerAddress == outerAddress) # [OK] From 56a901f9e197dd03f8c70ab4ef1042dc33548af0 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 31 Jan 2022 20:49:20 +0800 Subject: [PATCH 0679/1654] fix broken CI (#19472) * fix broken CI * fix * fix tests --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 2 ++ tests/stdlib/trst.nim | 12 ++++++------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 235922349..3e61032d8 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -237,6 +237,7 @@ template declareClosures = of meInvalidDirective: k = errRstInvalidDirectiveX of meInvalidField: k = errRstInvalidField of meFootnoteMismatch: k = errRstFootnoteMismatch + of meSandboxedDirective: k = errRstSandboxedDirective of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX of mwAmbiguousLink: k = warnRstAmbiguousLink diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index e13387be6..575be6196 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -39,6 +39,7 @@ type errRstInvalidDirectiveX, errRstInvalidField, errRstFootnoteMismatch, + errRstSandboxedDirective, errProveInit, # deadcode errGenerated, errUser, @@ -110,6 +111,7 @@ const errRstInvalidDirectiveX: "invalid directive: '$1'", errRstInvalidField: "invalid field: $1", errRstFootnoteMismatch: "number of footnotes and their references don't match: $1", + errRstSandboxedDirective: "disabled directive: '$1'", errProveInit: "Cannot prove that '$1' is initialized.", # deadcode errGenerated: "$1", errUser: "$1", diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 0a9eb8038..771e02477 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -23,7 +23,7 @@ import std/private/miscdollars import os proc toAst(input: string, - rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile}, + rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -866,7 +866,7 @@ suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") let input = ".. include:: other.rst" - doAssert "test1" == rstTohtml(input, {}, defaultConfig()) + doAssert "test1" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include starting from": @@ -880,7 +880,7 @@ OtherStart .. include:: other.rst :start-after: OtherStart """ - check "Visible" == rstTohtml(input, {}, defaultConfig()) + check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include everything before": @@ -894,7 +894,7 @@ And this should **NOT** be visible in `docs.html` .. include:: other.rst :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -912,7 +912,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - check "Visible" == rstTohtml(input, {}, defaultConfig()) + check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -932,7 +932,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {}, defaultConfig()) + doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") suite "RST escaping": From 22ae0bef63189dff28676b0dd5f6089864151120 Mon Sep 17 00:00:00 2001 From: tandy1000 Date: Tue, 1 Feb 2022 11:12:55 +0000 Subject: [PATCH 0680/1654] Update jsfetch with latest API and fix missing bindings (#19473) * Update with latest API and fix missing bindings remove deprecated `Body` remove implicit `cstring` convs add `Headers` to `FetchOptions` add `Request` init proc which takes `FetchOptions` * Update lib/std/jsfetch.nim Co-authored-by: Juan Carlos * Update lib/std/jsfetch.nim Co-authored-by: Juan Carlos * remove experimental flag Co-authored-by: Juan Carlos --- lib/std/jsfetch.nim | 46 ++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index a25fe7ca8..808d699b0 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -1,5 +1,4 @@ ## - Fetch for the JavaScript target: https://developer.mozilla.org/docs/Web/API/Fetch_API -## .. Note:: jsfetch is Experimental. when not defined(js): {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".} @@ -12,6 +11,7 @@ type keepalive*: bool metod* {.importjs: "method".}: cstring body*, integrity*, referrer*, mode*, credentials*, cache*, redirect*, referrerPolicy*: cstring + headers*: Headers FetchModes* = enum ## Mode options. fmCors = "cors" @@ -42,16 +42,13 @@ type frpOriginWhenCrossOrigin = "origin-when-cross-origin" frpUnsafeUrl = "unsafe-url" - Body* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Body - bodyUsed*: bool - Response* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Response bodyUsed*, ok*, redirected*: bool typ* {.importjs: "type".}: cstring url*, statusText*: cstring status*: cint headers*: Headers - body*: Body + body*: cstring Request* = ref object of JsRoot ## https://developer.mozilla.org/en-US/docs/Web/API/Request bodyUsed*, ok*, redirected*: bool @@ -59,8 +56,7 @@ type url*, statusText*: cstring status*: cint headers*: Headers - body*: Body - + body*: cstring func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response(#))".} ## Constructor for `Response`. This does *not* call `fetch()`. Same as `new Response()`. @@ -68,30 +64,34 @@ func newResponse*(body: cstring | FormData): Response {.importjs: "(new Response func newRequest*(url: cstring): Request {.importjs: "(new Request(#))".} ## Constructor for `Request`. This does *not* call `fetch()`. Same as `new Request()`. +func newRequest*(url: cstring; fetchOptions: FetchOptions): Request {.importjs: "(new Request(#, #))".} + ## Constructor for `Request` with `fetchOptions`. Same as `fetch(url, fetchOptions)`. + func clone*(self: Response | Request): Response {.importjs: "#.$1()".} ## https://developer.mozilla.org/en-US/docs/Web/API/Response/clone proc text*(self: Response): Future[cstring] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/text + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/text proc json*(self: Response): Future[JsObject] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/json + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/json -proc formData*(self: Body): Future[FormData] {.importjs: "#.$1()".} - ## https://developer.mozilla.org/en-US/docs/Web/API/Body/formData +proc formData*(self: Response): Future[FormData] {.importjs: "#.$1()".} + ## https://developer.mozilla.org/en-US/docs/Web/API/Response/formData proc unsafeNewFetchOptions*(metod, body, mode, credentials, cache, referrerPolicy: cstring; - keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring): FetchOptions {.importjs: - "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #}".} + keepalive: bool; redirect = "follow".cstring; referrer = "client".cstring; integrity = "".cstring; headers: Headers = newHeaders()): FetchOptions {.importjs: + "{method: #, body: #, mode: #, credentials: #, cache: #, referrerPolicy: #, keepalive: #, redirect: #, referrer: #, integrity: #, headers: #}".} ## .. warning:: Unsafe `newfetchOptions`. func newfetchOptions*(metod: HttpMethod; body: cstring; mode: FetchModes; credentials: FetchCredentials; cache: FetchCaches; referrerPolicy: FetchReferrerPolicies; - keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring): FetchOptions = + keepalive: bool; redirect = frFollow; referrer = "client".cstring; integrity = "".cstring, + headers: Headers = newHeaders()): FetchOptions = ## Constructor for `FetchOptions`. result = FetchOptions( - body: body, mode: $mode, credentials: $credentials, cache: $cache, referrerPolicy: $referrerPolicy, - keepalive: keepalive, redirect: $redirect, referrer: referrer, integrity: integrity, + body: body, mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, metod: (case metod of HttpHead: "HEAD".cstring of HttpGet: "GET".cstring @@ -109,9 +109,9 @@ proc fetch*(url: cstring | Request): Future[Response] {.importjs: "$1(#)".} proc fetch*(url: cstring | Request; options: FetchOptions): Future[Response] {.importjs: "$1(#, #)".} ## `fetch()` API that takes a `FetchOptions`, returns a `Future[Response]`. -func toCstring*(self: Request | Response | Body | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} +func toCstring*(self: Request | Response | FetchOptions): cstring {.importjs: "JSON.stringify(#)".} -func `$`*(self: Request | Response | Body | FetchOptions): string = $toCstring(self) +func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self) runnableExamples("-r:off"): @@ -131,7 +131,8 @@ runnableExamples("-r:off"): keepalive = false, redirect = "follow".cstring, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options0.keepalive == false assert options0.metod == "POST".cstring @@ -143,6 +144,7 @@ runnableExamples("-r:off"): assert options0.redirect == "follow".cstring assert options0.referrer == "client".cstring assert options0.integrity == "".cstring + assert options0.headers.len == 0 block: let options1: FetchOptions = newFetchOptions( @@ -155,7 +157,8 @@ runnableExamples("-r:off"): keepalive = false, redirect = frFollow, referrer = "client".cstring, - integrity = "".cstring + integrity = "".cstring, + headers = newHeaders() ) assert options1.keepalive == false assert options1.metod == $HttpPost @@ -167,6 +170,7 @@ runnableExamples("-r:off"): assert options1.redirect == $frFollow assert options1.referrer == "client".cstring assert options1.integrity == "".cstring + assert options1.headers.len == 0 block: let response: Response = newResponse(body = "-. .. --".cstring) @@ -182,7 +186,7 @@ runnableExamples("-r:off"): assert response.ok assert response.status == 200.cint assert response.headers is Headers - assert response.body is Body + assert response.body is cstring discard example() From 1830a3b505c33be5fc3b9e311c5e7720d9b37b32 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Wed, 2 Feb 2022 01:38:21 -0700 Subject: [PATCH 0681/1654] No longer segfault when using a typeclass with a self referencing type (#19467) --- compiler/sigmatch.nim | 3 ++- tests/generics/tgenerics_issues.nim | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index e1195d04a..14f4c5e19 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1688,7 +1688,8 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, var aa = a while aa.kind in {tyTypeDesc, tyGenericParam} and aa.len > 0: aa = lastSon(aa) - if aa.kind == tyGenericParam: + if aa.kind in {tyGenericParam} + tyTypeClasses: + # If the constraint is a genericParam or typeClass this isGeneric return isGeneric result = typeRel(c, f.base, aa, flags) if result > isGeneric: result = isGeneric diff --git a/tests/generics/tgenerics_issues.nim b/tests/generics/tgenerics_issues.nim index 092926c01..db7a16569 100644 --- a/tests/generics/tgenerics_issues.nim +++ b/tests/generics/tgenerics_issues.nim @@ -860,3 +860,15 @@ type var a: Tile3[int] +block: # Ensure no segfault from constraint + type + Regex[A: SomeOrdinal] = ref object + val: Regex[A] + MyConstraint = (seq or enum or set) + MyOtherType[A: MyConstraint] = ref object + val: MyOtherType[A] + + var + a = Regex[int]() + b = Regex[bool]() + c = MyOtherType[seq[int]]() From 486cb09ec2caef60011b3d182bfd188dadafdf62 Mon Sep 17 00:00:00 2001 From: Regis Caillaud <35006197+Clonkk@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:44:51 +0100 Subject: [PATCH 0682/1654] Clonkk fix2 11923 (#19451) * fix nnkBracketExpr not compiling for getImpl on customPragmaNode * fix test import * fix alias not working with hasCustomPragmas --- lib/core/macros.nim | 27 ++++++++++++++++++++++----- tests/pragmas/tcustom_pragma.nim | 6 ++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 28106493f..c0e6e5154 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1493,6 +1493,22 @@ macro expandMacros*(body: typed): untyped = echo body.toStrLit result = body +proc extractTypeImpl(n: NimNode): NimNode = + ## attempts to extract the type definition of the given symbol + case n.kind + of nnkSym: # can extract an impl + result = n.getImpl.extractTypeImpl() + of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n + of nnkBracketExpr: + if n.typeKind == ntyTypeDesc: + result = n[1].extractTypeImpl() + else: + doAssert n.typeKind == ntyGenericInst + result = n[0].getImpl() + of nnkTypeDef: + result = n[2] + else: error("Invalid node to retrieve type implementation of: " & $n.kind) + proc customPragmaNode(n: NimNode): NimNode = expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) let @@ -1501,7 +1517,10 @@ proc customPragmaNode(n: NimNode): NimNode = if typ.kind == nnkBracketExpr and typ.len > 1 and typ[1].kind == nnkProcTy: return typ[1][1] elif typ.typeKind == ntyTypeDesc: - let impl = typ[1].getImpl() + let impl = getImpl( + if kind(typ[1]) == nnkBracketExpr: typ[1][0] + else: typ[1] + ) if impl[0].kind == nnkPragmaExpr: return impl[0][1] else: @@ -1524,14 +1543,12 @@ proc customPragmaNode(n: NimNode): NimNode = let name = $(if n.kind == nnkCheckedFieldExpr: n[0][1] else: n[1]) let typInst = getTypeInst(if n.kind == nnkCheckedFieldExpr or n[0].kind == nnkHiddenDeref: n[0][0] else: n[0]) var typDef = getImpl( - if typInst.kind == nnkVarTy or - typInst.kind == nnkBracketExpr: - typInst[0] + if typInst.kind in {nnkVarTy, nnkBracketExpr}: typInst[0] else: typInst ) while typDef != nil: typDef.expectKind(nnkTypeDef) - let typ = typDef[2] + let typ = typDef[2].extractTypeImpl() typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) let isRef = typ.kind in {nnkRefTy, nnkPtrTy} if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) diff --git a/tests/pragmas/tcustom_pragma.nim b/tests/pragmas/tcustom_pragma.nim index 1c3709b26..db2536188 100644 --- a/tests/pragmas/tcustom_pragma.nim +++ b/tests/pragmas/tcustom_pragma.nim @@ -20,16 +20,22 @@ block: MyGenericObj[T] = object myField1, myField2 {.myAttr: "hi".}: int + MyOtherObj = MyObj + var o: MyObj static: doAssert o.myField2.hasCustomPragma(myAttr) doAssert(not o.myField1.hasCustomPragma(myAttr)) + doAssert(not o.myField1.hasCustomPragma(MyObj)) + doAssert(not o.myField1.hasCustomPragma(MyOtherObj)) var ogen: MyGenericObj[int] static: doAssert ogen.myField2.hasCustomPragma(myAttr) doAssert(not ogen.myField1.hasCustomPragma(myAttr)) + doAssert(not ogen.myField1.hasCustomPragma(MyGenericObj)) + doAssert(not ogen.myField1.hasCustomPragma(MyGenericObj)) import custom_pragma From 891329cd4b3e7e3b0d995cf342c1018239ebcf81 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Feb 2022 00:10:11 +0800 Subject: [PATCH 0683/1654] move io out of system (#19442) * move io out of system * fix tests * fix tests * next step * rename to syncio * rename * fix nimscript * comma * fix * fix parts of errors * good for now * fix test --- changelog.md | 2 ++ compiler/pathutils.nim | 3 ++ compiler/vmops.nim | 2 +- doc/tut1.rst | 8 ++--- lib/posix/posix.nim | 3 ++ lib/pure/htmlparser.nim | 3 ++ lib/pure/json.nim | 3 ++ lib/pure/logging.nim | 11 ++++--- lib/pure/memfiles.nim | 4 +-- lib/pure/os.nim | 3 ++ lib/pure/parsecfg.nim | 4 +++ lib/pure/parsecsv.nim | 3 ++ lib/pure/pegs.nim | 2 ++ lib/pure/ropes.nim | 3 ++ lib/pure/streams.nim | 9 ++++-- lib/pure/terminal.nim | 3 ++ lib/pure/xmlparser.nim | 3 ++ lib/{system/io.nim => std/syncio.nim} | 45 +++++++++++++-------------- lib/system.nim | 9 ++++-- lib/system_overview.rst | 1 - lib/windows/winlean.nim | 3 ++ tests/cpp/t6986.nim | 3 ++ tests/test_nimscript.nims | 3 ++ tools/kochdocs.nim | 1 - 24 files changed, 92 insertions(+), 42 deletions(-) rename lib/{system/io.nim => std/syncio.nim} (96%) diff --git a/changelog.md b/changelog.md index bbbaf6e97..412e046eb 100644 --- a/changelog.md +++ b/changelog.md @@ -20,6 +20,8 @@ - `addr` is now available for all addressable locations, `unsafeAddr` is deprecated and becomes an alias for `addr`. +- io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`. + ## Standard library additions and changes - `macros.parseExpr` and `macros.parseStmt` now accept an optional diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index d8f3613b0..8138a245a 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -12,6 +12,9 @@ import os, pathnorm +when defined(nimSlimSystem): + import std/syncio + type AbsoluteFile* = distinct string AbsoluteDir* = distinct string diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 018e7b9c6..6b06cc68d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -47,7 +47,7 @@ template systemop(op) {.dirty.} = registerCallback(c, "stdlib.system." & astToStr(op), `op Wrapper`) template ioop(op) {.dirty.} = - registerCallback(c, "stdlib.io." & astToStr(op), `op Wrapper`) + registerCallback(c, "stdlib.syncio." & astToStr(op), `op Wrapper`) template macrosop(op) {.dirty.} = registerCallback(c, "stdlib.macros." & astToStr(op), `op Wrapper`) diff --git a/doc/tut1.rst b/doc/tut1.rst index 3107aef45..405df57b1 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -84,8 +84,8 @@ done with spaces only, tabulators are not allowed. String literals are enclosed in double-quotes. The `var` statement declares a new variable named `name` of type `string` with the value that is -returned by the `readLine `_ procedure. Since the -compiler knows that `readLine `_ returns a string, +returned by the `readLine `_ procedure. Since the +compiler knows that `readLine `_ returns a string, you can leave out the type in the declaration (this is called `local type inference`:idx:). So this will work too: @@ -97,7 +97,7 @@ Note that this is basically the only form of type inference that exists in Nim: it is a good compromise between brevity and readability. The "hello world" program contains several identifiers that are already known -to the compiler: `echo`, `readLine `_, etc. +to the compiler: `echo`, `readLine `_, etc. These built-ins are declared in the system_ module which is implicitly imported by any other module. @@ -594,7 +594,7 @@ Procedures ========== To define new commands like `echo `_ -and `readLine `_ in the examples, the concept of a +and `readLine `_ in the examples, the concept of a *procedure* is needed. You might be used to them being called *methods* or *functions* in other languages, but Nim `differentiates these concepts `_. In diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 912e395a7..c10fc6ac9 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -37,6 +37,9 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} +when defined(nimSlimSystem): + import std/syncio + # TODO these constants don't seem to be fetched from a header file for unknown # platforms - where do they come from and why are they here? when false: diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 21dff69ff..6b1300f11 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -51,6 +51,9 @@ import strutils, streams, parsexml, xmltree, unicode, strtabs +when defined(nimPreviewSlimSystem): + import std/syncio + type HtmlTag* = enum ## list of all supported HTML tags; order will always be ## alphabetically diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 922cd4e2f..bdc9fe5ab 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -164,6 +164,9 @@ import hashes, tables, strutils, lexbase, streams, macros, parsejson import options # xxx remove this dependency using same approach as https://github.com/nim-lang/Nim/pull/14563 import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + export tables.`$` diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index b2ace79ab..6751a372a 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -48,7 +48,7 @@ ## .. warning:: ## For loggers that log to a console or to files, only error and fatal ## messages will cause their output buffers to be flushed immediately. -## Use the `flushFile proc `_ to flush the buffer +## Use the `flushFile proc `_ to flush the buffer ## manually if needed. ## ## Handlers @@ -146,6 +146,9 @@ import strutils, times when not defined(js): import os +when defined(nimPreviewSlimSystem): + import std/syncio + type Level* = enum ## \ ## Enumeration of logging levels. @@ -346,7 +349,7 @@ method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## ## **Note:** Only error and fatal messages will cause the output buffer ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## `_ to flush the buffer manually if needed. ## ## See also: ## * `log method<#log.e,FileLogger,Level,varargs[string,]>`_ @@ -422,7 +425,7 @@ when not defined(js): ## **Notes:** ## * Only error and fatal messages will cause the output buffer ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## `_ to flush the buffer manually if needed. ## * This method is not available for the JavaScript backend. ## ## See also: @@ -600,7 +603,7 @@ when not defined(js): ## **Notes:** ## * Only error and fatal messages will cause the output buffer ## to be flushed immediately. Use the `flushFile proc - ## `_ to flush the buffer manually if needed. + ## `_ to flush the buffer manually if needed. ## * This method is not available for the JavaScript backend. ## ## See also: diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 407a358fa..f65ca125e 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -431,7 +431,7 @@ iterator memSlices*(mfile: MemFile, delim = '\l', eat = '\r'): MemSlice {.inline iterator lines*(mfile: MemFile, buf: var string, delim = '\l', eat = '\r'): string {.inline.} = ## Replace contents of passed buffer with each new line, like - ## `readLine(File) `_. + ## `readLine(File) `_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## @@ -450,7 +450,7 @@ iterator lines*(mfile: MemFile, buf: var string, delim = '\l', iterator lines*(mfile: MemFile, delim = '\l', eat = '\r'): string {.inline.} = ## Return each line in a file as a Nim string, like - ## `lines(File) `_. + ## `lines(File) `_. ## `delim`, `eat`, and delimiting logic is exactly as for `memSlices ## <#memSlices.i,MemFile,char,char>`_, but Nim strings are returned. ## diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 07040d611..76164bc49 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -34,6 +34,9 @@ import std/private/since import strutils, pathnorm +when defined(nimPreviewSlimSystem): + import std/syncio + const weirdTarget = defined(nimscript) or defined(js) since (1, 1): diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 0ee19912c..54584a253 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -175,8 +175,12 @@ import strutils, lexbase, streams, tables import std/private/decode_helpers import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + include "system/inclrtl" + type CfgEventKind* = enum ## enumeration of all events that may occur when parsing cfgEof, ## end of file reached diff --git a/lib/pure/parsecsv.nim b/lib/pure/parsecsv.nim index 6db794682..a8d1cfaab 100644 --- a/lib/pure/parsecsv.nim +++ b/lib/pure/parsecsv.nim @@ -67,6 +67,9 @@ import lexbase, streams +when defined(nimPreviewSlimSystem): + import std/syncio + type CsvRow* = seq[string] ## A row in a CSV file. CsvParser* = object of BaseLexer ## The parser object. diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index bac8b1a0e..0eed1c388 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -16,6 +16,8 @@ ## include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/syncio const useUnicode = true ## change this to deactivate proper UTF-8 support diff --git a/lib/pure/ropes.nim b/lib/pure/ropes.nim index 1300b4479..84b5b47c2 100644 --- a/lib/pure/ropes.nim +++ b/lib/pure/ropes.nim @@ -19,6 +19,9 @@ include system/inclrtl import streams +when defined(nimPreviewSlimSystem): + import std/syncio + {.push debugger: off.} # the user does not want to trace a part # of the standard library! diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 7dc81148f..7ad81685f 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -92,10 +92,13 @@ ## See also ## ======== ## * `asyncstreams module `_ -## * `io module `_ for `FileMode enum `_ +## * `io module `_ for `FileMode enum `_ import std/private/since +when defined(nimPreviewSlimSystem): + import std/syncio + proc newEIO(msg: string): owned(ref IOError) = new(result) result.msg = msg @@ -1331,7 +1334,7 @@ proc newFileStream*(f: File): owned FileStream = ## * `newStringStream proc <#newStringStream,string>`_ creates a new stream ## from string. ## * `newFileStream proc <#newFileStream,string,FileMode,int>`_ is the same - ## as using `open proc `_ + ## as using `open proc `_ ## on Examples. ## * `openFileStream proc <#openFileStream,string,FileMode,int>`_ creates a ## file stream from the file name and the mode. @@ -1370,7 +1373,7 @@ proc newFileStream*(filename: string, mode: FileMode = fmRead, ## Creates a new stream from the file named `filename` with the mode `mode`. ## ## If the file cannot be opened, `nil` is returned. See the `io module - ## `_ for a list of available `FileMode enums `_. + ## `_ for a list of available `FileMode enums `_. ## ## **Note:** ## * **This function returns nil in case of failure.** diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index c9aafc037..5755e142a 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -66,6 +66,9 @@ import colors when defined(windows): import winlean +when defined(nimPreviewSlimSystem): + import std/syncio + type PTerminal = ref object trueColorIsSupported: bool diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 3d9c288ed..6785fa66e 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -11,6 +11,9 @@ import streams, parsexml, strtabs, xmltree +when defined(nimPreviewSlimSystem): + import std/syncio + type XmlError* = object of ValueError ## Exception that is raised ## for invalid XML. diff --git a/lib/system/io.nim b/lib/std/syncio.nim similarity index 96% rename from lib/system/io.nim rename to lib/std/syncio.nim index 04a43328d..eab96254a 100644 --- a/lib/system/io.nim +++ b/lib/std/syncio.nim @@ -1,18 +1,17 @@ # # # Nim's Runtime Library -# (c) Copyright 2019 Nim contributors +# (c) Copyright 2022 Nim contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## This is a part of `system.nim`, you should not manually import it. +## This module implements various synchronized I/O operations. - -include inclrtl +include system/inclrtl import std/private/since -import formatfloat +import system/formatfloat # ----------------- IO Part ------------------------------------------------ type @@ -179,7 +178,7 @@ proc checkErr(f: File) = {.push stackTrace: off, profiler: off.} proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer pointed to by `buffer`. Returns + ## Reads `len` bytes into the buffer pointed to by `buffer`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = cast[int](c_fread(buffer, 1, cast[csize_t](len), f)) @@ -188,13 +187,13 @@ proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. proc readBytes*(f: File, a: var openArray[int8|uint8], start, len: Natural): int {. tags: [ReadIOEffect], benign.} = - ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[start]), len) proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], benign.} = - ## reads up to `a.len` bytes into the buffer `a`. Returns + ## Reads up to `a.len` bytes into the buffer `a`. Returns ## the actual number of bytes that have been read which may be less than ## `a.len` (if not as many bytes are remaining), but not greater. result = readBuffer(f, addr(a[0]), a.len) @@ -202,7 +201,7 @@ proc readChars*(f: File, a: var openArray[char]): int {.tags: [ReadIOEffect], be proc readChars*(f: File, a: var openArray[char], start, len: Natural): int {. tags: [ReadIOEffect], benign, deprecated: "use other `readChars` overload, possibly via: readChars(toOpenArray(buf, start, len-1))".} = - ## reads `len` bytes into the buffer `a` starting at `a[start]`. Returns + ## Reads `len` bytes into the buffer `a` starting at `a[start]`. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. if (start + len) > len(a): @@ -216,7 +215,7 @@ proc write*(f: File, c: cstring) {.tags: [WriteIOEffect], benign.} = proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of buffer pointed to by the parameter `buffer` to the + ## Writes the bytes of buffer pointed to by the parameter `buffer` to the ## file `f`. Returns the number of actual written bytes, which may be less ## than `len` in case of an error. result = cast[int](c_fwrite(buffer, 1, cast[csize_t](len), f)) @@ -224,7 +223,7 @@ proc writeBuffer*(f: File, buffer: pointer, len: Natural): int {. proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -232,7 +231,7 @@ proc writeBytes*(f: File, a: openArray[int8|uint8], start, len: Natural): int {. proc writeChars*(f: File, a: openArray[char], start, len: Natural): int {. tags: [WriteIOEffect], benign.} = - ## writes the bytes of `a[start..start+len-1]` to the file `f`. Returns + ## Writes the bytes of `a[start..start+len-1]` to the file `f`. Returns ## the number of actual written bytes, which may be less than `len` in case ## of an error. var x = cast[ptr UncheckedArray[int8]](a) @@ -341,7 +340,7 @@ proc flushFile*(f: File) {.tags: [WriteIOEffect].} = discard c_fflush(f) proc getFileHandle*(f: File): FileHandle = - ## returns the file handle of the file `f`. This is only useful for + ## Returns the file handle of the file `f`. This is only useful for ## platform specific programming. ## Note that on Windows this doesn't return the Windows-specific handle, ## but the C library's notion of a handle, whatever that means. @@ -349,7 +348,7 @@ proc getFileHandle*(f: File): FileHandle = c_fileno(f) proc getOsFileHandle*(f: File): FileHandle = - ## returns the OS file handle of the file `f`. This is only useful for + ## Returns the OS file handle of the file `f`. This is only useful for ## platform specific programming. when defined(windows): result = FileHandle getOsfhandle(cint getFileHandle(f)) @@ -380,7 +379,7 @@ when defined(nimdoc) or (defined(posix) and not defined(nimscript)) or defined(w proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f` into `line`. May throw an IO + ## Reads a line of text from the file `f` into `line`. May throw an IO ## exception. ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. Returns `false` @@ -500,7 +499,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], line.setLen(pos+sp) proc readLine*(f: File): string {.tags: [ReadIOEffect], benign.} = - ## reads a line of text from the file `f`. May throw an IO exception. + ## Reads a line of text from the file `f`. May throw an IO exception. ## A line of text may be delimited by `LF` or `CRLF`. The newline ## character(s) are not part of the returned string. result = newStringOfCap(80) @@ -602,7 +601,7 @@ proc readAll*(file: File): string {.tags: [ReadIOEffect], benign.} = proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, tags: [WriteIOEffect], benign.} = - ## writes the values `x` to `f` and then writes "\\n". + ## Writes the values `x` to `f` and then writes "\\n". ## May throw an IO exception. for i in items(x): write(f, i) @@ -723,7 +722,7 @@ proc open*(f: var File, filename: string, proc reopen*(f: File, filename: string, mode: FileMode = fmRead): bool {. tags: [], benign.} = - ## reopens the file `f` with given `filename` and `mode`. This + ## Reopens the file `f` with given `filename` and `mode`. This ## is often used to redirect the `stdin`, `stdout` or `stderr` ## file variables. ## @@ -765,19 +764,19 @@ proc open*(filename: string, sysFatal(IOError, "cannot open: " & filename) proc setFilePos*(f: File, pos: int64, relativeTo: FileSeekPos = fspSet) {.benign.} = - ## sets the position of the file pointer that is used for read/write + ## Sets the position of the file pointer that is used for read/write ## operations. The file's first byte has the index zero. if c_fseek(f, pos, cint(relativeTo)) != 0: raiseEIO("cannot set file position") proc getFilePos*(f: File): int64 {.benign.} = - ## retrieves the current position of the file pointer that is used to + ## Retrieves the current position of the file pointer that is used to ## read from the file `f`. The file's first byte has the index zero. result = c_ftell(f) if result < 0: raiseEIO("cannot retrieve file position") proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} = - ## retrieves the file size (in bytes) of `f`. + ## Retrieves the file size (in bytes) of `f`. let oldPos = getFilePos(f) discard c_fseek(f, 0, 2) # seek the end of the file result = getFilePos(f) @@ -906,7 +905,7 @@ proc writeFile*(filename: string, content: openArray[byte]) {.since: (1, 1).} = raise newException(IOError, "cannot open: " & filename) proc readLines*(filename: string, n: Natural): seq[string] = - ## read `n` lines from the file named `filename`. Raises an IO exception + ## Reads `n` lines from the file named `filename`. Raises an IO exception ## in case of an error. Raises EOF if file does not contain at least `n` lines. ## Available at compile time. A line of text may be delimited by `LF` or `CRLF`. ## The newline character(s) are not part of the returned strings. @@ -948,7 +947,7 @@ iterator lines*(filename: string): string {.tags: [ReadIOEffect].} = close(f) iterator lines*(f: File): string {.tags: [ReadIOEffect].} = - ## Iterate over any line in the file `f`. + ## Iterates over any line in the file `f`. ## ## The trailing newline character(s) are removed from the iterated lines. ## diff --git a/lib/system.nim b/lib/system.nim index 63cdc3ca8..56a6e2887 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2680,7 +2680,7 @@ proc slurp*(filename: string): string {.magic: "Slurp".} ## This is an alias for `staticRead <#staticRead,string>`_. proc staticRead*(filename: string): string {.magic: "Slurp".} - ## Compile-time `readFile `_ proc for easy + ## Compile-time `readFile `_ proc for easy ## `resource`:idx: embedding: ## ## The maximum file size limit that `staticRead` and `slurp` can read is @@ -3150,8 +3150,11 @@ when defined(genode): import system/widestrs export widestrs -import system/io -export io +when not defined(nimPreviewSlimSystem): + {.deprecated: """io is about to move out of system; use `-d:nimPreviewSlimSystem` and + import `std/syncio`.""".} + import std/syncio + export syncio when not defined(createNimHcr) and not defined(nimscript): include nimhcr diff --git a/lib/system_overview.rst b/lib/system_overview.rst index 0e4ffaf17..768fdcd6d 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -7,7 +7,6 @@ is in separate files: * `iterators `_ * `assertions `_ * `dollars `_ -* `io `_ * `widestrs `_ diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 6e626e4ba..c02cd9775 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -17,6 +17,9 @@ when defined(nimHasStyleChecks): {.passc: "-DWIN32_LEAN_AND_MEAN".} +when defined(nimPreviewSlimSystem): + from std/syncio import FileHandle + const useWinUnicode* = not defined(useWinAnsi) diff --git a/tests/cpp/t6986.nim b/tests/cpp/t6986.nim index ffd277adb..16e455c3b 100644 --- a/tests/cpp/t6986.nim +++ b/tests/cpp/t6986.nim @@ -5,6 +5,9 @@ discard """ import sequtils, strutils +when defined(nimPreviewSlimSystem): + import std/syncio + let rules = toSeq(lines("input")) .mapIt(it.split(" => ").mapIt(it.replace("/", ""))) diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 22a566307..b380570ff 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -5,6 +5,9 @@ from stdtest/specialpaths import buildDir +when defined(nimSlimSystem): + import std/syncio + import std/[ # Core: bitops, typetraits, lenientops, macros, volatile, diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 4fbea8c77..8fd9c84a5 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -187,7 +187,6 @@ proc getDocList(): seq[string] = # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim From 772ed5eff29671b1e99ca17b63c646b341c88123 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 3 Feb 2022 11:15:46 +0800 Subject: [PATCH 0684/1654] correct typos (#19485) --- compiler/pathutils.nim | 2 +- lib/posix/posix.nim | 2 +- tests/test_nimscript.nims | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 8138a245a..3a501a417 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -12,7 +12,7 @@ import os, pathnorm -when defined(nimSlimSystem): +when defined(nimPreviewSlimSystem): import std/syncio type diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index c10fc6ac9..9c65d0732 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -37,7 +37,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(nimSlimSystem): +when defined(nimPreviewSlimSystem): import std/syncio # TODO these constants don't seem to be fetched from a header file for unknown diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index b380570ff..5fc77f6e4 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -5,7 +5,7 @@ from stdtest/specialpaths import buildDir -when defined(nimSlimSystem): +when defined(nimPreviewSlimSystem): import std/syncio import std/[ From 33bfe5b12cf771072d93d7316945f3f5e6c727ff Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 4 Feb 2022 20:47:03 +0800 Subject: [PATCH 0685/1654] fix parseEnum cannot parse enum with const fields (#19466) fix #19463 --- lib/std/enumutils.nim | 9 +++++++-- tests/stdlib/tstrutils.nim | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 81e602ad5..b7d2b9f89 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -14,7 +14,7 @@ from typetraits import OrdinalEnum, HoleyEnum macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, userMin, userMax: static[int], normalizer: static[proc(s :string): string]): untyped = - # generates a case stmt, which assigns the correct enum field given + # Generates a case stmt, which assigns the correct enum field given # a normalized string comparison to the `argSym` input. # string normalization is done using passed normalizer. # NOTE: for an enum with fields Foo, Bar, ... we cannot generate @@ -49,7 +49,12 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, of nnkIntLit: fStr = f[0].strVal fNum = f[1].intVal - else: error("Invalid tuple syntax!", f[1]) + else: + let fAst = f[0].getImpl + if fAst.kind == nnkStrLit: + fStr = fAst.strVal + else: + error("Invalid tuple syntax!", f[1]) else: error("Invalid node for enum type `" & $f.kind & "`!", f) # add field if string not already added if fNum >= userMin and fNum <= userMax: diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index 8000b6b07..26fc0c7b0 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -583,6 +583,17 @@ template main() = let g = parseEnum[Foo]("Bar", A) doAssert g == A + block: # bug #19463 + const CAMPAIGN_TABLE = "wikientries_campaign" + const CHARACTER_TABLE = "wikientries_character" + + type Tables = enum + a = CAMPAIGN_TABLE, + b = CHARACTER_TABLE, + + let myA = CAMPAIGN_TABLE + doAssert $parseEnum[Tables](myA) == "wikientries_campaign" + block: # check enum defined in block type Bar = enum From 7b09fd70ab8154eed3738dfd86f84c5642df3cf2 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 5 Feb 2022 06:10:24 +0800 Subject: [PATCH 0686/1654] undefine C symbols in JS backend [backport:1.6] (#19437) fix #19330; fix #19059 --- compiler/cmdlinehelper.nim | 3 ++- compiler/commands.nim | 5 +++-- tests/js/tcsymbol.nim | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 tests/js/tcsymbol.nim diff --git a/compiler/cmdlinehelper.nim b/compiler/cmdlinehelper.nim index 007dc2319..02162755c 100644 --- a/compiler/cmdlinehelper.nim +++ b/compiler/cmdlinehelper.nim @@ -64,7 +64,8 @@ proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: Confi if conf.cmd == cmdNimscript: return false # now process command line arguments again, because some options in the # command line can overwrite the config file's settings - extccomp.initVars(conf) + if conf.backend != backendJs: # bug #19059 + extccomp.initVars(conf) self.processCmdLine(passCmd2, "", conf) if conf.cmd == cmdNone: rawMessage(conf, errGenerated, "command missing") diff --git a/compiler/commands.nim b/compiler/commands.nim index a1fa55e74..fb2907bb1 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -899,8 +899,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; splitSwitch(conf, arg, key, val, pass, info) os.putEnv(key, val) of "cc": - expectArg(conf, switch, arg, pass, info) - setCC(conf, arg, info) + if conf.backend != backendJs: # bug #19330 + expectArg(conf, switch, arg, pass, info) + setCC(conf, arg, info) of "track": expectArg(conf, switch, arg, pass, info) track(conf, arg, info) diff --git a/tests/js/tcsymbol.nim b/tests/js/tcsymbol.nim new file mode 100644 index 000000000..07e52b9b6 --- /dev/null +++ b/tests/js/tcsymbol.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--cc:gcc; --cc:tcc" +""" + +doAssert not defined(gcc) +doAssert not defined(tcc) \ No newline at end of file From d0287748fefd9eae71cb3d66caef724473525b28 Mon Sep 17 00:00:00 2001 From: flywind Date: Sun, 6 Feb 2022 15:51:03 +0800 Subject: [PATCH 0687/1654] setjump => setjmp [backport: 1.2] (#19496) --- compiler/commands.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index fb2907bb1..6d3f23197 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -238,7 +238,7 @@ const errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found" errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found" errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found" - errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found" + errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found" template warningOptionNoop(switch: string) = warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch) From 6b302bb71b6d9aab708b91f8d68957652ec30480 Mon Sep 17 00:00:00 2001 From: Roj Date: Sun, 6 Feb 2022 18:28:54 -0800 Subject: [PATCH 0688/1654] Improve consistency (#19490) --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 6bc7a2385..7ff056a20 100644 --- a/readme.md +++ b/readme.md @@ -28,7 +28,7 @@ the latest release, check out [Nim's website][nim-site] or [bleeding edge docs]( is the official Telegram channel. Not bridged to IRC. * [Stack Overflow][nim-stackoverflow] - a popular Q/A site for programming related topics that includes posts about Nim. -* [Github Wiki][nim-wiki] - Misc user-contributed content. +* [GitHub Wiki][nim-wiki] - Misc user-contributed content. ## Compiling @@ -73,7 +73,7 @@ the installation instructions on the website to do so: https://nim-lang.org/inst For package maintainers: see [packaging guidelines](https://nim-lang.github.io/Nim/packaging.html). -First, get Nim from github: +First, get Nim from GitHub: ``` git clone https://github.com/nim-lang/Nim.git From 28180e47a99cda88abee3e2d8e130c18100f3a4f Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 7 Feb 2022 14:04:04 +0800 Subject: [PATCH 0689/1654] disable nimlsp (#19499) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 40a3aef36..456f42cd2 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -103,7 +103,7 @@ pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" -pkg "nimlsp" +pkg "nimlsp", allowFailure = true pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true From 4b0636fba018df407237811f9e687ed4fc249c10 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Mon, 7 Feb 2022 23:05:41 +0000 Subject: [PATCH 0690/1654] Add compilers and hints to default nim.cfg (#18424) --- config/nim.cfg | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 59d7f48d2..7fe279f56 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -2,6 +2,7 @@ # (c) 2017 Andreas Rumpf # Feel free to edit the default values as you need. +# See https://nim-lang.org/docs/nimc.html # You may set environment variables with # @putenv "key" "val" @@ -17,11 +18,21 @@ hint[LineTooLong]=off #hint[XDeclaredButNotUsed]=off # Examples of how to setup a cross-compiler: +# Nim can target architectures and OSes different than the local host +# Syntax: ..gcc.exe = "" +# ..gcc.linkerexe = "" -# Cross-compiling for Raspberry Pi. -# (This compiler is available in gcc-arm-linux-gnueabihf package on Ubuntu) +# ARM e.g. Raspberry Pi 2: gcc-arm-linux-gnueabihf package on Debian/Ubuntu arm.linux.gcc.exe = "arm-linux-gnueabihf-gcc" arm.linux.gcc.linkerexe = "arm-linux-gnueabihf-gcc" +# ARM64/aarch64 e.g. Raspberry Pi 3: gcc-aarch64-linux-gnu package on Debian/Ubuntu +arm64.linux.gcc.exe = "aarch64-linux-gnu-gcc" +arm64.linux.gcc.linkerexe = "aarch64-linux-gnu-gcc" +# RISC-V: gcc-riscv64-linux-gnu package on Debian/Ubuntu +riscv32.linux.gcc.exe = "riscv64-linux-gnu-gcc" +riscv32.linux.gcc.linkerexe = "riscv64-linux-gnu-gcc" +riscv64.linux.gcc.exe = "riscv64-linux-gnu-gcc" +riscv64.linux.gcc.linkerexe = "arm-linux-gnueabihf-gcc" # For OpenWRT, you will also need to adjust PATH to point to your toolchain. mips.linux.gcc.exe = "mips-openwrt-linux-gcc" From 801c0f0369779a47da2ef6618abea11feb390973 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 8 Feb 2022 02:11:53 +0300 Subject: [PATCH 0691/1654] Fix bug 27 of #17340 (#19433) Fixes silent disappearance of Markdown (pseudo-)link when it's detected as unsafe protocol. Now it will be converted to plain text in spirit of [the specification](https://spec.commonmark.org/0.30/#links). For that sake the check for protocol is added to rst.nim also. --- lib/packages/docutils/rst.nim | 66 +++++++++++++++++++++++--------- lib/packages/docutils/rstgen.nim | 17 ++------ tests/stdlib/trst.nim | 27 ++++++++++--- tests/stdlib/trstgen.nim | 11 +++++- 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index 93c2231dc..d4bfae48f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -222,7 +222,7 @@ import os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, tables + std/private/miscdollars, tables, strscans from highlite import SourceLanguage, getSourceLanguage type @@ -1347,7 +1347,20 @@ proc match(p: RstParser, start: int, expr: string): bool = inc i result = true -proc fixupEmbeddedRef(n, a, b: PRstNode) = +proc safeProtocol*(linkStr: var string): string = + # Returns link's protocol and, if it's not safe, clears `linkStr` + result = "" + if scanf(linkStr, "$w:", result): + # if it has a protocol at all, ensure that it's not 'javascript:' or worse: + if cmpIgnoreCase(result, "http") == 0 or + cmpIgnoreCase(result, "https") == 0 or + cmpIgnoreCase(result, "ftp") == 0: + discard "it's fine" + else: + linkStr = "" + +proc fixupEmbeddedRef(p: var RstParser, n, a, b: PRstNode): bool = + # Returns `true` if the link belongs to an allowed protocol var sep = - 1 for i in countdown(n.len - 2, 0): if n.sons[i].text == "<": @@ -1355,7 +1368,15 @@ proc fixupEmbeddedRef(n, a, b: PRstNode) = break var incr = if sep > 0 and n.sons[sep - 1].text[0] == ' ': 2 else: 1 for i in countup(0, sep - incr): a.add(n.sons[i]) - for i in countup(sep + 1, n.len - 2): b.add(n.sons[i]) + var linkStr = "" + for i in countup(sep + 1, n.len - 2): linkStr.add(n.sons[i].addNodes) + if linkStr != "": + let protocol = safeProtocol(linkStr) + result = linkStr != "" + if not result: + rstMessage(p, mwBrokenLink, protocol, + p.tok[p.idx-3].line, p.tok[p.idx-3].col) + b.add newLeaf(linkStr) proc whichRole(p: RstParser, sym: string): RstNodeKind = result = whichRoleAux(sym) @@ -1407,14 +1428,17 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode = if p.tok[p.idx-2].symbol == "`" and p.tok[p.idx-3].symbol == ">": var a = newRstNode(rnInner) var b = newRstNode(rnInner) - fixupEmbeddedRef(n, a, b) - if a.len == 0: - newKind = rnStandaloneHyperlink - newSons = @[b] - else: - newKind = rnHyperlink - newSons = @[a, b] - setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) + if fixupEmbeddedRef(p, n, a, b): + if a.len == 0: # e.g. ``_ + newKind = rnStandaloneHyperlink + newSons = @[b] + else: # e.g. `link title `_ + newKind = rnHyperlink + newSons = @[a, b] + setRef(p, rstnodeToRefname(a), b, implicitHyperlinkAlias) + else: # include as plain text, not a link + newKind = rnInner + newSons = n.sons result = newRstNode(newKind, newSons) else: # some link that will be resolved in `resolveSubs` newKind = rnRef @@ -1623,7 +1647,6 @@ proc parseMarkdownCodeblock(p: var RstParser): PRstNode = result.add(lb) proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = - result = true var desc, link = "" var i = p.idx @@ -1642,14 +1665,21 @@ proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = parse("]", desc) if p.tok[i].symbol != "(": return false + let linkIdx = i + 1 parse(")", link) - let child = newRstNode(rnHyperlink) - child.add desc - child.add link # only commit if we detected no syntax error: - father.add child - p.idx = i - result = true + let protocol = safeProtocol(link) + if link == "": + result = false + rstMessage(p, mwBrokenLink, protocol, + p.tok[linkIdx].line, p.tok[linkIdx].col) + else: + let child = newRstNode(rnHyperlink) + child.add desc + child.add link + father.add child + p.idx = i + result = true proc getFootnoteType(label: PRstNode): (FootnoteType, int) = if label.sons.len >= 1 and label.sons[0].kind == rnLeaf and diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index d2180cb91..d662a667c 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -40,7 +40,7 @@ ## can be done by simply searching for [footnoteName]. import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, - algorithm, parseutils, std/strbasics, strscans + algorithm, parseutils, std/strbasics import ../../std/private/since @@ -826,17 +826,6 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = "\\rstov$4[$5]{$3}$2\n", [$n.level, rstnodeToRefname(n).idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) - -proc safeProtocol(linkStr: var string) = - var protocol = "" - if scanf(linkStr, "$w:", protocol): - # if it has a protocol at all, ensure that it's not 'javascript:' or worse: - if cmpIgnoreCase(protocol, "http") == 0 or cmpIgnoreCase(protocol, "https") == 0 or - cmpIgnoreCase(protocol, "ftp") == 0: - discard "it's fine" - else: - linkStr = "" - proc renderTocEntry(d: PDoc, e: TocEntry, result: var string) = dispA(d.target, result, "

            • $2
              • \n", @@ -901,7 +890,7 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) = # support for `:target:` links for images: var target = esc(d.target, getFieldValue(n, "target").strip(), escMode=emUrl) - safeProtocol(target) + discard safeProtocol(target) if target.len > 0: # `htmlOut` needs to be of the following format for link to work for images: @@ -1205,7 +1194,7 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, d.escMode = emUrl renderRstToOut(d, link, linkStr) d.escMode = mode - safeProtocol(linkStr) + discard safeProtocol(linkStr) var textStr = "" renderRstToOut(d, text, textStr) let nimDocStr = if nimdoc: " nimdoc" else: "" diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 771e02477..8b98d1403 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -843,12 +843,7 @@ suite "Warnings": rnInner rnLeaf 'foo' rnInner - rnLeaf '#' - rnLeaf 'foo' - rnLeaf ',' - rnLeaf 'string' - rnLeaf ',' - rnLeaf 'string' + rnLeaf '#foo,string,string' rnParagraph anchor='foo' rnLeaf 'Paragraph' rnLeaf '.' @@ -1256,3 +1251,23 @@ suite "RST inline markup": rnLeaf 'my {link example' rnLeaf 'http://example.com/bracket_(symbol_[)' """) + + test "not a Markdown link": + # bug #17340 (27) `f` will be considered as a protocol and blocked as unsafe + var warnings = new seq[string] + check("[T](f: var Foo)".toAst(warnings = warnings) == + dedent""" + rnInner + rnLeaf '[' + rnLeaf 'T' + rnLeaf ']' + rnLeaf '(' + rnLeaf 'f' + rnLeaf ':' + rnLeaf ' ' + rnLeaf 'var' + rnLeaf ' ' + rnLeaf 'Foo' + rnLeaf ')' + """) + check(warnings[] == @["input(1, 5) Warning: broken link 'f'"]) diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 995c0a151..2c28fe506 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -1593,8 +1593,15 @@ suite "invalid targets": test "invalid links": check("(([Nim](https://nim-lang.org/)))".toHtml == """((Nim))""") - check("(([Nim](javascript://nim-lang.org/)))".toHtml == - """((Nim))""") + # unknown protocol is treated just like plain text, not a link + var warnings = new seq[string] + check("(([Nim](javascript://nim-lang.org/)))".toHtml(warnings=warnings) == + """(([Nim](javascript://nim-lang.org/)))""") + check(warnings[] == @["input(1, 9) Warning: broken link 'javascript'"]) + warnings[].setLen 0 + check("`Nim `_".toHtml(warnings=warnings) == + """Nim <javascript://nim-lang.org/>""") + check(warnings[] == @["input(1, 33) Warning: broken link 'javascript'"]) suite "local file inclusion": test "cannot include files in sandboxed mode": From 27e548140b0f51a8dbc6f6094764e02f08ae509e Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 8 Feb 2022 15:34:44 +0800 Subject: [PATCH 0692/1654] don't use a temp for addr [backport: 1.6] (#19503) * don't use a temp for addr fix #19497 * Update compiler/ccgcalls.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * add a test Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/ccgcalls.nim | 4 ++-- tests/arc/tarc_orc.nim | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/arc/tarc_orc.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 197728ccf..ce5fbfdd7 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -376,8 +376,8 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope = if not needTmp[i - 1]: needTmp[i - 1] = potentialAlias(n, potentialWrites) getPotentialWrites(ri[i], false, potentialWrites) - if ri[i].kind == nkHiddenAddr: - # Optimization: don't use a temp, if we would only take the adress anyway + if ri[i].kind in {nkHiddenAddr, nkAddr}: + # Optimization: don't use a temp, if we would only take the address anyway needTmp[i - 1] = false for i in 1.. Date: Fri, 11 Feb 2022 18:16:27 +0800 Subject: [PATCH 0693/1654] fix #16458; make useNimRtl compile for --gc:orc (#19512) * fix #16458; make useNimRtl compile for --gc:orc/arc * fix tests --- lib/system/mmdisp.nim | 5 +++-- lib/system/orc.nim | 4 ++-- lib/system/seqs_v2.nim | 3 ++- tests/arc/t16458.nim | 6 ++++++ 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 tests/arc/t16458.nim diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 3317ba627..e5038387f 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -82,8 +82,9 @@ else: # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here include "system/gc_regions" elif defined(nimV2) or usesDestructors: - var allocator {.rtlThreadVar.}: MemRegion - instantiateForRegion(allocator) + when not defined(useNimRtl): + var allocator {.rtlThreadVar.}: MemRegion + instantiateForRegion(allocator) when defined(gcHooks): include "system/gc_hooks" elif defined(gcMarkAndSweep): diff --git a/lib/system/orc.nim b/lib/system/orc.nim index f9f5cd81f..4c23aea6c 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -119,7 +119,7 @@ template orcAssert(cond, msg) = when logOrc: proc strstr(s, sub: cstring): cstring {.header: "", importc.} -proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inline.} = +proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inl.} = let p = cast[ptr pointer](q) if p[] != nil: @@ -128,7 +128,7 @@ proc nimTraceRef(q: pointer; desc: PNimTypeV2; env: pointer) {.compilerRtl, inli var j = cast[ptr GcEnv](env) j.traceStack.add(p, desc) -proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inline.} = +proc nimTraceRefDyn(q: pointer; env: pointer) {.compilerRtl, inl.} = let p = cast[ptr pointer](q) if p[] != nil: var j = cast[ptr GcEnv](env) diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 0c487b31d..1194f40ef 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -75,7 +75,8 @@ proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): poin proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = when nimvm: - setLen(x, newLen) + {.cast(tags: []).}: + setLen(x, newLen) else: #sysAssert newLen <= x.len, "invalid newLen parameter for 'shrink'" when not supportsCopyMem(T): diff --git a/tests/arc/t16458.nim b/tests/arc/t16458.nim new file mode 100644 index 000000000..6ae114287 --- /dev/null +++ b/tests/arc/t16458.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--gc:orc --d:useNimRtl" + action: "compile" +""" + +echo 134 \ No newline at end of file From ed0dce7292480002be96d0ea5b7775c38767134d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 12 Feb 2022 15:10:45 +0100 Subject: [PATCH 0694/1654] fixes #19404 by protecting the memory we borrow from. this replaces crashes with minor memory leaks which seems to be acceptable. In the longer run we need a better VM that didn't grow hacks over a decade. (#19515) Co-authored-by: flywind --- compiler/vm.nim | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 3343eb781..259648add 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -85,9 +85,9 @@ proc bailOut(c: PCtx; tos: PStackFrame) = when not defined(nimComputedGoto): {.pragma: computedGoto.} -proc ensureKind(n: var TFullReg, kind: TRegisterKind) = - if n.kind != kind: - n = TFullReg(kind: kind) +proc ensureKind(n: var TFullReg, k: TRegisterKind) {.inline.} = + if n.kind != k: + n = TFullReg(kind: k) template ensureKind(k: untyped) {.dirty.} = ensureKind(regs[ra], k) @@ -521,6 +521,11 @@ template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool = when not defined(nimHasSinkInference): {.pragma: nosinks.} +template takeAddress(reg, source) = + reg.nodeAddr = addr source + when defined(gcDestructors): + GC_ref source + proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var pc = start var tos = tos @@ -679,7 +684,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let idx = regs[rc].intVal.int let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[] if src.kind notin {nkEmpty..nkTripleStrLit} and idx <% src.len: - regs[ra].nodeAddr = addr src.sons[idx] + takeAddress regs[ra], src.sons[idx] else: stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.safeLen-1)) of opcLdStrIdx: @@ -747,11 +752,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of nkObjConstr: let n = src.sons[rc + 1] if n.kind == nkExprColonExpr: - regs[ra].nodeAddr = addr n.sons[1] + takeAddress regs[ra], n.sons[1] else: - regs[ra].nodeAddr = addr src.sons[rc + 1] + takeAddress regs[ra], src.sons[rc + 1] else: - regs[ra].nodeAddr = addr src.sons[rc] + takeAddress regs[ra], src.sons[rc] of opcWrObj: # a.b = c decodeBC(rkNode) @@ -778,7 +783,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeB(rkNodeAddr) case regs[rb].kind of rkNode: - regs[ra].nodeAddr = addr(regs[rb].node) + takeAddress regs[ra], regs[rb].node of rkNodeAddr: # bug #14339 regs[ra].nodeAddr = regs[rb].nodeAddr else: From 551225d88808c6029d9246cf2bea5da349d1eb43 Mon Sep 17 00:00:00 2001 From: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Date: Mon, 14 Feb 2022 08:50:38 +0100 Subject: [PATCH 0695/1654] Remove backslash in glob pattern (#19524) --- lib/pure/os.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 76164bc49..7c0c64276 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2131,7 +2131,7 @@ iterator walkPattern*(pattern: string): string {.tags: [ReadDirEffect], noWeirdT ## Iterate over all the files and directories that match the `pattern`. ## ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` + ## `pattern` is OS dependent, but at least the `"*.ext"` ## notation is supported. ## ## See also: @@ -2150,7 +2150,7 @@ iterator walkFiles*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTar ## Iterate over all the files that match the `pattern`. ## ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` + ## `pattern` is OS dependent, but at least the `"*.ext"` ## notation is supported. ## ## See also: @@ -2167,7 +2167,7 @@ iterator walkDirs*(pattern: string): string {.tags: [ReadDirEffect], noWeirdTarg ## Iterate over all the directories that match the `pattern`. ## ## On POSIX this uses the `glob`:idx: call. - ## `pattern` is OS dependent, but at least the `"\*.ext"` + ## `pattern` is OS dependent, but at least the `"*.ext"` ## notation is supported. ## ## See also: From cda77c15209519ea04f889a826ac79c7cbb7e152 Mon Sep 17 00:00:00 2001 From: David Krause Date: Mon, 14 Feb 2022 08:56:35 +0100 Subject: [PATCH 0696/1654] use OrderedTable instead of OrderedTableRef for mimedb (#19522) * use OrderedTable instead of OrderedTableRef for mimedb Signed-off-by: David Krause * added changelog entry for mimedb change Signed-off-by: David Krause --- changelog.md | 2 ++ lib/pure/mimetypes.nim | 4 ++-- tests/stdlib/tmimetypes.nim | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 412e046eb..b21784bae 100644 --- a/changelog.md +++ b/changelog.md @@ -40,6 +40,8 @@ becomes an alias for `addr`. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. +- Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, diff --git a/lib/pure/mimetypes.nim b/lib/pure/mimetypes.nim index 6ae63a0ae..d1566d897 100644 --- a/lib/pure/mimetypes.nim +++ b/lib/pure/mimetypes.nim @@ -31,7 +31,7 @@ from strutils import startsWith, toLowerAscii, strip type MimeDB* = object - mimes: OrderedTableRef[string, string] + mimes: OrderedTable[string, string] const mimes* = { "123": "application/vnd.lotus-1-2-3", @@ -1904,7 +1904,7 @@ func newMimetypes*(): MimeDB = ## Creates a new Mimetypes database. The database will contain the most ## common mimetypes. {.cast(noSideEffect).}: - result.mimes = mimes.newOrderedTable() + result.mimes = mimes.toOrderedTable() func getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string = ## Gets mimetype which corresponds to `ext`. Returns `default` if `ext` diff --git a/tests/stdlib/tmimetypes.nim b/tests/stdlib/tmimetypes.nim index 93c20d4a3..6435309e1 100644 --- a/tests/stdlib/tmimetypes.nim +++ b/tests/stdlib/tmimetypes.nim @@ -7,6 +7,8 @@ template main() = var m = newMimetypes() doAssert m.getMimetype("mp4") == "video/mp4" doAssert m.getExt("application/json") == "json" + m.register("foo", "baa") + doAssert m.getMimetype("foo") == "baa" # see also `runnableExamples`. # xxx we should have a way to avoid duplicating code between runnableExamples and tests From 42eeab5fd181ffca6d1197b704a15f056939cfa8 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Mon, 14 Feb 2022 00:09:21 -0800 Subject: [PATCH 0697/1654] Remove Deprecated oids.oidsToString (#19519) * Remove deprecated oids.oidToString * Remove deprecated oids.oidToString --- changelog.md | 2 ++ lib/pure/oids.nim | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index b21784bae..ec13152d0 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,8 @@ becomes an alias for `addr`. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. +- Removed deprecated `oids.oidToString`. + - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index da285a187..776c046b1 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -67,11 +67,6 @@ template toStringImpl[T: string | cstring](result: var T, oid: Oid) = when T is cstring: result[N] = '\0' -proc oidToString*(oid: Oid, str: cstring) {.deprecated: "unsafe; use `$`".} = - ## Converts an oid to a string which must have space allocated for 25 elements. - # work around a compiler bug: - var str = str - toStringImpl(str, oid) proc `$`*(oid: Oid): string = ## Converts an OID to a string. From 8e8231f9d6e586a32af7b80892244dc1033db871 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Mon, 14 Feb 2022 05:35:33 -0800 Subject: [PATCH 0698/1654] Remove deprecated math.c_frexp (#19518) * Remove Deprecated math proc * Remove Deprecated math proc * Remove Deprecated math proc --- changelog.md | 3 +++ lib/pure/math.nim | 5 ----- tests/stdlib/tmath.nim | 6 ------ 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index ec13152d0..3fd66dba8 100644 --- a/changelog.md +++ b/changelog.md @@ -44,6 +44,9 @@ becomes an alias for `addr`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. +- Removed deprecated `math.c_frexp`. + + ## Language changes - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 66f1a2e2c..1c47258bc 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -69,11 +69,6 @@ when defined(c) or defined(cpp): proc c_signbit(x: SomeFloat): cint {.importc: "signbit", header: "".} - func c_frexp*(x: cfloat, exponent: var cint): cfloat {. - importc: "frexpf", header: "", deprecated: "Use `frexp` instead".} - func c_frexp*(x: cdouble, exponent: var cint): cdouble {. - importc: "frexp", header: "", deprecated: "Use `frexp` instead".} - # don't export `c_frexp` in the future and remove `c_frexp2`. func c_frexp2(x: cfloat, exponent: var cint): cfloat {. importc: "frexpf", header: "".} diff --git a/tests/stdlib/tmath.nim b/tests/stdlib/tmath.nim index 9bc5b994f..5e501c09b 100644 --- a/tests/stdlib/tmath.nim +++ b/tests/stdlib/tmath.nim @@ -437,12 +437,6 @@ template main() = doAssert lgamma(-0.0) == Inf doAssert lgamma(-1.0) == Inf - when nimvm: discard - else: - var exponent: cint - doAssert c_frexp(0.0, exponent) == 0.0 - doAssert c_frexp(-0.0, exponent) == -0.0 - doAssert classify(c_frexp(-0.0, exponent)) == fcNegZero static: main() main() From 2ef71c0a46b2209c906921f10f585306ce6199cc Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 14 Feb 2022 21:36:01 +0800 Subject: [PATCH 0699/1654] [testcase] genSym fails to make unique identifier for ref object types (#19506) close #15118 --- tests/macros/tmacros_various.nim | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/macros/tmacros_various.nim b/tests/macros/tmacros_various.nim index 00d74cb5c..d702db56a 100644 --- a/tests/macros/tmacros_various.nim +++ b/tests/macros/tmacros_various.nim @@ -23,6 +23,8 @@ x: some string ([("key", "val"), ("keyB", "2")], [("val", "key"), ("2", "keyB")]) ([("key", "val"), ("keyB", "2")], [("val", "key"), ("2", "keyB")]) 0 +0 +0 ''' """ @@ -270,3 +272,61 @@ block: # bug #13511 debugAst: add(foo(), component) + +block: # bug #15118 + macro flop(name: static string) = + let id = genSym(nskType, "env") + let r = + nnkStmtList.newTree( + nnkTypeSection.newTree( + nnkTypeDef.newTree( + id, + newEmptyNode(), + nnkRefTy.newTree( + nnkObjectTy.newTree( + newEmptyNode(), + newEmptyNode(), + nnkRecList.newTree( + nnkIdentDefs.newTree( + newIdentNode(name), + newIdentNode("int"), + newEmptyNode() + ) + ) + ) + ) + ) + ), + + # var f: T + + nnkVarSection.newTree( + nnkIdentDefs.newTree( + newIdentNode("f"), + id, + newEmptyNode() + ) + ), + + # echo f.a + nnkCommand.newTree( + newIdentNode("new"), + newIdentNode("f") + ), + + nnkCommand.newTree( + newIdentNode("echo"), + nnkDotExpr.newTree( + newIdentNode("f"), + newIdentNode(name) + ) + ) + ) + r + + + block: + flop("a") + + block: + flop("b") From b2c5d7b4ff070abe2145e998d090fb15b4df522f Mon Sep 17 00:00:00 2001 From: Sizhe Zhao Date: Tue, 15 Feb 2022 06:15:59 +0800 Subject: [PATCH 0700/1654] Documentation: Fix word usage (#19529) --- doc/lib.rst | 2 +- lib/pure/base64.nim | 4 ++-- lib/pure/hashes.nim | 2 +- lib/pure/md5.nim | 4 ++-- lib/std/sha1.nim | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/lib.rst b/doc/lib.rst index 881d857d3..f72012777 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -446,7 +446,7 @@ Hashing from the MongoDB interface and it thus binary compatible with a MongoDB OID. * `sha1 `_ - This module implements a sha1 encoder and decoder. + This module implements the SHA-1 checksum algorithm. Miscellaneous diff --git a/lib/pure/base64.nim b/lib/pure/base64.nim index 513ec5826..9b7af88ef 100644 --- a/lib/pure/base64.nim +++ b/lib/pure/base64.nim @@ -56,8 +56,8 @@ runnableExamples: ## ======== ## ## * `hashes module`_ for efficient computations of hash values for diverse Nim types -## * `md5 module`_ implements the MD5 checksum algorithm -## * `sha1 module`_ implements a sha1 encoder and decoder +## * `md5 module`_ for the MD5 checksum algorithm +## * `sha1 module`_ for the SHA-1 checksum algorithm template cbBase(a, b): untyped = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index d9f534670..47dacec7d 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -62,7 +62,7 @@ runnableExamples: ## ======== ## * `md5 module `_ for the MD5 checksum algorithm ## * `base64 module `_ for a Base64 encoder and decoder -## * `std/sha1 module `_ for a SHA-1 encoder and decoder +## * `std/sha1 module `_ for the SHA-1 checksum algorithm ## * `tables module `_ for hash tables import std/private/since diff --git a/lib/pure/md5.nim b/lib/pure/md5.nim index 06c5608a4..cd4d1e6b8 100644 --- a/lib/pure/md5.nim +++ b/lib/pure/md5.nim @@ -13,8 +13,8 @@ ## ## See also ## ======== -## * `base64 module`_ implements a Base64 encoder and decoder -## * `std/sha1 module `_ for a SHA-1 encoder and decoder +## * `base64 module`_ for a Base64 encoder and decoder +## * `std/sha1 module `_ for the SHA-1 checksum algorithm ## * `hashes module`_ for efficient computations of hash values ## for diverse Nim types diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index 120a81702..ed962707b 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -12,9 +12,9 @@ ## ## See also ## ======== -## * `base64 module`_ implements a Base64 encoder and decoder +## * `base64 module`_ for a Base64 encoder and decoder ## * `hashes module`_ for efficient computations of hash values for diverse Nim types -## * `md5 module`_ implements the MD5 checksum algorithm +## * `md5 module`_ for the MD5 checksum algorithm runnableExamples: let accessName = secureHash("John Doe") From 30e9f23ec04618616dcf2aaa7980794bad2135ed Mon Sep 17 00:00:00 2001 From: rockcavera Date: Thu, 17 Feb 2022 08:48:57 -0300 Subject: [PATCH 0701/1654] Update chcks.nim (#19540) --- lib/system/chcks.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 8c7c95795..a404e9d40 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -29,7 +29,7 @@ proc raiseFieldError(f: string) {.compilerproc, noinline.} = ## remove after bootstrap > 1.5.1 sysFatal(FieldDefect, f) -when defined(gcdestructors): +when defined(nimV2): proc raiseFieldError2(f: string, discVal: int) {.compilerproc, noinline.} = ## raised when field is inaccessible given runtime value of discriminant sysFatal(FieldError, f & $discVal & "'") From f019e999d491ecf462421bd750aec0465d308094 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 18 Feb 2022 15:06:11 +0000 Subject: [PATCH 0702/1654] keep casing of noinit and noreturn pragmas consistently documented (#19535) --- doc/manual.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 2469a0525..e45133c38 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2818,7 +2818,7 @@ The implicit initialization can be avoided for optimization reasons with the .. code-block:: nim var - a {.noInit.}: array[0..1023, char] + a {.noinit.}: array[0..1023, char] If a proc is annotated with the `noinit` pragma, this refers to its implicit `result` variable: @@ -6636,7 +6636,7 @@ but accessed at runtime: doAssert nameToProc[2][1]() == "baz" -noReturn pragma +noreturn pragma --------------- The `noreturn` pragma is used to mark a proc that never returns. From 516db3bac389ea554166ee1a6671b5005159a30a Mon Sep 17 00:00:00 2001 From: locriacyber <74560659+locriacyber@users.noreply.github.com> Date: Fri, 18 Feb 2022 22:24:42 +0000 Subject: [PATCH 0703/1654] Remove trailing space in `nim r` command; Amend one error message (#19487) --- compiler/nim.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/nim.nim b/compiler/nim.nim index b8256d576..86c7ab524 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -116,7 +116,8 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if cmdPrefix.len > 0: cmdPrefix.add " " # without the `cmdPrefix.len > 0` check, on windows you'd get a cryptic: # `The parameter is incorrect` - execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments) + let cmd = cmdPrefix & output.quoteShell & ' ' & conf.arguments + execExternalProgram(conf, cmd.strip(leading=false,trailing=true)) of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing) if conf.arguments.len > 0: # reserved for future use From 0c915b5e47e554d657f6533f262cebbcf1554e8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 24 Feb 2022 08:31:40 +0100 Subject: [PATCH 0704/1654] compile pragma: cache the result sooner (#19554) extccomp.addExternalFileToCompile() relies on hashes to decide whether an external C file needs recompilation or not. Due to short-circuit evaluation of boolean expressions, the procedure that generates a corresponding hash file is not called the first time an external file is compiled, so an avoidable recompilation is triggered the next build. This patch fixes that by moving the proc call with a desired side effect from its boolean expression, so it's executed unconditionally. --- compiler/extccomp.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e7e1bcd51..124cc2c34 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -649,8 +649,10 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = close(f) proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) = + # we want to generate the hash file unconditionally + let extFileChanged = externalFileChanged(conf, c) if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and - not externalFileChanged(conf, c): + not extFileChanged: c.flags.incl CfileFlag.Cached else: # make sure Nim keeps recompiling the external file on reruns From 340b5a1676062aa9bc7f17a2e078eaba96821bf3 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 25 Feb 2022 07:16:46 +1100 Subject: [PATCH 0705/1654] Document links repect --out in .idx file (#19543) * Link in .idx file now respects --out:file switch * Use clearer chained calls Co-authored-by: Clay Sweetser --- compiler/docgen.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 3e61032d8..aad3b9409 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1499,7 +1499,10 @@ proc genOutFile(d: PDoc, groupedToc = false): string = # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: title = d.meta[metaTitle] - let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnixPath + let external = AbsoluteFile(d.destFile) + .relativeTo(d.conf.outDir, '/') + .changeFileExt(HtmlExt) + .string setIndexTerm(d[], external, "", title) else: # Modules get an automatic title for the HTML, but no entry in the index. From 9a49451124a781a2cd032b6e7616c67a7815fe57 Mon Sep 17 00:00:00 2001 From: PMunch Date: Fri, 25 Feb 2022 10:43:03 +0100 Subject: [PATCH 0706/1654] Remove volatiles when compiling with ARC/ORC (#19545) This removes volatiles on ARC/ORC targets in NimMain and PreMainInner. This avoids an issue where they couldn't be optimised out on microcontrollers leading to larger code. Since the stack bottom doesn't have to be initialised this way when using ARC or ORC (or None, which is also covered by this PR) these can be safely removed. --- compiler/cgen.nim | 68 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 10b4b55d0..b78ad10c4 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1363,16 +1363,25 @@ proc genMainProc(m: BModule) = # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. - PreMainBody = "$N" & + + PreMainVolatileBody = + "\tvoid (*volatile inner)(void);$N" & + "\tinner = PreMainInner;$N" & + "$1" & + "\t(*inner)();$N" + + PreMainNonVolatileBody = + "$1" & + "\tPreMainInner();$N" + + PreMainBodyStart = "$N" & "N_LIB_PRIVATE void PreMainInner(void) {$N" & "$2" & "}$N$N" & PosixCmdLine & - "N_LIB_PRIVATE void PreMain(void) {$N" & - "\tvoid (*volatile inner)(void);$N" & - "\tinner = PreMainInner;$N" & - "$1" & - "\t(*inner)();$N" & + "N_LIB_PRIVATE void PreMain(void) {$N" + + PreMainBodyEnd = "}$N$N" MainProcs = @@ -1385,17 +1394,32 @@ proc genMainProc(m: BModule) = "$1" & "}$N$N" - NimMainProc = - "N_CDECL(void, $5NimMain)(void) {$N" & - "\tvoid (*volatile inner)(void);$N" & - "$4" & - "\tinner = NimMainInner;$N" & - "$2" & - "\t(*inner)();$N" & + NimMainVolatileBody = + "\tvoid (*volatile inner)(void);$N" & + "$4" & + "\tinner = NimMainInner;$N" & + "$2" & + "\t(*inner)();$N" + + NimMainNonVolatileBody = + "$4" & + "$2" & + "\tNimMainInner();$N" + + NimMainProcStart = + "N_CDECL(void, $5NimMain)(void) {$N" + + NimMainProcEnd = "}$N$N" + NimMainProc = NimMainProcStart & NimMainVolatileBody & NimMainProcEnd + + NimSlimMainProc = NimMainProcStart & NimMainNonVolatileBody & NimMainProcEnd + NimMainBody = NimMainInner & NimMainProc + NimSlimMainBody = NimMainInner & NimSlimMainProc + PosixCMain = "int main(int argc, char** args, char** env) {$N" & "\tcmdLine = args;$N" & @@ -1456,10 +1480,13 @@ proc genMainProc(m: BModule) = m.includeHeader("") let initStackBottomCall = - if m.config.target.targetOS == osStandalone or m.config.selectedGC == gcNone: "".rope + if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcOrc}: "".rope else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []) inc(m.labels) - appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit]) + if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) + else: + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) if m.config.target.targetOS == osWindows and m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: @@ -1484,9 +1511,14 @@ proc genMainProc(m: BModule) = appcg(m, m.s[cfsProcs], nimMain, [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) else: - const nimMain = NimMainBody - appcg(m, m.s[cfsProcs], nimMain, - [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) + if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: + const nimMain = NimMainBody + appcg(m, m.s[cfsProcs], nimMain, + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) + else: + const nimMain = NimSlimMainBody + appcg(m, m.s[cfsProcs], nimMain, + [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix]) if optNoMain notin m.config.globalOptions: if m.config.cppCustomNamespace.len > 0: From fe791c67b4c028d2e439e6f74e0a4225ba6cc2f1 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 25 Feb 2022 01:43:35 -0800 Subject: [PATCH 0707/1654] Remove Deprecated jsre func (#19551) * Remove Deprecated jsre func * Remove Deprecated jsre func * ReStart CI, because OSX is dumb and I have no permission to restart --- changelog.md | 2 +- lib/js/jsre.nim | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 3fd66dba8..cbad9b4e6 100644 --- a/changelog.md +++ b/changelog.md @@ -43,7 +43,7 @@ becomes an alias for `addr`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - +- Removed deprecated `jsre.test` and `jsre.toString`. - Removed deprecated `math.c_frexp`. diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index a23fccaa8..cd8fb2be7 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -47,10 +47,6 @@ func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} func `$`*(self: RegExp): string = $toCstring(self) -func test*(self: RegExp; pattern: cstring): bool {.importjs: "#.test(#)", deprecated: "Use contains instead".} - -func toString*(self: RegExp): cstring {.importjs: "#.toString()", deprecated: "Use toCstring instead".} - func contains*(pattern: cstring; self: RegExp): bool = ## Tests for a substring match in its string parameter. runnableExamples: From ef3f343ec2979d22a5d4e51f4328e4a4c21d68dc Mon Sep 17 00:00:00 2001 From: Tanguy Date: Fri, 25 Feb 2022 12:57:58 +0100 Subject: [PATCH 0708/1654] Allow std/macros.params to work with nnkProcTy (#19563) * Allow std/macros.params to work with nnkProcTy * Add tests for proc params & pragma --- lib/core/macros.nim | 15 +++++++++++---- tests/macros/tprochelpers.nim | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/macros/tprochelpers.nim diff --git a/lib/core/macros.nim b/lib/core/macros.nim index c0e6e5154..4d040cebc 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1271,12 +1271,19 @@ proc `name=`*(someProc: NimNode; val: NimNode) = else: someProc[0] = val proc params*(someProc: NimNode): NimNode = - someProc.expectRoutine - result = someProc[3] + if someProc.kind == nnkProcTy: + someProc[0] + else: + someProc.expectRoutine + someProc[3] + proc `params=`* (someProc: NimNode; params: NimNode) = - someProc.expectRoutine expectKind(params, nnkFormalParams) - someProc[3] = params + if someProc.kind == nnkProcTy: + someProc[0] = params + else: + someProc.expectRoutine + someProc[3] = params proc pragma*(someProc: NimNode): NimNode = ## Get the pragma of a proc type. diff --git a/tests/macros/tprochelpers.nim b/tests/macros/tprochelpers.nim new file mode 100644 index 000000000..d95a2ced8 --- /dev/null +++ b/tests/macros/tprochelpers.nim @@ -0,0 +1,22 @@ +import std/macros +import stdtest/testutils + +macro test1(prc: untyped): untyped = + assertAll: + prc.params.len == 2 + prc.params[1].len == 4 + prc.pragma.len == 2 + + prc.params = nnkFormalParams.newTree( + ident("int") + ) + prc.pragma = newEmptyNode() + + assertAll: + prc.params.len == 1 + prc.pragma.len == 0 + prc + +proc test(a, b: int): int {.gcsafe, raises: [], test1.} = 5 + +type hello = proc(a, b: int): int {.gcsafe, raises: [], test1.} From 9c17a32e0ec45de83c21d84f98be7f351ccd1172 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 26 Feb 2022 03:33:28 +0800 Subject: [PATCH 0709/1654] fix #19266; allow reassign discriminant field (#19567) * add inUncheckedAssignSection * add one more test --- compiler/ccgexprs.nim | 16 +++++++++++++++- compiler/ccgstmts.nim | 2 +- compiler/cgendata.nim | 1 + tests/objvariant/treassign.nim | 22 ++++++++++++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1f1035115..9848366a1 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2932,7 +2932,21 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt: discard of nkPragma: genPragma(p, n) - of nkPragmaBlock: expr(p, n.lastSon, d) + of nkPragmaBlock: + var inUncheckedAssignSection = 0 + let pragmaList = n[0] + for pi in pragmaList: + if whichPragma(pi) == wCast: + case whichPragma(pi[1]) + of wUncheckedAssign: + inUncheckedAssignSection = 1 + else: + discard + + inc p.inUncheckedAssignSection, inUncheckedAssignSection + expr(p, n.lastSon, d) + dec p.inUncheckedAssignSection, inUncheckedAssignSection + of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef: if n[genericParamsPos].kind == nkEmpty: var prc = n[namePos].sym diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 58a6dd015..240fa55c8 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1566,7 +1566,7 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) = initLocExpr(p, e[0], a) getTemp(p, a.t, tmp) expr(p, e[1], tmp) - if optTinyRtti notin p.config.globalOptions: + if optTinyRtti notin p.config.globalOptions and p.inUncheckedAssignSection == 0: let field = dotExpr[1].sym genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field) message(p.config, e.info, warnCaseTransition) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 88b8d4090..449076725 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -99,6 +99,7 @@ type withinTryWithExcept*: int # required for goto based exception handling withinBlockLeaveActions*: int # complex to explain sigConflicts*: CountTable[string] + inUncheckedAssignSection*: int TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] diff --git a/tests/objvariant/treassign.nim b/tests/objvariant/treassign.nim index 9549cb29c..527204616 100644 --- a/tests/objvariant/treassign.nim +++ b/tests/objvariant/treassign.nim @@ -34,3 +34,25 @@ proc passToVar(x: var Token) = discard t.curr = TokenObject(kind: t.curr.kind, foo: "abc") t.curr.kind = Token.foo + + +block: + type + TokenKind = enum + strLit, intLit + Token = object + case kind*: TokenKind + of strLit: + s*: string + of intLit: + i*: int64 + + var t = Token(kind: strLit, s: "abc") + + {.cast(uncheckedAssign).}: + + # inside the 'cast' section it is allowed to assign to the 't.kind' field directly: + t.kind = intLit + + {.cast(uncheckedAssign).}: + t.kind = strLit \ No newline at end of file From f0bfc0bd3f619c0933df5b71a0a04d4181d99880 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 25 Feb 2022 11:34:16 -0800 Subject: [PATCH 0710/1654] Remove define for jsfetch (#19530) * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch * Remove define nimExperimentalAsyncjsThen for std/asyncjs.then and std/jsfetch --- changelog.md | 2 +- lib/js/asyncjs.nim | 148 ++++++++++++++++++++++---------------------- lib/std/jsfetch.nim | 15 +++-- tools/kochdocs.nim | 2 +- 4 files changed, 83 insertions(+), 84 deletions(-) diff --git a/changelog.md b/changelog.md index cbad9b4e6..011f95ddb 100644 --- a/changelog.md +++ b/changelog.md @@ -40,7 +40,7 @@ becomes an alias for `addr`. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - Removed deprecated `oids.oidToString`. - +- Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - Removed deprecated `jsre.test` and `jsre.toString`. diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 861fe3fe2..8f91e1acd 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -64,6 +64,7 @@ when not defined(js) and not defined(nimsuggest): import std/jsffi import std/macros +import std/private/since type Future*[T] = ref object @@ -163,92 +164,91 @@ template maybeFuture(T): untyped = when T is Future: T else: Future[T] -when defined(nimExperimentalAsyncjsThen): - import std/private/since - since (1, 5, 1): - #[ - TODO: - * map `Promise.all()` - * proc toString*(a: Error): cstring {.importjs: "#.toString()".} - Note: - We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible - in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options - and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho - ]# +since (1, 5, 1): + #[ + TODO: + * map `Promise.all()` + * proc toString*(a: Error): cstring {.importjs: "#.toString()".} - type Error* {.importjs: "Error".} = ref object of JsRoot - ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error - message*: cstring - name*: cstring + Note: + We probably can't have a `waitFor` in js in browser (single threaded), but maybe it would be possible + in in nodejs, see https://nodejs.org/api/child_process.html#child_process_child_process_execsync_command_options + and https://stackoverflow.com/questions/61377358/javascript-wait-for-async-call-to-finish-before-returning-from-function-witho + ]# - type OnReject* = proc(reason: Error) + type Error* {.importjs: "Error".} = ref object of JsRoot + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error + message*: cstring + name*: cstring - proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto = - ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then - ## Returns a `Future` from the return type of `onSuccess(T.default)`. - runnableExamples("-d:nimExperimentalAsyncjsThen"): - from std/sugar import `=>` + type OnReject* = proc(reason: Error) - proc fn(n: int): Future[int] {.async.} = - if n >= 7: raise newException(ValueError, "foobar: " & $n) - else: result = n * 2 + proc then*[T](future: Future[T], onSuccess: proc, onReject: OnReject = nil): auto = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then + ## Returns a `Future` from the return type of `onSuccess(T.default)`. + runnableExamples("-r:off"): + from std/sugar import `=>` - proc asyncFact(n: int): Future[int] {.async.} = - if n > 0: result = n * await asyncFact(n-1) - else: result = 1 + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 - proc main() {.async.} = - block: # then - assert asyncFact(3).await == 3*2 - assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2 - let x1 = await fn(3) - assert x1 == 3 * 2 - let x2 = await fn(4) - .then((a: int) => a.float) - .then((a: float) => $a) - assert x2 == "8.0" + proc asyncFact(n: int): Future[int] {.async.} = + if n > 0: result = n * await asyncFact(n-1) + else: result = 1 - block: # then with `onReject` callback - var witness = 1 - await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) - assert witness == 2 - await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) - assert witness == 3 + proc main() {.async.} = + block: # then + assert asyncFact(3).await == 3*2 + assert asyncFact(3).then(asyncFact).await == 6*5*4*3*2 + let x1 = await fn(3) + assert x1 == 3 * 2 + let x2 = await fn(4) + .then((a: int) => a.float) + .then((a: float) => $a) + assert x2 == "8.0" - template impl(call): untyped = - # see D20210421T014713 - when typeof(block: call) is void: - var ret: Future[void] - else: - var ret = default(maybeFuture(typeof(call))) - typeof(ret) - when T is void: - type A = impl(onSuccess()) + block: # then with `onReject` callback + var witness = 1 + await fn(6).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 2 + await fn(7).then((a: int) => (witness = 2), (r: Error) => (witness = 3)) + assert witness == 3 + + template impl(call): untyped = + # see D20210421T014713 + when typeof(block: call) is void: + var ret: Future[void] else: - type A = impl(onSuccess(default(T))) - var ret: A - asm "`ret` = `future`.then(`onSuccess`, `onReject`)" - return ret + var ret = default(maybeFuture(typeof(call))) + typeof(ret) + when T is void: + type A = impl(onSuccess()) + else: + type A = impl(onSuccess(default(T))) + var ret: A + asm "`ret` = `future`.then(`onSuccess`, `onReject`)" + return ret - proc catch*[T](future: Future[T], onReject: OnReject): Future[void] = - ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch - runnableExamples("-d:nimExperimentalAsyncjsThen"): - from std/sugar import `=>` - from std/strutils import contains + proc catch*[T](future: Future[T], onReject: OnReject): Future[void] = + ## See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/catch + runnableExamples("-r:off"): + from std/sugar import `=>` + from std/strutils import contains - proc fn(n: int): Future[int] {.async.} = - if n >= 7: raise newException(ValueError, "foobar: " & $n) - else: result = n * 2 + proc fn(n: int): Future[int] {.async.} = + if n >= 7: raise newException(ValueError, "foobar: " & $n) + else: result = n * 2 - proc main() {.async.} = - var reason: Error - await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work - assert reason == nil - await fn(7).catch((r: Error) => (reason = r)) - assert reason != nil - assert "foobar: 7" in $reason.message + proc main() {.async.} = + var reason: Error + await fn(6).catch((r: Error) => (reason = r)) # note: `()` are needed, `=> reason = r` would not work + assert reason == nil + await fn(7).catch((r: Error) => (reason = r)) + assert reason != nil + assert "foobar: 7" in $reason.message - discard main() + discard main() - asm "`result` = `future`.catch(`onReject`)" + asm "`result` = `future`.catch(`onReject`)" diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 808d699b0..034bb6332 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -190,12 +190,11 @@ runnableExamples("-r:off"): discard example() - when defined(nimExperimentalAsyncjsThen): - block: - proc example2 {.async.} = - await fetch("https://api.github.com/users/torvalds".cstring) - .then((response: Response) => response.json()) - .then((json: JsObject) => console.log(json)) - .catch((err: Error) => console.log("Request Failed", err)) + block: + proc example2 {.async.} = + await fetch("https://api.github.com/users/torvalds".cstring) + .then((response: Response) => response.json()) + .then((json: JsObject) => console.log(json)) + .catch((err: Error) => console.log("Request Failed", err)) - discard example2() + discard example2() diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 8fd9c84a5..764146cf5 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -11,7 +11,7 @@ const gaCode* = " --doc.googleAnalytics:UA-48159761-1" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - docDefines = "-d:nimExperimentalAsyncjsThen -d:nimExperimentalLinenoiseExtra" + docDefines = "-d:nimExperimentalLinenoiseExtra" nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" From c2c9fbd7458db7adb515aad30cfe65b98fba47bf Mon Sep 17 00:00:00 2001 From: solarizedalias <60473183+solarizedalias@users.noreply.github.com> Date: Sun, 27 Feb 2022 16:22:52 +0900 Subject: [PATCH 0711/1654] make compileOption("panics") work (#19568) --- compiler/commands.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/commands.nim b/compiler/commands.nim index 6d3f23197..8e1d96860 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -332,6 +332,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool result = contains(conf.options, optTrMacros) of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace) of "nilseqs", "nilchecks", "taintmode": warningOptionNoop(switch) + of "panics": result = contains(conf.globalOptions, optPanics) else: invalidCmdLineOption(conf, passCmd1, switch, info) proc processPath(conf: ConfigRef; path: string, info: TLineInfo, From c870ff5946731dd1deff828abfe07c1900a5b364 Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Sun, 27 Feb 2022 23:36:16 +0900 Subject: [PATCH 0712/1654] Add error message to pointer size check assert in lib/nimbase.h (#19549) --- lib/nimbase.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nimbase.h b/lib/nimbase.h index a83bd3006..e4dec0dd2 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -547,7 +547,7 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); } #endif // Test to see if Nim and the C compiler agree on the size of a pointer. -NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, ""); +NIM_STATIC_ASSERT(sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8, "Pointer size mismatch between Nim and C/C++ backend. You probably need to setup the backend compiler for target CPU."); #ifdef USE_NIM_NAMESPACE } From 207237cec229171eafbd3357dcf128bdd270332a Mon Sep 17 00:00:00 2001 From: Evan Typanski Date: Mon, 28 Feb 2022 07:18:41 -0500 Subject: [PATCH 0713/1654] Fix #18662 (#19534) --- compiler/jsgen.nim | 4 +++- compiler/ropes.nim | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 0a7036b46..f4097efa4 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2796,7 +2796,9 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = var map: SourceMap (code, map) = genSourceMap($(code), outFile.string) writeFile(outFile.string & ".map", $(%map)) - discard writeRopeIfNotEqual(code, outFile) + if not equalsFile(code, outFile): + if not writeRope(code, outFile): + rawMessage(m.config, errCannotOpenFile, outFile.string) proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = diff --git a/compiler/ropes.nim b/compiler/ropes.nim index 1e7957337..cd696a545 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -329,10 +329,3 @@ proc equalsFile*(r: Rope, filename: AbsoluteFile): bool = if result: result = equalsFile(r, f) close(f) - -proc writeRopeIfNotEqual*(r: Rope, filename: AbsoluteFile): bool = - # returns true if overwritten - if not equalsFile(r, filename): - result = writeRope(r, filename) - else: - result = false From d6d36093b18dfa1e2b8b6325016f1def015250b4 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 1 Mar 2022 14:46:08 +0800 Subject: [PATCH 0714/1654] apply changes from #18017 and some fixes (#19571) * implements https://github.com/nim-lang/RFCs/issues/369 * deprecate unsafeAddr; extend addr addr is now available for all addressable locations, unsafeAddr is deprecated and become an alias for addr * follow @Vindaar's advice * change the signature of addr * unsafeAddr => addr (stdlib) * Update changelog.md * unsafeAddr => addr (tests) * Revert "unsafeAddr => addr (stdlib)" This reverts commit ab83c99c507048a8396e636bf22d55fdd84d7d1c. * doc changes; thanks to @konsumlamm Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * merge * remove * fix bug Co-authored-by: Araq Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- compiler/parampatterns.nim | 60 ++++++++++++++++++++------------------ compiler/semexprs.nim | 14 +++++---- compiler/semmagic.nim | 6 ++-- lib/system.nim | 3 -- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index bfbc72675..cceb236ae 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -179,6 +179,7 @@ type arLocalLValue, # is an l-value, but local var; must not escape # its stack frame! arDiscriminant, # is a discriminant + arAddressableConst, # an addressable const arLentValue, # lent value arStrange # it is a strange beast like 'typedesc[var T]' @@ -212,7 +213,7 @@ proc exprRoot*(n: PNode): PSym = else: break -proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult = +proc isAssignable*(owner: PSym, n: PNode): TAssignableResult = ## 'owner' can be nil! result = arNone case n.kind @@ -220,20 +221,20 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult if n.typ != nil and n.typ.kind in {tyVar}: result = arLValue of nkSym: - let kinds = if isUnsafeAddr: {skVar, skResult, skTemp, skParam, skLet, skForVar} - else: {skVar, skResult, skTemp} - if n.sym.kind == skParam and n.sym.typ.kind in {tyVar, tySink}: - result = arLValue - elif isUnsafeAddr and n.sym.kind == skParam: - result = arLValue - elif isUnsafeAddr and n.sym.kind == skConst and dontInlineConstant(n, n.sym.ast): - result = arLValue + const kinds = {skVar, skResult, skTemp, skParam, skLet, skForVar} + if n.sym.kind == skParam: + result = if n.sym.typ.kind in {tyVar, tySink}: arLValue else: arAddressableConst + elif n.sym.kind == skConst and dontInlineConstant(n, n.sym.ast): + result = arAddressableConst elif n.sym.kind in kinds: - if owner != nil and owner == n.sym.owner and - sfGlobal notin n.sym.flags: - result = arLocalLValue + if n.sym.kind in {skParam, skLet, skForVar}: + result = arAddressableConst else: - result = arLValue + if owner != nil and owner == n.sym.owner and + sfGlobal notin n.sym.flags: + result = arLocalLValue + else: + result = arLValue elif n.sym.kind == skType: let t = n.sym.typ.skipTypes({tyTypeDesc}) if t.kind in {tyVar}: result = arStrange @@ -241,10 +242,10 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult let t = skipTypes(n[0].typ, abstractInst-{tyTypeDesc}) if t.kind in {tyVar, tySink, tyPtr, tyRef}: result = arLValue - elif isUnsafeAddr and t.kind == tyLent: - result = arLValue + elif t.kind == tyLent: + result = arAddressableConst else: - result = isAssignable(owner, n[0], isUnsafeAddr) + result = isAssignable(owner, n[0]) if result != arNone and n[1].kind == nkSym and sfDiscriminant in n[1].sym.flags: result = arDiscriminant @@ -252,23 +253,23 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult let t = skipTypes(n[0].typ, abstractInst-{tyTypeDesc}) if t.kind in {tyVar, tySink, tyPtr, tyRef}: result = arLValue - elif isUnsafeAddr and t.kind == tyLent: - result = arLValue + elif t.kind == tyLent: + result = arAddressableConst else: - result = isAssignable(owner, n[0], isUnsafeAddr) + result = isAssignable(owner, n[0]) of nkHiddenStdConv, nkHiddenSubConv, nkConv: # Object and tuple conversions are still addressable, so we skip them # XXX why is 'tyOpenArray' allowed here? if skipTypes(n.typ, abstractPtrs-{tyTypeDesc}).kind in {tyOpenArray, tyTuple, tyObject}: - result = isAssignable(owner, n[1], isUnsafeAddr) + result = isAssignable(owner, n[1]) elif compareTypes(n.typ, n[1].typ, dcEqIgnoreDistinct): # types that are equal modulo distinction preserve l-value: - result = isAssignable(owner, n[1], isUnsafeAddr) + result = isAssignable(owner, n[1]) of nkHiddenDeref: let n0 = n[0] if n0.typ.kind == tyLent: - if isUnsafeAddr or (n0.kind == nkSym and n0.sym.kind == skResult): + if n0.kind == nkSym and n0.sym.kind == skResult: result = arLValue else: result = arLentValue @@ -277,18 +278,19 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult of nkDerefExpr, nkHiddenAddr: result = arLValue of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr: - result = isAssignable(owner, n[0], isUnsafeAddr) + result = isAssignable(owner, n[0]) of nkCallKinds: # builtin slice keeps lvalue-ness: if getMagic(n) in {mArrGet, mSlice}: - result = isAssignable(owner, n[1], isUnsafeAddr) - elif n.typ != nil and n.typ.kind in {tyVar}: - result = arLValue - elif isUnsafeAddr and n.typ != nil and n.typ.kind == tyLent: - result = arLValue + result = isAssignable(owner, n[1]) + elif n.typ != nil: + case n.typ.kind + of tyVar: result = arLValue + of tyLent: result = arLentValue + else: discard of nkStmtList, nkStmtListExpr: if n.typ != nil: - result = isAssignable(owner, n.lastSon, isUnsafeAddr) + result = isAssignable(owner, n.lastSon) of nkVarTy: # XXX: The fact that this is here is a bit of a hack. # The goal is to allow the use of checks such as "foo(var T)" diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index c593efe55..5f3d065b6 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -650,8 +650,8 @@ proc fixAbstractType(c: PContext, n: PNode) = changeType(c, it[1], s, check=true) n[i] = it[1] -proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult = - result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr) +proc isAssignable(c: PContext, n: PNode): TAssignableResult = + result = parampatterns.isAssignable(c.p.owner, n) proc isUnresolvedSym(s: PSym): bool = result = s.kind == skGenericParam @@ -1658,9 +1658,11 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode = # `proc fun(a: var int): var int = a` discard else: discard - let valid = isAssignable(c, n, isLent) + let valid = isAssignable(c, n) if valid != arLValue: - if valid == arLocalLValue: + if valid in {arAddressableConst, arLentValue} and isLent: + discard "ok" + elif valid == arLocalLValue: localError(c.config, n.info, errXStackEscape % renderTree(n, {renderNoComments})) else: localError(c.config, n.info, errExprHasNoAddress) @@ -1784,7 +1786,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = if le == nil: localError(c.config, a.info, "expression has no type") elif (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind notin {tyVar} and - isAssignable(c, a) in {arNone, arLentValue}) or ( + isAssignable(c, a) in {arNone, arLentValue, arAddressableConst}) or ( skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs} and views notin c.features): # Direct assignment to a discriminant is allowed! localError(c.config, a.info, errXCannotBeAssignedTo % @@ -2267,7 +2269,7 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = markUsed(c, n.info, s) checkSonsLen(n, 2, c.config) result[0] = newSymNode(s, n[0].info) - result[1] = semAddrArg(c, n[1], s.name.s == "unsafeAddr") + result[1] = semAddrArg(c, n[1]) result.typ = makePtrType(c, result[1].typ) of mTypeOf: markUsed(c, n.info, s) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index c7fc75620..ed1826fd4 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -10,11 +10,11 @@ # This include file implements the semantic checking for magics. # included from sem.nim -proc semAddrArg(c: PContext; n: PNode; isUnsafeAddr = false): PNode = +proc semAddrArg(c: PContext; n: PNode): PNode = let x = semExprWithType(c, n) if x.kind == nkSym: x.sym.flags.incl(sfAddrTaken) - if isAssignable(c, x, true) notin {arLValue, arLocalLValue}: + if isAssignable(c, x) notin {arLValue, arLocalLValue, arAddressableConst, arLentValue}: localError(c.config, n.info, errExprHasNoAddress) result = x @@ -466,7 +466,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, of mAddr: checkSonsLen(n, 2, c.config) result = n - result[1] = semAddrArg(c, n[1], n[0].sym.name.s == "unsafeAddr") + result[1] = semAddrArg(c, n[1]) result.typ = makePtrType(c, result[1].typ) of mTypeOf: result = semTypeOf(c, n) diff --git a/lib/system.nim b/lib/system.nim index 56a6e2887..572768de2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -202,9 +202,6 @@ proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## ## Cannot be overloaded. ## - ## See also: - ## * `unsafeAddr <#unsafeAddr,T>`_ - ## ## .. code-block:: Nim ## var ## buf: seq[char] = @['a','b','c'] From 590d39785243afb6da3662cf77c842a5ff4e1412 Mon Sep 17 00:00:00 2001 From: huantian Date: Wed, 2 Mar 2022 05:53:59 +0000 Subject: [PATCH 0715/1654] Fix typo in Nim Tutorial pt3 (#19579) --- doc/tut3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut3.rst b/doc/tut3.rst index 6b5a3b5ca..1c497f1f5 100644 --- a/doc/tut3.rst +++ b/doc/tut3.rst @@ -244,7 +244,7 @@ Building Your First Macro ------------------------- To give a starting point to writing macros we will show now how to -implement the `myDebug` macro mentioned earlier. The first thing to +implement the `myAssert` macro mentioned earlier. The first thing to do is to build a simple example of the macro usage, and then just print the argument. This way it is possible to get an idea of what a correct argument should look like. From 488e18f90d342b5a3b528c0c84891b31b7d1e6ca Mon Sep 17 00:00:00 2001 From: VlkrS <47375452+VlkrS@users.noreply.github.com> Date: Thu, 3 Mar 2022 14:41:35 +0100 Subject: [PATCH 0716/1654] Fix CPU detection for i386 (#19577) * Fix CPU detection for i386 Commit 787def271b1cabc6f898caa42f892125de9fa908 breaks CPU detection for i386 on OpenBSD and probably on other platforms. [ isOpenIndiana -eq "yes" ] always returns 0, so mycpu is always set to "amd64". * Update buildsh.nimf Fix accidental bashism --- tools/niminst/buildsh.nimf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 5f1db9cd3..7a05ef342 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -164,7 +164,7 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - if [ isOpenIndiana -eq "yes" ] ; then + if [ "$isOpenIndiana" = "yes" ] ; then mycpu="amd64" else mycpu="i386" From 645447293851749fcc3394cd387d7070d8a9c735 Mon Sep 17 00:00:00 2001 From: Hamid Bluri Date: Mon, 7 Mar 2022 12:51:42 +0330 Subject: [PATCH 0717/1654] fix not flushing stdout in MSYS (#19590) I did this pull request according to what xflywind said: https://github.com/nim-lang/Nim/pull/19584#issuecomment-1060085141 --- lib/impure/rdstdin.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 32048b131..adc0e212d 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -27,6 +27,7 @@ when defined(windows): tags: [ReadIOEffect, WriteIOEffect].} = ## Reads a line from stdin. stdout.write(prompt) + stdout.flushFile() result = readLine(stdin) proc readLineFromStdin*(prompt: string, line: var string): bool {. From 8f9dd5b3492293e4e48647206c8858b58cd51666 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 9 Mar 2022 01:12:31 +0800 Subject: [PATCH 0718/1654] register callback for marshal in VM (#19578) * register callback for marshal in VM * remove unrelated code * polish * more tests * more tests * add loadVM and toVM --- compiler/vm.nim | 17 +++------------- compiler/vmdef.nim | 5 ++--- compiler/vmgen.nim | 27 ------------------------- compiler/vmops.nim | 36 ++++++++++++++++++++++++++++++++- lib/pure/marshal.nim | 28 ++++++++++++++++++-------- tests/stdlib/tmarshal.nim | 42 +++++++++++++++++++++++++++++++++------ 6 files changed, 96 insertions(+), 59 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 259648add..fa1c71c85 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -15,7 +15,7 @@ import std/[strutils, tables, parseutils], msgs, vmdef, vmgen, nimsets, types, passes, parser, vmdeps, idents, trees, renderer, options, transf, - vmmarshal, gorgeimpl, lineinfos, btrees, macrocacheimpl, + gorgeimpl, lineinfos, btrees, macrocacheimpl, modulegraphs, sighashes, int128, vmprofiler import ast except getstr @@ -1224,7 +1224,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = c.callbacks[-prc.offset-2].value( VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]), currentException: c.currentExceptionA, - currentLineInfo: c.debug[pc])) + currentLineInfo: c.debug[pc]) + ) elif importcCond(c, prc): if compiletimeFFI notin c.config.features: globalError(c.config, c.debug[pc], "VM not allowed to do FFI, see `compiletimeFFI`") @@ -2100,18 +2101,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = while typ.kind == tyTypeDesc and typ.len > 0: typ = typ[0] createStr regs[ra] regs[ra].node.strVal = typ.typeToString(preferExported) - of opcMarshalLoad: - let ra = instr.regA - let rb = instr.regB - inc pc - let typ = c.types[c.code[pc].regBx - wordExcess] - putIntoReg(regs[ra], loadAny(regs[rb].node.strVal, typ, c.cache, c.config, c.idgen)) - of opcMarshalStore: - decodeB(rkNode) - inc pc - let typ = c.types[c.code[pc].regBx - wordExcess] - createStrKeepNode(regs[ra]) - storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode, c.config) c.profiler.leave(c) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 2044b860a..ecdbeff89 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -180,7 +180,6 @@ type opcNBindSym, opcNDynBindSym, opcSetType, # dest.typ = types[Bx] opcTypeTrait, - opcMarshalLoad, opcMarshalStore, opcSymOwner, opcSymIsInstantiationOf @@ -307,8 +306,8 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.disca const firstABxInstr* = opcTJmp largeInstrs* = { # instructions which use 2 int32s instead of 1: - opcSubStr, opcConv, opcCast, opcNewSeq, opcOf, - opcMarshalLoad, opcMarshalStore} + opcSubStr, opcConv, opcCast, opcNewSeq, opcOf + } slotSomeTemp* = slotTempUnknown relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack} diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index c0c4bac19..1551fbb80 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -99,11 +99,6 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) = let idx = x.regBx-wordExcess result.addf("\t$#\tr$#, $# ($#)", opc.toStr, x.regA, c.constants[idx].renderTree, $idx) - elif opc in {opcMarshalLoad, opcMarshalStore}: - let y = c.code[i+1] - result.addf("\t$#\tr$#, r$#, $#", opc.toStr, x.regA, x.regB, - c.types[y.regBx-wordExcess].typeToString) - inc i else: result.addf("\t$#\tr$#, $#", opc.toStr, x.regA, x.regBx-wordExcess) result.add("\t# ") @@ -1383,22 +1378,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = # mGCref, mGCunref, globalError(c.config, n.info, "cannot generate code for: " & $m) -proc genMarshalLoad(c: PCtx, n: PNode, dest: var TDest) = - ## Signature: proc to*[T](data: string): T - if dest < 0: dest = c.getTemp(n.typ) - var tmp = c.genx(n[1]) - c.gABC(n, opcMarshalLoad, dest, tmp) - c.gABx(n, opcMarshalLoad, 0, c.genType(n.typ)) - c.freeTemp(tmp) - -proc genMarshalStore(c: PCtx, n: PNode, dest: var TDest) = - ## Signature: proc `$$`*[T](x: T): string - if dest < 0: dest = c.getTemp(n.typ) - var tmp = c.genx(n[1]) - c.gABC(n, opcMarshalStore, dest, tmp) - c.gABx(n, opcMarshalStore, 0, c.genType(n[1].typ)) - c.freeTemp(tmp) - proc unneededIndirection(n: PNode): bool = n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef @@ -2054,12 +2033,6 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = elif s.kind == skMethod: localError(c.config, n.info, "cannot call method " & s.name.s & " at compile time") - elif matches(s, "stdlib.marshal.to"): - # XXX marshal load&store should not be opcodes, but use the - # general callback mechanisms. - genMarshalLoad(c, n, dest) - elif matches(s, "stdlib.marshal.$$"): - genMarshalStore(c, n, dest) else: genCall(c, n, dest) clearDest(c, n, dest) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 6b06cc68d..f13e4b1cc 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -32,7 +32,7 @@ from system/formatfloat import addFloatRoundtrip, addFloatSprintf # There are some useful procs in vmconv. -import vmconv +import vmconv, vmmarshal template mathop(op) {.dirty.} = registerCallback(c, "stdlib.math." & astToStr(op), `op Wrapper`) @@ -152,6 +152,7 @@ when defined(nimHasInvariant): proc stackTrace2(c: PCtx, msg: string, n: PNode) = stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info) + proc registerAdditionalOps*(c: PCtx) = template wrapIterator(fqname: string, iter: untyped) = @@ -344,3 +345,36 @@ proc registerAdditionalOps*(c: PCtx) = addFloatSprintf(p.strVal, x) wrapIterator("stdlib.os.envPairsImplSeq"): envPairs() + + registerCallback c, "stdlib.marshal.toVM", proc(a: VmArgs) = + let typ = a.getNode(0).typ + case typ.kind + of tyInt..tyInt64, tyUInt..tyUInt64: + setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen).intVal) + of tyFloat..tyFloat128: + setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen).floatVal) + else: + setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen)) + + registerCallback c, "stdlib.marshal.loadVM", proc(a: VmArgs) = + let typ = a.getNode(0).typ + let p = a.getReg(1) + var res: string + + var node: PNode + case p.kind + of rkNone: + node = newNode(nkEmpty) + of rkInt: + node = newIntNode(nkIntLit, p.intVal) + of rkFloat: + node = newFloatNode(nkFloatLit, p.floatVal) + of rkNode: + node = p.node + of rkRegisterAddr: + node = p.regAddr.node + of rkNodeAddr: + node = p.nodeAddr[] + + storeAny(res, typ, node, c.config) + setResult(a, res) diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 993e0f510..452af54d5 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -298,6 +298,9 @@ proc store*[T](s: Stream, data: T) = shallowCopy(d, data) storeAny(s, toAny(d), stored) +proc loadVM[T](typ: typedesc[T], x: T): string = + discard "the implementation is in the compiler/vmops" + proc `$$`*[T](x: T): string = ## Returns a string representation of `x` (serialization, marshalling). ## @@ -313,12 +316,18 @@ proc `$$`*[T](x: T): string = let y = $$x assert y == """{"id": 1, "bar": "baz"}""" - var stored = initIntSet() - var d: T - shallowCopy(d, x) - var s = newStringStream() - storeAny(s, toAny(d), stored) - result = s.data + when nimvm: + result = loadVM(T, x) + else: + var stored = initIntSet() + var d: T + shallowCopy(d, x) + var s = newStringStream() + storeAny(s, toAny(d), stored) + result = s.data + +proc toVM[T](typ: typedesc[T], data: string): T = + discard "the implementation is in the compiler/vmops" proc to*[T](data: string): T = ## Reads data and transforms it to a type `T` (deserialization, unmarshalling). @@ -335,5 +344,8 @@ proc to*[T](data: string): T = assert z.id == 1 assert z.bar == "baz" - var tab = initTable[BiggestInt, pointer]() - loadAny(newStringStream(data), toAny(result), tab) + when nimvm: + result = toVM(T, data) + else: + var tab = initTable[BiggestInt, pointer]() + loadAny(newStringStream(data), toAny(result), tab) diff --git a/tests/stdlib/tmarshal.nim b/tests/stdlib/tmarshal.nim index 508205c3a..e539b1c34 100644 --- a/tests/stdlib/tmarshal.nim +++ b/tests/stdlib/tmarshal.nim @@ -4,12 +4,16 @@ import std/marshal proc testit[T](x: T): string = $$to[T]($$x) -let test1: array[0..1, array[0..4, string]] = [ - ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] -doAssert testit(test1) == - """[["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]""" -let test2: tuple[name: string, s: int] = ("tuple test", 56) -doAssert testit(test2) == """{"Field0": "tuple test", "Field1": 56}""" +template check1 = + let test1: array[0..1, array[0..4, string]] = [ + ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] + doAssert testit(test1) == + """[["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]""" + let test2: tuple[name: string, s: int] = ("tuple test", 56) + doAssert testit(test2) == """{"Field0": "tuple test", "Field1": 56}""" + +static: check1() +check1() type TE = enum @@ -146,3 +150,29 @@ block: let a: ref A = new(B) doAssert $$a[] == "{}" # not "{f: 0}" + +template checkMarshal(data: typed) = + let orig = data + let m = $$orig + + let old = to[typeof(orig)](m) + doAssert data == old + +template main() = + type + Book = object + page: int + name: string + + let book = Book(page: 12, name: "persona") + + checkMarshal(486) + checkMarshal(3.14) + checkMarshal("azure sky") + checkMarshal(book) + checkMarshal([1, 2, 3]) + checkMarshal(@[1.5, 2.7, 3.9, 4.2]) + checkMarshal(@["dream", "is", "possible"]) + +static: main() +main() From 0d6795a771e46b07244f3a3f43502602acbebdc3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 9 Mar 2022 07:22:10 +0100 Subject: [PATCH 0719/1654] fixes #19569 (#19595) [backport] * minor code refactorings * fixes #19569 --- compiler/cgen.nim | 2 +- compiler/sizealignoffsetimpl.nim | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b78ad10c4..2865473fb 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -881,7 +881,7 @@ proc containsResult(n: PNode): bool = if containsResult(n[i]): return true const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTemplateDef, - nkMacroDef, nkMixinStmt, nkBindStmt} + + nkMacroDef, nkMixinStmt, nkBindStmt, nkFormalParams} + declarativeDefs proc easyResultAsgn(n: PNode): PNode = diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index c5cd3ba64..c2e97aa53 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -40,15 +40,15 @@ proc inc(arg: var OffsetAccum; value: int) = else: arg.offset += value -proc alignmentMax(a,b: int): int = +proc alignmentMax(a, b: int): int = if unlikely(a == szIllegalRecursion or b == szIllegalRecursion): raiseIllegalTypeRecursion() if a == szUnknownSize or b == szUnknownSize: szUnknownSize else: - max(a,b) + max(a, b) proc align(arg: var OffsetAccum; value: int) = - if unlikely(value == szIllegalRecursion): raiseIllegalTypeRecursion() + if unlikely(value == szIllegalRecursion): raiseIllegalTypeRecursion() if value == szUnknownSize or arg.maxAlign == szUnknownSize or arg.offset == szUnknownSize: arg.maxAlign = szUnknownSize arg.offset = szUnknownSize @@ -112,7 +112,7 @@ proc setOffsetsToUnknown(n: PNode) = for i in 0.. Date: Wed, 9 Mar 2022 11:42:09 +0100 Subject: [PATCH 0720/1654] fixes #19575 (#19596) [backport] * fixes #19575 * better bugfix --- compiler/closureiters.nim | 6 ++++-- tests/iter/titer_issues.nim | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index a8da7485e..4d9aeb3f7 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -1351,13 +1351,15 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode = case n.kind of nkTryStmt: let f = n.lastSon + var didAddSomething = false if f.kind == nkFinally: c.finallys.add f.lastSon + didAddSomething = true for i in 0 ..< n.len: result[i] = preprocess(c, n[i]) - if f.kind == nkFinally: + if didAddSomething: discard c.finallys.pop() of nkWhileStmt, nkBlockStmt: @@ -1384,7 +1386,7 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode = result = newNodeI(nkStmtList, n.info) for i in countdown(c.finallys.high, fin): var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info, idgen: c.idgen) - result.add freshVars(preprocess(c, c.finallys[i]), vars) + result.add freshVars(copyTree(c.finallys[i]), vars) c.idgen = vars.idgen result.add n of nkSkip: discard diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 65f66ad26..15fe867c8 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -251,3 +251,26 @@ block: for x in ff(@[1, 2], @[1, 2, 3]): echo x + + +# bug #19575 + +iterator bb() {.closure.} = + while true: + try: discard + except: break + finally: break + +var a = bb + +iterator cc() {.closure.} = + while true: + try: discard + except: + if true: + break + finally: + if true: + break + +var a2 = cc From 6773ffa63d0b3ab8b8894e84ed417f4eaced9122 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 9 Mar 2022 19:55:20 +0800 Subject: [PATCH 0721/1654] enable nimPreviewDotLikeOps (#19598) --- config/config.nims | 1 + tests/config.nims | 1 - tests/stdlib/texperimental.nim | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/stdlib/texperimental.nim diff --git a/config/config.nims b/config/config.nims index 88c344642..08d9d9f55 100644 --- a/config/config.nims +++ b/config/config.nims @@ -21,3 +21,4 @@ when defined(nimStrictMode): # future work: XDeclaredButNotUsed switch("define", "nimVersion:" & NimVersion) +switch("define", "nimPreviewDotLikeOps") diff --git a/tests/config.nims b/tests/config.nims index 894c4bea0..8c4305521 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -37,6 +37,5 @@ switch("define", "nimExperimentalLinenoiseExtra") # preview APIs are expected to be the new default in upcoming versions switch("define", "nimPreviewFloatRoundtrip") -switch("define", "nimPreviewDotLikeOps") switch("define", "nimPreviewJsonutilsHoleyEnum") switch("define", "nimPreviewHashRef") diff --git a/tests/stdlib/texperimental.nim b/tests/stdlib/texperimental.nim new file mode 100644 index 000000000..ba8c4eb80 --- /dev/null +++ b/tests/stdlib/texperimental.nim @@ -0,0 +1 @@ +doAssert defined(nimPreviewDotLikeOps) \ No newline at end of file From 4c76f9f1523a72f560138709642c2d51ea365b85 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 11 Mar 2022 00:01:27 +0800 Subject: [PATCH 0722/1654] support useVersion:1.6 (#19601) --- compiler/commands.nim | 4 +++ compiler/options.nim | 1 + doc/advopt.txt | 2 +- doc/manual.rst | 5 ---- tests/system/tuse_version.nim | 5 ++-- tests/system/tuse_version16.nim | 49 +++++++++++++++++++++++++++++++++ 6 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 tests/system/tuse_version16.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 8e1d96860..b21b48801 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1043,6 +1043,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; defineSymbol(conf.symbols, "NimMajor", "1") defineSymbol(conf.symbols, "NimMinor", "2") conf.globalOptions.incl optNimV12Emulation + of "1.6": + defineSymbol(conf.symbols, "NimMajor", "1") + defineSymbol(conf.symbols, "NimMinor", "6") + conf.globalOptions.incl optNimV16Emulation else: localError(conf, info, "unknown Nim version; currently supported values are: `1.0`, `1.2`") # always be compatible with 1.x.100: diff --git a/compiler/options.nim b/compiler/options.nim index 5cfe58119..9c0c9df72 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -96,6 +96,7 @@ type # please make sure we have under 32 options optPanics # turn panics (sysFatal) into a process termination optNimV1Emulation # emulate Nim v1.0 optNimV12Emulation # emulate Nim v1.2 + optNimV16Emulation # emulate Nim v1.6 optSourcemap optProfileVM # enable VM profiler optEnableDeepCopy # ORC specific: enable 'deepcopy' for all types. diff --git a/doc/advopt.txt b/doc/advopt.txt index e27c75ada..0a2349597 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -161,7 +161,7 @@ Advanced options: enable experimental language feature --legacy:$2 enable obsolete/legacy language feature - --useVersion:1.0|1.2 emulate Nim version X of the Nim compiler, for testing + --useVersion:1.0|1.2|1.6 emulate Nim version X of the Nim compiler, for testing --benchmarkVM:on|off turn benchmarking of VM code with cpuTime() on|off --profileVM:on|off turn compile time VM profiler on|off --sinkInference:on|off turn sink parameter inference on|off (default: on) diff --git a/doc/manual.rst b/doc/manual.rst index e45133c38..4c4b9c506 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -5797,11 +5797,6 @@ However, this means that the method call syntax is not available for tmp(12) -**Note**: The Nim compiler prior to version 1 was more lenient about this -requirement. Use the `--useVersion:0.19`:option: switch for a transition period. - - - Limitations of the method call syntax ------------------------------------- diff --git a/tests/system/tuse_version.nim b/tests/system/tuse_version.nim index 36dac46ec..116ef3596 100644 --- a/tests/system/tuse_version.nim +++ b/tests/system/tuse_version.nim @@ -1,6 +1,5 @@ discard """ - cmd: "nim c --useVersion:1.0 -r $file" - output: "1.0.100" + matrix: "--useVersion:1.0" """ {.warning[UnusedImport]: off.} @@ -47,4 +46,4 @@ import std/[ ] -echo NimVersion +doAssert NimVersion == "1.0.100" diff --git a/tests/system/tuse_version16.nim b/tests/system/tuse_version16.nim new file mode 100644 index 000000000..d5430db32 --- /dev/null +++ b/tests/system/tuse_version16.nim @@ -0,0 +1,49 @@ +discard """ + matrix: "--useVersion:1.6" +""" + +{.warning[UnusedImport]: off.} + +import std/[ + # Core: + bitops, typetraits, lenientops, macros, volatile, + + # Algorithms: + algorithm, sequtils, + + # Collections: + critbits, deques, heapqueue, intsets, lists, options, sets, + sharedlist, tables, + + # Strings: + editdistance, wordwrap, parseutils, ropes, + pegs, punycode, strformat, strmisc, strscans, strtabs, + strutils, unicode, unidecode, + + # Generic operator system services: + os, streams, + + # Math libraries: + complex, math, mersenne, random, rationals, stats, sums, + + # Internet protocols: + httpcore, mimetypes, uri, + + # Parsers: + htmlparser, json, lexbase, parsecfg, parsecsv, parsesql, parsexml, + + # XML processing: + xmltree, xmlparser, + + # Generators: + htmlgen, + + # Hashing: + base64, hashes, + + # Miscellaneous: + colors, sugar, varints, +] + + +doAssert NimVersion == "1.6.100" From 7a50d663467f0e64ea8390e8c86269b25f312816 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Fri, 18 Mar 2022 03:54:20 -0300 Subject: [PATCH 0723/1654] Removed deprecated httpcore func (#19550) * Remove Deprecated httpcore func * Remove Deprecated httpcore func * Fix a test with Deprecated func * Restart CI, Apple can code shit anymore I tell you --- changelog.md | 1 + lib/pure/httpcore.nim | 10 ---------- tests/stdlib/tasynchttpserver.nim | 8 ++++---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/changelog.md b/changelog.md index 011f95ddb..a9bf3842a 100644 --- a/changelog.md +++ b/changelog.md @@ -45,6 +45,7 @@ becomes an alias for `addr`. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - Removed deprecated `jsre.test` and `jsre.toString`. - Removed deprecated `math.c_frexp`. +- Removed deprecated ``` httpcore.`==` ```. ## Language changes diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index fdd5926f7..7e995ac46 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -347,16 +347,6 @@ func `$`*(code: HttpCode): string = func `==`*(a, b: HttpCode): bool {.borrow.} -proc `==`*(rawCode: string, code: HttpCode): bool - {.deprecated: "Deprecated since v1.2; use rawCode == $code instead".} = - ## Compare the string form of the status code with a HttpCode - ## - ## **Note**: According to HTTP/1.1 specification, the reason phrase is - ## optional and should be ignored by the client, making this - ## proc only suitable for comparing the `HttpCode` against the - ## string form of itself. - return cmpIgnoreCase(rawCode, $code) == 0 - func is1xx*(code: HttpCode): bool {.inline, since: (1, 5).} = ## Determines whether `code` is a 1xx HTTP status code. runnableExamples: diff --git a/tests/stdlib/tasynchttpserver.nim b/tests/stdlib/tasynchttpserver.nim index 77ad7a071..d4176d55a 100644 --- a/tests/stdlib/tasynchttpserver.nim +++ b/tests/stdlib/tasynchttpserver.nim @@ -39,7 +39,7 @@ proc test200() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http200) + doAssert(response.status == $Http200) doAssert(body == "Hello World, 200") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "16") @@ -60,7 +60,7 @@ proc test404() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http404) + doAssert(response.status == $Http404) doAssert(body == "Hello World, 404") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "16") @@ -81,7 +81,7 @@ proc testCustomEmptyHeaders() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http200) + doAssert(response.status == $Http200) doAssert(body == "Hello World, 200") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "16") @@ -104,7 +104,7 @@ proc testCustomContentLength() {.async.} = return clientResponse proc test(response: AsyncResponse, body: string) {.async.} = - doAssert(response.status == Http200) + doAssert(response.status == $Http200) doAssert(body == "") doAssert(response.headers.hasKey("Content-Length")) doAssert(response.headers["Content-Length"] == "0") From 7b811deeffc762f8370b4cf7e8a353fd516d6543 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 18 Mar 2022 14:17:54 +0100 Subject: [PATCH 0724/1654] db_sqlite: added a space --- lib/impure/db_sqlite.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index b600576df..1638d38c6 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -700,7 +700,7 @@ proc insert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect], since: (1, 3).} = ## same as insertId - result = tryInsert(db, query,pkName, args) + result = tryInsert(db, query, pkName, args) if result < 0: dbError(db) proc execAffectedRows*(db: DbConn, query: SqlQuery, From 3e83d73f272afe8de85189da7d6d513916cc2efd Mon Sep 17 00:00:00 2001 From: Clyybber Date: Fri, 18 Mar 2022 16:41:45 +0100 Subject: [PATCH 0725/1654] compiler: Handle nkCheckedFieldExpr better in dfa (#19616) Simply recurse into their first child, which is always a nkDotExpr instead of treating them seperately. This fixes the rhs sym of a nkCheckedFieldExpr being checked twice in aliases. This double checking didn't cause any issues, but was unintentional and redundant. --- compiler/dfa.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/dfa.nim b/compiler/dfa.nim index 0539f6699..d68e7be82 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -605,11 +605,11 @@ proc aliases*(obj, field: PNode): AliasKind = var n = n while true: case n.kind - of PathKinds0 - {nkDotExpr, nkCheckedFieldExpr, nkBracketExpr}: + of PathKinds0 - {nkDotExpr, nkBracketExpr}: n = n[0] of PathKinds1: n = n[1] - of nkDotExpr, nkCheckedFieldExpr, nkBracketExpr: + of nkDotExpr, nkBracketExpr: result.add n n = n[0] of nkSym: @@ -642,8 +642,6 @@ proc aliases*(obj, field: PNode): AliasKind = if currFieldPath.sym != currObjPath.sym: return no of nkDotExpr: if currFieldPath[1].sym != currObjPath[1].sym: return no - of nkCheckedFieldExpr: - if currFieldPath[0][1].sym != currObjPath[0][1].sym: return no of nkBracketExpr: if currFieldPath[1].kind in nkLiterals and currObjPath[1].kind in nkLiterals: if currFieldPath[1].intVal != currObjPath[1].intVal: From 731eabc9309997775c8be41f3e5eb5512460aad0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 20 Mar 2022 14:02:44 +0100 Subject: [PATCH 0726/1654] fixes #19631 (#19618) Aliasing is hard and we have to watch out not to compile 'x = f(x.a)' into 'f(x.a, addr x)' --- compiler/ccgcalls.nim | 11 ++++++++--- tests/ccgbugs/tcgbug.nim | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index ce5fbfdd7..05be51a45 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -21,7 +21,7 @@ proc canRaiseDisp(p: BProc; n: PNode): bool = # we have to be *very* conservative: result = canRaiseConservative(n) -proc preventNrvo(p: BProc; le, ri: PNode): bool = +proc preventNrvo(p: BProc; dest, le, ri: PNode): bool = proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool = var n = le while true: @@ -54,6 +54,11 @@ proc preventNrvo(p: BProc; le, ri: PNode): bool = if canRaise(ri[0]) and locationEscapes(p, le, p.nestedTryStmts.len > 0): message(p.config, le.info, warnObservableStores, $le) + # bug #19613 prevent dangerous aliasing too: + if dest != nil and dest != le: + for i in 1.. 1: pl.add(~", ") # beware of 'result = p(result)'. We may need to allocate a temporary: - if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri): + if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri): # Great, we can use 'd': if d.k == locNone: getTemp(p, typ[0], d, needsInit=true) diff --git a/tests/ccgbugs/tcgbug.nim b/tests/ccgbugs/tcgbug.nim index db9c116be..0fe4b8852 100644 --- a/tests/ccgbugs/tcgbug.nim +++ b/tests/ccgbugs/tcgbug.nim @@ -91,3 +91,34 @@ proc test(c: Helper): string = c.formatted echo test(Helper(isKind: true, formatted: "ok")) + + +# bug #19613 + +type + Eth2Digest = object + data: array[42, byte] + + BlockId* = object + root*: Eth2Digest + + BlockSlotId* = object + bid*: BlockId + slot*: uint64 + +func init*(T: type BlockSlotId, bid: BlockId, slot: uint64): T = + #debugecho "init ", bid, " ", slot + BlockSlotId(bid: bid, slot: slot) + +proc bug19613 = + var x: BlockSlotId + x.bid.root.data[0] = 42 + + x = + if x.slot > 0: + BlockSlotId.init(x.bid, x.slot) + else: + BlockSlotId.init(x.bid, x.slot) + doAssert x.bid.root.data[0] == 42 + +bug19613() From c4a0d4c5e35f09430a1c3d465fc62eb1001b7f9f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 22 Mar 2022 15:36:49 +0100 Subject: [PATCH 0727/1654] =?UTF-8?q?fixes=20#19615;=20emit=20better=20cod?= =?UTF-8?q?e=20for=20integer=20divisions=20when=20the=20divisor=E2=80=A6?= =?UTF-8?q?=20(#19626)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixes #19615; emit better code for integer divisions when the divisor is known at compile-time * proper bugfix: unsigned numbers cannot be -1 --- compiler/ccgexprs.nim | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 9848366a1..1a3e217b2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -584,13 +584,23 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = else: # we handle div by zero here so that we know that the compilerproc's # result is only for overflows. + var needsOverflowCheck = true if m in {mDivI, mModI}: - linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n", - [rdLoc(b), raiseInstr(p)]) - - let res = binaryArithOverflowRaw(p, t, a, b, - if t.kind == tyInt64: prc64[m] else: prc[m]) - putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) + var canBeZero = true + if e[2].kind in {nkIntLit..nkUInt64Lit}: + canBeZero = e[2].intVal == 0 + if e[2].kind in {nkIntLit..nkInt64Lit}: + needsOverflowCheck = e[2].intVal == -1 + if canBeZero: + linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n", + [rdLoc(b), raiseInstr(p)]) + if needsOverflowCheck: + let res = binaryArithOverflowRaw(p, t, a, b, + if t.kind == tyInt64: prc64[m] else: prc[m]) + putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res]) + else: + let res = "($1)($2 $3 $4)" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)] + putIntoDest(p, d, e, res) proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) = var From 4c8934305c48bb52ed0e5a59b47229521cb333ce Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Tue, 22 Mar 2022 12:08:31 -0700 Subject: [PATCH 0728/1654] system: thread: stack dealloction on Zephyr (#19633) [backport:1.6] Try to free the stack allocation when a thread exits. Possibly works for FreeRTOS as well. --- lib/system/threads.nim | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/system/threads.nim b/lib/system/threads.nim index ac727be0a..1c1d1ca1c 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -47,7 +47,10 @@ when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} -when defined(zephyr) or defined(freertos): +const + hasAllocStack = defined(zephyr) # maybe freertos too? + +when hasAllocStack or defined(zephyr) or defined(freertos): const nimThreadStackSize {.intdefine.} = 8192 nimThreadStackGuard {.intdefine.} = 128 @@ -68,6 +71,12 @@ else: #const globalsSlot = ThreadVarSlot(0) #sysAssert checkSlot.int == globalsSlot.int +# Zephyr doesn't include this properly without some help +when defined(zephyr): + {.emit: """/*INCLUDESECTION*/ + #include + """.} + # create for the main thread. Note: do not insert this data into the list # of all threads; it's not to be stopped etc. when not defined(useNimRtl): @@ -98,6 +107,8 @@ type else: dataFn: proc (m: TArg) {.nimcall, gcsafe.} data: TArg + when hasAllocStack: + rawStack: pointer proc `=copy`*[TArg](x: var Thread[TArg], y: Thread[TArg]) {.error.} @@ -161,6 +172,8 @@ else: threadTrouble() finally: afterThreadRuns() + when hasAllocStack: + deallocShared(thrd.rawStack) proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = when defined(boehmgc): @@ -330,11 +343,12 @@ else: when hasSharedHeap: t.core.stackSize = ThreadStackSize var a {.noinit.}: Pthread_attr doAssert pthread_attr_init(a) == 0 - when defined(zephyr): + when hasAllocStack: var rawstk = allocShared0(ThreadStackSize + StackGuardSize) stk = cast[pointer](cast[uint](rawstk) + StackGuardSize) let setstacksizeResult = pthread_attr_setstack(addr a, stk, ThreadStackSize) + t.rawStack = rawstk else: let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize) From a8b5ad845c4218b4f20595df097c593acee53d50 Mon Sep 17 00:00:00 2001 From: Dominic Ward Date: Wed, 23 Mar 2022 06:50:36 +0000 Subject: [PATCH 0729/1654] Fix process lines iterator (#19605) * Ensure lines when process done * eliminate post-EOF exit test * Recommend fixes for execCmdEx/execProcess --- lib/pure/osproc.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6a0ac9a8b..3da9737ec 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -451,7 +451,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: [ReadIOEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -474,11 +474,11 @@ iterator lines*(p: Process): string {.since: (1, 3), tags: [ReadIOEffect].} = ## p.close var outp = p.outputStream var line = newStringOfCap(120) - while true: - if outp.readLine(line): - yield line - else: - if p.peekExitCode != -1: break + while outp.readLine(line): + if keepNewLines: + line.add("\n") + yield line + discard waitForExit(p) proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = ## Convenience function for working with `startProcess` to read data from a @@ -514,6 +514,7 @@ when not defined(useNimRtl): var outp = outputStream(p) result = "" var line = newStringOfCap(120) + # consider `p.lines(keepNewLines=true)` to circumvent `running` busy-wait while true: # FIXME: converts CR-LF to LF. if outp.readLine(line): @@ -1622,6 +1623,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { inputStream(p).write(input) close inputStream(p) + # consider `p.lines(keepNewLines=true)` to avoid exit code test result = ("", -1) var line = newStringOfCap(120) while true: From 7f6e800cafc7b73625893fb5280eb8b51a15b252 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 03:34:53 +0800 Subject: [PATCH 0730/1654] move assertions out of system (#19599) --- compiler/aliases.nim | 3 +++ compiler/ast.nim | 3 +++ compiler/astalgo.nim | 3 +++ compiler/bitsets.nim | 3 +++ compiler/btrees.nim | 3 +++ compiler/ccgutils.nim | 3 +++ compiler/cgen.nim | 4 ++++ compiler/cgmeth.nim | 4 ++++ compiler/closureiters.nim | 3 +++ compiler/commands.nim | 3 +++ compiler/concepts.nim | 3 +++ compiler/dfa.nim | 3 +++ compiler/enumtostr.nim | 4 ++++ compiler/errorhandling.nim | 3 +++ compiler/extccomp.nim | 3 +++ compiler/gorgeimpl.nim | 3 +++ compiler/guards.nim | 3 +++ compiler/hlo.nim | 3 +++ compiler/ic/bitabs.nim | 3 +++ compiler/ic/cbackend.nim | 3 +++ compiler/ic/dce.nim | 4 ++++ compiler/ic/ic.nim | 3 +++ compiler/ic/integrity.nim | 4 ++++ compiler/ic/navigator.nim | 3 +++ compiler/ic/packed_ast.nim | 3 +++ compiler/ic/replayer.nim | 3 +++ compiler/ic/rodfiles.nim | 3 +++ compiler/idents.nim | 3 +++ compiler/importer.nim | 3 +++ compiler/injectdestructors.nim | 3 +++ compiler/int128.nim | 3 +++ compiler/isolation_check.nim | 3 +++ compiler/lambdalifting.nim | 3 +++ compiler/lexer.nim | 3 +++ compiler/liftdestructors.nim | 3 +++ compiler/llstream.nim | 3 +++ compiler/lookups.nim | 4 ++++ compiler/lowerings.nim | 3 +++ compiler/main.nim | 4 ++++ compiler/modulegraphs.nim | 3 +++ compiler/modules.nim | 3 +++ compiler/msgs.nim | 4 ++++ compiler/ndi.nim | 3 +++ compiler/nilcheck.nim | 3 +++ compiler/nim.nim | 4 ++++ compiler/nimblecmd.nim | 3 +++ compiler/nimconf.nim | 3 +++ compiler/nimlexbase.nim | 3 +++ compiler/nimpaths.nim | 4 ++++ compiler/nimsets.nim | 3 +++ compiler/options.nim | 5 +++++ compiler/parser.nim | 3 +++ compiler/passes.nim | 3 +++ compiler/pathutils.nim | 2 +- compiler/patterns.nim | 3 +++ compiler/platform.nim | 4 ++++ compiler/pragmas.nim | 3 +++ compiler/renderer.nim | 3 +++ compiler/reorder.nim | 3 +++ compiler/rodutils.nim | 3 +++ compiler/ropes.nim | 4 ++++ compiler/scriptconfig.nim | 3 +++ compiler/semdata.nim | 3 +++ compiler/semfold.nim | 3 +++ compiler/sempass2.nim | 3 +++ compiler/semtypinst.nim | 3 +++ compiler/sighashes.nim | 4 ++++ compiler/sigmatch.nim | 3 +++ compiler/syntaxes.nim | 3 +++ compiler/transf.nim | 3 +++ compiler/treetab.nim | 3 +++ compiler/typeallowed.nim | 3 +++ compiler/types.nim | 3 +++ compiler/varpartitions.nim | 3 +++ compiler/vmdeps.nim | 3 +++ compiler/vmgen.nim | 3 +++ compiler/vmhooks.nim | 3 +++ compiler/vmmarshal.nim | 3 +++ compiler/vmops.nim | 4 ++++ lib/core/macros.nim | 4 ++++ lib/pure/algorithm.nim | 4 ++++ lib/pure/browsers.nim | 3 +++ lib/pure/collections/hashcommon.nim | 4 ++++ lib/pure/collections/sequtils.nim | 4 ++++ lib/pure/hashes.nim | 4 ++++ lib/pure/json.nim | 2 +- lib/pure/lexbase.nim | 3 +++ lib/pure/math.nim | 4 ++++ lib/pure/options.nim | 4 ++++ lib/pure/os.nim | 2 +- lib/pure/osproc.nim | 4 ++++ lib/pure/parsejson.nim | 3 +++ lib/pure/streamwrapper.nim | 4 ++++ lib/pure/strformat.nim | 4 ++++ lib/pure/strtabs.nim | 4 ++++ lib/pure/strutils.nim | 3 +++ lib/pure/terminal.nim | 2 +- lib/pure/times.nim | 4 ++++ lib/pure/typetraits.nim | 4 ++++ lib/{system => std}/assertions.nim | 14 ++++++++++---- lib/std/enumutils.nim | 4 ++++ lib/std/jsonutils.nim | 3 +++ lib/std/packedsets.nim | 3 +++ lib/std/private/globs.nim | 4 ++++ lib/std/sha1.nim | 3 +++ lib/system.nim | 7 +++++-- lib/system/dragonbox.nim | 2 ++ lib/system/formatfloat.nim | 3 +++ lib/system/iterators.nim | 3 +++ lib/system/schubfach.nim | 3 +++ 110 files changed, 361 insertions(+), 10 deletions(-) rename lib/{system => std}/assertions.nim (94%) diff --git a/compiler/aliases.nim b/compiler/aliases.nim index 0006c9fe6..9ec72faa4 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -12,6 +12,9 @@ import ast, astalgo, types, trees, intsets +when defined(nimPreviewSlimSystem): + import std/assertions + type TAnalysisResult* = enum arNo, arMaybe, arYes diff --git a/compiler/ast.nim b/compiler/ast.nim index ad5ec12f5..d1e5ae2bf 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -13,6 +13,9 @@ import lineinfos, hashes, options, ropes, idents, int128, tables from strutils import toLowerAscii +when defined(nimPreviewSlimSystem): + import std/assertions + export int128 type diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 94fa9da93..bc5b7d6e1 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -15,6 +15,9 @@ import ast, hashes, intsets, strutils, options, lineinfos, ropes, idents, rodutils, msgs +when defined(nimPreviewSlimSystem): + import std/assertions + proc hashNode*(p: RootRef): Hash proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope # Convert a tree into its YAML representation; this is used by the diff --git a/compiler/bitsets.nim b/compiler/bitsets.nim index e0d94409e..67598f9ca 100644 --- a/compiler/bitsets.nim +++ b/compiler/bitsets.nim @@ -10,6 +10,9 @@ # this unit handles Nim sets; it implements bit sets # the code here should be reused in the Nim standard library +when defined(nimPreviewSlimSystem): + import std/assertions + type ElemType = byte TBitSet* = seq[ElemType] # we use byte here to avoid issues with diff --git a/compiler/btrees.nim b/compiler/btrees.nim index d5e7e8099..1701f8d7b 100644 --- a/compiler/btrees.nim +++ b/compiler/btrees.nim @@ -10,6 +10,9 @@ ## BTree implementation with few features, but good enough for the ## Nim compiler's needs. +when defined(nimPreviewSlimSystem): + import std/assertions + const M = 512 # max children per B-tree node = M-1 # (must be even and greater than 2) diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index 06b75a20f..e19fccfa7 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -13,6 +13,9 @@ import ast, types, hashes, strutils, msgs, wordrecg, platform, trees, options, cgendata +when defined(nimPreviewSlimSystem): + import std/assertions + proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode = case n.kind of nkStmtList: diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 2865473fb..ad59b759f 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -17,6 +17,10 @@ import lowerings, tables, sets, ndi, lineinfos, pathutils, transf, injectdestructors, astmsgs + +when defined(nimPreviewSlimSystem): + import std/assertions + when not defined(leanCompiler): import spawn, semparallel diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 484bc9d97..dab8826c1 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -13,6 +13,10 @@ import intsets, options, ast, msgs, idents, renderer, types, magicsys, sempass2, strutils, modulegraphs, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + + proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode = var dest = skipTypes(d, abstractPtrs) var source = skipTypes(n.typ, abstractPtrs) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 4d9aeb3f7..6370d0dcb 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -133,6 +133,9 @@ import renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos, tables, options +when defined(nimPreviewSlimSystem): + import std/assertions + type Ctx = object g: ModuleGraph diff --git a/compiler/commands.nim b/compiler/commands.nim index b21b48801..4c1cec3d2 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -31,6 +31,9 @@ import from ast import setUseIc, eqTypeFlags, tfGcSafe, tfNoSideEffect +when defined(nimPreviewSlimSystem): + import std/assertions + # but some have deps to imported modules. Yay. bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc") bootSwitch(usedFFI, hasFFI, "-d:nimHasLibFFI") diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 885b69c60..89a2bdc10 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -15,6 +15,9 @@ import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, from magicsys import addSonSkipIntLit +when defined(nimPreviewSlimSystem): + import std/assertions + const logBindings = false diff --git a/compiler/dfa.nim b/compiler/dfa.nim index d68e7be82..5b048ff6e 100644 --- a/compiler/dfa.nim +++ b/compiler/dfa.nim @@ -32,6 +32,9 @@ import ast, intsets, lineinfos, renderer import std/private/asciitables +when defined(nimPreviewSlimSystem): + import std/assertions + type InstrKind* = enum goto, fork, def, use diff --git a/compiler/enumtostr.nim b/compiler/enumtostr.nim index 9bfa7001a..4a28d066c 100644 --- a/compiler/enumtostr.nim +++ b/compiler/enumtostr.nim @@ -1,6 +1,10 @@ import ast, idents, lineinfos, modulegraphs, magicsys +when defined(nimPreviewSlimSystem): + import std/assertions + + proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGenerator): PSym = result = newSym(skProc, getIdent(g.cache, "$"), nextSymId idgen, t.owner, info) diff --git a/compiler/errorhandling.nim b/compiler/errorhandling.nim index cda7ab3f4..a8361105e 100644 --- a/compiler/errorhandling.nim +++ b/compiler/errorhandling.nim @@ -12,6 +12,9 @@ import ast, renderer, options, strutils, types +when defined(nimPreviewSlimSystem): + import std/assertions + type ErrorKind* = enum ## expand as you need. RawTypeMismatchError diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 124cc2c34..09a1dfbb3 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -16,6 +16,9 @@ import ropes, platform, condsyms, options, msgs, lineinfos, pathutils import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] +when defined(nimPreviewSlimSystem): + import std/syncio + type TInfoCCProp* = enum # properties of the C compiler: hasSwitchRange, # CC allows ranges in switch statements (GNU C) diff --git a/compiler/gorgeimpl.nim b/compiler/gorgeimpl.nim index d4aeb6a77..f58ac5de9 100644 --- a/compiler/gorgeimpl.nim +++ b/compiler/gorgeimpl.nim @@ -12,6 +12,9 @@ import msgs, std / sha1, os, osproc, streams, options, lineinfos, pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + proc readOutput(p: Process): (string, int) = result[0] = "" var output = p.outputStream diff --git a/compiler/guards.nim b/compiler/guards.nim index d1265d42c..a50593aca 100644 --- a/compiler/guards.nim +++ b/compiler/guards.nim @@ -12,6 +12,9 @@ import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents, saturate, modulegraphs, options, lineinfos, int128 +when defined(nimPreviewSlimSystem): + import std/assertions + const someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc, mEqStr, mEqSet, mEqCString} diff --git a/compiler/hlo.nim b/compiler/hlo.nim index af54cabbb..8620c4c4d 100644 --- a/compiler/hlo.nim +++ b/compiler/hlo.nim @@ -9,6 +9,9 @@ # This include implements the high level optimization pass. +when defined(nimPreviewSlimSystem): + import std/assertions + proc hlo(c: PContext, n: PNode): PNode proc evalPattern(c: PContext, n, orig: PNode): PNode = diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index 0bce30b5d..ae673b574 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -3,6 +3,9 @@ import hashes, rodfiles +when defined(nimPreviewSlimSystem): + import std/assertions + type LitId* = distinct uint32 diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index f5811cb3b..e7ab000e6 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -20,6 +20,9 @@ import std/packedsets, algorithm, tables +when defined(nimPreviewSlimSystem): + import std/assertions + import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs] diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim index 0a436a5d1..bc61a38de 100644 --- a/compiler/ic/dce.nim +++ b/compiler/ic/dce.nim @@ -10,6 +10,10 @@ ## Dead code elimination (=DCE) for IC. import std/[intsets, tables] + +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, options, lineinfos, types] import packed_ast, ic, bitabs diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 7387d165b..193e5f517 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -14,6 +14,9 @@ import ".." / [ast, idents, lineinfos, msgs, ropes, options, #import ".." / [renderer, astalgo] from os import removeFile, isAbsolute +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + type PackedConfig* = object backend: TBackend diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim index ed367ef61..d341fd653 100644 --- a/compiler/ic/integrity.nim +++ b/compiler/ic/integrity.nim @@ -11,6 +11,10 @@ ## The set must cover a complete Nim project. import sets + +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, modulegraphs] import packed_ast, bitabs, ic diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index a1a14885d..cbba591c5 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -16,6 +16,9 @@ import sets from os import nil from std/private/miscdollars import toLocation +when defined(nimPreviewSlimSystem): + import std/assertions + import ".." / [ast, modulegraphs, msgs, options] import packed_ast, bitabs, ic diff --git a/compiler/ic/packed_ast.nim b/compiler/ic/packed_ast.nim index 17beda2c1..c78fe56f5 100644 --- a/compiler/ic/packed_ast.nim +++ b/compiler/ic/packed_ast.nim @@ -16,6 +16,9 @@ import hashes, tables, strtabs import bitabs import ".." / [ast, options] +when defined(nimPreviewSlimSystem): + import std/assertions + type SymId* = distinct int32 ModuleId* = distinct int32 diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index 61aa0e697..0188eaee3 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -16,6 +16,9 @@ import ".." / [ast, modulegraphs, trees, extccomp, btrees, import tables +when defined(nimPreviewSlimSystem): + import std/assertions + import packed_ast, ic, bitabs proc replayStateChanges*(module: PSym; g: ModuleGraph) = diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index a52bd18b3..d811b9b99 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -16,6 +16,9 @@ from typetraits import supportsCopyMem +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + ## Overview ## ======== ## `RodFile` represents a Rod File (versioned binary format), and the diff --git a/compiler/idents.nim b/compiler/idents.nim index d2a84fd36..f36ce09f3 100644 --- a/compiler/idents.nim +++ b/compiler/idents.nim @@ -14,6 +14,9 @@ import hashes, wordrecg +when defined(nimPreviewSlimSystem): + import std/assertions + type PIdent* = ref TIdent TIdent*{.acyclic.} = object diff --git a/compiler/importer.nim b/compiler/importer.nim index acca2c644..719b75f0f 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -15,6 +15,9 @@ import modulegraphs, wordrecg, tables from strutils import `%` +when defined(nimPreviewSlimSystem): + import std/assertions + proc readExceptSet*(c: PContext, n: PNode): IntSet = assert n.kind in {nkImportExceptStmt, nkExportExceptStmt} result = initIntSet() diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 985f27878..6500c5bc7 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -19,6 +19,9 @@ import lineinfos, parampatterns, sighashes, liftdestructors, optimizer, varpartitions +when defined(nimPreviewSlimSystem): + import std/assertions + from trees import exprStructuralEquivalent, getRoot type diff --git a/compiler/int128.nim b/compiler/int128.nim index 6ba7c1961..afa07094b 100644 --- a/compiler/int128.nim +++ b/compiler/int128.nim @@ -5,6 +5,9 @@ from math import trunc +when defined(nimPreviewSlimSystem): + import std/assertions + type Int128* = object udata: array[4, uint32] diff --git a/compiler/isolation_check.nim b/compiler/isolation_check.nim index 68a212794..2674605dc 100644 --- a/compiler/isolation_check.nim +++ b/compiler/isolation_check.nim @@ -13,6 +13,9 @@ import ast, types, renderer, intsets +when defined(nimPreviewSlimSystem): + import std/assertions + proc canAlias(arg, ret: PType; marker: var IntSet): bool proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool = diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index a622f6de6..6838fd80d 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -14,6 +14,9 @@ import idents, renderer, types, magicsys, lowerings, tables, modulegraphs, lineinfos, transf, liftdestructors, typeallowed +when defined(nimPreviewSlimSystem): + import std/assertions + discard """ The basic approach is that captured vars need to be put on the heap and that the calling chain needs to be explicitly modelled. Things to consider: diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 506b0e924..e795d52c0 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -19,6 +19,9 @@ import hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream, wordrecg, lineinfos, pathutils, parseutils +when defined(nimPreviewSlimSystem): + import std/assertions + const MaxLineLength* = 80 # lines longer than this lead to a warning numChars*: set[char] = {'0'..'9', 'a'..'z', 'A'..'Z'} diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 51b4ddfb0..68c93a179 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -15,6 +15,9 @@ import modulegraphs, lineinfos, idents, ast, renderer, semdata, from trees import isCaseObj +when defined(nimPreviewSlimSystem): + import std/assertions + type TLiftCtx = object g: ModuleGraph diff --git a/compiler/llstream.nim b/compiler/llstream.nim index 865a98ee0..004d990fa 100644 --- a/compiler/llstream.nim +++ b/compiler/llstream.nim @@ -12,6 +12,9 @@ import pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + # support `useGnuReadline`, `useLinenoise` for backwards compatibility const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and not defined(windows) diff --git a/compiler/lookups.nim b/compiler/lookups.nim index fc30408e5..d61e15915 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -9,6 +9,10 @@ # This module implements lookup helpers. import std/[algorithm, strutils] + +when defined(nimPreviewSlimSystem): + import std/assertions + import intsets, ast, astalgo, idents, semdata, types, msgs, options, renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 37405d8d9..20003b946 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -15,6 +15,9 @@ const import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + proc newDeref*(n: PNode): PNode {.inline.} = result = newNodeIT(nkHiddenDeref, n.info, n.typ[0]) result.add n diff --git a/compiler/main.nim b/compiler/main.nim index e4ec4d729..3cddbffb3 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -22,6 +22,10 @@ import modules, modulegraphs, lineinfos, pathutils, vmprofiler + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 28e0ddb82..c44908dc3 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -15,6 +15,9 @@ import intsets, tables, hashes, md5_old import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils import ic / [packed_ast, ic] +when defined(nimPreviewSlimSystem): + import std/assertions + type SigHash* = distinct MD5Digest diff --git a/compiler/modules.nim b/compiler/modules.nim index 6fba606b2..dd5db63fa 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -14,6 +14,9 @@ import idents, lexer, passes, syntaxes, llstream, modulegraphs, lineinfos, pathutils, tables +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + import ic / replayer proc resetSystemArtifacts*(g: ModuleGraph) = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 56531eb68..9d111b2e2 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -12,6 +12,10 @@ import std/private/miscdollars, options, ropes, lineinfos, pathutils, strutils2 +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + type InstantiationInfo* = typeof(instantiationInfo()) template instLoc*(): InstantiationInfo = instantiationInfo(-2, fullPaths = true) diff --git a/compiler/ndi.nim b/compiler/ndi.nim index 5af87237f..a9d9cfe79 100644 --- a/compiler/ndi.nim +++ b/compiler/ndi.nim @@ -12,6 +12,9 @@ import ast, msgs, ropes, options, pathutils +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + type NdiFile* = object enabled: bool diff --git a/compiler/nilcheck.nim b/compiler/nilcheck.nim index 9c2d091f7..49ceb8942 100644 --- a/compiler/nilcheck.nim +++ b/compiler/nilcheck.nim @@ -10,6 +10,9 @@ import ast, renderer, intsets, tables, msgs, options, lineinfos, strformat, idents, treetab, hashes import sequtils, strutils, sets +when defined(nimPreviewSlimSystem): + import std/assertions + # IMPORTANT: notes not up to date, i'll update this comment again # # notes: diff --git a/compiler/nim.nim b/compiler/nim.nim index 86c7ab524..bfb07ba20 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -8,6 +8,10 @@ # import std/[os, strutils, parseopt] + +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(windows) and not defined(nimKochBootstrap): # remove workaround pending bootstrap >= 1.5.1 # refs https://github.com/nim-lang/Nim/issues/18334#issuecomment-867114536 diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim index 9cd2941ba..b6b08ccd4 100644 --- a/compiler/nimblecmd.nim +++ b/compiler/nimblecmd.nim @@ -12,6 +12,9 @@ import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils, std/sha1, tables +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) = if not conf.searchPaths.contains(path): conf.searchPaths.insert(path, 0) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 1cf22e20a..319a3de6d 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -13,6 +13,9 @@ import llstream, commands, os, strutils, msgs, lexer, ast, options, idents, wordrecg, strtabs, lineinfos, pathutils, scriptconfig +when defined(nimPreviewSlimSystem): + import std/syncio + # ---------------- configuration file parser ----------------------------- # we use Nim's lexer here to save space and work diff --git a/compiler/nimlexbase.nim b/compiler/nimlexbase.nim index b8cb9f78a..3bc9af9c9 100644 --- a/compiler/nimlexbase.nim +++ b/compiler/nimlexbase.nim @@ -15,6 +15,9 @@ import llstream, strutils +when defined(nimPreviewSlimSystem): + import std/assertions + const Lrz* = ' ' Apo* = '\'' diff --git a/compiler/nimpaths.nim b/compiler/nimpaths.nim index a93b488fd..216071c5c 100644 --- a/compiler/nimpaths.nim +++ b/compiler/nimpaths.nim @@ -19,6 +19,10 @@ Unstable API import os, strutils +when defined(nimPreviewSlimSystem): + import std/assertions + + const docCss* = "$nimr/doc/nimdoc.css" docHackNim* = "$nimr/tools/dochack/dochack.nim" diff --git a/compiler/nimsets.nim b/compiler/nimsets.nim index 8683604af..49c80065a 100644 --- a/compiler/nimsets.nim +++ b/compiler/nimsets.nim @@ -12,6 +12,9 @@ import ast, astalgo, lineinfos, bitsets, types, options +when defined(nimPreviewSlimSystem): + import std/assertions + proc inSet*(s: PNode, elem: PNode): bool = assert s.kind == nkCurly if s.kind != nkCurly: diff --git a/compiler/options.nim b/compiler/options.nim index 9c0c9df72..69eafb67f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -14,6 +14,11 @@ import from terminal import isatty from times import utc, fromUnix, local, getTime, format, DateTime from std/private/globs import nativeToUnixPath + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + const hasTinyCBackend* = defined(tinyc) useEffectSystem* = true diff --git a/compiler/parser.nim b/compiler/parser.nim index 0fb60440c..7973f7d37 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -36,6 +36,9 @@ import when defined(nimpretty): import layouter +when defined(nimPreviewSlimSystem): + import std/assertions + type Parser* = object # A Parser object represents a file that # is being parsed diff --git a/compiler/passes.nim b/compiler/passes.nim index 3debce1f6..ce52b10c5 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -16,6 +16,9 @@ import syntaxes, modulegraphs, reorder, lineinfos, pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + type TPassData* = tuple[input: PNode, closeOutput: PNode] diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 3a501a417..1ef0143d5 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -13,7 +13,7 @@ import os, pathnorm when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] type AbsoluteFile* = distinct string diff --git a/compiler/patterns.nim b/compiler/patterns.nim index 4b39de3ba..87e9c825c 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -13,6 +13,9 @@ import ast, types, semdata, sigmatch, idents, aliases, parampatterns, trees +when defined(nimPreviewSlimSystem): + import std/assertions + type TPatternContext = object owner: PSym diff --git a/compiler/platform.nim b/compiler/platform.nim index 4e6054d5c..4b4316bc2 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -16,6 +16,10 @@ import strutils +when defined(nimPreviewSlimSystem): + import std/assertions + + type TSystemOS* = enum # Also add OS in initialization section and alias # conditionals to condsyms (end of module). diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 702a9e082..01399d4ff 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -14,6 +14,9 @@ import wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees, types, lookups, lineinfos, pathutils, linter +when defined(nimPreviewSlimSystem): + import std/assertions + from ic / ic import addCompilerProc const diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 22a2d4cbd..3503c4bf6 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -17,6 +17,9 @@ when defined(nimHasUsed): import lexer, options, idents, strutils, ast, msgs, lineinfos +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + type TRenderFlag* = enum renderNone, renderNoBody, renderNoComments, renderDocComments, diff --git a/compiler/reorder.nim b/compiler/reorder.nim index 65c1fb9ee..daeb48248 100644 --- a/compiler/reorder.nim +++ b/compiler/reorder.nim @@ -4,6 +4,9 @@ import msgs, modulegraphs, syntaxes, options, modulepaths, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + type DepN = ref object pnode: PNode diff --git a/compiler/rodutils.nim b/compiler/rodutils.nim index a4f7a5146..95b7b2d9e 100644 --- a/compiler/rodutils.nim +++ b/compiler/rodutils.nim @@ -10,6 +10,9 @@ ## Serialization utilities for the compiler. import strutils, math +when defined(nimPreviewSlimSystem): + import std/assertions + # bcc on windows doesn't have C99 functions when defined(windows) and defined(bcc): {.emit: """#if defined(_MSC_VER) && _MSC_VER < 1900 diff --git a/compiler/ropes.nim b/compiler/ropes.nim index cd696a545..a44d84ddc 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -60,6 +60,10 @@ import from pathutils import AbsoluteFile +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + + type FormatStr* = string # later we may change it to CString for better # performance of the code generator (assignments diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 89510f6b2..adc228d1e 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -16,6 +16,9 @@ import os, times, osproc, wordrecg, strtabs, modulegraphs, pathutils +when defined(nimPreviewSlimSystem): + import std/syncio + # we support 'cmpIgnoreStyle' natively for efficiency: from strutils import cmpIgnoreStyle, contains diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 422d1223a..d7a4d2c1c 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -11,6 +11,9 @@ import tables +when defined(nimPreviewSlimSystem): + import std/assertions + import intsets, options, ast, astalgo, msgs, idents, renderer, magicsys, vmdef, modulegraphs, lineinfos, sets, pathutils diff --git a/compiler/semfold.nim b/compiler/semfold.nim index d0473b5ce..d2bbc63be 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -17,6 +17,9 @@ import from system/memory import nimCStrLen +when defined(nimPreviewSlimSystem): + import std/assertions + proc errorType*(g: ModuleGraph): PType = ## creates a type representing an error state result = newType(tyError, nextTypeId(g.idgen), g.owners[^1]) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index d6cc4180b..9bc811a03 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -12,6 +12,9 @@ import wordrecg, strutils, options, guards, lineinfos, semfold, semdata, modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling, tables +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(useDfa): import dfa diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 5d300fc58..504b83b4c 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -14,6 +14,9 @@ import ast, astalgo, msgs, types, magicsys, semdata, renderer, options, from concepts import makeTypeDesc +when defined(nimPreviewSlimSystem): + import std/assertions + const tfInstClearedFlags = {tfHasMeta, tfUnresolved} proc checkPartialConstructedType(conf: ConfigRef; info: TLineInfo, t: PType) = diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 3d36d2533..1835d9d0f 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -13,6 +13,10 @@ import ast, tables, ropes, md5_old, modulegraphs from hashes import Hash import types +when defined(nimPreviewSlimSystem): + import std/assertions + + proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len) proc `&=`(c: var MD5Context, ch: char) = # XXX suspicious code here; relies on ch being zero terminated? diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 14f4c5e19..f195c4e45 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -15,6 +15,9 @@ import magicsys, idents, lexer, options, parampatterns, strutils, trees, linter, lineinfos, lowerings, modulegraphs, concepts +when defined(nimPreviewSlimSystem): + import std/assertions + type MismatchKind* = enum kUnknown, kAlreadyGiven, kUnknownNamedParam, kTypeMismatch, kVarNeeded, diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index 03a9702a3..e47b0483d 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -13,6 +13,9 @@ import strutils, llstream, ast, idents, lexer, options, msgs, parser, filters, filter_tmpl, renderer, lineinfos, pathutils +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + export Parser, parseAll, parseTopLevelStmt, closeParser type diff --git a/compiler/transf.nim b/compiler/transf.nim index edb8d3573..bdd7c680c 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -24,6 +24,9 @@ import lowerings, liftlocals, modulegraphs, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, cache: bool): PNode import closureiters, lambdalifting diff --git a/compiler/treetab.nim b/compiler/treetab.nim index 26afd102c..d8dd8d33e 100644 --- a/compiler/treetab.nim +++ b/compiler/treetab.nim @@ -12,6 +12,9 @@ import hashes, ast, astalgo, types +when defined(nimPreviewSlimSystem): + import std/assertions + proc hashTree*(n: PNode): Hash = if n.isNil: return diff --git a/compiler/typeallowed.nim b/compiler/typeallowed.nim index 3d6ea0edb..57dd039ad 100644 --- a/compiler/typeallowed.nim +++ b/compiler/typeallowed.nim @@ -13,6 +13,9 @@ import intsets, ast, renderer, options, semdata, types +when defined(nimPreviewSlimSystem): + import std/assertions + type TTypeAllowedFlag* = enum taField, diff --git a/compiler/types.nim b/compiler/types.nim index 007a61356..4d2801633 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -13,6 +13,9 @@ import intsets, ast, astalgo, trees, msgs, strutils, platform, renderer, options, lineinfos, int128, modulegraphs, astmsgs +when defined(nimPreviewSlimSystem): + import std/assertions + type TPreferedDesc* = enum preferName, # default diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index 721de900e..d04a7b40c 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -32,6 +32,9 @@ import ast, types, lineinfos, options, msgs, renderer, typeallowed, modulegraphs from trees import getMagic, isNoSideEffectPragma, stupidStmtListExpr from isolation_check import canAlias +when defined(nimPreviewSlimSystem): + import std/assertions + type AbstractTime = distinct int diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 1afda14b0..118a3031e 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -10,6 +10,9 @@ import ast, types, msgs, os, options, idents, lineinfos from pathutils import AbsoluteFile +when defined(nimPreviewSlimSystem): + import std/syncio + proc opSlurp*(file: string, info: TLineInfo, module: PSym; conf: ConfigRef): string = try: var filename = parentDir(toFullPath(conf, info)) / file diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 1551fbb80..72bcef4db 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -29,6 +29,9 @@ import tables +when defined(nimPreviewSlimSystem): + import std/assertions + import strutils, ast, types, msgs, renderer, vmdef, intsets, magicsys, options, lowerings, lineinfos, transf, astmsgs diff --git a/compiler/vmhooks.nim b/compiler/vmhooks.nim index 1ede87e5e..84ecf586f 100644 --- a/compiler/vmhooks.nim +++ b/compiler/vmhooks.nim @@ -9,6 +9,9 @@ import pathutils +when defined(nimPreviewSlimSystem): + import std/assertions + template setX(k, field) {.dirty.} = a.slots[a.ra].ensureKind(k) a.slots[a.ra].field = v diff --git a/compiler/vmmarshal.nim b/compiler/vmmarshal.nim index d28f0325b..83c441283 100644 --- a/compiler/vmmarshal.nim +++ b/compiler/vmmarshal.nim @@ -12,6 +12,9 @@ import streams, json, intsets, tables, ast, astalgo, idents, types, msgs, options, lineinfos +when defined(nimPreviewSlimSystem): + import std/assertions + proc ptrToInt(x: PNode): int {.inline.} = result = cast[int](x) # don't skip alignment diff --git a/compiler/vmops.nim b/compiler/vmops.nim index f13e4b1cc..640826cc8 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -31,6 +31,10 @@ from std/osproc import nil from system/formatfloat import addFloatRoundtrip, addFloatSprintf +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + # There are some useful procs in vmconv. import vmconv, vmmarshal diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4d040cebc..f20ca9385 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -10,6 +10,10 @@ include "system/inclrtl" import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + ## This module contains the interface to the compiler's abstract syntax ## tree (`AST`:idx:). Macros operate on this tree. ## diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index 1ddcc9843..c43545f78 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -44,6 +44,10 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type SortOrder* = enum Descending, Ascending diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index 08f5208d2..c36e31b11 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -16,6 +16,9 @@ import std/private/since import strutils +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(windows): import winlean from os import absolutePath diff --git a/lib/pure/collections/hashcommon.nim b/lib/pure/collections/hashcommon.nim index a169418ce..deff8fa21 100644 --- a/lib/pure/collections/hashcommon.nim +++ b/lib/pure/collections/hashcommon.nim @@ -10,6 +10,10 @@ # An `include` file which contains common code for # hash sets and tables. +when defined(nimPreviewSlimSystem): + import std/assertions + + const growthFactor = 2 diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 64a7be7a9..5e9b492c2 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -84,6 +84,10 @@ import std/private/since import macros +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(nimHasEffectsOf): {.experimental: "strictEffects".} else: diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 47dacec7d..e88210757 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -67,6 +67,10 @@ runnableExamples: import std/private/since +when defined(nimPreviewSlimSystem): + import std/assertions + + type Hash* = int ## A hash value. Hash tables using these values should ## always have a size of a power of two so they can use the `and` diff --git a/lib/pure/json.nim b/lib/pure/json.nim index bdc9fe5ab..dd9232ea6 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -165,7 +165,7 @@ import options # xxx remove this dependency using same approach as https://githu import std/private/since when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] export tables.`$` diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index bbc0a38ae..336a57ec1 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -14,6 +14,9 @@ import strutils, streams +when defined(nimPreviewSlimSystem): + import std/assertions + const EndOfFile* = '\0' ## end of file marker NewLines* = {'\c', '\L'} diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 1c47258bc..15324f882 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -60,6 +60,10 @@ import std/private/since import bitops, fenv +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(c) or defined(cpp): proc c_isnan(x: float): bool {.importc: "isnan", header: "".} # a generic like `x: SomeFloat` might work too if this is implemented via a C macro. diff --git a/lib/pure/options.nim b/lib/pure/options.nim index 850bfa555..562ed6361 100644 --- a/lib/pure/options.nim +++ b/lib/pure/options.nim @@ -71,6 +71,10 @@ supports pattern matching on `Option`s, with the `Some()` and import typetraits +when defined(nimPreviewSlimSystem): + import std/assertions + + when (NimMajor, NimMinor) >= (1, 1): type SomePointer = ref | ptr | pointer | proc diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 7c0c64276..fa379a228 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -35,7 +35,7 @@ import std/private/since import strutils, pathnorm when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] const weirdTarget = defined(nimscript) or defined(js) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 3da9737ec..dca5099a4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -31,6 +31,10 @@ else: when defined(linux) and defined(useClone): import linux +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + + type ProcessOption* = enum ## Options that can be passed to `startProcess proc ## <#startProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_. diff --git a/lib/pure/parsejson.nim b/lib/pure/parsejson.nim index 196d8c360..c92eac26e 100644 --- a/lib/pure/parsejson.nim +++ b/lib/pure/parsejson.nim @@ -14,6 +14,9 @@ import strutils, lexbase, streams, unicode import std/private/decode_helpers +when defined(nimPreviewSlimSystem): + import std/assertions + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing diff --git a/lib/pure/streamwrapper.nim b/lib/pure/streamwrapper.nim index 7a501760b..a6c1901d2 100644 --- a/lib/pure/streamwrapper.nim +++ b/lib/pure/streamwrapper.nim @@ -13,6 +13,10 @@ import deques, streams +when defined(nimPreviewSlimSystem): + import std/assertions + + type PipeOutStream*[T] = ref object of T # When stream peek operation is called, it reads from base stream diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index 40a33951c..ce3439600 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -316,6 +316,10 @@ single letter DSLs. import macros, parseutils, unicode import strutils except format +when defined(nimPreviewSlimSystem): + import std/assertions + + proc mkDigit(v: int, typ: char): string {.inline.} = assert(v < 26) if v < 10: diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 3b90fea50..aa2886cfa 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -53,6 +53,10 @@ import std/private/since import hashes, strutils +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js) or defined(nimscript) or defined(Standalone): {.pragma: rtlFunc.} else: diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 5e505ec3a..bf7bd6aa8 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -82,6 +82,9 @@ include "system/inclrtl" import std/private/since from std/private/strimpl import cmpIgnoreStyleImpl, cmpIgnoreCaseImpl, startsWithImpl, endsWithImpl +when defined(nimPreviewSlimSystem): + import std/assertions + const Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'} diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 5755e142a..571c9b13c 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -67,7 +67,7 @@ when defined(windows): import winlean when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] type PTerminal = ref object diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 7851bf158..b70c5cedc 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -201,6 +201,10 @@ import strutils, math, options import std/private/since include "system/inclrtl" +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(js): import jscore diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 8dc1b8cf2..3fc1c7c5c 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -15,6 +15,10 @@ import std/private/since export system.`$` # for backward compatibility +when defined(nimPreviewSlimSystem): + import std/assertions + + type HoleyEnum* = (not Ordinal) and enum ## Enum with holes. type OrdinalEnum* = Ordinal and enum ## Enum without holes. diff --git a/lib/system/assertions.nim b/lib/std/assertions.nim similarity index 94% rename from lib/system/assertions.nim rename to lib/std/assertions.nim index 6f64a55b7..3d2112b1a 100644 --- a/lib/system/assertions.nim +++ b/lib/std/assertions.nim @@ -1,7 +1,13 @@ -## This module provides various assertion utilities. -## -## **Note:** This module is reexported by `system` and thus does not need to be -## imported directly (with `system/assertions`). +# +# +# Nim's Runtime Library +# (c) Copyright 2022 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements assertion handling. when not declared(sysFatal): include "system/fatal" diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index b7d2b9f89..9d4ff1bcf 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -10,6 +10,10 @@ import macros from typetraits import OrdinalEnum, HoleyEnum +when defined(nimPreviewSlimSystem): + import std/assertions + + # xxx `genEnumCaseStmt` needs tests and runnableExamples macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index 19384b5d1..722ea49b5 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -34,6 +34,9 @@ import macros from enumutils import symbolName from typetraits import OrdinalEnum +when defined(nimPreviewSlimSystem): + import std/assertions + when not defined(nimFixedForwardGeneric): # xxx remove pending csources_v1 update >= 1.2.0 proc to[T](node: JsonNode, t: typedesc[T]): T = diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index b2ee917eb..1e2892658 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -24,6 +24,9 @@ import std/private/since import hashes +when defined(nimPreviewSlimSystem): + import std/assertions + type BitScalar = uint diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index 190316f93..28a810372 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -8,6 +8,10 @@ import os when defined(windows): from strutils import replace +when defined(nimPreviewSlimSystem): + import std/assertions + + when defined(nimHasEffectsOf): {.experimental: "strictEffects".} else: diff --git a/lib/std/sha1.nim b/lib/std/sha1.nim index ed962707b..50175024c 100644 --- a/lib/std/sha1.nim +++ b/lib/std/sha1.nim @@ -29,6 +29,9 @@ runnableExamples("-r:off"): import strutils from endians import bigEndian32, bigEndian64 +when defined(nimPreviewSlimSystem): + import std/syncio + const Sha1DigestSize = 20 type diff --git a/lib/system.nim b/lib/system.nim index 572768de2..4080fee06 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1830,8 +1830,11 @@ when not defined(nimscript): when defined(nimV2): include system/arc -import system/assertions -export assertions +when not defined(nimPreviewSlimSystem): + {.deprecated: """assertions is about to move out of system; use `-d:nimPreviewSlimSystem` and + import `std/assertions`.""".} + import std/assertions + export assertions import system/iterators export iterators diff --git a/lib/system/dragonbox.nim b/lib/system/dragonbox.nim index 336c982c1..34ae9e210 100644 --- a/lib/system/dragonbox.nim +++ b/lib/system/dragonbox.nim @@ -24,6 +24,8 @@ import std/private/digitsutils +when defined(nimPreviewSlimSystem): + import std/assertions const dtoaMinBufferLength*: cint = 64 diff --git a/lib/system/formatfloat.nim b/lib/system/formatfloat.nim index 3bcd3257b..aada3e1bf 100644 --- a/lib/system/formatfloat.nim +++ b/lib/system/formatfloat.nim @@ -7,6 +7,9 @@ # distribution, for details about the copyright. # +when defined(nimPreviewSlimSystem): + import std/assertions + proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "", discardable.} proc addCstringN(result: var string, buf: cstring; buflen: int) = diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index f23f7aa11..6d33fc0bc 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -1,3 +1,6 @@ +when defined(nimPreviewSlimSystem): + import std/assertions + when defined(nimHasLentIterators) and not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: diff --git a/lib/system/schubfach.nim b/lib/system/schubfach.nim index 06813f632..7d6861e35 100644 --- a/lib/system/schubfach.nim +++ b/lib/system/schubfach.nim @@ -12,6 +12,9 @@ import std/private/digitsutils +when defined(nimPreviewSlimSystem): + import std/assertions + template sf_Assert(x: untyped): untyped = assert(x) From e93eaac223045543e753643765b6982a109b41a1 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 03:42:08 +0800 Subject: [PATCH 0731/1654] build external deps with userversion:1.6 (#19612) --- koch.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index 295b1584b..a3cfd5930 100644 --- a/koch.nim +++ b/koch.nim @@ -142,7 +142,7 @@ proc csource(args: string) = proc bundleC2nim(args: string) = cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git") nimCompile("dist/c2nim/c2nim", - options = "--noNimblePath --path:. " & args) + options = "--noNimblePath --useVersion:1.6 --path:. " & args) proc bundleNimbleExe(latest: bool, args: string) = let commit = if latest: "HEAD" else: NimbleStableCommit @@ -150,7 +150,7 @@ proc bundleNimbleExe(latest: bool, args: string) = commit = commit, allowBundled = true) # installer.ini expects it under $nim/bin nimCompile("dist/nimble/src/nimble.nim", - options = "-d:release --noNimblePath " & args) + options = "-d:release --useVersion:1.6 --noNimblePath " & args) proc bundleNimsuggest(args: string) = nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", From d14e8e8dccc87671acb832b4aea8e736ce60c13e Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 23 Mar 2022 16:42:54 -0300 Subject: [PATCH 0732/1654] Remove Deprecated Nimscript proc (#19629) --- lib/system/nimscript.nim | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index b4554d778..0ef5fc584 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -136,14 +136,6 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -template existsFile*(args: varargs[untyped]): untyped {.deprecated: "use fileExists".} = - # xxx: warning won't be shown for nimsscript because of current logic handling - # `foreignPackageNotes` - fileExists(args) - -template existsDir*(args: varargs[untyped]): untyped {.deprecated: "use dirExists".} = - dirExists(args) - proc selfExe*(): string = ## Returns the currently running nim or nimble executable. # TODO: consider making this as deprecated alias of `getCurrentCompilerExe` From d7a966c4ebdb13a1cc414ce98bbff87ffa922261 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 23 Mar 2022 16:43:28 -0300 Subject: [PATCH 0733/1654] Remove Deprecated domextension (#19630) * Remove Deprecated domextension * Remove Deprecated domextension --- changelog.md | 1 + lib/js/dom_extensions.nim | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 lib/js/dom_extensions.nim diff --git a/changelog.md b/changelog.md index a9bf3842a..73cb5e2b5 100644 --- a/changelog.md +++ b/changelog.md @@ -46,6 +46,7 @@ becomes an alias for `addr`. - Removed deprecated `jsre.test` and `jsre.toString`. - Removed deprecated `math.c_frexp`. - Removed deprecated ``` httpcore.`==` ```. +- Removed deprecated `std/dom_extensions`. ## Language changes diff --git a/lib/js/dom_extensions.nim b/lib/js/dom_extensions.nim deleted file mode 100644 index c15a03195..000000000 --- a/lib/js/dom_extensions.nim +++ /dev/null @@ -1,3 +0,0 @@ -import std/dom -export elementsFromPoint -{.deprecated: "use `std/dom` instead".} From b0bd4320a0cd9a90acf44bca0dddf10fa5022969 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 23 Mar 2022 16:43:58 -0300 Subject: [PATCH 0734/1654] Remove deprecated typo poDemon (#19631) * Remove Deprecated poDemon * Remove Deprecated poDemon --- changelog.md | 2 ++ lib/pure/osproc.nim | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/changelog.md b/changelog.md index 73cb5e2b5..92d2612e8 100644 --- a/changelog.md +++ b/changelog.md @@ -48,6 +48,8 @@ becomes an alias for `addr`. - Removed deprecated ``` httpcore.`==` ```. - Removed deprecated `std/dom_extensions`. +- Remove deprecated `osproc.poDemon`, symbol with typo. + ## Language changes diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index dca5099a4..512db92a4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -70,11 +70,6 @@ type Process* = ref ProcessObj ## Represents an operating system process. -const poDemon* {.deprecated.} = poDaemon ## Nim versions before 0.20 - ## used the wrong spelling ("demon"). - ## Now `ProcessOption` uses the correct spelling ("daemon"), - ## and this is needed just for backward compatibility. - proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, From 2c01c9c4c8a20772ebbb91f3333feb5dbcc94e9e Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 03:57:13 +0800 Subject: [PATCH 0735/1654] output byref types into --header file [backport: 1.6] (#19505) * output byref types into --header file fix #19445 * fix comments * set targets --- compiler/ccgtypes.nim | 9 +++++++-- tests/ccgbugs/m19445.c | 3 +++ tests/ccgbugs/t19445.nim | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/ccgbugs/m19445.c create mode 100644 tests/ccgbugs/t19445.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index f010b25c9..c5b4d3857 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -471,8 +471,13 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope, var arr = t[0] if params != nil: params.add(", ") if mapReturnType(m.config, t[0]) != ctArray: - params.add(getTypeDescWeak(m, arr, check, skResult)) - params.add("*") + if isHeaderFile in m.flags: + # still generates types for `--header` + params.add(getTypeDescAux(m, arr, check, skResult)) + params.add("*") + else: + params.add(getTypeDescWeak(m, arr, check, skResult)) + params.add("*") else: params.add(getTypeDescAux(m, arr, check, skResult)) params.addf(" Result", []) diff --git a/tests/ccgbugs/m19445.c b/tests/ccgbugs/m19445.c new file mode 100644 index 000000000..74c23d4b4 --- /dev/null +++ b/tests/ccgbugs/m19445.c @@ -0,0 +1,3 @@ +#include "m19445.h" + +const Foo f = {10, 20, 30, 40}; \ No newline at end of file diff --git a/tests/ccgbugs/t19445.nim b/tests/ccgbugs/t19445.nim new file mode 100644 index 000000000..b6e8d028c --- /dev/null +++ b/tests/ccgbugs/t19445.nim @@ -0,0 +1,13 @@ +discard """ + matrix: "--nimcache:tests/ccgbugs/nimcache19445 --cincludes:nimcache19445 --header:m19445" + targets: "c" +""" + +# bug #19445 +type + Foo* {.exportc.} = object + a*, b*, c*, d*: int + +proc dummy(): Foo {.exportc.} = discard + +{.compile:"m19445.c".} \ No newline at end of file From a262a87bbebdaf41ea248f7caa780f8ea27e20e1 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 04:07:05 +0800 Subject: [PATCH 0736/1654] [add testcase] arc problems with recursive types (#19456) * [add testcase] arc problems with recursive types close #9650 * do test * expand * Update tests/arc/t9650.nim --- tests/arc/t9650.nim | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/arc/t9650.nim diff --git a/tests/arc/t9650.nim b/tests/arc/t9650.nim new file mode 100644 index 000000000..a8182db68 --- /dev/null +++ b/tests/arc/t9650.nim @@ -0,0 +1,87 @@ +discard """ + matrix: "--gc:arc" +""" + +import typetraits + +# bug #9650 +type + SharedPtr*[T] = object + val: ptr tuple[atomicCounter: int, value: T] + + Node*[T] = object + value: T + next: SharedPtr[Node[T]] + + ForwardList*[T] = object + first: SharedPtr[Node[T]] + +proc `=destroy`*[T](p: var SharedPtr[T]) = + if p.val != nil: + let c = atomicDec(p.val[].atomicCounter) + if c == 0: + when not supportsCopyMem(T): + `=destroy`(p.val[]) + dealloc(p.val) + p.val = nil + +proc `=`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if dest.val != src.val: + if dest.val != nil: + `=destroy`(dest) + if src.val != nil: + discard atomicInc(src.val[].atomicCounter) + dest.val = src.val + +proc `=sink`*[T](dest: var SharedPtr[T], src: SharedPtr[T]) = + if dest.val != nil and dest.val != src.val: + `=destroy`(dest) + dest.val = src.val + + +proc newSharedPtr*[T](val: sink T): SharedPtr[T] = + result.val = cast[type(result.val)](alloc(sizeof(result.val[]))) + reset(result.val[]) + result.val.atomicCounter = 1 + result.val.value = val + +proc isNil*[T](p: SharedPtr[T]): bool = + p.val == nil + +template `->`*[T](p: SharedPtr[T], name: untyped): untyped = + p.val.value.name + +proc createNode[T](val: T): SharedPtr[ Node[T] ]= + result = newSharedPtr(Node[T](value: val)) + +proc push_front*[T](list: var ForwardList[T], val: T) = + var newElem = createNode(val) + newElem->next = list.first + list.first = newElem + +proc pop_front*[T](list: var ForwardList[T]) = + let head = list.first + list.first = head->next + +proc toString*[T](list: ForwardList[T]): string = + result = "[" + var head = list.first + while not head.isNil: + result &= $(head->value) & ", " + head = head->next + result &= ']' + +block: + var x: ForwardList[int] + x.push_front(1) + x.push_front(2) + x.push_front(3) + + doAssert toString(x) == "[3, 2, 1, ]" + + x.pop_front() + x.pop_front() + doAssert toString(x) == "[1, ]" + + x.pop_front() + doAssert toString(x) == "[]" From 7c3c61f2f1e898e5c70dfd619d94d09e339836e4 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 24 Mar 2022 04:09:44 +0800 Subject: [PATCH 0737/1654] fix nim check nimscript [backport: 1.6] (#19444) fix #19440; fix #3858 --- compiler/commands.nim | 2 ++ compiler/main.nim | 6 +++++- tests/newconfig/tconfigcheck.nims | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/newconfig/tconfigcheck.nims diff --git a/compiler/commands.nim b/compiler/commands.nim index 4c1cec3d2..98537e9ef 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1112,6 +1112,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; else: if pass == passCmd1: config.commandArgs.add p.key if argsCount == 1: + if p.key.endsWith(".nims"): + incl(config.globalOptions, optWasNimscript) # support UNIX style filenames everywhere for portable build scripts: if config.projectName.len == 0: config.projectName = unixToNativePath(p.key) diff --git a/compiler/main.nim b/compiler/main.nim index 3cddbffb3..e27a4f6d1 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -62,6 +62,9 @@ proc commandCheck(graph: ModuleGraph) = let conf = graph.config conf.setErrorMaxHighMaybe defineSymbol(conf.symbols, "nimcheck") + if optWasNimscript in conf.globalOptions: + defineSymbol(conf.symbols, "nimscript") + defineSymbol(conf.symbols, "nimconfig") semanticPasses(graph) # use an empty backend for semantic checking only compileProject(graph) @@ -367,7 +370,8 @@ proc mainCommand*(graph: ModuleGraph) = msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook}) for it in conf.searchPaths: msgWriteln(conf, it.string) - of cmdCheck: commandCheck(graph) + of cmdCheck: + commandCheck(graph) of cmdParse: wantMainModule(conf) discard parseFile(conf.projectMainIdx, cache, conf) diff --git a/tests/newconfig/tconfigcheck.nims b/tests/newconfig/tconfigcheck.nims new file mode 100644 index 000000000..2e12a0bda --- /dev/null +++ b/tests/newconfig/tconfigcheck.nims @@ -0,0 +1,11 @@ +discard """ + cmd: "nim check $file" +""" + +mode = ScriptMode.Verbose +proc build() = + echo "building nim... " + exec "sleep 10" + echo getCurrentDir() + +echo "hello" From 400e0260b854f34a91af47bbe0a1b1c9d112e51b Mon Sep 17 00:00:00 2001 From: treeform Date: Wed, 23 Mar 2022 23:35:29 -0700 Subject: [PATCH 0738/1654] Add more info for {.bycopy.} (#18815) * Add more info for {.bycopy.} See confusion here: https://github.com/nim-lang/Nim/issues/18807 I hope this will help people googling to find this. * Update doc/manual.rst Co-authored-by: Andreas Rumpf --- doc/manual.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual.rst b/doc/manual.rst index 4c4b9c506..8206d8342 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7912,6 +7912,7 @@ instructs the compiler to pass the type by value to procs: Vector {.bycopy.} = object x, y, z: float +The Nim compiler automatically determines whether a parameter is passed by value or by reference based on the parameter type's size. If a parameter must be passed by value or by reference, (such as when interfacing with a C library) use the bycopy or byref pragmas. Byref pragma ------------ From afbcba909b37b8c06250b141ddb9da4bf5bb9922 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 25 Mar 2022 17:46:42 +0800 Subject: [PATCH 0739/1654] remove unnecessary framePtr code (#19645) --- compiler/jsgen.nim | 9 ++++++--- lib/system/jssys.nim | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f4097efa4..d7955b2f9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -773,7 +773,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = var tmpFramePtr = rope"F" if optStackTrace notin p.options: tmpFramePtr = p.getTemp(true) - line(p, tmpFramePtr & " = framePtr;\L") + if hasFrameInfo(p): + line(p, tmpFramePtr & " = framePtr;\L") lineF(p, "try {$n", []) var a: TCompRes gen(p, n[0], a) @@ -782,7 +783,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = if catchBranchesExist: p.body.addf("--excHandler;$n} catch (EXCEPTION) {$n var prevJSError = lastJSError;$n" & " lastJSError = EXCEPTION;$n --excHandler;$n", []) - line(p, "framePtr = $1;$n" % [tmpFramePtr]) + if hasFrameInfo(p): + line(p, "framePtr = $1;$n" % [tmpFramePtr]) while i < n.len and n[i].kind == nkExceptBranch: if n[i].len == 1: # general except section: @@ -841,7 +843,8 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = line(p, "}\L") lineF(p, "lastJSError = prevJSError;$n") line(p, "} finally {\L") - line(p, "framePtr = $1;$n" % [tmpFramePtr]) + if hasFrameInfo(p): + line(p, "framePtr = $1;$n" % [tmpFramePtr]) if i < n.len and n[i].kind == nkFinally: genStmt(p, n[i][0]) line(p, "}\L") diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 6608a2927..00a4a8ab6 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -130,7 +130,8 @@ proc unhandledException(e: ref Exception) {. when NimStackTrace: add(buf, rawWriteStackTrace()) let cbuf = cstring(buf) - framePtr = nil + when NimStackTrace: + framePtr = nil {.emit: """ if (typeof(Error) !== "undefined") { throw new Error(`cbuf`); From 12a0f88a52bae74e31d6d612f0e2aae3f90340a9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Mar 2022 06:37:04 +0100 Subject: [PATCH 0740/1654] mitigates #19364 [backport]; we make this bug more unlikely to appear by producing better code to begin with; real fix will come later (#19647) --- compiler/lowerings.nim | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 20003b946..382e48a16 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -74,14 +74,20 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P let value = n.lastSon result = newNodeI(nkStmtList, n.info) - var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen), - owner, value.info, g.config.options) - temp.typ = skipTypes(value.typ, abstractInst) - incl(temp.flags, sfFromGeneric) + var tempAsNode: PNode + let avoidTemp = value.kind == nkSym + if avoidTemp: + tempAsNode = value + else: + var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen), + owner, value.info, g.config.options) + temp.typ = skipTypes(value.typ, abstractInst) + incl(temp.flags, sfFromGeneric) + tempAsNode = newSymNode(temp) var v = newNodeI(nkVarSection, value.info) - let tempAsNode = newSymNode(temp) - v.addVar(tempAsNode, value) + if not avoidTemp: + v.addVar(tempAsNode, value) result.add(v) for i in 0.. Date: Sat, 26 Mar 2022 06:47:04 +0100 Subject: [PATCH 0741/1654] fixes #19603; some pragmas were really only supported as top level statements. Now this is enforced properly. (#19646) --- compiler/pragmas.nim | 15 +++++++++------ compiler/semexprs.nim | 8 +++++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 01399d4ff..81c6a42d3 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -47,18 +47,21 @@ const wDiscardable, wGensym, wInject, wRaises, wEffectsOf, wTags, wLocks, wGcSafe, wRequires, wEnsures} exprPragmas* = {wLine, wLocks, wNoRewrite, wGcSafe, wNoSideEffect} - stmtPragmas* = {wChecks, wObjChecks, wFieldChecks, wRangeChecks, - wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks, - wStyleChecks, wAssertions, - wWarnings, wHints, - wLineDir, wStackTrace, wLineTrace, wOptimization, wHint, wWarning, wError, + stmtPragmas* = { + wHint, wWarning, wError, wFatal, wDefine, wUndef, wCompile, wLink, wLinksys, wPure, wPush, wPop, wPassl, wPassc, wLocalPassc, wDeadCodeElimUnused, # deprecated, always on wDeprecated, - wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll, + wPragma, wEmit, wUnroll, wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto, wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert} + stmtPragmasTopLevel* = {wChecks, wObjChecks, wFieldChecks, wRangeChecks, + wBoundChecks, wOverflowChecks, wNilChecks, wStaticBoundchecks, + wStyleChecks, wAssertions, + wWarnings, wHints, + wLineDir, wStackTrace, wLineTrace, wOptimization, + wFloatChecks, wInfChecks, wNanChecks} lambdaPragmas* = {FirstCallConv..LastCallConv, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader, wThread, wAsmNoStackFrame, diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 5f3d065b6..f224bd0b3 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2769,6 +2769,12 @@ proc enumFieldSymChoice(c: PContext, n: PNode, s: PSym): PNode = onUse(info, a) a = nextOverloadIter(o, c, n) +proc semPragmaStmt(c: PContext; n: PNode) = + if c.p.owner.kind == skModule: + pragma(c, c.p.owner, n, stmtPragmas+stmtPragmasTopLevel, true) + else: + pragma(c, c.p.owner, n, stmtPragmas, true) + proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = when defined(nimCompilerStacktraceHints): setFrameMsg c.config$n.info & " " & $n.kind @@ -3017,7 +3023,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkUsingStmt: result = semUsing(c, n) of nkAsmStmt: result = semAsm(c, n) of nkYieldStmt: result = semYield(c, n) - of nkPragma: pragma(c, c.p.owner, n, stmtPragmas, true) + of nkPragma: semPragmaStmt(c, n) of nkIteratorDef: result = semIterator(c, n) of nkProcDef: result = semProc(c, n) of nkFuncDef: result = semFunc(c, n) From 4dfe420c2f1791c722ff949072e1a74f3e7c1499 Mon Sep 17 00:00:00 2001 From: tandy1000 Date: Sat, 26 Mar 2022 05:57:39 +0000 Subject: [PATCH 0742/1654] Add support for `Window.matchMedia` (#19648) --- lib/js/dom.nim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index aca79faca..c1a677258 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -1322,6 +1322,11 @@ type `block`*: cstring inline*: cstring + MediaQueryList* = ref MediaQueryListObj + MediaQueryListObj {.importc.} = object of EventTargetObj + matches*: bool + media*: cstring + since (1, 3): type DomParser* = ref object @@ -1529,6 +1534,7 @@ proc setTimeout*(w: Window, function: proc (), pause: int): Interval proc stop*(w: Window) proc requestAnimationFrame*(w: Window, function: proc (time: float)): int proc cancelAnimationFrame*(w: Window, id: int) +proc matchMedia*(w: Window, mediaQueryString: cstring): MediaQueryList # Node "methods" proc appendData*(n: Node, data: cstring) From 84ac0035190e81cb991544460786bd5618386bec Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 26 Mar 2022 15:57:29 +0800 Subject: [PATCH 0743/1654] remove tmpFramePtr when optStackTrace is absent (#19649) --- compiler/jsgen.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d7955b2f9..b9a73daf9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -771,10 +771,6 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = if catchBranchesExist: p.body.add("++excHandler;\L") var tmpFramePtr = rope"F" - if optStackTrace notin p.options: - tmpFramePtr = p.getTemp(true) - if hasFrameInfo(p): - line(p, tmpFramePtr & " = framePtr;\L") lineF(p, "try {$n", []) var a: TCompRes gen(p, n[0], a) From 86e93eb2938329db921d5edb718fa7f3ecee83b1 Mon Sep 17 00:00:00 2001 From: Omar Flores <47699779+yoyojambo@users.noreply.github.com> Date: Sat, 26 Mar 2022 02:56:10 -0600 Subject: [PATCH 0744/1654] Fixed description as option in advopt.txt (#19635) * Fixed formatting error in warningAsError. There was only a single space character between the warning and its description, so it shows up as part of the name (in bold) and with no description. Copied the way hotCodeReloading was formatted, with the description in a new line. Also changed descriptions from 'ditto' to what the description it references says. * Fixed typo in advopt.txt Corrected 'hints' to 'warnings' in updated advopt.txt. --- doc/advopt.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 0a2349597..244fe035b 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -55,9 +55,13 @@ Advanced options: (only `all:off` is supported). --hintAsError:X:on|off turn specific hint X into an error on|off -w:on|off|list, --warnings:on|off|list - same as `--hints` but for warnings. - --warning:X:on|off ditto - --warningAsError:X:on|off ditto + `on|off` enables or disables warnings. + `list` reports which warnings are selected. + --warning:X:on|off turn specific warning X on|off. `warning:X` means `warning:X:on`, + as with similar flags. `all` is the set of all warning + (only `all:off` is supported). + --warningAsError:X:on|off + turn specific warning X into an error on|off --styleCheck:off|hint|error produce hints or errors for Nim identifiers that do not adhere to Nim's official style guide From 8cdd8867c0237c28f7a43e6f871507eda207a15d Mon Sep 17 00:00:00 2001 From: John Titor Date: Sat, 26 Mar 2022 10:43:26 +0000 Subject: [PATCH 0745/1654] Fix dial ignoring buffered parameter (#19650) [backport] --- lib/pure/net.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c1d4c6228..9922bbfa4 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1963,7 +1963,7 @@ proc dial*(address: string, port: Port, closeUnusedFds(ord(domain)) if success: - result = newSocket(lastFd, domain, sockType, protocol) + result = newSocket(lastFd, domain, sockType, protocol, buffered) elif lastError != 0.OSErrorCode: raiseOSError(lastError) else: From 82319ef00dc9ce6f65b1664024a10ee5d9465f38 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 26 Mar 2022 18:50:15 +0800 Subject: [PATCH 0746/1654] fix #8219; nim check/dump shouldn't run single nimscript project [backport: 1.6] (#19641) * fix #8219; nim check/dump shouldn't run single nimscript project [backport: 1.6] --- compiler/nimconf.nim | 3 ++- tests/misc/trunner.nim | 5 +++++ tests/newconfig/{tconfigcheck.nims => mconfigcheck.nims} | 6 ++---- 3 files changed, 9 insertions(+), 5 deletions(-) rename tests/newconfig/{tconfigcheck.nims => mconfigcheck.nims} (74%) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 319a3de6d..fceedb2c4 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -301,7 +301,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: if conf.cmd == cmdNimscript: showHintConf() conf.configFiles.setLen 0 - if conf.cmd != cmdIdeTools: + if conf.cmd notin {cmdIdeTools, cmdCheck, cmdDump}: if conf.cmd == cmdNimscript: runNimScriptIfExists(conf.projectFull, isMain = true) else: @@ -311,5 +311,6 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: runNimScriptIfExists(scriptFile, isMain = true) else: # 'nimsuggest foo.nims' means to just auto-complete the NimScript file + # `nim check foo.nims' means to check the syntax of the NimScript file discard showHintConf() diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index f874d38d9..5d12c38b6 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -249,6 +249,11 @@ tests/newconfig/bar/mfoo.nims""".splitLines expected.add &"Hint: used config file '{b}' [Conf]\n" doAssert outp.endsWith expected, outp & "\n" & expected + block: # bug #8219 + let file = "tests/newconfig/mconfigcheck.nims" + let cmd = fmt"{nim} check --hints:off {file}" + check execCmdEx(cmd) == ("", 0) + block: # mfoo2.customext let filename = testsDir / "newconfig/foo2/mfoo2.customext" let cmd = fmt"{nim} e --hint:conf {filename}" diff --git a/tests/newconfig/tconfigcheck.nims b/tests/newconfig/mconfigcheck.nims similarity index 74% rename from tests/newconfig/tconfigcheck.nims rename to tests/newconfig/mconfigcheck.nims index 2e12a0bda..8df6715f6 100644 --- a/tests/newconfig/tconfigcheck.nims +++ b/tests/newconfig/mconfigcheck.nims @@ -1,11 +1,9 @@ -discard """ - cmd: "nim check $file" -""" - mode = ScriptMode.Verbose proc build() = echo "building nim... " exec "sleep 10" + exec "nonexistant command" echo getCurrentDir() echo "hello" +build() From 7d32425a577053bda356ca71c81d6701d8ad6372 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Mar 2022 13:29:04 +0100 Subject: [PATCH 0747/1654] fixes #14318 (#14335) --- compiler/pragmas.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 81c6a42d3..458c7547a 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -656,7 +656,7 @@ proc pragmaRaisesOrTags(c: PContext, n: PNode) = x.typ = makeTypeFromExpr(c, x) else: var t = skipTypes(c.semTypeNode(c, x, nil), skipPtrs) - if t.kind != tyObject and not t.isMetaType: + if t.kind notin {tyObject, tyOr}: localError(c.config, x.info, errGenerated, "invalid type for raises/tags list") x.typ = t From eae29e8eafded41ce966c131bff5e66964354805 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Mon, 28 Mar 2022 03:37:59 -0700 Subject: [PATCH 0748/1654] Embedded Network patches - eventfd & socket getters (#19632) --- lib/pure/ioselects/ioselectors_poll.nim | 35 ++++++++++++++++++------- lib/pure/net.nim | 10 +++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index 0d8fef78a..12812ac80 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -14,6 +14,12 @@ import posix, times # Maximum number of events that can be returned const MAX_POLL_EVENTS = 64 +const hasEventFds = defined(zephyr) or defined(nimPollHasEventFds) + +when hasEventFds: + proc eventfd(count: cuint, flags: cint): cint + {.cdecl, importc: "eventfd", header: "".} + when hasThreadSupport: type SelectorImpl[T] = object @@ -184,14 +190,22 @@ proc unregister*[T](s: Selector[T], ev: SelectEvent) = s.pollRemove(fdi.cint) proc newSelectEvent*(): SelectEvent = - var fds: array[2, cint] - if posix.pipe(fds) != 0: - raiseIOSelectorsError(osLastError()) - setNonBlocking(fds[0]) - setNonBlocking(fds[1]) - result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) - result.rfd = fds[0] - result.wfd = fds[1] + when not hasEventFds: + var fds: array[2, cint] + if posix.pipe(fds) != 0: + raiseIOSelectorsError(osLastError()) + setNonBlocking(fds[0]) + setNonBlocking(fds[1]) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fds[0] + result.wfd = fds[1] + else: + let fdci = eventfd(0, posix.O_NONBLOCK) + if fdci == -1: + raiseIOSelectorsError(osLastError()) + result = cast[SelectEvent](allocShared0(sizeof(SelectEventImpl))) + result.rfd = fdci + result.wfd = fdci proc trigger*(ev: SelectEvent) = var data: uint64 = 1 @@ -200,7 +214,10 @@ proc trigger*(ev: SelectEvent) = proc close*(ev: SelectEvent) = let res1 = posix.close(ev.rfd) - let res2 = posix.close(ev.wfd) + let res2 = + when hasEventFds: 0 + else: posix.close(ev.wfd) + deallocShared(cast[pointer](ev)) if res1 != 0 or res2 != 0: raiseIOSelectorsError(osLastError()) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 9922bbfa4..c59a4e563 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1798,6 +1798,16 @@ proc isSsl*(socket: Socket): bool = proc getFd*(socket: Socket): SocketHandle = return socket.fd ## Returns the socket's file descriptor +when defined(zephyr) or defined(nimNetSocketExtras): # Remove in future + proc getDomain*(socket: Socket): Domain = return socket.domain + ## Returns the socket's domain + + proc getType*(socket: Socket): SockType = return socket.sockType + ## Returns the socket's type + + proc getProtocol*(socket: Socket): Protocol = return socket.protocol + ## Returns the socket's protocol + when defined(nimHasStyleChecks): {.push styleChecks: off.} From 49844415ce829cb627d0b02dd6b057a1574f44d6 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 31 Mar 2022 16:06:13 -0300 Subject: [PATCH 0749/1654] Deprecate selfExe (#19660) * Deprecate selfExe Nimscript * Deprecate selfExe Nimscript --- changelog.md | 2 ++ lib/system/nimscript.nim | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 92d2612e8..8b9d0bc29 100644 --- a/changelog.md +++ b/changelog.md @@ -50,6 +50,8 @@ becomes an alias for `addr`. - Remove deprecated `osproc.poDemon`, symbol with typo. +- Deprecated `selfExe` for Nimscript. + ## Language changes diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 0ef5fc584..5d1916c7f 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -136,9 +136,8 @@ proc dirExists*(dir: string): bool {. ## Checks if the directory `dir` exists. builtin -proc selfExe*(): string = +proc selfExe*(): string {.deprecated: "Deprecated since v1.7; Use getCurrentCompilerExe".} = ## Returns the currently running nim or nimble executable. - # TODO: consider making this as deprecated alias of `getCurrentCompilerExe` builtin proc toExe*(filename: string): string = From 1275763284d540c0e8ad53b3c27e52743ee08f2b Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 1 Apr 2022 03:01:51 -0300 Subject: [PATCH 0750/1654] fix 19655 - fixing url fragment (#19667) --- lib/pure/parseopt.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index b2d024a39..aff9312f9 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -16,7 +16,7 @@ ## ## The following syntax is supported when arguments for the `shortNoVal` and ## `longNoVal` parameters, which are -## `described later<#shortnoval-and-longnoval>`_, are not provided: +## `described later<#nimshortnoval-and-nimlongnoval>`_, are not provided: ## ## 1. Short options: `-abcd`, `-e:5`, `-e=5` ## 2. Long options: `--foo:bar`, `--foo=bar`, `--foo` From a7024f49afe5623ce90a800986dea7447e249d3b Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Sat, 2 Apr 2022 01:29:15 +0800 Subject: [PATCH 0751/1654] fix 19655 - fixing more url fragments (#19669) --- lib/pure/parseopt.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index aff9312f9..71deaa535 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -203,7 +203,7 @@ proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, ## ## `shortNoVal` and `longNoVal` are used to specify which options ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## ## See also: @@ -465,7 +465,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## ## `shortNoVal` and `longNoVal` are used to specify which options ## do not take values. See the `documentation about these - ## parameters<#shortnoval-and-longnoval>`_ for more information on + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## ## There is no need to check for `cmdEnd` while iterating. From c3f03cfa5dfa2ca47f8e4cf99bbcdbf5a7d16eda Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 2 Apr 2022 01:30:02 +0800 Subject: [PATCH 0752/1654] add somes links to docs (#19668) --- lib/core/macros.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index f20ca9385..4d0e64868 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -933,21 +933,21 @@ proc treeTraverse(n: NimNode; res: var string; level = 0; isLisp = false, indent proc treeRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to a human-readable tree-like string. ## - ## See also `repr`, `lispRepr`, and `astGenRepr`. + ## See also `repr`, `lispRepr`_, and `astGenRepr`_. result = "" n.treeTraverse(result, isLisp = false, indented = true) proc lispRepr*(n: NimNode; indented = false): string {.benign.} = ## Convert the AST `n` to a human-readable lisp-like string. ## - ## See also `repr`, `treeRepr`, and `astGenRepr`. + ## See also `repr`, `treeRepr`_, and `astGenRepr`_. result = "" n.treeTraverse(result, isLisp = true, indented = indented) proc astGenRepr*(n: NimNode): string {.benign.} = ## Convert the AST `n` to the code required to generate that AST. ## - ## See also `repr`, `treeRepr`, and `lispRepr`. + ## See also `repr`_, `treeRepr`_, and `lispRepr`_. const NodeKinds = {nnkEmpty, nnkIdent, nnkSym, nnkNone, nnkCommentStmt} @@ -1602,7 +1602,7 @@ macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped = ## Expands to `true` if expression `n` which is expected to be `nnkDotExpr` ## (if checking a field), a proc or a type has custom pragma `cp`. ## - ## See also `getCustomPragmaVal`. + ## See also `getCustomPragmaVal`_. ## ## .. code-block:: nim ## template myAttr() {.pragma.} @@ -1626,7 +1626,7 @@ macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped = ## Expands to value of custom pragma `cp` of expression `n` which is expected ## to be `nnkDotExpr`, a proc or a type. ## - ## See also `hasCustomPragma` + ## See also `hasCustomPragma`_. ## ## .. code-block:: nim ## template serializationKey(key: string) {.pragma.} From 83dabb69ae0f6c0bb269594a5b73af964b809bc7 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 4 Apr 2022 18:05:23 +0800 Subject: [PATCH 0753/1654] Fix bug in freshVarForClosureIter. Fixes #18474 (#19675) [backport] * Fix bug in freshVarForClosureIter. Fixes #18474. freshVarForClosureIter was returning non-fresh symbols sometimes. Fixed by making addField return the generated PSym. * remove discardable Co-authored-by: Nick Smallbone --- compiler/lambdalifting.nim | 13 ++++--------- compiler/lowerings.nim | 3 ++- compiler/spawn.nim | 22 +++++++++++----------- tests/iter/tclosureiters.nim | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 6838fd80d..4b7ca89cd 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -275,16 +275,11 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode = let envParam = getHiddenParam(g, owner) let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr}) - addField(obj, s, g.cache, idgen) + let field = addField(obj, s, g.cache, idgen) var access = newSymNode(envParam) assert obj.kind == tyObject - let field = getFieldFromObj(obj, s) - if field != nil: - result = rawIndirectAccess(access, field, s.info) - else: - localError(g.config, s.info, "internal error: cannot generate fresh variable") - result = access + result = rawIndirectAccess(access, field, s.info) # ------------------ new stuff ------------------------------------------- @@ -452,7 +447,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = if s.name.id == getIdent(c.graph.cache, ":state").id: obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item) else: - addField(obj, s, c.graph.cache, c.idgen) + discard addField(obj, s, c.graph.cache, c.idgen) # direct or indirect dependency: elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s): discard """ @@ -474,7 +469,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr}) #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr}) - addField(obj, s, c.graph.cache, c.idgen) + discard addField(obj, s, c.graph.cache, c.idgen) # create required upFields: var w = owner.skipGenericOwner if isInnerProc(w) or owner.isIterator: diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 382e48a16..de2f678d7 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -230,7 +230,7 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym = if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym else: discard -proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) = +proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym = # because of 'gensym' support, we have to mangle the name with its ID. # This is hacky but the clean solution is much more complex than it looks. var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), @@ -244,6 +244,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) = field.flags = s.flags * {sfCursor} obj.n.add newSymNode(field) fieldCheck() + result = field proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} = result = lookupInRecord(obj.n, s.itemId) diff --git a/compiler/spawn.nim b/compiler/spawn.nim index e57686cb4..581f722d5 100644 --- a/compiler/spawn.nim +++ b/compiler/spawn.nim @@ -221,7 +221,7 @@ proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType; let fieldname = if i < formals.len: formals[i].sym.name else: tmpName var field = newSym(skField, fieldname, nextSymId idgen, objType.owner, n.info, g.config.options) field.typ = argType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i]) let temp = addLocalVar(g, varSection, varInit, idgen, owner, argType, @@ -260,17 +260,17 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; slice[0].typ = getSysType(g, n.info, tyInt) # fake type var fieldB = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options) fieldB.typ = getSysType(g, n.info, tyInt) - objType.addField(fieldB, g.cache, idgen) + discard objType.addField(fieldB, g.cache, idgen) if getMagic(n) == mSlice: let a = genAddrOf(n[1], idgen) field.typ = a.typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a) var fieldA = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options) fieldA.typ = getSysType(g, n.info, tyInt) - objType.addField(fieldA, g.cache, idgen) + discard objType.addField(fieldA, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2]) result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3]) @@ -281,7 +281,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; else: let a = genAddrOf(n, idgen) field.typ = a.typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a) result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n)) @@ -299,7 +299,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; # it is more efficient to pass a pointer instead: let a = genAddrOf(n, idgen) field.typ = a.typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a) let threadLocal = addLocalVar(g, varSection, nil, idgen, owner, field.typ, indirectAccess(castExpr, field, n.info), @@ -308,7 +308,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType; else: # boring case field.typ = argType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n) let threadLocal = addLocalVar(g, varSection, varInit, idgen, owner, field.typ, @@ -377,7 +377,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp var argType = n[0].typ.skipTypes(abstractInst) var field = newSym(skField, getIdent(g.cache, "fn"), nextSymId idgen, owner, n.info, g.config.options) field.typ = argType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0]) fn = indirectAccess(castExpr, field, n.info) elif fn.kind == nkSym and fn.sym.kind == skIterator: @@ -399,7 +399,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ) var field = newSym(skField, getIdent(g.cache, "barrier"), nextSymId idgen, owner, n.info, g.config.options) field.typ = typ - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier) barrierAsExpr = indirectAccess(castExpr, field, n.info) @@ -407,7 +407,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp if spawnKind == srFlowVar: var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options) field.typ = retType - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) fvField = newDotExpr(scratchObj, field) fvAsExpr = indirectAccess(castExpr, field, n.info) # create flowVar: @@ -419,7 +419,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options) field.typ = newType(tyPtr, nextTypeId idgen, objType.owner) field.typ.rawAddSon(retType) - objType.addField(field, g.cache, idgen) + discard objType.addField(field, g.cache, idgen) fvAsExpr = indirectAccess(castExpr, field, n.info) result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest, idgen)) diff --git a/tests/iter/tclosureiters.nim b/tests/iter/tclosureiters.nim index afeaabc7d..85611373c 100644 --- a/tests/iter/tclosureiters.nim +++ b/tests/iter/tclosureiters.nim @@ -21,6 +21,15 @@ discard """ 2 70 0 +(1, 1) +(1, 2) +(1, 3) +(2, 1) +(2, 2) +(2, 3) +(3, 1) +(3, 2) +(3, 3) ''' """ @@ -152,3 +161,12 @@ var love = iterator: int {.closure.} = for i in love(): echo i + +# bug #18474 +iterator pairs(): (int, int) {.closure.} = + for i in 1..3: + for j in 1..3: + yield (i, j) + +for pair in pairs(): + echo pair From 5a995ffc53ac7c1a51ab62440a58af2f4e43963b Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 6 Apr 2022 22:17:09 +0800 Subject: [PATCH 0754/1654] fix #18986; Import/except doesn't work on devel [backport: 1.6] (#19687) * fix #18986; Import/except doesn't work on devel [backport: 1.6] * add testcase --- compiler/importer.nim | 2 +- tests/converter/m18986.nim | 3 +++ tests/converter/t18986.nim | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/converter/m18986.nim create mode 100644 tests/converter/t18986.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index 719b75f0f..c4e37c269 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -202,7 +202,7 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} = proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) = c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet) - addUnnamedIt(c, fromMod, it.sym.id notin exceptSet) + addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet) proc importAllSymbols*(c: PContext, fromMod: PSym) = c.addImport ImportedModule(m: fromMod, mode: importAll) diff --git a/tests/converter/m18986.nim b/tests/converter/m18986.nim new file mode 100644 index 000000000..0ebf343ae --- /dev/null +++ b/tests/converter/m18986.nim @@ -0,0 +1,3 @@ +import std/macros + +converter Lit*(x: uint): NimNode = newLit(x) diff --git a/tests/converter/t18986.nim b/tests/converter/t18986.nim new file mode 100644 index 000000000..ef300fa49 --- /dev/null +++ b/tests/converter/t18986.nim @@ -0,0 +1,10 @@ +discard """ + output: "Found a 0" +""" + +import m18986 except Lit +import std/macros + +# bug #18986 +var x = 0.uint +echo "Found a ", x From c322faaf384bfcde4dd5a6e6eb9355d10c2e9597 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 6 Apr 2022 16:18:10 +0200 Subject: [PATCH 0755/1654] [backport] fix broken SSL tests (#19684) * [backport] fix broken SSL tests * remove a flaky one --- tests/untestable/thttpclient_ssl_remotenetwork.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim index d8137e516..cbb86dc13 100644 --- a/tests/untestable/thttpclient_ssl_remotenetwork.nim +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -32,7 +32,7 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win good, bad, dubious, good_broken, bad_broken, dubious_broken CertTest = tuple[url:string, category:Category, desc: string] - const certificate_tests: array[0..55, CertTest] = [ + const certificate_tests: array[0..54, CertTest] = [ ("https://wrong.host.badssl.com/", bad, "wrong.host"), ("https://captive-portal.badssl.com/", bad, "captive-portal"), ("https://expired.badssl.com/", bad, "expired"), @@ -41,13 +41,12 @@ when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(win ("https://untrusted-root.badssl.com/", bad, "untrusted-root"), ("https://revoked.badssl.com/", bad_broken, "revoked"), ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"), - ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"), - ("https://no-subject.badssl.com/", dubious_broken, "no-subject"), - ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"), + ("https://no-common-name.badssl.com/", bad, "no-common-name"), + ("https://no-subject.badssl.com/", bad, "no-subject"), ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"), ("https://sha256.badssl.com/", good, "sha256"), - ("https://sha384.badssl.com/", good, "sha384"), - ("https://sha512.badssl.com/", good, "sha512"), + ("https://sha384.badssl.com/", bad, "sha384"), + ("https://sha512.badssl.com/", bad, "sha512"), ("https://1000-sans.badssl.com/", bad, "1000-sans"), ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), ("https://ecc256.badssl.com/", good, "ecc256"), From abe3b0eb644c60be4423bd3595382a126c4746db Mon Sep 17 00:00:00 2001 From: locriacyber <74560659+locriacyber@users.noreply.github.com> Date: Wed, 6 Apr 2022 16:08:27 +0000 Subject: [PATCH 0756/1654] Load Google Fonts last (#19683) --- config/nimdoc.cfg | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index c1074f344..0ae768bfd 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -233,10 +233,6 @@ doc.file = """ - - - - $title @@ -282,8 +278,6 @@ function main() { } } } - -window.addEventListener('DOMContentLoaded', main); @@ -301,7 +295,15 @@ window.addEventListener('DOMContentLoaded', main); + + + $analytics + + + + + """ From e649ddd09ba0b1195163f456dbf6b1308affec37 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 7 Apr 2022 02:27:51 +0800 Subject: [PATCH 0757/1654] Revert "Load Google Fonts last (#19683)" (#19693) This reverts commit abe3b0eb644c60be4423bd3595382a126c4746db. --- config/nimdoc.cfg | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 0ae768bfd..c1074f344 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -233,6 +233,10 @@ doc.file = """ + + + + $title @@ -278,6 +282,8 @@ function main() { } } } + +window.addEventListener('DOMContentLoaded', main); @@ -295,15 +301,7 @@ function main() { - - - $analytics - - - - - """ From cb6ce80cb8dcf21733b437606ada1dfff9884316 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 6 Apr 2022 20:28:37 +0200 Subject: [PATCH 0758/1654] std/tasks: fix spelling (#19691) [backport] why aren't these not being caught by style check options? --styleCheck:usages finds it. Co-authored-by: flywind --- lib/std/tasks.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index e2ea5377f..dbe2e265b 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -87,7 +87,7 @@ template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr let isolatedTemp = genSym(nskTemp, "isoTemp") scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam)) scratchAssignList.add newAssignment(scratchDotExpr, - newcall(newIdentNode("extract"), isolatedTemp)) + newCall(newIdentNode("extract"), isolatedTemp)) template addAllNode(assignParam: NimNode, procParam: NimNode) = let scratchDotExpr = newDotExpr(scratchIdent, formalParams[i][0]) @@ -154,7 +154,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") let seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) - seqCallNode = newcall("@", e[i]) + seqCallNode = newCall("@", e[i]) addAllNode(seqType, seqCallNode) else: addAllNode(param, e[i]) From 065f568470af944cfc43ecfed8df6ac8021b7cbe Mon Sep 17 00:00:00 2001 From: Nan Xiao Date: Thu, 7 Apr 2022 02:28:58 +0800 Subject: [PATCH 0759/1654] No need to export pos from OptParser (#19688) Co-authored-by: flywind --- lib/pure/parseopt.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 71deaa535..fef354733 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -164,7 +164,7 @@ type ## ## To initialize it, use the ## `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_. - pos*: int + pos: int inShortState: bool allowWhitespaceAfterColon: bool shortNoVal: set[char] From e1929deac7dd41595e998ce68ee703a113ce8746 Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 7 Apr 2022 07:11:25 +0800 Subject: [PATCH 0760/1654] remove useless compilation condition (#19690) ref https://github.com/nim-lang/Nim/pull/18546 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 456f42cd2..6b0b15fd8 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -100,7 +100,7 @@ pkg "nimcrypto", "nim r --path:. tests/testall.nim" # `--path:.` workaround need pkg "NimData", "nim c -o:nimdataa src/nimdata.nim" pkg "nimes", "nim c src/nimes.nim" pkg "nimfp", "nim c -o:nfp -r src/fp.nim" -pkg "nimgame2", "nim c -d:nimLegacyConvEnumEnum nimgame2/nimgame.nim" +pkg "nimgame2", "nim c nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" pkg "nimlsp", allowFailure = true From 0978276ed9882a25782f91795b355a0dca7b0d3b Mon Sep 17 00:00:00 2001 From: flywind Date: Thu, 7 Apr 2022 13:07:39 +0800 Subject: [PATCH 0761/1654] use two spaces indentation (#19696) --- lib/core/macros.nim | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4d0e64868..ba19712cf 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1505,20 +1505,20 @@ macro expandMacros*(body: typed): untyped = result = body proc extractTypeImpl(n: NimNode): NimNode = - ## attempts to extract the type definition of the given symbol - case n.kind - of nnkSym: # can extract an impl - result = n.getImpl.extractTypeImpl() - of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n - of nnkBracketExpr: - if n.typeKind == ntyTypeDesc: - result = n[1].extractTypeImpl() - else: - doAssert n.typeKind == ntyGenericInst - result = n[0].getImpl() - of nnkTypeDef: - result = n[2] - else: error("Invalid node to retrieve type implementation of: " & $n.kind) + ## attempts to extract the type definition of the given symbol + case n.kind + of nnkSym: # can extract an impl + result = n.getImpl.extractTypeImpl() + of nnkObjectTy, nnkRefTy, nnkPtrTy: result = n + of nnkBracketExpr: + if n.typeKind == ntyTypeDesc: + result = n[1].extractTypeImpl() + else: + doAssert n.typeKind == ntyGenericInst + result = n[0].getImpl() + of nnkTypeDef: + result = n[2] + else: error("Invalid node to retrieve type implementation of: " & $n.kind) proc customPragmaNode(n: NimNode): NimNode = expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) From 810d5e91e435fffb7898ac26b5e4cc5fbb1678d7 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Thu, 7 Apr 2022 14:39:27 +0300 Subject: [PATCH 0762/1654] [nimsuggest] return the type when on symbol in let/var (#19639) - make sure `suggestSym` is called after `PSym.typ` is set. --- compiler/semstmts.nim | 9 ++++++--- nimsuggest/tests/tdef_let.nim | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 nimsuggest/tests/tdef_let.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c2b385c80..3513fd767 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -306,7 +306,7 @@ proc identWithin(n: PNode, s: PIdent): bool = if identWithin(n[i], s): return true result = n.kind == nkSym and n.sym.name.id == s.id -proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = +proc semIdentDef(c: PContext, n: PNode, kind: TSymKind, reportToNimsuggest = true): PSym = if isTopLevel(c): result = semIdentWithPragma(c, kind, n, {sfExported}) incl(result.flags, sfGlobal) @@ -330,7 +330,8 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = discard result = n.info let info = getLineInfo(n) - suggestSym(c.graph, info, result, c.graph.usageSym) + if reportToNimsuggest: + suggestSym(c.graph, info, result, c.graph.usageSym) proc checkNilable(c: PContext; v: PSym) = if {sfGlobal, sfImportc} * v.flags == {sfGlobal} and v.typ.requiresInit: @@ -660,7 +661,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if a.kind != nkVarTuple: typ else: tup[j]) addToVarSection(c, result, n, a) continue - var v = semIdentDef(c, a[j], symkind) + var v = semIdentDef(c, a[j], symkind, false) styleCheckDef(c.config, v) onDef(a[j].info, v) if sfGenSym notin v.flags: @@ -725,6 +726,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if v.flags * {sfGlobal, sfThread} == {sfGlobal}: message(c.config, v.info, hintGlobalVar) + suggestSym(c.graph, v.info, v, c.graph.usageSym) + proc semConst(c: PContext, n: PNode): PNode = result = copyNode(n) inc c.inStaticContext diff --git a/nimsuggest/tests/tdef_let.nim b/nimsuggest/tests/tdef_let.nim new file mode 100644 index 000000000..3e9456d2f --- /dev/null +++ b/nimsuggest/tests/tdef_let.nim @@ -0,0 +1,7 @@ +discard """ +$nimsuggest --tester $file +>def $1 +def;;skLet;;tdef_let.intVar;;int;;$file;;7;;4;;"";;100 +""" + +let int#[!]#Var = 10 From c8aeea9d6205198306c3cd802136915f909ba5dc Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Apr 2022 02:56:34 +0800 Subject: [PATCH 0763/1654] improve the error messages for std/tasks [backport: 1.6] (#19695) --- lib/std/tasks.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index dbe2e265b..3ceb4c8ba 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -111,10 +111,10 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC when compileOption("threads"): if not isGcSafe(e[0]): - error("'toTask' takes a GC safe call expression") + error("'toTask' takes a GC safe call expression", e) if hasClosure(e[0]): - error("closure call is not allowed") + error("closure call is not allowed", e) if e.len > 1: let scratchIdent = genSym(kind = nskTemp, ident = "scratch") @@ -141,17 +141,17 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC param = param[0] if param.typeKind in {ntyExpr, ntyStmt}: - error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) case param.kind of nnkVarTy: - error("'toTask'ed function cannot have a 'var' parameter") + error("'toTask'ed function cannot have a 'var' parameter", e) of nnkBracketExpr: if param[0].typeKind == ntyTypeDesc: callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) elif param[0].typeKind in {ntyVarargs, ntyOpenArray}: if param[1].typeKind in {ntyExpr, ntyStmt}: - error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter") + error("'toTask'ed function cannot have a 'typed' or 'untyped' parameter", e) let seqType = nnkBracketExpr.newTree(newIdentNode("seq"), param[1]) seqCallNode = newCall("@", e[i]) @@ -167,7 +167,7 @@ macro toTask*(e: typed{nkCall | nkInfix | nkPrefix | nkPostfix | nkCommand | nkC of nnkCharLit..nnkNilLit: callNode.add nnkExprEqExpr.newTree(formalParams[i][0], e[i]) else: - error("not supported type kinds") + error("'toTask'ed function cannot have a parameter of " & $param.kind & " kind", e) let scratchObjType = genSym(kind = nskType, ident = "ScratchObj") let scratchObj = nnkTypeSection.newTree( From 1807de38e52f45c2fb88dac9b99b47729b12ebae Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Apr 2022 02:57:50 +0800 Subject: [PATCH 0764/1654] add testcase for #16462 (#19692) --- tests/enum/m16462_1.nim | 3 +++ tests/enum/m16462_2.nim | 3 +++ tests/enum/t16462.nim | 5 +++++ 3 files changed, 11 insertions(+) create mode 100644 tests/enum/m16462_1.nim create mode 100644 tests/enum/m16462_2.nim create mode 100644 tests/enum/t16462.nim diff --git a/tests/enum/m16462_1.nim b/tests/enum/m16462_1.nim new file mode 100644 index 000000000..631c63256 --- /dev/null +++ b/tests/enum/m16462_1.nim @@ -0,0 +1,3 @@ +type + Scancode* {.pure.} = enum + SCANCODE_LEFT = 80 \ No newline at end of file diff --git a/tests/enum/m16462_2.nim b/tests/enum/m16462_2.nim new file mode 100644 index 000000000..631c63256 --- /dev/null +++ b/tests/enum/m16462_2.nim @@ -0,0 +1,3 @@ +type + Scancode* {.pure.} = enum + SCANCODE_LEFT = 80 \ No newline at end of file diff --git a/tests/enum/t16462.nim b/tests/enum/t16462.nim new file mode 100644 index 000000000..9f38286bb --- /dev/null +++ b/tests/enum/t16462.nim @@ -0,0 +1,5 @@ +import m16462_1 except Scancode +import m16462_2 + +# bug #16462 +let a = SCANCODE_LEFT \ No newline at end of file From e78ef57c93d66da483e0482ce0907dfe16dc8d27 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 7 Apr 2022 14:38:01 -0700 Subject: [PATCH 0765/1654] typetraits: add toSigned, toUnsigned (#18445) * typetraits: add toSigned, toUnsigned * improve and add tests Co-authored-by: Andreas Rumpf Co-authored-by: flywind --- lib/pure/bitops.nim | 26 ++++++++-------------- lib/pure/typetraits.nim | 38 +++++++++++++++++++++++++++----- lib/std/private/bitops_utils.nim | 15 ++++++++----- lib/system/countbits_impl.nim | 6 ++--- tests/metatype/ttypetraits.nim | 15 +++++++++++++ tests/stdlib/tbitops_utils.nim | 14 ++++++++++++ tests/stdlib/ttypetraits.nim | 4 ++-- 7 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 tests/stdlib/tbitops_utils.nim diff --git a/lib/pure/bitops.nim b/lib/pure/bitops.nim index 1b3838621..a518c25d2 100644 --- a/lib/pure/bitops.nim +++ b/lib/pure/bitops.nim @@ -27,7 +27,7 @@ import macros import std/private/since -from std/private/bitops_utils import forwardImpl, toUnsigned +from std/private/bitops_utils import forwardImpl, castToUnsigned func bitnot*[T: SomeInteger](x: T): T {.magic: "BitnotI".} ## Computes the `bitwise complement` of the integer `x`. @@ -72,7 +72,7 @@ func bitsliced*[T: SomeInteger](v: T; slice: Slice[int]): T {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned + uv = v.castToUnsigned (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, 3).} = @@ -84,7 +84,7 @@ proc bitslice*[T: SomeInteger](v: var T; slice: Slice[int]) {.inline, since: (1, let upmost = sizeof(T) * 8 - 1 - uv = when v is SomeUnsignedInt: v else: v.toUnsigned + uv = v.castToUnsigned v = (uv shl (upmost - slice.b) shr (upmost - slice.b + slice.a)).T func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = @@ -95,10 +95,7 @@ func toMask*[T: SomeInteger](slice: Slice[int]): T {.inline, since: (1, 3).} = let upmost = sizeof(T) * 8 - 1 - bitmask = when T is SomeUnsignedInt: - bitnot(0.T) - else: - bitnot(0.T).toUnsigned + bitmask = bitnot(0.T).castToUnsigned (bitmask shl (upmost - slice.b + slice.a) shr (upmost - slice.b)).T proc masked*[T: SomeInteger](v, mask :T): T {.inline, since: (1, 3).} = @@ -505,8 +502,7 @@ func parityBits*(x: SomeInteger): int {.inline.} = # Can be used a base if creating ASM version. # https://stackoverflow.com/questions/21617970/how-to-check-if-value-has-even-parity-of-bits-or-odd - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(parityImpl, x) else: @@ -529,8 +525,7 @@ func firstSetBit*(x: SomeInteger): int {.inline.} = doAssert firstSetBit(0b0000_1111'u8) == 1 # GCC builtin 'builtin_ffs' already handle zero input. - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: when noUndefined: if x == 0: @@ -572,8 +567,7 @@ func fastLog2*(x: SomeInteger): int {.inline.} = doAssert fastLog2(0b0000_1000'u8) == 3 doAssert fastLog2(0b0000_1111'u8) == 3 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return -1 @@ -615,8 +609,7 @@ func countLeadingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countLeadingZeroBits(0b0000_1000'u8) == 4 doAssert countLeadingZeroBits(0b0000_1111'u8) == 4 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 @@ -644,8 +637,7 @@ func countTrailingZeroBits*(x: SomeInteger): int {.inline.} = doAssert countTrailingZeroBits(0b0000_1000'u8) == 3 doAssert countTrailingZeroBits(0b0000_1111'u8) == 0 - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when noUndefined: if x == 0: return 0 diff --git a/lib/pure/typetraits.nim b/lib/pure/typetraits.nim index 3fc1c7c5c..c20f9e645 100644 --- a/lib/pure/typetraits.nim +++ b/lib/pure/typetraits.nim @@ -35,7 +35,7 @@ runnableExamples: assert C[float] is HoleyEnum proc name*(t: typedesc): string {.magic: "TypeTrait".} = - ## Returns the name of the given type. + ## Returns the name of `t`. ## ## Alias for `system.\`$\`(t) `_ since Nim v0.20. runnableExamples: @@ -43,7 +43,7 @@ proc name*(t: typedesc): string {.magic: "TypeTrait".} = doAssert name(seq[string]) == "seq[string]" proc arity*(t: typedesc): int {.magic: "TypeTrait".} = - ## Returns the arity of the given type. This is the number of "type" + ## Returns the arity of `t`. This is the number of "type" ## components or the number of generic parameters a given type `t` has. runnableExamples: doAssert arity(int) == 0 @@ -92,8 +92,7 @@ proc stripGenericParams*(t: typedesc): typedesc {.magic: "TypeTrait".} = doAssert stripGenericParams(int) is int proc supportsCopyMem*(t: typedesc): bool {.magic: "TypeTrait".} - ## This trait returns true if the type `t` is safe to use for - ## `copyMem`:idx:. + ## Returns true if `t` is safe to use for `copyMem`:idx:. ## ## Other languages name a type like these `blob`:idx:. @@ -289,9 +288,38 @@ since (1, 1): proc hasClosureImpl(n: NimNode): bool = discard "see compiler/vmops.nim" proc hasClosure*(fn: NimNode): bool {.since: (1, 5, 1).} = - ## Return true if the func/proc/etc `fn` has `closure`. + ## Returns true if the func/proc/etc `fn` has `closure`. ## `fn` has to be a resolved symbol of kind `nnkSym`. This ## implies that the macro that calls this proc should accept `typed` ## arguments and not `untyped` arguments. expectKind fn, nnkSym result = hasClosureImpl(fn) + +template toUnsigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns an unsigned type with same bit size as `T`. + runnableExamples: + assert int8.toUnsigned is uint8 + assert uint.toUnsigned is uint + assert int.toUnsigned is uint + # range types are currently unsupported: + assert not compiles(toUnsigned(range[0..7])) + when T is int8: uint8 + elif T is int16: uint16 + elif T is int32: uint32 + elif T is int64: uint64 + elif T is int: uint + else: T + +template toSigned*(T: typedesc[SomeInteger and not range]): untyped = + ## Returns a signed type with same bit size as `T`. + runnableExamples: + assert int8.toSigned is int8 + assert uint16.toSigned is int16 + # range types are currently unsupported: + assert not compiles(toSigned(range[0..7])) + when T is uint8: int8 + elif T is uint16: int16 + elif T is uint32: int32 + elif T is uint64: int64 + elif T is uint: int + else: T diff --git a/lib/std/private/bitops_utils.nim b/lib/std/private/bitops_utils.nim index d54977b4e..0b9484416 100644 --- a/lib/std/private/bitops_utils.nim +++ b/lib/std/private/bitops_utils.nim @@ -10,8 +10,13 @@ template forwardImpl*(impl, arg) {.dirty.} = else: impl(x.uint64) -template toUnsigned*(x: int8): uint8 = cast[uint8](x) -template toUnsigned*(x: int16): uint16 = cast[uint16](x) -template toUnsigned*(x: int32): uint32 = cast[uint32](x) -template toUnsigned*(x: int64): uint64 = cast[uint64](x) -template toUnsigned*(x: int): uint = cast[uint](x) +# this could also be implemented via: +# import std/typetraits +# template castToUnsigned*(x: SomeInteger): auto = cast[toUnsigned(typeof(x))](x) + +template castToUnsigned*(x: int8): uint8 = cast[uint8](x) +template castToUnsigned*(x: int16): uint16 = cast[uint16](x) +template castToUnsigned*(x: int32): uint32 = cast[uint32](x) +template castToUnsigned*(x: int64): uint64 = cast[uint64](x) +template castToUnsigned*(x: int): uint = cast[uint](x) +template castToUnsigned*[T: SomeUnsignedInt](x: T): T = x diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 020423e01..34969cb32 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -9,8 +9,7 @@ ## Contains the used algorithms for counting bits. -from std/private/bitops_utils import forwardImpl, toUnsigned - +from std/private/bitops_utils import forwardImpl, castToUnsigned const useBuiltins* = not defined(noIntrinsicsBitOpts) const noUndefined* = defined(noUndefinedBitOpts) @@ -65,8 +64,7 @@ func countSetBitsImpl*(x: SomeInteger): int {.inline.} = ## Counts the set bits in an integer (also called `Hamming weight`:idx:). # TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT. # like GCC and MSVC - when x is SomeSignedInt: - let x = x.toUnsigned + let x = x.castToUnsigned when nimvm: result = forwardImpl(countBitsImpl, x) else: diff --git a/tests/metatype/ttypetraits.nim b/tests/metatype/ttypetraits.nim index 3ff5c5ea6..bfaa23057 100644 --- a/tests/metatype/ttypetraits.nim +++ b/tests/metatype/ttypetraits.nim @@ -1,6 +1,21 @@ import typetraits import macros +block: # toUnsigned, toSigned + var a1: toSigned(int16) + doAssert a1 is int16 + var a2: toSigned(uint16) + doAssert $a2.typeof == "int16" + doAssert toSigned(uint32) is int32 + doAssert uint64.toSigned is int64 + doAssert int64.toSigned is int64 + doAssert int64.toUnsigned is uint64 + doAssert int.toUnsigned is uint + doAssert $uint.toUnsigned == "uint" + # disallowed for now + doAssert not compiles(toUnsigned(range[0..7])) + doAssert not compiles(toSigned(range[0..7])) + block: # isNamedTuple type Foo1 = (a:1,).type type Foo2 = (Field0:1,).type diff --git a/tests/stdlib/tbitops_utils.nim b/tests/stdlib/tbitops_utils.nim new file mode 100644 index 000000000..b571baeae --- /dev/null +++ b/tests/stdlib/tbitops_utils.nim @@ -0,0 +1,14 @@ +import std/private/bitops_utils + +template chk(a, b) = + let a2 = castToUnsigned(a) + doAssert a2 == b + doAssert type(a2) is type(b) + doAssert type(b) is type(a2) + +chk 1'i8, 1'u8 +chk -1'i8, 255'u8 +chk 1'u8, 1'u8 +chk 1'u, 1'u +chk -1, cast[uint](-1) +chk -1'i64, cast[uint64](-1) diff --git a/tests/stdlib/ttypetraits.nim b/tests/stdlib/ttypetraits.nim index de8259ab0..799bcf6e2 100644 --- a/tests/stdlib/ttypetraits.nim +++ b/tests/stdlib/ttypetraits.nim @@ -2,10 +2,10 @@ discard """ targets: "c cpp js" """ +# xxx merge with tests/metatype/ttypetraits.nim + import std/typetraits - - macro testClosure(fn: typed, flag: static bool) = if flag: doAssert hasClosure(fn) From 00775f6880733695d187d84bb742e8c9d6c65d6d Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 8 Apr 2022 14:04:46 +0800 Subject: [PATCH 0766/1654] fix stylecheck bug with nre (#19356) * stylecheck usages part two: stdlib cleanup typeinfo.nim: importCompilerProc => importcompilerproc nre.nim: newLineFlags => newlineFlags system.nim: JSRoot => JsRoot ref #19319 * prefer importCompilerProc * fix stylecheck error with asyncdispatch it is a partial regression since #12842 * add tests * don't use echo in tests * fix stylecheck bug with nre * Update compiler/linter.nim * no need to check dotexpr again * neither did let/var/const --- compiler/semexprs.nim | 9 ++++++--- tests/stylecheck/taccept.nim | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f224bd0b3..702b47929 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1188,7 +1188,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = let s = getGenSym(c, sym) case s.kind of skConst: - markUsed(c, n.info, s) + if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess + markUsed(c, n.info, s) onUse(n.info, s) let typ = skipTypes(s.typ, abstractInst-{tyTypeDesc}) case typ.kind @@ -1249,7 +1250,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = if s.magic == mNimvm: localError(c.config, n.info, "illegal context for 'nimvm' magic") - markUsed(c, n.info, s) + if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess + markUsed(c, n.info, s) onUse(n.info, s) result = newSymNode(s, n.info) # We cannot check for access to outer vars for example because it's still @@ -1270,7 +1272,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode = n.typ = s.typ return n of skType: - markUsed(c, n.info, s) + if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess + markUsed(c, n.info, s) onUse(n.info, s) if s.typ.kind == tyStatic and s.typ.base.kind != tyNone and s.typ.n != nil: return s.typ.n diff --git a/tests/stylecheck/taccept.nim b/tests/stylecheck/taccept.nim index 6fb55f835..afe02a65c 100644 --- a/tests/stylecheck/taccept.nim +++ b/tests/stylecheck/taccept.nim @@ -2,7 +2,7 @@ discard """ matrix: "--styleCheck:error --styleCheck:usages" """ -import asyncdispatch +import std/[asyncdispatch, nre] type Name = object From ded8b0e5410518ffed24b46565eb6e42f18a63f1 Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 9 Apr 2022 23:50:57 +0800 Subject: [PATCH 0767/1654] rewrite docs JS in Nim (#19701) * rewrite docs JS in Nim * fixup * fix nimdoc/rsttester --- config/nimdoc.cfg | 43 ------------------- nimdoc/rst2html/expected/rst_examples.html | 43 ------------------- .../expected/index.html | 43 ------------------- .../expected/theindex.html | 43 ------------------- .../expected/subdir/subdir_b/utils.html | 43 ------------------- nimdoc/testproject/expected/testproject.html | 43 ------------------- nimdoc/testproject/expected/theindex.html | 43 ------------------- tools/dochack/dochack.nim | 36 +++++++++++++++- 8 files changed, 35 insertions(+), 302 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index c1074f344..4efa1f637 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -243,49 +243,6 @@ doc.file = """ - -
              diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 1df91ff1f..2b5218d9f 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 69dd2eb2e..c6a116bd6 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index 34ddf8f6a..8ee62a330 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 574ac5b9b..f94da7f40 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index e45492dac..cba9391af 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -22,49 +22,6 @@ - -
              diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 88d4b86aa..47fae2491 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -22,49 +22,6 @@ - -
              diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 3a663808d..c7b8232ef 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,6 +1,39 @@ import dom import fuzzysearch + +proc switchTheme(event: Event) = + if event.target.checked: + document.documentElement.setAttribute("data-theme", "dark") + window.localStorage.setItem("theme", "dark") + else: + document.documentElement.setAttribute("data-theme", "light") + window.localStorage.setItem("theme", "light") + + +proc nimThemeSwitch(event: Event) {.exportC.} = + var pragmaDots = document.getElementsByClassName("pragmadots") + for i in 0.. 0: + document.documentElement.setAttribute("data-theme", currentTheme); + + if currentTheme == "dark" and toggleSwitch != nil: + toggleSwitch.checked = true + proc textContent(e: Element): cstring {. importcpp: "#.textContent", nodecl.} @@ -396,5 +429,6 @@ proc copyToClipboard*() {.exportc.} = """ .} - + copyToClipboard() +window.addEventListener("DOMContentLoaded", nimThemeSwitch) \ No newline at end of file From 152dab09838adc5a4ab8dc220c5017c868563467 Mon Sep 17 00:00:00 2001 From: Leon Date: Sun, 10 Apr 2022 01:51:44 +1000 Subject: [PATCH 0768/1654] fix: Fix introductory macro examples (#19706) Co-authored-by: adigitoleo --- doc/tut3.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/doc/tut3.rst b/doc/tut3.rst index 1c497f1f5..c2c956107 100644 --- a/doc/tut3.rst +++ b/doc/tut3.rst @@ -188,21 +188,35 @@ Backticks are used to insert code from `NimNode` symbols into the generated expression. .. code-block:: nim - macro a(i) = quote do: let `i` = 0 + :test: "nim c $1" + import std/macros + macro a(i) = quote do: + let `i` = 0 + a b + doAssert b == 0 A custom prefix operator can be defined whenever backticks are needed. .. code-block:: nim - macro a(i) = quote("@") do: assert @i == 0 + :test: "nim c $1" + import std/macros + macro a(i) = quote("@") do: + assert @i == 0 + let b = 0 a b The injected symbol needs accent quoted when it resolves to a symbol. .. code-block:: nim - macro a(i) = quote("@") do: let `@i` == 0 + :test: "nim c $1" + import std/macros + macro a(i) = quote("@") do: + let `@i` = 0 + a b + doAssert b == 0 Make sure to inject only symbols of type `NimNode` into the generated syntax tree. You can use `newLit` to convert arbitrary values into From a77ffdb7ea4ec66983d3481cc486c821a5df6b1b Mon Sep 17 00:00:00 2001 From: flywind Date: Sat, 9 Apr 2022 23:52:11 +0800 Subject: [PATCH 0769/1654] fix #17286 nim check -b:js works (#19704) * fix #17286 nim check -b:js works * fix --- compiler/main.nim | 2 ++ tests/misc/t17286.nim | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/misc/t17286.nim diff --git a/compiler/main.nim b/compiler/main.nim index e27a4f6d1..a4425e510 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -65,6 +65,8 @@ proc commandCheck(graph: ModuleGraph) = if optWasNimscript in conf.globalOptions: defineSymbol(conf.symbols, "nimscript") defineSymbol(conf.symbols, "nimconfig") + elif conf.backend == backendJs: + setTarget(conf.target, osJS, cpuJS) semanticPasses(graph) # use an empty backend for semantic checking only compileProject(graph) diff --git a/tests/misc/t17286.nim b/tests/misc/t17286.nim new file mode 100644 index 000000000..3a54e624e --- /dev/null +++ b/tests/misc/t17286.nim @@ -0,0 +1,16 @@ +discard """ + cmd: "nim check -b:js $file" + action: "compile" +""" + +# bug #17286 + +import std/compilesettings + +static: + doAssert querySetting(backend) == "js" + doAssert defined(js) + doAssert not defined(c) + +import random +randomize() \ No newline at end of file From 26acc97864ede8e19dddd52aa04e0139da356e3e Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Mon, 11 Apr 2022 06:27:12 -0600 Subject: [PATCH 0770/1654] StringStreams no longer errors when intialized with literals on arc/orc (#19708) --- lib/pure/streams.nim | 2 ++ tests/stdlib/tstreams.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 7ad81685f..84451f989 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1274,6 +1274,8 @@ else: # after 1.3 or JS not defined new(result) result.data = s + when defined(gcOrc) or defined(gcArc): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index c2ceed624..d70d4995a 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -74,3 +74,5 @@ block: doAssert(ss.peekLine(str)) doAssert(str == "uick brown fox jumped over the lazy dog.") doAssert(ss.getPosition == 5) # haven't moved + ss.setPosition(0) # Ensure we dont error with writing over literals on arc/orc #19707 + ss.write("hello") From 26bcf18f918b2bcc7b6b2e230ae74d7726dca6d0 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 12 Apr 2022 15:48:48 +0800 Subject: [PATCH 0771/1654] fix #19680; check if stderr is static (#19709) --- compiler/vmgen.nim | 2 +- tests/misc/tvarious1.nim | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 72bcef4db..dadcc3865 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1479,7 +1479,7 @@ proc checkCanEval(c: PCtx; n: PNode) = # proc foo() = var x ... let s = n.sym if {sfCompileTime, sfGlobal} <= s.flags: return - if s.importcCondVar: return + if compiletimeFFI in c.config.features and s.importcCondVar: return if s.kind in {skVar, skTemp, skLet, skParam, skResult} and not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl: # little hack ahead for bug #12612: assume gensym'ed variables diff --git a/tests/misc/tvarious1.nim b/tests/misc/tvarious1.nim index 9c0b541db..9c912ee8e 100644 --- a/tests/misc/tvarious1.nim +++ b/tests/misc/tvarious1.nim @@ -46,3 +46,13 @@ echo value var ys = @[4.1, 5.6, 7.2, 1.7, 9.3, 4.4, 3.2] #var x = int(ys.high / 2) #echo ys[x] # Works echo ys[int(ys.high / 2)] # Doesn't work + + +# bug #19680 +var here = "" +when stderr is static: + doAssert false +else: + here = "works" + +doAssert here == "works" From 98cebad7debddfb147ee22bc6f3d81221582c4d6 Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 13 Apr 2022 17:53:02 +0800 Subject: [PATCH 0772/1654] enable style:usages for stdlib tests [backport: 1.6] (#19715) * enable style:usages for stdlib tests * freeAddrInfo * more tests * importc * bufSize * fix more * => parseSql and renderSql --- lib/impure/db_mysql.nim | 20 +++--- lib/posix/posix.nim | 2 +- lib/pure/asyncdispatch.nim | 6 +- lib/pure/asyncnet.nim | 6 +- lib/pure/cookies.nim | 2 +- lib/pure/ioselects/ioselectors_epoll.nim | 2 +- lib/pure/ioselects/ioselectors_select.nim | 16 ++--- lib/pure/json.nim | 12 ++-- lib/pure/nativesockets.nim | 4 +- lib/pure/net.nim | 14 ++-- lib/pure/parsesql.nim | 12 ++-- lib/pure/pegs.nim | 4 +- lib/pure/segfaults.nim | 2 +- lib/std/strbasics.nim | 2 +- lib/std/tasks.nim | 2 +- lib/windows/registry.nim | 18 ++--- lib/windows/winlean.nim | 2 +- tests/stdlib/config.nims | 2 + tests/stdlib/nre/misc.nim | 4 +- tests/stdlib/talgorithm.nim | 4 +- tests/stdlib/tasynchttpserver.nim | 8 +-- tests/stdlib/tbase64.nim | 6 +- tests/stdlib/tbitops.nim | 4 +- tests/stdlib/thttpclient.nim | 2 +- tests/stdlib/tjsonmacro.nim | 4 +- tests/stdlib/tjsonutils.nim | 2 +- tests/stdlib/tnet.nim | 2 +- tests/stdlib/tnetdial.nim | 2 +- tests/stdlib/tosproc.nim | 2 +- tests/stdlib/tparsesql.nim | 86 +++++++++++------------ tests/stdlib/trst.nim | 10 +-- tests/stdlib/trstgen.nim | 8 +-- tests/stdlib/tsha1.nim | 2 +- tests/stdlib/tssl.nim | 4 +- tests/stdlib/tstrbasics.nim | 2 +- tests/stdlib/tstreams.nim | 2 +- tests/stdlib/tstrset.nim | 2 +- tests/stdlib/ttimes.nim | 2 +- tests/stdlib/ttypeinfo.nim | 2 +- tests/stdlib/ttypeinfo.nims | 1 + tests/stdlib/tunicode.nim | 4 +- tests/stdlib/tvarints.nim | 2 +- 42 files changed, 149 insertions(+), 146 deletions(-) create mode 100644 tests/stdlib/config.nims create mode 100644 tests/stdlib/ttypeinfo.nims diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 242ea1b0d..df878e25a 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -150,17 +150,17 @@ proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) - return mysql.realQuery(PMySQL db, q, q.len) == 0'i32 + return mysql.real_query(PMySQL db, q, q.len) == 0'i32 proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) + if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db) + if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db) proc newRow(L: int): Row = newSeq(result, L) @@ -168,7 +168,7 @@ proc newRow(L: int): Row = proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = if row != nil: - while mysql.fetchRow(sqlres) != nil: discard + while mysql.fetch_row(sqlres) != nil: discard mysql.freeResult(sqlres) iterator fastRows*(db: DbConn, query: SqlQuery, @@ -190,7 +190,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery, backup: Row newSeq(result, L) while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break for i in 0..L-1: setLen(result[i], 0) @@ -209,7 +209,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery, let L = int(mysql.numFields(sqlres)) var row: cstringArray while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break yield InstantRow(row: row, len: L) properFreeResult(sqlres, row) @@ -290,7 +290,7 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery; setColumnInfo(columns, sqlres, L) var row: cstringArray while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break yield InstantRow(row: row, len: L) properFreeResult(sqlres, row) @@ -317,7 +317,7 @@ proc getRow*(db: DbConn, query: SqlQuery, if sqlres != nil: var L = int(mysql.numFields(sqlres)) result = newRow(L) - var row = mysql.fetchRow(sqlres) + var row = mysql.fetch_row(sqlres) if row != nil: for i in 0..L-1: setLen(result[i], 0) @@ -335,7 +335,7 @@ proc getAllRows*(db: DbConn, query: SqlQuery, var row: cstringArray var j = 0 while true: - row = mysql.fetchRow(sqlres) + row = mysql.fetch_row(sqlres) if row == nil: break setLen(result, j+1) newSeq(result[j], L) @@ -361,7 +361,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery, ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) - if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: + if mysql.real_query(PMySQL db, q, q.len) != 0'i32: result = -1'i64 else: result = mysql.insertId(PMySQL db) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 9c65d0732..146ba886f 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1016,7 +1016,7 @@ proc endhostent*() {.importc, header: "".} proc endnetent*() {.importc, header: "".} proc endprotoent*() {.importc, header: "".} proc endservent*() {.importc, header: "".} -proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "".} +proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "".} proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "".} diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 5e2a8d98e..b67b0c308 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1791,7 +1791,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, curAddrInfo = curAddrInfo.ai_next if curAddrInfo == nil: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds() if lastException != nil: @@ -1807,7 +1807,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, try: curFd = createAsyncNativeSocket(domain, sockType, protocol) except: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) closeUnusedFds() raise getCurrentException() when defined(windows): @@ -1817,7 +1817,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped, doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo curAddrInfo = curAddrInfo.ai_next else: - freeaddrinfo(addrInfo) + freeAddrInfo(addrInfo) when shouldCreateFd: closeUnusedFds(ord(domain)) retFuture.complete(curFd) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index abb4dd075..61c2fc75a 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -648,9 +648,9 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) raiseOSError(osLastError()) - freeaddrinfo(aiList) + freeAddrInfo(aiList) proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} = ## Determines whether an AsyncSocket has data buffered. @@ -859,7 +859,7 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string, it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: if lastException != nil: diff --git a/lib/pure/cookies.nim b/lib/pure/cookies.nim index 7bf34f75b..0306558d6 100644 --- a/lib/pure/cookies.nim +++ b/lib/pure/cookies.nim @@ -75,4 +75,4 @@ proc setCookie*(key, value: string, expires: DateTime|Time, ## `Set-Cookie: key=value; Domain=...; ...` result = setCookie(key, value, domain, path, format(expires.utc, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'"), - noname, secure, httpOnly, maxAge, sameSite) + noName, secure, httpOnly, maxAge, sameSite) diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 05d9d19a8..8526eb8a3 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -526,4 +526,4 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value, body1, body2 proc getFd*[T](s: Selector[T]): int = - return s.epollFd.int + return s.epollFD.int diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 88cbaf28c..4c38ba604 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -26,27 +26,27 @@ else: #include #include """ type - Fdset {.importc: "fd_set", header: platformHeaders, pure, final.} = object + FdSet {.importc: "fd_set", header: platformHeaders, pure, final.} = object var FD_SETSIZE {.importc: "FD_SETSIZE", header: platformHeaders.}: cint -proc IOFD_SET(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_SET(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_SET", header: platformHeaders, inline.} -proc IOFD_CLR(fd: SocketHandle, fdset: ptr Fdset) +proc IOFD_CLR(fd: SocketHandle, fdset: ptr FdSet) {.cdecl, importc: "FD_CLR", header: platformHeaders, inline.} -proc IOFD_ZERO(fdset: ptr Fdset) +proc IOFD_ZERO(fdset: ptr FdSet) {.cdecl, importc: "FD_ZERO", header: platformHeaders, inline.} when defined(windows): - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.stdcall, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.stdcall, importc: "select", header: platformHeaders.} else: - proc IOFD_ISSET(fd: SocketHandle, fdset: ptr Fdset): cint + proc IOFD_ISSET(fd: SocketHandle, fdset: ptr FdSet): cint {.cdecl, importc: "FD_ISSET", header: platformHeaders, inline.} - proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr Fdset, + proc ioselect(nfds: cint, readFds, writeFds, exceptFds: ptr FdSet, timeout: ptr Timeval): cint {.cdecl, importc: "select", header: platformHeaders.} diff --git a/lib/pure/json.nim b/lib/pure/json.nim index dd9232ea6..8e4db93b0 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -958,12 +958,12 @@ proc parseJson*(s: Stream, filename: string = ""; rawIntegers = false, rawFloats when defined(js): from math import `mod` - from std/jsffi import JSObject, `[]`, to + from std/jsffi import JsObject, `[]`, to from std/private/jsutils import getProtoName, isInteger, isSafeInteger - proc parseNativeJson(x: cstring): JSObject {.importjs: "JSON.parse(#)".} + proc parseNativeJson(x: cstring): JsObject {.importjs: "JSON.parse(#)".} - proc getVarType(x: JSObject, isRawNumber: var bool): JsonNodeKind = + proc getVarType(x: JsObject, isRawNumber: var bool): JsonNodeKind = result = JNull case $getProtoName(x) # TODO: Implicit returns fail here. of "[object Array]": return JArray @@ -982,12 +982,12 @@ when defined(js): of "[object String]": return JString else: assert false - proc len(x: JSObject): int = + proc len(x: JsObject): int = asm """ `result` = `x`.length; """ - proc convertObject(x: JSObject): JsonNode = + proc convertObject(x: JsObject): JsonNode = var isRawNumber = false case getVarType(x, isRawNumber) of JArray: @@ -1000,7 +1000,7 @@ when defined(js): if (`x`.hasOwnProperty(property)) { """ var nimProperty: cstring - var nimValue: JSObject + var nimValue: JsObject asm "`nimProperty` = property; `nimValue` = `x`[property];" result[$nimProperty] = nimValue.convertObject() asm "}}" diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index eb0514a8f..5453226a0 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -633,8 +633,8 @@ when useNimNetLite: INET_ADDRSTRLEN = 16 INET6_ADDRSTRLEN = 46 # it's actually 46 in both cases - proc sockAddrToStr(sa: ptr Sockaddr): string {.noinit.} = - let af_family = sa.sa_family + proc sockAddrToStr(sa: ptr SockAddr): string {.noinit.} = + let af_family = sa.sa_family var nl, v4Slice: cint var si_addr: ptr InAddr diff --git a/lib/pure/net.nim b/lib/pure/net.nim index c59a4e563..370b83e54 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -984,11 +984,11 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: - freeaddrinfo(aiList) + freeAddrInfo(aiList) var address2: string address2.addQuoted address raiseOSError(osLastError(), "address: $# port: $#" % [address2, $port]) - freeaddrinfo(aiList) + freeAddrInfo(aiList) proc acceptAddr*(server: Socket, client: var owned(Socket), address: var string, flags = {SocketFlag.SafeDisconn}, @@ -1745,7 +1745,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer, it = it.ai_next let osError = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(osError) @@ -1960,7 +1960,7 @@ proc dial*(address: string, port: Port, # network system problem (e.g. not enough FDs), and not an unreachable # address. let err = osLastError() - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds() raiseOSError(err) fdPerDomain[ord(domain)] = lastFd @@ -1969,7 +1969,7 @@ proc dial*(address: string, port: Port, break lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) closeUnusedFds(ord(domain)) if success: @@ -1999,7 +1999,7 @@ proc connect*(socket: Socket, address: string, else: lastError = osLastError() it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) when defineSsl: @@ -2051,7 +2051,7 @@ proc connectAsync(socket: Socket, name: string, port = Port(0), it = it.ai_next - freeaddrinfo(aiList) + freeAddrInfo(aiList) if not success: raiseOSError(lastError) proc connect*(socket: Socket, address: string, port = Port(0), diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 9d74d08d0..61196aead 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -1451,7 +1451,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("enum") rs(n, s) -proc renderSQL*(n: SqlNode, upperCase = false): string = +proc renderSql*(n: SqlNode, upperCase = false): string = ## Converts an SQL abstract syntax tree to its string representation. var s: SqlWriter s.buffer = "" @@ -1460,8 +1460,8 @@ proc renderSQL*(n: SqlNode, upperCase = false): string = return s.buffer proc `$`*(n: SqlNode): string = - ## an alias for `renderSQL`. - renderSQL(n) + ## an alias for `renderSql`. + renderSql(n) proc treeReprAux(s: SqlNode, level: int, result: var string) = result.add('\n') @@ -1493,7 +1493,7 @@ proc open(p: var SqlParser, input: Stream, filename: string) = p.tok.literal = "" getTok(p) -proc parseSQL*(input: Stream, filename: string): SqlNode = +proc parseSql*(input: Stream, filename: string): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. @@ -1504,8 +1504,8 @@ proc parseSQL*(input: Stream, filename: string): SqlNode = finally: close(p) -proc parseSQL*(input: string, filename = ""): SqlNode = +proc parseSql*(input: string, filename = ""): SqlNode = ## parses the SQL from `input` into an AST and returns the AST. ## `filename` is only used for error messages. ## Syntax errors raise an `SqlParseError` exception. - parseSQL(newStringStream(input), "") + parseSql(newStringStream(input), "") diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 0eed1c388..78f03d308 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -1023,7 +1023,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## Symbols declared in an *enter* handler can be made visible in the ## corresponding *leave* handler by annotating them with an *inject* pragma. proc rawParse(s: string, p: Peg, start: int, c: var Captures): int - {.genSym.} = + {.gensym.} = # binding from *macros* bind strVal @@ -1058,7 +1058,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = matchOrParse(parseIt) parseIt(s, p, start, c) - proc parser(s: string): int {.genSym.} = + proc parser(s: string): int {.gensym.} = # the proc to be returned var ms: array[MaxSubpatterns, (int, int)] diff --git a/lib/pure/segfaults.nim b/lib/pure/segfaults.nim index e0da8b81d..b0eac2299 100644 --- a/lib/pure/segfaults.nim +++ b/lib/pure/segfaults.nim @@ -30,7 +30,7 @@ when defined(windows): const EXCEPTION_ACCESS_VIOLATION = DWORD(0xc0000005'i32) - EXCEPTION_CONTINUE_SEARCH = Long(0) + EXCEPTION_CONTINUE_SEARCH = LONG(0) type PEXCEPTION_RECORD = ptr object diff --git a/lib/std/strbasics.nim b/lib/std/strbasics.nim index 0d4c96f44..6f6db5c37 100644 --- a/lib/std/strbasics.nim +++ b/lib/std/strbasics.nim @@ -46,7 +46,7 @@ func setSlice*(s: var string, slice: Slice[int]) = import std/sugar var a = "Hello, Nim!" - doassert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" doAssert a.dup(setSlice(0 .. 0)) == "H" doAssert a.dup(setSlice(0 .. 1)) == "He" doAssert a.dup(setSlice(0 .. 10)) == a diff --git a/lib/std/tasks.nim b/lib/std/tasks.nim index 3ceb4c8ba..7b931ab14 100644 --- a/lib/std/tasks.nim +++ b/lib/std/tasks.nim @@ -85,7 +85,7 @@ template checkIsolate(scratchAssignList: seq[NimNode], procParam, scratchDotExpr # var isoTempB = isolate(literal) # scratch.b = extract(isolateB) let isolatedTemp = genSym(nskTemp, "isoTemp") - scratchAssignList.add newVarStmt(isolatedTemp, newCall(newidentNode("isolate"), procParam)) + scratchAssignList.add newVarStmt(isolatedTemp, newCall(newIdentNode("isolate"), procParam)) scratchAssignList.add newAssignment(scratchDotExpr, newCall(newIdentNode("extract"), isolatedTemp)) diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim index ab7d0b1fe..44e4e4420 100644 --- a/lib/windows/registry.nim +++ b/lib/windows/registry.nim @@ -46,26 +46,26 @@ template call(f) = proc getUnicodeValue*(path, key: string; handle: HKEY): string = let hh = newWideCString path let kk = newWideCString key - var bufsize: int32 + var bufSize: int32 # try a couple of different flag settings: var flags: int32 = RRF_RT_ANY - let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufsize) + let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufSize) if err != 0: var newHandle: HKEY call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) - call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufsize) + call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufSize) if bufSize > 0: - var res = newWideCString(bufsize) + var res = newWideCString(bufSize) call regGetValue(newHandle, nil, kk, flags, nil, addr res[0], - addr bufsize) - result = res $ bufsize + addr bufSize) + result = res $ bufSize call regCloseKey(newHandle) else: if bufSize > 0: - var res = newWideCString(bufsize) + var res = newWideCString(bufSize) call regGetValue(handle, hh, kk, flags, nil, addr res[0], - addr bufsize) - result = res $ bufsize + addr bufSize) + result = res $ bufSize proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, dwType: int32; lpData: WideCString; cbData: int32): int32 {. diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index c02cd9775..cd59b68a4 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -670,7 +670,7 @@ proc getaddrinfo*(nodename, servname: cstring, hints: ptr AddrInfo, res: var ptr AddrInfo): cint {. stdcall, importc: "getaddrinfo", dynlib: ws2dll.} -proc freeaddrinfo*(ai: ptr AddrInfo) {. +proc freeAddrInfo*(ai: ptr AddrInfo) {. stdcall, importc: "freeaddrinfo", dynlib: ws2dll.} proc inet_ntoa*(i: InAddr): cstring {. diff --git a/tests/stdlib/config.nims b/tests/stdlib/config.nims new file mode 100644 index 000000000..0c0c5bd88 --- /dev/null +++ b/tests/stdlib/config.nims @@ -0,0 +1,2 @@ +switch("styleCheck", "usages") +switch("styleCheck", "error") \ No newline at end of file diff --git a/tests/stdlib/nre/misc.nim b/tests/stdlib/nre/misc.nim index dbb0ecdf9..b7df08ee9 100644 --- a/tests/stdlib/nre/misc.nim +++ b/tests/stdlib/nre/misc.nim @@ -6,8 +6,8 @@ block: # Misc tests check("перевірка".replace(re"(*U)\w", "") == "") block: # empty or non-empty match - check("abc".findall(re"|.").join(":") == ":a::b::c:") - check("abc".findall(re".|").join(":") == "a:b:c:") + check("abc".findAll(re"|.").join(":") == ":a::b::c:") + check("abc".findAll(re".|").join(":") == "a:b:c:") check("abc".replace(re"|.", "x") == "xxxxxxx") check("abc".replace(re".|", "x") == "xxxx") diff --git a/tests/stdlib/talgorithm.nim b/tests/stdlib/talgorithm.nim index 5bb5a6bdf..61d2bc62f 100644 --- a/tests/stdlib/talgorithm.nim +++ b/tests/stdlib/talgorithm.nim @@ -101,10 +101,10 @@ block: doAssert binarySearch(noData, 7) == -1 let oneData = @[1] doAssert binarySearch(oneData, 1) == 0 - doAssert binarySearch(onedata, 7) == -1 + doAssert binarySearch(oneData, 7) == -1 let someData = @[1, 3, 4, 7] doAssert binarySearch(someData, 1) == 0 - doAssert binarySearch(somedata, 7) == 3 + doAssert binarySearch(someData, 7) == 3 doAssert binarySearch(someData, -1) == -1 doAssert binarySearch(someData, 5) == -1 doAssert binarySearch(someData, 13) == -1 diff --git a/tests/stdlib/tasynchttpserver.nim b/tests/stdlib/tasynchttpserver.nim index d4176d55a..aed21099d 100644 --- a/tests/stdlib/tasynchttpserver.nim +++ b/tests/stdlib/tasynchttpserver.nim @@ -111,9 +111,9 @@ proc testCustomContentLength() {.async.} = runTest(handler, request, test) -waitfor(test200()) -waitfor(test404()) -waitfor(testCustomEmptyHeaders()) -waitfor(testCustomContentLength()) +waitFor(test200()) +waitFor(test404()) +waitFor(testCustomEmptyHeaders()) +waitFor(testCustomContentLength()) echo "OK" diff --git a/tests/stdlib/tbase64.nim b/tests/stdlib/tbase64.nim index 491ec5db0..8ef5adf57 100644 --- a/tests/stdlib/tbase64.nim +++ b/tests/stdlib/tbase64.nim @@ -27,13 +27,13 @@ template main() = const tests = ["", "abc", "xyz", "man", "leasure.", "sure.", "easure.", "asure.", longText, testInputExpandsTo76, testInputExpands] - doAssert encodeMIME("foobarbaz", lineLen=4) == "Zm9v\r\nYmFy\r\nYmF6" + doAssert encodeMime("foobarbaz", lineLen=4) == "Zm9v\r\nYmFy\r\nYmF6" doAssert decode("Zm9v\r\nYmFy\r\nYmF6") == "foobarbaz" for t in items(tests): doAssert decode(encode(t)) == t - doAssert decode(encodeMIME(t, lineLen=40)) == t - doAssert decode(encodeMIME(t, lineLen=76)) == t + doAssert decode(encodeMime(t, lineLen=40)) == t + doAssert decode(encodeMime(t, lineLen=76)) == t doAssertRaises(ValueError): discard decode("SGVsbG\x008gV29ybGQ=") diff --git a/tests/stdlib/tbitops.nim b/tests/stdlib/tbitops.nim index 6b5e0aeb9..f807993d6 100644 --- a/tests/stdlib/tbitops.nim +++ b/tests/stdlib/tbitops.nim @@ -263,8 +263,8 @@ proc main() = doAssert v == 0b1000_0010 v.flipBit(1) doAssert v == 0b1000_0000 - doAssert v.testbit(7) - doAssert not v.testbit(6) + doAssert v.testBit(7) + doAssert not v.testBit(6) block: # multi bit operations var v: uint8 diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index e81590d95..8563a0c26 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -21,7 +21,7 @@ proc makeIPv6HttpServer(hostname: string, port: Port, let fd = createNativeSocket(AF_INET6) setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1) var aiList = getAddrInfo(hostname, port, AF_INET6) - if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: + if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeAddrInfo(aiList) raiseOSError(osLastError()) freeAddrInfo(aiList) diff --git a/tests/stdlib/tjsonmacro.nim b/tests/stdlib/tjsonmacro.nim index 779c0ce65..9b59c7dc3 100644 --- a/tests/stdlib/tjsonmacro.nim +++ b/tests/stdlib/tjsonmacro.nim @@ -259,7 +259,7 @@ proc testJson() = colors: array[2, BirdColor] var red = BirdColor(name: "red", rgb: [1.0, 0.0, 0.0]) - var blue = Birdcolor(name: "blue", rgb: [0.0, 0.0, 1.0]) + var blue = BirdColor(name: "blue", rgb: [0.0, 0.0, 1.0]) var b = Bird(age: 3, height: 1.734, name: "bardo", colors: [red, blue]) let jnode = %b let data = jnode.to(Bird) @@ -497,7 +497,7 @@ proc testJson() = doAssert array[3, float](t.arr) == [1.0,2.0,7.0] doAssert MyRef(t.person).name == "boney" - doAssert MyObj(t.distFruit).color == 11 + doAssert MyObj(t.distfruit).color == 11 doAssert t.dog.name == "honey" doAssert t.fruit.color == 10 doAssert seq[string](t.emails) == @["abc", "123"] diff --git a/tests/stdlib/tjsonutils.nim b/tests/stdlib/tjsonutils.nim index 205160471..d5809ee73 100644 --- a/tests/stdlib/tjsonutils.nim +++ b/tests/stdlib/tjsonutils.nim @@ -320,7 +320,7 @@ template fn() = b: int var a = A() - fromJson(a, """{"is_a": true, "a":1, "extra_key": 1}""".parse_json, Joptions(allowExtraKeys: true)) + fromJson(a, """{"is_a": true, "a":1, "extra_key": 1}""".parseJson, Joptions(allowExtraKeys: true)) doAssert $a[] == "(is_a: true, a: 1)" block testAllowMissingKeys: diff --git a/tests/stdlib/tnet.nim b/tests/stdlib/tnet.nim index 671e0f7bc..4ec62d88f 100644 --- a/tests/stdlib/tnet.nim +++ b/tests/stdlib/tnet.nim @@ -93,7 +93,7 @@ block: # "IpAddress/Sockaddr conversion" doAssert($ipaddrstr == $ipaddr_1) var sockaddr: Sockaddr_storage - var socklen: Socklen + var socklen: SockLen var ipaddr_2: IpAddress var port_2: Port diff --git a/tests/stdlib/tnetdial.nim b/tests/stdlib/tnetdial.nim index 3de00ff99..b836fb78d 100644 --- a/tests/stdlib/tnetdial.nim +++ b/tests/stdlib/tnetdial.nim @@ -14,7 +14,7 @@ proc initIPv6Server(hostname: string, port: Port): AsyncFD = let fd = createNativeSocket(AF_INET6) setSockOptInt(fd, SOL_SOCKET, SO_REUSEADDR, 1) var aiList = getAddrInfo(hostname, port, AF_INET6) - if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: + if bindAddr(fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32: freeAddrInfo(aiList) raiseOSError(osLastError()) freeAddrInfo(aiList) diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index d767aac80..c54e0d112 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -12,7 +12,7 @@ see also: tests/osproc/*.nim; consider merging those into a single test here when defined(case_testfile): # compiled test file for child process from posix import exitnow - proc c_exit2(code: c_int): void {.importc: "_exit", header: "".} + proc c_exit2(code: cint): void {.importc: "_exit", header: "".} import os var a = 0 proc fun(b = 0) = diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index b440dc90f..8ef67f5dc 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -14,8 +14,8 @@ nkStmtList nkIntegerLit 10 nkNumericLit 5.5""" -doAssert $parseSQL("SELECT foo FROM table;") == "select foo from table;" -doAssert $parseSQL(""" +doAssert $parseSql("SELECT foo FROM table;") == "select foo from table;" +doAssert $parseSql(""" SELECT CustomerName, ContactName, @@ -31,26 +31,26 @@ SELECT Country FROM table;""") == "select CustomerName, ContactName, Address, City, PostalCode, Country, CustomerName, ContactName, Address, City, PostalCode, Country from table;" -doAssert $parseSQL("SELECT foo FROM table limit 10") == "select foo from table limit 10;" -doAssert $parseSQL("SELECT foo, bar, baz FROM table limit 10") == "select foo, bar, baz from table limit 10;" -doAssert $parseSQL("SELECT foo AS bar FROM table") == "select foo as bar from table;" -doAssert $parseSQL("SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table") == "select foo as foo_prime, bar as bar_prime, baz as baz_prime from table;" -doAssert $parseSQL("SELECT * FROM table") == "select * from table;" -doAssert $parseSQL("SELECT count(*) FROM table") == "select count(*) from table;" -doAssert $parseSQL("SELECT count(*) as 'Total' FROM table") == "select count(*) as 'Total' from table;" -doAssert $parseSQL("SELECT count(*) as 'Total', sum(a) as 'Aggr' FROM table") == "select count(*) as 'Total', sum(a) as 'Aggr' from table;" +doAssert $parseSql("SELECT foo FROM table limit 10") == "select foo from table limit 10;" +doAssert $parseSql("SELECT foo, bar, baz FROM table limit 10") == "select foo, bar, baz from table limit 10;" +doAssert $parseSql("SELECT foo AS bar FROM table") == "select foo as bar from table;" +doAssert $parseSql("SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table") == "select foo as foo_prime, bar as bar_prime, baz as baz_prime from table;" +doAssert $parseSql("SELECT * FROM table") == "select * from table;" +doAssert $parseSql("SELECT count(*) FROM table") == "select count(*) from table;" +doAssert $parseSql("SELECT count(*) as 'Total' FROM table") == "select count(*) as 'Total' from table;" +doAssert $parseSql("SELECT count(*) as 'Total', sum(a) as 'Aggr' FROM table") == "select count(*) as 'Total', sum(a) as 'Aggr' from table;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE a = b and c = d """) == "select * from table where a = b and c = d;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE not b """) == "select * from table where not b;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM @@ -59,63 +59,63 @@ WHERE a and not b """) == "select * from table where a and not b;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table ORDER BY 1 """) == "select * from table order by 1;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table GROUP BY 1 ORDER BY 1 """) == "select * from table group by 1 order by 1;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table ORDER BY 1 LIMIT 100 """) == "select * from table order by 1 limit 100;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE a = b and c = d or n is null and not b + 1 = 3 """) == "select * from table where a = b and c = d or n is null and not b + 1 = 3;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table WHERE (a = b and c = d) or (n is null and not b + 1 = 3) """) == "select * from table where(a = b and c = d) or (n is null and not b + 1 = 3);" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM table HAVING a = b and c = d """) == "select * from table having a = b and c = d;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM table GROUP BY a """) == "select a, b from table group by a;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM table GROUP BY 1, 2 """) == "select a, b from table group by 1, 2;" -doAssert $parseSQL("SELECT t.a FROM t as t") == "select t.a from t as t;" +doAssert $parseSql("SELECT t.a FROM t as t") == "select t.a from t as t;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM ( SELECT * FROM t ) """) == "select a, b from(select * from t);" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM ( SELECT * FROM t ) as foo """) == "select a, b from(select * from t) as foo;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM ( SELECT * FROM ( SELECT * FROM ( @@ -127,49 +127,49 @@ SELECT a, b FROM ( ) as inner5 """) == "select a, b from(select * from(select * from(select * from(select * from innerTable as inner1) as inner2) as inner3) as inner4) as inner5;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT a, b FROM (SELECT * FROM a), (SELECT * FROM b), (SELECT * FROM c) """) == "select a, b from(select * from a),(select * from b),(select * from c);" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM Products WHERE Price BETWEEN 10 AND 20; """) == "select * from Products where Price between 10 and 20;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a JOIN b ON a.id == b.id """) == "select id from a join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a JOIN (SELECT id from c) as b ON a.id == b.id """) == "select id from a join(select id from c) as b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a INNER JOIN b ON a.id == b.id """) == "select id from a inner join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a OUTER JOIN b ON a.id == b.id """) == "select id from a outer join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT id FROM a CROSS JOIN b ON a.id == b.id """) == "select id from a cross join b on a.id == b.id;" -doAssert $parseSQL(""" +doAssert $parseSql(""" CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); CREATE TABLE holidays ( num_weeks int, @@ -179,16 +179,16 @@ CREATE INDEX table1_attr1 ON table1(attr1); SELECT * FROM myTab WHERE col1 = 'happy'; """) == "create type happiness as enum ('happy' , 'very happy' , 'ecstatic' ); create table holidays(num_weeks int , happiness happiness );; create index table1_attr1 on table1(attr1 );; select * from myTab where col1 = 'happy';" -doAssert $parseSQL(""" +doAssert $parseSql(""" INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); """) == "insert into Customers (CustomerName , ContactName , Address , City , PostalCode , Country ) values ('Cardinal' , 'Tom B. Erichsen' , 'Skagen 21' , 'Stavanger' , '4006' , 'Norway' );" -doAssert $parseSQL(""" +doAssert $parseSql(""" INSERT INTO TableName DEFAULT VALUES """) == "insert into TableName default values;" -doAssert $parseSQL(""" +doAssert $parseSql(""" UPDATE Customers SET ContactName = 'Alfred Schmidt', City= 'Frankfurt' WHERE CustomerID = 1; @@ -209,9 +209,9 @@ nkStmtList nkStringLit Frankfurt nkNone""" -doAssert $parseSQL("DELETE FROM table_name;") == "delete from table_name;" +doAssert $parseSql("DELETE FROM table_name;") == "delete from table_name;" -doAssert treeRepr(parseSQL("DELETE FROM table_name;") +doAssert treeRepr(parseSql("DELETE FROM table_name;") ) == """ nkStmtList @@ -219,14 +219,14 @@ nkStmtList nkIdent table_name nkNone""" -doAssert $parseSQL("DELETE * FROM table_name;") == "delete from table_name;" +doAssert $parseSql("DELETE * FROM table_name;") == "delete from table_name;" -doAssert $parseSQL(""" +doAssert $parseSql(""" --Select all: SELECT * FROM Customers; """) == "select * from Customers;" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT * FROM Customers WHERE (CustomerName LIKE 'L%' OR CustomerName LIKE 'R%' /*OR CustomerName LIKE 'S%' OR CustomerName LIKE 'T%'*/ OR CustomerName LIKE 'W%') @@ -235,9 +235,9 @@ ORDER BY CustomerName; """) == "select * from Customers where(CustomerName like 'L%' or CustomerName like 'R%' or CustomerName like 'W%') and Country = 'USA' order by CustomerName;" # parse quoted keywords as identifires -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT `SELECT`, `FROM` as `GROUP` FROM `WHERE`; """) == """select "SELECT", "FROM" as "GROUP" from "WHERE";""" -doAssert $parseSQL(""" +doAssert $parseSql(""" SELECT "SELECT", "FROM" as "GROUP" FROM "WHERE"; """) == """select "SELECT", "FROM" as "GROUP" from "WHERE";""" diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 8b98d1403..9787c13eb 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -861,7 +861,7 @@ suite "RST include directive": test "Include whole": "other.rst".writeFile("**test1**") let input = ".. include:: other.rst" - doAssert "test1" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + doAssert "test1" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include starting from": @@ -875,7 +875,7 @@ OtherStart .. include:: other.rst :start-after: OtherStart """ - check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + check "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") test "Include everything before": @@ -889,7 +889,7 @@ And this should **NOT** be visible in `docs.html` .. include:: other.rst :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + doAssert "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -907,7 +907,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - check "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + check "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") @@ -927,7 +927,7 @@ And this should **NOT** be visible in `docs.html` :start-after: OtherStart :end-before: OtherEnd """ - doAssert "Visible" == rstTohtml(input, {roSandboxDisabled}, defaultConfig()) + doAssert "Visible" == rstToHtml(input, {roSandboxDisabled}, defaultConfig()) removeFile("other.rst") suite "RST escaping": diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index 2c28fe506..d91b5615e 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -417,7 +417,7 @@ Some chapter """ var error9Bad = new string - let output9Bad = input9bad.toHtml(error=error9Bad) + let output9Bad = input9Bad.toHtml(error=error9Bad) check(error9Bad[] == "input(15, 1) Error: new section expected (section " & "level inconsistent: underline ~~~~~ unexpectedly found, while " & "the following intermediate section level(s) are missing on " & @@ -464,7 +464,7 @@ Some chapter rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames = files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" - doAssert rstGenera.meta[metaSubTitle] == "SubTitle0" + doAssert rstGenera.meta[metaSubtitle] == "SubTitle0" doAssert "

              Level1

              " in output doAssert "

              Level2

              " in output doAssert "

              Level3

              " in output @@ -491,7 +491,7 @@ Some chapter rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "" - doAssert rstGenera.meta[metaSubTitle] == "" + doAssert rstGenera.meta[metaSubtitle] == "" doAssert "

              Title0

              " in output doAssert "

              SubTitle0

              " in output @@ -521,7 +521,7 @@ Some chapter rstGenera.initRstGenerator(outHtml, defaultConfig(), "input", filenames=files) rstGenera.renderRstToOut(rst, output) doAssert rstGenera.meta[metaTitle] == "Title0" - doAssert rstGenera.meta[metaSubTitle] == "" + doAssert rstGenera.meta[metaSubtitle] == "" doAssert output == "\n

              MySection1a

              " & # RST "\n

              MySection1b

              " & # Markdown diff --git a/tests/stdlib/tsha1.nim b/tests/stdlib/tsha1.nim index a20183039..6dca1f197 100644 --- a/tests/stdlib/tsha1.nim +++ b/tests/stdlib/tsha1.nim @@ -15,7 +15,7 @@ checkVector("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", proc testIsValidSha1Hash = doAssert not isValidSha1Hash("") doAssert not isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD11") - doAssert not isValidSha1hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1") + doAssert not isValidSha1Hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1") doAssert isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD1") doAssert isValidSha1Hash("042d4be2b90ed0672e717d71850abdb0a2d19cd1") doAssert isValidSha1Hash("042d4be2b90ed0672e717D71850ABDB0A2D19CD1") diff --git a/tests/stdlib/tssl.nim b/tests/stdlib/tssl.nim index 7625f3694..379c1b1e5 100644 --- a/tests/stdlib/tssl.nim +++ b/tests/stdlib/tssl.nim @@ -37,8 +37,8 @@ proc notifiedShutdown(port: Port) {.thread.} = proc main() = when defined(posix): var - ignoreAction = SigAction(sa_handler: SIG_IGN) - oldSigPipeHandler: SigAction + ignoreAction = Sigaction(sa_handler: SIG_IGN) + oldSigPipeHandler: Sigaction if sigemptyset(ignoreAction.sa_mask) == -1: raiseOSError(osLastError(), "Couldn't create an empty signal set") if sigaction(SIGPIPE, ignoreAction, oldSigPipeHandler) == -1: diff --git a/tests/stdlib/tstrbasics.nim b/tests/stdlib/tstrbasics.nim index b340ad509..bf562d9ec 100644 --- a/tests/stdlib/tstrbasics.nim +++ b/tests/stdlib/tstrbasics.nim @@ -73,7 +73,7 @@ proc main() = block: # setSlice var a = "Hello, Nim!" - doassert a.dup(setSlice(7 .. 9)) == "Nim" + doAssert a.dup(setSlice(7 .. 9)) == "Nim" doAssert a.dup(setSlice(0 .. 0)) == "H" doAssert a.dup(setSlice(0 .. 1)) == "He" doAssert a.dup(setSlice(0 .. 10)) == a diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index d70d4995a..d199037e3 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -40,7 +40,7 @@ block tstreams2: block tstreams3: try: var fs = openFileStream("shouldneverexist.txt") - except IoError: + except IOError: echo "threw exception" static: diff --git a/tests/stdlib/tstrset.nim b/tests/stdlib/tstrset.nim index 1b253f862..71a5520c2 100644 --- a/tests/stdlib/tstrset.nim +++ b/tests/stdlib/tstrset.nim @@ -50,7 +50,7 @@ proc search(r: PRadixNode, s: string): PRadixNode = proc contains*(r: PRadixNode, s: string): bool = return search(r, s) != nil -proc testOrincl*(r: var PRadixNode, s: string): bool = +proc testOrIncl*(r: var PRadixNode, s: string): bool = nil proc incl*(r: var PRadixNode, s: string) = discard testOrIncl(r, s) diff --git a/tests/stdlib/ttimes.nim b/tests/stdlib/ttimes.nim index 2cf945786..fd440eb20 100644 --- a/tests/stdlib/ttimes.nim +++ b/tests/stdlib/ttimes.nim @@ -20,7 +20,7 @@ proc staticTz(hours, minutes, seconds: int = 0): Timezone {.noSideEffect.} = result.utcOffset = offset result.time = time - newTimezone("", zonedTimeFromTime, zonedTImeFromAdjTime) + newTimezone("", zonedTimeFromTime, zonedTimeFromAdjTime) template parseTest(s, f, sExpected: string, ydExpected: int) = let diff --git a/tests/stdlib/ttypeinfo.nim b/tests/stdlib/ttypeinfo.nim index 61c661a58..1cddea781 100644 --- a/tests/stdlib/ttypeinfo.nim +++ b/tests/stdlib/ttypeinfo.nim @@ -50,7 +50,7 @@ doAssert($x4[].kind() == "akString") block: # gimme a new scope dammit - var myarr: array[0..4, array[0..4, string]] = [ + var myArr: array[0..4, array[0..4, string]] = [ ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]] diff --git a/tests/stdlib/ttypeinfo.nims b/tests/stdlib/ttypeinfo.nims new file mode 100644 index 000000000..d31d0b26f --- /dev/null +++ b/tests/stdlib/ttypeinfo.nims @@ -0,0 +1 @@ +--styleCheck:off \ No newline at end of file diff --git a/tests/stdlib/tunicode.nim b/tests/stdlib/tunicode.nim index 33405cf38..3a8206c8f 100644 --- a/tests/stdlib/tunicode.nim +++ b/tests/stdlib/tunicode.nim @@ -218,5 +218,5 @@ block: # bug #17768 let s1 = "abcdef" let s2 = "abcdéf" - doAssert s1.runeSubstr(0, -1) == "abcde" - doAssert s2.runeSubstr(0, -1) == "abcdé" + doAssert s1.runeSubStr(0, -1) == "abcde" + doAssert s2.runeSubStr(0, -1) == "abcdé" diff --git a/tests/stdlib/tvarints.nim b/tests/stdlib/tvarints.nim index 772121f9f..3bba4f457 100644 --- a/tests/stdlib/tvarints.nim +++ b/tests/stdlib/tvarints.nim @@ -48,7 +48,7 @@ block: chk 0.1 chk Inf chk NegInf - chk Nan + chk NaN chk 3.1415926535897932384626433 block: From ef7d7f24594b1ea560fbf25d3fa0d6c51122725f Mon Sep 17 00:00:00 2001 From: huantian Date: Wed, 13 Apr 2022 14:03:46 -0700 Subject: [PATCH 0773/1654] Better error message and tests for bad await (#19622) * Better error message and tests for bad await * Use compiles to check if await is valid * temp: disable windows noasync test * Better error report, simplify test Co-authored-by: flywind --- lib/pure/asyncmacro.nim | 16 +++++++++++++--- tests/async/tasync_noasync.nim | 35 +++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index ce538913f..63c8e6e5c 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -126,9 +126,19 @@ template await*(f: typed): untyped {.used.} = error "await expects Future[T], got " & $typeof(f) template await*[T](f: Future[T]): auto {.used.} = - var internalTmpFuture: FutureBase = f - yield internalTmpFuture - (cast[typeof(f)](internalTmpFuture)).read() + template yieldFuture = yield FutureBase() + + when compiles(yieldFuture): + var internalTmpFuture: FutureBase = f + yield internalTmpFuture + (cast[typeof(f)](internalTmpFuture)).read() + else: + macro errorAsync(futureError: Future[T]) = + error( + "Can only 'await' inside a proc marked as 'async'. Use " & + "'waitFor' when calling an 'async' proc in a non-async scope instead", + futureError) + errorAsync(f) proc asyncSingleProc(prc: NimNode): NimNode = ## This macro transforms a single procedure into a closure iterator. diff --git a/tests/async/tasync_noasync.nim b/tests/async/tasync_noasync.nim index 54f4f597f..0927148bf 100644 --- a/tests/async/tasync_noasync.nim +++ b/tests/async/tasync_noasync.nim @@ -1,13 +1,42 @@ discard """ - errormsg: "'yield' only allowed in an iterator" - cmd: "nim c $file" - file: "asyncmacro.nim" + cmd: "nim check --hints:off --warnings:off $file" + action: "reject" + nimout: ''' +tasync_noasync.nim(21, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(25, 12) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(28, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(31, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(35, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(38, 10) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +tasync_noasync.nim(40, 8) Error: Can only 'await' inside a proc marked as 'async'. Use 'waitFor' when calling an 'async' proc in a non-async scope instead +''' """ import async proc a {.async.} = discard +# Bad await usage +proc nonAsyncProc = + await a() + +proc nestedNonAsyncProc {.async.} = + proc nested = + await a() + +iterator customIterator: int = + await a() + +macro awaitInMacro = + await a() + +type DummyRef = ref object of RootObj +method awaitInMethod(_: DummyRef) {.base.} = + await a() + +proc improperMultisync {.multisync.} = + await a() + await a() # if we overload a fallback handler to get From dc4cc2dca53e3772efb3654a4ddbbe8350d1db43 Mon Sep 17 00:00:00 2001 From: Jason Beetham Date: Thu, 14 Apr 2022 03:30:59 -0600 Subject: [PATCH 0774/1654] Fix string stream crashing when created on nimscript due to last fix (#19717) --- lib/pure/streams.nim | 2 +- tests/stdlib/tstreams.nim | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 84451f989..2ac6a82f1 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1274,7 +1274,7 @@ else: # after 1.3 or JS not defined new(result) result.data = s - when defined(gcOrc) or defined(gcArc): + when declared(prepareMutation): prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index d199037e3..6fda30f51 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -76,3 +76,7 @@ block: doAssert(ss.getPosition == 5) # haven't moved ss.setPosition(0) # Ensure we dont error with writing over literals on arc/orc #19707 ss.write("hello") + +static: # Ensure streams it doesnt break with nimscript on arc/orc #19716 + let s = newStringStream("a") + discard s.data From bb839029d5065bcd8367661802bf04e1e7eebb58 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 21 Apr 2022 22:06:09 +0300 Subject: [PATCH 0775/1654] Move common Latex code into class nimdoc.cls (#19734) --- compiler/docgen.nim | 6 +- compiler/nimpaths.nim | 2 + config/nimdoc.tex.cfg | 188 +---------------------------------------- doc/nimdoc.cls | 190 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 198 insertions(+), 188 deletions(-) create mode 100644 doc/nimdoc.cls diff --git a/compiler/docgen.nim b/compiler/docgen.nim index aad3b9409..ecf98d0b6 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1576,7 +1576,11 @@ proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = outfile.string) if not d.wroteSupportFiles: # nimdoc.css + dochack.js let nimr = $d.conf.getPrefixDir() - copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss) + case d.target + of outHtml: + copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss) + of outLatex: + copyFile(docCls.interp(nimr = nimr), $d.conf.outDir / nimdocOutCls) if optGenIndex in d.conf.globalOptions: let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename()) copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart) diff --git a/compiler/nimpaths.nim b/compiler/nimpaths.nim index 216071c5c..c6e188289 100644 --- a/compiler/nimpaths.nim +++ b/compiler/nimpaths.nim @@ -25,11 +25,13 @@ when defined(nimPreviewSlimSystem): const docCss* = "$nimr/doc/nimdoc.css" + docCls* = "$nimr/doc/nimdoc.cls" docHackNim* = "$nimr/tools/dochack/dochack.nim" docHackJs* = docHackNim.changeFileExt("js") docHackJsFname* = docHackJs.lastPathPart theindexFname* = "theindex.html" nimdocOutCss* = "nimdoc.out.css" + nimdocOutCls* = "nimdoc.cls" # `out` to make it easier to use with gitignore in user's repos htmldocsDirname* = "htmldocs" dotdotMangle* = "_._" ## refs #13223 diff --git a/config/nimdoc.tex.cfg b/config/nimdoc.tex.cfg index 3912d1279..4fb1aec90 100644 --- a/config/nimdoc.tex.cfg +++ b/config/nimdoc.tex.cfg @@ -69,198 +69,12 @@ doc.file = """ % makeindex file % xelatex file.tex % -\documentclass[a4paper,11pt]{article} -\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry} - -% for 2-sided printing with larger inner "binding" margin -%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry} -% for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080) -%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} -% for e-readers with 1.45:1 aspect ratio (e.g. 1200x825) -%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} -% for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404) -%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} - -\usepackage{fontspec} -% logic to select default font with some fall-back fonts. -\IfFontExistsTF{Times New Roman}{% - \setmainfont{Times New Roman} % the default font - \typeout{========================================= nim: using Times New Roman} -}{ - \IfFontExistsTF{FreeSerif}{% - \setmainfont{FreeSerif} % fallback #1 - official GNU font, resembles Times - \typeout{========================================= nim: using FreeSerif} - }{ - \IfFontExistsTF{DejaVuSerif}{% - \setmainfont{DejaVuSerif} % fallback #2 - very widespread free font - \typeout{========================================= nim: using DejaVuSerif} - }{ - \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!} - } - } -} - -% default monospace font for code: -\usepackage{GoMono} -\usepackage{relsize} -% make this monospace font 2 steps smaller to hold 80-character line -\newcommand{\rstverbblockfont}{\smaller[2]} -\newcommand{\rstverbinlinefont}{\smaller} - -\usepackage{parskip} % paragraphs delimited by vertical space, no indent -\usepackage{graphicx} - -\usepackage{makeidx} -\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}} -\makeindex - -\usepackage{dingbat} % for \carriagereturn, etc -\usepackage{fvextra} % for code blocks (works better than original fancyvrb) -\fvset{ - breaklines, - breakafter={=}:|\_\{\}[](){,}.;+-*/'", - breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, - breaksymbolright=\color{red}{\small\carriagereturn} -} -\fvinlineset{% - breaklines, - breakafter={=}:|\_\{\}[](){,}.;+-*/'", - % that does not work at all when we underline inline code by ulem :-( - commandchars=\\\{\} -} - -\usepackage{scrextend} % for the `addmargin` environment - -\usepackage{xcolor} -\usepackage[urlbordercolor=blue,linkbordercolor=cyan, - pdfborderstyle={/S/U/W 1}]{hyperref} -\usepackage{enumitem} % for option list, enumList, and rstfootnote - -\usepackage[most]{tcolorbox} % boxes around admonitions, code blocks, doc.item - -\newtcolorbox{rstadmonition}[1][]{blanker, breakable, - left=3mm, right=0mm, top=1mm, bottom=1mm, - before upper=\indent, parbox=false, #1} - -\newtcolorbox{rstquote}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, - parbox=false, - borderline west={0.3em}{0pt}{lightgray}, - borderline north={0.05em}{0pt}{lightgray}, - borderline east={0.05em}{0pt}{lightgray}, - borderline south={0.05em}{0pt}{lightgray}} - -\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} - -\newtcolorbox{rstprebox}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, - borderline ={0.1em}{0pt}{rstframecolor}, - before upper=\indent, parbox=false, #1} - -\newenvironment{rstpre}{% -\VerbatimEnvironment\begingroup\begin{rstprebox}% -\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% -{\end{Verbatim}\end{rstprebox}\endgroup} - -\newtcolorbox{rstdocitembox}[1][]{blanker, breakable, - left=3mm, right=3mm, top=1mm, bottom=1mm, - borderline ={1pt}{0pt}{cyan}, - before upper=\indent, parbox=false, #1} - -% Inline code formatting: grey underline, -% use \Verb from fvextras e.g. to display -- correctly as double - -\usepackage[normalem]{ulem} -\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon} - -\newcommand{\rstcode}[1]{% -{\rstverbinlinefont\Verb{\rstuline{#1}}}% -} - -\newcommand{\rstcodeitem}[1]{\Verb{#1}} - -\newenvironment{rstdocitem}{% -\VerbatimEnvironment\begingroup\begin{rstdocitembox}% -\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% -{\end{Verbatim}\end{rstdocitembox}\endgroup} - - -\newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}} -\ifdim\linewidth<30em - \def\rstoptleftmargin{0.4\linewidth} - \def\rstoptlabelwidth{0.35\linewidth} -\else - \def\rstoptleftmargin{12em} - \def\rstoptlabelwidth{10.5em} -\fi -\newenvironment{rstoptlist}{% -\begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} - -\usepackage{tabulary} % tables with adjustable cell width and no overflow -% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088) -\tymin=60pt -\tymax=\maxdimen -% to pack tabulary into a new environment, special syntax is needed :-( -\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary} - -\newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}} -\newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}} - -\newcommand{\rsthA}[2][]{\section[#1]{#2}} -\newcommand{\rsthB}[2][]{\subsection[#1]{#2}} -\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}} -\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}} -\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}} - -\newcommand{\rstovA}[2][]{\section*[#1]{#2}} -\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}} -\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}} -\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}} -\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}} - -% Syntax highlighting: -\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanIdentifier}[1]{#1} -\newcommand{\spanKeyword}[1]{\textbf{#1}} -\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanCharLit}[1]{#1} -\newcommand{\spanEscapeSequence}[1]{#1} -\newcommand{\spanOperator}[1]{\textbf{#1}} -\newcommand{\spanPunctuation}[1]{#1} -\newcommand{\spanComment}[1]{\emph{#1}} -\newcommand{\spanLongComment}[1]{\emph{#1}} -\newcommand{\spanRegularExpression}[1]{#1} -\newcommand{\spanTagStart}[1]{#1} -\newcommand{\spanTagEnd}[1]{#1} -\newcommand{\spanKey}[1]{#1} -\newcommand{\spanValue}[1]{#1} -\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}} -\newcommand{\spanAssembler}[1]{#1} -\newcommand{\spanPreprocessor}[1]{#1} -\newcommand{\spanDirective}[1]{#1} -\newcommand{\spanCommand}[1]{#1} -\newcommand{\spanRule}[1]{#1} -\newcommand{\spanHyperlink}[1]{#1} -\newcommand{\spanLabel}[1]{#1} -\newcommand{\spanReference}[1]{#1} -\newcommand{\spanOther}[1]{#1} -\newcommand{\spantok}[1]{\fbox{#1}} -\newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}} -\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}} -\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}} -\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}} +\documentclass{nimdoc} \begin{document} \title{$title $version $subtitle} \author{$author} -% Never allow text overflow to margin: -\setlength\emergencystretch{\hsize}\hbadness=10000 - \maketitle $content diff --git a/doc/nimdoc.cls b/doc/nimdoc.cls new file mode 100644 index 000000000..271dc5dc9 --- /dev/null +++ b/doc/nimdoc.cls @@ -0,0 +1,190 @@ +\ProvidesClass{nimdoc}[2022/04/17, 2018/01/01 LaTeX2e nonstandard class] + +\LoadClass[a4paper,11pt]{article} + +\usepackage[a4paper,xetex,left=3cm,right=3cm,top=1.5cm,bottom=2cm]{geometry} + +% for 2-sided printing with larger inner "binding" margin +%\usepackage[a4paper,xetex,twoside,left=4cm,right=2cm,top=1.5cm,bottom=2cm]{geometry} +% for e-readers with 1.77:1 aspect ratio (e.g. 1920x1080) +%\usepackage[xetex,paperheight=27.6cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} +% for e-readers with 1.45:1 aspect ratio (e.g. 1200x825) +%\usepackage[xetex,paperheight=22.5cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} +% for e-readers with 1.33:1 aspect ratio (e.g. 1872x1404) +%\usepackage[xetex,paperheight=20.7cm,paperwidth=15.5cm,left=3mm,right=3mm,top=3mm,bottom=3mm]{geometry} + +\usepackage{fontspec} +% logic to select default font with some fall-back fonts. +\IfFontExistsTF{Times New Roman}{% + \setmainfont{Times New Roman} % the default font + \typeout{========================================= nim: using Times New Roman} +}{ + \IfFontExistsTF{FreeSerif}{% + \setmainfont{FreeSerif} % fallback #1 - official GNU font, resembles Times + \typeout{========================================= nim: using FreeSerif} + }{ + \IfFontExistsTF{DejaVuSerif}{% + \setmainfont{DejaVuSerif} % fallback #2 - very widespread free font + \typeout{========================================= nim: using DejaVuSerif} + }{ + \typeout{!!!!!!!!!!!!!!!!!!! Fonts not found !!!!!!!!!!!!!!!!!!!!!!!} + } + } +} + +% default monospace font for code: +\usepackage{GoMono} +\usepackage{relsize} +% make this monospace font 2 steps smaller to hold 80-character line +\newcommand{\rstverbblockfont}{\smaller[2]} +\newcommand{\rstverbinlinefont}{\smaller} + +\usepackage{parskip} % paragraphs delimited by vertical space, no indent +\usepackage{graphicx} + +\usepackage{makeidx} +\newcommand{\nimindexterm}[2]{#2\index{#2}\label{#1}} +\makeindex + +\usepackage{dingbat} % for \carriagereturn, etc +\usepackage{fvextra} % for code blocks (works better than original fancyvrb) +\fvset{ + breaklines, + breakafter={=}:|\_\{\}[](){,}.;+-*/'", + breaksymbolleft=\color{red}{\ensuremath{\hookrightarrow}}, + breaksymbolright=\color{red}{\small\carriagereturn} +} +\fvinlineset{% + breaklines, + breakafter={=}:|\_\{\}[](){,}.;+-*/'", + % that does not work at all when we underline inline code by ulem :-( + commandchars=\\\{\} +} + +\usepackage{scrextend} % for the `addmargin` environment + +\usepackage{xcolor} +\usepackage[urlbordercolor=blue,linkbordercolor=cyan, + pdfborderstyle={/S/U/W 1}]{hyperref} +\usepackage{enumitem} % for option list, enumList, and rstfootnote + +\usepackage[most]{tcolorbox} % boxes around admonitions, code blocks, doc.item + +\newtcolorbox{rstadmonition}[1][]{blanker, breakable, + left=3mm, right=0mm, top=1mm, bottom=1mm, + before upper=\indent, parbox=false, #1} + +\newtcolorbox{rstquote}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + parbox=false, + borderline west={0.3em}{0pt}{lightgray}, + borderline north={0.05em}{0pt}{lightgray}, + borderline east={0.05em}{0pt}{lightgray}, + borderline south={0.05em}{0pt}{lightgray}} + +\definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} + +\newtcolorbox{rstprebox}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + borderline ={0.1em}{0pt}{rstframecolor}, + before upper=\indent, parbox=false, #1} + +\newenvironment{rstpre}{% +\VerbatimEnvironment\begingroup\begin{rstprebox}% +\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% +{\end{Verbatim}\end{rstprebox}\endgroup} + +\newtcolorbox{rstdocitembox}[1][]{blanker, breakable, + left=3mm, right=3mm, top=1mm, bottom=1mm, + borderline ={1pt}{0pt}{cyan}, + before upper=\indent, parbox=false, #1} + +% Inline code formatting: grey underline, +% use \Verb from fvextras e.g. to display -- correctly as double - +\usepackage[normalem]{ulem} +\newcommand\rstuline{\bgroup\markoverwith{\textcolor{rstframecolor}{\rule[-0.8ex]{2pt}{1.0pt}}}\ULon} + +\newcommand{\rstcode}[1]{% +{\rstverbinlinefont\Verb{\rstuline{#1}}}% +} + +\newcommand{\rstcodeitem}[1]{\Verb{#1}} + +\newenvironment{rstdocitem}{% +\VerbatimEnvironment\begingroup\begin{rstdocitembox}% +\begin{Verbatim}[fontsize=\rstverbblockfont , commandchars=\\\{\}]}% +{\end{Verbatim}\end{rstdocitembox}\endgroup} + + +\newenvironment{rstfootnote}{\begin{description}[labelindent=1em,leftmargin=1em,labelwidth=2.6em]}{\end{description}} +\ifdim\linewidth<30em + \def\rstoptleftmargin{0.4\linewidth} + \def\rstoptlabelwidth{0.35\linewidth} +\else + \def\rstoptleftmargin{12em} + \def\rstoptlabelwidth{10.5em} +\fi +\newenvironment{rstoptlist}{% +\begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} + +\usepackage{tabulary} % tables with adjustable cell width and no overflow +% make tabulary prevent overflows (https://tex.stackexchange.com/a/195088) +\tymin=60pt +\tymax=\maxdimen +% to pack tabulary into a new environment, special syntax is needed :-( +\newenvironment{rsttab}[1]{\tabulary{\linewidth}{#1}}{\endtabulary} + +\newcommand{\rstsub}[1]{\raisebox{-0.5ex}{\scriptsize{#1}}} +\newcommand{\rstsup}[1]{\raisebox{0.5ex}{\scriptsize{#1}}} + +\newcommand{\rsthA}[2][]{\section[#1]{#2}} +\newcommand{\rsthB}[2][]{\subsection[#1]{#2}} +\newcommand{\rsthC}[2][]{\subsubsection[#1]{#2}} +\newcommand{\rsthD}[2][]{\paragraph[#1]{#2}} +\newcommand{\rsthE}[2][]{\paragraph[#1]{#2}} + +\newcommand{\rstovA}[2][]{\section*[#1]{#2}} +\newcommand{\rstovB}[2][]{\subsection*[#1]{#2}} +\newcommand{\rstovC}[2][]{\subsubsection*[#1]{#2}} +\newcommand{\rstovD}[2][]{\paragraph*[#1]{#2}} +\newcommand{\rstovE}[2][]{\paragraph*[#1]{#2}} + +% Syntax highlighting: +\newcommand{\spanDecNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanBinNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanHexNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanOctNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanFloatNumber}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanIdentifier}[1]{#1} +\newcommand{\spanKeyword}[1]{\textbf{#1}} +\newcommand{\spanStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanLongStringLit}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanCharLit}[1]{#1} +\newcommand{\spanEscapeSequence}[1]{#1} +\newcommand{\spanOperator}[1]{\textbf{#1}} +\newcommand{\spanPunctuation}[1]{#1} +\newcommand{\spanComment}[1]{\emph{#1}} +\newcommand{\spanLongComment}[1]{\emph{#1}} +\newcommand{\spanRegularExpression}[1]{#1} +\newcommand{\spanTagStart}[1]{#1} +\newcommand{\spanTagEnd}[1]{#1} +\newcommand{\spanKey}[1]{#1} +\newcommand{\spanValue}[1]{#1} +\newcommand{\spanRawData}[1]{\textbf{\textcolor{darkgray}{#1}}} +\newcommand{\spanAssembler}[1]{#1} +\newcommand{\spanPreprocessor}[1]{#1} +\newcommand{\spanDirective}[1]{#1} +\newcommand{\spanCommand}[1]{#1} +\newcommand{\spanRule}[1]{#1} +\newcommand{\spanHyperlink}[1]{#1} +\newcommand{\spanLabel}[1]{#1} +\newcommand{\spanReference}[1]{#1} +\newcommand{\spanOther}[1]{#1} +\newcommand{\spantok}[1]{\fbox{#1}} +\newcommand{\spanPrompt}[1]{\textcolor{red}{\textbf{#1}}} +\newcommand{\spanProgramOutput}[1]{\textcolor{darkgray}{\textbf{#1}}} +\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}} +\newcommand{\spanoption}[1]{\textbf{\textcolor{darkgray}{#1}}} + +% Never allow text overflow to margin: +\setlength\emergencystretch{\hsize}\hbadness=10000 From 02e8aa9660ae78a4d6a67656709f9379f30c5197 Mon Sep 17 00:00:00 2001 From: huantian Date: Sun, 24 Apr 2022 02:47:39 -0700 Subject: [PATCH 0776/1654] Fix doc: list of async backends (#19741) --- lib/pure/asyncdispatch.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b67b0c308..b1e1c19fd 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -187,9 +187,9 @@ ## ## Known `async` backends include: ## -## * `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely -## * `asyncdispatch -``-d:asyncBackend=asyncdispatch`` -## * `chronos ` - ``-d:asyncBackend=chronos`` +## * `-d:asyncBackend=none`: disable `async` support completely +## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html +## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/ ## ## ``none`` can be used when a library supports both a synchronous and ## asynchronous API, to disable the latter. From b10f0e7bca43761316f6424786a771af33254e19 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 02:20:18 +0800 Subject: [PATCH 0777/1654] fix db_sqlite.tryInsertID does raise exceptions in 1.6.0 #19743 (#19744) --- lib/impure/db_sqlite.nim | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 1638d38c6..db40dddd8 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -635,7 +635,7 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared): string proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [DbError].} = + {.tags: [WriteDbEffect], raises: [].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. ## @@ -650,16 +650,19 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, ## 1, "item#1") == -1 ## db.close() assert(not db.isNil, "Database not connected.") - var q = dbFormat(query, args) - var stmt: sqlite3.PStmt result = -1 - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: - result = -1 - else: - discard finalize(stmt) + try: + var q = dbFormat(query, args) + var stmt: sqlite3.PStmt + if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: + if step(stmt) == SQLITE_DONE: + result = last_insert_rowid(db) + if finalize(stmt) != SQLITE_OK: + result = -1 + else: + discard finalize(stmt) + except DbError: + discard proc insertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = @@ -692,7 +695,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [DbError], since: (1, 3).} = + {.tags: [WriteDbEffect], raises: [], since: (1, 3).} = ## same as tryInsertID tryInsertID(db, query, args) From 8e6136dd59c737dab7c244e69f52864bd3fa65c9 Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 25 Apr 2022 09:06:53 +0000 Subject: [PATCH 0778/1654] document localPassC consistently with compiler; fix ANSI capitalization (#19721) --- doc/manual.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 8206d8342..adc1d9d16 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1370,7 +1370,7 @@ cstring type The `cstring` type meaning `compatible string` is the native representation of a string for the compilation backend. For the C backend the `cstring` type represents a pointer to a zero-terminated char array -compatible with the type `char*` in Ansi C. Its primary purpose lies in easy +compatible with the type `char*` in ANSI C. Its primary purpose lies in easy interfacing with C. The index operation `s[i]` means the i-th *char* of `s`; however no bounds checking for `cstring` is performed making the index operation unsafe. @@ -7218,16 +7218,16 @@ during semantic analysis: {.passc: gorge("pkg-config --cflags sdl").} -LocalPassc pragma +localPassC pragma ----------------- -The `localPassc` pragma can be used to pass additional parameters to the C +The `localPassC` pragma can be used to pass additional parameters to the C compiler, but only for the C/C++ file that is produced from the Nim module the pragma resides in: .. code-block:: Nim # Module A.nim # Produces: A.nim.cpp - {.localPassc: "-Wall -Werror".} # Passed when compiling A.nim.cpp + {.localPassC: "-Wall -Werror".} # Passed when compiling A.nim.cpp PassL pragma From efaa6777a464b94d88cb948390397b490ccee5b8 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 17:07:55 +0800 Subject: [PATCH 0779/1654] fix #19435; don't create TypeBoundOps for tyOpenArray, tyVarargs [backport: 1.6] (#19723) * fix #19435; openArray wronyly registers typebounds * add testcase * don't create TypeBoundOps for tyOpenArray, tyVarargs --- compiler/sempass2.nim | 2 +- tests/arc/t19435.nim | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/arc/t19435.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 9bc811a03..0ee806f36 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1054,7 +1054,7 @@ proc track(tracked: PEffects, n: PNode) = addAsgnFact(tracked.guards, n[0], n[1]) notNilCheck(tracked, n[1], n[0].typ) when false: cstringCheck(tracked, n) - if tracked.owner.kind != skMacro: + if tracked.owner.kind != skMacro and n[0].typ.kind notin {tyOpenArray, tyVarargs}: createTypeBoundOps(tracked, n[0].typ, n.info) if n[0].kind != nkSym or not isLocalVar(tracked, n[0].sym): checkForSink(tracked.config, tracked.c.idgen, tracked.owner, n[1]) diff --git a/tests/arc/t19435.nim b/tests/arc/t19435.nim new file mode 100644 index 000000000..519216bad --- /dev/null +++ b/tests/arc/t19435.nim @@ -0,0 +1,29 @@ +discard """ + matrix: "--gc:arc" +""" + +# bug #19435 +{.experimental: "views".} + +type + Bar = object + placeholder: int + Foo = object + placeholder: int + c: seq[Bar] # remove this line to make things right + +func children*(s: var seq[Foo]): openArray[Foo] = + s.toOpenArray(0, s.len-1) + +proc test = + var foos = @[Foo(), Foo()] + + assert foos.children.len == 2 + var flag = true + for a in foos.children: + flag = false + + if flag: + doAssert false + +test() \ No newline at end of file From 2f32b450d39030c287df4d56be9d06473772f5cd Mon Sep 17 00:00:00 2001 From: Danil Yarantsev Date: Mon, 25 Apr 2022 12:15:03 +0300 Subject: [PATCH 0780/1654] Really fix StringStream with ARC at compile-time, improve streams test (#19739) * Fix compile-time StringStream with ARC * make readDataStr work with ARC, improve test --- lib/pure/streams.nim | 10 ++++++++-- tests/stdlib/tstreams.nim | 9 +++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 2ac6a82f1..f58273ee8 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -243,6 +243,9 @@ proc readDataStr*(s: Stream, buffer: var string, slice: Slice[int]): int = result = s.readDataStrImpl(s, buffer, slice) else: # fallback + when declared(prepareMutation): + # buffer might potentially be a CoW literal with ARC + prepareMutation(buffer) result = s.readData(addr buffer[slice.a], slice.b + 1 - slice.a) template jsOrVmBlock(caseJsOrVm, caseElse: untyped): untyped = @@ -1274,8 +1277,11 @@ else: # after 1.3 or JS not defined new(result) result.data = s - when declared(prepareMutation): - prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(result.data) # Allows us to mutate using `addr` logic like `copyMem`, otherwise it errors. result.pos = 0 result.closeImpl = ssClose result.atEndImpl = ssAtEnd diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index 6fda30f51..d9857926e 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--gc:refc; --gc:arc" input: "Arne" output: ''' Hello! What is your name? @@ -74,9 +75,13 @@ block: doAssert(ss.peekLine(str)) doAssert(str == "uick brown fox jumped over the lazy dog.") doAssert(ss.getPosition == 5) # haven't moved - ss.setPosition(0) # Ensure we dont error with writing over literals on arc/orc #19707 + # bug #19707 - Ensure we dont error with writing over literals on arc/orc + ss.setPosition(0) ss.write("hello") + ss.setPosition(0) + doAssert(ss.peekStr(5) == "hello") +# bug #19716 static: # Ensure streams it doesnt break with nimscript on arc/orc #19716 let s = newStringStream("a") - discard s.data + doAssert s.data == "a" From 465fd06d50d0dd2e65762b718ea9a726d94c110b Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 17:18:26 +0800 Subject: [PATCH 0781/1654] add warnings for gc:option (#19722) --- compiler/commands.nim | 91 +++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 98537e9ef..bab98b376 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -527,6 +527,50 @@ proc unregisterArcOrc(conf: ConfigRef) = excl conf.globalOptions, optSeqDestructors excl conf.globalOptions, optTinyRtti +proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass, + info: TLineInfo; conf: ConfigRef) = + if conf.backend == backendJs: return # for: bug #16033 + expectArg(conf, switch, arg, pass, info) + if pass in {passCmd2, passPP}: + case arg.normalize + of "boehm": + unregisterArcOrc(conf) + conf.selectedGC = gcBoehm + defineSymbol(conf.symbols, "boehmgc") + incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS + of "refc": + unregisterArcOrc(conf) + conf.selectedGC = gcRefc + of "markandsweep": + unregisterArcOrc(conf) + conf.selectedGC = gcMarkAndSweep + defineSymbol(conf.symbols, "gcmarkandsweep") + of "destructors", "arc": + registerArcOrc(pass, conf, false) + of "orc": + registerArcOrc(pass, conf, true) + of "hooks": + conf.selectedGC = gcHooks + defineSymbol(conf.symbols, "gchooks") + incl conf.globalOptions, optSeqDestructors + processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info) + if pass in {passCmd2, passPP}: + defineSymbol(conf.symbols, "nimSeqsV2") + of "go": + unregisterArcOrc(conf) + conf.selectedGC = gcGo + defineSymbol(conf.symbols, "gogc") + of "none": + unregisterArcOrc(conf) + conf.selectedGC = gcNone + defineSymbol(conf.symbols, "nogc") + of "stack", "regions": + unregisterArcOrc(conf) + conf.selectedGC = gcRegions + defineSymbol(conf.symbols, "gcregions") + of "v2": warningOptionNoop(arg) + else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf: ConfigRef) = var @@ -626,48 +670,11 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info) of "project": processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info) - of "gc", "mm": - if conf.backend == backendJs: return # for: bug #16033 - expectArg(conf, switch, arg, pass, info) - if pass in {passCmd2, passPP}: - case arg.normalize - of "boehm": - unregisterArcOrc(conf) - conf.selectedGC = gcBoehm - defineSymbol(conf.symbols, "boehmgc") - incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS - of "refc": - unregisterArcOrc(conf) - conf.selectedGC = gcRefc - of "markandsweep": - unregisterArcOrc(conf) - conf.selectedGC = gcMarkAndSweep - defineSymbol(conf.symbols, "gcmarkandsweep") - of "destructors", "arc": - registerArcOrc(pass, conf, false) - of "orc": - registerArcOrc(pass, conf, true) - of "hooks": - conf.selectedGC = gcHooks - defineSymbol(conf.symbols, "gchooks") - incl conf.globalOptions, optSeqDestructors - processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info) - if pass in {passCmd2, passPP}: - defineSymbol(conf.symbols, "nimSeqsV2") - of "go": - unregisterArcOrc(conf) - conf.selectedGC = gcGo - defineSymbol(conf.symbols, "gogc") - of "none": - unregisterArcOrc(conf) - conf.selectedGC = gcNone - defineSymbol(conf.symbols, "nogc") - of "stack", "regions": - unregisterArcOrc(conf) - conf.selectedGC = gcRegions - defineSymbol(conf.symbols, "gcregions") - of "v2": warningOptionNoop(arg) - else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) + of "gc": + warningDeprecated(conf, info, "`gc:option` is deprecated; use `mm:option` instead") + processMemoryManagementOption(switch, arg, pass, info, conf) + of "mm": + processMemoryManagementOption(switch, arg, pass, info, conf) of "warnings", "w": if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf) of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf) From 15ae9323e8f1e045bdd68e16231528f8c08fb611 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 17:19:21 +0800 Subject: [PATCH 0782/1654] fix NimNode comment repr() regression [backport: 1.2] (#19726) --- compiler/renderer.nim | 2 ++ tests/macros/tmacros1.nim | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 3503c4bf6..f847aa094 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -165,6 +165,7 @@ proc putNL(g: var TSrcGen) = proc optNL(g: var TSrcGen, indent: int) = g.pendingNL = indent g.lineLen = indent + g.col = g.indent when defined(nimpretty): g.pendingNewlineCount = 0 proc optNL(g: var TSrcGen) = @@ -173,6 +174,7 @@ proc optNL(g: var TSrcGen) = proc optNL(g: var TSrcGen; a, b: PNode) = g.pendingNL = g.indent g.lineLen = g.indent + g.col = g.indent when defined(nimpretty): g.pendingNewlineCount = lineDiff(a, b) proc indentNL(g: var TSrcGen) = diff --git a/tests/macros/tmacros1.nim b/tests/macros/tmacros1.nim index 53af4287c..c588ff7e6 100644 --- a/tests/macros/tmacros1.nim +++ b/tests/macros/tmacros1.nim @@ -60,3 +60,22 @@ macro foo(t: static Tuple): untyped = doAssert t.b == 12345 foo((a: "foo", b: 12345)) + + +# bug #16307 + +macro bug(x: untyped): string = + newLit repr(x) + +let res = bug: + block: + ## one + ## two + ## three + +doAssert res == """ + +block: + ## one + ## two + ## three""" From ee0a47029443e6d717ed4a85825b7dcf971dbdf8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 25 Apr 2022 11:59:37 +0200 Subject: [PATCH 0783/1654] Revert "fix db_sqlite.tryInsertID does raise exceptions in 1.6.0 #19743 (#19744)" (#19745) This reverts commit b10f0e7bca43761316f6424786a771af33254e19. --- lib/impure/db_sqlite.nim | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index db40dddd8..1638d38c6 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -635,7 +635,7 @@ proc getValue*(db: DbConn, stmtName: SqlPrepared): string proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [].} = + {.tags: [WriteDbEffect], raises: [DbError].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. ## @@ -650,19 +650,16 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, ## 1, "item#1") == -1 ## db.close() assert(not db.isNil, "Database not connected.") + var q = dbFormat(query, args) + var stmt: sqlite3.PStmt result = -1 - try: - var q = dbFormat(query, args) - var stmt: sqlite3.PStmt - if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: - if step(stmt) == SQLITE_DONE: - result = last_insert_rowid(db) - if finalize(stmt) != SQLITE_OK: - result = -1 - else: - discard finalize(stmt) - except DbError: - discard + if prepare_v2(db, q, q.len.cint, stmt, nil) == SQLITE_OK: + if step(stmt) == SQLITE_DONE: + result = last_insert_rowid(db) + if finalize(stmt) != SQLITE_OK: + result = -1 + else: + discard finalize(stmt) proc insertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = @@ -695,7 +692,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc tryInsert*(db: DbConn, query: SqlQuery, pkName: string, args: varargs[string, `$`]): int64 - {.tags: [WriteDbEffect], raises: [], since: (1, 3).} = + {.tags: [WriteDbEffect], raises: [DbError], since: (1, 3).} = ## same as tryInsertID tryInsertID(db, query, args) From 42ac50e988a4734624b204740613c795a0e789a3 Mon Sep 17 00:00:00 2001 From: flywind Date: Mon, 25 Apr 2022 18:00:23 +0800 Subject: [PATCH 0784/1654] fixes #19662; Indent level errored for first line (#19718) --- compiler/parser.nim | 4 ++++ compiler/passes.nim | 1 + compiler/syntaxes.nim | 2 +- tests/parser/t19662.nim | 6 ++++++ 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/parser/t19662.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index 7973f7d37..52466d9fa 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -2393,6 +2393,10 @@ proc parseAll(p: var Parser): PNode = if p.tok.indent != 0: parMessage(p, errInvalidIndentation) +proc checkFirstLineIndentation*(p: var Parser) = + if p.tok.indent != 0 and p.tok.strongSpaceA > 0: + parMessage(p, errInvalidIndentation) + proc parseTopLevelStmt(p: var Parser): PNode = ## Implements an iterator which, when called repeatedly, returns the next ## top-level statement or emptyNode if end of stream. diff --git a/compiler/passes.nim b/compiler/passes.nim index ce52b10c5..7fb2842f5 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -150,6 +150,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; processImplicits graph, graph.config.implicitImports, nkImportStmt, a, module processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module + checkFirstLineIndentation(p) while true: if graph.stopCompile(): break var n = parseTopLevelStmt(p) diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index e47b0483d..c00fe8b67 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -16,7 +16,7 @@ import when defined(nimPreviewSlimSystem): import std/[syncio, assertions] -export Parser, parseAll, parseTopLevelStmt, closeParser +export Parser, parseAll, parseTopLevelStmt, checkFirstLineIndentation, closeParser type FilterKind = enum diff --git a/tests/parser/t19662.nim b/tests/parser/t19662.nim new file mode 100644 index 000000000..7a1864ac6 --- /dev/null +++ b/tests/parser/t19662.nim @@ -0,0 +1,6 @@ + var i: int # bug #19662 +echo i + +discard """ + errormsg: "invalid indentation" +""" \ No newline at end of file From 82680a12a7a6acfbb6f5fdd22c042e409081b812 Mon Sep 17 00:00:00 2001 From: ehmry Date: Mon, 25 Apr 2022 15:16:11 -0500 Subject: [PATCH 0785/1654] macros: make hasCustomPragma more permissive (#19747) Make hasCustomPragma return false rather than fail for invalid parameters. --- lib/core/macros.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index ba19712cf..b370aaa7b 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1521,7 +1521,7 @@ proc extractTypeImpl(n: NimNode): NimNode = else: error("Invalid node to retrieve type implementation of: " & $n.kind) proc customPragmaNode(n: NimNode): NimNode = - expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkCheckedFieldExpr}) + expectKind(n, {nnkSym, nnkDotExpr, nnkBracketExpr, nnkTypeOfExpr, nnkType, nnkCheckedFieldExpr}) let typ = n.getTypeInst() @@ -1532,7 +1532,9 @@ proc customPragmaNode(n: NimNode): NimNode = if kind(typ[1]) == nnkBracketExpr: typ[1][0] else: typ[1] ) - if impl[0].kind == nnkPragmaExpr: + if impl.kind == nnkNilLit: + return impl + elif impl[0].kind == nnkPragmaExpr: return impl[0][1] else: return impl[0] # handle types which don't have macro at all @@ -1560,7 +1562,7 @@ proc customPragmaNode(n: NimNode): NimNode = while typDef != nil: typDef.expectKind(nnkTypeDef) let typ = typDef[2].extractTypeImpl() - typ.expectKind({nnkRefTy, nnkPtrTy, nnkObjectTy}) + if typ.kind notin {nnkRefTy, nnkPtrTy, nnkObjectTy}: break let isRef = typ.kind in {nnkRefTy, nnkPtrTy} if isRef and typ[0].kind in {nnkSym, nnkBracketExpr}: # defines ref type for another object(e.g. X = ref X) typDef = getImpl(typ[0]) From ef4ac5a0d23fcffac9963670b73fa69ad0d0aa73 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 26 Apr 2022 08:58:15 +0200 Subject: [PATCH 0786/1654] use signed comparisons for the index checking in the hope it improves the code generation (#19712) --- compiler/ccgexprs.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1a3e217b2..5a58b4bcd 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -988,12 +988,12 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = if reifiedOpenArray(arr.lode): linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3.Field1) || (NU)($2) >= (NU)($3.Field1))){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError(); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) else: linefmt(p, cpsStmts, - "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))){ #raiseIndexError(); $4}$n", + "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" & + "{ #raiseIndexError(); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) of tyArray: let first = intLiteral(firstOrd(p.config, ty)) @@ -1004,7 +1004,7 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = of tySequence, tyString: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError(); $4}$n", [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)]) else: discard @@ -1015,14 +1015,14 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) = if not reifiedOpenArray(x): # emit range check: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n", + linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2Len_0){ #raiseIndexError2($1,$2Len_0-1); $3}$n", [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage) else: if optBoundsCheck in p.options: - linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n", + linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2.Field1){ #raiseIndexError2($1,$2.Field1-1); $3}$n", [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``! inheritLocation(d, a) putIntoDest(p, d, n, @@ -1037,7 +1037,7 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) = ty = skipTypes(ty.lastSon, abstractVarRange) # emit range check: if optBoundsCheck in p.options: linefmt(p, cpsStmts, - "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n", + "if ($1 < 0 || $1 >= $2){ #raiseIndexError2($1,$2-1); $3}$n", [rdCharLoc(b), lenExpr(p, a), raiseInstr(p)]) if d.k == locNone: d.storage = OnHeap if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: From 8bfc396a4dfa722239818f399a119452a53fe07f Mon Sep 17 00:00:00 2001 From: flywind Date: Wed, 27 Apr 2022 02:14:39 +0800 Subject: [PATCH 0787/1654] fixes #18612; apply cache and memcmp for methods in arc/orc (#19749) * try using endsWith * use memcmp * add cache * cleanup * better * minor * fix * improve test coverage for methods with ARC --- compiler/ccgexprs.nim | 7 +++++-- lib/system/arc.nim | 30 +++++++++++++++++++++++++++--- tests/metatype/ttypedesc3.nim | 3 ++- tests/method/tgeneric_methods.nim | 1 + tests/method/tmethod_issues.nim | 1 + tests/method/tmethod_various.nim | 1 + tests/method/tsingle_methods.nim | 2 +- tests/misc/parsecomb.nim | 4 ++++ tests/pragmas/tlocks.nim | 3 +++ 9 files changed, 45 insertions(+), 7 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 5a58b4bcd..8e7a21c67 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1615,8 +1615,11 @@ proc genNewFinalize(p: BProc, e: PNode) = proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo): Rope = if optTinyRtti in p.config.globalOptions: - result = ropecg(p.module, "#isObj($1.m_type, $2)", - [a, genTypeInfo2Name(p.module, dest)]) + let ti = genTypeInfo2Name(p.module, dest) + inc p.module.labels + let cache = "Nim_OfCheck_CACHE" & p.module.labels.rope + p.module.s[cfsVars].addf("static TNimTypeV2* $#[2];$n", [cache]) + result = ropecg(p.module, "#isObjWithCache($#.m_type, $#, $#)", [a, ti, cache]) else: # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we # have to call it here first: diff --git a/lib/system/arc.nim b/lib/system/arc.nim index d66f4b997..17142b277 100644 --- a/lib/system/arc.nim +++ b/lib/system/arc.nim @@ -227,10 +227,34 @@ template tearDownForeignThreadGc* = ## With `--gc:arc` a nop. discard -proc isObj(obj: PNimTypeV2, subclass: cstring): bool {.compilerRtl, inl.} = - proc strstr(s, sub: cstring): cstring {.header: "", importc.} +type ObjCheckCache = array[0..1, PNimTypeV2] - result = strstr(obj.name, subclass) != nil +proc memcmp(str1, str2: cstring, n: csize_t): cint {.importc, header: "".} + +func endsWith(s, suffix: cstring): bool {.inline.} = + let + sLen = s.len + suffixLen = suffix.len + + if suffixLen <= sLen: + result = memcmp(cstring(addr s[sLen - suffixLen]), suffix, csize_t(suffixLen)) == 0 + +proc isObj(obj: PNimTypeV2, subclass: cstring): bool {.compilerRtl, inl.} = + result = endsWith(obj.name, subclass) + +proc isObjSlowPath(obj: PNimTypeV2, subclass: cstring, cache: var ObjCheckCache): bool {.compilerRtl, inline.} = + if endsWith(obj.name, subclass): + cache[1] = obj + result = true + else: + cache[0] = obj + result = false + +proc isObjWithCache(obj: PNimTypeV2, subclass: cstring, cache: var ObjCheckCache): bool {.compilerRtl.} = + if cache[0] == obj: result = false + elif cache[1] == obj: result = true + else: + result = isObjSlowPath(obj, subclass, cache) proc chckObj(obj: PNimTypeV2, subclass: cstring) {.compilerRtl.} = # checks if obj is of type subclass: diff --git a/tests/metatype/ttypedesc3.nim b/tests/metatype/ttypedesc3.nim index 98a59f613..d3a58853d 100644 --- a/tests/metatype/ttypedesc3.nim +++ b/tests/metatype/ttypedesc3.nim @@ -1,5 +1,6 @@ discard """ -output: ''' + matrix: "--mm:arc; --mm:refc" + output: ''' proc Base proc Child method Base diff --git a/tests/method/tgeneric_methods.nim b/tests/method/tgeneric_methods.nim index 0e2aeeede..ab92c66d8 100644 --- a/tests/method/tgeneric_methods.nim +++ b/tests/method/tgeneric_methods.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:arc; --mm:refc" output: '''wow2 X 1 X 3''' diff --git a/tests/method/tmethod_issues.nim b/tests/method/tmethod_issues.nim index df4c3771a..13467f2b3 100644 --- a/tests/method/tmethod_issues.nim +++ b/tests/method/tmethod_issues.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:arc; --mm:refc" output: ''' wof! wof! diff --git a/tests/method/tmethod_various.nim b/tests/method/tmethod_various.nim index fd022717b..f9d067a72 100644 --- a/tests/method/tmethod_various.nim +++ b/tests/method/tmethod_various.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--mm:arc; --mm:refc" output: ''' do nothing HELLO WORLD! diff --git a/tests/method/tsingle_methods.nim b/tests/method/tsingle_methods.nim index 40269559a..b564e7d87 100644 --- a/tests/method/tsingle_methods.nim +++ b/tests/method/tsingle_methods.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim c --multimethods:off $file" + matrix: "--mm:arc --multimethods:off; --mm:refc --multimethods:off" output: '''base base base diff --git a/tests/misc/parsecomb.nim b/tests/misc/parsecomb.nim index 4ff2f65d2..4f149cbf6 100644 --- a/tests/misc/parsecomb.nim +++ b/tests/misc/parsecomb.nim @@ -1,3 +1,7 @@ +discard """ + matrix: "--mm:arc; --mm:refc" +""" + type Input[T] = object toks: seq[T] index: int diff --git a/tests/pragmas/tlocks.nim b/tests/pragmas/tlocks.nim index ba66a2dca..6c2a9f9e9 100644 --- a/tests/pragmas/tlocks.nim +++ b/tests/pragmas/tlocks.nim @@ -1,3 +1,6 @@ +discard """ + matrix: "--mm:arc; --mm:refc" +""" type SomeBase* = ref object of RootObj type SomeDerived* = ref object of SomeBase From e4a2c2d474f725b01acfa3a6169686f5e16f2ce9 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Fri, 29 Apr 2022 12:16:07 +0300 Subject: [PATCH 0788/1654] Make sure that field usage preserves the original line info (#19751) Currently `struct.field` will generate a node with `info` that points to the symbol definition instead of having the actual node location. --- compiler/semexprs.nim | 2 ++ nimsuggest/tests/tuse_structure.nim | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 nimsuggest/tests/tuse_structure.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 702b47929..b8742ff65 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1436,8 +1436,10 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = # is the access to a public field or in the same module or in a friend? markUsed(c, n[1].info, f) onUse(n[1].info, f) + let info = n[1].info n[0] = makeDeref(n[0]) n[1] = newSymNode(f) # we now have the correct field + n[1].info = info # preserve the original info n.typ = f.typ if check == nil: result = n diff --git a/nimsuggest/tests/tuse_structure.nim b/nimsuggest/tests/tuse_structure.nim new file mode 100644 index 000000000..f65ab9060 --- /dev/null +++ b/nimsuggest/tests/tuse_structure.nim @@ -0,0 +1,15 @@ +# tests for use and structures + +type + Foo* = ref object of RootObj + bar*: string + +proc test(f: Foo) = + echo f.#[!]#bar + +discard """ +$nimsuggest --tester $file +>use $1 +def skField tuse_structure.Foo.bar string $file 5 4 "" 100 +use skField tuse_structure.Foo.bar string $file 8 9 "" 100 +""" From 4680ab61c06782d142492d1fcdebf8e942373c09 Mon Sep 17 00:00:00 2001 From: nc-x Date: Sat, 30 Apr 2022 19:28:58 +0530 Subject: [PATCH 0789/1654] Fix fixAbstractType for user defined typeclasses, fixes #19730 & #18409 (#19732) --- compiler/semexprs.nim | 2 +- tests/concepts/t18409.nim | 37 +++++++++++++++++++++++++++++++++++++ tests/concepts/t19730.nim | 20 ++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/concepts/t18409.nim create mode 100644 tests/concepts/t19730.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index b8742ff65..ed28d8145 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -645,7 +645,7 @@ proc fixAbstractType(c: PContext, n: PNode) = skipTypes(it.typ, abstractVar).kind notin {tyOpenArray, tyVarargs}: if skipTypes(it[1].typ, abstractVar).kind in {tyNil, tyTuple, tySet} or it[1].isArrayConstr: - var s = skipTypes(it.typ, abstractVar) + var s = skipTypes(it.typ, abstractVar + tyUserTypeClasses) if s.kind != tyUntyped: changeType(c, it[1], s, check=true) n[i] = it[1] diff --git a/tests/concepts/t18409.nim b/tests/concepts/t18409.nim new file mode 100644 index 000000000..0edba2d31 --- /dev/null +++ b/tests/concepts/t18409.nim @@ -0,0 +1,37 @@ +discard """ + action: "compile" +""" + +# A vector space over a field F concept. +type VectorSpace*[F] = concept x, y, type V + vector_add(x, y) is V + scalar_mul(x, F) is V + dimension(V) is Natural + +# Real numbers (here floats) form a vector space. +func vector_add*(v: float, w: float): float = v + w +func scalar_mul*(v: float, s: float): float = v * s +func dimension*(x: typedesc[float]): Natural = 1 + +# 2-tuples of real numbers form a vector space. +func vector_add*(v, w: (float, float)): (float, float) = + (vector_add(v[0], w[0]), vector_add(v[1], w[1])) + +func scalar_mul*(v: (float, float), s: float): (float, float) = + (scalar_mul(v[0], s), scalar_mul(v[1], s)) + +func dimension*(x: typedesc[(float, float)]): Natural = 2 + +# Check concept requirements. +assert float is VectorSpace +assert (float, float) is VectorSpace + +# Commutivity axiom for vector spaces over the same field. +func axiom_commutivity*[F](u, v: VectorSpace[F]): bool = + vector_add(u, v) == vector_add(v, u) + +# This is okay. +assert axiom_commutivity(2.2, 3.3) + +# This is not. +assert axiom_commutivity((2.2, 3.3), (4.4, 5.5)) diff --git a/tests/concepts/t19730.nim b/tests/concepts/t19730.nim new file mode 100644 index 000000000..575d45dda --- /dev/null +++ b/tests/concepts/t19730.nim @@ -0,0 +1,20 @@ +discard """ + output: '''1.01.01.01.0 +1.01.01.01.0 +''' +""" + +type + Color = concept c + c.r is SomeFloat + c.g is SomeFloat + c.b is SomeFloat + c.a is SomeFloat + +proc useColor(color: Color) = + echo(color.r, color.g, color.b, color.a) + +let color = (r: 1.0, g: 1.0, b: 1.0, a: 1.0) +useColor(color) + +useColor((r: 1.0, g: 1.0, b: 1.0, a: 1.0)) From 927978345bbf57631828287850eda13b82277eb7 Mon Sep 17 00:00:00 2001 From: huantian Date: Mon, 2 May 2022 09:06:57 -0700 Subject: [PATCH 0790/1654] Rework discarding futures documentation in asyncdispatch (#19738) * Rework discarding futures docs in asyncdispatch * Fix typos Co-authored-by: Danil Yarantsev * Use rst note:: Co-authored-by: flywind * Split discarding and handling futures. * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim * Update lib/pure/asyncdispatch.nim Co-authored-by: Danil Yarantsev Co-authored-by: flywind Co-authored-by: Dominik Picheta --- lib/pure/asyncdispatch.nim | 39 ++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index b1e1c19fd..755f59ff7 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -123,15 +123,42 @@ ## if future.failed: ## # Handle exception ## -## ## Discarding futures ## ================== ## -## Futures should **never** be discarded. This is because they may contain -## errors. If you do not care for the result of a Future then you should -## use the `asyncCheck` procedure instead of the `discard` keyword. Note -## however that this does not wait for completion, and you should use -## `waitFor` for that purpose. +## Futures should **never** be discarded directly because they may contain +## errors. If you do not care for the result of a Future then you should use +## the `asyncCheck` procedure instead of the `discard` keyword. Note that this +## does not wait for completion, and you should use `waitFor` or `await` for that purpose. +## +## .. note:: `await` also checks if the future fails, so you can safely discard +## its result. +## +## Handling futures +## ================ +## +## There are many different operations that apply to a future. +## The three primary high-level operations are `asyncCheck`, +## `waitFor`, and `await`. +## +## * `asyncCheck`: Raises an exception if the future fails. It neither waits +## for the future to finish nor returns the result of the future. +## * `waitFor`: Polls the event loop and blocks the current thread until the +## future finishes. This is often used to call an async procedure from a +## synchronous context and should never be used in an `async` proc. +## * `await`: Pauses execution in the current async procedure until the future +## finishes. While the current procedure is paused, other async procedures will +## continue running. Should be used instead of `waitFor` in an async +## procedure. +## +## Here is a handy quick reference chart showing their high-level differences: +## ============== ===================== ======================= +## Procedure Context Blocking +## ============== ===================== ======================= +## `asyncCheck` non-async and async non-blocking +## `waitFor` non-async blocks current thread +## `await` async suspends current proc +## ============== ===================== ======================= ## ## Examples ## ======== From 832b8c50345eab6e6f3d037d6e521831007cb6ed Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 3 May 2022 12:08:39 +1000 Subject: [PATCH 0791/1654] Fix link to experimental manual in macro pragmas section [skip ci] (#19753) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index adc1d9d16..a92d6e18f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -7814,7 +7814,7 @@ the same way. There are a few more applications of macro pragmas, such as in type, variable and constant declarations, but this behavior is considered to be experimental and is documented in the `experimental manual -` instead. +`_ instead. Foreign function interface From 2ecae20c437179ff84e255f952dae7000f7ff316 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 3 May 2022 18:35:16 +0800 Subject: [PATCH 0792/1654] switch to mainline bigints (#19756) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 6b0b15fd8..a55327211 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -39,7 +39,7 @@ pkg "arraymancer", "nim c tests/tests_cpu.nim" pkg "ast_pattern_matching", "nim c -r --oldgensym:on tests/test1.nim", allowFailure = true pkg "asyncthreadpool" pkg "awk" -pkg "bigints", url = "https://github.com/Araq/nim-bigints" +pkg "bigints" pkg "binaryheap", "nim c -r binaryheap.nim" pkg "BipBuffer" pkg "blscurve", allowFailure = true # pending https://github.com/status-im/nim-blscurve/issues/39 From 278ecad973c6581aeea0a6ff9372109b0dd6df5e Mon Sep 17 00:00:00 2001 From: Zoom Date: Wed, 4 May 2022 14:24:52 +0000 Subject: [PATCH 0793/1654] Add 'usages' option to the --stylechecks error msg (#19759) --- compiler/commands.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index bab98b376..2e08ad615 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -116,7 +116,7 @@ const errInvalidCmdLineOption = "invalid command line option: '$1'" errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found" errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found" - errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found" + errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found" proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) = if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-") @@ -508,7 +508,7 @@ proc registerArcOrc(pass: TCmdLinePass, conf: ConfigRef, isOrc: bool) = else: conf.selectedGC = gcArc defineSymbol(conf.symbols, "gcarc") - + defineSymbol(conf.symbols, "gcdestructors") incl conf.globalOptions, optSeqDestructors incl conf.globalOptions, optTinyRtti From a4401054cc7fbd199a8dfa3ec0f9760ca7498b8e Mon Sep 17 00:00:00 2001 From: Anthony Dario Date: Fri, 6 May 2022 02:16:00 -0700 Subject: [PATCH 0794/1654] Fix broken link in sets documentation. (#19769) --- lib/pure/collections/sets.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 7b1c58ac8..114e4582a 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -61,7 +61,7 @@ type HashSet*[A] {.myShallow.} = object ## \ ## A generic hash set. ## - ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet,int>`_ + ## Use `init proc <#init,HashSet[A]>`_ or `initHashSet proc <#initHashSet>`_ ## before calling other procs on it. data: KeyValuePairSeq[A] counter: int From 0455d24d555e4f83537af535c0b2e44291d1fe97 Mon Sep 17 00:00:00 2001 From: Zoom Date: Fri, 6 May 2022 09:19:27 +0000 Subject: [PATCH 0795/1654] Fix questionable suggestion in `strutils` docs (#19765) * Fix questionable suggestion in `stutils` docs - Removes the recommendation to pass a string slice for getting a relative index for `find` and `rfind` functions, as this currently makes a string copy, while a simple subtraction is enough. - Docstring for `SkipTable` type. * Doc layout fixup --- lib/pure/strutils.nim | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index bf7bd6aa8..b98bd2bec 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -278,7 +278,7 @@ func nimIdentNormalize*(s: string): string = ## except first one. ## ## .. Warning:: Backticks (`) are not handled: they remain *as is* and - ## spaces are preserved. See `nimIdentBackticksNormalize + ## spaces are preserved. See `nimIdentBackticksNormalize ## `_ for ## an alternative approach. runnableExamples: @@ -1808,11 +1808,15 @@ func join*[T: not string](a: openArray[T], sep: string = ""): string = add(result, $x) type - SkipTable* = array[char, int] + SkipTable* = array[char, int] ## Character table for efficient substring search. func initSkipTable*(a: var SkipTable, sub: string) {.rtl, extern: "nsuInitSkipTable".} = - ## Preprocess table `a` for `sub`. + ## Initializes table `a` for efficient search of substring `sub`. + ## + ## See also: + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + # TODO: this should be the `default()` initializer for the type. let m = len(sub) fill(a, m) @@ -1826,6 +1830,9 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. ## element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let last = if last == 0: s.high else: last subLast = sub.len - 1 @@ -1865,7 +1872,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `rfind func<#rfind,string,char,Natural,int>`_ @@ -1893,7 +1900,7 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `rfind func<#rfind,string,set[char],Natural,int>`_ @@ -1910,7 +1917,7 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `rfind func<#rfind,string,string,Natural,int>`_ @@ -1950,7 +1957,7 @@ func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].find` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,char,Natural,int>`_ @@ -1968,7 +1975,7 @@ func rfind*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,set[char],Natural,int>`_ @@ -1986,7 +1993,7 @@ func rfind*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. - ## Use `s[start..last].rfind` for a `start`-origin index. + ## Subtract `start` from the result for a `start`-origin index. ## ## See also: ## * `find func<#find,string,string,Natural,int>`_ From 2c73e84436a11cae1676c7da0228158ba1a885cc Mon Sep 17 00:00:00 2001 From: kraptor Date: Sat, 7 May 2022 20:46:45 +0200 Subject: [PATCH 0796/1654] "ip" protocol as id 0 is long gone (#19760) IANA is using id 0 for "HOPOPT" instead of "ip" for some time now and those systems that still support the old mapping will stop doing so at some point in the future. Some BSDs and openSUSE are already following this change as per IANA list here: https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml As new unixes (and maybe Windows, who knows) start to adopt the updated IANA list, this will keep failing from time to time, so it's better to remove the "ip" check altogether. --- tests/stdlib/tgetprotobyname.nim | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/stdlib/tgetprotobyname.nim b/tests/stdlib/tgetprotobyname.nim index 0b60d059f..014c18845 100644 --- a/tests/stdlib/tgetprotobyname.nim +++ b/tests/stdlib/tgetprotobyname.nim @@ -1,9 +1,5 @@ import nativesockets -when not defined(netbsd): - # Ref: https://github.com/nim-lang/Nim/issues/15452 - NetBSD doesn't define an `ip` protocol - doAssert getProtoByName("ip") == 0 - doAssert getProtoByName("ipv6") == 41 doAssert getProtoByName("tcp") == 6 doAssert getProtoByName("udp") == 17 From 8c100a37b93607806acec51733fe5e2fda392d44 Mon Sep 17 00:00:00 2001 From: kraptor Date: Sun, 8 May 2022 11:54:04 +0200 Subject: [PATCH 0797/1654] Add new self-signed 4096 bit certificate to testdata (#19758) The test tasyncssl may fail on modern linux versions as they require at least 2048 bit certificates. A new certificate and private key with default values has been added to meet this new requirement. --- tests/testdata/mycert.pem | 105 ++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 28 deletions(-) diff --git a/tests/testdata/mycert.pem b/tests/testdata/mycert.pem index 69039fe04..61dcb685c 100644 --- a/tests/testdata/mycert.pem +++ b/tests/testdata/mycert.pem @@ -1,32 +1,81 @@ -----BEGIN PRIVATE KEY----- -MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAL9QXX/nuiFbizdI -Uhg1D9gG0GIANENvKwdWTlOlZAoOqvjXPFLGh+87yhvkq4f5FcICkSDao2SfeZcP -JsgD7T01owt8x48898+d91i7nIpr6IXGPyBxHOuaxAITY1D+MbbkhIGUVrEqKEOm -qfS9cqPZaDNkx8xVef0HPCmqEme9AgMBAAECgYBxqrQCvJFQJG3QiL2N+GjTdyj0 -MR7cOf6cu2CKPifz+ccHVgpXO/Gj6Cgq7nAjt5B/1rqXhI+zxzSc1bm6+OpIfakS -E0DLCFacECmL0v3c+XLxTtMhFZF5u7Yq0UMsuWmDSfRb4sbRjC+s+c51i5N0485k -b3un/MDI/i/jD/YZGQJBAPLtcuMIwEblUR1uw7NFezXdauXCRFkekoSlJNvpdM/Z -XDRcuWioek5yD8FvMpTz7H2e26Ev645JT5lIuN4Eti8CQQDJm+Qt9NYUohRsU279 -GYI3vXsXKKqmA22at4I3KRXPSeYV1vtQLYWWqGAXzgGkUEVBY0chmHyDcNwkUsNw -svHTAkEAwOTpD/vX6bOXOD7GqKgoULozcqNScE2FXExhuzliJtTakT17f+4fyABs -IFWynXIevBUTIqeRbJcr3HRRTwIAwwJBAJQ8XkL4IaxcG/4mPpY0ek13sZiumwKj -xKQcx869E78tS9LFFlW2kuHafYUjQIvLRZC1aWinUO3oPsUqYW9s82cCQFjoods5 -YsWEJB2RKCT5nhyAXEZLehxF+FXr+JjLMHkuEINKTnHHKjHJ7LbMcTCKUJAcKDTA -qZFEq5N1aT6DrAU= +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDgpWhnNdNSUbwJ +39mKf+PA879K1wyT23LC3Ipo5g5vaRjB4vrSh5vzrjK5Q/z42yRDLQ2ojEdUne19 +1xMaM+WqTXnfkf9sou4VJ0HsdQ6jOU1LDRPrPvCjHLZjyMu18sGvQf4FATBwWN3n +QhmvcbleTeyv2pgMNeQDfjuIhRJ/aCIi/WYQ466+Rcj0y3/udYX1yPYf2mszXnSE +i2iWgdsjx0qkDU/nJhnKXhKfucAm9Ds0YfQdZYN3rEzfGkJzZUhDG1n30ghu+vUg +qqLi6HU/HqBxKxBlJf2HdMNK4VLkJuFC+wCI0JfF6VYhcNmtZzlQy3M9WjYvbXUY +nvMkQzUECaSaAtmbhFFfgey1Q3IEQS/j1lsVxtuGUE3YaoyaK22/CkVInDz0aka9 +CKq77x7S/V5PP/wwPV4s8XsbT6/34KemuY3oU84+bf5SBuWdzBY0hnDO9c/FaOnZ +4yjLMQi285HLpK417Z32DmzIkb7GjKtP9WLvsWGgbgquEjcFaIT+erkN09cDjsG/ +URWbKcsQkz8v9E4zj0rxdanXoJEtg1gm+dPsiXokZOLTeaMVtDoTkWeCM/VAp25J +MTc656QRUTSzUODK5tXQ3JgE2XIPGV/PsMi7T/q9xFBQmhUk5kfLNlQ1cy/aiTWK +8pjHcmxbyD121AimuTtv9CFYnYtzyQIDAQABAoICAAMYZFrfs/yzYZrlObMd1f6H +nUAjvGmhIXCr50BQwywnz46EWR5jffOal9pTpH2tT0+ZpFGJNUZmMqqENyAqTOTO +0noRIerWR9+EvfTLHBuFo5oAISEhqeEleSHg12W3ZZHLn/tjq84we0Y/c+kl8P7q +pfM6WNP6Ph0KNTnJU5rrzWScBzb+XB8FCSLOVwHrHqBnV3TS5p07lPFqllNUkLdq +fI3MHSi7LqnKKAmJXqtqvBIZs2pgRrJ0bk64pue+IoPCMbgnbbRRwuTjVQE5YLww +6NcGV+B86IRgSHyUpDa+jmYE3VoFPcIdV/F4A5fPD0wcsYbL4mk+4dkn/4OlZWqV +NZILp2IGejKaKtM1fr7fV2IRUbGUBN/+rX4I0SRnuq6Q4Ipx8VASbpgXQPBo9XTg +SHHsFbEu2DL8BHVgXdOy4PW6jQ0Ux2LhuJk6AQ5nIlFBYA+c8rSlZQXJbEbk1VZu +1i7iSOn/kx3ULMjUfhI/Ddm6rQqtiLbXSubXCzu1HMPT1FG9LUfAbq6EpiVkpAk1 +TqlciBHsyz//mk2RmIEx0Bt+0bX8FFGTIUiGyrp5s4hAHbgQZbXBAUYMRzWxhZ2Q +G0KBXx46bv3hJUb0GOgbNVxcaPnyrXaS/Hafcbx2LXlEtKiwGnC/yKJ7Hmcrt+AQ +RTaqNU1o/bkSYC7vHMZdAoIBAQD79uYPZPv5GLCKPZ64gc3+tbFXNqkmL7qv14yD +Gr3VubRbJe3Fx+T1cS+t9cjgOofhjFnwsDaFRoyWOYqRV/znwFsvvsDhHOLBr1u6 +qWQiF2CT1uMdXR3P6KD8h2DUVNNccxKqqIJNCR5oD/ngnnByWkQobzlsnoIdXgZm +ozBZjGr2XUMO5dJqUxaXZwY3j4I2hk/Ka9uroApyptl+DTVbPHvjk3MzU9QKUNor +vXEtQ8EmM8Oy6v/33HBmNs6cF5fMpgWz6u+B357OTxAfu8B42jZ18OeLrvkHFxzu +phOB1uXvqtQ0tdksSHHWj3IIZRK7GDGudnDEZ23vbCaxH4zbAoIBAQDkPn+5N3px +7UAECWrvT10TD3xKeqMkFhqRA9gmDE4N9AdoN6T6PzD7Tr3gOgGLq1tXjCjBqAlx +ZIDTnih3IK3xoRk2zmhq5+LfM8LQRAxAC8IsoQMXAsmW1KlS6MR70m50pFR8NK6r +UmOdrwVUKp3K6Mecid3LmMVLXGMUKwIJc1k7LJHtwrfi0i5xfBtiqQeaR0lJg4Zr ++zEL/4rHfcq06/P3k0+4uLKZ1LGOvwLPiTA3DADPWZbzUXo7McKOFWF/ycGQBrJq +AJikx15dVLnB16bnHXdxrlrd0LON2R+XfT4+dfRymqZLzrBI3L39t/elTmVYnD3P +punkmZuVwNErAoIBAQD5xOiOPibh6S2n/CmI8XQImIgx0kefSRUhFuV9WVbxtOMq +r9CijONUw1zmb40vahYk6gKGa8fAGg1nJadNKRHVkoNSMx/0h7PpGDIwOZa/jLj6 +FLyS8SmKXiqn6nN8SJI1RQUuE1kHkJCJy7yCg80oLn7+LjOYjxCgmAJ0YDSfsGif +zBebfws0xyTP9RrenO4RqtcR7BWYbk+tE+Tp5aIMzUpqcFJ0gRbjGv8K+QJmQpIH +kqzegcI4LFdnm9D4PxMFlVZ14eCGt+wuy4VKT84efwIZrDN77nmCI9FUaWFRBnxt +NsShc9rS4QWoEg6Sb88/lF47ecGTkIwUGPvJ/WKdAoIBAQClF7/zDPn4Zg+j29wJ +dXJxUwYoKUTP2V0l/43dF5Ft7lFdRMKEcCjR3kbhZZOwnyXW0X65dP4/kt7MMt46 +LN0kpc5DIlHM4iXsJNiJJG9n9BljhqNhhZajDvfbDJrypWdX33Vs0f511YZQjERi +eODh4DZiOCbCGaK7u/u+ns0+YLzuXHLBc9Lmsfj+BTMZzgG9ykpsbkJQ4MS9VP3h +BlAVRYaWUWucxZwKQRqdkfRKgYTqjDgZw0e4f/rVzkxX0YdQk3L65p0up3fB2KOd +BqfGWmJTUbEP/XmkcE0wERkUznazX0aNjucydjJ0wZZ7axIp8+bCjWD4TldoDuPH +Ek05AoIBAECgPfBHLQTAsI+wHbFsu/are28BOiJCSEXjRv88CbUbj/qgAppFuXbx +900WwJ1rVWd5x3LFa3VfyuAqYMi5jzmX9kWgEsC0WhgfyIRFiynw45LlcT4u3fWg +vJEx01lGgFVjnYfFUDS9d1MuiXGxIHrNhzHOP2x2CsS5vrFHav7iwG9YULEk8tJr +My0wzjF3UJ2/5DjGK56WuzauLYKrQ6Faw8dWUy4e/bNYId8wglhQQW548JwJEGmq +nq+EzTfEupXH57Bw7MGEOfdlhv98zNT9VcvBAN09vHeF3Hh6AM4aiGSUIt2HIkto +zvw+fqZ2Sk9O5qva+KE1QMVtY1EICI8= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICgDCCAemgAwIBAgIJANpVfZSDAyNgMA0GCSqGSIb3DQEBBQUAMFkxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xMjA2MDMxMjI4 -MDhaFw0xMzA2MDMxMjI4MDhaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21l -LVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNV -BAMMCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAv1Bdf+e6 -IVuLN0hSGDUP2AbQYgA0Q28rB1ZOU6VkCg6q+Nc8UsaH7zvKG+Srh/kVwgKRINqj -ZJ95lw8myAPtPTWjC3zHjzz3z533WLucimvohcY/IHEc65rEAhNjUP4xtuSEgZRW -sSooQ6ap9L1yo9loM2THzFV5/Qc8KaoSZ70CAwEAAaNQME4wHQYDVR0OBBYEFF2n -Of61swO+XSNrYb4T02tGx8afMB8GA1UdIwQYMBaAFF2nOf61swO+XSNrYb4T02tG -x8afMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAEPHofdf4acaph5/e -+BzZGsMfRqdPgwp5sxjFKeTQI1A49VL7ykkb0iLKGfKZtvE8MjMrYjzt20E2bIZj -8eCivT6TbNrVRoACCly/lH9fZfWOG6dBu/85IrTAhSKi8yjbRzmjWUkdrcEJ+ZtV -1cahfFar4l4QwYgqp2pDd6ie+zE= +MIIFCTCCAvGgAwIBAgIUKqDcJ71wiMObIQ5sga2sZItNseowDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUwNDExMzAwN1oXDTIyMDYw +MzExMzAwN1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEA4KVoZzXTUlG8Cd/Zin/jwPO/StcMk9tywtyKaOYOb2kY +weL60oeb864yuUP8+NskQy0NqIxHVJ3tfdcTGjPlqk1535H/bKLuFSdB7HUOozlN +Sw0T6z7woxy2Y8jLtfLBr0H+BQEwcFjd50IZr3G5Xk3sr9qYDDXkA347iIUSf2gi +Iv1mEOOuvkXI9Mt/7nWF9cj2H9prM150hItoloHbI8dKpA1P5yYZyl4Sn7nAJvQ7 +NGH0HWWDd6xM3xpCc2VIQxtZ99IIbvr1IKqi4uh1Px6gcSsQZSX9h3TDSuFS5Cbh +QvsAiNCXxelWIXDZrWc5UMtzPVo2L211GJ7zJEM1BAmkmgLZm4RRX4HstUNyBEEv +49ZbFcbbhlBN2GqMmittvwpFSJw89GpGvQiqu+8e0v1eTz/8MD1eLPF7G0+v9+Cn +prmN6FPOPm3+UgblncwWNIZwzvXPxWjp2eMoyzEItvORy6SuNe2d9g5syJG+xoyr +T/Vi77FhoG4KrhI3BWiE/nq5DdPXA47Bv1EVmynLEJM/L/ROM49K8XWp16CRLYNY +JvnT7Il6JGTi03mjFbQ6E5FngjP1QKduSTE3OuekEVE0s1DgyubV0NyYBNlyDxlf +z7DIu0/6vcRQUJoVJOZHyzZUNXMv2ok1ivKYx3JsW8g9dtQIprk7b/QhWJ2Lc8kC +AwEAAaNTMFEwHQYDVR0OBBYEFEZcUeqH6MfIzC56BlD3NSs2mCgGMB8GA1UdIwQY +MBaAFEZcUeqH6MfIzC56BlD3NSs2mCgGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggIBADGc7WONP6I6Trb7ici9fQ9qT3wh/RGDcmUDmDtARn9SFtOF +hsbszOMZg1Flj10fuD6OYDonKz4rv+Ieo5VkAYXxxd3J+bx2x1pqd1YSIsvugTwv +pnx39uBR9cjOmt4W7RyzhFnXoVfuSBE6LpkBUjcrqi5xwrQ31mOSCPwe8uDZYEWS +pX49MiHXGTyjQ481QLiOtTBZJa5igfnHUJbJbyZMa86zBQ/clS7+OeDwkvaEpjov +2VQf3QouVLghfLZYWSxWdEKD9+IWHn8rV6qksEb/Ogu4ZtzDRGqJow4j0DeSSEu7 +ns1YeT2mVTFwHjGXCWS+0iE885NDVX/b5YptlwH5PW7aqeXyCS9Hrd1C1GnXoXGp +NHltYRTyNWm974xWg7eu2gbbB8Ng02chXysdkBq7l+7OyA0a2EfX3Cbz3/49+Mqn +viqwNO5toSHVCdfV9Jd0p0CcqryYgyt2YNpJB+2nUQpiW4jviAs49PZg2PpCVw/2 +0cqtaPeUh26Si8UzDOuT697PIuGkZ9Q9QVwccVXtCyA0UpJ13P0fMrA+yEMhtwSs +k1tRm0pUQa6t3v26/cAy+kMhviHBJFwi5dx+y3OMvqQqpQJrgfZawm/o2ZQHy1KP +8m4ngrJzb13evKf216qCwllmQo6Ts4yeI1Ddx8UpdX7RUWpD8Uw4zSi7Th4r -----END CERTIFICATE----- From 61c1142667fabec11a7440b0404ce5688cad54d2 Mon Sep 17 00:00:00 2001 From: huantian Date: Sun, 8 May 2022 02:54:25 -0700 Subject: [PATCH 0798/1654] Update nimscript `withDir` doc example (#19776) --- lib/system/nimscript.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index 5d1916c7f..0b49ea2e7 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -342,17 +342,17 @@ template withDir*(dir: string; body: untyped): untyped = ## Usage example: ## ## .. code-block:: nim + ## # inside /some/path/ ## withDir "foo": - ## # inside foo - ## #back to last dir - var curDir = getCurrentDir() + ## # move to /some/path/foo/ + ## # back in /some/path/ + let curDir = getCurrentDir() try: cd(dir) body finally: cd(curDir) - proc writeTask(name, desc: string) = if desc.len > 0: var spaces = " " From b2b878f4d62ff2dcaf69979bb0f96e6015b47605 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sun, 8 May 2022 06:56:46 -0300 Subject: [PATCH 0799/1654] Remove deprecated posix proc (#19774) * Remove deprecated posix proc that takes wrong argument types * Remove deprecated posix proc that takes wrong argument types --- changelog.md | 2 +- lib/posix/posix.nim | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/changelog.md b/changelog.md index 8b9d0bc29..3b2685547 100644 --- a/changelog.md +++ b/changelog.md @@ -47,7 +47,7 @@ becomes an alias for `addr`. - Removed deprecated `math.c_frexp`. - Removed deprecated ``` httpcore.`==` ```. - Removed deprecated `std/dom_extensions`. - +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. - Remove deprecated `osproc.poDemon`, symbol with typo. - Deprecated `selfExe` for Nimscript. diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 146ba886f..4ebae4361 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -894,19 +894,9 @@ proc CMSG_NXTHDR*(mhdr: ptr Tmsghdr, cmsg: ptr Tcmsghdr): ptr Tcmsghdr {. proc CMSG_FIRSTHDR*(mhdr: ptr Tmsghdr): ptr Tcmsghdr {. importc, header: "".} -{.push warning[deprecated]: off.} -proc CMSG_SPACE*(len: csize): csize {. - importc, header: "", deprecated: "argument `len` should be of type `csize_t`".} -{.pop.} - proc CMSG_SPACE*(len: csize_t): csize_t {. importc, header: "".} -{.push warning[deprecated]: off.} -proc CMSG_LEN*(len: csize): csize {. - importc, header: "", deprecated: "argument `len` should be of type `csize_t`".} -{.pop.} - proc CMSG_LEN*(len: csize_t): csize_t {. importc, header: "".} From 85bc8326acc9bf18e748055e770a40890e7ac069 Mon Sep 17 00:00:00 2001 From: Alfred Morgan Date: Tue, 10 May 2022 05:21:35 -0700 Subject: [PATCH 0800/1654] varargs example erroneously transformed "abc" to "def" (#19781) --- doc/manual.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual.rst b/doc/manual.rst index a92d6e18f..126b0f0d6 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -1556,7 +1556,7 @@ type conversions in this context: myWriteln(stdout, 123, "abc", 4.0) # is transformed to: - myWriteln(stdout, [$123, $"def", $4.0]) + myWriteln(stdout, [$123, $"abc", $4.0]) In this example `$` is applied to any argument that is passed to the parameter `a`. (Note that `$` applied to strings is a nop.) From b9f243eb2aeb4930ba552f8bf487719d628f4a2a Mon Sep 17 00:00:00 2001 From: Alfred Morgan Date: Tue, 10 May 2022 08:45:57 -0700 Subject: [PATCH 0801/1654] string is missing formatting when calling fmt (#19780) it appears the documentation intends to compare & with .fmt but there is no formatting in the string. even though the assert is true it doesn't quite prove that .fmt is an equivalent formatter. --- lib/pure/strformat.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index ce3439600..247d0a296 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -705,6 +705,6 @@ macro `&`*(pattern: string{lit}): string = runnableExamples: let x = 7 assert &"{x}\n" == "7\n" # regular string literal - assert &"{x}\n" == "7\n".fmt # `fmt` can be used instead - assert &"{x}\n" != fmt"7\n" # see `fmt` docs, this would use a raw string literal + assert &"{x}\n" == "{x}\n".fmt # `fmt` can be used instead + assert &"{x}\n" != fmt"{x}\n" # see `fmt` docs, this would use a raw string literal strformatImpl(pattern.strVal, '{', '}') From c61eb5df32c66431d0d08417c39e268fdf236132 Mon Sep 17 00:00:00 2001 From: flywind Date: Fri, 13 May 2022 01:03:30 +0800 Subject: [PATCH 0802/1654] disable polypbren (#19787) * disable polypbren * Update important_packages.nim --- testament/important_packages.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index a55327211..c8fb5ad3c 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -103,7 +103,7 @@ pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" -pkg "nimlsp", allowFailure = true +pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true pkg "nimph", "nimble test", "https://github.com/disruptek/nimph", allowFailure = true @@ -125,7 +125,7 @@ pkg "patty" pkg "pixie" pkg "plotly", "nim c examples/all.nim" pkg "pnm" -pkg "polypbren" +pkg "polypbren", allowFailure = true pkg "prologue", "nimble tcompile" pkg "protobuf", "nim c -o:protobuff -r src/protobuf.nim" pkg "pylib" From 19001c070bbe7645ff45fcbd66ab221235715302 Mon Sep 17 00:00:00 2001 From: Anthony Dario Date: Thu, 12 May 2022 20:19:42 -0700 Subject: [PATCH 0803/1654] Fix typo in sequtils documentation (#19789) Found another small typo. --- lib/pure/collections/sequtils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 5e9b492c2..39908e9c1 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -914,7 +914,7 @@ template foldl*(sequence, operation, first): untyped = ## ## The `operation` parameter should be an expression which uses the variables ## `a` and `b` for each step of the fold. The `first` parameter is the - ## start value (the first `a`) and therefor defines the type of the result. + ## start value (the first `a`) and therefore defines the type of the result. ## ## **See also:** ## * `foldr template<#foldr.t,untyped,untyped>`_ From 33888a73840a7f9fa46f79e613d488de2d193916 Mon Sep 17 00:00:00 2001 From: flywind Date: Tue, 17 May 2022 13:37:26 +0800 Subject: [PATCH 0804/1654] [manual] TLock => Lock (#19785) * [manual] TLock => Lock * minor --- doc/manual.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 126b0f0d6..bfcb41b7a 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -8112,7 +8112,9 @@ Object fields and global variables can be annotated via a `guard` pragma: .. code-block:: nim - var glock: TLock + import std/locks + + var glock: Lock var gdata {.guard: glock.}: int The compiler then ensures that every access of `gdata` is within a `locks` @@ -8139,7 +8141,7 @@ that also implement some form of locking at runtime: .. code-block:: nim - template lock(a: TLock; body: untyped) = + template lock(a: Lock; body: untyped) = pthread_mutex_lock(a) {.locks: [a].}: try: @@ -8181,10 +8183,12 @@ the expressivity of the language: .. code-block:: nim + import std/locks + type ProtectedCounter = object v {.guard: L.}: int - L: TLock + L: Lock proc incCounters(counters: var openArray[ProtectedCounter]) = for i in 0..counters.high: From 06f02bb7716066241b04b2a92a3a61f539eadff2 Mon Sep 17 00:00:00 2001 From: Yardanico Date: Tue, 17 May 2022 10:56:39 +0300 Subject: [PATCH 0805/1654] Always use httpclient in nimgrab (#19767) --- tools/nimgrab.nim | 37 +--- tools/urldownloader.nim | 431 ---------------------------------------- 2 files changed, 9 insertions(+), 459 deletions(-) delete mode 100644 tools/urldownloader.nim diff --git a/tools/nimgrab.nim b/tools/nimgrab.nim index ee5eced1e..7e4161faf 100644 --- a/tools/nimgrab.nim +++ b/tools/nimgrab.nim @@ -1,33 +1,14 @@ +import std/[os, httpclient] +proc syncDownload(url, file: string) = + var client = newHttpClient() + proc onProgressChanged(total, progress, speed: BiggestInt) = + echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" + echo clamp(int(progress*100 div total), 0, 100), "%" -when defined(windows): - import os, urldownloader - - proc syncDownload(url, file: string) = - proc progress(status: DownloadStatus, progress: uint, total: uint, - message: string) {.gcsafe.} = - echo "Downloading " & url - let t = total.BiggestInt - if t != 0: - echo clamp(int(progress.BiggestInt*100 div t), 0, 100), "%" - else: - echo "0%" - - downloadToFile(url, file, {optUseCache}, progress) - echo "100%" - -else: - import os, asyncdispatch, httpclient - - proc syncDownload(url, file: string) = - var client = newHttpClient() - proc onProgressChanged(total, progress, speed: BiggestInt) = - echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" - echo clamp(int(progress*100 div total), 0, 100), "%" - - client.onProgressChanged = onProgressChanged - client.downloadFile(url, file) - echo "100%" + client.onProgressChanged = onProgressChanged + client.downloadFile(url, file) + echo "100%" if os.paramCount() != 2: quit "Usage: nimgrab " diff --git a/tools/urldownloader.nim b/tools/urldownloader.nim deleted file mode 100644 index 73e4034c9..000000000 --- a/tools/urldownloader.nim +++ /dev/null @@ -1,431 +0,0 @@ - -# -# -# Windows native FTP/HTTP/HTTPS file downloader -# (c) Copyright 2017 Eugene Kabanov -# -# See the file "LICENSE", included in this -# distribution, for details about the copyright. -# - -## This module implements native Windows FTP/HTTP/HTTPS downloading feature, -## using ``urlmon.UrlDownloadToFile()``. -## -## - -when not (defined(windows) or defined(nimdoc)): - {.error: "Platform is not supported.".} - -import os - -type - DownloadOptions* = enum - ## Available download options - optUseCache, ## Use Windows cache. - optUseProgressCallback, ## Report progress via callback. - optIgnoreSecurity ## Ignore HTTPS security problems. - - DownloadStatus* = enum - ## Available download status sent to ``progress`` callback. - statusProxyDetecting, ## Automatic Proxy detection. - statusCookieSent ## Cookie will be sent with request. - statusResolving, ## Resolving URL with DNS. - statusConnecting, ## Establish connection to server. - statusRedirecting ## HTTP redirection pending. - statusRequesting, ## Sending request to server. - statusMimetypeAvailable, ## Mimetype received from server. - statusBeginDownloading, ## Download process starting. - statusDownloading, ## Download process pending. - statusEndDownloading, ## Download process finished. - statusCacheAvailable ## File found in Windows cache. - statusUnsupported ## Unsupported status. - statusError ## Error happens. - - DownloadProgressCallback* = proc(status: DownloadStatus, progress: uint, - progressMax: uint, - message: string) - ## Progress callback. - ## - ## status - ## Indicate current stage of downloading process. - ## - ## progress - ## Number of bytes currently downloaded. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## progressMax - ## Number of bytes expected to download. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## message - ## Status message, which depends on ``status`` code. - ## - ## Available messages' values: - ## - ## statusResolving - ## URL hostname to be resolved. - ## statusConnecting - ## IP address - ## statusMimetypeAvailable - ## Downloading resource MIME type. - ## statusCacheAvailable - ## Path to filename stored in Windows cache. - -type - UUID = array[4, uint32] - - LONG = clong - ULONG = culong - HRESULT = clong - DWORD = uint32 - OLECHAR = uint16 - OLESTR = ptr OLECHAR - LPWSTR = OLESTR - UINT = cuint - REFIID = ptr UUID - -const - E_NOINTERFACE = 0x80004002'i32 - E_NOTIMPL = 0x80004001'i32 - S_OK = 0x00000000'i32 - - CP_UTF8 = 65001'u32 - - IID_IUnknown = UUID([0'u32, 0'u32, 192'u32, 1174405120'u32]) - IID_IBindStatusCallback = UUID([2045430209'u32, 298760953'u32, - 2852160140'u32, 195644160'u32]) - - BINDF_GETNEWESTVERSION = 0x00000010'u32 - BINDF_IGNORESECURITYPROBLEM = 0x00000100'u32 - BINDF_RESYNCHRONIZE = 0x00000200'u32 - BINDF_NO_UI = 0x00000800'u32 - BINDF_SILENTOPERATION = 0x00001000'u32 - BINDF_PRAGMA_NO_CACHE = 0x00002000'u32 - - ERROR_FILE_NOT_FOUND = 2 - ERROR_ACCESS_DENIED = 5 - - BINDSTATUS_FINDINGRESOURCE = 1 - BINDSTATUS_CONNECTING = 2 - BINDSTATUS_REDIRECTING = 3 - BINDSTATUS_BEGINDOWNLOADDATA = 4 - BINDSTATUS_DOWNLOADINGDATA = 5 - BINDSTATUS_ENDDOWNLOADDATA = 6 - BINDSTATUS_SENDINGREQUEST = 11 - BINDSTATUS_MIMETYPEAVAILABLE = 13 - BINDSTATUS_CACHEFILENAMEAVAILABLE = 14 - BINDSTATUS_PROXYDETECTING = 32 - BINDSTATUS_COOKIE_SENT = 34 - -type - STGMEDIUM = object - tymed: DWORD - pstg: pointer - pUnkForRelease: pointer - - SECURITY_ATTRIBUTES = object - nLength*: uint32 - lpSecurityDescriptor*: pointer - bInheritHandle*: int32 - - BINDINFO = object - cbSize: ULONG - stgmedData: STGMEDIUM - szExtraInfo: LPWSTR - grfBindInfoF: DWORD - dwBindVerb: DWORD - szCustomVerb: LPWSTR - cbstgmedData: DWORD - dwOptions: DWORD - dwOptionsFlags: DWORD - dwCodePage: DWORD - securityAttributes: SECURITY_ATTRIBUTES - iid: UUID - pUnk: pointer - dwReserved: DWORD - - IBindStatusCallback = object - vtable: ptr IBindStatusCallbackVTable - options: set[DownloadOptions] - objectRefCount: ULONG - binfoFlags: DWORD - progressCallback: DownloadProgressCallback - - PIBindStatusCallback = ptr IBindStatusCallback - LPBINDSTATUSCALLBACK = PIBindStatusCallback - - IBindStatusCallbackVTable = object - QueryInterface: proc (self: PIBindStatusCallback, - riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} - AddRef: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - Release: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - OnStartBinding: proc(self: PIBindStatusCallback, - dwReserved: DWORD, pib: pointer): HRESULT - {.gcsafe, stdcall.} - GetPriority: proc(self: PIBindStatusCallback, pnPriority: ptr LONG): HRESULT - {.gcsafe, stdcall.} - OnLowResource: proc(self: PIBindStatusCallback, dwReserved: DWORD): HRESULT - {.gcsafe, stdcall.} - OnProgress: proc(self: PIBindStatusCallback, ulProgress: ULONG, - ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT - {.gcsafe, stdcall.} - OnStopBinding: proc(self: PIBindStatusCallback, hresult: HRESULT, - szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} - GetBindInfo: proc(self: PIBindStatusCallback, grfBINDF: ptr DWORD, - pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} - OnDataAvailable: proc(self: PIBindStatusCallback, grfBSCF: DWORD, - dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT - {.gcsafe, stdcall.} - OnObjectAvailable: proc(self: PIBindStatusCallback, riid: REFIID, - punk: pointer): HRESULT - {.gcsafe, stdcall.} - -template FAILED(hr: HRESULT): bool = - (hr < 0) - -proc URLDownloadToFile(pCaller: pointer, szUrl: LPWSTR, szFileName: LPWSTR, - dwReserved: DWORD, - lpfnCb: LPBINDSTATUSCALLBACK): HRESULT - {.stdcall, dynlib: "urlmon.dll", importc: "URLDownloadToFileW".} - -proc WideCharToMultiByte(CodePage: UINT, dwFlags: DWORD, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpDefaultChar: ptr char, - lpUsedDefaultChar: ptr uint32): cint - {.stdcall, dynlib: "kernel32.dll", importc: "WideCharToMultiByte".} - -proc MultiByteToWideChar(CodePage: UINT, dwFlags: DWORD, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint): cint - {.stdcall, dynlib: "kernel32.dll", importc: "MultiByteToWideChar".} -proc DeleteUrlCacheEntry(lpszUrlName: LPWSTR): int32 - {.stdcall, dynlib: "wininet.dll", importc: "DeleteUrlCacheEntryW".} - -proc `==`(a, b: UUID): bool = - result = false - if a[0] == b[0] and a[1] == b[1] and - a[2] == b[2] and a[3] == b[3]: - result = true - -proc `$`(bstr: LPWSTR): string = - var buffer: char - var count = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(buffer), 0, - nil, nil) - if count == 0: - raiseOsError(osLastError()) - else: - result = newString(count + 8) - let res = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(result[0]), count, - nil, nil) - if res == 0: - raiseOsError(osLastError()) - result.setLen(res - 1) - -proc toBstring(str: string): LPWSTR = - var buffer: OLECHAR - var count = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - addr(buffer), 0) - if count == 0: - raiseOsError(osLastError()) - else: - result = cast[LPWSTR](alloc0((count + 1) * sizeof(OLECHAR))) - let res = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - result, count) - if res == 0: - raiseOsError(osLastError()) - -proc freeBstring(bstr: LPWSTR) = - dealloc(bstr) - -proc getStatus(scode: ULONG): DownloadStatus = - case scode - of 0: result = statusError - of BINDSTATUS_PROXYDETECTING: result = statusProxyDetecting - of BINDSTATUS_REDIRECTING: result = statusRedirecting - of BINDSTATUS_COOKIE_SENT: result = statusCookieSent - of BINDSTATUS_FINDINGRESOURCE: result = statusResolving - of BINDSTATUS_CONNECTING: result = statusConnecting - of BINDSTATUS_SENDINGREQUEST: result = statusRequesting - of BINDSTATUS_MIMETYPEAVAILABLE: result = statusMimetypeAvailable - of BINDSTATUS_BEGINDOWNLOADDATA: result = statusBeginDownloading - of BINDSTATUS_DOWNLOADINGDATA: result = statusDownloading - of BINDSTATUS_ENDDOWNLOADDATA: result = statusEndDownloading - of BINDSTATUS_CACHEFILENAMEAVAILABLE: result = statusCacheAvailable - else: result = statusUnsupported - -proc addRef(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - inc(self.objectRefCount) - result = self.objectRefCount - -proc release(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - dec(self.objectRefCount) - result = self.objectRefCount - -proc queryInterface(self: PIBindStatusCallback, riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} = - pvObject[] = nil - - if riid[] == IID_IUnknown: - pvObject[] = cast[pointer](self) - elif riid[] == IID_IBindStatusCallback: - pvObject[] = cast[pointer](self) - - if not isNil(pvObject[]): - discard addRef(self) - result = S_OK - else: - result = E_NOINTERFACE - -proc onStartBinding(self: PIBindStatusCallback, dwReserved: DWORD, - pib: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc getPriority(self: PIBindStatusCallback, - pnPriority: ptr LONG): HRESULT {.gcsafe, stdcall.} = - result = E_NOTIMPL - -proc onLowResource(self: PIBindStatusCallback, - dwReserved: DWORD): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onStopBinding(self: PIBindStatusCallback, - hresult: HRESULT, szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc getBindInfo(self: PIBindStatusCallback, - grfBINDF: ptr DWORD, pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} = - var cbSize = pbindinfo.cbSize - zeroMem(cast[pointer](pbindinfo), cbSize) - pbindinfo.cbSize = cbSize - grfBINDF[] = self.binfoFlags - result = S_OK - -proc onDataAvailable(self: PIBindStatusCallback, - grfBSCF: DWORD, dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onObjectAvailable(self: PIBindStatusCallback, - riid: REFIID, punk: pointer): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc onProgress(self: PIBindStatusCallback, - ulProgress: ULONG, ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT {.gcsafe, stdcall.} = - var message: string - if optUseProgressCallback in self.options: - if not isNil(szStatusText): - message = $szStatusText - else: - message = "" - self.progressCallback(getStatus(ulStatusCode), uint(ulProgress), - uint(ulProgressMax), message) - result = S_OK - -proc newBindStatusCallback(): IBindStatusCallback = - result = IBindStatusCallback() - result.vtable = cast[ptr IBindStatusCallbackVTable]( - alloc0(sizeof(IBindStatusCallbackVTable)) - ) - result.vtable.QueryInterface = queryInterface - result.vtable.AddRef = addRef - result.vtable.Release = release - result.vtable.OnStartBinding = onStartBinding - result.vtable.GetPriority = getPriority - result.vtable.OnLowResource = onLowResource - result.vtable.OnStopBinding = onStopBinding - result.vtable.GetBindInfo = getBindInfo - result.vtable.OnDataAvailable = onDataAvailable - result.vtable.OnObjectAvailable = onObjectAvailable - result.vtable.OnProgress = onProgress - result.objectRefCount = 1 - -proc freeBindStatusCallback(v: var IBindStatusCallback) = - dealloc(v.vtable) - -proc downloadToFile*(szUrl: string, szFileName: string, - options: set[DownloadOptions] = {}, - progresscb: DownloadProgressCallback = nil) = - ## Downloads from URL specified in ``szUrl`` to local filesystem path - ## specified in ``szFileName``. - ## - ## szUrl - ## URL to download, international names are supported. - ## szFileName - ## Destination path for downloading resource. - ## options - ## Downloading options. Currently only 2 options supported. - ## progresscb - ## Callback procedure, which will be called throughout the download - ## process, indicating status and progress. - ## - ## Available downloading options: - ## - ## optUseCache - ## Try to use Windows cache when downloading. - ## optIgnoreSecurity - ## Ignore HTTPS security problems, e.g. self-signed HTTPS certificate. - ## - var bszUrl = szUrl.toBstring() - var bszFile = szFileName.toBstring() - var bstatus = newBindStatusCallback() - - bstatus.options = {} - - if optUseCache notin options: - bstatus.options.incl(optUseCache) - let res = DeleteUrlCacheEntry(bszUrl) - if res == 0: - let err = osLastError() - if err.int notin {ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND}: - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(err) - - bstatus.binfoFlags = BINDF_GETNEWESTVERSION or BINDF_RESYNCHRONIZE or - BINDF_PRAGMA_NO_CACHE or BINDF_NO_UI or - BINDF_SILENTOPERATION - - if optIgnoreSecurity in options: - bstatus.binfoFlags = bstatus.binfoFlags or BINDF_IGNORESECURITYPROBLEM - - if not isNil(progresscb): - bstatus.options.incl(optUseProgressCallback) - bstatus.progressCallback = progresscb - - let res = URLDownloadToFile(nil, bszUrl, bszFile, 0, addr bstatus) - if FAILED(res): - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(OSErrorCode(res)) - - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - -when isMainModule: - proc progress(status: DownloadStatus, progress: uint, progressMax: uint, - message: string) {.gcsafe.} = - const downset: set[DownloadStatus] = {statusBeginDownloading, - statusDownloading, statusEndDownloading} - if status in downset: - var message = "Downloaded " & $progress & " of " & $progressMax & "\c" - stdout.write(message) - else: - echo "Status [" & $status & "] message = [" & $message & "]" - - downloadToFile("https://nim-lang.org/download/mingw64.7z", - "test.zip", {optUseCache}, progress) From 63cca93ea9bcab3c1fea39ff3789fd3a656d2db2 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 19 May 2022 17:11:53 +0200 Subject: [PATCH 0806/1654] testament: include extra options in test name (#19801) there's currently no (simple) way to disambiguate which option failed --- testament/categories.nim | 12 +++--- testament/testament.nim | 79 +++++++++++++++++++--------------------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index 46cc3b210..a092fec84 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -437,7 +437,7 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = if pkg.allowFailure: inc r.passed inc r.failedButAllowed - addResult(r, test, targetC, "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) + addResult(r, test, targetC, "", "", cmd & "\n" & outp, reFailed, allowFailure = pkg.allowFailure) continue outp @@ -450,21 +450,21 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = discard tryCommand("nimble install --depsOnly -y", maxRetries = 3) discard tryCommand(pkg.cmd, reFailed = reBuildFailed) inc r.passed - r.addResult(test, targetC, "", "", reSuccess, allowFailure = pkg.allowFailure) + r.addResult(test, targetC, "", "", "", reSuccess, allowFailure = pkg.allowFailure) errors = r.total - r.passed if errors == 0: - r.addResult(packageFileTest, targetC, "", "", reSuccess) + r.addResult(packageFileTest, targetC, "", "", "", reSuccess) else: - r.addResult(packageFileTest, targetC, "", "", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "", reBuildFailed) except JsonParsingError: errors = 1 - r.addResult(packageFileTest, targetC, "", "Invalid package file", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "Invalid package file", reBuildFailed) raise except ValueError: errors = 1 - r.addResult(packageFileTest, targetC, "", "Unknown package", reBuildFailed) + r.addResult(packageFileTest, targetC, "", "", "Unknown package", reBuildFailed) raise # bug #18805 finally: if errors == 0: removeDir(packagesDir) diff --git a/testament/testament.nim b/testament/testament.nim index 4b5a8a147..a4329b982 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -259,7 +259,8 @@ Tests skipped: $4 / $1
              """ % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum, allowFailure = false, givenSpec: ptr TSpec = nil) = + extraOptions, expected, given: string, successOrig: TResultEnum, + allowFailure = false, givenSpec: ptr TSpec = nil) = # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename @@ -269,6 +270,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, if allowFailure: name.add " (allowed to fail) " if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout @@ -333,7 +335,8 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, discard waitForExit(p) close(p) -proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = +proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}" var covered = initIntSet() for line in splitLines(given.nimout): @@ -367,10 +370,10 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, e.add ": " e.add expected.inlineErrors[j].msg - r.addResult(test, target, e, given.nimout, reMsgsDiffer) + r.addResult(test, target, extraOptions, e, given.nimout, reMsgsDiffer) break coverCheck - r.addResult(test, target, "", given.msg, reSuccess) + r.addResult(test, target, extraOptions, "", given.msg, reSuccess) inc(r.passed) proc nimoutCheck(expected, given: TSpec): bool = @@ -381,22 +384,23 @@ proc nimoutCheck(expected, given: TSpec): bool = elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): result = false -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = +proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, + target: TTarget, extraOptions: string) = if expected.inlineErrors.len > 0: - checkForInlineErrors(r, expected, given, test, target) + checkForInlineErrors(r, expected, given, test, target, extraOptions) elif strip(expected.msg) notin strip(given.msg): - r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reMsgsDiffer) elif not nimoutCheck(expected, given): - r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) + r.addResult(test, target, extraOptions, expected.nimout, given.nimout, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, target, expected.file, given.file, reFilesDiffer) + r.addResult(test, target, extraOptions, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0 or expected.column != given.column and expected.column != 0: - r.addResult(test, target, $expected.line & ':' & $expected.column, + r.addResult(test, target, extraOptions, $expected.line & ':' & $expected.column, $given.line & ':' & $given.column, reLinesDiffer) else: - r.addResult(test, target, expected.msg, given.msg, reSuccess) + r.addResult(test, target, extraOptions, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = @@ -434,8 +438,8 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st given.err = reCodeNotFound echo getCurrentExceptionMsg() -proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, - expected: TSpec; r: var TResults) = +proc compilerOutputTests(test: TTest, target: TTarget, extraOptions: string, + given: var TSpec, expected: TSpec; r: var TResults) = var expectedmsg: string = "" var givenmsg: string = "" if given.err == reSuccess: @@ -449,7 +453,7 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, else: givenmsg = "$ " & given.cmd & '\n' & given.nimout if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + r.addResult(test, target, extraOptions, expectedmsg, givenmsg, given.err) proc getTestSpecTarget(): TTarget = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": @@ -457,16 +461,6 @@ proc getTestSpecTarget(): TTarget = else: result = targetC -proc checkDisabled(r: var TResults, test: TTest): bool = - if test.spec.err in {reDisabled, reJoined}: - # targetC is a lie, but parameter is required - r.addResult(test, targetC, "", "", test.spec.err) - inc(r.skipped) - inc(r.total) - result = false - else: - result = true - var count = 0 proc equalModuloLastNewline(a, b: string): bool = @@ -474,8 +468,13 @@ proc equalModuloLastNewline(a, b: string): bool = result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, - target: TTarget, nimcache: string, extraOptions = "") = + target: TTarget, extraOptions: string, nimcache: string) = test.startTime = epochTime() + if test.spec.err in {reDisabled, reJoined}: + r.addResult(test, target, extraOptions, "", "", test.spec.err) + inc(r.skipped) + return + template callNimCompilerImpl(): untyped = # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently # with other branches @@ -483,21 +482,21 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, case expected.action of actionCompile: var given = callNimCompilerImpl() - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionRun: var given = callNimCompilerImpl() if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) + r.addResult(test, target, extraOptions, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) if not fileExists(exeFile): - r.addResult(test, target, expected.output, + r.addResult(test, target, extraOptions, expected.output, "executable not found: " & exeFile, reExeNotFound) else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", + r.addResult(test, target, extraOptions, expected.output, "nodejs binary not in PATH", reExeNotFound) else: var exeCmd: string @@ -528,24 +527,24 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, else: buf if exitCode != expected.exitCode: - r.addResult(test, target, "exitcode: " & $expected.exitCode, + r.addResult(test, target, extraOptions, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or (expected.outputCheck == ocSubstr and expected.output notin bufB): given.err = reOutputsDiffer - r.addResult(test, target, expected.output, bufB, reOutputsDiffer) + r.addResult(test, target, extraOptions, expected.output, bufB, reOutputsDiffer) else: - compilerOutputTests(test, target, given, expected, r) + compilerOutputTests(test, target, extraOptions, given, expected, r) of actionReject: let given = callNimCompilerImpl() - cmpMsgs(r, expected, given, test, target) + cmpMsgs(r, expected, given, test, target, extraOptions) -proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = +proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions: string) = for target in expected.targets: inc(r.total) if target notin gTargets: - r.addResult(test, target, "", "", reDisabled) + r.addResult(test, target, extraOptions, "", "", reDisabled) inc(r.skipped) elif simulate: inc count @@ -553,16 +552,15 @@ proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = else: let nimcache = nimcacheDir(test.name, test.options, target) var testClone = test - testSpecHelper(r, testClone, expected, target, nimcache, extraOptions) + testSpecHelper(r, testClone, expected, target, extraOptions, nimcache) proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: # targetC is a lie, but a parameter is required - r.addResult(test, targetC, "", expected.parseErrors, reInvalidSpec) + r.addResult(test, targetC, "", "", expected.parseErrors, reInvalidSpec) inc(r.total) return - if not checkDisabled(r, test): return expected.targets.incl targets # still no target specified at all @@ -572,14 +570,13 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = for m in test.spec.matrix: targetHelper(r, test, expected, m) else: - targetHelper(r, test, expected) + targetHelper(r, test, expected, "") proc testSpecWithNimcache(r: var TResults, test: TTest; nimcache: string) {.used.} = - if not checkDisabled(r, test): return for target in test.spec.targets: inc(r.total) var testClone = test - testSpecHelper(r, testClone, test.spec, target, nimcache) + testSpecHelper(r, testClone, test.spec, target, "", nimcache) proc makeTest(test, options: string, cat: Category): TTest = result.cat = cat From a8426fc7890f1bea1c507ddb2fe42662f7a574f8 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 23 May 2022 00:12:34 -0400 Subject: [PATCH 0807/1654] Fix default testament target in docs and cli help (#19796) Co-authored-by: quantimnot --- doc/testament.rst | 4 ++-- testament/testament.nim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/testament.rst b/doc/testament.rst index 5590cc6d7..427a7ff71 100644 --- a/doc/testament.rst +++ b/doc/testament.rst @@ -36,7 +36,7 @@ Options (for debugging) --failing Only show failing/ignored tests --targets:"c cpp js objc" - Run tests for specified targets (default: all) + Run tests for specified targets (default: c) --nim:path Use a particular nim executable (default: $PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. @@ -164,7 +164,7 @@ Example "template" **to edit** and write a Testament unittest: # Timeout seconds to run the test. Fractional values are supported. timeout: 1.5 - # Targets to run the test into (c, cpp, objc, js). + # Targets to run the test into (c, cpp, objc, js). Defaults to c. targets: "c js" # flags with which to run the test, delimited by `;` diff --git a/testament/testament.nim b/testament/testament.nim index a4329b982..d2b0e7fb4 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -54,7 +54,7 @@ Options: --verbose print commands (compiling and running tests) --simulate see what tests would be run but don't run them (for debugging) --failing only show failing/ignored tests - --targets:"c cpp js objc" run tests for specified targets (default: all) + --targets:"c cpp js objc" run tests for specified targets (default: c) --nim:path use a particular nim executable (default: $$PATH/nim) --directory:dir Change to directory dir before reading the tests or doing anything else. --colors:on|off Turn messages coloring on|off. From 05c0419658e3f97c29a12a16e99c0f8b842622c8 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 23 May 2022 00:17:32 -0400 Subject: [PATCH 0808/1654] Fix global destructor injection for JS backend (#19797) * Fix global destructor injection for JS backend * Moved global destructors injection before the final call to transform and generate JS code. It had previously been after and thus not no JS was generated for them. * Added some internal documentation of `jsgen`. * Enable a current destructor test to cover the JS backend as well. * Fixes the JS aspect of #17237. * Fixed global destructor injection order for JS backend Co-authored-by: quantimnot --- compiler/jsgen.nim | 30 +++++++++++++++++++++++++----- tests/destructor/t5342.nim | 3 ++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index b9a73daf9..6d34fcae2 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2392,6 +2392,7 @@ proc optionalLine(p: Rope): Rope = return p & "\L" proc genProc(oldProc: PProc, prc: PSym): Rope = + ## Generate a JS procedure ('function'). var resultSym: PSym a: TCompRes @@ -2673,6 +2674,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = else: internalError(p.config, n.info, "gen: unknown node type: " & $n.kind) proc newModule(g: ModuleGraph; module: PSym): BModule = + ## Create a new JS backend module node. new(result) result.module = module result.sigConflicts = initCountTable[SigHash]() @@ -2684,6 +2686,7 @@ proc newModule(g: ModuleGraph; module: PSym): BModule = PGlobals(g.backend).inSystem = true proc genHeader(): Rope = + ## Generate the JS header. result = rope("""/* Generated by the Nim Compiler v$1 */ var framePtr = null; var excHandler = 0; @@ -2714,6 +2717,8 @@ proc addHcrInitGuards(p: PProc, n: PNode, genStmt(p, n) proc genModule(p: PProc, n: PNode) = + ## Generate the JS module code. + ## Called for each top level node in a Nim module. if optStackTrace in p.options: p.body.add(frameCreate(p, makeJSString("module " & p.module.module.name.s), @@ -2742,6 +2747,7 @@ proc genModule(p: PProc, n: PNode) = p.body.add(frameDestroy(p)) proc myProcess(b: PPassContext, n: PNode): PNode = + ## Generate JS code for a node. result = n let m = BModule(b) if passes.skipCodegen(m.config, n): return n @@ -2754,6 +2760,7 @@ proc myProcess(b: PPassContext, n: PNode): PNode = p.g.code.add(p.body) proc wholeCode(graph: ModuleGraph; m: BModule): Rope = + ## Combine source code from all nodes. let globals = PGlobals(graph.backend) for prc in globals.forwarded: if not globals.generatedSyms.containsOrIncl(prc.id): @@ -2779,28 +2786,41 @@ proc getClassName(t: PType): Rope = else: result = rope(s.name.s) proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = - result = myProcess(b, n) + ## Finalize JS code generation of a Nim module. + ## Param `n` may contain nodes returned from the last module close call. var m = BModule(b) if sfMainModule in m.module.flags: - for destructorCall in graph.globalDestructors: - n.add destructorCall + # Add global destructors to the module. + # This must come before the last call to `myProcess`. + for i in countdown(high(graph.globalDestructors), 0): + n.add graph.globalDestructors[i] + # Process any nodes left over from the last call to `myClose`. + result = myProcess(b, n) + # Some codegen is different (such as no stacktraces; see `initProcOptions`) + # when `std/system` is being processed. if sfSystemModule in m.module.flags: PGlobals(graph.backend).inSystem = false + # Check if codegen should continue before any files are generated. + # It may bail early is if too many errors have been raised. if passes.skipCodegen(m.config, n): return n + # Nim modules are compiled into a single JS file. + # If this is the main module, then this is the final call to `myClose`. if sfMainModule in m.module.flags: var code = genHeader() & wholeCode(graph, m) let outFile = m.config.prepareToWriteOutput() - + # Generate an optional source map. if optSourcemap in m.config.globalOptions: var map: SourceMap (code, map) = genSourceMap($(code), outFile.string) writeFile(outFile.string & ".map", $(%map)) + # Check if the generated JS code matches the output file, or else + # write it to the file. if not equalsFile(code, outFile): if not writeRope(code, outFile): rawMessage(m.config, errCannotOpenFile, outFile.string) - proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = + ## Create the JS backend pass context `BModule` for a Nim module. result = newModule(graph, s) result.idgen = idgen diff --git a/tests/destructor/t5342.nim b/tests/destructor/t5342.nim index 19354ea64..0acd5ef9d 100644 --- a/tests/destructor/t5342.nim +++ b/tests/destructor/t5342.nim @@ -1,5 +1,6 @@ discard """ - matrix: "--gc:refc; --gc:arc" + matrix: "--mm:refc; --mm:arc" + targets: "c js" output: ''' 1 2 From b1b5e3ab10749851e942d624ee85188d8b06822e Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 24 May 2022 12:37:19 -0300 Subject: [PATCH 0809/1654] Add Array.shift (#19811) * Add Array.shift for JavaScript targets * Add Array.shift for JavaScript targets --- changelog.md | 2 +- lib/js/jscore.nim | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 3b2685547..e120dcfb8 100644 --- a/changelog.md +++ b/changelog.md @@ -37,7 +37,7 @@ becomes an alias for `addr`. - Added `initDateTime` in `times` to create a datetime from a weekday, and ISO 8601 week number and week-based year. - Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. - +- Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - Removed deprecated `oids.oidToString`. - Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index 61c188431..4518f32ce 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -123,3 +123,12 @@ since (1, 5): assert [1, 2, 3, 4, 5].copyWithin(0, 3) == @[4, 5, 3, 4, 5] assert [1, 2, 3, 4, 5].copyWithin(0, 3, 4) == @[4, 2, 3, 4, 5] assert [1, 2, 3, 4, 5].copyWithin(-2, -3, -1) == @[1, 2, 3, 3, 4] + + +since (1, 7): + func shift*[T](self: seq[T]): T {.importjs: "#.$1()".} = + ## https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift + runnableExamples: + var arrai = @[1, 2, 3] + assert arrai.shift() == 1 + assert arrai == @[2, 3] From ec0cec3170a3f175b9e02ca98e82c963fe64df4f Mon Sep 17 00:00:00 2001 From: tandy1000 Date: Tue, 24 May 2022 16:37:39 +0100 Subject: [PATCH 0810/1654] Add `document.hidden` and `document.visibilityState` properties (#19817) --- lib/js/dom.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/js/dom.nim b/lib/js/dom.nim index c1a677258..1a62780a7 100644 --- a/lib/js/dom.nim +++ b/lib/js/dom.nim @@ -216,11 +216,13 @@ type defaultCharset*: cstring fgColor*: cstring head*: Element + hidden*: bool lastModified*: cstring linkColor*: cstring referrer*: cstring title*: cstring URL*: cstring + visibilityState*: cstring vlinkColor*: cstring anchors*: seq[AnchorElement] forms*: seq[FormElement] From d81edcacc6740a99b0360cf31ddc2f2a1626006c Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 24 May 2022 23:53:01 +0800 Subject: [PATCH 0811/1654] Add link to std/tempfiles in the docs; fix #19155 (#19807) --- doc/lib.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/lib.rst b/doc/lib.rst index f72012777..2f3a315a8 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -251,6 +251,10 @@ Generic Operating System Services This module contains a few procedures to control the *terminal* (also called *console*). The implementation simply uses ANSI escape sequences and does not depend on any other module. + +* `tempfiles `_ + This module provides some utils to generate temporary path names and + create temporary files and directories. Math libraries From 14960fa754f42c820727bed9308e075b2cb923f6 Mon Sep 17 00:00:00 2001 From: Michael New Date: Sun, 29 May 2022 19:40:15 -0700 Subject: [PATCH 0812/1654] Fix typo and incorrect pragma name (#19847) --- doc/manual_experimental_strictnotnil.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.rst index b14e5f9f3..ebfca7e51 100644 --- a/doc/manual_experimental_strictnotnil.rst +++ b/doc/manual_experimental_strictnotnil.rst @@ -172,7 +172,7 @@ We might need to check for `strictFuncs` pure funcs and not do that then. For field expressions `a.field`, we calculate an integer value based on a hash of the tree and just accept equivalent trees as equivalent expressions. For item expression `a[index]`, we also calculate an integer value based on a hash of the tree and accept equivalent trees as equivalent expressions: for static values only. -For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictCheckNotNil]:off}.`. +For now we support only constant indices: we dont track expression with no-const indices. For those we just report a warning even if they are safe for now: one can use a local variable to workaround. For loops this might be annoying: so one should be able to turn off locally the warning using the `{.warning[StrictNotNil]:off.}`. For bracket expressions, in the future we might count `a[]` as the same general expression. This means we should should the index but otherwise handle it the same for assign (maybe "aliasing" all the non-static elements) and differentiate only for static: e.g. `a[0]` and `a[1]`. From 004fc23a3fa79fc32e0fb5d4f438ddc22964b0da Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 30 May 2022 16:41:24 +0800 Subject: [PATCH 0813/1654] [vm] remove unused opcSubstr opcode (#19834) remove unused opcSubstr --- compiler/vm.nim | 8 -------- compiler/vmdef.nim | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index fa1c71c85..bbcff77ba 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1185,14 +1185,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcContainsSet: decodeBC(rkInt) regs[ra].intVal = ord(inSet(regs[rb].node, regs[rc].regToNode)) - of opcSubStr: - decodeBC(rkNode) - inc pc - assert c.code[pc].opcode == opcSubStr - let rd = c.code[pc].regA - createStr regs[ra] - regs[ra].node.strVal = substr(regs[rb].node.strVal, - regs[rc].intVal.int, regs[rd].intVal.int) of opcParseFloat: decodeBC(rkInt) inc pc diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index ecdbeff89..c653501ca 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -102,7 +102,7 @@ type opcMulSet, opcPlusSet, opcMinusSet, opcConcatStr, opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq, opcIsNil, opcOf, opcIs, - opcSubStr, opcParseFloat, opcConv, opcCast, + opcParseFloat, opcConv, opcCast, opcQuit, opcInvalidField, opcNarrowS, opcNarrowU, opcSignExtend, @@ -306,7 +306,7 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.disca const firstABxInstr* = opcTJmp largeInstrs* = { # instructions which use 2 int32s instead of 1: - opcSubStr, opcConv, opcCast, opcNewSeq, opcOf + opcConv, opcCast, opcNewSeq, opcOf } slotSomeTemp* = slotTempUnknown relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack} From 497af2c0d9f311e050ba57842a6326ca70489d55 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 30 May 2022 05:07:47 -0400 Subject: [PATCH 0814/1654] Doc preferred import style in compiler (#19832) Doc prefered import style in compiler Co-authored-by: quantimnot --- doc/intern.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/intern.rst b/doc/intern.rst index a8846ae02..4ca7eff20 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -186,6 +186,7 @@ Coding Guidelines * Use a space after a colon, but not before it. * [deprecated] Start types with a capital `T`, unless they are pointers/references which start with `P`. +* Prefer `import package`:nim: over `from package import symbol`:nim:. See also the `API naming design `_ document. From 15f0b4867679120580b2f14bbb7a8b302505b34d Mon Sep 17 00:00:00 2001 From: Alfred Morgan Date: Mon, 30 May 2022 03:09:18 -0700 Subject: [PATCH 0815/1654] Zectbumo fixes 19824 (#19825) * borrowed `$` to make Time string friendly * added sep character parameter * Revert "added sep character parameter" This reverts commit 45f4b019a4883b6ba577ade1f94677266beb5960. * added sep character parameter * Revert "borrowed `$` to make Time string friendly" This reverts commit 10e2e44c9a04970f38cf66556635bdbb50b69136. * added uri tests and made changelong entry * Update lib/pure/uri.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update lib/pure/uri.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update tests/stdlib/turi.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> * Update tests/stdlib/turi.nim Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 1 + lib/pure/uri.nim | 14 ++++++++------ tests/stdlib/turi.nim | 4 ++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index e120dcfb8..9548f3034 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,7 @@ becomes an alias for `addr`. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. - Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. +- Added `sep` parameter in `std/uri` to specify the query separator. - Removed deprecated `oids.oidToString`. - Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index 7583dbd1e..1dbb018c2 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -123,13 +123,13 @@ func decodeUrl*(s: string, decodePlus = true): string = setLen(result, j) func encodeQuery*(query: openArray[(string, string)], usePlus = true, - omitEq = true): string = + omitEq = true, sep = '&'): string = ## Encodes a set of (key, value) parameters into a URL query string. ## ## Every (key, value) pair is URL-encoded and written as `key=value`. If the ## value is an empty string then the `=` is omitted, unless `omitEq` is ## false. - ## The pairs are joined together by a `&` character. + ## The pairs are joined together by the `sep` character. ## ## The `usePlus` parameter is passed down to the `encodeUrl` function that ## is used for the URL encoding of the string values. @@ -140,9 +140,10 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, assert encodeQuery({: }) == "" assert encodeQuery({"a": "1", "b": "2"}) == "a=1&b=2" assert encodeQuery({"a": "1", "b": ""}) == "a=1&b" + assert encodeQuery({"a": "1", "b": ""}, omitEq = false, sep = ';') == "a=1;b=" for elem in query: - # Encode the `key = value` pairs and separate them with a '&' - if result.len > 0: result.add('&') + # Encode the `key = value` pairs and separate them with 'sep' + if result.len > 0: result.add(sep) let (key, val) = elem result.add(encodeUrl(key, usePlus)) # Omit the '=' if the value string is empty @@ -150,7 +151,7 @@ func encodeQuery*(query: openArray[(string, string)], usePlus = true, result.add('=') result.add(encodeUrl(val, usePlus)) -iterator decodeQuery*(data: string): tuple[key, value: string] = +iterator decodeQuery*(data: string, sep = '&'): tuple[key, value: string] = ## Reads and decodes the query string `data` and yields the `(key, value)` pairs ## the data consists of. If compiled with `-d:nimLegacyParseQueryStrict`, ## a `UriParseError` is raised when there is an unencoded `=` character in a decoded @@ -158,6 +159,7 @@ iterator decodeQuery*(data: string): tuple[key, value: string] = runnableExamples: import std/sequtils assert toSeq(decodeQuery("foo=1&bar=2=3")) == @[("foo", "1"), ("bar", "2=3")] + assert toSeq(decodeQuery("foo=1;bar=2=3", ';')) == @[("foo", "1"), ("bar", "2=3")] assert toSeq(decodeQuery("&a&=b&=&&")) == @[("", ""), ("a", ""), ("", "b"), ("", ""), ("", "")] proc parseData(data: string, i: int, field: var string, sep: char): int = @@ -186,7 +188,7 @@ iterator decodeQuery*(data: string): tuple[key, value: string] = when defined(nimLegacyParseQueryStrict): i = parseData(data, i, value, '=') else: - i = parseData(data, i, value, '&') + i = parseData(data, i, value, sep) yield (name, value) if i < data.len: when defined(nimLegacyParseQueryStrict): diff --git a/tests/stdlib/turi.nim b/tests/stdlib/turi.nim index a3b6afe2c..79ddd773b 100644 --- a/tests/stdlib/turi.nim +++ b/tests/stdlib/turi.nim @@ -274,7 +274,9 @@ template main() = doAssert encodeQuery({"foo": ""}) == "foo" doAssert encodeQuery({"foo": ""}, omitEq = false) == "foo=" doAssert encodeQuery({"a": "1", "b": "", "c": "3"}) == "a=1&b&c=3" + doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, sep = ';') == "a=1;b;c=3" doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, omitEq = false) == "a=1&b=&c=3" + doAssert encodeQuery({"a": "1", "b": "", "c": "3"}, omitEq = false, sep = ';') == "a=1;b=;c=3" block: # `?` block: @@ -300,7 +302,9 @@ template main() = block: # decodeQuery doAssert toSeq(decodeQuery("a=1&b=0")) == @[("a", "1"), ("b", "0")] + doAssert toSeq(decodeQuery("a=1;b=0", sep = ';')) == @[("a", "1"), ("b", "0")] doAssert toSeq(decodeQuery("a=1&b=2c=6")) == @[("a", "1"), ("b", "2c=6")] + doAssert toSeq(decodeQuery("a=1;b=2c=6", sep = ';')) == @[("a", "1"), ("b", "2c=6")] block: # bug #17481 let u1 = parseUri("./") From d30c6419a051a815e3fdb354ac79522f17e55bda Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Mon, 30 May 2022 12:52:19 -0400 Subject: [PATCH 0816/1654] Refactor and doc package handling, module name mangling (#19821) * Refactor and doc package handling, module name mangling * Consolidate, de-duplicate and extend package handling * Alter how duplicate module names of a package are handled * Alter how module names are mangled * Fix crash when another package is named 'stdlib' (test case added) * Doc what defines a package in the manual Modules with duplicate names within a package used to be given 'fake' packages to resolve conflicts. That prevented the ability to discern if a module belonged to the current project package or a foreign package. They now have the proper package owner and the names are mangled in a consistent manner to prevent codegen clashes. All module names are now mangled the same. Stdlib was treated special before, but now it is same as any other package. This fixes a crash when a foreign package is named 'stdlib'. Module mangling is altered for both file paths and symbols used by the backends. Removed an unused module name to package mapping that may have been intended for IC. The mapping was removed because it wasn't being used and was complicating the issue of package modules with duplicate names not having the proper package owner assigned. * Fix some tests * Refactor `packagehandling` * Remove `packagehandling.withPackageName` and its uses * Move module path mangling from `packagehandling` to `modulepaths` * Move `options.toRodFile` to `ic` to break import cycle * Changed import style to match preferred style Co-authored-by: quantimnot --- compiler/ast.nim | 21 ---- compiler/cgen.nim | 27 ++--- compiler/docgen.nim | 13 +-- compiler/docgen2.nim | 4 +- compiler/extccomp.nim | 11 ++- compiler/ic/cbackend.nim | 5 +- compiler/ic/ic.nim | 19 ++-- compiler/modulegraphs.nim | 18 +++- compiler/modulepaths.nim | 109 +++------------------ compiler/modules.nim | 51 +--------- compiler/options.nim | 7 +- compiler/packagehandling.nim | 23 +---- compiler/packages.nim | 49 +++++++++ compiler/passes.nim | 12 +-- compiler/pragmas.nim | 3 +- doc/manual.rst | 13 +++ tests/ccgbugs/tforward_decl_only.nim | 2 +- tests/modules/a/module_name_clashes.nim | 8 ++ tests/modules/b/module_name_clashes.nim | 3 + tests/modules/tmodule_name_clashes.nim | 16 +++ tests/package/stdlib/stdlib.nimble | 0 tests/package/stdlib/system.nim | 2 + tests/package/tstdlib_name_not_special.nim | 3 + 23 files changed, 175 insertions(+), 244 deletions(-) create mode 100644 compiler/packages.nim create mode 100644 tests/modules/a/module_name_clashes.nim create mode 100644 tests/modules/b/module_name_clashes.nim create mode 100644 tests/modules/tmodule_name_clashes.nim create mode 100644 tests/package/stdlib/stdlib.nimble create mode 100644 tests/package/stdlib/system.nim create mode 100644 tests/package/tstdlib_name_not_special.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index d1e5ae2bf..f8343c1a3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1108,21 +1108,6 @@ proc getPIdent*(a: PNode): PIdent {.inline.} = of nkIdent: a.ident else: nil -proc getnimblePkg*(a: PSym): PSym = - result = a - while result != nil: - case result.kind - of skModule: - result = result.owner - assert result.kind == skPackage - of skPackage: - if result.owner == nil: - break - else: - result = result.owner - else: - assert false, $result.kind - const moduleShift = when defined(cpu32): 20 else: 24 @@ -1167,13 +1152,7 @@ when false: assert dest.ItemId.item <= src.ItemId.item dest = src -proc getnimblePkgId*(a: PSym): int = - let b = a.getnimblePkg - result = if b == nil: -1 else: b.id - var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things -#var -# gMainPackageId*: int proc isCallExpr*(n: PNode): bool = result = n.kind in nkCallKinds diff --git a/compiler/cgen.nim b/compiler/cgen.nim index ad59b759f..8d24f30c5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -15,8 +15,7 @@ import ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, rodutils, renderer, cgendata, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf, - injectdestructors, astmsgs - + injectdestructors, astmsgs, modulepaths when defined(nimPreviewSlimSystem): import std/assertions @@ -1306,17 +1305,19 @@ proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope = if conf.hcrOn: result.add("#define NIM_HOT_CODE_RELOADING\L") addNimDefines(result, conf) -proc getSomeNameForModule(m: PSym): Rope = - assert m.kind == skModule - assert m.owner.kind == skPackage - if {sfSystemModule, sfMainModule} * m.flags == {}: - result = m.owner.name.s.mangle.rope - result.add "_" - result.add m.name.s.mangle +proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope = + ## Returns a mangled module name. + result.add mangleModuleName(conf, filename).mangle + +proc getSomeNameForModule(m: BModule): Rope = + ## Returns a mangled module name. + assert m.module.kind == skModule + assert m.module.owner.kind == skPackage + result.add mangleModuleName(m.g.config, m.filename).mangle proc getSomeInitName(m: BModule, suffix: string): Rope = if not m.hcrOn: - result = getSomeNameForModule(m.module) + result = getSomeNameForModule(m) result.add suffix proc getInitName(m: BModule): Rope = @@ -1555,11 +1556,11 @@ proc genMainProc(m: BModule) = proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) = ## Called from the IC backend. if HasDatInitProc in flags: - let datInit = getSomeNameForModule(m) & "DatInit000" + let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000" g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) g.mainDatInit.addf("\t$1();$N", [datInit]) if HasModuleInitProc in flags: - let init = getSomeNameForModule(m) & "Init000" + let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000" g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.flags: @@ -1940,7 +1941,7 @@ proc getCFile(m: BModule): AbsoluteFile = if m.compileToCpp: ".nim.cpp" elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m" else: ".nim.c" - result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext) + result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext) when false: proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = diff --git a/compiler/docgen.nim b/compiler/docgen.nim index ecf98d0b6..f633a57a0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -16,7 +16,7 @@ import packages/docutils/[rst, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, tables, nimpaths, renderverbatim, osproc + pathutils, tables, nimpaths, renderverbatim, osproc, packages import packages/docutils/rstast except FileIndex, TLineInfo from uri import encodeUrl @@ -413,9 +413,6 @@ proc getPlainDocstring(n: PNode): string = result = getPlainDocstring(n[i]) if result.len > 0: return -proc belongsToPackage(conf: ConfigRef; module: PSym): bool = - result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId - proc externalDep(d: PDoc; module: PSym): string = if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0: let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position) @@ -471,7 +468,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var string; "\\spanIdentifier{$1}", [escLit, procLink]) elif s != nil and s.kind in {skType, skVar, skLet, skConst} and sfExported in s.flags and s.owner != nil and - belongsToPackage(d.conf, s.owner) and d.target == outHtml: + belongsToProjectPackage(d.conf, s.owner) and d.target == outHtml: let external = externalDep(d, s.owner) result.addf "$3", [changeFileExt(external, "html"), literal, @@ -1131,7 +1128,7 @@ proc traceDeps(d: PDoc, it: PNode) = for x in it[2]: a[2] = x traceDeps(d, a) - elif it.kind == nkSym and belongsToPackage(d.conf, it.sym): + elif it.kind == nkSym and belongsToProjectPackage(d.conf, it.sym): let external = externalDep(d, it.sym) if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") dispA(d.conf, d.section[k].finalMarkup, @@ -1141,7 +1138,7 @@ proc traceDeps(d: PDoc, it: PNode) = proc exportSym(d: PDoc; s: PSym) = const k = exportSection - if s.kind == skModule and belongsToPackage(d.conf, s): + if s.kind == skModule and belongsToProjectPackage(d.conf, s): let external = externalDep(d, s) if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ") dispA(d.conf, d.section[k].finalMarkup, @@ -1150,7 +1147,7 @@ proc exportSym(d: PDoc; s: PSym) = changeFileExt(external, "html")]) elif s.kind != skModule and s.owner != nil: let module = originatingModule(s) - if belongsToPackage(d.conf, module): + if belongsToProjectPackage(d.conf, module): let complexSymbol = complexName(s.kind, s.ast, s.name.s) symbolOrId = d.newUniquePlainSymbol(complexSymbol) diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index bfdb4568c..9abde9f52 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -11,7 +11,7 @@ # semantic checking. import - options, ast, msgs, passes, docgen, lineinfos, pathutils + options, ast, msgs, passes, docgen, lineinfos, pathutils, packages from modulegraphs import ModuleGraph, PPassContext @@ -23,7 +23,7 @@ type PGen = ref TGen proc shouldProcess(g: PGen): bool = - (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or + (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex template closeImpl(body: untyped) {.dirty.} = diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 09a1dfbb3..515c7ba29 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -12,7 +12,7 @@ # from a lineinfos file, to provide generalized procedures to compile # nim files. -import ropes, platform, condsyms, options, msgs, lineinfos, pathutils +import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar] @@ -370,6 +370,7 @@ proc initVars*(conf: ConfigRef) = proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile, createSubDir: bool = true): AbsoluteFile = + ## Generate the absolute file path to the generated modules. result = completeGeneratedFilePath(conf, cfile, createSubDir) proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile = @@ -380,7 +381,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) = conf.toCompile.add(cf) proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) = - let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string + let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string var value = conf.cfileSpecificOptions.getOrDefault(key) if strutils.find(value, option, 0) < 0: addOpt(value, option) @@ -637,7 +638,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash = proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good - let hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1") + let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1") let currentHash = footprint(conf, cfile) var f: File if open(f, hashFile.string, fmRead): @@ -845,9 +846,9 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = if conf.hasHint(hintCC): if optListCmd in conf.globalOptions or conf.verbosity > 1: - result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd) + result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd) else: - result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name) + result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name) proc callCCompiler*(conf: ConfigRef) = var diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index e7ab000e6..815078a36 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -24,7 +24,7 @@ when defined(nimPreviewSlimSystem): import std/assertions import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, - pathutils, extccomp, msgs] + pathutils, extccomp, msgs, modulepaths] import packed_ast, ic, dce, rodfiles @@ -64,7 +64,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) = if config.backend == backendCpp: ".nim.cpp" elif config.backend == backendObjc: ".nim.m" else: ".nim.c" - let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext) + let cfile = changeFileExt(completeCfilePath(config, + mangleModuleName(config, filename).AbsoluteFile), ext) let objFile = completeCfilePath(config, toObjFile(config, cfile)) if fileExists(objFile): var cf = Cfile(nimname: m.name.s, cname: cfile, diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim index 193e5f517..38b6987f9 100644 --- a/compiler/ic/ic.nim +++ b/compiler/ic/ic.nim @@ -10,7 +10,7 @@ import hashes, tables, intsets, std/sha1 import packed_ast, bitabs, rodfiles import ".." / [ast, idents, lineinfos, msgs, ropes, options, - pathutils, condsyms] + pathutils, condsyms, packages, modulepaths] #import ".." / [renderer, astalgo] from os import removeFile, isAbsolute @@ -551,6 +551,10 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) = rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err) #echo "Error: ", $err, " loading file: ", filename.string +proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile = + result = changeFileExt(completeGeneratedFilePath(conf, + mangleModuleName(conf, f).AbsoluteFile), ext) + proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef; ignoreConfig = false): RodFileError = var f = rodfiles.open(filename.string) @@ -930,17 +934,6 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t result = g[si].types[t.item] assert result.itemId.item > 0 -proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym = - let filename = AbsoluteFile toFullPath(config, fileIdx) - let name = getIdent(cache, splitFile(filename).name) - let info = newLineInfo(fileIdx, 1, 1) - let - pck = getPackageName(config, filename.string) - pck2 = if pck.len > 0: pck else: "unknown" - pack = getIdent(cache, pck2) - result = newSym(skPackage, getIdent(cache, pck2), - ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info) - proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex; m: var LoadedModule) = m.iface = initTable[PIdent, seq[PackedItemId]]() @@ -968,7 +961,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa name: getIdent(cache, splitFile(filename).name), info: newLineInfo(fileIdx, 1, 1), position: int(fileIdx)) - m.module.owner = newPackage(conf, cache, fileIdx) + m.module.owner = getPackage(conf, cache, fileIdx) m.module.flags = m.fromDisk.moduleFlags proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index c44908dc3..147381910 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -12,7 +12,7 @@ ## or stored in a rod-file. import intsets, tables, hashes, md5_old -import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils +import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages import ic / [packed_ast, ic] when defined(nimPreviewSlimSystem): @@ -67,7 +67,6 @@ type startupPackedConfig*: PackedConfig packageSyms*: TStrTable - modulesPerPackage*: Table[ItemId, TStrTable] deps*: IntSet # the dependency graph or potentially its transitive closure. importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. @@ -597,3 +596,18 @@ proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)" let mode = if isNimscript: "(nims) " else: "" rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path]) + +proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym = + ## Returns a package symbol for yet to be defined module for fileIdx. + ## The package symbol is added to the graph if it doesn't exist. + let pkgSym = getPackage(graph.config, graph.cache, fileIdx) + # check if the package is already in the graph + result = graph.packageSyms.strTableGet(pkgSym.name) + if result == nil: + # the package isn't in the graph, so create and add it + result = pkgSym + graph.packageSyms.strTableAdd(pkgSym) + +func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool = + ## Check if symbol belongs to the 'stdlib' package. + sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim index a16b669c4..e80ea3fa6 100644 --- a/compiler/modulepaths.nim +++ b/compiler/modulepaths.nim @@ -10,100 +10,6 @@ import ast, renderer, strutils, msgs, options, idents, os, lineinfos, pathutils -when false: - const - considerParentDirs = not defined(noParentProjects) - considerNimbleDirs = not defined(noNimbleDirs) - - proc findInNimbleDir(pkg, subdir, dir: string): string = - var best = "" - var bestv = "" - for k, p in os.walkDir(dir, relative=true): - if k == pcDir and p.len > pkg.len+1 and - p[pkg.len] == '-' and p.startsWith(pkg): - let (_, a, _) = getPathVersionChecksum(p) - if bestv.len == 0 or bestv < a: - bestv = a - best = dir / p - - if best.len > 0: - var f: File - if open(f, best / changeFileExt(pkg, ".nimble-link")): - # the second line contains what we're interested in, see: - # https://github.com/nim-lang/nimble#nimble-link - var override = "" - discard readLine(f, override) - discard readLine(f, override) - close(f) - if not override.isAbsolute(): - best = best / override - else: - best = override - let f = if subdir.len == 0: pkg else: subdir - let res = addFileExt(best / f, "nim") - if best.len > 0 and fileExists(res): - result = res - -when false: - proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string = - template attempt(a) = - let x = addFileExt(a, "nim") - if fileExists(x): return x - - case pkg - of "stdlib": - if subdir.len == 0: - return options.libpath - else: - for candidate in stdlibDirs: - attempt(options.libpath / candidate / subdir) - of "root": - let root = project.splitFile.dir - if subdir.len == 0: - return root - else: - attempt(root / subdir) - else: - when considerParentDirs: - var p = parentDir(source.splitFile.dir) - # support 'import $karax': - let f = if subdir.len == 0: pkg else: subdir - - while p.len > 0: - let dir = p / pkg - if dirExists(dir): - attempt(dir / f) - # 2nd attempt: try to use 'karax/karax' - attempt(dir / pkg / f) - # 3rd attempt: try to use 'karax/src/karax' - attempt(dir / "src" / f) - attempt(dir / "src" / pkg / f) - p = parentDir(p) - - when considerNimbleDirs: - if not options.gNoNimblePath: - var nimbleDir = getEnv("NIMBLE_DIR") - if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble" - result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs") - if result.len > 0: return result - when not defined(windows): - result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs") - if result.len > 0: return result - - proc scriptableImport(pkg, sub: string; info: TLineInfo): string = - resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info) - - proc lookupPackage(pkg, subdir: PNode): string = - let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: "" - case pkg.kind - of nkStrLit, nkRStrLit, nkTripleStrLit: - result = scriptableImport(pkg.strVal, sub, pkg.info) - of nkIdent: - result = scriptableImport(pkg.ident.s, sub, pkg.info) - else: - localError(pkg.info, "package name must be an identifier or string literal") - result = "" - proc getModuleName*(conf: ConfigRef; n: PNode): string = # This returns a short relative module name without the nim extension # e.g. like "system", "importer" or "somepath/module" @@ -163,3 +69,18 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex = result = InvalidFileIdx else: result = fileInfoIdx(conf, fullPath) + +proc mangleModuleName*(conf: ConfigRef; path: AbsoluteFile): string = + ## Mangle a relative module path to avoid path and symbol collisions. + ## + ## Used by backends that need to generate intermediary files from Nim modules. + ## This is needed because the compiler uses a flat cache file hierarchy. + ## + ## Example: + ## `foo-#head/../bar` becomes `@foo-@hhead@s..@sbar` + "@m" & relativeTo(path, conf.projectPath).string.multiReplace( + {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) + +proc demangleModuleName*(path: string): string = + ## Demangle a relative module path. + result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"}) diff --git a/compiler/modules.nim b/compiler/modules.nim index dd5db63fa..2becef38f 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -12,7 +12,7 @@ import ast, astalgo, magicsys, msgs, options, idents, lexer, passes, syntaxes, llstream, modulegraphs, - lineinfos, pathutils, tables + lineinfos, pathutils, tables, packages when defined(nimPreviewSlimSystem): import std/[syncio, assertions] @@ -25,56 +25,11 @@ proc resetSystemArtifacts*(g: ModuleGraph) = template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent = getIdent(graph.cache, splitFile(filename).name) -template packageId(): untyped {.dirty.} = ItemId(module: PackageModuleId, item: int32(fileIdx)) - -proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym = - ## returns package symbol (skPackage) for yet to be defined module for fileIdx - let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - let name = getModuleIdent(graph, filename) - let info = newLineInfo(fileIdx, 1, 1) - let - pck = getPackageName(graph.config, filename.string) - pck2 = if pck.len > 0: pck else: "unknown" - pack = getIdent(graph.cache, pck2) - result = graph.packageSyms.strTableGet(pack) - if result == nil: - result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info) - #initStrTable(packSym.tab) - graph.packageSyms.strTableAdd(result) - else: - let modules = graph.modulesPerPackage.getOrDefault(result.itemId) - let existing = if modules.data.len > 0: strTableGet(modules, name) else: nil - if existing != nil and existing.info.fileIndex != info.fileIndex: - when false: - # we used to produce an error: - localError(graph.config, info, - "module names need to be unique per Nimble package; module clashes with " & - toFullPath(graph.config, existing.info.fileIndex)) - else: - # but starting with version 0.20 we now produce a fake Nimble package instead - # to resolve the conflicts: - let pck3 = fakePackageName(graph.config, filename) - # this makes the new `result`'s owner be the original `result` - result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info) - #initStrTable(packSym.tab) - graph.packageSyms.strTableAdd(result) - proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = let packSym = getPackage(graph, fileIdx) result.owner = packSym result.position = int fileIdx - #initStrTable(result.tab(graph)) - when false: - strTableAdd(result.tab, result) # a module knows itself - # This is now implemented via - # c.moduleScope.addSym(module) # a module knows itself - # in sem.nim, around line 527 - - if graph.modulesPerPackage.getOrDefault(packSym.itemId).data.len == 0: - graph.modulesPerPackage[packSym.itemId] = newStrTable() - graph.modulesPerPackage[packSym.itemId].strTableAdd(result) - proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = let filename = AbsoluteFile toFullPath(graph.config, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID @@ -136,7 +91,7 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: graph.config.notes = - if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes + if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes else: graph.config.foreignPackageNotes proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode = @@ -171,7 +126,7 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = conf.projectMainIdx2 = projectFile let packSym = getPackage(graph, projectFile) - graph.config.mainPackageId = packSym.getnimblePkgId + graph.config.mainPackageId = packSym.getPackageId graph.importStack.add projectFile if projectFile == systemFileIdx: diff --git a/compiler/options.nim b/compiler/options.nim index 69eafb67f..545dfb1d1 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -807,6 +807,8 @@ proc toGeneratedFile*(conf: ConfigRef; path: AbsoluteFile, proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile, createSubDir: bool = true): AbsoluteFile = + ## Return an absolute path of a generated intermediary file. + ## Optionally creates the cache directory if `createSubDir` is `true`. let subdir = getNimcacheDir(conf) if createSubDir: try: @@ -814,11 +816,6 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile, except OSError: conf.quitOrRaise "cannot create directory: " & subdir.string result = subdir / RelativeFile f.string.splitPath.tail - #echo "completeGeneratedFilePath(", f, ") = ", result - -proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile = - result = changeFileExt(completeGeneratedFilePath(conf, - withPackageName(conf, f)), ext) proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile = for it in conf.searchPaths: diff --git a/compiler/packagehandling.nim b/compiler/packagehandling.nim index 4af0c28fa..8cf209779 100644 --- a/compiler/packagehandling.nim +++ b/compiler/packagehandling.nim @@ -37,24 +37,7 @@ proc getNimbleFile*(conf: ConfigRef; path: string): string = proc getPackageName*(conf: ConfigRef; path: string): string = ## returns nimble package name, e.g.: `cligen` let path = getNimbleFile(conf, path) - result = path.splitFile.name - -proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string = - # Convert `path` so that 2 modules with same name - # in different directory get different name and they can be - # placed in a directory. - # foo-#head/../bar becomes @foo-@hhead@s..@sbar - result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace( - {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) - -proc demanglePackageName*(path: string): string = - result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"}) - -proc withPackageName*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile = - let x = getPackageName(conf, path.string) - let (p, file, ext) = path.splitFile - if x == "stdlib": - # Hot code reloading now relies on 'stdlib_system' names etc. - result = p / RelativeFile((x & '_' & file) & ext) + if path.len > 0: + return path.splitFile.name else: - result = p / RelativeFile(fakePackageName(conf, path)) + return "unknown" diff --git a/compiler/packages.nim b/compiler/packages.nim new file mode 100644 index 000000000..6ceeb1ccc --- /dev/null +++ b/compiler/packages.nim @@ -0,0 +1,49 @@ +# +# +# The Nim Compiler +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Package related procs. +## +## See Also: +## * `packagehandling` for package path handling +## * `modulegraphs.getPackage` +## * `modulegraphs.belongsToStdlib` + +import "." / [options, ast, lineinfos, idents, pathutils, msgs] + +proc getPackage*(conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym = + ## Return a new package symbol. + ## + ## See Also: + ## * `modulegraphs.getPackage` + let + filename = AbsoluteFile toFullPath(conf, fileIdx) + name = getIdent(cache, splitFile(filename).name) + info = newLineInfo(fileIdx, 1, 1) + pkgName = getPackageName(conf, filename.string) + pkgIdent = getIdent(cache, pkgName) + newSym(skPackage, pkgIdent, ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info) + +func getPackageSymbol*(sym: PSym): PSym = + ## Return the owning package symbol. + assert sym != nil + result = sym + while result.kind != skPackage: + result = result.owner + assert result != nil, repr(sym.info) + +func getPackageId*(sym: PSym): int = + ## Return the owning package ID. + sym.getPackageSymbol.id + +func belongsToProjectPackage*(conf: ConfigRef, sym: PSym): bool = + ## Return whether the symbol belongs to the project's package. + ## + ## See Also: + ## * `modulegraphs.belongsToStdlib` + conf.mainPackageId == sym.getPackageId diff --git a/compiler/passes.nim b/compiler/passes.nim index 7fb2842f5..3de27575b 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -14,7 +14,7 @@ import options, ast, llstream, msgs, idents, syntaxes, modulegraphs, reorder, - lineinfos, pathutils + lineinfos, pathutils, packages when defined(nimPreviewSlimSystem): import std/syncio @@ -104,7 +104,7 @@ const proc prepareConfigNotes(graph: ModuleGraph; module: PSym) = # don't be verbose unless the module belongs to the main package: - if module.getnimblePkgId == graph.config.mainPackageId: + if graph.config.belongsToProjectPackage(module): graph.config.notes = graph.config.mainPackageNotes else: if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes @@ -114,12 +114,6 @@ proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = result = true #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") -proc partOfStdlib(x: PSym): bool = - var it = x.owner - while it != nil and it.kind == skPackage and it.owner != nil: - it = it.owner - result = it != nil and it.name.s == "stdlib" - proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; stream: PLLStream): bool {.discardable.} = if graph.stopCompile(): return true @@ -141,7 +135,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; while true: openParser(p, fileIdx, s, graph.cache, graph.config) - if not partOfStdlib(module) or module.name.s == "distros": + if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"): # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 458c7547a..2262e441b 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -641,12 +641,13 @@ proc pragmaLine(c: PContext, n: PNode) = n.info = getInfoContext(c.config, -1) proc processPragma(c: PContext, n: PNode, i: int) = + ## Create and add a new custom pragma `{.pragma: name.}` node to the module's context. let it = n[i] if it.kind notin nkPragmaCallKinds and it.safeLen == 2: invalidPragma(c, n) elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent: invalidPragma(c, n) - var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), nil, it.info, c.config.options) + var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), c.module, it.info, c.config.options) userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1]) strTableAdd(c.userPragmas, userPragma) diff --git a/doc/manual.rst b/doc/manual.rst index bfcb41b7a..5ab257fec 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -6551,6 +6551,19 @@ iterator in which case the overloading resolution takes place: write(stdout, x) # not ambiguous: uses the module C's x +Packages +-------- +A collection of modules in a file tree with an ``identifier.nimble`` file in the +root of the tree is called a Nimble package. A valid package name can only be a +valid Nim identifier and thus its filename is ``identifier.nimble`` where +``identifier`` is the desired package name. A module without a ``.nimble`` file +is assigned the package identifier: `unknown`. + +The distinction between packages allows diagnostic compiler messages to be +scoped to the current project's package vs foreign packages. + + + Compiler Messages ================= diff --git a/tests/ccgbugs/tforward_decl_only.nim b/tests/ccgbugs/tforward_decl_only.nim index 74fbae303..416e50eb5 100644 --- a/tests/ccgbugs/tforward_decl_only.nim +++ b/tests/ccgbugs/tforward_decl_only.nim @@ -1,7 +1,7 @@ discard """ ccodecheck: "\\i !@('struct tyObject_MyRefObject'[0-z]+' {')" ccodecheck: "\\i !@('mymoduleInit')" -ccodecheck: "\\i @('mymoduleDatInit')" +ccodecheck: "\\i @('atmmymoduledotnim_DatInit000')" output: "hello" """ diff --git a/tests/modules/a/module_name_clashes.nim b/tests/modules/a/module_name_clashes.nim new file mode 100644 index 000000000..209526e22 --- /dev/null +++ b/tests/modules/a/module_name_clashes.nim @@ -0,0 +1,8 @@ +# See `tmodule_name_clashes` + +import ../b/module_name_clashes +type A* = object + b*: B + +proc print*(a: A) = + echo repr a diff --git a/tests/modules/b/module_name_clashes.nim b/tests/modules/b/module_name_clashes.nim new file mode 100644 index 000000000..6a10cac33 --- /dev/null +++ b/tests/modules/b/module_name_clashes.nim @@ -0,0 +1,3 @@ +# See `tmodule_name_clashes` + +type B* = object diff --git a/tests/modules/tmodule_name_clashes.nim b/tests/modules/tmodule_name_clashes.nim new file mode 100644 index 000000000..73b166c77 --- /dev/null +++ b/tests/modules/tmodule_name_clashes.nim @@ -0,0 +1,16 @@ +discard """ +targets: "c" +ccodecheck: "\\i @('atmaatsmodule_name_clashesdotnim_DatInit000')" +ccodecheck: "\\i @('atmbatsmodule_name_clashesdotnim_DatInit000')" +joinable: false +""" + +# Test module name clashes within same package. +# This was created to test that module symbol mangling functioned correctly +# for the C backend when there are one or more modules with the same name in +# a package, and more than one of them require module initialization procs. +# I'm not sure of the simplest method to cause the init procs to be generated. + +import a/module_name_clashes + +print A() diff --git a/tests/package/stdlib/stdlib.nimble b/tests/package/stdlib/stdlib.nimble new file mode 100644 index 000000000..e69de29bb diff --git a/tests/package/stdlib/system.nim b/tests/package/stdlib/system.nim new file mode 100644 index 000000000..475f8ec5b --- /dev/null +++ b/tests/package/stdlib/system.nim @@ -0,0 +1,2 @@ +# this module is part of tstdlib_name_not_special +doAssert true \ No newline at end of file diff --git a/tests/package/tstdlib_name_not_special.nim b/tests/package/tstdlib_name_not_special.nim new file mode 100644 index 000000000..e8226a82d --- /dev/null +++ b/tests/package/tstdlib_name_not_special.nim @@ -0,0 +1,3 @@ +# Test whether a another package named 'stdlib' can be imported and used. +# This caused a crash in the past. +import stdlib/system \ No newline at end of file From 4e3eb7414cfedad3d29b2eeab451de4206d91df4 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 1 Jun 2022 20:44:26 +0800 Subject: [PATCH 0817/1654] [Minor] remove unused and unnecessary local variable (#19853) --- lib/system/reprjs.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/system/reprjs.nim b/lib/system/reprjs.nim index 28935a4ef..f5a0ed3ea 100644 --- a/lib/system/reprjs.nim +++ b/lib/system/reprjs.nim @@ -115,7 +115,6 @@ proc reprArray(a: pointer, typ: PNimType, # We prepend @ to seq, the C backend prepends the pointer to the seq. result = if typ.kind == tySequence: "@[" else: "[" var len: int = 0 - var i: int = 0 {. emit: "`len` = `a`.length;\n" .} var dereffed: pointer = a From 68aeb4c1a64ce73f38a471372277f5ec788f5a6e Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 1 Jun 2022 21:01:17 +0800 Subject: [PATCH 0818/1654] change the type of mangleJsName; fixes CStringConv warnings (#19852) change the type of mangleJsName since mangleJsName is used in macros, there is no need to use cstring. Using cstring may increase conversions and cause warnings. --- lib/js/jsffi.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index ac963eb89..35cbf2864 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -64,7 +64,7 @@ proc validJsName(name: string): bool = if chr notin {'A'..'Z','a'..'z','_','$','0'..'9'}: return false -template mangleJsName(name: cstring): cstring = +template mangleJsName(name: string): string = inc nameCounter "mangledName" & $nameCounter @@ -233,7 +233,7 @@ macro `.`*(obj: JsObject, field: untyped): JsObject = helper(`obj`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) let importString = "#." & mangledNames[$field] result = quote do: proc helper(o: JsObject): JsObject @@ -251,7 +251,7 @@ macro `.=`*(obj: JsObject, field, value: untyped): untyped = helper(`obj`, `value`) else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) let importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: JsObject, v: auto) @@ -282,7 +282,7 @@ macro `.()`*(obj: JsObject, importString = "#." & $field & "(@)" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] & "(@)" result = quote: proc helper(o: JsObject): JsObject @@ -302,7 +302,7 @@ macro `.`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & $field else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] result = quote do: proc helper(o: type(`obj`)): `obj`.V @@ -319,7 +319,7 @@ macro `.=`*[K: cstring, V](obj: JsAssoc[K, V], importString = "#." & $field & " = #" else: if not mangledNames.hasKey($field): - mangledNames[$field] = $mangleJsName($field) + mangledNames[$field] = mangleJsName($field) importString = "#." & mangledNames[$field] & " = #" result = quote do: proc helper(o: type(`obj`), v: `obj`.V) From f7a13f62d634300c3cf68e36dd7926a6a235d52f Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Sat, 4 Jun 2022 00:25:21 -0400 Subject: [PATCH 0819/1654] Stop type aliases from inheriting sfUsed (#19861) Fixes #18201 Co-authored-by: quantimnot --- compiler/ast.nim | 2 +- tests/errmsgs/treportunused.nim | 53 +++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index f8343c1a3..6610a1333 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1506,7 +1506,7 @@ proc assignType*(dest, src: PType) = # this fixes 'type TLock = TSysLock': if src.sym != nil: if dest.sym != nil: - dest.sym.flags.incl src.sym.flags-{sfExported} + dest.sym.flags.incl src.sym.flags-{sfUsed, sfExported} if dest.sym.annex == nil: dest.sym.annex = src.sym.annex mergeLoc(dest.sym.loc, src.sym.loc) else: diff --git a/tests/errmsgs/treportunused.nim b/tests/errmsgs/treportunused.nim index f5ee79afa..46afe163d 100644 --- a/tests/errmsgs/treportunused.nim +++ b/tests/errmsgs/treportunused.nim @@ -2,19 +2,27 @@ discard """ matrix: "--hint:all:off --hint:XDeclaredButNotUsed" nimoutFull: true nimout: ''' -treportunused.nim(23, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(24, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(25, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(26, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(27, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(28, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(29, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(30, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(31, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(32, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(33, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(37, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed] -treportunused.nim(38, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(51, 5) Hint: 'A' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(52, 5) Hint: 'B' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(55, 5) Hint: 'D' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(56, 5) Hint: 'E' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(59, 5) Hint: 'G' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(60, 5) Hint: 'H' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(64, 5) Hint: 'K' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(65, 5) Hint: 'L' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(31, 10) Hint: 's1' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(32, 10) Hint: 's2' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(33, 10) Hint: 's3' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(34, 6) Hint: 's4' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(35, 6) Hint: 's5' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(36, 7) Hint: 's6' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(37, 7) Hint: 's7' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(38, 5) Hint: 's8' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(39, 5) Hint: 's9' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(40, 6) Hint: 's10' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(41, 6) Hint: 's11' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(45, 3) Hint: 'v0.99' is declared but not used [XDeclaredButNotUsed] +treportunused.nim(46, 3) Hint: 'v0.99.99' is declared but not used [XDeclaredButNotUsed] ''' action: compile """ @@ -36,3 +44,22 @@ type s11 = type(1.2) let `v0.99` = "0.99" `v0.99.99` = "0.99.99" + +block: # bug #18201 + # Test that unused type aliases raise hint XDeclaredButNotUsed. + type + A = int + B = distinct int + + C = object + D = C + E = distinct C + + F = string + G = F + H = distinct F + + J = enum + Foo + K = J + L = distinct J From 4341b06f65396c63eec3689d0f8bb00db26c362f Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 4 Jun 2022 08:03:03 +0300 Subject: [PATCH 0820/1654] RST: improve simple tables (#19859) * RST: improve simple tables * nim 1.0 gotchas * Still allow legacy boundaries like `----` --- compiler/docgen.nim | 1 + compiler/lineinfos.nim | 2 + doc/apis.rst | 6 +- doc/astspec.txt | 12 +- doc/manual.rst | 2 +- doc/nep1.rst | 6 +- doc/nimdoc.cls | 8 +- doc/niminst.rst | 22 +-- doc/tut1.rst | 6 +- lib/packages/docutils/rst.nim | 223 +++++++++++++++++++++--------- lib/packages/docutils/rstast.nim | 13 ++ lib/packages/docutils/rstgen.nim | 55 +++++--- lib/pure/coro.nim | 4 +- lib/pure/os.nim | 12 +- lib/pure/xmltree.nim | 6 +- tests/stdlib/trst.nim | 229 +++++++++++++++++++++++++++++++ 16 files changed, 487 insertions(+), 120 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f633a57a0..390f44f2e 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -232,6 +232,7 @@ template declareClosures = of meExpected: k = errXExpected of meGridTableNotImplemented: k = errRstGridTableNotImplemented of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable + of meIllformedTable: k = errRstIllformedTable of meNewSectionExpected: k = errRstNewSectionExpected of meGeneralParseError: k = errRstGeneralParseError of meInvalidDirective: k = errRstInvalidDirectiveX diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 575be6196..105de1636 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -34,6 +34,7 @@ type errXExpected, errRstGridTableNotImplemented, errRstMarkdownIllformedTable, + errRstIllformedTable, errRstNewSectionExpected, errRstGeneralParseError, errRstInvalidDirectiveX, @@ -106,6 +107,7 @@ const errXExpected: "'$1' expected", errRstGridTableNotImplemented: "grid table is not implemented", errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table", + errRstIllformedTable: "Illformed table: $1", errRstNewSectionExpected: "new section expected $1", errRstGeneralParseError: "general parse error", errRstInvalidDirectiveX: "invalid directive: '$1'", diff --git a/doc/apis.rst b/doc/apis.rst index e8313749d..f0b8c93e5 100644 --- a/doc/apis.rst +++ b/doc/apis.rst @@ -18,9 +18,9 @@ been renamed to fit this scheme. The ultimate goal is that the programmer can *guess* a name. -------------------- ------------ -------------------------------------- +=================== ============ ====================================== English word To use Notes -------------------- ------------ -------------------------------------- +=================== ============ ====================================== initialize initT `init` is used to create a value type `T` new newP `new` is used to create a @@ -82,4 +82,4 @@ literal lit string str identifier ident indentation indent -------------------- ------------ -------------------------------------- +=================== ============ ====================================== diff --git a/doc/astspec.txt b/doc/astspec.txt index 6d3fa1f8c..dbbe2799d 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -50,9 +50,9 @@ A leaf of the AST often corresponds to a terminal symbol in the concrete syntax. Note that the default ``float`` in Nim maps to ``float64`` such that the default AST for a float is ``nnkFloat64Lit`` as below. ------------------ --------------------------------------------- +================= ============================================= Nim expression Corresponding AST ------------------ --------------------------------------------- +================= ============================================= ``42`` ``nnkIntLit(intVal = 42)`` ``42'i8`` ``nnkInt8Lit(intVal = 42)`` ``42'i16`` ``nnkInt16Lit(intVal = 42)`` @@ -72,7 +72,7 @@ Nim expression Corresponding AST ``nil`` ``nnkNilLit()`` ``myIdentifier`` ``nnkIdent(strVal = "myIdentifier")`` ``myIdentifier`` after lookup pass: ``nnkSym(strVal = "myIdentifier", ...)`` ------------------ --------------------------------------------- +================= ============================================= Identifiers are ``nnkIdent`` nodes. After the name lookup pass these nodes get transferred into ``nnkSym`` nodes. @@ -1211,9 +1211,9 @@ AST: In general, declaring types mirrors this syntax (i.e., ``nnkStaticTy`` for ``static``, etc.). Examples follow (exceptions marked by ``*``): -------------- --------------------------------------------- +============= ============================================= Nim type Corresponding AST -------------- --------------------------------------------- +============= ============================================= ``static`` ``nnkStaticTy`` ``tuple`` ``nnkTupleTy`` ``var`` ``nnkVarTy`` @@ -1226,7 +1226,7 @@ Nim type Corresponding AST ``proc`` ``nnkProcTy`` ``iterator`` ``nnkIteratorTy`` ``object`` ``nnkObjectTy`` -------------- --------------------------------------------- +============= ============================================= Take special care when declaring types as ``proc``. The behavior is similar to ``Procedure declaration``, below, but does not treat ``nnkGenericParams``. diff --git a/doc/manual.rst b/doc/manual.rst index 5ab257fec..82c9f5758 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -363,7 +363,7 @@ contain the following `escape sequences`:idx:\ : ``\\`` `backslash`:idx: ``\"`` `quotation mark`:idx: ``\'`` `apostrophe`:idx: - ``\`` '0'..'9'+ `character with decimal value d`:idx:; + ``\`` '0'..'9'+ `character with decimal value d`:idx:; all decimal digits directly following are used for the character ``\a`` `alert`:idx: diff --git a/doc/nep1.rst b/doc/nep1.rst index 8eb31332f..9627071bf 100644 --- a/doc/nep1.rst +++ b/doc/nep1.rst @@ -153,9 +153,9 @@ The library uses a simple naming scheme that makes use of common abbreviations to keep the names short but meaningful. -------------------- ------------ -------------------------------------- +=================== ============ ====================================== English word To use Notes -------------------- ------------ -------------------------------------- +=================== ============ ====================================== initialize initFoo initializes a value type `Foo` new newFoo initializes a reference type `Foo` via `new` or a value type `Foo` @@ -220,7 +220,7 @@ literal lit string str identifier ident indentation indent -------------------- ------------ -------------------------------------- +=================== ============ ====================================== Coding Conventions diff --git a/doc/nimdoc.cls b/doc/nimdoc.cls index 271dc5dc9..37039f130 100644 --- a/doc/nimdoc.cls +++ b/doc/nimdoc.cls @@ -63,7 +63,7 @@ \usepackage{scrextend} % for the `addmargin` environment -\usepackage{xcolor} +\usepackage[table]{xcolor} \usepackage[urlbordercolor=blue,linkbordercolor=cyan, pdfborderstyle={/S/U/W 1}]{hyperref} \usepackage{enumitem} % for option list, enumList, and rstfootnote @@ -84,6 +84,11 @@ \definecolor{rstframecolor}{rgb}{0.85, 0.8, 0.6} +\usepackage{booktabs} +\belowrulesep=0ex +\aboverulesep=0ex +\renewcommand{\arraystretch}{1.1} + \newtcolorbox{rstprebox}[1][]{blanker, breakable, left=3mm, right=3mm, top=1mm, bottom=1mm, borderline ={0.1em}{0pt}{rstframecolor}, @@ -127,6 +132,7 @@ \newenvironment{rstoptlist}{% \begin{description}[font=\sffamily\bfseries,style=nextline,leftmargin=\rstoptleftmargin,labelwidth=\rstoptlabelwidth]}{\end{description}} +\usepackage{multirow} \usepackage{tabulary} % tables with adjustable cell width and no overflow % make tabulary prevent overflows (https://tex.stackexchange.com/a/195088) \tymin=60pt diff --git a/doc/niminst.rst b/doc/niminst.rst index 8fff1f0e8..2da8664a9 100644 --- a/doc/niminst.rst +++ b/doc/niminst.rst @@ -49,20 +49,20 @@ contain the following key-value pairs: ==================== ======================================================= Key description ==================== ======================================================= -`Name` the project's name; this needs to be a single word -`DisplayName` the project's long name; this can contain spaces. If +`Name` the project's name; this needs to be a single word +`DisplayName` the project's long name; this can contain spaces. If not specified, this is the same as `Name`. -`Version` the project's version -`OS` the OSes to generate C code for; for example: +`Version` the project's version +`OS` the OSes to generate C code for; for example: `"windows;linux;macosx"` -`CPU` the CPUs to generate C code for; for example: +`CPU` the CPUs to generate C code for; for example: `"i386;amd64;powerpc"` -`Authors` the project's authors -`Description` the project's description -`App` the application's type: "Console" or "GUI". If +`Authors` the project's authors +`Description` the project's description +`App` the application's type: "Console" or "GUI". If "Console", niminst generates a special batch file for Windows to open up the command-line shell. -`License` the filename of the application's license +`License` the filename of the application's license ==================== ======================================================= @@ -149,9 +149,9 @@ Possible options are: ==================== ======================================================= Key description ==================== ======================================================= -`InstallScript` boolean flag whether an installation shell script +`InstallScript` boolean flag whether an installation shell script should be generated. Example: `InstallScript: "Yes"` -`UninstallScript` boolean flag whether a de-installation shell script +`UninstallScript` boolean flag whether a de-installation shell script should be generated. Example: `UninstallScript: "Yes"` ==================== ======================================================= diff --git a/doc/tut1.rst b/doc/tut1.rst index 405df57b1..66a4c3274 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1185,9 +1185,9 @@ subranges) are called ordinal types. Ordinal types have quite a few special operations: ------------------ -------------------------------------------------------- +================= ======================================================== Operation Comment ------------------ -------------------------------------------------------- +================= ======================================================== `ord(x)` returns the integer value that is used to represent `x`'s value `inc(x)` increments `x` by one @@ -1198,7 +1198,7 @@ Operation Comment `succ(x, n)` returns the `n`'th successor of `x` `pred(x)` returns the predecessor of `x` `pred(x, n)` returns the `n`'th predecessor of `x` ------------------ -------------------------------------------------------- +================= ======================================================== The `inc `_, `dec `_, `succ diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index d4bfae48f..cd5f26278 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -255,6 +255,7 @@ type meExpected = "'$1' expected", meGridTableNotImplemented = "grid table is not implemented", meMarkdownIllformedTable = "illformed delimiter row of a Markdown table", + meIllformedTable = "Illformed table: $1", meNewSectionExpected = "new section expected $1", meGeneralParseError = "general parse error", meInvalidDirective = "invalid directive: '$1'", @@ -2467,81 +2468,175 @@ proc parseOverline(p: var RstParser): PRstNode = anchorType=headlineAnchor) type - IntSeq = seq[int] - ColumnLimits = tuple + ColSpec = object + start, stop: int + RstCols = seq[ColSpec] + ColumnLimits = tuple # for Markdown first, last: int ColSeq = seq[ColumnLimits] -proc tokEnd(p: RstParser): int = - result = currentTok(p).col + currentTok(p).symbol.len - 1 +proc tokStart(p: RstParser, idx: int): int = + result = p.tok[idx].col -proc getColumns(p: var RstParser, cols: var IntSeq) = +proc tokStart(p: RstParser): int = + result = tokStart(p, p.idx) + +proc tokEnd(p: RstParser, idx: int): int = + result = p.tok[idx].col + p.tok[idx].symbol.len - 1 + +proc tokEnd(p: RstParser): int = + result = tokEnd(p, p.idx) + +proc getColumns(p: RstParser, cols: var RstCols, startIdx: int): int = + # Fills table column specification (or separator) `cols` and returns + # the next parser index after it. var L = 0 + result = startIdx while true: inc L setLen(cols, L) - cols[L - 1] = tokEnd(p) - assert(currentTok(p).kind == tkAdornment) - inc p.idx - if currentTok(p).kind != tkWhite: break - inc p.idx - if currentTok(p).kind != tkAdornment: break - if currentTok(p).kind == tkIndent: inc p.idx - # last column has no limit: - cols[L - 1] = 32000 + cols[L - 1].start = tokStart(p, result) + cols[L - 1].stop = tokEnd(p, result) + assert(p.tok[result].kind == tkAdornment) + inc result + if p.tok[result].kind != tkWhite: break + inc result + if p.tok[result].kind != tkAdornment: break + if p.tok[result].kind == tkIndent: inc result + +proc checkColumns(p: RstParser, cols: RstCols) = + var + i = p.idx + col = 0 + if p.tok[i].symbol[0] != '=': + rstMessage(p, mwRstStyle, + "only tables with `=` columns specification are allowed") + for col in 0 ..< cols.len: + if tokEnd(p, i) != cols[col].stop: + rstMessage(p, meIllformedTable, + "end of table column #$1 should end at position $2" % [ + $(col+1), $(cols[col].stop+ColRstOffset)], + p.tok[i].line, tokEnd(p, i)) + inc i + if col == cols.len - 1: + if p.tok[i].kind == tkWhite: + inc i + if p.tok[i].kind notin {tkIndent, tkEof}: + rstMessage(p, meIllformedTable, "extraneous column specification") + elif p.tok[i].kind == tkWhite: + inc i + else: + rstMessage(p, meIllformedTable, "no enough table columns", + p.tok[i].line, p.tok[i].col) + +proc getSpans(p: RstParser, nextLine: int, + cols: RstCols, unitedCols: RstCols): seq[int] = + ## Calculates how many columns a joined cell occupies. + if unitedCols.len > 0: + result = newSeq[int](unitedCols.len) + var + iCell = 0 + jCell = 0 + uCell = 0 + while jCell < cols.len: + if cols[jCell].stop < unitedCols[uCell].stop: + inc jCell + elif cols[jCell].stop == unitedCols[uCell].stop: + result[uCell] = jCell - iCell + 1 + iCell = jCell + 1 + jCell = jCell + 1 + inc uCell + else: + rstMessage(p, meIllformedTable, + "spanning underline does not match main table columns", + p.tok[nextLine].line, p.tok[nextLine].col) + +proc parseSimpleTableRow(p: var RstParser, cols: RstCols, colChar: char): PRstNode = + ## Parses 1 row in RST simple table. + # Consider that columns may be spanning (united by using underline like ----): + let nextLine = tokenAfterNewline(p) + var unitedCols: RstCols + var afterSpan: int + if p.tok[nextLine].kind == tkAdornment and p.tok[nextLine].symbol[0] == '-': + afterSpan = getColumns(p, unitedCols, nextLine) + if unitedCols == cols and p.tok[nextLine].symbol[0] == colChar: + # legacy rst.nim compat.: allow punctuation like `----` in main boundaries + afterSpan = nextLine + unitedCols.setLen 0 + else: + afterSpan = nextLine + template colEnd(i): int = + if i == cols.len - 1: high(int) # last column has no limit + elif unitedCols.len > 0: unitedCols[i].stop else: cols[i].stop + template colStart(i): int = + if unitedCols.len > 0: unitedCols[i].start else: cols[i].start + var row = newSeq[string](if unitedCols.len > 0: unitedCols.len else: cols.len) + var spans: seq[int] = getSpans(p, nextLine, cols, unitedCols) + + let line = currentTok(p).line + # Iterate over the lines a single cell may span: + while true: + var nCell = 0 + # distribute tokens between cells in the current line: + while currentTok(p).kind notin {tkIndent, tkEof}: + if tokEnd(p) <= colEnd(nCell): + if tokStart(p) < colStart(nCell): + if currentTok(p).kind != tkWhite: + rstMessage(p, meIllformedTable, + "this word crosses table column from the left") + else: + inc p.idx + else: + row[nCell].add(currentTok(p).symbol) + inc p.idx + else: + if tokStart(p) < colEnd(nCell) and currentTok(p).kind != tkWhite: + rstMessage(p, meIllformedTable, + "this word crosses table column from the right") + inc nCell + if currentTok(p).kind == tkIndent: inc p.idx + if tokEnd(p) <= colEnd(0): break + # Continued current cells because the 1st column is empty. + if currentTok(p).kind in {tkEof, tkAdornment}: + break + for nCell in countup(1, high(row)): row[nCell].add('\n') + result = newRstNode(rnTableRow) + var q: RstParser + for uCell in 0 ..< row.len: + initParser(q, p.s) + q.col = colStart(uCell) + q.line = line - 1 + getTokens(row[uCell], q.tok) + let cell = newRstNode(rnTableDataCell) + cell.span = if spans.len == 0: 0 else: spans[uCell] + cell.add(parseDoc(q)) + result.add(cell) + if afterSpan > p.idx: + p.idx = afterSpan proc parseSimpleTable(p: var RstParser): PRstNode = - var - cols: IntSeq - row: seq[string] - i, last, line: int - c: char - q: RstParser - a, b: PRstNode + var cols: RstCols result = newRstNodeA(p, rnTable) - cols = @[] - row = @[] - a = nil - c = currentTok(p).symbol[0] + let startIdx = getColumns(p, cols, p.idx) + let colChar = currentTok(p).symbol[0] + checkColumns(p, cols) + p.idx = startIdx + result.colCount = cols.len while true: if currentTok(p).kind == tkAdornment: - last = tokenAfterNewline(p) - if p.tok[last].kind in {tkEof, tkIndent}: + checkColumns(p, cols) + p.idx = tokenAfterNewline(p) + if currentTok(p).kind in {tkEof, tkIndent}: # skip last adornment line: - p.idx = last break - getColumns(p, cols) - setLen(row, cols.len) - if a != nil: - for j in 0 ..< a.len: # fix rnTableDataCell -> rnTableHeaderCell - a.sons[j] = newRstNode(rnTableHeaderCell, a.sons[j].sons) + if result.sons.len > 0: result.sons[^1].endsHeader = true + # fix rnTableDataCell -> rnTableHeaderCell for previous table rows: + for nRow in 0 ..< result.sons.len: + for nCell in 0 ..< result.sons[nRow].len: + result.sons[nRow].sons[nCell].kind = rnTableHeaderCell if currentTok(p).kind == tkEof: break - for j in countup(0, high(row)): row[j] = "" - # the following while loop iterates over the lines a single cell may span: - line = currentTok(p).line - while true: - i = 0 - while currentTok(p).kind notin {tkIndent, tkEof}: - if tokEnd(p) <= cols[i]: - row[i].add(currentTok(p).symbol) - inc p.idx - else: - if currentTok(p).kind == tkWhite: inc p.idx - inc i - if currentTok(p).kind == tkIndent: inc p.idx - if tokEnd(p) <= cols[0]: break - if currentTok(p).kind in {tkEof, tkAdornment}: break - for j in countup(1, high(row)): row[j].add('\n') - a = newRstNode(rnTableRow) - for j in countup(0, high(row)): - initParser(q, p.s) - q.col = cols[j] - q.line = line - 1 - getTokens(row[j], q.tok) - b = newRstNode(rnTableDataCell) - b.add(parseDoc(q)) - a.add(b) - result.add(a) + let tabRow = parseSimpleTableRow(p, cols, colChar) + result.add tabRow proc readTableRow(p: var RstParser): ColSeq = if currentTok(p).symbol == "|": inc p.idx @@ -2574,17 +2669,16 @@ proc isValidDelimiterRow(p: var RstParser, colNum: int): bool = proc parseMarkdownTable(p: var RstParser): PRstNode = var row: ColSeq - colNum: int a, b: PRstNode q: RstParser result = newRstNodeA(p, rnMarkdownTable) proc parseRow(p: var RstParser, cellKind: RstNodeKind, result: PRstNode) = row = readTableRow(p) - if colNum == 0: colNum = row.len # table header - elif row.len < colNum: row.setLen(colNum) + if result.colCount == 0: result.colCount = row.len # table header + elif row.len < result.colCount: row.setLen(result.colCount) a = newRstNode(rnTableRow) - for j in 0 ..< colNum: + for j in 0 ..< result.colCount: b = newRstNode(cellKind) initParser(q, p.s) q.col = p.col @@ -2595,7 +2689,8 @@ proc parseMarkdownTable(p: var RstParser): PRstNode = result.add(a) parseRow(p, rnTableHeaderCell, result) - if not isValidDelimiterRow(p, colNum): rstMessage(p, meMarkdownIllformedTable) + if not isValidDelimiterRow(p, result.colCount): + rstMessage(p, meMarkdownIllformedTable) while predNL(p) and currentTok(p).symbol == "|": parseRow(p, rnTableDataCell, result) diff --git a/lib/packages/docutils/rstast.nim b/lib/packages/docutils/rstast.nim index 1d5da5e1c..bf0bc1be5 100644 --- a/lib/packages/docutils/rstast.nim +++ b/lib/packages/docutils/rstast.nim @@ -112,6 +112,12 @@ type ## nodes that are post-processed after parsing of rnNimdocRef: tooltip*: string + of rnTable, rnGridTable, rnMarkdownTable: + colCount*: int ## Number of (not-united) cells in the table + of rnTableRow: + endsHeader*: bool ## Is last row in the header of table? + of rnTableHeaderCell, rnTableDataCell: + span*: int ## Number of table columns that the cell occupies else: discard anchor*: string ## anchor, internal link target @@ -416,6 +422,13 @@ proc treeRepr*(node: PRstNode, indent=0): string = result.add (if node.order == 0: "" else: " order=" & $node.order) of rnMarkdownBlockQuoteItem: result.add " quotationDepth=" & $node.quotationDepth + of rnTable, rnGridTable, rnMarkdownTable: + result.add " colCount=" & $node.colCount + of rnTableHeaderCell, rnTableDataCell: + if node.span > 0: + result.add " span=" & $node.span + of rnTableRow: + if node.endsHeader: result.add " endsHeader" else: discard result.add (if node.anchor == "": "" else: " anchor='" & node.anchor & "'") diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index d662a667c..8eb4be675 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -1091,10 +1091,6 @@ proc renderContainer(d: PDoc, n: PRstNode, result: var string) = else: dispA(d.target, result, "
              $2
              ", "$2", [arg, tmp]) -proc texColumns(n: PRstNode): string = - let nColumns = if n.sons.len > 0: len(n.sons[0]) else: 1 - result = "L".repeat(nColumns) - proc renderField(d: PDoc, n: PRstNode, result: var string) = var b = false if d.target == outLatex: @@ -1323,24 +1319,49 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = renderAux(d, n, "$1", "\n$2\n\\begin{rsttab}{" & - texColumns(n) & "}\n\\hline\n$1\\end{rsttab}", result) + "L".repeat(n.colCount) & "}\n\\toprule\n$1" & + "\\addlinespace[0.1em]\\bottomrule\n\\end{rsttab}", result) of rnTableRow: if len(n) >= 1: - if d.target == outLatex: - #var tmp = "" - renderRstToOut(d, n.sons[0], result) - for i in countup(1, len(n) - 1): - result.add(" & ") - renderRstToOut(d, n.sons[i], result) - result.add("\\\\\n\\hline\n") - else: + case d.target + of outHtml: result.add("") renderAux(d, n, result) result.add("\n") - of rnTableDataCell: - renderAux(d, n, "$1", "$1", result) - of rnTableHeaderCell: - renderAux(d, n, "$1", "\\textbf{$1}", result) + of outLatex: + if n.sons[0].kind == rnTableHeaderCell: + result.add "\\rowcolor{gray!15} " + var spanLines: seq[(int, int)] + var nCell = 0 + for uCell in 0 .. n.len - 1: + renderRstToOut(d, n.sons[uCell], result) + if n.sons[uCell].span > 0: + spanLines.add (nCell + 1, nCell + n.sons[uCell].span) + nCell += n.sons[uCell].span + else: + nCell += 1 + if uCell != n.len - 1: + result.add(" & ") + result.add("\\\\") + if n.endsHeader: result.add("\\midrule\n") + for (start, stop) in spanLines: + result.add("\\cmidrule(lr){$1-$2}" % [$start, $stop]) + result.add("\n") + of rnTableHeaderCell, rnTableDataCell: + case d.target + of outHtml: + let tag = if n.kind == rnTableHeaderCell: "th" else: "td" + var spanSpec: string + if n.span <= 1: spanSpec = "" + else: + spanSpec = " colspan=\"" & $n.span & "\" style=\"text-align: center\"" + renderAux(d, n, "<$1$2>$$1" % [tag, spanSpec], "", result) + of outLatex: + let text = if n.kind == rnTableHeaderCell: "\\textbf{$1}" else: "$1" + var latexStr: string + if n.span <= 1: latexStr = text + else: latexStr = "\\multicolumn{" & $n.span & "}{c}{" & text & "}" + renderAux(d, n, "", latexStr, result) of rnFootnoteGroup: renderAux(d, n, "
              " & diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index f4495a536..aaf442a83 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -8,11 +8,11 @@ # ## Nim coroutines implementation, supports several context switching methods: -## -------- ------------ +## ======== ============ ## ucontext available on unix and alike (default) ## setjmp available on unix and alike (x86/64 only) ## fibers available and required on windows. -## -------- ------------ +## ======== ============ ## ## -d:nimCoroutines Required to build this module. ## -d:nimCoroutinesUcontext Use ucontext backend. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index fa379a228..247a0d089 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2383,21 +2383,21 @@ iterator walkDirRec*(dir: string, ## ## Walking is recursive. `followFilter` controls the behaviour of the iterator: ## - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## yieldFilter meaning - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## ``pcFile`` yield real files (default) ## ``pcLinkToFile`` yield symbolic links to files ## ``pcDir`` yield real directories ## ``pcLinkToDir`` yield symbolic links to directories - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## followFilter meaning - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## ``pcDir`` follow real directories (default) ## ``pcLinkToDir`` follow symbolic links to directories - ## --------------------- --------------------------------------------- + ## ===================== ============================================= ## ## ## See also: diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index fdb2c3fc5..9a9ecde57 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -552,15 +552,15 @@ proc escape*(s: string): string = ## ## Escapes these characters: ## - ## ------------ ------------------- + ## ============ =================== ## char is converted to - ## ------------ ------------------- + ## ============ =================== ## ``<`` ``<`` ## ``>`` ``>`` ## ``&`` ``&`` ## ``"`` ``"`` ## ``'`` ``'`` - ## ------------ ------------------- + ## ============ =================== ## ## You can also use `addEscaped proc <#addEscaped,string,string>`_. result = newStringOfCap(s.len) diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 9787c13eb..24bc03027 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -3,6 +3,8 @@ discard """ [Suite] RST parsing +[Suite] RST tables + [Suite] RST indentation [Suite] Warnings @@ -618,6 +620,233 @@ suite "RST parsing": rnLeaf 'code' """) +suite "RST tables": + + test "formatting in tables works": + check( + dedent""" + ========= === + `build` `a` + ========= === + """.toAst == + dedent""" + rnTable colCount=2 + rnTableRow + rnTableDataCell + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'build' + rnTableDataCell + rnInlineCode + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf 'a' + """) + + test "tables with slightly overflowed cells cause an error (1)": + var error = new string + check( + dedent""" + ====== ====== + Inputs Output + ====== ====== + """.toAst(error=error) == "") + check(error[] == "input(2, 2) Error: Illformed table: " & + "this word crosses table column from the right") + + test "tables with slightly overflowed cells cause an error (2)": + var error = new string + check("" == dedent""" + ===== ===== ====== + Input Output + ===== ===== ====== + False False False + ===== ===== ====== + """.toAst(error=error)) + check(error[] == "input(2, 8) Error: Illformed table: " & + "this word crosses table column from the right") + + test "tables with slightly underflowed cells cause an error": + var error = new string + check("" == dedent""" + ===== ===== ====== + Input Output + ===== ===== ====== + False False False + ===== ===== ====== + """.toAst(error=error)) + check(error[] == "input(2, 7) Error: Illformed table: " & + "this word crosses table column from the left") + + test "tables with unequal underlines should be reported (1)": + var error = new string + error[] = "none" + check("" == dedent""" + ===== ====== + Input Output + ===== ====== + False False + ===== ======= + """.toAst(error=error)) + check(error[] == "input(5, 14) Error: Illformed table: " & + "end of table column #2 should end at position 13") + + test "tables with unequal underlines should be reported (2)": + var error = new string + check("" == dedent""" + ===== ====== + Input Output + ===== ======= + False False + ===== ====== + """.toAst(error=error)) + check(error[] == "input(3, 14) Error: Illformed table: " & + "end of table column #2 should end at position 13") + + test "tables with empty first cells": + check( + dedent""" + = = = + x y z + t + = = = + """.toAst == + dedent""" + rnTable colCount=3 + rnTableRow + rnTableDataCell + rnLeaf 'x' + rnTableDataCell + rnInner + rnLeaf 'y' + rnLeaf ' ' + rnTableDataCell + rnInner + rnLeaf 'z' + rnLeaf ' ' + rnLeaf 't' + """) + + test "tables with spanning cells & separators": + check( + dedent""" + ===== ===== ====== + Inputs Output + ------------ ------ + A B A or B + ===== ===== ====== + False False False + True False True + ----- ----- ------ + False True True + True True True + ===== ===== ====== + """.toAst == + dedent""" + rnTable colCount=3 + rnTableRow + rnTableHeaderCell span=2 + rnLeaf 'Inputs' + rnTableHeaderCell span=1 + rnLeaf 'Output' + rnTableRow endsHeader + rnTableHeaderCell + rnLeaf 'A' + rnTableHeaderCell + rnLeaf 'B' + rnTableHeaderCell + rnInner + rnLeaf 'A' + rnLeaf ' ' + rnLeaf 'or' + rnLeaf ' ' + rnLeaf 'B' + rnTableRow + rnTableDataCell + rnLeaf 'False' + rnTableDataCell + rnLeaf 'False' + rnTableDataCell + rnLeaf 'False' + rnTableRow + rnTableDataCell span=1 + rnLeaf 'True' + rnTableDataCell span=1 + rnLeaf 'False' + rnTableDataCell span=1 + rnLeaf 'True' + rnTableRow + rnTableDataCell + rnLeaf 'False' + rnTableDataCell + rnLeaf 'True' + rnTableDataCell + rnLeaf 'True' + rnTableRow + rnTableDataCell + rnLeaf 'True' + rnTableDataCell + rnLeaf 'True' + rnTableDataCell + rnLeaf 'True' + """) + + test "tables with spanning cells with uneqal underlines cause an error": + var error = new string + check( + dedent""" + ===== ===== ====== + Inputs Output + ------------- ------ + A B A or B + ===== ===== ====== + """.toAst(error=error) == "") + check(error[] == "input(3, 1) Error: Illformed table: " & + "spanning underline does not match main table columns") + + let expTable = dedent""" + rnTable colCount=2 + rnTableRow + rnTableDataCell + rnLeaf 'Inputs' + rnTableDataCell + rnLeaf 'Output' + """ + + test "only tables with `=` columns specs are allowed (1)": + var warnings = new seq[string] + check( + dedent""" + ------ ------ + Inputs Output + ------ ------ + """.toAst(warnings=warnings) == + expTable) + check(warnings[] == + @["input(1, 1) Warning: RST style: " & + "only tables with `=` columns specification are allowed", + "input(3, 1) Warning: RST style: " & + "only tables with `=` columns specification are allowed"]) + + test "only tables with `=` columns specs are allowed (2)": + var warnings = new seq[string] + check( + dedent""" + ====== ====== + Inputs Output + ~~~~~~ ~~~~~~ + """.toAst(warnings=warnings) == + expTable) + check(warnings[] == + @["input(3, 1) Warning: RST style: "& + "only tables with `=` columns specification are allowed"]) + + suite "RST indentation": test "nested bullet lists": let input = dedent """ From d0232f0e5b97b697a640c2f622c7596ce6d10eb5 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Sat, 4 Jun 2022 02:03:25 -0300 Subject: [PATCH 0821/1654] Add Microtasks (#19860) * Add Microtasks * Add Microtasks --- changelog.md | 2 +- lib/js/jscore.nim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 9548f3034..2ae64d543 100644 --- a/changelog.md +++ b/changelog.md @@ -50,7 +50,7 @@ becomes an alias for `addr`. - Removed deprecated `std/dom_extensions`. - Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. - Remove deprecated `osproc.poDemon`, symbol with typo. - +- Added [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) for JavaScript targets. - Deprecated `selfExe` for Nimscript. diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index 4518f32ce..e6060535b 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -132,3 +132,8 @@ since (1, 7): var arrai = @[1, 2, 3] assert arrai.shift() == 1 assert arrai == @[2, 3] + + func queueMicrotask*(function: proc) {.importjs: "$1(#)".} = + ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask + ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide + runnableExamples: queueMicrotask(proc() = echo "Microtask") From b024a45163ce1a2224ec5891c81cf2cc7d399f11 Mon Sep 17 00:00:00 2001 From: Zoom Date: Tue, 7 Jun 2022 08:35:06 +0000 Subject: [PATCH 0822/1654] Fix `find` routines' api to default to `last=-1` (#19761) This changes the default for the `last` parameter of various `find` routines from `0` to `-1`. Previous default prevents limiting the search to the first character. This is a logic error, as full text search was performed for 2 *valid* values of `last`: `0` and `last.high()`. Adds an overload for `initSkipTable` which returns a newly initialized table. This encapsulates every single usage of a `var`-acting original func in this module. Co-authored-by: flywind --- lib/pure/strutils.nim | 91 +++++++++++++++++++++----------------- tests/stdlib/tstrutils.nim | 42 ++++++++++-------- 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index b98bd2bec..ffbefd0fb 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1815,6 +1815,7 @@ func initSkipTable*(a: var SkipTable, sub: string) {.rtl, ## Initializes table `a` for efficient search of substring `sub`. ## ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ ## * `find func<#find,SkipTable,string,string,Natural,int>`_ # TODO: this should be the `default()` initializer for the type. let m = len(sub) @@ -1823,7 +1824,16 @@ func initSkipTable*(a: var SkipTable, sub: string) {.rtl, for i in 0 ..< m - 1: a[sub[i]] = m - 1 - i -func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. +func initSkipTable*(sub: string): SkipTable {.noinit, rtl, + extern: "nsuInitNewSkipTable".} = + ## Returns a new table initialized for `sub`. + ## + ## See also: + ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ + ## * `find func<#find,SkipTable,string,string,Natural,int>`_ + initSkipTable(result, sub) + +func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindStrA".} = ## Searches for `sub` in `s` inside range `start..last` using preprocessed ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last @@ -1832,9 +1842,10 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## ## See also: + ## * `initSkipTable func<#initSkipTable,string>`_ ## * `initSkipTable func<#initSkipTable,SkipTable,string>`_ let - last = if last == 0: s.high else: last + last = if last < 0: s.high else: last subLast = sub.len - 1 if subLast == -1: @@ -1844,6 +1855,7 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. # This is an implementation of the Boyer-Moore Horspool algorithms # https://en.wikipedia.org/wiki/Boyer%E2%80%93Moore%E2%80%93Horspool_algorithm + result = -1 var skip = start while last - skip >= subLast: @@ -1853,7 +1865,6 @@ func find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int {. return skip dec i inc skip, a[s[skip + subLast]] - return -1 when not (defined(js) or defined(nimdoc) or defined(nimscript)): func c_memchr(cstr: pointer, c: char, n: csize_t): pointer {. @@ -1865,10 +1876,10 @@ when not (defined(js) or defined(nimdoc) or defined(nimscript)): else: const hasCStringBuiltin = false -func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, +func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindChar".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. @@ -1877,26 +1888,30 @@ func find*(s: string, sub: char, start: Natural = 0, last = 0): int {.rtl, ## See also: ## * `rfind func<#rfind,string,char,Natural,int>`_ ## * `replace func<#replace,string,char,char>`_ - let last = if last == 0: s.high else: last - when nimvm: + result = -1 + let last = if last < 0: s.high else: last + + template findImpl = for i in int(start)..last: - if sub == s[i]: return i + if s[i] == sub: + return i + + when nimvm: + findImpl() else: when hasCStringBuiltin: - let L = last-start+1 - if L > 0: - let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) + let length = last-start+1 + if length > 0: + let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) else: - for i in int(start)..last: - if sub == s[i]: return i - return -1 + findImpl() -func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. +func find*(s: string, chars: set[char], start: Natural = 0, last = -1): int {. rtl, extern: "nsuFindCharSet".} = ## Searches for `chars` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## If `s` contains none of the characters in `chars`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. @@ -1905,15 +1920,16 @@ func find*(s: string, chars: set[char], start: Natural = 0, last = 0): int {. ## See also: ## * `rfind func<#rfind,string,set[char],Natural,int>`_ ## * `multiReplace func<#multiReplace,string,varargs[]>`_ - let last = if last == 0: s.high else: last + result = -1 + let last = if last < 0: s.high else: last for i in int(start)..last: - if s[i] in chars: return i - return -1 + if s[i] in chars: + return i -func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, +func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, extern: "nsuFindStr".} = ## Searches for `sub` in `s` inside range `start..last` (both ends included). - ## If `last` is unspecified, it defaults to `s.high` (the last element). + ## If `last` is unspecified or negative, it defaults to `s.high` (the last element). ## ## Searching is case-sensitive. If `sub` is not in `s`, -1 is returned. ## Otherwise the index returned is relative to `s[0]`, not `start`. @@ -1925,28 +1941,23 @@ func find*(s, sub: string, start: Natural = 0, last = 0): int {.rtl, if sub.len > s.len - start: return -1 if sub.len == 1: return find(s, sub[0], start, last) - template useSkipTable {.dirty.} = - var a {.noinit.}: SkipTable - initSkipTable(a, sub) - result = find(a, s, sub, start, last) + template useSkipTable = + result = find(initSkipTable(sub), s, sub, start, last) - when not hasCStringBuiltin: + when nimvm: useSkipTable() else: - when nimvm: - useSkipTable() - else: - when hasCStringBuiltin: - if last == 0 and s.len > start: - let found = c_strstr(s[start].unsafeAddr, sub) - if not found.isNil: - result = cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + when hasCStringBuiltin: + if last < 0 and start < s.len: + let found = c_strstr(s[start].unsafeAddr, sub) + result = if not found.isNil: + cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) else: - result = -1 - else: - useSkipTable() + -1 else: useSkipTable() + else: + useSkipTable() func rfind*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, extern: "nsuRFindChar".} = @@ -2121,8 +2132,7 @@ func replace*(s, sub: string, by = ""): string {.rtl, # copy the rest: add result, substr(s, i) else: - var a {.noinit.}: SkipTable - initSkipTable(a, sub) + var a = initSkipTable(sub) let last = s.high var i = 0 while true: @@ -2162,9 +2172,8 @@ func replaceWord*(s, sub: string, by = ""): string {.rtl, ## replaced. if sub.len == 0: return s const wordChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\128'..'\255'} - var a {.noinit.}: SkipTable result = "" - initSkipTable(a, sub) + var a = initSkipTable(sub) var i = 0 let last = s.high let sublen = sub.len diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index 26fc0c7b0..e17277ef2 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -232,18 +232,22 @@ template main() = {.pop.} block: # find - doAssert "0123456789ABCDEFGH".find('A') == 10 - doAssert "0123456789ABCDEFGH".find('A', 5) == 10 - doAssert "0123456789ABCDEFGH".find('A', 5, 10) == 10 - doAssert "0123456789ABCDEFGH".find('A', 5, 9) == -1 - doAssert "0123456789ABCDEFGH".find("A") == 10 - doAssert "0123456789ABCDEFGH".find("A", 5) == 10 - doAssert "0123456789ABCDEFGH".find("A", 5, 10) == 10 - doAssert "0123456789ABCDEFGH".find("A", 5, 9) == -1 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}) == 10 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}, 5) == 10 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}, 5, 10) == 10 - doAssert "0123456789ABCDEFGH".find({'A'..'C'}, 5, 9) == -1 + const haystack: string = "0123456789ABCDEFGH" + doAssert haystack.find('A') == 10 + doAssert haystack.find('A', 5) == 10 + doAssert haystack.find('A', 5, 10) == 10 + doAssert haystack.find('A', 5, 9) == -1 + doAssert haystack.find("A") == 10 + doAssert haystack.find("A", 5) == 10 + doAssert haystack.find("A", 5, 10) == 10 + doAssert haystack.find("A", 5, 9) == -1 + doAssert haystack.find({'A'..'C'}) == 10 + doAssert haystack.find({'A'..'C'}, 5) == 10 + doAssert haystack.find({'A'..'C'}, 5, 10) == 10 + doAssert haystack.find({'A'..'C'}, 5, 9) == -1 + doAssert haystack.find('A', 0, 0) == -1 # search limited to the first char + doAssert haystack.find('A', 5, 0) == -1 # last < start + doAssert haystack.find('A', 5, 4) == -1 # last < start block: const haystack: string = "ABCABABABABCAB" @@ -290,17 +294,17 @@ template main() = # when last <= start, searching for non-empty string block: - let last: int = -1 - doAssert "abcd".find("ab", start=0, last=last) == -1 - doAssert "abcd".find("ab", start=1, last=last) == -1 - doAssert "abcd".find("bc", start=1, last=last) == -1 - doAssert "abcd".find("bc", start=2, last=last) == -1 - block: - let last: int = 0 + let last: int = -1 # searching through whole line doAssert "abcd".find("ab", start=0, last=last) == 0 doAssert "abcd".find("ab", start=1, last=last) == -1 doAssert "abcd".find("bc", start=1, last=last) == 1 doAssert "abcd".find("bc", start=2, last=last) == -1 + block: + let last: int = 0 + doAssert "abcd".find("ab", start=0, last=last) == -1 + doAssert "abcd".find("ab", start=1, last=last) == -1 + doAssert "abcd".find("bc", start=1, last=last) == -1 + doAssert "abcd".find("bc", start=2, last=last) == -1 block: let last: int = 1 doAssert "abcd".find("ab", start=0, last=last) == 0 From f2b16c490df858328cbd884ff9ca9d1a675a4a31 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Wed, 8 Jun 2022 11:10:52 -0300 Subject: [PATCH 0823/1654] Add Microtasks docs (#19870) * Add Microtasks * Add Microtasks * Workaround to build js docs in older NodeJS versions --- lib/js/jscore.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/js/jscore.nim b/lib/js/jscore.nim index e6060535b..781e8fd57 100644 --- a/lib/js/jscore.nim +++ b/lib/js/jscore.nim @@ -136,4 +136,4 @@ since (1, 7): func queueMicrotask*(function: proc) {.importjs: "$1(#)".} = ## * https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask ## * https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide - runnableExamples: queueMicrotask(proc() = echo "Microtask") + runnableExamples"-r:off": queueMicrotask(proc() = echo "Microtask") From e1702ae1e6da5bf560d7b692bcd5fa513ece8a69 Mon Sep 17 00:00:00 2001 From: Zoom Date: Wed, 8 Jun 2022 14:11:22 +0000 Subject: [PATCH 0824/1654] `changelog.md`: Add `strutils.find` changes (#19868) Added `strutils.find` changes Standard library section restructured to group by `changed/added/deprecated/removed` with paragraphs marked by markdown comments. --- changelog.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/changelog.md b/changelog.md index 2ae64d543..356ee9263 100644 --- a/changelog.md +++ b/changelog.md @@ -24,35 +24,39 @@ becomes an alias for `addr`. ## Standard library additions and changes -- `macros.parseExpr` and `macros.parseStmt` now accept an optional +[//]: # "Changes:" +- `macros.parseExpr` and `macros.parseStmt` now accept an optional. filename argument for more informative errors. - Module `colors` expanded with missing colors from the CSS color standard. - Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). - `md5` now works at compile time and in JavaScript. - - `std/smtp` sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. +- Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. +- `strutils.find` now use and default to `last=-1` for whole string searches, making limiting it to just the first char (`last=0`) valid. +[//]: # "Additions:" - Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. - Added `IsoYear`, a distinct int type to prevent bugs from confusing the week-based year and the regular year. - Added `initDateTime` in `times` to create a datetime from a weekday, and ISO 8601 week number and week-based year. - Added `getIsoWeekAndYear` in `times` to get an ISO week number along with the corresponding ISO week-based year from a datetime. - Added `getIsoWeeksInYear` in `times` to return the number of weeks in an ISO week-based year. -- Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - Added `sep` parameter in `std/uri` to specify the query separator. -- Removed deprecated `oids.oidToString`. -- Remove define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. - -- Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. -- Removed deprecated `jsre.test` and `jsre.toString`. -- Removed deprecated `math.c_frexp`. -- Removed deprecated ``` httpcore.`==` ```. -- Removed deprecated `std/dom_extensions`. -- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. -- Remove deprecated `osproc.poDemon`, symbol with typo. +- Added [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) for JavaScript targets. - Added [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) for JavaScript targets. + +[//]: # "Deprecations:" - Deprecated `selfExe` for Nimscript. +[//]: # "Removals:" +- Removed deprecated `oids.oidToString`. +- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. +- Removed deprecated `jsre.test` and `jsre.toString`. +- Removed deprecated `math.c_frexp`. +- Removed deprecated `` httpcore.`==` ``. +- Removed deprecated `std/dom_extensions`. +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. +- Removed deprecated `osproc.poDemon`, symbol with typo. ## Language changes From 415689323c625afb7d39d923c4165a541beb9927 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 9 Jun 2022 00:17:25 +0800 Subject: [PATCH 0825/1654] clarify how to restart CI (#19845) --- doc/contributing.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/contributing.rst b/doc/contributing.rst index d3b2770f3..0ca0d0cbc 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -539,6 +539,7 @@ Debugging CI failures, flaky tests, etc 2. If CI failure seems unrelated to your PR, it could be caused by a flaky test. File a bug for it if it isn't already reported. A PR push (or opening/closing PR) will re-trigger all CI jobs (even successful ones, which can be wasteful). Instead, + request collaboration from the Nim team. The Nim team should follow these instructions to only restart the jobs that failed: * Azure: if on your own fork, it's possible from inside azure console From 836b95c896087a3f24bedad9bc1bcb2c5db1c5a8 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Thu, 9 Jun 2022 09:27:31 +0300 Subject: [PATCH 0826/1654] Deprecate std/sums (#18439) * Deprecate sums * Update changelog.md * Update lib/std/sums.nim * log * format * remove * Update changelog.md Co-authored-by: sandytypical <43030857+xflywind@users.noreply.github.com> --- changelog.md | 2 ++ lib/std/sums.nim | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changelog.md b/changelog.md index 356ee9263..6bd3c99e3 100644 --- a/changelog.md +++ b/changelog.md @@ -11,6 +11,8 @@ or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). Nim uses `Math.trunc` for the division and modulo operators for integers. +- Deprecated `std/sums`. + - Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via `experimental:flexibleOptionalParams`. diff --git a/lib/std/sums.nim b/lib/std/sums.nim index b68858ef7..a6ce1b85d 100644 --- a/lib/std/sums.nim +++ b/lib/std/sums.nim @@ -8,6 +8,8 @@ ## Accurate summation functions. +{.deprecated: "use the nimble package `sums` instead.".} + runnableExamples: import std/math From 25d89269eb7fe7ebed142f2d3136959db11c9f02 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 9 Jun 2022 14:21:18 +0200 Subject: [PATCH 0827/1654] give a better error message for some template expansions (#19871) --- compiler/semstmts.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 3513fd767..0516d08ce 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -140,7 +140,9 @@ proc discardCheck(c: PContext, result: PNode, flags: TExprFlags) = n[0] = result elif result.typ.kind != tyError and c.config.cmd != cmdInteractive: var n = result - while n.kind in skipForDiscardable: n = n.lastSon + while n.kind in skipForDiscardable: + if n.kind == nkTryStmt: n = n[0] + else: n = n.lastSon var s = "expression '" & $n & "' is of type '" & result.typ.typeToString & "' and has to be used (or discarded)" if result.info.line != n.info.line or @@ -483,7 +485,7 @@ proc semVarMacroPragma(c: PContext, a: PNode, n: PNode): PNode = for i in 0 ..< pragmas.len: let it = pragmas[i] let key = if it.kind in nkPragmaCallKinds and it.len >= 1: it[0] else: it - + when false: let lhs = b[0] let clash = strTableGet(c.currentScope.symbols, lhs.ident) From 1972005439306a12088400ef338c40c3d3c2ce28 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 9 Jun 2022 22:51:17 +0800 Subject: [PATCH 0828/1654] fix #19862; make widestrs consistent between refc and orc (#19874) [backport] fix #19862; make widestrs consistent in refc and orc --- lib/system/widestrs.nim | 3 +++ tests/arc/t19862.nim | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/arc/t19862.nim diff --git a/lib/system/widestrs.nim b/lib/system/widestrs.nim index 8b08959b5..bb348fd67 100644 --- a/lib/system/widestrs.nim +++ b/lib/system/widestrs.nim @@ -224,3 +224,6 @@ when defined(nimv2): proc `$`*(s: WideCStringObj): string = $(s.data) + + proc len*(w: WideCStringObj): int {.inline.} = + len(w.data) diff --git a/tests/arc/t19862.nim b/tests/arc/t19862.nim new file mode 100644 index 000000000..f7146ec26 --- /dev/null +++ b/tests/arc/t19862.nim @@ -0,0 +1,13 @@ +discard """ + matrix: "--gc:refc; --gc:arc" +""" + +# bug #19862 +type NewString = object + +proc len(s: NewString): int = 10 + +converter toNewString(x: WideCStringObj): NewString = discard + +let w = newWideCString("test") +doAssert len(w) == 4 From eefca1b81f334a5ccdb7d180de5b6a9969f12793 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Fri, 10 Jun 2022 21:24:28 +0800 Subject: [PATCH 0829/1654] not generate initStackBottomWith in arc/orc [backport] (#19875) not generate initStackBottomWith in arc/orc --- compiler/cgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 8d24f30c5..862f31107 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -2094,7 +2094,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = discard cgsym(m, "rawWrite") # raise dependencies on behalf of genMainProc - if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone: + if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}: discard cgsym(m, "initStackBottomWith") if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone: discard cgsym(m, "initThreadVarsEmulation") From 8cde6ba6bcf236478f1d5d5aa722977ce9723a57 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 11 Jun 2022 02:32:27 +0800 Subject: [PATCH 0830/1654] remove noop option `gc:v2` (#19810) * remove noop option gc:v2 * changelog --- changelog.md | 2 ++ compiler/ccgstmts.nim | 2 +- compiler/ccgtypes.nim | 4 ++-- compiler/commands.nim | 2 -- compiler/options.nim | 1 - lib/system/sysstr.nim | 14 ++------------ 6 files changed, 7 insertions(+), 18 deletions(-) diff --git a/changelog.md b/changelog.md index 6bd3c99e3..98655eaa3 100644 --- a/changelog.md +++ b/changelog.md @@ -24,6 +24,8 @@ becomes an alias for `addr`. - io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`. +- The `gc:v2` option is removed. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 240fa55c8..e791318ee 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -16,7 +16,7 @@ const # above X strings a hash-switch for strings is generated proc getTraverseProc(p: BProc, v: PSym): Rope = - if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcV2, gcRefc} and + if p.config.selectedGC in {gcMarkAndSweep, gcHooks, gcRefc} and optOwnedRefs notin p.config.globalOptions and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index c5b4d3857..7c26db40d 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1483,12 +1483,12 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = genTupleInfo(m, x, x, result, info) of tySequence: genTypeInfoAux(m, t, t, result, info) - if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}: + if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcGo}: let markerProc = genTraverseProc(m, origType, sig) m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc]) of tyRef: genTypeInfoAux(m, t, t, result, info) - if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}: + if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcGo}: let markerProc = genTraverseProc(m, origType, sig) m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc]) of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info) diff --git a/compiler/commands.nim b/compiler/commands.nim index 2e08ad615..31e637aba 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -262,7 +262,6 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo of "go": result = conf.selectedGC == gcGo of "none": result = conf.selectedGC == gcNone of "stack", "regions": result = conf.selectedGC == gcRegions - of "v2", "generational": warningOptionNoop(arg) else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) of "opt": case arg.normalize @@ -568,7 +567,6 @@ proc processMemoryManagementOption(switch, arg: string, pass: TCmdLinePass, unregisterArcOrc(conf) conf.selectedGC = gcRegions defineSymbol(conf.symbols, "gcregions") - of "v2": warningOptionNoop(arg) else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg) proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; diff --git a/compiler/options.nim b/compiler/options.nim index 545dfb1d1..89fb66d5f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -183,7 +183,6 @@ type gcMarkAndSweep = "markAndSweep" gcHooks = "hooks" gcRefc = "refc" - gcV2 = "v2" gcGo = "go" # gcRefc and the GCs that follow it use a write barrier, # as far as usesWriteBarrier() is concerned diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 49fff41e9..c4f0db718 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -304,20 +304,10 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, elemAlign, newLen: int): PGenericS when not defined(boehmGC) and not defined(nogc) and not defined(gcMarkAndSweep) and not defined(gogc) and not defined(gcRegions): - when false: # deadcode: was used by `compileOption("gc", "v2")` + if ntfNoRefs notin extGetCellType(result).base.flags: for i in newLen..result.len-1: - let len0 = gch.tempStack.len forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waPush) - let len1 = gch.tempStack.len - for i in len0 ..< len1: - doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic) - gch.tempStack.len = len0 - else: - if ntfNoRefs notin extGetCellType(result).base.flags: - for i in newLen..result.len-1: - forAllChildrenAux(dataPointer(result, elemAlign, elemSize, i), - extGetCellType(result).base, waZctDecRef) + extGetCellType(result).base, waZctDecRef) # XXX: zeroing out the memory can still result in crashes if a wiped-out # cell is aliased by another pointer (ie proc parameter or a let variable). From 1e5dd9022b0d86d63d616431a0f9959ca4c10f40 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 11 Jun 2022 02:33:44 +0800 Subject: [PATCH 0831/1654] [js] add testcase for array indexDefect and remove todo (#19838) * remove unused opcSubstr * [js] add testcase for array indexDefect * Revert "remove unused opcSubstr" This reverts commit cb461f2545234d62c1e0b83318f3e5495c97de52. --- lib/system/jssys.nim | 4 ---- tests/js/tindexdefect.nim | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/js/tindexdefect.nim diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 00a4a8ab6..a31de0d86 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -731,10 +731,6 @@ proc parseFloatNative(a: string): float = `result` = Number(`a2`); """ -#[ -xxx how come code like this doesn't give IndexDefect ? -let z = s[10000] == 'a' -]# proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start: int): int {.compilerproc.} = var sign: bool var i = start diff --git a/tests/js/tindexdefect.nim b/tests/js/tindexdefect.nim new file mode 100644 index 000000000..37994ec2e --- /dev/null +++ b/tests/js/tindexdefect.nim @@ -0,0 +1,9 @@ +discard """ + outputsub: "unhandled exception: index 10000 not in 0 .. 0 [IndexDefect]" + exitcode: 1 + joinable: false +""" + +var s = ['a'] +let z = s[10000] == 'a' +echo z \ No newline at end of file From 6f4bacff67e2e219ef914e24d9f9aaf34a6d1eb1 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Fri, 10 Jun 2022 14:40:08 -0400 Subject: [PATCH 0832/1654] Extend and document compiler debugging utilities (#19841) * Add two debugutils procs that native debuggers can break on use to execute commands when code of interest is being compiled. * Add GDB and LLDB programs to disable and enable breakpoints and watchpoints when code of interest is being compiled. * Extend the `intern.rst` docs regarding debugging the compiler. Co-authored-by: quantimnot --- compiler/debugutils.nim | 16 +++ doc/intern.rst | 222 +++++++++++++++++++++++++++++++++------- doc/koch.rst | 3 + tools/compiler.gdb | 39 +++++++ tools/compiler.lldb | 40 ++++++++ 5 files changed, 281 insertions(+), 39 deletions(-) create mode 100644 tools/compiler.gdb create mode 100644 tools/compiler.lldb diff --git a/compiler/debugutils.nim b/compiler/debugutils.nim index d109d2121..adbb0517f 100644 --- a/compiler/debugutils.nim +++ b/compiler/debugutils.nim @@ -54,3 +54,19 @@ proc isCompilerDebug*(): bool = {.undef(nimCompilerDebug).} echo 'x' conf0.isDefined("nimCompilerDebug") + +proc enteringDebugSection*() {.exportc, dynlib.} = + ## Provides a way for native debuggers to enable breakpoints, watchpoints, etc + ## when code of interest is being compiled. + ## + ## Set your debugger to break on entering `nimCompilerIsEnteringDebugSection` + ## and then execute a desired command. + discard + +proc exitingDebugSection*() {.exportc, dynlib.} = + ## Provides a way for native debuggers to disable breakpoints, watchpoints, etc + ## when code of interest is no longer being compiled. + ## + ## Set your debugger to break on entering `exitingDebugSection` + ## and then execute a desired command. + discard diff --git a/doc/intern.rst b/doc/intern.rst index 4ca7eff20..9103c694c 100644 --- a/doc/intern.rst +++ b/doc/intern.rst @@ -79,55 +79,216 @@ Set the compilation timestamp with the `SOURCE_DATE_EPOCH` environment variable. koch boot # or `./build_all.sh` -Developing the compiler -======================= +Debugging the compiler +====================== -To create a new compiler for each run, use `koch temp`:cmd:\: + +Bisecting for regressions +------------------------- + +There are often times when there is a bug that is caused by a regression in the +compiler or stdlib. Bisecting the Nim repo commits is a usefull tool to identify +what commit introduced the regression. + +Even if it's not known whether a bug is caused by a regression, bisection can reduce +debugging time by ruling it out. If the bug is found to be a regression, then you +focus on the changes introduced by that one specific commit. + +`koch temp`:cmd: returns 125 as the exit code in case the compiler +compilation fails. This exit code tells `git bisect`:cmd: to skip the +current commit: .. code:: cmd - koch temp c test.nim + git bisect start bad-commit good-commit + git bisect run ./koch temp -r c test-source.nim -`koch temp`:cmd: creates a debug build of the compiler, which is useful -to create stacktraces for compiler debugging. +You can also bisect using custom options to build the compiler, for example if +you don't need a debug version of the compiler (which runs slower), you can replace +`./koch temp`:cmd: by explicit compilation command, see `Bootstrapping the compiler`_. -You can of course use GDB or Visual Studio to debug the -compiler (via `--debuginfo --lineDir:on`:option:). However, there -are also lots of procs that aid in debugging: +Building an instrumented compiler +--------------------------------- + +Considering that a useful method of debugging the compiler is inserting debug +logging, or changing code and then observing the outcome of a testcase, it is +fastest to build a compiler that is instrumented for debugging from an +existing release build. `koch temp`:cmd: provides a convenient method of doing +just that. + +By default running `koch temp`:cmd: will build a lean version of the compiler +with `-d:debug`:option: enabled. The compiler is written to `bin/nim_temp` by +default. A lean version of the compiler lacks JS and documentation generation. + +`bin/nim_temp` can be directly used to run testcases, or used with testament +with `testament --nim:bin/nim_temp r tests/category/tsometest`:cmd:. + +`koch temp`:cmd: will build the temporary compiler with the `-d:debug`:option: +enabled. Here are compiler options that are of interest for debugging: + +* `-d:debug`:option:\: enables `assert` statements and stacktraces and all + runtime checks +* `--opt:speed`:option:\: build with optimizations enabled +* `--debugger:native`:option:\: enables `--debuginfo --lineDir:on`:option: for using + a native debugger like GDB, LLDB or CDB +* `-d:nimDebug`:option: cause calls to `quit` to raise an assertion exception +* `-d:nimDebugUtils`:option:\: enables various debugging utilities; + see `compiler/debugutils` +* `-d:stacktraceMsgs -d:nimCompilerStacktraceHints`:option:\: adds some additional + stacktrace hints; see https://github.com/nim-lang/Nim/pull/13351 +* `-u:leanCompiler`:option:\: enable JS and doc generation + +Another method to build and run the compiler is directly through `koch`:cmd:\: + +.. code:: cmd + + koch temp [options] c test.nim + + # (will build with js support) + koch temp [options] js test.nim + + # (will build with doc support) + koch temp [options] doc test.nim + +Debug logging +------------- + +"Printf debugging" is still the most appropriate way to debug many problems +arising in compiler development. The typical usage of breakpoints to debug +the code is often less practical, because almost all of the code paths in the +compiler will be executed hundreds of times before a particular section of the +tested program is reached where the newly developed code must be activated. + +To work-around this problem, you'll typically introduce an if statement in the +compiler code detecting more precisely the conditions where the tested feature +is being used. One very common way to achieve this is to use the `mdbg` condition, +which will be true only in contexts, processing expressions and statements from +the currently compiled main module: .. code-block:: nim - # dealing with PNode: - echo renderTree(someNode) - debug(someNode) # some JSON representation + # inside some compiler module + if mdbg: + debug someAstNode - # dealing with PType: +Using the `isCompilerDebug`:nim: condition along with inserting some statements +into the testcase provides more granular logging: + +.. code-block:: nim + + # compilermodule.nim + if isCompilerDebug(): + debug someAstNode + + # testcase.nim + proc main = + {.define(nimCompilerDebug).} + let a = 2.5 * 3 + {.undef(nimCompilerDebug).} + +Logging can also be scoped to a specific filename as well. This will of course +match against every module with that name. + +.. code-block:: nim + + if `??`(conf, n.info, "module.nim"): + debug(n) + +The above examples also makes use of the `debug`:nim: proc, which is able to +print a human-readable form of an arbitrary AST tree. Other common ways to print +information about the internal compiler types include: + +.. code-block:: nim + + # pretty print PNode + + # pretty prints the Nim ast + echo renderTree(someNode) + + # pretty prints the Nim ast, but annotates symbol IDs + echo renderTree(someNode, {renderIds}) + + # pretty print ast as JSON + debug(someNode) + + # print as YAML + echo treeToYaml(config, someNode) + + + # pretty print PType + + # print type name echo typeToString(someType) + + # pretty print as JSON debug(someType) - # dealing with PSym: + # print as YAML + echo typeToYaml(config, someType) + + + # pretty print PSym + + # print the symbol's name echo symbol.name.s + + # pretty print as JSON debug(symbol) - # pretty prints the Nim ast, but annotates symbol IDs: - echo renderTree(someNode, {renderIds}) - if `??`(conf, n.info, "temp.nim"): - # only output when it comes from "temp.nim" - echo renderTree(n) - if `??`(conf, n.info, "temp.nim"): - # why does it process temp.nim here? - writeStackTrace() + # print as YAML + echo symToYaml(config, symbol) + + + # pretty print TLineInfo + lineInfoToStr(lineInfo) + + + # print the structure of any type + repr(someVar) + +Here are some other helpful utilities: + +.. code-block:: nim + + # how did execution reach this location? + writeStackTrace() These procs may not already be imported by the module you're editing. You can import them directly for debugging: .. code-block:: nim + from astalgo import debug from types import typeToString from renderer import renderTree from msgs import `??` +Native debugging +---------------- + +Stepping through the compiler with a native debugger is a very powerful tool to +both learn and debug it. However, there is still the need to constrain when +breakpoints are triggered. The same methods as in `Debug logging`_ can be applied +here when combined with calls to the debug helpers `enteringDebugSection()`:nim: +and `exitingDebugSection()`:nim:. + +#. Compile the temp compiler with `--debugger:native -d:nimDebugUtils`:option: +#. Set your desired breakpoints or watchpoints. +#. Configure your debugger: + * GDB: execute `source tools/compiler.gdb` at startup + * LLDB execute `command source tools/compiler.lldb` at startup +#. Use one of the scoping helpers like so: + +.. code-block:: nim + + if isCompilerDebug(): + enteringDebugSection() + else: + exitingDebugSection() + +A caveat of this method is that all breakpoints and watchpoints are enabled or +disabled. Also, due to a bug, only breakpoints can be constrained for LLDB. The compiler's architecture =========================== @@ -152,23 +313,6 @@ for the type definitions. The `macros `_ module contains many examples how the AST represents each syntactic structure. -Bisecting for regressions -========================= - -`koch temp`:cmd: returns 125 as the exit code in case the compiler -compilation fails. This exit code tells `git bisect`:cmd: to skip the -current commit: - -.. code:: cmd - - git bisect start bad-commit good-commit - git bisect run ./koch temp -r c test-source.nim - -You can also bisect using custom options to build the compiler, for example if -you don't need a debug version of the compiler (which runs slower), you can replace -`./koch temp`:cmd: by explicit compilation command, see `Bootstrapping the compiler`_. - - Runtimes ======== diff --git a/doc/koch.rst b/doc/koch.rst index b8d85dff4..91dd5d570 100644 --- a/doc/koch.rst +++ b/doc/koch.rst @@ -38,6 +38,9 @@ options: unless you are debugging the compiler. -d:nimUseLinenoise Use the linenoise library for interactive mode (not needed on Windows). +-d:leanCompiler Produce a compiler without JS codegen or + documentation generator in order to use less RAM + for bootstrapping. After compilation is finished you will hopefully end up with the nim compiler in the `bin` directory. You can add Nim's `bin` directory to diff --git a/tools/compiler.gdb b/tools/compiler.gdb new file mode 100644 index 000000000..c81f47152 --- /dev/null +++ b/tools/compiler.gdb @@ -0,0 +1,39 @@ +# create a breakpoint on `debugutils.enteringDebugSection` +define enable_enteringDebugSection + break -function enteringDebugSection + # run these commands once breakpoint enteringDebugSection is hit + command + # enable all breakpoints and watchpoints + enable + # continue execution + cont + end +end + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +define enable_exitingDebugSection + break -function exitingDebugSection + # run these commands once breakpoint exitingDebugSection is hit + command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # continue execution + cont + end +end + +# some commands can't be set until the process is running, so set an entry breakpoint +break -function NimMain +# run these commands once breakpoint NimMain is hit +command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # no longer need this breakpoint + delete -function NimMain + # continue execution + cont +end diff --git a/tools/compiler.lldb b/tools/compiler.lldb new file mode 100644 index 000000000..e0b375055 --- /dev/null +++ b/tools/compiler.lldb @@ -0,0 +1,40 @@ +# create a breakpoint on `debugutils.enteringDebugSection` named enteringDebugSection +breakpoint set -n 'enteringDebugSection' -N enteringDebugSection +# run these commands once breakpoint enteringDebugSection is hit +breakpoint command add enteringDebugSection + # enable all breakpoints + breakpoint enable + # enable all watchpoints + # watchpoint enable # FIXME: not currently working for unknown reason + # continue execution + continue +DONE + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +breakpoint set -n 'exitingDebugSection' -N exitingDebugSection +# run these commands once breakpoint exitingDebugSection is hit +breakpoint command add exitingDebugSection + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + breakpoint enable enteringDebugSection + # continue execution + continue +DONE + +# some commands can't be set until the process is running, so set an entry breakpoint +breakpoint set -n NimMain -N NimMain +# run these commands once breakpoint NimMain is hit +breakpoint command add NimMain + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + # enable the enteringDebugSection breakpoint though + breakpoint enable enteringDebugSection + # no longer need this breakpoint + breakpoint delete NimMain + # continue execution + continue +DONE From ac89eead8a40a1a8ee170a4650a1d47e203da14c Mon Sep 17 00:00:00 2001 From: Bung Date: Sat, 11 Jun 2022 02:40:34 +0800 Subject: [PATCH 0833/1654] test for #19046 (#19839) * test for #19046 * add threads flag --- tests/misc/t19046.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/misc/t19046.nim diff --git a/tests/misc/t19046.nim b/tests/misc/t19046.nim new file mode 100644 index 000000000..b3bfec7ae --- /dev/null +++ b/tests/misc/t19046.nim @@ -0,0 +1,19 @@ +discard """ + targets: "c cpp" + matrix: "--threads:on" + disabled: "win" + disabled: "osx" + action: compile +""" + +# bug #19046 + +import std/os + +var t: Thread[void] + +proc test = discard +proc main = + createThread(t, test) + pinToCpu(t, 1) +main() \ No newline at end of file From e2e663a14389500b07e2f6fcd5c231536bdf76a0 Mon Sep 17 00:00:00 2001 From: Carlo Capocasa Date: Sat, 11 Jun 2022 18:23:31 +0200 Subject: [PATCH 0834/1654] Friendlier error message with solution (#19880) * Add helpful suggestion, should always apply * mention var param limitation in async docs * Update compiler/lambdalifting.nim whoops thanks Co-authored-by: flywind <43030857+xflywind@users.noreply.github.com> Co-authored-by: flywind <43030857+xflywind@users.noreply.github.com> --- compiler/lambdalifting.nim | 3 ++- lib/pure/asyncdispatch.nim | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 4b7ca89cd..96edba8c8 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -288,7 +288,8 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) = if illegalCapture(s): localError(g.config, n.info, ("'$1' is of type <$2> which cannot be captured as it would violate memory" & - " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases") % + " safety, declared here: $3; using '-d:nimNoLentIterators' helps in some cases." & + " Consider using a which can be captured.") % [s.name.s, typeToString(s.typ), g.config$s.info]) elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags): localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" % diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 755f59ff7..0d406b271 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -98,6 +98,10 @@ ## `await`. The following section shows different ways that you can handle ## exceptions in async procs. ## +## .. caution:: +## Procedures marked {.async.} do not support mutable parameters such +## as `var int`. References such as `ref int` should be used instead. +## ## Handling Exceptions ## ------------------- ## @@ -192,6 +196,7 @@ ## ================ ## ## * The effect system (`raises: []`) does not work with async procedures. +## * Mutable parameters are not supported by async procedures. ## ## ## Multiple async backend support From 8fa2c0b532be9fdfd7682e5a8cc846b60bd4ea6c Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Mon, 13 Jun 2022 16:03:40 +1000 Subject: [PATCH 0835/1654] Pass headers and body correctly to FetchOptions (#19884) [backport] * Pass headers to FetchOptions Don't pass body if method is HttpGet or HttpHead * Syntax fixes * Restart CI --- lib/std/jsfetch.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/std/jsfetch.nim b/lib/std/jsfetch.nim index 034bb6332..7fe154325 100644 --- a/lib/std/jsfetch.nim +++ b/lib/std/jsfetch.nim @@ -90,8 +90,9 @@ func newfetchOptions*(metod: HttpMethod; body: cstring; headers: Headers = newHeaders()): FetchOptions = ## Constructor for `FetchOptions`. result = FetchOptions( - body: body, mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), - keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, + body: if metod notin {HttpHead, HttpGet}: body else: nil, + mode: cstring($mode), credentials: cstring($credentials), cache: cstring($cache), referrerPolicy: cstring($referrerPolicy), + keepalive: keepalive, redirect: cstring($redirect), referrer: referrer, integrity: integrity, headers: headers, metod: (case metod of HttpHead: "HEAD".cstring of HttpGet: "GET".cstring From 2f4900615ab0c42c9608910dd30a14be52455f91 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 13 Jun 2022 14:10:40 +0800 Subject: [PATCH 0836/1654] [cleanup] remove unnecessary procs in vm (#19888) remove unused procs --- compiler/vm.nim | 1 - compiler/vmgen.nim | 7 ++----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index bbcff77ba..1b11cb4f3 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2252,7 +2252,6 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg = else: var n = x if n.kind in {nkHiddenSubConv, nkHiddenStdConv}: n = n[1] - n = n.canonValue n.flags.incl nfIsRef n.typ = x.typ result = TFullReg(kind: rkNode, node: n) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index dadcc3865..2e7f6d8b0 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -439,14 +439,11 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) = c.gABC(n, opcAsgnInt, dest, tmp) freeTemp(c, tmp) -proc canonValue*(n: PNode): PNode = - result = n - proc rawGenLiteral(c: PCtx; n: PNode): int = result = c.constants.len #assert(n.kind != nkCall) n.flags.incl nfAllConst - c.constants.add n.canonValue + c.constants.add n internalAssert c.config, result < regBxMax proc sameConstant*(a, b: PNode): bool = @@ -1857,7 +1854,7 @@ proc genVarSection(c: PCtx; n: PNode) = else: let sa = getNullValue(s.typ, a.info, c.config) #if s.ast.isNil: getNullValue(s.typ, a.info) - #else: canonValue(s.ast) + #else: s.ast assert sa.kind != nkCall c.globals.add(sa) s.position = c.globals.len From ab477075860364db2690a0411c0b2c330532042f Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:01:40 +0800 Subject: [PATCH 0837/1654] [semfold] fix #19199; properly fold uint to float conversion (#19890) [backport] fix #19199; properly fold float conversion --- compiler/semfold.nim | 2 +- tests/vm/t19199.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/vm/t19199.nim diff --git a/compiler/semfold.nim b/compiler/semfold.nim index d2bbc63be..72c8abaa9 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -409,7 +409,7 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P rangeCheck(n, getInt(result), g) of tyFloat..tyFloat64: case srcTyp.kind - of tyInt..tyInt64, tyEnum, tyBool, tyChar: + of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyBool, tyChar: result = newFloatNodeT(toFloat64(getOrdValue(a)), n, g) else: result = a diff --git a/tests/vm/t19199.nim b/tests/vm/t19199.nim new file mode 100644 index 000000000..6ae48cb54 --- /dev/null +++ b/tests/vm/t19199.nim @@ -0,0 +1,6 @@ +# bug #19199 +proc mikasa(x: float) = doAssert x == 42 + +static: + mikasa 42.uint.float +mikasa 42.uint.float From a4fdaa88ccfa03bf30113ffc194284fcb45d3054 Mon Sep 17 00:00:00 2001 From: Ardek Romak Date: Mon, 13 Jun 2022 10:22:20 +0300 Subject: [PATCH 0838/1654] Correctly import libcrypto functions using dynlib (#19881) --- lib/wrappers/openssl.nim | 72 +++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 00ec160cb..1db3fc239 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -93,6 +93,9 @@ else: import dynlib +{.pragma: lcrypto, cdecl, dynlib: DLLUtilName, importc.} +{.pragma: lssl, cdecl, dynlib: DLLSSLName, importc.} + type SslStruct {.final, pure.} = object SslPtr* = ptr SslStruct @@ -102,7 +105,6 @@ type PSTACK* = SslPtr PX509* = SslPtr PX509_NAME* = SslPtr - PEVP_MD* = SslPtr PBIO_METHOD* = SslPtr BIO* = SslPtr EVP_PKEY* = SslPtr @@ -684,48 +686,48 @@ proc RSA_free*(rsa: PRSA) {.cdecl, dynlib: DLLUtilName, importc.} proc RSA_size*(rsa: PRSA): cint {.cdecl, dynlib: DLLUtilName, importc.} # sha types -proc EVP_md_null*(): EVP_MD {.cdecl, importc.} -proc EVP_md2*(): EVP_MD {.cdecl, importc.} -proc EVP_md4*(): EVP_MD {.cdecl, importc.} -proc EVP_md5*(): EVP_MD {.cdecl, importc.} -proc EVP_sha*(): EVP_MD {.cdecl, importc.} -proc EVP_sha1*(): EVP_MD {.cdecl, importc.} -proc EVP_dss*(): EVP_MD {.cdecl, importc.} -proc EVP_dss1*(): EVP_MD {.cdecl, importc.} -proc EVP_ecdsa*(): EVP_MD {.cdecl, importc.} -proc EVP_sha224*(): EVP_MD {.cdecl, importc.} -proc EVP_sha256*(): EVP_MD {.cdecl, importc.} -proc EVP_sha384*(): EVP_MD {.cdecl, importc.} -proc EVP_sha512*(): EVP_MD {.cdecl, importc.} -proc EVP_mdc2*(): EVP_MD {.cdecl, importc.} -proc EVP_ripemd160*(): EVP_MD {.cdecl, importc.} -proc EVP_whirlpool*(): EVP_MD {.cdecl, importc.} -proc EVP_MD_size*(md: EVP_MD): cint {.cdecl, importc.} +proc EVP_md_null*(): EVP_MD {.lcrypto.} +proc EVP_md2*(): EVP_MD {.lcrypto.} +proc EVP_md4*(): EVP_MD {.lcrypto.} +proc EVP_md5*(): EVP_MD {.lcrypto.} +proc EVP_sha*(): EVP_MD {.lcrypto.} +proc EVP_sha1*(): EVP_MD {.lcrypto.} +proc EVP_dss*(): EVP_MD {.lcrypto.} +proc EVP_dss1*(): EVP_MD {.lcrypto.} +proc EVP_ecdsa*(): EVP_MD {.lcrypto.} +proc EVP_sha224*(): EVP_MD {.lcrypto.} +proc EVP_sha256*(): EVP_MD {.lcrypto.} +proc EVP_sha384*(): EVP_MD {.lcrypto.} +proc EVP_sha512*(): EVP_MD {.lcrypto.} +proc EVP_mdc2*(): EVP_MD {.lcrypto.} +proc EVP_ripemd160*(): EVP_MD {.lcrypto.} +proc EVP_whirlpool*(): EVP_MD {.lcrypto.} +proc EVP_MD_size*(md: EVP_MD): cint {.lcrypto.} # hmac functions -proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.cdecl, importc.} +proc HMAC*(evp_md: EVP_MD; key: pointer; key_len: cint; d: cstring; n: csize_t; md: cstring; md_len: ptr cuint): cstring {.lcrypto.} # RSA key functions -proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.cdecl, importc.} -proc EVP_PKEY_free*(p: EVP_PKEY) {.cdecl, importc.} -proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.cdecl, importc.} -proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: PEVP_MD, engine: SslPtr = nil): cint {.cdecl, importc.} -proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.cdecl, importc.} -proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.cdecl, importc.} -proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.cdecl, importc.} -proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.cdecl, importc.} -proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.cdecl, importc.} -proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.cdecl, importc.} +proc PEM_read_bio_PrivateKey*(bp: BIO, x: ptr EVP_PKEY, cb: pointer, u: pointer): EVP_PKEY {.lcrypto.} +proc EVP_PKEY_free*(p: EVP_PKEY) {.lcrypto.} +proc EVP_DigestSignInit*(ctx: EVP_MD_CTX, pctx: ptr EVP_PKEY_CTX, typ: EVP_MD, e: ENGINE, pkey: EVP_PKEY): cint {.lcrypto.} +proc EVP_DigestInit_ex*(ctx: EVP_MD_CTX, typ: EVP_MD, engine: SslPtr = nil): cint {.lcrypto.} +proc EVP_DigestUpdate*(ctx: EVP_MD_CTX, data: pointer, len: cuint): cint {.lcrypto.} +proc EVP_DigestFinal_ex*(ctx: EVP_MD_CTX, buffer: pointer, size: ptr cuint): cint {.lcrypto.} +proc EVP_DigestSignFinal*(ctx: EVP_MD_CTX, data: pointer, len: ptr csize_t): cint {.lcrypto.} +proc EVP_PKEY_CTX_new*(pkey: EVP_PKEY, e: ENGINE): EVP_PKEY_CTX {.lcrypto.} +proc EVP_PKEY_CTX_free*(pkeyCtx: EVP_PKEY_CTX) {.lcrypto.} +proc EVP_PKEY_sign_init*(c: EVP_PKEY_CTX): cint {.lcrypto.} when defined(macosx) or defined(windows): - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc.} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc.} - proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc.} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.lcrypto.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.lcrypto.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.lcrypto.} else: # some times you will need this instead: - proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new".} - proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free".} - proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup".} + proc EVP_MD_CTX_create*(): EVP_MD_CTX {.cdecl, importc: "EVP_MD_CTX_new", dynlib: DLLUtilName.} + proc EVP_MD_CTX_destroy*(ctx: EVP_MD_CTX) {.cdecl, importc: "EVP_MD_CTX_free", dynlib: DLLUtilName.} + proc EVP_MD_CTX_cleanup*(ctx: EVP_MD_CTX): cint {.cdecl, importc: "EVP_MD_CTX_cleanup", dynlib: DLLUtilName.} # type From b41226001ce596fe520fd326a302c6f4e91c2504 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Mon, 13 Jun 2022 12:33:44 +0300 Subject: [PATCH 0839/1654] Initial implementation of nimsuggest v3 (#19826) * Initial implementation of nimsuggest v3 Rework `nimsuggest` to use caching to make usage of ide commands more efficient. Previously, all commands no matter what the state of the process is were causing clean build. In the context of Language Server Protocol(LSP) and lsp clients this was causing perf issues and overall instability. Overall, the goal of v3 is to fit to LSP Server needs - added two new commands: - `recompile` to do clean compilation - `changed` which can be used by the IDEs to notify that a particular file has been changed. The later can be utilized when using LSP file watches. - `globalSymbols` - searching global references - added `segfaults` dependency to allow fallback to clean build when incremental fails. I wish the error to be propagated to the client so we can work on fixing the incremental build failures (typically hitting pointer) - more efficient rebuild flow. ATM incremental rebuild is triggered when the command needs that(i. e. it is global) while the commands that work on the current source rebuild only it Things missing in this PR: - Documentation - Extensive unit testing. Although functional I still see this more as a POC that this approach can work. Next steps: - Implement `sug` request. - Rework/extend the protocol to allow better client/server communication. Ideally we will need push events, diagnostics should be restructored to allow per file notifications, etc. - implement v3 test suite. - better logging * Add tests for v3 and implement ideSug * Remove typeInstCache/procInstCache cleanup * Add ideChkFile command * Avoid contains call when adding symbol info * Remove log * Remove segfaults --- compiler/modulegraphs.nim | 82 ++++++++++--- compiler/options.nim | 13 +- compiler/passes.nim | 7 +- compiler/suggest.nim | 12 +- nimsuggest/nimsuggest.nim | 242 +++++++++++++++++++++++++++++++++++--- nimsuggest/tester.nim | 5 +- nimsuggest/tests/tv3.nim | 25 ++++ 7 files changed, 349 insertions(+), 37 deletions(-) create mode 100644 nimsuggest/tests/tv3.nim diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 147381910..8294d863e 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import intsets, tables, hashes, md5_old +import intsets, tables, hashes, md5_old, sequtils import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages import ic / [packed_ast, ic] @@ -83,6 +83,8 @@ type doStopCompile*: proc(): bool {.closure.} usageSym*: PSym # for nimsuggest owners*: seq[PSym] + suggestSymbols*: Table[FileIndex, seq[tuple[sym: PSym, info: TLineInfo]]] + suggestErrors*: Table[FileIndex, seq[Suggest]] methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization! systemModule*: PSym sysTypes*: array[TTypeKind, PType] @@ -385,9 +387,19 @@ when defined(nimfind): c.graph.onDefinitionResolveForward(c.graph, s, info) else: - template onUse*(info: TLineInfo; s: PSym) = discard - template onDef*(info: TLineInfo; s: PSym) = discard - template onDefResolveForward*(info: TLineInfo; s: PSym) = discard + when defined(nimsuggest): + template onUse*(info: TLineInfo; s: PSym) = discard + + template onDef*(info: TLineInfo; s: PSym) = + let c = getPContext() + if c.graph.config.suggestVersion == 3: + suggestSym(c.graph, info, s, c.graph.usageSym) + + template onDefResolveForward*(info: TLineInfo; s: PSym) = discard + else: + template onUse*(info: TLineInfo; s: PSym) = discard + template onDef*(info: TLineInfo; s: PSym) = discard + template onDefResolveForward*(info: TLineInfo; s: PSym) = discard proc stopCompile*(g: ModuleGraph): bool {.inline.} = result = g.doStopCompile != nil and g.doStopCompile() @@ -434,8 +446,7 @@ proc initOperators*(g: ModuleGraph): Operators = result.opNot = createMagic(g, "not", mNot) result.opContains = createMagic(g, "contains", mInSet) -proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = - result = ModuleGraph() +proc initModuleGraphFields(result: ModuleGraph) = # A module ID of -1 means that the symbol is not attached to a module at all, # but to the module graph: result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32) @@ -445,9 +456,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.ifaces = @[] result.importStack = @[] result.inclToMod = initTable[FileIndex, FileIndex]() - result.config = config - result.cache = cache result.owners = @[] + result.suggestSymbols = initTable[FileIndex, seq[tuple[sym: PSym, info: TLineInfo]]]() + result.suggestErrors = initTable[FileIndex, seq[Suggest]]() result.methods = @[] initStrTable(result.compilerprocs) initStrTable(result.exposed) @@ -461,6 +472,12 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result.operators = initOperators(result) result.emittedTypeInfo = initTable[string, FileIndex]() +proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = + result = ModuleGraph() + result.config = config + result.cache = cache + initModuleGraphFields(result) + proc resetAllModules*(g: ModuleGraph) = initStrTable(g.packageSyms) g.deps = initIntSet() @@ -472,6 +489,7 @@ proc resetAllModules*(g: ModuleGraph) = g.methods = @[] initStrTable(g.compilerprocs) initStrTable(g.exposed) + initModuleGraphFields(g) proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = if fileIdx.int32 >= 0: @@ -550,7 +568,19 @@ proc transitiveClosure(g: var IntSet; n: int) = proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) = let m = g.getModule fileIdx - if m != nil: incl m.flags, sfDirty + if m != nil: + g.suggestSymbols.del(fileIdx) + g.suggestErrors.del(fileIdx) + incl m.flags, sfDirty + +proc unmarkAllDirty*(g: ModuleGraph) = + for i in 0i32.. 100: + break + else: + myLog fmt "Discarding {cmd}" + +# v3 end when isMainModule: handleCmdLine(newIdentCache(), newConfigRef()) else: @@ -726,8 +939,9 @@ else: if self.loadConfigsAndProcessCmdLine(cache, conf, graph): mockCommand(graph) if gLogging: + log("Search paths:") for it in conf.searchPaths: - log(it.string) + log(" " & it.string) retval.doStopCompile = proc (): bool = false return NimSuggest(graph: retval, idle: 0, cachedMsgs: @[]) diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 1db33706a..fea0a8d45 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -252,7 +252,10 @@ proc runEpcTest(filename: string): int = for cmd in s.startup: if not runCmd(cmd, s.dest): quit "invalid command: " & cmd - let epccmd = s.cmd.replace("--tester", "--epc --v2 --log") + let epccmd = if s.cmd.contains("--v3"): + s.cmd.replace("--tester", "--epc --log") + else: + s.cmd.replace("--tester", "--epc --v2 --log") let cl = parseCmdLine(epccmd) var p = startProcess(command=cl[0], args=cl[1 .. ^1], options={poStdErrToStdOut, poUsePath, diff --git a/nimsuggest/tests/tv3.nim b/nimsuggest/tests/tv3.nim new file mode 100644 index 000000000..99caa987b --- /dev/null +++ b/nimsuggest/tests/tv3.nim @@ -0,0 +1,25 @@ +# tests v3 + +type + Foo* = ref object of RootObj + bar*: string + +proc test(f: Foo) = + echo f.ba#[!]#r + +discard """ +$nimsuggest --v3 --tester $file +>use $1 +def skField tv3.Foo.bar string $file 5 4 "" 100 +use skField tv3.Foo.bar string $file 8 9 "" 100 +>def $1 +def skField tv3.Foo.bar string $file 5 4 "" 100 +>outline $1 +outline skType tv3.Foo Foo $file 4 2 "" 100 +outline skField tv3.Foo.bar string $file 5 4 "" 100 +outline skProc tv3.test proc (f: Foo){.gcsafe, locks: 0.} $file 7 5 "" 100 +>sug $1 +sug skField bar string $file 5 4 "" 100 Prefix +>globalSymbols test +def skProc tv3.test proc (f: Foo){.gcsafe, locks: 0.} $file 7 5 "" 100 +""" From 251bdc1d5aed98fccb31677dda3c3ead9be2f6bf Mon Sep 17 00:00:00 2001 From: Tanguy Date: Tue, 14 Jun 2022 12:37:31 +0200 Subject: [PATCH 0840/1654] Windows: enable nimRawSetjmp by default [backport] (#19891) * Windows: enable nimRawSetjmp by default See #19197. The default setjmp can randomly segfault on windows * Attempt to disable the flag for bootstraping * Disable styleCheck for c_setjmp --- config/config.nims | 5 +++++ lib/system/ansi_c.nim | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/config/config.nims b/config/config.nims index 08d9d9f55..b50181eaf 100644 --- a/config/config.nims +++ b/config/config.nims @@ -20,5 +20,10 @@ when defined(nimStrictMode): switch("hintAsError", "ConvFromXtoItselfNotNeeded") # future work: XDeclaredButNotUsed +when defined(windows) and not defined(booting): + # Avoid some rare stack corruption while using exceptions with a SEH-enabled + # toolchain: https://github.com/nim-lang/Nim/pull/19197 + switch("define", "nimRawSetjmp") + switch("define", "nimVersion:" & NimVersion) switch("define", "nimPreviewDotLikeOps") diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 23fb9fdef..5d0ecc01c 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -116,6 +116,7 @@ elif defined(nimBuiltinSetjmp): proc c_builtin_setjmp(jmpb: ptr pointer): cint {. importc: "__builtin_setjmp", nodecl.} c_builtin_setjmp(unsafeAddr jmpb[0]) + elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): when defined(windows): # No `_longjmp()` on Windows. @@ -127,10 +128,16 @@ elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): # prone to stack corruption during unwinding, so we disable that by setting # it to NULL. # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} + proc c_setjmp*(jmpb: C_JmpBuf): cint = proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. header: "", importc: "_setjmp".} c_setjmp_win(jmpb, nil) + + when defined(nimHasStyleChecks): + {.pop.} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "_longjmp".} From 789b1bcbb60278b335a81df416be1d5a663d7e30 Mon Sep 17 00:00:00 2001 From: Don-Duong Quach Date: Tue, 14 Jun 2022 23:20:34 -0700 Subject: [PATCH 0841/1654] Fixes #19883 so genscript works with vcc (#19893) --- compiler/extccomp.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 515c7ba29..5c074d284 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -434,7 +434,13 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} = # really: Cross compilation from Linux to Linux for example is entirely # reasonable. # `optGenMapping` is included here for niminst. - result = conf.globalOptions * {optGenScript, optGenMapping} != {} + # We use absolute paths for vcc / cl, see issue #19883. + let options = + if conf.cCompiler == ccVcc: + {optGenMapping} + else: + {optGenScript, optGenMapping} + result = conf.globalOptions * options != {} proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string = result = conf.compileOptions @@ -921,7 +927,7 @@ proc callCCompiler*(conf: ConfigRef) = objfiles.add(' ') objfiles.add(quoteShell(objFile)) let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput - else: AbsoluteFile(conf.projectName) + else: AbsoluteFile(conf.outFile) linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true) extraCmds = getExtraCmds(conf, mainOutput) From e7e8f437c4f95f4f5d038cdddf5036462733288a Mon Sep 17 00:00:00 2001 From: JJ <35242550+j-james@users.noreply.github.com> Date: Wed, 15 Jun 2022 06:40:56 -0700 Subject: [PATCH 0842/1654] Keep the doc sidebar on the screen while scrolling (#19851) * [docgen] Group sidebar sections into
              (open by default) * [docgen] Consistent indentation in generated HTML (this is a boon for working on docgen's html/css output) * [docgen] Move Source/Edit buttons inside main div This makes styling the documentation significantly easier. * [docgen] Somewhat consistent CSS formatting * [docgen] Keep the sidebar onscreen while scrolling * [docgen] Tweak CSS for the sticky sidebar * [docgen] search type=text ==> type=search * [docgen] Update expected doc output * [docgen] Fix Group by Type sidebar placement bug * [docgen] Curse you, whitespace (fix tests) * [docgen] Fix rst2html tests Co-authored-by: sandytypical <43030857+xflywind@users.noreply.github.com> --- config/nimdoc.cfg | 216 ++- doc/nimdoc.css | 164 +- nimdoc/rst2html/expected/rst_examples.html | 85 +- .../expected/index.html | 122 +- .../expected/theindex.html | 24 +- nimdoc/testproject/expected/nimdoc.out.css | 164 +- .../expected/subdir/subdir_b/utils.html | 758 ++++----- nimdoc/testproject/expected/testproject.html | 1454 ++++++++--------- nimdoc/testproject/expected/theindex.html | 24 +- 9 files changed, 1401 insertions(+), 1610 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 4efa1f637..725f9e0a5 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -9,25 +9,28 @@ split.item.toc = "20" doc.section = """
              -

              $sectionTitle

              -
              -$content -
              +

              $sectionTitle

              +
              + $content +
              +
              """ doc.section.toc = """
            • - $sectionTitle -
                - $content -
              +
              + $sectionTitle +
                + $content +
              +
              • """ doc.section.toc2 = """ -
                $plainName - $content -
              +
                $plainName + $content +
              """ # Chunk of HTML emitted for each entry in the HTML table of contents. @@ -47,12 +50,12 @@ doc.section.toc2 = """ doc.item = """
              -
              $header
              -
              -$deprecationMsg -$desc -$seeSrc -
              + 
              $header
              +
              + $deprecationMsg + $desc + $seeSrc +
              """ @@ -61,9 +64,8 @@ $seeSrc # * $overloadGroupName - the anchor for this whole group # * $content - string containing `doc.item`s themselves doc.item2 = """ -
              -$content + $content
              """ @@ -73,18 +75,14 @@ $content # This is used for TOC items which are not overloadable (e.g. types). # `$header_plain` would be too verbose here, so we use $name. doc.item.toc = """ -
            • $name
              • +
            • $name
              • """ # This is used for TOC items which are grouped by the same name (e.g. procs). doc.item.tocTable = """ -
            • $header_plain
              • +
            • $header_plain
              • """ - - # HTML rendered for doc.item's seeSrc variable. Note that this will render to # the empty string if you don't pass anything through --git.url. Available # substitutaion variables here are: @@ -94,32 +92,31 @@ doc.item.tocTable = """ # * $line: line of the item in the original source file. # * $url: whatever you did pass through the --git.url switch (which also # gets variables path/line replaced!) -doc.item.seesrc = """  Source -  Edit +doc.item.seesrc = """ +Source   +Edit   """ doc.deprecationmsg = """ -
              - $label $message -
              +
              + $label $message +
              """ doc.toc = """
                -$content + $content
              """ doc.body_toc_groupsection = """ -
              - Group by: - -
              +
              + Group by: + +
              """ @if boot: @@ -130,36 +127,36 @@ doc.body_toc_groupsection = """ doc.body_toc_group = """
              -
              - -     Dark Mode +
              + +     Dark Mode +
              + +
              + Search: +
              + $body_toc_groupsection +
              + $tableofcontents
              - -
              - Search: -
              - $body_toc_groupsection - $tableofcontents -
              - $seeSrc
              -
              - $deprecationMsg -

              $moduledesc

              - $content + $seeSrc + $deprecationMsg +

              $moduledesc

              + $content
              """ @@ -169,39 +166,36 @@ doc.body_toc_group = """ doc.body_toc_group = """
              -
              - -     Dark Mode +
              + +     Dark Mode +
              + +
              + Search: +
              +
              + Group by: + +
              +
              + $tableofcontents
              - -
              - Search: -
              -
              - Group by: - -
              - $tableofcontents -
              - $seeSrc
              -
              - $deprecationMsg -

              $moduledesc

              - $content + $seeSrc + $deprecationMsg +

              $moduledesc

              + $content
              """ @@ -220,14 +214,13 @@ doc.listing_end = "" # * $analytics: Google analytics location, includes - -
              -
              -

              $title

              $subtitle - $content -
              +
              +
              +

              $title

              $subtitle + $content
              -
              +
              Made with Nim. Generated: $date $time UTC
              -
              -$analytics + $analytics """ diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 0014cf196..e72c4a213 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -159,24 +159,28 @@ body { padding: 0; box-sizing: border-box; } -.column, -.columns { +.column, .columns { width: 100%; float: left; box-sizing: border-box; - margin-left: 1%; -} + margin-left: 1%; } -.column:first-child, -.columns:first-child { +.column:first-child, .columns:first-child { margin-left: 0; } +.container .row { + display: flex; } + .three.columns { - width: 22%; -} + width: 25.0%; + height: 100vh; + position: sticky; + top: 0px; + overflow-y: auto; } .nine.columns { - width: 77.0%; } + width: 75.0%; + padding-left: 1.5em; } .twelve.columns { width: 100%; @@ -269,25 +273,26 @@ a.nimdoc { a.toc-backref { text-decoration: none; - color: var(--text); } + color: var(--text); +} a.link-seesrc { color: #607c9f; font-size: 0.9em; - font-style: italic; } + font-style: italic; +} -a:hover, -a:focus { +a:hover, a:focus { color: var(--anchor-focus); - text-decoration: underline; } + text-decoration: underline; +} a:hover span.Identifier { color: var(--anchor); } -sub, -sup { +sub, sup { position: relative; font-size: 75%; line-height: 0; @@ -314,8 +319,7 @@ img { background: transparent !important; box-shadow: none !important; } - a, - a:visited { + a, a:visited { text-decoration: underline; } a[href]:after { @@ -329,16 +333,14 @@ img { a[href^="#"]:after { content: ""; } - pre, - blockquote { + pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } thead { display: table-header-group; } - tr, - img { + tr, img { page-break-inside: avoid; } img { @@ -353,22 +355,18 @@ img { h1.title { page-break-before: avoid; } - p, - h2, - h3 { + p, h2, h3 { orphans: 3; widows: 3; } - h2, - h3 { + h2, h3 { page-break-after: avoid; } } p { margin-top: 0.5em; - margin-bottom: 0.5em; -} + margin-bottom: 0.5em; } small { font-size: 85%; } @@ -376,8 +374,7 @@ small { strong { font-weight: 600; font-size: 0.95em; - color: var(--strong); -} + color: var(--strong); } em { font-style: italic; } @@ -398,8 +395,7 @@ h1.title { text-align: center; font-weight: 900; margin-top: 0.75em; - margin-bottom: 0em; -} + margin-bottom: 0em; } h2 { font-size: 1.3em; @@ -426,36 +422,29 @@ h6 { font-size: 1.1em; } -ul, -ol { +ul, ol { padding: 0; margin-top: 0.5em; margin-left: 0.75em; } -ul ul, -ul ol, -ol ol, -ol ul { +ul ul, ul ol, ol ol, ol ul { margin-bottom: 0; margin-left: 1.25em; } ul.simple > li { - list-style-type: circle; -} + list-style-type: circle; } ul.simple-boot li { - list-style-type: none; - margin-left: 0em; - margin-bottom: 0.5em; -} + list-style-type: none; + margin-left: 0em; + margin-bottom: 0.5em; } ol.simple > li, ul.simple > li { margin-bottom: 0.2em; margin-left: 0.4em } ul.simple.simple-toc > li { - margin-top: 1em; -} + margin-top: 1em; } ul.simple-toc { list-style: none; @@ -464,8 +453,7 @@ ul.simple-toc { margin-top: 1em; } ul.simple-toc > li { - list-style-type: none; -} + list-style-type: none; } ul.simple-toc-section { list-style-type: circle; @@ -475,12 +463,10 @@ ul.simple-toc-section { ul.nested-toc-section { list-style-type: circle; margin-left: -0.75em; - color: var(--text); -} + color: var(--text); } ul.nested-toc-section > li { - margin-left: 1.25em; -} + margin-left: 1.25em; } ol.arabic { @@ -527,7 +513,8 @@ hr.footnote { margin-top: 0.15em; } div.footnote-group { - margin-left: 1em; } + margin-left: 1em; +} div.footnote-label { display: inline-block; min-width: 1.7em; @@ -611,7 +598,7 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; + border-radius: 6px; } .copyToClipBoardBtn { @@ -629,7 +616,7 @@ pre { .copyToClipBoard:hover .copyToClipBoardBtn { visibility: visible; -} +} .pre-scrollable { max-height: 340px; @@ -694,8 +681,8 @@ table th { font-weight: bold; } table th.docinfo-name { - background-color: transparent; - text-align: right; + background-color: transparent; + text-align: right; } table tr:hover { @@ -712,31 +699,31 @@ table.borderless td, table.borderless th { padding: 0 0.5em 0 0 !important; } .admonition { - padding: 0.3em; - background-color: var(--secondary-background); - border-left: 0.4em solid #7f7f84; - margin-bottom: 0.5em; - -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + padding: 0.3em; + background-color: var(--secondary-background); + border-left: 0.4em solid #7f7f84; + margin-bottom: 0.5em; + -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); } .admonition-info { - border-color: var(--info-background); + border-color: var(--info-background); } .admonition-info-text { - color: var(--info-background); + color: var(--info-background); } .admonition-warning { - border-color: var(--warning-background); + border-color: var(--warning-background); } .admonition-warning-text { - color: var(--warning-background); + color: var(--warning-background); } .admonition-error { - border-color: var(--error-background); + border-color: var(--error-background); } .admonition-error-text { - color: var(--error-background); + color: var(--error-background); } .first { @@ -770,8 +757,7 @@ div.footer, div.header { font-size: smaller; } div.footer { - padding-top: 5em; -} + padding-top: 5em; } div.line-block { display: block; @@ -790,17 +776,14 @@ div.search_results { background-color: var(--third-background); margin: 3em; padding: 1em; - border: 1px solid #4d4d4d; -} + border: 1px solid #4d4d4d; } div#global-links ul { margin-left: 0; - list-style-type: none; -} + list-style-type: none; } div#global-links > simple-boot { - margin-left: 3em; -} + margin-left: 3em; } hr.docutils { width: 75%; } @@ -980,8 +963,7 @@ span.Directive { span.option { font-weight: bold; font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; - color: var(--option); -} + color: var(--option); } span.Prompt { font-weight: bold; @@ -997,11 +979,10 @@ span.program { text-decoration: underline; text-decoration-color: var(--hint); text-decoration-thickness: 0.05em; - text-underline-offset: 0.15em; -} + text-underline-offset: 0.15em; } -span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, -span.Other { +span.Command, span.Rule, span.Hyperlink, +span.Label, span.Reference, span.Other { color: var(--other); } /* Pop type, const, proc, and iterator defs in nim def blocks */ @@ -1039,17 +1020,14 @@ span.pragmadots { border-radius: 4px; margin: 0 2px; cursor: pointer; - font-size: 0.8em; -} + font-size: 0.8em; } span.pragmadots:hover { - background-color: var(--hint); -} + background-color: var(--hint); } + span.pragmawrap { - display: none; -} + display: none; } span.attachedType { display: none; - visibility: hidden; -} + visibility: hidden; } diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 2b5218d9f..532917055 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -1,12 +1,11 @@ - + - + - +Not a Nim Manual @@ -17,45 +16,42 @@ -Not a Nim Manual + - -
              -
              -

              Not a Nim Manual

              -
              +
              +
              +

              Not a Nim Manual

              +
              -
              - -     Dark Mode -
              - -
              - Search: -
              -
              - Group by: - -
              -
              -
              -
              - -

              + + +

              Authors:Andreas Rumpf, Zahary Karadjov
              Authors:Andreas Rumpf, Zahary Karadjov
              Version:|nimversion|

              "Complexity" seems to be a lot like "energy": you can transfer it from the end-user to one/some of the other players, but the total amount seems to remain pretty much constant for a given task. -- Ran

              @@ -262,19 +257,17 @@ stmt = IND{>} stmt ^+ IND{=} DED # list of statements F2 without pipe

              not in table

              - +
              -
              -
              +
              Made with Nim. Generated: 1970-01-02 03:46:40 UTC
              -
              - + diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index c6a116bd6..0c0dc2268 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -1,12 +1,11 @@ - + - + - +nimdoc/test_out_index_dot_html/foo @@ -17,94 +16,89 @@ -nimdoc/test_out_index_dot_html/foo + - -
              -
              -

              nimdoc/test_out_index_dot_html/foo

              -
              +
              +
              +

              nimdoc/test_out_index_dot_html/foo

              +
              -
              - -     Dark Mode -
              - -
              - Search: -
              -
              - Group by: - -
              -
                -
              • - Procs -
                  +
                  + +     Dark Mode +
                  + +
                  + Search: +
                  +
                  + Group by: + +
                  +
                  + +
                +
              -
              -
              - -

              -
              -

              Procs

              -
              - -
              -
              -
              proc foo() {....raises: [], tags: [].}
              -
              - -I do foo - -
              + + +

              +
              +

              Procs

              +
              +
              +
              + 
              proc foo() {....raises: [], tags: [].}
              +
              + + I do foo + +
              -
              +
              +
              -
              -
              +
              Made with Nim. Generated: 1970-01-02 03:46:40 UTC
              -
              - + diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index 8ee62a330..00c81189e 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -1,12 +1,11 @@ - + - + - +Index @@ -17,31 +16,28 @@ -Index + - -
              -
              -

              Index

              - Modules: index.

              API symbols

              +
              +
              +

              Index

              + Modules: index.

              API symbols

              foo:
              -
              -
              +
              Made with Nim. Generated: 1970-01-02 03:46:40 UTC
              -
              - + diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 0014cf196..e72c4a213 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -159,24 +159,28 @@ body { padding: 0; box-sizing: border-box; } -.column, -.columns { +.column, .columns { width: 100%; float: left; box-sizing: border-box; - margin-left: 1%; -} + margin-left: 1%; } -.column:first-child, -.columns:first-child { +.column:first-child, .columns:first-child { margin-left: 0; } +.container .row { + display: flex; } + .three.columns { - width: 22%; -} + width: 25.0%; + height: 100vh; + position: sticky; + top: 0px; + overflow-y: auto; } .nine.columns { - width: 77.0%; } + width: 75.0%; + padding-left: 1.5em; } .twelve.columns { width: 100%; @@ -269,25 +273,26 @@ a.nimdoc { a.toc-backref { text-decoration: none; - color: var(--text); } + color: var(--text); +} a.link-seesrc { color: #607c9f; font-size: 0.9em; - font-style: italic; } + font-style: italic; +} -a:hover, -a:focus { +a:hover, a:focus { color: var(--anchor-focus); - text-decoration: underline; } + text-decoration: underline; +} a:hover span.Identifier { color: var(--anchor); } -sub, -sup { +sub, sup { position: relative; font-size: 75%; line-height: 0; @@ -314,8 +319,7 @@ img { background: transparent !important; box-shadow: none !important; } - a, - a:visited { + a, a:visited { text-decoration: underline; } a[href]:after { @@ -329,16 +333,14 @@ img { a[href^="#"]:after { content: ""; } - pre, - blockquote { + pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } thead { display: table-header-group; } - tr, - img { + tr, img { page-break-inside: avoid; } img { @@ -353,22 +355,18 @@ img { h1.title { page-break-before: avoid; } - p, - h2, - h3 { + p, h2, h3 { orphans: 3; widows: 3; } - h2, - h3 { + h2, h3 { page-break-after: avoid; } } p { margin-top: 0.5em; - margin-bottom: 0.5em; -} + margin-bottom: 0.5em; } small { font-size: 85%; } @@ -376,8 +374,7 @@ small { strong { font-weight: 600; font-size: 0.95em; - color: var(--strong); -} + color: var(--strong); } em { font-style: italic; } @@ -398,8 +395,7 @@ h1.title { text-align: center; font-weight: 900; margin-top: 0.75em; - margin-bottom: 0em; -} + margin-bottom: 0em; } h2 { font-size: 1.3em; @@ -426,36 +422,29 @@ h6 { font-size: 1.1em; } -ul, -ol { +ul, ol { padding: 0; margin-top: 0.5em; margin-left: 0.75em; } -ul ul, -ul ol, -ol ol, -ol ul { +ul ul, ul ol, ol ol, ol ul { margin-bottom: 0; margin-left: 1.25em; } ul.simple > li { - list-style-type: circle; -} + list-style-type: circle; } ul.simple-boot li { - list-style-type: none; - margin-left: 0em; - margin-bottom: 0.5em; -} + list-style-type: none; + margin-left: 0em; + margin-bottom: 0.5em; } ol.simple > li, ul.simple > li { margin-bottom: 0.2em; margin-left: 0.4em } ul.simple.simple-toc > li { - margin-top: 1em; -} + margin-top: 1em; } ul.simple-toc { list-style: none; @@ -464,8 +453,7 @@ ul.simple-toc { margin-top: 1em; } ul.simple-toc > li { - list-style-type: none; -} + list-style-type: none; } ul.simple-toc-section { list-style-type: circle; @@ -475,12 +463,10 @@ ul.simple-toc-section { ul.nested-toc-section { list-style-type: circle; margin-left: -0.75em; - color: var(--text); -} + color: var(--text); } ul.nested-toc-section > li { - margin-left: 1.25em; -} + margin-left: 1.25em; } ol.arabic { @@ -527,7 +513,8 @@ hr.footnote { margin-top: 0.15em; } div.footnote-group { - margin-left: 1em; } + margin-left: 1em; +} div.footnote-label { display: inline-block; min-width: 1.7em; @@ -611,7 +598,7 @@ pre { border: 1px solid var(--border); -webkit-border-radius: 6px; -moz-border-radius: 6px; - border-radius: 6px; + border-radius: 6px; } .copyToClipBoardBtn { @@ -629,7 +616,7 @@ pre { .copyToClipBoard:hover .copyToClipBoardBtn { visibility: visible; -} +} .pre-scrollable { max-height: 340px; @@ -694,8 +681,8 @@ table th { font-weight: bold; } table th.docinfo-name { - background-color: transparent; - text-align: right; + background-color: transparent; + text-align: right; } table tr:hover { @@ -712,31 +699,31 @@ table.borderless td, table.borderless th { padding: 0 0.5em 0 0 !important; } .admonition { - padding: 0.3em; - background-color: var(--secondary-background); - border-left: 0.4em solid #7f7f84; - margin-bottom: 0.5em; - -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); - box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + padding: 0.3em; + background-color: var(--secondary-background); + border-left: 0.4em solid #7f7f84; + margin-bottom: 0.5em; + -webkit-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); + box-shadow: 0 5px 8px -6px rgba(0,0,0,.2); } .admonition-info { - border-color: var(--info-background); + border-color: var(--info-background); } .admonition-info-text { - color: var(--info-background); + color: var(--info-background); } .admonition-warning { - border-color: var(--warning-background); + border-color: var(--warning-background); } .admonition-warning-text { - color: var(--warning-background); + color: var(--warning-background); } .admonition-error { - border-color: var(--error-background); + border-color: var(--error-background); } .admonition-error-text { - color: var(--error-background); + color: var(--error-background); } .first { @@ -770,8 +757,7 @@ div.footer, div.header { font-size: smaller; } div.footer { - padding-top: 5em; -} + padding-top: 5em; } div.line-block { display: block; @@ -790,17 +776,14 @@ div.search_results { background-color: var(--third-background); margin: 3em; padding: 1em; - border: 1px solid #4d4d4d; -} + border: 1px solid #4d4d4d; } div#global-links ul { margin-left: 0; - list-style-type: none; -} + list-style-type: none; } div#global-links > simple-boot { - margin-left: 3em; -} + margin-left: 3em; } hr.docutils { width: 75%; } @@ -980,8 +963,7 @@ span.Directive { span.option { font-weight: bold; font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace; - color: var(--option); -} + color: var(--option); } span.Prompt { font-weight: bold; @@ -997,11 +979,10 @@ span.program { text-decoration: underline; text-decoration-color: var(--hint); text-decoration-thickness: 0.05em; - text-underline-offset: 0.15em; -} + text-underline-offset: 0.15em; } -span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference, -span.Other { +span.Command, span.Rule, span.Hyperlink, +span.Label, span.Reference, span.Other { color: var(--other); } /* Pop type, const, proc, and iterator defs in nim def blocks */ @@ -1039,17 +1020,14 @@ span.pragmadots { border-radius: 4px; margin: 0 2px; cursor: pointer; - font-size: 0.8em; -} + font-size: 0.8em; } span.pragmadots:hover { - background-color: var(--hint); -} + background-color: var(--hint); } + span.pragmawrap { - display: none; -} + display: none; } span.attachedType { display: none; - visibility: hidden; -} + visibility: hidden; } diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index f94da7f40..d5a3b84c7 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -1,12 +1,11 @@ - + - + - +subdir/subdir_b/utils @@ -17,215 +16,189 @@ -subdir/subdir_b/utils + - -
              -
              -

              subdir/subdir_b/utils

              -
              +
              +
              +

              subdir/subdir_b/utils

              +
              -
              - -     Dark Mode -
              - -
              - Search: -
              -
              - Group by: - -
              - +
            • - Templates - +
            +
          -
          -
          - -

          This is a description of the utils module.

          + + +

          This is a description of the utils module.

          Links work:

          • other module: iterators (not in this dir, just an example)
          • internal: fn2(x)
            • @@ -256,377 +229,356 @@

            Ref. type like G and type G and G[T] and type G*[T].

            Group ref. with capital letters works: fN11 or fn11

            Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

            -
            -

            Types

            -
            -
            -
            G[T] = object
+    
            
+  
            Types
            
+  
            
+    
            
+  
            G[T] = object
   val: T
 
            
-
            
-
-
-
-
            
+  
            
+    
+    
+    
+  
            
 
            
 
            
-
            SomeType = enum
+  
            SomeType = enum
   enumValueA, enumValueB, enumValueC
            
-
            
-
-
-
-
            
+  
            
+    
+    
+    
+  
            
 
            
 
-
            
+
            +
            -

            Procs

            -
            - -
            -
            -
            proc `$`[T](a: G[T]): string
            -
            - - - -
            +

            Procs

            +
            +
            +
            + 
            proc `$`[T](a: G[T]): string
            +
            + + + +
            -
            proc `$`[T](a: ref SomeType): string
            -
            - - - -
            + 
            proc `$`[T](a: ref SomeType): string
            +
            + + + +
            -
            -
            -
            func `'big`(a: string): SomeType {....raises: [], tags: [].}
            -
            - - - -
            +
            + 
            func `'big`(a: string): SomeType {....raises: [], tags: [].}
            +
            + + + +
            -
            -
            -
            proc `[]`[T](x: G[T]): T
            -
            - - - -
            +
            + 
            proc `[]`[T](x: G[T]): T
            +
            + + + +
            -
            -
            -
            proc `[]=`[T](a: var G[T]; index: int; value: T)
            -
            - - - -
            +
            + 
            proc `[]=`[T](a: var G[T]; index: int; value: T)
            +
            + + + +
            -
            -
            -
            proc binarySearch[T, K](a: openArray[T]; key: K;
+  
            
+  
            proc binarySearch[T, K](a: openArray[T]; key: K;
                         cmp: proc (x: T; y: K): int {.closure.}): int
            
-
            
-
-
-
-
            
+  
            
+    
+    
+    
+
            -
            -
            -
            proc f(x: G[int]) {....raises: [], tags: [].}
            -
            - -There is also variant f(G[string]) - -
            +
            + 
            proc f(x: G[int]) {....raises: [], tags: [].}
            +
            + + There is also variant f(G[string]) + +
            -
            proc f(x: G[string]) {....raises: [], tags: [].}
            -
            - -See also f(G[int]). - -
            + 
            proc f(x: G[string]) {....raises: [], tags: [].}
            +
            + + See also f(G[int]). + +
            -
            -
            -
            proc fn[T; U, V: SomeFloat]()
            -
            - - - -
            +
            + 
            proc fn[T; U, V: SomeFloat]()
            +
            + + + +
            -
            -
            -
            proc fn2() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn2() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            proc fn2(x: int) {....raises: [], tags: [].}
            -
            - -fn2 comment - -
            + 
            proc fn2(x: int) {....raises: [], tags: [].}
            +
            + + fn2 comment + +
            -
            proc fn2(x: int; y: float) {....raises: [], tags: [].}
            -
            - - - -
            + 
            proc fn2(x: int; y: float) {....raises: [], tags: [].}
            +
            + + + +
            -
            -
            -
            proc fn3(): auto {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn3(): auto {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn4(): auto {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn4(): auto {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn5() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn5() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn6() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn6() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn7() {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn7() {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            proc fn8(): auto {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            proc fn8(): auto {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            func fn9(a: int): int {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            func fn9(a: int): int {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            func fn10(a: int): int {....raises: [], tags: [].}
            -
            - -comment - -
            +
            + 
            func fn10(a: int): int {....raises: [], tags: [].}
            +
            + + comment + +
            -
            -
            -
            func fN11() {....raises: [], tags: [].}
            -
            - - - -
            +
            + 
            func fN11() {....raises: [], tags: [].}
            +
            + + + +
            -
            func fN11(x: int) {....raises: [], tags: [].}
            -
            - - - -
            + 
            func fN11(x: int) {....raises: [], tags: [].}
            +
            + + + +
            -
            -
            -
            proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
            -
            - - - -
            +
            + 
            proc funWithGenerics[T, U: SomeFloat](a: T; b: U)
            +
            + + + +
            -
            -
            -
            proc someType(): SomeType {....raises: [], tags: [].}
            -
            - -constructor. - -
            +
            + 
            proc someType(): SomeType {....raises: [], tags: [].}
            +
            + + constructor. + +
            -
            +
            +
            -

            Iterators

            -
            - -
            -
            -
            iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
            -
            - - - -
            +

            Iterators

            +
            +
            +
            + 
            iterator fooBar(a: seq[SomeType]): int {....raises: [], tags: [].}
            +
            + + + +
            -
            +
            +
            -

            Templates

            -
            - -
            -
            -
            template aEnum(): untyped
            -
            - - - -
            +

            Templates

            +
            +
            +
            + 
            template aEnum(): untyped
            +
            + + + +
            -
            -
            -
            template bEnum(): untyped
            -
            - - - -
            +
            + 
            template bEnum(): untyped
            +
            + + + +
            -
            -
            -
            template fromUtilsGen(): untyped
            -
            - -should be shown in utils.html only +
            + 
            template fromUtilsGen(): untyped
            +
            + + should be shown in utils.html only

            Example:

            discard "should be in utils.html only, not in module that calls fromUtilsGen"
            ditto - -
            + +
            -
            +
            +
          -
          -
          +
          Made with Nim. Generated: 1970-01-02 03:46:40 UTC
          - - + diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index cba9391af..49f24d204 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -1,12 +1,11 @@ - + - + - +testproject @@ -17,372 +16,327 @@ -testproject + - -
          -
          -

          testproject

          -
          +
          +
          +

          testproject

          +
          -
          - -     Dark Mode -
          - -
          - Search: -
          -
          - Group by: - -
          - +
        • - Methods - +
        +
      • - Iterators - +
      +
    • - Macros - +
    +
  • - Templates - + +
    • -
    -
    - -

    This is the top level module. + + +

    This is the top level module.

    Example:

    import testproject
 import subdir / subdir_b / utils
@@ -396,133 +350,136 @@
 
    Example:
    
 
    import testproject
 discard "in top3"
    top3 after
    
-  
+    
 
    
-
    Types
    
-
    
-
    
-
    A {.inject.} = enum
+  
    Types
    
+  
    
+    
    
+  
    A {.inject.} = enum
   aA
    
-
    
-
-The enum A.
-
-
    
+  
    
+    
+    The enum A.
+    
+  
    
 
    
 
    
-
    B {.inject.} = enum
+  
    B {.inject.} = enum
   bB
    
-
    
-
-The enum B.
-
-
    
+  
    
+    
+    The enum B.
+    
+  
    
 
    
 
    
-
    Foo = enum
+  
    Foo = enum
   enumValueA2
    
-
    
-
-
-
-
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    FooBuzz {....deprecated: "FooBuzz msg".} = int
    
-
    
-  
    
-    Deprecated: FooBuzz msg
-  
    
+  
    FooBuzz {....deprecated: "FooBuzz msg".} = int
    
+  
    
+    
    
+  Deprecated: FooBuzz msg
+
    
 
-
-
-
    
+    
+    
+  
 
    
 
    
-
    Shapes = enum
+  
    Shapes = enum
   Circle,                   ## A circle
   Triangle,                 ## A three-sided shape
   Rectangle                  ## A four-sided shape
    
-
    
-
-Some shapes.
-
-
    
+  
    
+    
+    Some shapes.
+    
+  
    
 
    
 
-
    
+  
    
+
    
 
    
-
    Vars
    
-
    
-
    
-
    aVariable: array[1, int]
    
-
    
-
-
-
-
    
+  
    Vars
    
+  
    
+    
    
+  
    aVariable: array[1, int]
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    someVariable: bool
    
-
    
-
-This should be visible.
-
-
    
+  
    someVariable: bool
    
+  
    
+    
+    This should be visible.
+    
+  
    
 
    
 
-
    
+  
    
+
    
 
    
-
    Consts
    
-
    
-
    
-
    C_A = 0x7FF0000000000000'f64
    
-
    
-
-
-
-
    
+  
    Consts
    
+  
    
+    
    
+  
    C_A = 0x7FF0000000000000'f64
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    C_B = 0o377'i8
    
-
    
-
-
-
-
    
+  
    C_B = 0o377'i8
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    C_C = 0o277'i8
    
-
    
-
-
-
-
    
+  
    C_C = 0o277'i8
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    C_D = 0o177777'i16
    
-
    
-
-
-
-
    
+  
    C_D = 0o177777'i16
    
+  
    
+    
+    
+    
+  
    
 
    
 
-
    
+  
    
+
    
 
    
-
    Procs
    
-
    
-
-
    
-
    
-
    proc addfBug14485() {....raises: [], tags: [].}
    
-
    
-
-Some proc
+  
    Procs
    
+  
    
+    
    
+  
    
+  
    proc addfBug14485() {....raises: [], tags: [].}
    
+  
    
+    
+    Some proc
 
    Example:
    
 
    discard "foo() = " & $[1]
 #[
@@ -535,208 +492,194 @@ Some proc
 6: </script
 7: end of broken html
 ]#
    
-
-
    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc anything() {....raises: [], tags: [].}
    
-
    
-
-There is no block quote after blank lines at the beginning.
-
-
    
+  
    
+  
    proc anything() {....raises: [], tags: [].}
    
+  
    
+    
+    There is no block quote after blank lines at the beginning.
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+  
    
+  
    proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
                                 tags: [RootEffect].}
    
-
    
-
-ok1
-
-
    
+  
    
+    
+    ok1
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
-
    
-
-
-
-
    
+  
    
+  
    proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
-
    
-
-
+  
    
+  
    proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect].}
    
+  
    
+    
+    
 
    Example:
    
 
    discard
    ok1
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc bar[T](a, b: T): T
    
-
    
-
-
-
-
    
+  
    
+  
    proc bar[T](a, b: T): T
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc baz() {....raises: [], tags: [].}
    
-
    
-
-
-
-
    
+  
    
+  
    proc baz() {....raises: [], tags: [].}
    
+  
    
+    
+    
+    
+  
    
 
    
 
    
-
    proc baz[T](a, b: T): T {....deprecated.}
    
-
    
-  
    
-    Deprecated 
-  
    
+  
    proc baz[T](a, b: T): T {....deprecated.}
    
+  
    
+    
    
+  Deprecated 
+
    
 
-This is deprecated without message.
-
-
    
+    This is deprecated without message.
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    
-
    
-  
    
-    Deprecated: since v0.20
-  
    
+  
    
+  
    proc buzz[T](a, b: T): T {....deprecated: "since v0.20".}
    
+  
    
+    
    
+  Deprecated: since v0.20
+
    
 
-This is deprecated with a message.
-
-
    
+    This is deprecated with a message.
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
+  
    
+  
    proc c_nonexistent(frmt: cstring): cint {.importc: "nonexistent",
     header: "<stdio.h>", varargs, discardable, ...raises: [], tags: [].}
    
-
    
-
-
-
-
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
+  
    
+  
    proc c_printf(frmt: cstring): cint {.importc: "printf", header: "<stdio.h>",
                                      varargs, discardable, ...raises: [], tags: [].}
    
-
    
-
-the c printf. etc.
-
-
    
+  
    
+    
+    the c printf. etc.
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc fromUtils3() {....raises: [], tags: [].}
    
-
    
-
-came form utils but should be shown where fromUtilsGen is called
+  
    
+  
    proc fromUtils3() {....raises: [], tags: [].}
    
+  
    
+    
+    came form utils but should be shown where fromUtilsGen is called
 
    Example:
    
 
    discard """should be shown as examples for fromUtils3
        in module calling fromUtilsGen"""
    
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc isValid[T](x: T): bool
    
-
    
-
-
-
-
    
+  
    
+  
    proc isValid[T](x: T): bool
    
+  
    
+    
+    
+    
+  
    
 
    
 
 
    
-
 
    
-
    
-
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+  
    
+  
    proc low[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}
    
-
    
-
-
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
+  
    
+    
+    
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
 
    See also:
    
 
 
    low(2) # => -9223372036854775808
    
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
+  
    
+  
    proc low2[T: Ordinal | enum | range](x: T): T {.magic: "Low", noSideEffect,
     ...raises: [], tags: [].}
    
-
    
-
-
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
+  
    
+    
+    
    Returns the lowest possible value of an ordinal value x. As a special semantic rule, x may also be a type identifier.
    
 
    See also:
    
 
 
    low2(2) # => -9223372036854775808
    
 
    Example:
    
 
    discard "in low2"
    
-
-
    
+    
+  
 
    
 
 
    
-
 
    
-
    
-
    proc p1() {....raises: [], tags: [].}
    
-
    
-
-cp1
+  
    
+  
    proc p1() {....raises: [], tags: [].}
    
+  
    
+    
+    cp1
 
    Example:
    
 
    doAssert 1 == 1 # regular comments work here
    c4
 
    Example:
    
@@ -753,30 +696,28 @@ this is a nested doc comment
 ]##
 discard "c9"
 # also work after
    - - + +
    -
    -
    -
    func someFunc() {....raises: [], tags: [].}
    -
    - -My someFunc. Stuff in quotes here. Some link - -
    +
    + 
    func someFunc() {....raises: [], tags: [].}
    +
    + + My someFunc. Stuff in quotes here. Some link + +
    -
    -
    -
    proc tripleStrLitTest() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc tripleStrLitTest() {....raises: [], tags: [].}
    +
    + +

    Example: cmd: --hint:XDeclaredButNotUsed:off

    ## mullitline string litterals are tricky as their indentation can span
 ## below that of the runnableExamples
@@ -813,365 +754,343 @@ at indent 0
   """ ]
 discard
 # should be in
    - -
    + +
    -
    -
    -
    proc z1(): Foo {....raises: [], tags: [].}
    -
    - -cz1 - -
    +
    + 
    proc z1(): Foo {....raises: [], tags: [].}
    +
    + + cz1 + +
    -
    -
    -
    proc z2() {....raises: [], tags: [].}
    -
    - -cz2 +
    + 
    proc z2() {....raises: [], tags: [].}
    +
    + + cz2

    Example:

    discard "in cz2"
    - -
    + +
    -
    -
    -
    proc z3() {....raises: [], tags: [].}
    -
    - -cz3 - -
    +
    + 
    proc z3() {....raises: [], tags: [].}
    +
    + + cz3 + +
    -
    -
    -
    proc z4() {....raises: [], tags: [].}
    -
    - -cz4 - -
    +
    + 
    proc z4() {....raises: [], tags: [].}
    +
    + + cz4 + +
    -
    -
    -
    proc z5(): int {....raises: [], tags: [].}
    -
    - -cz5 - -
    +
    + 
    proc z5(): int {....raises: [], tags: [].}
    +
    + + cz5 + +
    -
    -
    -
    proc z6(): int {....raises: [], tags: [].}
    -
    - -cz6 - -
    +
    + 
    proc z6(): int {....raises: [], tags: [].}
    +
    + + cz6 + +
    -
    -
    -
    proc z7(): int {....raises: [], tags: [].}
    -
    - -cz7 - -
    +
    + 
    proc z7(): int {....raises: [], tags: [].}
    +
    + + cz7 + +
    -
    -
    -
    proc z8(): int {....raises: [], tags: [].}
    -
    - -cz8 - -
    +
    + 
    proc z8(): int {....raises: [], tags: [].}
    +
    + + cz8 + +
    -
    -
    -
    proc z9() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z9() {....raises: [], tags: [].}
    +
    + +

    Example:

    doAssert 1 + 1 == 2
    - -
    + +
    -
    -
    -
    proc z10() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z10() {....raises: [], tags: [].}
    +
    + +

    Example: cmd: -d:foobar

    discard 1
    cz10 - -
    + +
    -
    -
    -
    proc z11() {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z11() {....raises: [], tags: [].}
    +
    + +

    Example:

    discard 1
    - -
    + +
    -
    -
    -
    proc z12(): int {....raises: [], tags: [].}
    -
    - - +
    + 
    proc z12(): int {....raises: [], tags: [].}
    +
    + +

    Example:

    discard 1
    - -
    + +
    -
    -
    -
    proc z13() {....raises: [], tags: [].}
    -
    - -cz13 +
    + 
    proc z13() {....raises: [], tags: [].}
    +
    + + cz13

    Example:

    discard
    - -
    + +
    -
    -
    -
    proc z17() {....raises: [], tags: [].}
    -
    - -cz17 rest +
    + 
    proc z17() {....raises: [], tags: [].}
    +
    + + cz17 rest

    Example:

    discard 1
    rest - -
    + +
    -
    + +
    -

    Methods

    -
    - -
    -
    -
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    -
    - -foo1 - -
    +

    Methods

    +
    +
    +
    + 
    method method1(self: Moo) {.base, ...raises: [], tags: [].}
    +
    + + foo1 + +
    -
    -
    -
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    -
    - -foo2 - -
    +
    + 
    method method2(self: Moo): int {.base, ...raises: [], tags: [].}
    +
    + + foo2 + +
    -
    -
    -
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    -
    - -foo3 - -
    +
    + 
    method method3(self: Moo): int {.base, ...raises: [], tags: [].}
    +
    + + foo3 + +
    -
    +
    +
    -

    Iterators

    -
    - -
    -
    -
    iterator fromUtils1(): int {....raises: [], tags: [].}
    -
    - - +

    Iterators

    +
    +
    +
    + 
    iterator fromUtils1(): int {....raises: [], tags: [].}
    +
    + +

    Example:

    # ok1
 assert 1 == 1
 # ok2
    - -
    + +
    -
    -
    -
    iterator iter1(n: int): int {....raises: [], tags: [].}
    -
    - -foo1 - -
    +
    + 
    iterator iter1(n: int): int {....raises: [], tags: [].}
    +
    + + foo1 + +
    -
    -
    -
    iterator iter2(n: int): int {....raises: [], tags: [].}
    -
    - -foo2 +
    + 
    iterator iter2(n: int): int {....raises: [], tags: [].}
    +
    + + foo2

    Example:

    discard # bar
    - -
    + +
    -
    + +
    -

    Macros

    -
    - -
    -
    -
    macro bar(): untyped
    -
    - - - -
    +

    Macros

    +
    +
    +
    + 
    macro bar(): untyped
    +
    + + + +
    -
    -
    -
    macro z16()
    -
    - - +
    + 
    macro z16()
    +
    + +

    Example:

    discard 1
    cz16 after

    Example:

    doAssert 2 == 1 + 1
    - -
    + +
    -
    -
    -
    macro z18(): int
    -
    - -cz18 - -
    +
    + 
    macro z18(): int
    +
    + + cz18 + +
    -
    +
    +
    -

    Templates

    -
    - -
    -
    -
    template foo(a, b: SomeType)
    -
    - -This does nothing - -
    +

    Templates

    +
    +
    +
    + 
    template foo(a, b: SomeType)
    +
    + + This does nothing + +
    -
    -
    -
    template fromUtils2()
    -
    - -ok3 +
    + 
    template fromUtils2()
    +
    + + ok3

    Example:

    discard """should be shown as examples for fromUtils2
        in module calling fromUtilsGen"""
    - -
    + +
    -
    -
    -
    template myfn()
    -
    - - +
    + 
    template myfn()
    +
    + +

    Example:

    import std/strutils
 ## issue #8871 preserve formatting
@@ -1187,58 +1106,54 @@ bar
 block:
   discard 0xff # elu par cette crapule
 # should be in
    should be still in - -
    + +
    -
    -
    -
    template testNimDocTrailingExample()
    -
    - - +
    + 
    template testNimDocTrailingExample()
    +
    + +

    Example:

    discard 2
    - -
    + +
    -
    -
    -
    template z6t(): int
    -
    - -cz6t - -
    +
    + 
    template z6t(): int
    +
    + + cz6t + +
    -
    -
    -
    template z14()
    -
    - -cz14 +
    + 
    template z14()
    +
    + + cz14

    Example:

    discard
    - -
    + +
    -
    -
    -
    template z15()
    -
    - -cz15 +
    + 
    template z15()
    +
    + + cz15

    Example:

    discard

    Example:

    @@ -1249,26 +1164,25 @@ cz15 
    assert true

    Example:

    discard 1
    in or out? - -
    + +
    -
    +
    +
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    - - + diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 47fae2491..c62b4c7db 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -1,12 +1,11 @@ - + - + - +Index @@ -17,17 +16,16 @@ -Index + - -
    -
    -

    Index

    - Modules: subdir/subdir_b/utils, testproject.

    API symbols

    +
    +
    +

    Index

    + Modules: subdir/subdir_b/utils, testproject.

    API symbols

    `$`:
    -
    -
    +
    Made with Nim. Generated: 1970-01-02 03:46:40 UTC
    -
    - + From d33e1127666be23ad2dc879c2f89a41c179e2093 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Wed, 15 Jun 2022 16:38:12 +0200 Subject: [PATCH 0843/1654] Better range error messages (#19867) * Better range error messages * Revert to old behavior for arrays * Small corrections --- compiler/ccgexprs.nim | 6 +++--- lib/system/chcks.nim | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 8e7a21c67..4c15101a9 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -988,12 +988,12 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = if reifiedOpenArray(arr.lode): linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError4($1, $2, $3.Field1); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) else: linefmt(p, cpsStmts, "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" & - "{ #raiseIndexError(); $4}$n", + "{ #raiseIndexError4($1, $2, $3Len_0); $4}$n", [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)]) of tyArray: let first = intLiteral(firstOrd(p.config, ty)) @@ -1004,7 +1004,7 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) = of tySequence, tyString: linefmt(p, cpsStmts, "if ($2-$1 != -1 && " & - "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError(); $4}$n", + "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError4($1, $2, $3); $4}$n", [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)]) else: discard diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index a404e9d40..1e1ce9c87 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -16,6 +16,9 @@ proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = else: sysFatal(RangeDefect, "value out of range: ", $val) +proc raiseIndexError4(l1, h1, h2: int) {.compilerproc, noinline.} = + sysFatal(IndexDefect, "index out of bounds: " & $l1 & ".." & $h1 & " notin 0.." & $(h2 - 1)) + proc raiseIndexError3(i, a, b: int) {.compilerproc, noinline.} = sysFatal(IndexDefect, formatErrorIndexBound(i, a, b)) From ef2dd2e473d7082d60088ece05beef81062850bb Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 16 Jun 2022 14:46:59 +0800 Subject: [PATCH 0844/1654] document nimTestsNimdocFixup for rsttester (#19894) --- nimdoc/rsttester.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nimdoc/rsttester.nim b/nimdoc/rsttester.nim index daca3dfc7..a0bdfca1e 100644 --- a/nimdoc/rsttester.nim +++ b/nimdoc/rsttester.nim @@ -1,9 +1,14 @@ +# To run this, cd to the git repo root, and run "nim r nimdoc/rsttester.nim". +# to change expected results (after carefully verifying everything), use -d:nimTestsNimdocFixup + import os, strutils from std/private/gitutils import diffFiles const baseDir = "nimdoc/rst2html" +const fixup = defined(nimTestsNimdocFixup) + var failures = 0 proc exec(cmd: string) = @@ -29,7 +34,7 @@ proc testRst2Html(fixup = false) = if failures == 0: removeDir(baseDir / "source/htmldocs") -testRst2Html(defined(fixup)) +testRst2Html(fixup) # Check for failures if failures > 0: quit($failures & " failures occurred.") From 332aa0af75baf328416068459931d5a6e4292d34 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sun, 19 Jun 2022 13:35:25 +0800 Subject: [PATCH 0845/1654] [vm] refactor `stdlib.marshal.load` (#19905) refactor stdlib.marshal.load --- compiler/vmops.nim | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 640826cc8..04b753ef6 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -364,21 +364,5 @@ proc registerAdditionalOps*(c: PCtx) = let typ = a.getNode(0).typ let p = a.getReg(1) var res: string - - var node: PNode - case p.kind - of rkNone: - node = newNode(nkEmpty) - of rkInt: - node = newIntNode(nkIntLit, p.intVal) - of rkFloat: - node = newFloatNode(nkFloatLit, p.floatVal) - of rkNode: - node = p.node - of rkRegisterAddr: - node = p.regAddr.node - of rkNodeAddr: - node = p.nodeAddr[] - - storeAny(res, typ, node, c.config) + storeAny(res, typ, regToNode(p[]), c.config) setResult(a, res) From dd4cc266cd39d7cecfd7215e683b0c6499fc4a36 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Sun, 19 Jun 2022 15:36:33 +1000 Subject: [PATCH 0846/1654] Fixes #19900 (#19906) Fixed tocRoot placement --- config/nimdoc.cfg | 4 ++-- nimdoc/rst2html/expected/rst_examples.html | 2 +- nimdoc/test_out_index_dot_html/expected/index.html | 2 +- nimdoc/testproject/expected/subdir/subdir_b/utils.html | 2 +- nimdoc/testproject/expected/testproject.html | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 725f9e0a5..3b0afe8d6 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -149,11 +149,11 @@ doc.body_toc_group = """ onkeyup="search()" />
    $body_toc_groupsection -
    $tableofcontents
    $seeSrc +
    $deprecationMsg

    $moduledesc

    $content @@ -188,11 +188,11 @@ doc.body_toc_group = """
    -
    $tableofcontents
    $seeSrc +
    $deprecationMsg

    $moduledesc

    $content diff --git a/nimdoc/rst2html/expected/rst_examples.html b/nimdoc/rst2html/expected/rst_examples.html index 532917055..efc8ac414 100644 --- a/nimdoc/rst2html/expected/rst_examples.html +++ b/nimdoc/rst2html/expected/rst_examples.html @@ -49,7 +49,6 @@
    -
    • About this document
      • Encoding
        • @@ -75,6 +74,7 @@
        +

        diff --git a/nimdoc/test_out_index_dot_html/expected/index.html b/nimdoc/test_out_index_dot_html/expected/index.html index 0c0dc2268..43285db6b 100644 --- a/nimdoc/test_out_index_dot_html/expected/index.html +++ b/nimdoc/test_out_index_dot_html/expected/index.html @@ -49,7 +49,6 @@ -
        • @@ -69,6 +68,7 @@
          +

          diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index d5a3b84c7..4d753ad8f 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -49,7 +49,6 @@
          -
          +

          This is a description of the utils module.

          Links work:

          diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 49f24d204..24dfdc13f 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -49,7 +49,6 @@
          -
          • @@ -335,6 +334,7 @@
            +

            This is the top level module.

            Example:

            From 5e32fc785527d9b71ebab643f142fbab13eb29ce Mon Sep 17 00:00:00 2001 From: tersec Date: Mon, 20 Jun 2022 06:01:41 +0000 Subject: [PATCH 0847/1654] Recommend gcc 5.x or newer (#19910) --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 7ff056a20..8999091e6 100644 --- a/readme.md +++ b/readme.md @@ -51,8 +51,8 @@ Nim programming language. Those C sources are available within the Next, to build from source you will need: - * A C compiler such as ``gcc`` 3.x/later or an alternative such as ``clang``, - ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 3.x or + * A C compiler such as ``gcc`` 5.x/later or an alternative such as ``clang``, + ``Visual C++`` or ``Intel C++``. It is recommended to use ``gcc`` 5.x or later. * Either ``git`` or ``wget`` to download the needed source repositories. * The ``build-essential`` package when using ``gcc`` on Ubuntu (and likely From 40464fa7626a616a3881c0b16a74b9ec7d788720 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 20 Jun 2022 08:21:20 +0200 Subject: [PATCH 0848/1654] Fix nimRawSetjmp for VCC [backport: 1.2] (#19899) --- lib/system/ansi_c.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 5d0ecc01c..74f79167a 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -118,7 +118,7 @@ elif defined(nimBuiltinSetjmp): c_builtin_setjmp(unsafeAddr jmpb[0]) elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - when defined(windows): + when defined(windows) and not defined(vcc): # No `_longjmp()` on Windows. proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "longjmp".} From 0ae44e562f693d270e3cad77675ce5f68474921d Mon Sep 17 00:00:00 2001 From: flintforge Date: Tue, 21 Jun 2022 03:53:46 +0200 Subject: [PATCH 0849/1654] fix typo in nre.nim (#19915) Update nre.nim typo in proc replace description --- lib/impure/nre.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 03cbf5220..1012c7c36 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -695,8 +695,7 @@ proc replace*(str: string, pattern: Regex, ## each match and the return value is the replacement value. ## ## If `subproc` is a `proc (string): string`, then it is executed with the - ## full text of the match and and the return value is the replacement - ## value. + ## full text of the match and the return value is the replacement value. ## ## If `subproc` is a string, the syntax is as follows: ## From a65db5e2e924c51af7c654f8e564332299b95cb3 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 21 Jun 2022 15:37:23 +0800 Subject: [PATCH 0850/1654] [Tiny] correct comment opcDeref => opcLdDeref (#19908) correct comment opcDeref => opcLdDeref --- compiler/vm.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 1b11cb4f3..656508793 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -642,7 +642,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcNodeToReg: let ra = instr.regA let rb = instr.regB - # opcDeref might already have loaded it into a register. XXX Let's hope + # opcLdDeref might already have loaded it into a register. XXX Let's hope # this is still correct this way: if regs[rb].kind != rkNode: regs[ra] = regs[rb] From 3cb2d7af0591b12044d4be20dfb6d7f7ff79a4ff Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:43:58 +0800 Subject: [PATCH 0851/1654] [vm]fixes #15974 #12551 #19464 #16020 #16780 #16613 #14553 #19909 #18641 (#19902) [backport] * revert #12217 since the root problem seems to have been fixed; fix #15974;fix #12551; fix #19464 * fix #16020; fix #16780 * fix tests and #16613 * fix #14553 * fix #19909; skip skipRegisterAddr * fix #18641 --- compiler/vm.nim | 28 +++--- compiler/vmgen.nim | 3 - tests/vm/tissues.nim | 50 ++++++++++- tests/vm/tmisc_vm.nim | 196 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 263 insertions(+), 14 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 656508793..e681bbc96 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -523,8 +523,7 @@ when not defined(nimHasSinkInference): template takeAddress(reg, source) = reg.nodeAddr = addr source - when defined(gcDestructors): - GC_ref source + GC_ref source proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var pc = start @@ -1011,6 +1010,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeBC(rkInt) template getTyp(n): untyped = n.typ.skipTypes(abstractInst) + template skipRegisterAddr(n: TFullReg): TFullReg = + var tmp = n + while tmp.kind == rkRegisterAddr: + tmp = tmp.regAddr[] + tmp + proc ptrEquality(n1: ptr PNode, n2: PNode): bool = ## true if n2.intVal represents a ptr equal to n1 let p1 = cast[int](n1) @@ -1024,16 +1029,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = return t2.kind in PtrLikeKinds and n2.intVal == p1 else: return false - if regs[rb].kind == rkNodeAddr: - if regs[rc].kind == rkNodeAddr: - ret = regs[rb].nodeAddr == regs[rc].nodeAddr + let rbReg = skipRegisterAddr(regs[rb]) + let rcReg = skipRegisterAddr(regs[rc]) + + if rbReg.kind == rkNodeAddr: + if rcReg.kind == rkNodeAddr: + ret = rbReg.nodeAddr == rcReg.nodeAddr else: - ret = ptrEquality(regs[rb].nodeAddr, regs[rc].node) - elif regs[rc].kind == rkNodeAddr: - ret = ptrEquality(regs[rc].nodeAddr, regs[rb].node) + ret = ptrEquality(rbReg.nodeAddr, rcReg.node) + elif rcReg.kind == rkNodeAddr: + ret = ptrEquality(rcReg.nodeAddr, rbReg.node) else: - let nb = regs[rb].node - let nc = regs[rc].node + let nb = rbReg.node + let nc = rcReg.node if nb.kind != nc.kind: discard elif (nb == nc) or (nb.kind == nkNilLit): ret = true # intentional elif nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ != nil and nb.typ.kind == tyProc and sameConstant(nb, nc): diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 2e7f6d8b0..aa2bdbe4c 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1382,9 +1382,6 @@ proc unneededIndirection(n: PNode): bool = n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef proc canElimAddr(n: PNode): PNode = - if n[0].typ.skipTypes(abstractInst).kind in {tyObject, tyTuple, tyArray}: - # objects are reference types in the VM - return n[0] case n[0].kind of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64: var m = n[0][0] diff --git a/tests/vm/tissues.nim b/tests/vm/tissues.nim index 1cf3afc00..f0ae6c296 100644 --- a/tests/vm/tissues.nim +++ b/tests/vm/tissues.nim @@ -1,6 +1,6 @@ import macros -block t9043: # issue #9043 +block t9043: # bug #9043 proc foo[N: static[int]](dims: array[N, int]): string = const N1 = N const N2 = dims.len @@ -26,3 +26,51 @@ block t4952: let tree = newTree(nnkExprColonExpr) let t = (n: tree) doAssert: t.n.kind == tree.kind + + +# bug #19909 +type + SinglyLinkedList[T] = ref object + SinglyLinkedListObj[T] = ref object + + +proc addMoved[T](a, b: var SinglyLinkedList[T]) = + if a.addr != b.addr: discard + +proc addMoved[T](a, b: var SinglyLinkedListObj[T]) = + if a.addr != b.addr: discard + +proc main = + var a: SinglyLinkedList[int]; new a + var b: SinglyLinkedList[int]; new b + a.addMoved b + + var a0: SinglyLinkedListObj[int] + var b0: SinglyLinkedListObj[int] + a0.addMoved b0 + +static: main() + + +# bug #18641 + +type A = object + ha1: int +static: + var a = A() + var a2 = a.addr + a2.ha1 = 11 + doAssert a2.ha1 == 11 + a.ha1 = 12 + doAssert a.ha1 == 12 + doAssert a2.ha1 == 12 # ok +static: + proc fn() = + var a = A() + var a2 = a.addr + a2.ha1 = 11 + doAssert a2.ha1 == 11 + a.ha1 = 12 + doAssert a.ha1 == 12 + doAssert a2.ha1 == 12 # fails + fn() diff --git a/tests/vm/tmisc_vm.nim b/tests/vm/tmisc_vm.nim index bbf618622..af09e8871 100644 --- a/tests/vm/tmisc_vm.nim +++ b/tests/vm/tmisc_vm.nim @@ -1,7 +1,12 @@ discard """ + targets: "c js" output: ''' [127, 127, 0, 255][127, 127, 0, 255] (data: 1) +(2, 1) +(2, 1) +(2, 1) +(f0: 5) ''' nimout: '''caught Exception @@ -17,6 +22,19 @@ foo4 (a: 0, b: 0) (a: 0, b: 0) (a: 0, b: 0) +z1 m: (lo: 12) +z2 a: (lo: 3) +x1 a: (lo: 3) +x2 a: (lo: 6) +x3 a: (lo: 0) +z3 a: (lo: 3) +x1 a: (lo: 3) +x2 a: (lo: 6) +x3 a: (lo: 0) +(2, 1) +(2, 1) +(2, 1) +(f0: 5) ''' """ import std/sets @@ -249,3 +267,181 @@ static: echo x2[] let x3 = new(ref MyObject) # cannot generate VM code for ref MyObject echo x3[] + +# bug #19464 +type + Wrapper = object + inner: int + +proc assign(r: var Wrapper, a: Wrapper) = + r = a + +proc myEcho(a: Wrapper) = + var tmp = a + assign(tmp, Wrapper(inner: 0)) # this shouldn't modify `a` + doAssert a.inner == 1 + +static: + var result: Wrapper + assign(result, Wrapper(inner: 1)) + myEcho(result) + +when true: + # bug #15974 + type Foo = object + f0: int + + proc fn(a: var Foo) = + var s: Foo + a = Foo(f0: 2) + s = a + doAssert s.f0 == 2 + a = Foo(f0: 3) + doAssert s.f0 == 2 + + proc test2()= + var a = Foo(f0: 1) + fn(a) + + static: test2() + test2() + +# bug #12551 +type + StUint = object + lo: uint64 + +func `+=`(x: var Stuint, y: Stuint) = + x.lo += y.lo + +func `-`(x, y: Stuint): Stuint = + result.lo = x.lo - y.lo + +func `+`(x, y: Stuint): Stuint = + result.lo = x.lo + y.lo + +func `-=`(x: var Stuint, y: Stuint) = + x = x - y + +func `<`(x, y: Stuint): bool= + x.lo < y.lo + +func `==`(x, y: Stuint): bool = + x.lo == y.lo + +func `<=`(x, y: Stuint): bool = + x.lo <= y.lo + +proc div3n2n(r: var Stuint, b: Stuint) = + var d: Stuint + r = d + r += b + +func div2n1n(r: var Stuint, b: Stuint) = + div3n2n(r, b) + +func divmodBZ(x, y: Stuint, r: var Stuint)= + div2n1n(r, y) + r.lo = 3 + +func `mod`(x, y: Stuint): Stuint = + divmodBZ(x, y, result) + +func doublemod_internal(a, m: Stuint): Stuint = + result = a + if a >= m - a: + result -= m + result += a + +func mulmod_internal(a, b, m: Stuint): Stuint = + var (a, b) = (a, b) + swap(a, b) + debugEcho "x1 a: ", a + a = doublemod_internal(a, m) + debugEcho "x2 a: ", a + a = doublemod_internal(a, m) + debugEcho "x3 a: ", a + +func powmod_internal(a, m: Stuint): Stuint = + var a = a + debugEcho "z1 m: ", m + debugEcho "z2 a: ", a + result = mulmod_internal(result, a, m) + debugEcho "z3 a: ", a + a = mulmod_internal(a, a, m) + +func powmod*(a, m: Stuint) = + discard powmod_internal(a mod m, m) + +static: + var x = Stuint(lo: high(uint64)) + var y = Stuint(lo: 12) + + powmod(x, y) + +# bug #16780 +when true: + template swap*[T](a, b: var T) = + var a2 = addr(a) + var b2 = addr(b) + var aOld = a2[] + a2[] = b2[] + b2[] = aOld + + proc rather = + block: + var a = 1 + var b = 2 + swap(a, b) + echo (a,b) + + block: + type Foo = ref object + x: int + var a = Foo(x:1) + var b = Foo(x:2) + swap(a, b) + echo (a.x, b.x) + + block: + type Foo = object + x: int + var a = Foo(x:1) + var b = Foo(x:2) + swap(a, b) + echo (a.x,b.x) + + static: rather() + rather() + +# bug #16020 +when true: + block: + type Foo = object + f0: int + proc main= + var f = Foo(f0: 3) + var f2 = f.addr + f2[].f0 += 1 + f2.f0 += 1 + echo f + static: main() + main() + +import tables, strutils + +# bug #14553 +const PpcPatterns = @[("aaaa", "bbbb"), ("aaaaa", "bbbbb"), ("aaaaaa", "bbbbbb"), ("aaaaaaa", "bbbbbbb"), ("aaaaaaaa", "bbbbb")] + +static: + var + needSecondIdentifier = initTable[uint32, seq[(string, string)]]() + + for (name, pattern) in PpcPatterns: + let + firstPart = 0'u32 + lastPart = "test" + + needSecondIdentifier.mgetOrPut(firstPart, @[]).add((name, pattern)) + + doAssert needSecondIdentifier[0] == @[("aaaa", "bbbb"), ("aaaaa", "bbbbb"), ("aaaaaa", "bbbbbb"), ("aaaaaaa", "bbbbbbb"), ("aaaaaaaa", "bbbbb")] From caf6aff06b6dd36ef0bbe8c1f8b527952790e208 Mon Sep 17 00:00:00 2001 From: Khaled Hammouda Date: Wed, 22 Jun 2022 06:36:30 -0400 Subject: [PATCH 0852/1654] Fix distinct requiresInit test and manual (#19901) fix distinct test and manual --- doc/manual.rst | 11 +++++++---- tests/distinct/tdistinct.nim | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual.rst b/doc/manual.rst index 82c9f5758..b63a2f68f 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -2851,7 +2851,10 @@ Given the following distinct type definitions: .. code-block:: nim type - DistinctObject {.requiresInit, borrow: `.`.} = distinct MyObject + Foo = object + x: string + + DistinctFoo {.requiresInit, borrow: `.`.} = distinct Foo DistinctString {.requiresInit.} = distinct string The following code blocks will fail to compile: @@ -2864,7 +2867,7 @@ The following code blocks will fail to compile: .. code-block:: nim var s: DistinctString s = "test" - doAssert s == "test" + doAssert string(s) == "test" But these ones will compile successfully: @@ -2873,8 +2876,8 @@ But these ones will compile successfully: doAssert foo.x == "test" .. code-block:: nim - let s = "test" - doAssert s == "test" + let s = DistinctString("test") + doAssert string(s) == "test" Let statement ------------- diff --git a/tests/distinct/tdistinct.nim b/tests/distinct/tdistinct.nim index dd8237854..8ec083020 100644 --- a/tests/distinct/tdistinct.nim +++ b/tests/distinct/tdistinct.nim @@ -135,11 +135,11 @@ block tRequiresInit: reject: var s: DistinctString s = "test" - doAssert s == "test" + doAssert string(s) == "test" accept: - let s = "test" - doAssert s == "test" + let s = DistinctString("test") + doAssert string(s) == "test" block: #17322 type From 0189122d4fadc73a3de18a7a781997c506f837aa Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 27 Jun 2022 23:57:07 +0800 Subject: [PATCH 0853/1654] ref #19830; multiple definition of in Nim generated static libraries (#19934) * ref #19830; multiple definition of in Nim generated static libraries * fix compile errors --- compiler/cgen.nim | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 862f31107..c0b94ebbb 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1323,7 +1323,7 @@ proc getSomeInitName(m: BModule, suffix: string): Rope = proc getInitName(m: BModule): Rope = if sfMainModule in m.module.flags: # generate constant name for main module, for "easy" debugging. - result = rope"NimMainModule" + result = rope(m.config.nimMainPrefix) & rope"NimMainModule" else: result = getSomeInitName(m, "Init000") @@ -1356,35 +1356,35 @@ proc genMainProc(m: BModule) = preMainCode.add("\tinitStackBottomWith_actual((void *)&inner);\L") preMainCode.add("\t(*inner)();\L") else: - preMainCode.add("\tPreMain();\L") + preMainCode.add("\t$1PreMain();\L" % [rope m.config.nimMainPrefix]) + + var posixCmdLine: Rope + if optNoMain notin m.config.globalOptions: + posixCmdLine.add "\tN_LIB_PRIVATE int cmdCount;\L" + posixCmdLine.add "\tN_LIB_PRIVATE char** cmdLine;\L" + posixCmdLine.add "\tN_LIB_PRIVATE char** gEnv;\L" const - # not a big deal if we always compile these 3 global vars... makes the HCR code easier - PosixCmdLine = - "N_LIB_PRIVATE int cmdCount;$N" & - "N_LIB_PRIVATE char** cmdLine;$N" & - "N_LIB_PRIVATE char** gEnv;$N" - # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. PreMainVolatileBody = "\tvoid (*volatile inner)(void);$N" & - "\tinner = PreMainInner;$N" & + "\tinner = $3PreMainInner;$N" & "$1" & "\t(*inner)();$N" PreMainNonVolatileBody = "$1" & - "\tPreMainInner();$N" + "\t$3PreMainInner();$N" PreMainBodyStart = "$N" & - "N_LIB_PRIVATE void PreMainInner(void) {$N" & + "N_LIB_PRIVATE void $3PreMainInner(void) {$N" & "$2" & "}$N$N" & - PosixCmdLine & - "N_LIB_PRIVATE void PreMain(void) {$N" + "$4" & + "N_LIB_PRIVATE void $3PreMain(void) {$N" PreMainBodyEnd = "}$N$N" @@ -1395,21 +1395,21 @@ proc genMainProc(m: BModule) = MainProcsWithResult = MainProcs & ("\treturn $1nim_program_result;$N") - NimMainInner = "N_LIB_PRIVATE N_CDECL(void, NimMainInner)(void) {$N" & + NimMainInner = "N_LIB_PRIVATE N_CDECL(void, $5NimMainInner)(void) {$N" & "$1" & "}$N$N" NimMainVolatileBody = "\tvoid (*volatile inner)(void);$N" & "$4" & - "\tinner = NimMainInner;$N" & + "\tinner = $5NimMainInner;$N" & "$2" & "\t(*inner)();$N" NimMainNonVolatileBody = "$4" & "$2" & - "\tNimMainInner();$N" + "\t$5NimMainInner();$N" NimMainProcStart = "N_CDECL(void, $5NimMain)(void) {$N" @@ -1489,9 +1489,9 @@ proc genMainProc(m: BModule) = else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []) inc(m.labels) if m.config.selectedGC notin {gcNone, gcArc, gcOrc}: - appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine]) else: - appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit]) + appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit, m.config.nimMainPrefix, posixCmdLine]) if m.config.target.targetOS == osWindows and m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}: From 7c31b6a47b5cfb6f2d1f296390eb457bd75e55c2 Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Tue, 28 Jun 2022 03:13:17 -0300 Subject: [PATCH 0854/1654] Fix jsre (#19917) * Fixes for jsre to make it more safe at runtime on some edge cases * https://github.com/nim-lang/Nim/pull/19917#issuecomment-1162692893 --- lib/js/jsre.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index cd8fb2be7..0cc2f8a87 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -33,13 +33,13 @@ func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.com func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} ## Returns a new string with some or all matches of a pattern replaced by given replacement -func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "#.split(#)".} +func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.split(#) || [])".} ## Divides a string into an ordered list of substrings and returns the array -func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "#.match(#)".} +func match*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.match(#) || [])".} ## Returns an array of matches of a RegExp against given string -func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "#.exec(#)".} +func exec*(self: RegExp; pattern: cstring): seq[cstring] {.importjs: "(#.exec(#) || [])".} ## Executes a search for a match in its string parameter. func toCstring*(self: RegExp): cstring {.importjs: "#.toString()".} @@ -87,3 +87,5 @@ runnableExamples: assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] jsregex.compile(r"[lw]", r"i") assert "hello world".replace(jsregex,"X") == "heXlo world" + let digitsRegex: RegExp = newRegExp(r"\d") + assert "foo".match(digitsRegex) == @[] From 8a344cb25b9b9d1b185ba6e5a8e1cf14421fedb4 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 28 Jun 2022 16:49:31 +0800 Subject: [PATCH 0855/1654] closes #11042; add testcase (#19935) close #11042; add testcase --- tests/closure/t11042.nim | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/closure/t11042.nim diff --git a/tests/closure/t11042.nim b/tests/closure/t11042.nim new file mode 100644 index 000000000..6a3928316 --- /dev/null +++ b/tests/closure/t11042.nim @@ -0,0 +1,55 @@ +discard """ + output:''' +foo: 1 +foo: 2 +bar: 1 +bar: 2 +foo: 1 +foo: 2 +bar: 1 +bar: 2 +bar: 3 +bar: 4 +bar: 5 +bar: 6 +bar: 7 +bar: 8 +bar: 9 +''' +""" + +# bug #11042 +block: + iterator foo: int = + for x in 1..2: + echo "foo: ", x + for y in 1..2: + discard + + for x in foo(): discard + + let bar = iterator: int = + for x in 1..2: + echo "bar: ", x + for y in 1..2: + discard + + for x in bar(): discard + + +block: + iterator foo: int = + for x in 1..2: + echo "foo: ", x + for y in 1..2: + discard + + for x in foo(): discard + + let bar = iterator: int = + for x in 1..9: + echo "bar: ", x + for y in 1..2: + discard + + for x in bar(): discard \ No newline at end of file From bcff13debcbdd8108237c8033e4dc9c38bb331e0 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 29 Jun 2022 22:37:24 +0800 Subject: [PATCH 0856/1654] dec inLoop after exiting the while scope in computeLiveRanges [backport] (#19918) * dec inLoop after exiting the while scope in computeLiveRanges * add testcase --- compiler/varpartitions.nim | 2 +- tests/arc/tcursorloop.nim | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/arc/tcursorloop.nim diff --git a/compiler/varpartitions.nim b/compiler/varpartitions.nim index d04a7b40c..a93104b06 100644 --- a/compiler/varpartitions.nim +++ b/compiler/varpartitions.nim @@ -851,7 +851,7 @@ proc computeLiveRanges(c: var Partitions; n: PNode) = # connect(graph, cursorVar) inc c.inLoop for child in n: computeLiveRanges(c, child) - inc c.inLoop + dec c.inLoop of nkElifBranch, nkElifExpr, nkElse, nkOfBranch: inc c.inConditional for child in n: computeLiveRanges(c, child) diff --git a/tests/arc/tcursorloop.nim b/tests/arc/tcursorloop.nim new file mode 100644 index 000000000..a37a6a036 --- /dev/null +++ b/tests/arc/tcursorloop.nim @@ -0,0 +1,45 @@ +discard """ + cmd: '''nim c --gc:arc --expandArc:traverse --hint:Performance:off $file''' + nimout: ''' +--expandArc: traverse + +var + it + jt_cursor +try: + `=copy`(it, root) + block :tmp: + while ( + not (it == nil)): + if true: + echo [it.s] + `=copy`(it, it.ri) + jt_cursor = root + if ( + not (jt_cursor == nil)): + echo [jt_cursor.s] + jt_cursor = jt_cursor.ri +finally: + `=destroy`(it) +-- end of expandArc ------------------------ +''' +""" + +type + Node = ref object + le, ri: Node + s: string + +proc traverse(root: Node) = + var it = root + while it != nil: + if true: + echo it.s + it = it.ri + + var jt = root + if jt != nil: + echo jt.s + jt = jt.ri + +traverse(nil) From ad430c0daad16019a6c440ab735a61856523329a Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 30 Jun 2022 10:20:19 +0200 Subject: [PATCH 0857/1654] once C++, always C++ [backport] (#19938) * once C++, always C++ When using `{.compile: "file.cc".}` in a nim module, even when compiling with `nim c` the C++ compiler should be used - once any C++ file has been compiled, the C++ linker also needs to be used. * more strict C++ check * simplify code --- compiler/extccomp.nim | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 5c074d284..f1dc01f44 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -493,7 +493,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} = (conf.target.hostOS == osWindows) proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool = - conf.backend == backendCpp and not cfile.string.endsWith(".c") + # List of possible file extensions taken from gcc + for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]: + if cfile.string.endsWith(ext): return true + false proc envFlags(conf: ConfigRef): string = result = if conf.backend == backendCpp: @@ -501,14 +504,14 @@ proc envFlags(conf: ConfigRef): string = else: getEnv("CFLAGS") -proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string = +proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string = if compiler == ccEnv: - result = if useCpp(conf, cfile): + result = if isCpp: getEnv("CXX") else: getEnv("CC") else: - result = if useCpp(conf, cfile): + result = if isCpp: CC[compiler].cppCompiler else: CC[compiler].compilerExe @@ -547,22 +550,25 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool = proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe - elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler - else: getCompilerExe(conf, compiler, AbsoluteFile"") + else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp) proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false; produceOutput = false): string = - let c = conf.cCompiler + let + c = conf.cCompiler + isCpp = useCpp(conf, cfile.cname) # We produce files like module.nim.cpp, so the absolute Nim filename is not # cfile.name but `cfile.cname.changeFileExt("")`: var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string) - if useCpp(conf, cfile.cname): + if isCpp: # needs to be prepended so that --passc:-std=c++17 can override default. # we could avoid allocation by making cFileSpecificOptions inplace options = CC[c].cppXsupport & ' ' & options + # If any C++ file was compiled, we need to use C++ driver for linking as well + incl conf.globalOptions, optMixedMode var exe = getConfigVar(conf, c, ".exe") - if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname) + if exe.len == 0: exe = getCompilerExe(conf, c, isCpp) if needsExeExt(conf): exe = addFileExt(exe, "exe") if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and @@ -582,7 +588,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, compilePattern = joinPath(conf.cCompilerPath, exe) else: - compilePattern = getCompilerExe(conf, c, cfile.cname) + compilePattern = getCompilerExe(conf, c, isCpp) includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)])) From 306810a748ab3e0182f798bea7d2792c64ab856a Mon Sep 17 00:00:00 2001 From: Sam Zaydel Date: Thu, 30 Jun 2022 14:16:50 -0700 Subject: [PATCH 0858/1654] Enable nim-lang to build correctly on illumos-based systems (#19952) --- tools/niminst/buildsh.nimf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 7a05ef342..6b99c49ee 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -164,7 +164,7 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - if [ "$isOpenIndiana" = "yes" ] ; then + if [ "$isOpenIndiana" = "yes" ] || [ `uname -o` == "illumos" ] ; then mycpu="amd64" else mycpu="i386" From 2c0aaac3045a63b5e4a24468341bf9ccfc8820cc Mon Sep 17 00:00:00 2001 From: Juan Carlos Date: Thu, 30 Jun 2022 18:18:11 -0300 Subject: [PATCH 0859/1654] jsffi add missing braces (#19948) js codegen add missing whitespaces and braces --- lib/js/jsffi.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/js/jsffi.nim b/lib/js/jsffi.nim index 35cbf2864..aca4fc292 100644 --- a/lib/js/jsffi.nim +++ b/lib/js/jsffi.nim @@ -348,8 +348,8 @@ iterator pairs*(obj: JsObject): (cstring, JsObject) = var k: cstring var v: JsObject {.emit: "for (var `k` in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(`k`)) continue;".} - {.emit: " `v`=`obj`[`k`];".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `obj`[`k`];".} yield (k, v) {.emit: "}".} @@ -357,8 +357,8 @@ iterator items*(obj: JsObject): JsObject = ## Yields the `values` of each field in a JsObject, wrapped into a JsObject. var v: JsObject {.emit: "for (var k in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(k)) continue;".} - {.emit: " `v`=`obj`[k];".} + {.emit: " if (!`obj`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `obj`[k];".} yield v {.emit: "}".} @@ -366,7 +366,7 @@ iterator keys*(obj: JsObject): cstring = ## Yields the `names` of each field in a JsObject. var k: cstring {.emit: "for (var `k` in `obj`) {".} - {.emit: " if (!`obj`.hasOwnProperty(`k`)) continue;".} + {.emit: " if (!`obj`.hasOwnProperty(`k`)) { continue; }".} yield k {.emit: "}".} @@ -376,8 +376,8 @@ iterator pairs*[K: JsKey, V](assoc: JsAssoc[K, V]): (K,V) = var k: cstring var v: V {.emit: "for (var `k` in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(`k`)) continue;".} - {.emit: " `v`=`assoc`[`k`];".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} + {.emit: " `v` = `assoc`[`k`];".} yield (k.toJsKey(K), v) {.emit: "}".} @@ -385,8 +385,8 @@ iterator items*[K, V](assoc: JsAssoc[K, V]): V = ## Yields the `values` in a JsAssoc. var v: V {.emit: "for (var k in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(k)) continue;".} - {.emit: " `v`=`assoc`[k];".} + {.emit: " if (!`assoc`.hasOwnProperty(k)) { continue; }".} + {.emit: " `v` = `assoc`[k];".} yield v {.emit: "}".} @@ -394,7 +394,7 @@ iterator keys*[K: JsKey, V](assoc: JsAssoc[K, V]): K = ## Yields the `keys` in a JsAssoc. var k: cstring {.emit: "for (var `k` in `assoc`) {".} - {.emit: " if (!`assoc`.hasOwnProperty(`k`)) continue;".} + {.emit: " if (!`assoc`.hasOwnProperty(`k`)) { continue; }".} yield k.toJsKey(K) {.emit: "}".} From ce4078acd40bb27de5d05832f486dbc95918a3c9 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Thu, 30 Jun 2022 23:19:04 +0200 Subject: [PATCH 0860/1654] Allow recursive closure iterators (#19939) --- compiler/semexprs.nim | 3 ++- doc/manual.rst | 5 +---- tests/iter/titer_issues.nim | 13 +++++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index ed28d8145..e07a98417 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -886,7 +886,8 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, case callee.kind of skMacro, skTemplate: discard else: - if callee.kind == skIterator and callee.id == c.p.owner.id: + if callee.kind == skIterator and callee.id == c.p.owner.id and + not isClosureIterator(c.p.owner.typ): localError(c.config, n.info, errRecursiveDependencyIteratorX % callee.name.s) # error correction, prevents endless for loop elimination in transf. # See bug #2051: diff --git a/doc/manual.rst b/doc/manual.rst index b63a2f68f..571379a87 100644 --- a/doc/manual.rst +++ b/doc/manual.rst @@ -4326,13 +4326,10 @@ Closure iterators and inline iterators have some restrictions: 1. For now, a closure iterator cannot be executed at compile time. 2. `return` is allowed in a closure iterator but not in an inline iterator (but rarely useful) and ends the iteration. -3. Neither inline nor closure iterators can be (directly)* recursive. +3. Inline iterators cannot be recursive. 4. Neither inline nor closure iterators have the special `result` variable. 5. Closure iterators are not supported by the JS backend. -(*) Closure iterators can be co-recursive with a factory proc which results -in similar syntax to a recursive iterator. More details follow. - Iterators that are neither marked `{.closure.}` nor `{.inline.}` explicitly default to being inline, but this may change in future versions of the implementation. diff --git a/tests/iter/titer_issues.nim b/tests/iter/titer_issues.nim index 15fe867c8..1f7e41e69 100644 --- a/tests/iter/titer_issues.nim +++ b/tests/iter/titer_issues.nim @@ -29,6 +29,7 @@ end 9018 @[1, 2] @[1, 2, 3] +1 ''' """ @@ -274,3 +275,15 @@ iterator cc() {.closure.} = break var a2 = cc + +# bug #16876 +block: + iterator a(num: int): int {.closure.} = + if num == 1: + yield num + else: + for i in a(num - 1): + yield i + + for i in a(5): + echo i From 4897c47c8004e85474b760d770d9d09ae1d58771 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 2 Jul 2022 16:51:21 +0800 Subject: [PATCH 0861/1654] closes #15897; add testcase (#19961) --- tests/views/tviews1.nim | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/views/tviews1.nim b/tests/views/tviews1.nim index 249058eb6..b81b17f30 100644 --- a/tests/views/tviews1.nim +++ b/tests/views/tviews1.nim @@ -66,3 +66,14 @@ proc mainB = assert foo.x.y == @[1, 2, 3] mainB() + + +# bug #15897 +type Outer = ref object + value: int +type Inner = object + owner: var Outer + +var o = Outer(value: 1234) +var v = Inner(owner: o).owner.value +doAssert v == 1234 From d2d8f1342b3125aac8e0592da25df61e6feb95b2 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Mon, 4 Jul 2022 08:52:44 -0300 Subject: [PATCH 0862/1654] Fixing `nimRawSetJmp` for vcc and clangcl on Windows (#19959) * fix vcc rawsetjmp * changing `_longjmp()` to `longjmp()` and `_setjmp()` to `setjmp()` * fix * fix setjmp to clangcl on Windows * fix genTrySetjmp() to clangcl on Windows --- compiler/ccgstmts.nim | 19 ++++++++++++------- lib/system/ansi_c.nim | 34 +++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index e791318ee..3ecf68402 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1367,13 +1367,18 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint]) elif isDefined(p.config, "nimRawSetjmp"): if isDefined(p.config, "mswindows"): - # The Windows `_setjmp()` takes two arguments, with the second being an - # undocumented buffer used by the SEH mechanism for stack unwinding. - # Mingw-w64 has been trying to get it right for years, but it's still - # prone to stack corruption during unwinding, so we disable that by setting - # it to NULL. - # More details: https://github.com/status-im/nimbus-eth2/issues/3121 - linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) + if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"): + # For the vcc compiler, use `setjmp()` with one argument. + # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170 + linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint]) + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint]) else: linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint]) else: diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 74f79167a..0dbded126 100644 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -118,26 +118,30 @@ elif defined(nimBuiltinSetjmp): c_builtin_setjmp(unsafeAddr jmpb[0]) elif defined(nimRawSetjmp) and not defined(nimStdSetjmp): - when defined(windows) and not defined(vcc): + when defined(windows): # No `_longjmp()` on Windows. proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "longjmp".} - # The Windows `_setjmp()` takes two arguments, with the second being an - # undocumented buffer used by the SEH mechanism for stack unwinding. - # Mingw-w64 has been trying to get it right for years, but it's still - # prone to stack corruption during unwinding, so we disable that by setting - # it to NULL. - # More details: https://github.com/status-im/nimbus-eth2/issues/3121 - when defined(nimHasStyleChecks): - {.push styleChecks: off.} + when defined(vcc) or defined(clangcl): + proc c_setjmp*(jmpb: C_JmpBuf): cint {. + header: "", importc: "setjmp".} + else: + # The Windows `_setjmp()` takes two arguments, with the second being an + # undocumented buffer used by the SEH mechanism for stack unwinding. + # Mingw-w64 has been trying to get it right for years, but it's still + # prone to stack corruption during unwinding, so we disable that by setting + # it to NULL. + # More details: https://github.com/status-im/nimbus-eth2/issues/3121 + when defined(nimHasStyleChecks): + {.push styleChecks: off.} - proc c_setjmp*(jmpb: C_JmpBuf): cint = - proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. - header: "", importc: "_setjmp".} - c_setjmp_win(jmpb, nil) + proc c_setjmp*(jmpb: C_JmpBuf): cint = + proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {. + header: "", importc: "_setjmp".} + c_setjmp_win(jmpb, nil) - when defined(nimHasStyleChecks): - {.pop.} + when defined(nimHasStyleChecks): + {.pop.} else: proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {. header: "", importc: "_longjmp".} From 7d0285853f4327a0a4f683e13971dc3429aad1f3 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 5 Jul 2022 16:11:34 +0800 Subject: [PATCH 0863/1654] rename gc to mm (#19971) --- compiler/msgs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 9d111b2e2..371a886f4 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -677,7 +677,7 @@ proc genSuccessX*(conf: ConfigRef) = const debugModeHints = "none (DEBUG BUILD, `-d:release` generates faster code)" if conf.cmd in cmdBackends: if conf.backend != backendJs: - build.add "gc: $#; " % $conf.selectedGC + build.add "mm: $#; " % $conf.selectedGC if optThreads in conf.globalOptions: build.add "threads: on; " build.add "opt: " if optOptimizeSpeed in conf.options: build.add "speed" From 01b40dc1d7c7c5331361341bdf305084c799c05b Mon Sep 17 00:00:00 2001 From: Daniel Clarke Date: Wed, 6 Jul 2022 06:29:05 +1000 Subject: [PATCH 0864/1654] Fixes return values of execCmd on macos (#19963) * Fixes return values of execCmd on macos * update tests to use existing structure Co-authored-by: daniel --- lib/pure/osproc.nim | 2 +- tests/stdlib/tosproc.nim | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 512db92a4..666ccdda8 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -1523,7 +1523,7 @@ elif not defined(useNimRtl): header: "".} proc execCmd(command: string): int = - when defined(linux): + when defined(posix): let tmp = csystem(command) result = if tmp == -1: tmp else: exitStatusLikeShell(tmp) else: diff --git a/tests/stdlib/tosproc.nim b/tests/stdlib/tosproc.nim index c54e0d112..f55dd3e21 100644 --- a/tests/stdlib/tosproc.nim +++ b/tests/stdlib/tosproc.nim @@ -29,6 +29,12 @@ when defined(case_testfile): # compiled test file for child process case arg of "exit_0": if true: quit(0) + of "exit_1": + if true: quit(1) + of "exit_2": + if true: quit(2) + of "exit_42": + if true: quit(42) of "exitnow_139": if true: exitnow(139) of "c_exit2_139": @@ -115,6 +121,13 @@ else: # main driver runTest("c_exit2_139", 139) runTest("quit_139", 139) + block execCmdTest: + let output = compileNimProg("-d:release -d:case_testfile", "D20220705T221100") + doAssert execCmd(output & " exit_0") == 0 + doAssert execCmd(output & " exit_1") == 1 + doAssert execCmd(output & " exit_2") == 2 + doAssert execCmd(output & " exit_42") == 42 + import std/streams block execProcessTest: From 430a1793075866179b34790a461936023fca1c0a Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 6 Jul 2022 19:06:41 +0800 Subject: [PATCH 0865/1654] default threads on (#19368) * default threads on * make rst gcsafe * ignore threads option for nimscript * threads off * use createShared for threads * test without threads * avr threds off * avr threads off * async threads off * threads off * fix ci * restore option * make CI pleased * fix ic tests * Update config.nims * add changelog * Update changelog.md Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> Co-authored-by: konsumlamm <44230978+konsumlamm@users.noreply.github.com> --- changelog.md | 2 ++ config/nim.cfg | 2 ++ testament/categories.nim | 10 +++++----- tests/avr/thello.nim | 2 +- tests/destructor/trecursive.nim | 2 +- tests/ic/config.nims | 2 ++ tests/manyloc/standalone2/tavr.nim.cfg | 1 + tests/niminaction/Chapter7/Tweeter/src/tweeter.nim | 1 + tests/stdlib/ttasks.nim | 2 +- tests/system/tgcnone.nim | 2 +- 10 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 tests/ic/config.nims diff --git a/changelog.md b/changelog.md index 98655eaa3..c35a403e6 100644 --- a/changelog.md +++ b/changelog.md @@ -26,6 +26,8 @@ becomes an alias for `addr`. - The `gc:v2` option is removed. +- The `threads:on` option becomes the default. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/config/nim.cfg b/config/nim.cfg index 7fe279f56..37d12a02d 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -17,6 +17,8 @@ cc = gcc hint[LineTooLong]=off #hint[XDeclaredButNotUsed]=off +threads:on + # Examples of how to setup a cross-compiler: # Nim can target architectures and OSes different than the local host # Syntax: ..gcc.exe = "" diff --git a/testament/categories.nim b/testament/categories.nim index a092fec84..43ea9f0ee 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -59,10 +59,10 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) test3.spec.action = actionCompile testSpec c, test3 - var test4 = makeTest("tests/dll/visibility.nim", options & " --app:lib" & rpath, cat) + var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 @@ -77,13 +77,13 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = defer: putEnv(libpathenv, libpath) testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) - testSpec r, makeTest("tests/dll/visibility.nim", options & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: # force build required - see the comments in the .nim file for more details var hcri = makeTest("tests/dll/nimhcr_integration.nim", - options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + options & " --threads:off --forceBuild --hotCodeReloading:on" & rpath, cat) let nimcache = nimcacheDir(hcri.name, hcri.options, getTestSpecTarget()) let cmd = prepareTestCmd(hcri.spec.getCmd, hcri.name, hcri.options, nimcache, getTestSpecTarget()) diff --git a/tests/avr/thello.nim b/tests/avr/thello.nim index a0191815c..7ebaeae5f 100644 --- a/tests/avr/thello.nim +++ b/tests/avr/thello.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim c --compileOnly --os:standalone --exceptions:quirky -d:noSignalHandler -d:danger $file" + cmd: "nim c --compileOnly --os:standalone --exceptions:quirky -d:noSignalHandler -d:danger --threads:off $file" action: "compile" """ diff --git a/tests/destructor/trecursive.nim b/tests/destructor/trecursive.nim index 17a40e5a9..e7afa6ba9 100644 --- a/tests/destructor/trecursive.nim +++ b/tests/destructor/trecursive.nim @@ -47,7 +47,7 @@ proc `=destroy`(x: var MyObject) = proc `=`(x: var MyObject, y: MyObject) {.error.} proc newMyObject(i: int): MyObject = - result.p = create(int) + result.p = createShared(int) result.p[] = i proc test: seq[MyObject] = diff --git a/tests/ic/config.nims b/tests/ic/config.nims new file mode 100644 index 000000000..76b29a6aa --- /dev/null +++ b/tests/ic/config.nims @@ -0,0 +1,2 @@ +when defined(windows): + --threads:off diff --git a/tests/manyloc/standalone2/tavr.nim.cfg b/tests/manyloc/standalone2/tavr.nim.cfg index e5291969d..2a31618d0 100644 --- a/tests/manyloc/standalone2/tavr.nim.cfg +++ b/tests/manyloc/standalone2/tavr.nim.cfg @@ -2,3 +2,4 @@ --cpu:avr --os:standalone --compileOnly +--threads:off diff --git a/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim b/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim index fe39278fb..2fac949b9 100644 --- a/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim +++ b/tests/niminaction/Chapter7/Tweeter/src/tweeter.nim @@ -1,5 +1,6 @@ discard """ action: compile +matrix: "--threads:off" """ import asyncdispatch, times diff --git a/tests/stdlib/ttasks.nim b/tests/stdlib/ttasks.nim index 75fed9f9b..e90823aba 100644 --- a/tests/stdlib/ttasks.nim +++ b/tests/stdlib/ttasks.nim @@ -1,6 +1,6 @@ discard """ targets: "c cpp" - matrix: "--gc:orc" + matrix: "--gc:orc --threads:off" """ import std/[tasks, strformat] diff --git a/tests/system/tgcnone.nim b/tests/system/tgcnone.nim index 700176d5f..47c6c6014 100644 --- a/tests/system/tgcnone.nim +++ b/tests/system/tgcnone.nim @@ -1,5 +1,5 @@ discard """ - matrix: "--gc:none -d:useMalloc" + matrix: "--gc:none -d:useMalloc --threads:off" """ # bug #15617 let x = 4 From dcb28fd061743ba141b3914501512b5eb352aa1b Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 7 Jul 2022 20:14:05 +0800 Subject: [PATCH 0866/1654] cache rope when threads are enabled (#19981) * cache rope * add threadvar --- compiler/ropes.nim | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index a44d84ddc..610159c75 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -86,13 +86,12 @@ proc newRope(data: string = ""): Rope = result.L = -data.len result.data = data -when not compileOption("threads"): - var - cache: array[0..2048*2 - 1, Rope] +var + cache {.threadvar.} : array[0..2048*2 - 1, Rope] - proc resetRopeCache* = - for i in low(cache)..high(cache): - cache[i] = nil +proc resetRopeCache* = + for i in low(cache)..high(cache): + cache[i] = nil proc ropeInvariant(r: Rope): bool = if r == nil: From ad0aee535435ac9b22c2ee9ef02085ffdc901d48 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 7 Jul 2022 15:26:58 +0200 Subject: [PATCH 0867/1654] sysrand: fix syscall signature [backport] (#19982) sysrand: fix syscall signature `syscall` is a `C` varags function --- lib/std/sysrand.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/std/sysrand.nim b/lib/std/sysrand.nim index 1b7b2c024..4ee25d01e 100644 --- a/lib/std/sysrand.nim +++ b/lib/std/sysrand.nim @@ -170,9 +170,8 @@ elif defined(linux) and not defined(nimNoGetRandom) and not defined(emscripten): const syscallHeader = """#include #include """ - proc syscall( - n: clong, buf: pointer, bufLen: cint, flags: cuint - ): clong {.importc: "syscall", header: syscallHeader.} + proc syscall(n: clong): clong {. + importc: "syscall", varargs, header: syscallHeader.} # When reading from the urandom source (GRND_RANDOM is not set), # getrandom() will block until the entropy pool has been # initialized (unless the GRND_NONBLOCK flag was specified). If a @@ -211,7 +210,7 @@ elif defined(zephyr): proc sys_csrand_get(dst: pointer, length: csize_t): cint {.importc: "sys_csrand_get", header: "".} # Fill the destination buffer with cryptographically secure # random data values - # + # proc getRandomImpl(p: pointer, size: int): int {.inline.} = # 0 if success, -EIO if entropy reseed error From 7a29a782f84ebceec0fb5c3d39bb484621831fca Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 9 Jul 2022 06:46:05 +0200 Subject: [PATCH 0868/1654] removed caching logic; saves 400MB for an ORC booting compiler (#19989) --- compiler/injectdestructors.nim | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 6500c5bc7..867d30d96 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -80,12 +80,6 @@ import sets, hashes proc hash(n: PNode): Hash = hash(cast[pointer](n)) -proc aliasesCached(cache: var Table[(PNode, PNode), AliasKind], obj, field: PNode): AliasKind = - let key = (obj, field) - if not cache.hasKey(key): - cache[key] = aliases(obj, field) - cache[key] - type State = ref object lastReads: IntSet @@ -116,9 +110,8 @@ proc mergeStates(a: var State, b: sink State) = a.alreadySeen.incl b.alreadySeen proc computeLastReadsAndFirstWrites(cfg: ControlFlowGraph) = - var cache = initTable[(PNode, PNode), AliasKind]() template aliasesCached(obj, field: PNode): AliasKind = - aliasesCached(cache, obj, field) + aliases(obj, field) var cfg = cfg preprocessCfg(cfg) From e8ee2f9c2ad06cad2f62fe7505acbb43530f28d7 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sat, 9 Jul 2022 17:28:14 +0800 Subject: [PATCH 0869/1654] update section regarding `std/assertions` in changelog (#19992) Update changelog.md --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index c35a403e6..770393e13 100644 --- a/changelog.md +++ b/changelog.md @@ -22,7 +22,7 @@ - `addr` is now available for all addressable locations, `unsafeAddr` is deprecated and becomes an alias for `addr`. -- io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`. +- `io` and `assertions` are about to move out of system; use `-d:nimPreviewSlimSystem`, import `std/syncio` and import `std/assertions`. - The `gc:v2` option is removed. From ed2bf02a511ae230243ce384ad16c318ca3fe661 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Sun, 10 Jul 2022 15:37:15 +0800 Subject: [PATCH 0870/1654] remove `when declared(cache)`; cache is always there (#19991) --- compiler/ropes.nim | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index 610159c75..677a3ce09 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -111,16 +111,13 @@ var gCacheMisses* = 0 var gCacheIntTries* = 0 proc insertInCache(s: string): Rope = - when declared(cache): - inc gCacheTries - var h = hash(s) and high(cache) - result = cache[h] - if isNil(result) or result.data != s: - inc gCacheMisses - result = newRope(s) - cache[h] = result - else: + inc gCacheTries + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or result.data != s: + inc gCacheMisses result = newRope(s) + cache[h] = result proc rope*(s: string): Rope = ## Converts a string to a rope. From a90763ebd762a4862e2af1dab7d4429f4e984a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20M=20G=C3=B3mez?= Date: Sun, 10 Jul 2022 15:40:26 +0200 Subject: [PATCH 0871/1654] Fixes Compilation error with --app:lib (#19965) Fixes Compilation error with --app:lib when a module tries to pull os.paramStr on posix by throwing a runtime exception instead. More details here: #19964 --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 247a0d089..a60e913f1 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2939,7 +2939,7 @@ elif defined(genode): proc paramCount*(): int = raise newException(OSError, "paramCount is not implemented on Genode") -elif weirdTarget: +elif weirdTarget or (defined(posix) and appType == "lib"): proc paramStr*(i: int): string {.tags: [ReadIOEffect].} = raise newException(OSError, "paramStr is not implemented on current platform") From fb5fbf1e087563f0288b8ed684c8dcc1891730b0 Mon Sep 17 00:00:00 2001 From: Tanguy Date: Mon, 11 Jul 2022 11:28:52 +0200 Subject: [PATCH 0872/1654] Fix nested finally handling in closureiters [backport] (#19933) * Fix nested finally handling in closureiters * Fix CI * review comment * third time the charm * Update compiler/closureiters.nim Co-authored-by: Dominik Picheta Co-authored-by: Dominik Picheta --- compiler/closureiters.nim | 31 ++++++++++++--- tests/iter/titer_issues.nim | 75 +++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 6370d0dcb..2848942fa 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -121,7 +121,10 @@ # yield 2 # if :unrollFinally: # This node is created by `newEndFinallyNode` # if :curExc.isNil: -# return :tmpResult +# if nearestFinally == 0: +# return :tmpResult +# else: +# :state = nearestFinally # bubble up # else: # closureIterSetupExc(nil) # raise @@ -807,7 +810,10 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode = # Generate the following code: # if :unrollFinally: # if :curExc.isNil: - # return :tmpResult + # if nearestFinally == 0: + # return :tmpResult + # else: + # :state = nearestFinally # bubble up # else: # raise let curExc = ctx.newCurExcAccess() @@ -816,11 +822,17 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode = let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode) cmp.typ = ctx.g.getSysType(info, tyBool) - let asgn = newTree(nkFastAsgn, - newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info), - ctx.newTmpResultAccess()) + let retStmt = + if ctx.nearestFinally == 0: + # last finally, we can return + let asgn = newTree(nkFastAsgn, + newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info), + ctx.newTmpResultAccess()) + newTree(nkReturnStmt, asgn) + else: + # bubble up to next finally + newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally)) - let retStmt = newTree(nkReturnStmt, asgn) let branch = newTree(nkElifBranch, cmp, retStmt) let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode) @@ -864,6 +876,13 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode = of nkSkip: discard + of nkTryStmt: + if n.hasYields: + # the inner try will handle these transformations + discard + else: + for i in 0.. try + iterator p1: int {.closure.} = + try: + yield 0 + try: + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p1(): + discard + + # try -> try yield + iterator p2: int {.closure.} = + try: + try: + yield 0 + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p2(): + discard + + # try yield -> try yield + iterator p3: int {.closure.} = + try: + yield 0 + try: + yield 0 + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p3(): + discard + + # try -> try + iterator p4: int {.closure.} = + try: + try: + return + finally: + echo "nested finally" + echo "shouldn't run" + finally: + echo "outer finally" + echo "shouldn't run" + + for _ in p4(): + discard From cf1c14936670ef0c175149f6fba4dffc1cf4ba43 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Mon, 11 Jul 2022 23:27:01 +0800 Subject: [PATCH 0873/1654] tracking the memory usage of orc-booting compiler for each commit (#19941) * yaml * pub * redo * let's comment * now action * newly * code name * build * ready * remove submodule * build * modify name * fix * rephrase * trigger when PR is merged --- .github/workflows/ci_publish.yml | 90 ++++++++++++++++++++++++++++++++ ci/action.nim | 28 ++++++++++ 2 files changed, 118 insertions(+) create mode 100644 .github/workflows/ci_publish.yml create mode 100644 ci/action.nim diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml new file mode 100644 index 000000000..94ea5ff8f --- /dev/null +++ b/.github/workflows/ci_publish.yml @@ -0,0 +1,90 @@ +name: Tracking orc-booting compiler memory usage + +on: + push: + branches: + - devel + + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + cpu: [amd64] + name: '${{ matrix.os }}' + runs-on: ${{ matrix.os }} + steps: + - name: 'Checkout' + uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: 'Install node.js 16.x' + uses: actions/setup-node@v2 + with: + node-version: '16.x' + + - name: 'Install dependencies (Linux amd64)' + if: runner.os == 'Linux' && matrix.cpu == 'amd64' + run: | + sudo apt-fast update -qq + DEBIAN_FRONTEND='noninteractive' \ + sudo apt-fast install --no-install-recommends -yq \ + libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \ + valgrind libc6-dbg libblas-dev xorg-dev + - name: 'Install dependencies (macOS)' + if: runner.os == 'macOS' + run: brew install boehmgc make sfml gtk+3 + - name: 'Install dependencies (Windows)' + if: runner.os == 'Windows' + shell: bash + run: | + set -e + . ci/funs.sh + nimInternalInstallDepsWindows + echo_run echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" + + - name: 'Add build binaries to PATH' + shell: bash + run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" + + - name: 'System information' + shell: bash + run: . ci/funs.sh && nimCiSystemInfo + + - name: 'Build csourcesAny' + shell: bash + run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}' + + - name: 'Build koch' + shell: bash + run: nim c koch + + - name: 'Build Nim' + shell: bash + run: ./koch boot -d:release -d:nimStrictMode --lib:lib + + - name: 'Action' + shell: bash + run: nim c -r -d:release ci/action.nim + + - name: 'Comment' + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + + try { + const data = fs.readFileSync('ci/nimcache/results.txt', 'utf8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: data + }) + } catch (err) { + console.error(err); + } + diff --git a/ci/action.nim b/ci/action.nim new file mode 100644 index 000000000..8c3260096 --- /dev/null +++ b/ci/action.nim @@ -0,0 +1,28 @@ +import std/[strutils, os, osproc, parseutils, strformat] + + +proc main() = + var msg = "" + const cmd = "./koch boot --gc:orc -d:release" + + let (output, exitCode) = execCmdEx(cmd) + + doAssert exitCode == 0, output + + var start = rfind(output, "Hint: gc") + if start < 0: + start = rfind(output, "Hint: mm") + doAssert parseUntil(output, msg, "; proj", start) > 0, output + + let (commitHash, _) = execCmdEx("""git log --format="%H" -n 1""") + + let welcomeMessage = fmt"""Thanks for your hard work on this PR! +The lines below are statistics of the Nim compiler built from {commitHash} + +{msg} +""" + createDir "ci/nimcache" + writeFile "ci/nimcache/results.txt", welcomeMessage + +when isMainModule: + main() \ No newline at end of file From d0bae989d63f91345cd0991e00502c05d18dbf6b Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 12 Jul 2022 09:05:50 +0800 Subject: [PATCH 0874/1654] fix github script failure (#20006) * yaml * pub * redo * let's comment * now action * newly * code name * build * ready * remove submodule * build * modify name * fix * rephrase * trigger when PR is merged * fix CI failure --- .github/workflows/ci_publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml index 94ea5ff8f..90a3438a7 100644 --- a/.github/workflows/ci_publish.yml +++ b/.github/workflows/ci_publish.yml @@ -1,13 +1,13 @@ name: Tracking orc-booting compiler memory usage on: - push: - branches: - - devel + pull_request: + types: [closed] jobs: build: + if: github.event.pull_request.merged == true strategy: fail-fast: false matrix: From a97b00ad8d18d836d693c224909b607d2af0c4b1 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Tue, 12 Jul 2022 19:07:18 +0800 Subject: [PATCH 0875/1654] try to optimize hot spots for orc-booting compiler (#20001) * optimize hot spots fro orc-booting compiler * remove GC_ref * minor * remove comments * Revert "minor" This reverts commit 4965a190a2d6457044faa1442795e55bdad57602. * emulate cursor --- compiler/astalgo.nim | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index bc5b7d6e1..22f4257fc 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -18,6 +18,9 @@ import when defined(nimPreviewSlimSystem): import std/assertions +when not defined(nimHasCursor): + {.pragma: cursor.} + proc hashNode*(p: RootRef): Hash proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope # Convert a tree into its YAML representation; this is used by the @@ -815,16 +818,21 @@ type name*: PIdent proc nextIdentIter*(ti: var TIdentIter, tab: TStrTable): PSym = + # hot spots var h = ti.h and high(tab.data) var start = h - result = tab.data[h] - while result != nil: - if result.name.id == ti.name.id: break + var p {.cursor.} = tab.data[h] + while p != nil: + if p.name.id == ti.name.id: break h = nextTry(h, high(tab.data)) if h == start: - result = nil + p = nil break - result = tab.data[h] + p = tab.data[h] + if p != nil: + result = p # increase the count + else: + result = nil ti.h = nextTry(h, high(tab.data)) proc initIdentIter*(ti: var TIdentIter, tab: TStrTable, s: PIdent): PSym = From 5c510a9ab96f265ffb0323a18deadb5bb175cfdc Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Tue, 12 Jul 2022 19:03:58 +0300 Subject: [PATCH 0876/1654] allow dots in defined() (#20010) * allow dots in defined() refs https://github.com/nim-lang/RFCs/issues/181 * mention accents in older versions --- changelog.md | 3 +++ compiler/semexprs.nim | 14 +++++++++++++- tests/misc/tdefine.nim | 19 +++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 770393e13..7498a9567 100644 --- a/changelog.md +++ b/changelog.md @@ -101,6 +101,9 @@ becomes an alias for `addr`. - Full command syntax and block arguments i.e. `foo a, b: c` are now allowed for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". +- `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use accents as in ``defined(`a.b.c`)`` to access such defines. ## Compiler changes diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index e07a98417..fed9cae21 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1938,11 +1938,23 @@ proc semYield(c: PContext, n: PNode): PNode = elif c.p.owner.typ[0] != nil: localError(c.config, n.info, errGenerated, "yield statement must yield a value") +proc considerQuotedIdentOrDot(c: PContext, n: PNode, origin: PNode = nil): PIdent = + if n.kind == nkDotExpr: + let a = considerQuotedIdentOrDot(c, n[0], origin).s + let b = considerQuotedIdentOrDot(c, n[1], origin).s + var s = newStringOfCap(a.len + b.len + 1) + s.add(a) + s.add('.') + s.add(b) + result = getIdent(c.cache, s) + else: + result = considerQuotedIdent(c, n, origin) + proc semDefined(c: PContext, n: PNode): PNode = checkSonsLen(n, 2, c.config) # we replace this node by a 'true' or 'false' node: result = newIntNode(nkIntLit, 0) - result.intVal = ord isDefined(c.config, considerQuotedIdent(c, n[1], n).s) + result.intVal = ord isDefined(c.config, considerQuotedIdentOrDot(c, n[1], n).s) result.info = n.info result.typ = getSysType(c.graph, n.info, tyBool) diff --git a/tests/misc/tdefine.nim b/tests/misc/tdefine.nim index f1c6e7a96..c4d11c941 100644 --- a/tests/misc/tdefine.nim +++ b/tests/misc/tdefine.nim @@ -1,6 +1,6 @@ discard """ joinable: false -cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -r $file" +cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -d:namespaced.define=false -d:double.namespaced.define -r $file" """ const booldef {.booldefine.} = false @@ -27,4 +27,19 @@ type T = object when intdef2 == 1: field2: int when strdef2 == "abc": - field3: int \ No newline at end of file + field3: int + +doAssert not defined(booldef3) +doAssert not defined(intdef2) +doAssert not defined(strdef2) +discard T(field1: 1, field2: 2, field3: 3) + +doAssert defined(namespaced.define) +const `namespaced.define` {.booldefine.} = true +doAssert not `namespaced.define` + +doAssert defined(double.namespaced.define) +const `double.namespaced.define` {.booldefine.} = false +doAssert `double.namespaced.define` + +doAssert not defined(namespaced.butnotdefined) From 0180c6179aa0ff2dacaa6c063c0fa24cf459a84d Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 13 Jul 2022 01:35:08 +0800 Subject: [PATCH 0877/1654] fix #18735; genDepend broken for duplicate module names in separate folders (#19988) --- compiler/depends.nim | 58 ++++++++++++++++++++++++++++++++++++++------ compiler/options.nim | 2 +- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/compiler/depends.nim b/compiler/depends.nim index 30fc961c5..1dcdec989 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -9,10 +9,12 @@ # This module implements a dependency file generator. -import - options, ast, ropes, idents, passes, modulepaths, pathutils +import options, ast, ropes, passes, pathutils, msgs, lineinfos -from modulegraphs import ModuleGraph, PPassContext +import modulegraphs + +import std/[os, strutils, parseutils] +import std/private/globs type TGen = object of PPassContext @@ -28,6 +30,50 @@ proc addDependencyAux(b: Backend; importing, imported: string) = b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)]) # s1 -> s2_4[label="[0-9]"]; +proc toNimblePath(s: string, isStdlib: bool): string = + const stdPrefix = "std/" + const pkgPrefix = "pkg/" + if isStdlib: + let sub = "lib/" + var start = s.find(sub) + if start < 0: + doAssert false + else: + start += sub.len + let base = s[start..^1] + + if base.startsWith("system") or base.startsWith("std"): + result = base + else: + for dir in stdlibDirs: + if base.startsWith(dir): + return stdPrefix & base.splitFile.name + + result = stdPrefix & base + else: + var sub = getEnv("NIMBLE_DIR") + if sub.len == 0: + sub = ".nimble/pkgs/" + else: + sub.add "/pkgs/" + var start = s.find(sub) + if start < 0: + result = s + else: + start += sub.len + start += skipUntil(s, '/', start) + start += 1 + result = pkgPrefix & s[start..^1] + +proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) = + doAssert n.kind == nkSym, $n.kind + + let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex)) + let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex)) + let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module)) + let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym)) + addDependencyAux(b, parent, child) + proc addDotDependency(c: PPassContext, n: PNode): PNode = result = n let g = PGen(c) @@ -35,11 +81,9 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode = case n.kind of nkImportStmt: for i in 0.. 0 and not ret.startsWith ".." -const stdlibDirs = [ +const stdlibDirs* = [ "pure", "core", "arch", "pure/collections", "pure/concurrency", From af140966ea5b58ea248459acb74bf84537deb423 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Wed, 13 Jul 2022 00:24:17 +0300 Subject: [PATCH 0878/1654] Add #19406 changes to changelog (#20011) --- changelog.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 7498a9567..68b133040 100644 --- a/changelog.md +++ b/changelog.md @@ -66,6 +66,12 @@ becomes an alias for `addr`. ## Language changes +- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Templates now accept [macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas). +- Macro pragmas for var/let/const sections have been redesigned in a way that works + similarly to routine macro pragmas. The new behavior is documented in the + [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, allowing multiple type definitions to be injected in place of the original type definition. @@ -96,8 +102,6 @@ becomes an alias for `addr`. x, y, z: int Baz = object ``` -- [Case statement macros](manual.html#macros-case-statement-macros) are no longer experimental, - meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. - Full command syntax and block arguments i.e. `foo a, b: c` are now allowed for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". From 489f6ddfefc4993af97eb2884b6d8173a4ee8b9d Mon Sep 17 00:00:00 2001 From: Daniel Clarke Date: Wed, 13 Jul 2022 22:55:33 +1000 Subject: [PATCH 0879/1654] fix #20012 (#20013) * replace gcc asm with __asm__ and add a test * update test case to specify gcc or clang and not cpp Co-authored-by: daniel --- compiler/extccomp.nim | 2 +- tests/compiler/tasm.nim | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/compiler/tasm.nim diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index f1dc01f44..23c43cb67 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -86,7 +86,7 @@ compiler gcc: linkLibCmd: " -l$1", debug: "", pic: "-fPIC", - asmStmtFrmt: "asm($1);$n", + asmStmtFrmt: "__asm__($1);$n", structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name produceAsm: gnuAsmListing, cppXsupport: "-std=gnu++14 -funsigned-char", diff --git a/tests/compiler/tasm.nim b/tests/compiler/tasm.nim new file mode 100644 index 000000000..9f60231e0 --- /dev/null +++ b/tests/compiler/tasm.nim @@ -0,0 +1,15 @@ +proc testAsm() = + let src = 41 + var dst = 0 + + asm """ + mov %1, %0\n\t + add $1, %0 + : "=r" (`dst`) + : "r" (`src`)""" + + doAssert dst == 42 + +when defined(gcc) or defined(clang) and not defined(cpp): + {.passc: "-std=c99".} + testAsm() \ No newline at end of file From 73ee34f56fc267e6b8b29d12b22f969f7f8a7788 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Wed, 13 Jul 2022 22:37:31 +0800 Subject: [PATCH 0880/1654] use pull_request_target (#20020) --- .github/workflows/ci_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_publish.yml b/.github/workflows/ci_publish.yml index 90a3438a7..a56ff9a7e 100644 --- a/.github/workflows/ci_publish.yml +++ b/.github/workflows/ci_publish.yml @@ -1,7 +1,7 @@ name: Tracking orc-booting compiler memory usage on: - pull_request: + pull_request_target: types: [closed] From 93211a2bddf8e6a806c3fd4e9322ca21f0a2a9c4 Mon Sep 17 00:00:00 2001 From: silent-observer Date: Thu, 14 Jul 2022 13:03:16 +0300 Subject: [PATCH 0881/1654] Add sink and lent annotations to the critbits module (#20021) * Add sink and lent to critbits * Remove lent for pairs I guess lent doesn't work well inside tuples * Remove lent from template in critbits Apparently this also doesn't work, because some checks failed --- lib/pure/collections/critbits.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim index e12984995..dca60b37b 100644 --- a/lib/pure/collections/critbits.nim +++ b/lib/pure/collections/critbits.nim @@ -197,7 +197,7 @@ proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool = discard exclImpl(c, key) result = c.count == oldCount -proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: T): bool = +proc containsOrIncl*[T](c: var CritBitTree[T], key: string, val: sink T): bool = ## Returns true if `c` contains the given `key`. If the key does not exist, ## `c[key] = val` is performed. ## @@ -270,7 +270,7 @@ proc incl*(c: var CritBitTree[void], key: string) = discard rawInsert(c, key) -proc incl*[T](c: var CritBitTree[T], key: string, val: T) = +proc incl*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Inserts `key` with value `val` into `c`. ## ## **See also:** @@ -284,7 +284,7 @@ proc incl*[T](c: var CritBitTree[T], key: string, val: T) = var n = rawInsert(c, key) n.val = val -proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) = +proc `[]=`*[T](c: var CritBitTree[T], key: string, val: sink T) = ## Alias for `incl <#incl,CritBitTree[T],string,T>`_. ## ## **See also:** @@ -300,7 +300,7 @@ template get[T](c: CritBitTree[T], key: string): T = n.val -func `[]`*[T](c: CritBitTree[T], key: string): T {.inline.} = +func `[]`*[T](c: CritBitTree[T], key: string): lent T {.inline.} = ## Retrieves the value at `c[key]`. If `key` is not in `t`, the ## `KeyError` exception is raised. One can check with `hasKey` whether ## the key exists. @@ -342,7 +342,7 @@ iterator keys*[T](c: CritBitTree[T]): string = for x in leaves(c.root): yield x.key -iterator values*[T](c: CritBitTree[T]): T = +iterator values*[T](c: CritBitTree[T]): lent T = ## Yields all values of `c` in the lexicographical order of the ## corresponding keys. ## @@ -415,7 +415,7 @@ iterator keysWithPrefix*[T](c: CritBitTree[T], prefix: string): string = let top = allprefixedAux(c, prefix) for x in leaves(top): yield x.key -iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): T = +iterator valuesWithPrefix*[T](c: CritBitTree[T], prefix: string): lent T = ## Yields all values of `c` starting with `prefix` of the ## corresponding keys. ## @@ -518,7 +518,7 @@ func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} = else: c.root.byte else: 0 -proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = +proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `pairs`. runnableExamples: doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string] @@ -526,7 +526,7 @@ proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: ( for item in pairs: result.incl item[0], item[1] -proc toCritBitTree*(items: openArray[string]): CritBitTree[void] {.since: (1, 3).} = +proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} = ## Creates a new `CritBitTree` that contains the given `items`. runnableExamples: doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void] From 10c8e2037db9ec3dc53fdaf1cc27e837e204af76 Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Thu, 14 Jul 2022 18:42:56 +0800 Subject: [PATCH 0882/1654] fixes #20015; document `shallowCopy` does a deep copy with ARC/ORC (#20025) --- lib/system.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index 4080fee06..6e8519cf6 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -473,6 +473,8 @@ proc shallowCopy*[T](x: var T, y: T) {.noSideEffect, magic: "ShallowCopy".} ## Be careful with the changed semantics though! ## There is a reason why the default assignment does a deep copy of sequences ## and strings. + ## + ## .. warning:: `shallowCopy` does a deep copy with ARC/ORC. # :array|openArray|string|seq|cstring|tuple proc `[]`*[I: Ordinal;T](a: T; i: I): T {. From 800cb006e74f8c52bde69cfc1eb9e55fcc439633 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Thu, 14 Jul 2022 08:20:40 -0400 Subject: [PATCH 0883/1654] Change `styleCheck` to ignore foreign packages (#19822) * Change `styleCheck` to ignore foreign packages * Symbols from foreign packages are now ignored. * Fixed `styleCheck` violations in `compiler` package. * Added symbol ownership to custom annotation pragmas. * Minor refactors to cleanup style check callsites. * Minor internal documentation of reasons why a symbol isn't checked. Style violations were fixed in the compiler after thet were exposed by the changes. The compiler wouldn't compile otherwise. Symbol ownership for custom pragma annotations is needed for checking the annotation's style. A NPE was raised otherwise. Fixes #10201 See also nim-lang/RFCs#456 * Fix a misunderstanding about excluding field style checks I had refactored the callsites of `styleCheckUse` to apply the DRY principle, but I misunderstood the field access handling in a template as a general case. This corrects it. * Fix some `styleCheck` violations in `compiler/evalffi` The violations were exposed in CI when the compiler was built with libffi. * Removed some uneeded transitionary code * Add changelog entry Co-authored-by: quantimnot --- changelog.md | 2 ++ compiler/evalffi.nim | 8 ++--- compiler/ic/bitabs.nim | 8 ++--- compiler/linter.nim | 67 +++++++++++++++++++++++++----------------- compiler/msgs.nim | 4 +-- compiler/pragmas.nim | 6 ++-- compiler/semexprs.nim | 2 +- compiler/semgnrc.nim | 2 +- compiler/semmagic.nim | 2 +- compiler/semstmts.nim | 14 ++++----- compiler/semtempl.nim | 12 ++++---- compiler/semtypes.nim | 8 ++--- compiler/suggest.nim | 3 +- compiler/vm.nim | 8 ++--- compiler/vmops.nim | 64 ++++++++++++++++++++-------------------- compiler/wordrecg.nim | 14 ++++----- 16 files changed, 118 insertions(+), 106 deletions(-) diff --git a/changelog.md b/changelog.md index 68b133040..47479fc5c 100644 --- a/changelog.md +++ b/changelog.md @@ -114,6 +114,8 @@ becomes an alias for `addr`. - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, without requiring `-d:nimVersion140` which is now a noop. +- `--styleCheck` now only applies to the current package. + ## Tool changes diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 5004011ed..d1d88a1fa 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -110,8 +110,8 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI else: globalError(conf, info, "cannot map calling convention to FFI") -template rd(T, p: untyped): untyped = (cast[ptr T](p))[] -template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v +template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[] +template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v template `+!`(x, y: untyped): untyped = cast[pointer](cast[ByteAddress](x) + y) @@ -177,8 +177,8 @@ const maxPackDepth = 20 var packRecCheck = 0 proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) = - template awr(T, v: untyped): untyped = - wr(T, res, v) + template awr(typ, v: untyped): untyped = + wr(typ, res, v) case typ.kind of tyBool: awr(bool, v.intVal != 0) diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim index ae673b574..8adab8388 100644 --- a/compiler/ic/bitabs.nim +++ b/compiler/ic/bitabs.nim @@ -93,13 +93,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId = t.vals.add v -proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} = - let idx = idToIdx LitId +proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} = + let idx = idToIdx litId assert idx < t.vals.len result = t.vals[idx] -proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} = - let idx = idToIdx LitId +proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} = + let idx = idToIdx litId assert idx < t.vals.len result = t.vals[idx] diff --git a/compiler/linter.nim b/compiler/linter.nim index 5fb646051..2c0ad4d6f 100644 --- a/compiler/linter.nim +++ b/compiler/linter.nim @@ -12,7 +12,8 @@ import std/strutils from std/sugar import dup -import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs +import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages +export packages const Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'} @@ -85,24 +86,32 @@ proc differ*(line: string, a, b: int, x: string): string = result = y proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) = - # operators stay as they are: - if k in {skResult, skTemp} or s.name.s[0] notin Letters: return - if k in {skType, skGenericParam} and sfAnon in s.flags: return - if s.typ != nil and s.typ.kind == tyTypeDesc: return - if {sfImportc, sfExportc} * s.flags != {}: return - if optStyleCheck notin s.options: return let beau = beautifyName(s.name.s, k) if s.name.s != beau: lintReport(conf, info, beau, s.name.s) -template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) = - if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions: - nep1CheckDefImpl(conf, info, s, k) +template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind) = + ## Check symbol definitions adhere to NEP1 style rules. + if optStyleCheck in ctx.config.options and # ignore if styleChecks are off + hintName in ctx.config.notes and # ignore if name checks are not requested + ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages + optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage + sym.kind != skResult and # ignore `result` + sym.kind != skTemp and # ignore temporary variables created by the compiler + sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols??? + k notin {skType, skGenericParam} and # ignore types and generic params + (sym.typ == nil or sym.typ.kind != tyTypeDesc) and # ignore `typedesc` + {sfImportc, sfExportc} * sym.flags == {} and # ignore FFI + sfAnon notin sym.flags: # ignore if created by compiler + nep1CheckDefImpl(ctx.config, info, sym, k) -template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) = - styleCheckDef(conf, info, s, s.kind) -template styleCheckDef*(conf: ConfigRef; s: PSym) = - styleCheckDef(conf, s.info, s, s.kind) +template styleCheckDef*(ctx: PContext; info: TLineInfo; s: PSym) = + ## Check symbol definitions adhere to NEP1 style rules. + styleCheckDef(ctx, info, s, s.kind) + +template styleCheckDef*(ctx: PContext; s: PSym) = + ## Check symbol definitions adhere to NEP1 style rules. + styleCheckDef(ctx, s.info, s, s.kind) proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string = let line = sourceLine(conf, info) @@ -116,23 +125,27 @@ proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string = let last = first+identLen(line, first)-1 result = differ(line, first, last, newName) -proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) = - if info.fileIndex.int < 0: return - # we simply convert it to what it looks like in the definition - # for consistency - - # operators stay as they are: - if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags: - return - +proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) = let newName = s.name.s let badName = differs(conf, info, newName) if badName.len > 0: - # special rules for historical reasons - let forceHint = badName == "nnkArgList" and newName == "nnkArglist" or badName == "nnkArglist" and newName == "nnkArgList" - lintReport(conf, info, newName, badName, forceHint = forceHint, extraMsg = "".dup(addDeclaredLoc(conf, s))) + lintReport(conf, info, newName, badName, "".dup(addDeclaredLoc(conf, s))) -proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = +template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) = + ## Check symbol uses match their definition's style. + if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off + hintName in ctx.config.notes and # ignore if name checks are not requested + ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages + sym.kind != skTemp and # ignore temporary variables created by the compiler + sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols??? + sfAnon notin sym.flags: # ignore temporary variables created by the compiler + styleCheckUseImpl(ctx.config, info, sym) + +proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = let wanted = $w if pragmaName != wanted: lintReport(conf, info, wanted, pragmaName) + +template checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) = + if {optStyleHint, optStyleError} * conf.globalOptions != {}: + checkPragmaUseImpl(conf, info, w, pragmaName) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 371a886f4..6d770e72f 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -622,9 +622,9 @@ template internalAssert*(conf: ConfigRef, e: bool) = let arg = info2.toFileLineCol internalErrorImpl(conf, unknownLineInfo, arg, info2) -template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, forceHint = false, extraMsg = "") = +template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, extraMsg = "") = let m = "'$1' should be: '$2'$3" % [got, beau, extraMsg] - let msg = if optStyleError in conf.globalOptions and not forceHint: errGenerated else: hintName + let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName liMessage(conf, info, msg, m, doNothing, instLoc()) proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope = diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 2262e441b..1487a871d 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -829,8 +829,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, let ident = considerQuotedIdent(c, key) var userPragma = strTableGet(c.userPragmas, ident) if userPragma != nil: - if {optStyleHint, optStyleError} * c.config.globalOptions != {}: - styleCheckUse(c.config, key.info, userPragma) + styleCheckUse(c, key.info, userPragma) # number of pragmas increase/decrease with user pragma expansion inc c.instCounter @@ -844,8 +843,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, else: let k = whichKeyword(ident) if k in validPragmas: - if {optStyleHint, optStyleError} * c.config.globalOptions != {}: - checkPragmaUse(c.config, key.info, k, ident.s) + checkPragmaUse(c.config, key.info, k, ident.s) case k of wExportc, wExportCpp: makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index fed9cae21..1bb4f49ce 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2599,7 +2599,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode = labl.owner = c.p.owner n[0] = newSymNode(labl, n[0].info) suggestSym(c.graph, n[0].info, labl, c.graph.usageSym) - styleCheckDef(c.config, labl) + styleCheckDef(c, labl) onDef(n[0].info, labl) n[1] = semExpr(c, n[1], flags) n.typ = n[1].typ diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index e249d88e8..ca0b05fa2 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -179,7 +179,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags, proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) = let s = newSymS(skUnknown, getIdentNode(c, n), c) addPrelimDecl(c, s) - styleCheckDef(c.config, n.info, s, kind) + styleCheckDef(c, n.info, s, kind) onDef(n.info, s) proc semGenericStmt(c: PContext, n: PNode, diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index ed1826fd4..d3e12670b 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -428,7 +428,7 @@ proc semQuantifier(c: PContext; n: PNode): PNode = let op = considerQuotedIdent(c, it[0]) if op.id == ord(wIn): let v = newSymS(skForVar, it[1], c) - styleCheckDef(c.config, v) + styleCheckDef(c, v) onDef(it[1].info, v) let domain = semExprWithType(c, it[2], {efWantIterator}) v.typ = domain.typ diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 0516d08ce..03c30b6e6 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -375,7 +375,7 @@ proc semUsing(c: PContext; n: PNode): PNode = let typ = semTypeNode(c, a[^2], nil) for j in 0.. 0: @@ -271,8 +271,8 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode = when defined(nimsuggest): suggestSym(c.graph, n.info, s, c.graph.usageSym, false) # field access (dot expr) will be handled by builtinFieldAccess - if not isField and {optStyleHint, optStyleError} * c.config.globalOptions != {}: - styleCheckUse(c.config, n.info, s) + if not isField: + styleCheckUse(c, n.info, s) proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode = result = n @@ -297,7 +297,7 @@ proc semRoutineInTemplBody(c: var TemplCtx, n: PNode, k: TSymKind): PNode = var s = newGenSym(k, ident, c) s.ast = n addPrelimDecl(c.c, s) - styleCheckDef(c.c.config, n.info, s) + styleCheckDef(c.c, n.info, s) onDef(n.info, s) n[namePos] = newSymNode(s, n[namePos].info) else: @@ -431,7 +431,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = # labels are always 'gensym'ed: let s = newGenSym(skLabel, n[0], c) addPrelimDecl(c.c, s) - styleCheckDef(c.c.config, s) + styleCheckDef(c.c, s) onDef(n[0].info, s) n[0] = newSymNode(s, n[0].info) n[1] = semTemplBody(c, n[1]) @@ -624,7 +624,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = s.owner.name.s == "vm" and s.name.s == "stackTrace": incl(s.flags, sfCallsite) - styleCheckDef(c.config, s) + styleCheckDef(c, s) onDef(n[namePos].info, s) # check parameter list: #s.scope = c.currentScope diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index b4f385fe6..584da1305 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -139,7 +139,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = e.flags.incl {sfUsed, sfExported} result.n.add symNode - styleCheckDef(c.config, e) + styleCheckDef(c, e) onDef(e.info, e) if sfGenSym notin e.flags: if not isPure: @@ -476,7 +476,7 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType = else: result.n.add newSymNode(field) addSonSkipIntLit(result, typ, c.idgen) - styleCheckDef(c.config, a[j].info, field) + styleCheckDef(c, a[j].info, field) onDef(field.info, field) if result.n.len == 0: result.n = nil if isTupleRecursive(result): @@ -808,7 +808,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int, localError(c.config, info, "attempt to redefine: '" & f.name.s & "'") if a.kind == nkEmpty: father.add newSymNode(f) else: a.add newSymNode(f) - styleCheckDef(c.config, f) + styleCheckDef(c, f) onDef(f.info, f) if a.kind != nkEmpty: father.add a of nkSym: @@ -1315,7 +1315,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, result.n.add newSymNode(arg) rawAddSon(result, finalType) addParamOrResult(c, arg, kind) - styleCheckDef(c.config, a[j].info, arg) + styleCheckDef(c, a[j].info, arg) onDef(a[j].info, arg) if {optNimV1Emulation, optNimV12Emulation} * c.config.globalOptions == {}: a[j] = newSymNode(arg) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 38751fcc7..4b48ef5bb 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -598,8 +598,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) = if sfError in s.flags: userError(conf, info, s) when defined(nimsuggest): suggestSym(c.graph, info, s, c.graph.usageSym, false) - if {optStyleHint, optStyleError} * conf.globalOptions != {}: - styleCheckUse(conf, info, s) + styleCheckUse(c, info, s) markOwnerModuleAsUsed(c, s) proc safeSemExpr*(c: PContext, n: PNode): PNode = diff --git a/compiler/vm.nim b/compiler/vm.nim index e681bbc96..3cb699482 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -119,13 +119,13 @@ template move(a, b: untyped) {.dirty.} = system.shallowCopy(a, b) proc derefPtrToReg(address: BiggestInt, typ: PType, r: var TFullReg, isAssign: bool): bool = # nim bug: `isAssign: static bool` doesn't work, giving odd compiler error - template fun(field, T, rkind) = + template fun(field, typ, rkind) = if isAssign: - cast[ptr T](address)[] = T(r.field) + cast[ptr typ](address)[] = typ(r.field) else: r.ensureKind(rkind) - let val = cast[ptr T](address)[] - when T is SomeInteger | char: + let val = cast[ptr typ](address)[] + when typ is SomeInteger | char: r.field = BiggestInt(val) else: r.field = val diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 04b753ef6..69599ece3 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -56,13 +56,13 @@ template ioop(op) {.dirty.} = template macrosop(op) {.dirty.} = registerCallback(c, "stdlib.macros." & astToStr(op), `op Wrapper`) -template wrap1f_math(op) {.dirty.} = +template wrap1fMath(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = doAssert a.numArgs == 1 setResult(a, op(getFloat(a, 0))) mathop op -template wrap2f_math(op) {.dirty.} = +template wrap2fMath(op) {.dirty.} = proc `op Wrapper`(a: VmArgs) {.nimcall.} = setResult(a, op(getFloat(a, 0), getFloat(a, 1))) mathop op @@ -172,40 +172,40 @@ proc registerAdditionalOps*(c: PCtx) = proc getProjectPathWrapper(a: VmArgs) = setResult a, c.config.projectPath.string - wrap1f_math(sqrt) - wrap1f_math(cbrt) - wrap1f_math(ln) - wrap1f_math(log10) - wrap1f_math(log2) - wrap1f_math(exp) - wrap1f_math(arccos) - wrap1f_math(arcsin) - wrap1f_math(arctan) - wrap1f_math(arcsinh) - wrap1f_math(arccosh) - wrap1f_math(arctanh) - wrap2f_math(arctan2) - wrap1f_math(cos) - wrap1f_math(cosh) - wrap2f_math(hypot) - wrap1f_math(sinh) - wrap1f_math(sin) - wrap1f_math(tan) - wrap1f_math(tanh) - wrap2f_math(pow) - wrap1f_math(trunc) - wrap1f_math(floor) - wrap1f_math(ceil) - wrap1f_math(erf) - wrap1f_math(erfc) - wrap1f_math(gamma) - wrap1f_math(lgamma) + wrap1fMath(sqrt) + wrap1fMath(cbrt) + wrap1fMath(ln) + wrap1fMath(log10) + wrap1fMath(log2) + wrap1fMath(exp) + wrap1fMath(arccos) + wrap1fMath(arcsin) + wrap1fMath(arctan) + wrap1fMath(arcsinh) + wrap1fMath(arccosh) + wrap1fMath(arctanh) + wrap2fMath(arctan2) + wrap1fMath(cos) + wrap1fMath(cosh) + wrap2fMath(hypot) + wrap1fMath(sinh) + wrap1fMath(sin) + wrap1fMath(tan) + wrap1fMath(tanh) + wrap2fMath(pow) + wrap1fMath(trunc) + wrap1fMath(floor) + wrap1fMath(ceil) + wrap1fMath(erf) + wrap1fMath(erfc) + wrap1fMath(gamma) + wrap1fMath(lgamma) when declared(copySign): - wrap2f_math(copySign) + wrap2fMath(copySign) when declared(signbit): - wrap1f_math(signbit) + wrap1fMath(signbit) registerCallback c, "stdlib.math.round", proc (a: VmArgs) {.nimcall.} = let n = a.numArgs diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index c819f4306..cf2f768b1 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -89,23 +89,23 @@ type wLiftLocals = "liftlocals", wEnforceNoRaises = "enforceNoRaises", wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char", - wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default", - wDelete = "delete", wDouble = "double", wDynamic_cast = "dynamic_cast", + wClass = "class", wCompl = "compl", wConstCast = "const_cast", wDefault = "default", + wDelete = "delete", wDouble = "double", wDynamicCast = "dynamic_cast", wExplicit = "explicit", wExtern = "extern", wFalse = "false", wFloat = "float", wFriend = "friend", wGoto = "goto", wInt = "int", wLong = "long", wMutable = "mutable", wNamespace = "namespace", wNew = "new", wOperator = "operator", wPrivate = "private", wProtected = "protected", wPublic = "public", wRegister = "register", - wReinterpret_cast = "reinterpret_cast", wRestrict = "restrict", wShort = "short", - wSigned = "signed", wSizeof = "sizeof", wStatic_cast = "static_cast", wStruct = "struct", + wReinterpretCast = "reinterpret_cast", wRestrict = "restrict", wShort = "short", + wSigned = "signed", wSizeof = "sizeof", wStaticCast = "static_cast", wStruct = "struct", wSwitch = "switch", wThis = "this", wThrow = "throw", wTrue = "true", wTypedef = "typedef", wTypeid = "typeid", wTypeof = "typeof", wTypename = "typename", wUnion = "union", wPacked = "packed", wUnsigned = "unsigned", wVirtual = "virtual", - wVoid = "void", wVolatile = "volatile", wWchar_t = "wchar_t", + wVoid = "void", wVolatile = "volatile", wWchar = "wchar_t", wAlignas = "alignas", wAlignof = "alignof", wConstexpr = "constexpr", wDecltype = "decltype", wNullptr = "nullptr", wNoexcept = "noexcept", - wThread_local = "thread_local", wStatic_assert = "static_assert", - wChar16_t = "char16_t", wChar32_t = "char32_t", + wThreadLocal = "thread_local", wStaticAssert = "static_assert", + wChar16 = "char16_t", wChar32 = "char32_t", wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr", From 286fcef68ef6f0e3a20ab5b25307c9b4705ce54d Mon Sep 17 00:00:00 2001 From: flywind <43030857+xflywind@users.noreply.github.com> Date: Fri, 15 Jul 2022 15:42:54 +0800 Subject: [PATCH 0884/1654] [Orc] fixes "streams.readDataStr segafaults" when accepting a string literal (#20019) [backport] fixes streams.readDataStr accept a string literal --- lib/pure/streams.nim | 5 +++++ tests/stdlib/tstreams.nim | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index f58273ee8..50b3085d2 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -1197,6 +1197,11 @@ else: # after 1.3 or JS not defined proc ssReadDataStr(s: Stream, buffer: var string, slice: Slice[int]): int = var s = StringStream(s) + when nimvm: + discard + else: + when declared(prepareMutation): + prepareMutation(buffer) # buffer might potentially be a CoW literal with ARC result = min(slice.b + 1 - slice.a, s.data.len - s.pos) if result > 0: jsOrVmBlock: diff --git a/tests/stdlib/tstreams.nim b/tests/stdlib/tstreams.nim index d9857926e..cc1343651 100644 --- a/tests/stdlib/tstreams.nim +++ b/tests/stdlib/tstreams.nim @@ -85,3 +85,13 @@ block: static: # Ensure streams it doesnt break with nimscript on arc/orc #19716 let s = newStringStream("a") doAssert s.data == "a" + +template main = + var strm = newStringStream("abcde") + var buffer = "12345" + doAssert strm.readDataStr(buffer, 0..3) == 4 + doAssert buffer == "abcd5" + strm.close() + +static: main() +main() From f35c9cf73ddb3150ab6dbe449db4975866ee8a11 Mon Sep 17 00:00:00 2001 From: metagn <10591326+metagn@users.noreply.github.com> Date: Fri, 15 Jul 2022 13:37:08 +0300 Subject: [PATCH 0885/1654] fix #20002 (#20004) While this fix seems innocent, this unlocks the hidden behavior of method calls not being able to call gensym'ed routines inside templates. --- compiler/semtempl.nim | 2 +- tests/template/tinnerouterproc.nim | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/template/tinnerouterproc.nim diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index a599edb9c..1f76aff75 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -78,7 +78,7 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule; result = newNodeIT(kind, info, newTypeS(tyNone, c)) a = initOverloadIter(o, c, n) while a != nil: - if a.kind != skModule and (not isField or sfGenSym notin s.flags): + if a.kind != skModule and (not isField or sfGenSym notin a.flags): incl(a.flags, sfUsed) markOwnerModuleAsUsed(c, a) result.add newSymNode(a, info) diff --git a/tests/template/tinnerouterproc.nim b/tests/template/tinnerouterproc.nim new file mode 100644 index 000000000..1f15fb13e --- /dev/null +++ b/tests/template/tinnerouterproc.nim @@ -0,0 +1,8 @@ +block: # #20002 + proc bar(x: int): int = 10 + template foo = + proc bar(x: int): int {.gensym.} = x + 2 + doAssert bar(3) == 5 + discard 3.bar # evaluates to 10 but only check if it compiles for now + block: + foo() From 417b90a7e5b88bfc0ad1bfbbc81a3205c99e128e Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Fri, 15 Jul 2022 20:27:54 +0300 Subject: [PATCH 0886/1654] Improve Markdown code blocks & start moving docs to Markdown style (#19954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add additional parameters parsing (other implementations will just ignore them). E.g. if in RST we have: .. code:: nim :test: "nim c $1" ... then in Markdown that will be: ```nim test="nim c $1" ... ``` - implement Markdown interpretation of additional indentation which is less than 4 spaces (>=4 spaces is a code block but it's not implemented yet). RST interpretes it as quoted block, for Markdown it's just normal paragraphs. - add separate `md2html` and `md2tex` commands. This is to separate Markdown behavior in cases when it diverges w.r.t. RST significantly — most conspicously like in the case of additional indentation above, and also currently the contradicting inline rule of Markdown is also turned on only in `md2html` and `md2tex`. **Rationale:** mixing Markdown and RST arbitrarily is a way to nowhere, we need to provide a way to fix the particular behavior. Note that still all commands have **both** Markdown and RST features **enabled**. In this PR `*.nim` files can be processed only in Markdown mode, while `md2html` is for `*.md` files and `rst2html` for `*.rst` files. - rename `*.rst` files to `.*md` as our current default behavior is already Markdown-ish - convert code blocks in `docgen.rst` to Markdown style as an example. Other code blocks will be converted in the follow-up PRs - fix indentation inside Markdown code blocks — additional indentation is preserved there - allow more than 3 backticks open/close blocks (tildas \~ are still not allowed to avoid conflict with RST adornment headings) see also https://github.com/nim-lang/RFCs/issues/355 - better error messages - (other) fix a bug that admonitions cannot be used in sandbox mode; fix annoying warning on line 2711 --- compiler/commands.nim | 5 +- compiler/docgen.nim | 33 ++-- compiler/lineinfos.nim | 2 + compiler/main.nim | 13 +- compiler/nim.nim | 2 +- compiler/options.nim | 2 + doc/{apis.rst => apis.md} | 0 doc/{backends.rst => backends.md} | 3 +- doc/{contributing.rst => contributing.md} | 2 +- doc/{destructors.rst => destructors.md} | 0 doc/{docgen.rst => docgen.md} | 63 ++++---- doc/{docs.rst => docs.md} | 0 doc/{docstyle.rst => docstyle.md} | 0 doc/{drnim.rst => drnim.md} | 0 doc/{estp.rst => estp.md} | 0 doc/{filters.rst => filters.md} | 0 doc/{hcr.rst => hcr.md} | 0 doc/{idetools.rst => idetools.md} | 0 doc/{intern.rst => intern.md} | 6 +- doc/{koch.rst => koch.md} | 0 doc/{lib.rst => lib.md} | 0 doc/{manual.rst => manual.md} | 8 +- .../{var_t_return.rst => var_t_return.md} | 0 ...xperimental.rst => manual_experimental.md} | 2 +- ...st => manual_experimental_strictnotnil.md} | 0 doc/{mm.rst => mm.md} | 2 +- doc/{nep1.rst => nep1.md} | 0 doc/{nimc.rst => nimc.md} | 6 +- doc/{nimfix.rst => nimfix.md} | 0 doc/{nimgrep.rst => nimgrep.md} | 2 +- doc/{niminst.rst => niminst.md} | 0 doc/{nims.rst => nims.md} | 0 doc/{nimsuggest.rst => nimsuggest.md} | 0 doc/{overview.rst => overview.md} | 2 +- doc/{packaging.rst => packaging.md} | 0 doc/{refc.rst => refc.md} | 0 doc/{testament.rst => testament.md} | 0 doc/{tools.rst => tools.md} | 0 doc/{tut1.rst => tut1.md} | 0 doc/{tut2.rst => tut2.md} | 2 +- doc/{tut3.rst => tut3.md} | 2 +- lib/packages/docutils/highlite.nim | 4 +- lib/packages/docutils/rst.nim | 151 ++++++++++++++---- tests/stdlib/thighlite.nim | 6 + tests/stdlib/trst.nim | 117 +++++++++++++- tests/stdlib/trstgen.nim | 16 +- tools/kochdocs.nim | 16 +- 47 files changed, 341 insertions(+), 126 deletions(-) rename doc/{apis.rst => apis.md} (100%) rename doc/{backends.rst => backends.md} (99%) rename doc/{contributing.rst => contributing.md} (99%) rename doc/{destructors.rst => destructors.md} (100%) rename doc/{docgen.rst => docgen.md} (98%) rename doc/{docs.rst => docs.md} (100%) rename doc/{docstyle.rst => docstyle.md} (100%) rename doc/{drnim.rst => drnim.md} (100%) rename doc/{estp.rst => estp.md} (100%) rename doc/{filters.rst => filters.md} (100%) rename doc/{hcr.rst => hcr.md} (100%) rename doc/{idetools.rst => idetools.md} (100%) rename doc/{intern.rst => intern.md} (99%) rename doc/{koch.rst => koch.md} (100%) rename doc/{lib.rst => lib.md} (100%) rename doc/{manual.rst => manual.md} (99%) rename doc/manual/{var_t_return.rst => var_t_return.md} (100%) rename doc/{manual_experimental.rst => manual_experimental.md} (99%) rename doc/{manual_experimental_strictnotnil.rst => manual_experimental_strictnotnil.md} (100%) rename doc/{mm.rst => mm.md} (98%) rename doc/{nep1.rst => nep1.md} (100%) rename doc/{nimc.rst => nimc.md} (99%) rename doc/{nimfix.rst => nimfix.md} (100%) rename doc/{nimgrep.rst => nimgrep.md} (99%) rename doc/{niminst.rst => niminst.md} (100%) rename doc/{nims.rst => nims.md} (100%) rename doc/{nimsuggest.rst => nimsuggest.md} (100%) rename doc/{overview.rst => overview.md} (86%) rename doc/{packaging.rst => packaging.md} (100%) rename doc/{refc.rst => refc.md} (100%) rename doc/{testament.rst => testament.md} (100%) rename doc/{tools.rst => tools.md} (100%) rename doc/{tut1.rst => tut1.md} (100%) rename doc/{tut2.rst => tut2.md} (99%) rename doc/{tut3.rst => tut3.md} (99%) diff --git a/compiler/commands.nim b/compiler/commands.nim index 31e637aba..b849f503d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -449,6 +449,8 @@ proc parseCommand*(command: string): Command = of "doc2", "doc": cmdDoc of "doc2tex": cmdDoc2tex of "rst2html": cmdRst2html + of "md2tex": cmdMd2tex + of "md2html": cmdMd2html of "rst2tex": cmdRst2tex of "jsondoc0": cmdJsondoc0 of "jsondoc2", "jsondoc": cmdJsondoc @@ -480,7 +482,8 @@ proc setCommandEarly*(conf: ConfigRef, command: string) = # command early customizations # must be handled here to honor subsequent `--hint:x:on|off` case conf.cmd - of cmdRst2html, cmdRst2tex: # xxx see whether to add others: cmdGendepend, etc. + of cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex: + # xxx see whether to add others: cmdGendepend, etc. conf.foreignPackageNotes = {hintSuccessX} else: conf.foreignPackageNotes = foreignPackageNotesDefault diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 390f44f2e..ed5fe06ef 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -88,7 +88,7 @@ type jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable sharedState: PRstSharedState - isPureRst: bool + standaloneDoc: bool conf*: ConfigRef cache*: IdentCache exampleCounter: int @@ -230,6 +230,7 @@ template declareClosures = case msgKind of meCannotOpenFile: k = errCannotOpenFile of meExpected: k = errXExpected + of meMissingClosing: k = errRstMissingClosing of meGridTableNotImplemented: k = errRstGridTableNotImplemented of meMarkdownIllformedTable: k = errRstMarkdownIllformedTable of meIllformedTable: k = errRstIllformedTable @@ -276,16 +277,18 @@ proc isLatexCmd(conf: ConfigRef): bool = conf.cmd in {cmdRst2tex, cmdDoc2tex} proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, outExt: string = HtmlExt, module: PSym = nil, - isPureRst = false): PDoc = + standaloneDoc = false, preferMarkdown = true): PDoc = declareClosures() new(result) result.module = module result.conf = conf result.cache = cache result.outDir = conf.outDir.string - result.isPureRst = isPureRst - var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled} - if not isPureRst: options.incl roNimFile + result.standaloneDoc = standaloneDoc + var options= {roSupportRawDirective, roSupportMarkdown, roSandboxDisabled} + if preferMarkdown: + options.incl roPreferMarkdown + if not standaloneDoc: options.incl roNimFile result.sharedState = newRstSharedState( options, filename.string, docgenFindFile, compilerMsgHandler) @@ -333,7 +336,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef, # Make sure the destination directory exists createDir(outp.splitFile.dir) # Include the current file if we're parsing a nim file - let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")] + let importStmt = if d.standaloneDoc: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")] writeFile(outp, importStmt & content) proc interpSnippetCmd(cmd: string): string = @@ -1512,7 +1515,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = "\\\\\\vspace{0.5em}\\large $1", [esc(d.target, d.meta[metaSubtitle])]) var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection") - let bodyname = if d.hasToc and not d.isPureRst and not d.conf.isLatexCmd: + let bodyname = if d.hasToc and not d.standaloneDoc and not d.conf.isLatexCmd: groupsection.setLen 0 "doc.body_toc_group" elif d.hasToc: "doc.body_toc" @@ -1626,9 +1629,11 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) = generateIndex(d) proc commandRstAux(cache: IdentCache, conf: ConfigRef; - filename: AbsoluteFile, outExt: string) = + filename: AbsoluteFile, outExt: string, + preferMarkdown: bool) = var filen = addFileExt(filename, "txt") - var d = newDocumentor(filen, cache, conf, outExt, isPureRst = true) + var d = newDocumentor(filen, cache, conf, outExt, standaloneDoc = true, + preferMarkdown = preferMarkdown) let rst = parseRst(readFile(filen.string), line=LineRstInit, column=ColRstInit, conf, d.sharedState) @@ -1637,11 +1642,13 @@ proc commandRstAux(cache: IdentCache, conf: ConfigRef; writeOutput(d) generateIndex(d) -proc commandRst2Html*(cache: IdentCache, conf: ConfigRef) = - commandRstAux(cache, conf, conf.projectFull, HtmlExt) +proc commandRst2Html*(cache: IdentCache, conf: ConfigRef, + preferMarkdown=false) = + commandRstAux(cache, conf, conf.projectFull, HtmlExt, preferMarkdown) -proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef) = - commandRstAux(cache, conf, conf.projectFull, TexExt) +proc commandRst2TeX*(cache: IdentCache, conf: ConfigRef, + preferMarkdown=false) = + commandRstAux(cache, conf, conf.projectFull, TexExt, preferMarkdown) proc commandJson*(cache: IdentCache, conf: ConfigRef) = ## implementation of a deprecated jsondoc0 command diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 105de1636..071316f15 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -32,6 +32,7 @@ type # non-fatal errors errIllFormedAstX, errCannotOpenFile, errXExpected, + errRstMissingClosing, errRstGridTableNotImplemented, errRstMarkdownIllformedTable, errRstIllformedTable, @@ -105,6 +106,7 @@ const errIllFormedAstX: "illformed AST: $1", errCannotOpenFile: "cannot open '$1'", errXExpected: "'$1' expected", + errRstMissingClosing: "$1", errRstGridTableNotImplemented: "grid table is not implemented", errRstMarkdownIllformedTable: "illformed delimiter row of a markdown table", errRstIllformedTable: "Illformed table: $1", diff --git a/compiler/main.nim b/compiler/main.nim index a4425e510..0354bec9c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -276,7 +276,8 @@ proc mainCommand*(graph: ModuleGraph) = var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf) else: conf.projectPath doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee - if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex}: ret = ret / htmldocsDir + if conf.cmd in cmdDocLike + {cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex}: + ret = ret / htmldocsDir conf.outDir = ret ## process all commands @@ -302,7 +303,7 @@ proc mainCommand*(graph: ModuleGraph) = commandDoc2(graph, HtmlExt) if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions: commandBuildIndex(conf, $conf.outDir) - of cmdRst2html: + of cmdRst2html, cmdMd2html: # XXX: why are warnings disabled by default for rst2html and rst2tex? for warn in rstWarnings: conf.setNoteDefaults(warn, true) @@ -311,16 +312,16 @@ proc mainCommand*(graph: ModuleGraph) = conf.quitOrRaise "compiler wasn't built with documentation generator" else: loadConfigs(DocConfig, cache, conf, graph.idgen) - commandRst2Html(cache, conf) - of cmdRst2tex, cmdDoc2tex: + commandRst2Html(cache, conf, preferMarkdown = (conf.cmd == cmdMd2html)) + of cmdRst2tex, cmdMd2tex, cmdDoc2tex: for warn in rstWarnings: conf.setNoteDefaults(warn, true) when defined(leanCompiler): conf.quitOrRaise "compiler wasn't built with documentation generator" else: - if conf.cmd == cmdRst2tex: + if conf.cmd in {cmdRst2tex, cmdMd2tex}: loadConfigs(DocTexConfig, cache, conf, graph.idgen) - commandRst2TeX(cache, conf) + commandRst2TeX(cache, conf, preferMarkdown = (conf.cmd == cmdMd2tex)) else: docLikeCmd commandDoc2(graph, TexExt) of cmdJsondoc0: docLikeCmd commandJson(cache, conf) diff --git a/compiler/nim.nim b/compiler/nim.nim index bfb07ba20..48472507d 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -122,7 +122,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = # `The parameter is incorrect` let cmd = cmdPrefix & output.quoteShell & ' ' & conf.arguments execExternalProgram(conf, cmd.strip(leading=false,trailing=true)) - of cmdDocLike, cmdRst2html, cmdRst2tex: # bugfix(cmdRst2tex was missing) + of cmdDocLike, cmdRst2html, cmdRst2tex, cmdMd2html, cmdMd2tex: # bugfix(cmdRst2tex was missing) if conf.arguments.len > 0: # reserved for future use rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd]) diff --git a/compiler/options.nim b/compiler/options.nim index 1ba157587..d567927cb 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -153,6 +153,8 @@ type cmdDoc2tex # convert .nim doc comments to LaTeX cmdRst2html # convert a reStructuredText file to HTML cmdRst2tex # convert a reStructuredText file to TeX + cmdMd2html # convert a Markdown file to HTML + cmdMd2tex # convert a Markdown file to TeX cmdJsondoc0 cmdJsondoc cmdCtags diff --git a/doc/apis.rst b/doc/apis.md similarity index 100% rename from doc/apis.rst rename to doc/apis.md diff --git a/doc/backends.rst b/doc/backends.md similarity index 99% rename from doc/backends.rst rename to doc/backends.md index 65dd8a6f2..a135e78b0 100644 --- a/doc/backends.rst +++ b/doc/backends.md @@ -10,7 +10,8 @@ .. no syntax highlighting here by default: .. contents:: - "Heresy grows from idleness." -- Unknown. + +> "Heresy grows from idleness." -- Unknown. Introduction diff --git a/doc/contributing.rst b/doc/contributing.md similarity index 99% rename from doc/contributing.rst rename to doc/contributing.md index 0ca0d0cbc..51d1d5065 100644 --- a/doc/contributing.rst +++ b/doc/contributing.md @@ -581,7 +581,7 @@ Code reviews -.. include:: docstyle.rst +.. include:: docstyle.md Evolving the stdlib diff --git a/doc/destructors.rst b/doc/destructors.md similarity index 100% rename from doc/destructors.rst rename to doc/destructors.md diff --git a/doc/docgen.rst b/doc/docgen.md similarity index 98% rename from doc/docgen.rst rename to doc/docgen.md index 48166b2c5..5f5c4ecc0 100644 --- a/doc/docgen.rst +++ b/doc/docgen.md @@ -35,14 +35,13 @@ Quick start Generate HTML documentation for a file: -.. code:: cmd - + ```cmd nim doc .nim + ``` Generate HTML documentation for a whole project: -.. code:: cmd - + ```cmd # delete any htmldocs/*.idx file before starting nim doc --project --index:on --git.url: --git.commit: --outdir:htmldocs .nim # this will generate html files, a theindex.html index, css and js under `htmldocs` @@ -54,7 +53,7 @@ Generate HTML documentation for a whole project: # or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources; # and likewise without `--project`. # Adding `-r` will open in a browser directly. - + ``` Documentation Comments ---------------------- @@ -120,8 +119,8 @@ Example of Nim file input The following examples will generate documentation for this sample *Nim* module, aptly named ``doc/docgen_sample.nim``: -.. code:: nim - :file: docgen_sample.nim + ```nim file=docgen_sample.nim + ``` All the below commands save their output to ``htmldocs`` directory relative to the directory of file; @@ -137,9 +136,9 @@ optionally, an index file. The `doc`:option: command: -.. code:: cmd - + ```cmd nim doc docgen_sample.nim + ``` Partial Output:: ... @@ -159,8 +158,7 @@ HTML -> PDF conversion). The `doc2tex`:option: command: -.. code:: cmd - + ```cmd nim doc2tex docgen_sample.nim cd htmldocs xelatex docgen_sample.tex @@ -169,6 +167,7 @@ The `doc2tex`:option: command: # large documents) to get all labels generated. # That depends on this warning in the end of `xelatex` output: # LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. + ``` The output is ``docgen_sample.pdf``. @@ -183,9 +182,9 @@ Note that this tool is built off of the `doc`:option: command The `jsondoc`:option: command: -.. code:: cmd - + ```cmd nim jsondoc docgen_sample.nim + ``` Output:: { @@ -209,9 +208,9 @@ renamed to `jsondoc0`:option:. The `jsondoc0`:option: command: -.. code:: cmd - + ```cmd nim jsondoc0 docgen_sample.nim + ``` Output:: [ @@ -249,9 +248,9 @@ the anchor [*]_ of Nim symbol that corresponds to link text. If you have a constant: -.. code:: Nim - + ```Nim const pi* = 3.14 + ``` then it should be referenced in one of the 2 forms: @@ -262,9 +261,9 @@ B. qualified (with symbol kind specification):: For routine kinds there are more options. Consider this definition: -.. code:: Nim - + ```Nim proc foo*(a: int, b: float): string + ``` Generally following syntax is allowed for referencing `foo`: @@ -352,11 +351,11 @@ recognized fine:: (without parameter names, see form A.2 above). E.g. for this signature: - .. code:: Nim - + ```Nim proc binarySearch*[T, K](a: openArray[T]; key: K; cmp: proc (x: T; y: K): int {.closure.}): int ~~ ~~ ~~~~~ + ``` you cannot use names underlined by `~~` so it must be referenced with ``cmp: proc(T, K)``. Hence these forms are valid:: @@ -379,10 +378,10 @@ recognized fine:: .. Note:: A bit special case is operators (as their signature is also defined with `\``): - .. code:: Nim - + ```Nim func `$`(x: MyType): string func `[]`*[T](x: openArray[T]): T + ``` A short form works without additional backticks:: @@ -412,9 +411,9 @@ Related Options Project switch -------------- -.. code:: cmd - + ```cmd nim doc --project filename.nim + ``` This will recursively generate documentation of all Nim modules imported into the input module that belong to the Nimble package that ``filename.nim`` @@ -425,9 +424,9 @@ also be generated. Index switch ------------ -.. code:: cmd - + ```cmd nim doc --index:on filename.nim + ``` This will generate an index of all the exported symbols in the input Nim module, and put it into a neighboring file with the extension of ``.idx``. The @@ -443,9 +442,9 @@ file. See source switch ----------------- -.. code:: cmd - + ```cmd nim doc --git.url: filename.nim + ``` With the `git.url`:option: switch the *See source* hyperlink will appear below each documented item in your source code pointing to the implementation of that @@ -490,9 +489,9 @@ supports highlighting of a few other languages supported by the Usage: -.. code:: cmd - + ```cmd nim rst2html docgen.rst + ``` Output:: You're reading it! @@ -528,7 +527,7 @@ HTML file, most browsers will go to the first one. To differentiate the rest, you will need to use the complex name. A complex name for a callable type is made up of several parts: - (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\* + (**plain symbol**)(**.type**),(**first param**)?(**,param type**)\* The first thing to note is that all callable types have at least a comma, even if they don't have any parameters. If there are parameters, they are diff --git a/doc/docs.rst b/doc/docs.md similarity index 100% rename from doc/docs.rst rename to doc/docs.md diff --git a/doc/docstyle.rst b/doc/docstyle.md similarity index 100% rename from doc/docstyle.rst rename to doc/docstyle.md diff --git a/doc/drnim.rst b/doc/drnim.md similarity index 100% rename from doc/drnim.rst rename to doc/drnim.md diff --git a/doc/estp.rst b/doc/estp.md similarity index 100% rename from doc/estp.rst rename to doc/estp.md diff --git a/doc/filters.rst b/doc/filters.md similarity index 100% rename from doc/filters.rst rename to doc/filters.md diff --git a/doc/hcr.rst b/doc/hcr.md similarity index 100% rename from doc/hcr.rst rename to doc/hcr.md diff --git a/doc/idetools.rst b/doc/idetools.md similarity index 100% rename from doc/idetools.rst rename to doc/idetools.md diff --git a/doc/intern.rst b/doc/intern.md similarity index 99% rename from doc/intern.rst rename to doc/intern.md index 9103c694c..0fd995582 100644 --- a/doc/intern.rst +++ b/doc/intern.md @@ -10,7 +10,7 @@ .. include:: rstcommon.rst .. contents:: - "Abstraction is layering ignorance on top of reality." -- Richard Gabriel +> "Abstraction is layering ignorance on top of reality." -- Richard Gabriel Directory structure @@ -276,8 +276,8 @@ and `exitingDebugSection()`:nim:. #. Compile the temp compiler with `--debugger:native -d:nimDebugUtils`:option: #. Set your desired breakpoints or watchpoints. #. Configure your debugger: - * GDB: execute `source tools/compiler.gdb` at startup - * LLDB execute `command source tools/compiler.lldb` at startup + * GDB: execute `source tools/compiler.gdb` at startup + * LLDB execute `command source tools/compiler.lldb` at startup #. Use one of the scoping helpers like so: .. code-block:: nim diff --git a/doc/koch.rst b/doc/koch.md similarity index 100% rename from doc/koch.rst rename to doc/koch.md diff --git a/doc/lib.rst b/doc/lib.md similarity index 100% rename from doc/lib.rst rename to doc/lib.md diff --git a/doc/manual.rst b/doc/manual.md similarity index 99% rename from doc/manual.rst rename to doc/manual.md index 571379a87..503b9538b 100644 --- a/doc/manual.rst +++ b/doc/manual.md @@ -10,9 +10,9 @@ Nim Manual .. contents:: - "Complexity" seems to be a lot like "energy": you can transfer it from the - end-user to one/some of the other players, but the total amount seems to remain - pretty much constant for a given task. -- Ran +> "Complexity" seems to be a lot like "energy": you can transfer it from the +> end-user to one/some of the other players, but the total amount seems to remain +> pretty much constant for a given task. -- Ran About this document @@ -4025,7 +4025,7 @@ In the standard library every name of a routine that returns a `var` type starts with the prefix `m` per convention. -.. include:: manual/var_t_return.rst +.. include:: manual/var_t_return.md Future directions ~~~~~~~~~~~~~~~~~ diff --git a/doc/manual/var_t_return.rst b/doc/manual/var_t_return.md similarity index 100% rename from doc/manual/var_t_return.rst rename to doc/manual/var_t_return.md diff --git a/doc/manual_experimental.rst b/doc/manual_experimental.md similarity index 99% rename from doc/manual_experimental.rst rename to doc/manual_experimental.md index 3089755cb..3968163c5 100644 --- a/doc/manual_experimental.rst +++ b/doc/manual_experimental.md @@ -505,7 +505,7 @@ The compiler ensures that every code path initializes variables which contain non-nilable pointers. The details of this analysis are still to be specified here. -.. include:: manual_experimental_strictnotnil.rst +.. include:: manual_experimental_strictnotnil.md Aliasing restrictions in parameter passing diff --git a/doc/manual_experimental_strictnotnil.rst b/doc/manual_experimental_strictnotnil.md similarity index 100% rename from doc/manual_experimental_strictnotnil.rst rename to doc/manual_experimental_strictnotnil.md diff --git a/doc/mm.rst b/doc/mm.md similarity index 98% rename from doc/mm.rst rename to doc/mm.md index b6941a901..09b235228 100644 --- a/doc/mm.rst +++ b/doc/mm.md @@ -11,7 +11,7 @@ Nim's Memory Management .. - "The road to hell is paved with good intentions." +> "The road to hell is paved with good intentions." Multi-paradigm Memory Management Strategies diff --git a/doc/nep1.rst b/doc/nep1.md similarity index 100% rename from doc/nep1.rst rename to doc/nep1.md diff --git a/doc/nimc.rst b/doc/nimc.md similarity index 99% rename from doc/nimc.rst rename to doc/nimc.md index aa6650491..28a917d92 100644 --- a/doc/nimc.rst +++ b/doc/nimc.md @@ -11,9 +11,9 @@ .. - "Look at you, hacker. A pathetic creature of meat and bone, panting and - sweating as you run through my corridors. How can you challenge a perfect, - immortal machine?" +> "Look at you, hacker. A pathetic creature of meat and bone, panting and +> sweating as you run through my corridors. How can you challenge a perfect, +> immortal machine?" Introduction diff --git a/doc/nimfix.rst b/doc/nimfix.md similarity index 100% rename from doc/nimfix.rst rename to doc/nimfix.md diff --git a/doc/nimgrep.rst b/doc/nimgrep.md similarity index 99% rename from doc/nimgrep.rst rename to doc/nimgrep.md index 6088a4c45..ff2bf3a8d 100644 --- a/doc/nimgrep.rst +++ b/doc/nimgrep.md @@ -54,7 +54,7 @@ All examples below use default PCRE Regex patterns: nimgrep --excludeDir:'^\.git$' --excludeDir:'^\.hg$' --excludeDir:'^\.svn$' # short: --ed:'^\.git$' --ed:'^\.hg$' --ed:'^\.svn$' -+ To search only in paths containing the `tests` sub-directory recursively:: ++ To search only in paths containing the `tests` sub-directory recursively: .. code:: cmd nimgrep --recursive --includeDir:'(^|/)tests($|/)' diff --git a/doc/niminst.rst b/doc/niminst.md similarity index 100% rename from doc/niminst.rst rename to doc/niminst.md diff --git a/doc/nims.rst b/doc/nims.md similarity index 100% rename from doc/nims.rst rename to doc/nims.md diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.md similarity index 100% rename from doc/nimsuggest.rst rename to doc/nimsuggest.md diff --git a/doc/overview.rst b/doc/overview.md similarity index 86% rename from doc/overview.rst rename to doc/overview.md index e01520d7c..b21eb1e68 100644 --- a/doc/overview.rst +++ b/doc/overview.md @@ -5,5 +5,5 @@ Nim Documentation Overview :Author: Andreas Rumpf :Version: |nimversion| -.. include:: docs.rst +.. include:: docs.md diff --git a/doc/packaging.rst b/doc/packaging.md similarity index 100% rename from doc/packaging.rst rename to doc/packaging.md diff --git a/doc/refc.rst b/doc/refc.md similarity index 100% rename from doc/refc.rst rename to doc/refc.md diff --git a/doc/testament.rst b/doc/testament.md similarity index 100% rename from doc/testament.rst rename to doc/testament.md diff --git a/doc/tools.rst b/doc/tools.md similarity index 100% rename from doc/tools.rst rename to doc/tools.md diff --git a/doc/tut1.rst b/doc/tut1.md similarity index 100% rename from doc/tut1.rst rename to doc/tut1.md diff --git a/doc/tut2.rst b/doc/tut2.md similarity index 99% rename from doc/tut2.rst rename to doc/tut2.md index 725f68dd5..d37d6a16a 100644 --- a/doc/tut2.rst +++ b/doc/tut2.md @@ -13,7 +13,7 @@ Nim Tutorial (Part II) Introduction ============ - "Repetition renders the ridiculous reasonable." -- Norman Wildberger +> "Repetition renders the ridiculous reasonable." -- Norman Wildberger This document is a tutorial for the advanced constructs of the *Nim* programming language. **Note that this document is somewhat obsolete as the** diff --git a/doc/tut3.rst b/doc/tut3.md similarity index 99% rename from doc/tut3.rst rename to doc/tut3.md index c2c956107..2fcfd4220 100644 --- a/doc/tut3.rst +++ b/doc/tut3.md @@ -13,7 +13,7 @@ Nim Tutorial (Part III) Introduction ============ - "With Great Power Comes Great Responsibility." -- Spider Man's Uncle +> "With Great Power Comes Great Responsibility." -- Spider Man's Uncle This document is a tutorial about Nim's macro system. A macro is a function that is executed at compile-time and transforms diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index d36b2c877..3af94ab21 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -125,9 +125,7 @@ proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: cstring) = g.length = 0 g.state = low(TokenClass) g.lang = low(SourceLanguage) - var pos = 0 # skip initial whitespace: - while g.buf[pos] in {' ', '\t'..'\r'}: inc(pos) - g.pos = pos + g.pos = 0 proc initGeneralTokenizer*(g: var GeneralTokenizer, buf: string) = initGeneralTokenizer(g, cstring(buf)) diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index cd5f26278..1721674c8 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -8,20 +8,23 @@ # ## ================================== -## rst +## packages/docutils/rst ## ================================== ## ## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ## Nim-flavored reStructuredText and Markdown ## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ## -## This module implements a `reStructuredText`:idx: (RST) parser. +## This module implements a `reStructuredText`:idx: (RST) and +## `Markdown`:idx: parser. ## A large subset is implemented with some limitations_ and ## `Nim-specific features`_. -## A few `extra features`_ of the `Markdown`:idx: syntax are -## also supported. +## Both Markdown and RST are mark-up languages whose goal is to +## typeset texts with complex structure, formatting and references +## using simple plaintext representation. ## -## Nim can output the result to HTML [#html]_ or Latex [#latex]_. +## This module is also embedded into Nim compiler; the compiler can output +## the result to HTML [#html]_ or Latex [#latex]_. ## ## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and ## `nim rst2html`:cmd: for ``*.rst`` files @@ -29,11 +32,13 @@ ## .. [#latex] commands `nim doc2tex`:cmd: for ``*.nim`` and ## `nim rst2tex`:cmd: for ``*.rst``. ## -## If you are new to RST please consider reading the following: +## If you are new to Markdown/RST please consider reading the following: ## -## 1) a short `quick introduction`_ -## 2) an `RST reference`_: a comprehensive cheatsheet for RST -## 3) a more formal 50-page `RST specification`_. +## 1) `Markdown Basic Syntax`_ +## 2) a long specification of Markdown: `CommonMark Spec`_ +## 3) a short `quick introduction`_ to RST +## 4) an `RST reference`_: a comprehensive cheatsheet for RST +## 5) a more formal 50-page `RST specification`_. ## ## Features ## -------- @@ -120,7 +125,13 @@ ## ## * emoji / smiley symbols ## * Markdown tables -## * Markdown code blocks +## * Markdown code blocks. For them the same additional arguments as for RST +## code blocks can be provided (e.g. `test` or `number-lines`) but with +## a one-line syntax like this:: +## +## ```nim test number-lines=10 +## echo "ok" +## ``` ## * Markdown links ## * Markdown headlines ## * Markdown block quotes @@ -211,6 +222,8 @@ ## See `packages/docutils/rstgen module `_ to know how to ## generate HTML or Latex strings to embed them into your documents. ## +## .. _Markdown Basic Syntax: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax +## .. _CommonMark Spec: https://spec.commonmark.org/0.30 ## .. _quick introduction: https://docutils.sourceforge.io/docs/user/rst/quickstart.html ## .. _RST reference: https://docutils.sourceforge.io/docs/user/rst/quickref.html ## .. _RST specification: https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html @@ -253,6 +266,7 @@ type MsgKind* = enum ## the possible messages meCannotOpenFile = "cannot open '$1'", meExpected = "'$1' expected", + meMissingClosing = "$1", meGridTableNotImplemented = "grid table is not implemented", meMarkdownIllformedTable = "illformed delimiter row of a Markdown table", meIllformedTable = "Illformed table: $1", @@ -323,7 +337,10 @@ const ":geek:": "icon_e_geek", ":ugeek:": "icon_e_ugeek" } - SandboxDirAllowlist = ["image", "code", "code-block"] + SandboxDirAllowlist = [ + "image", "code", "code-block", "admonition", "attention", "caution", + "container", "contents", "danger", "default-role", "error", "figure", + "hint", "important", "index", "note", "role", "tip", "title", "warning"] type TokType = enum @@ -1616,35 +1633,89 @@ proc parseUntil(p: var RstParser, father: PRstNode, postfix: string, inc p.idx else: rstMessage(p, meExpected, postfix, line, col) +proc parseMarkdownCodeblockFields(p: var RstParser): PRstNode = + ## Parses additional (after language string) code block parameters + ## in a format *suggested* in the `CommonMark Spec`_ with handling of `"`. + if currentTok(p).kind == tkIndent: + result = nil + else: + result = newRstNode(rnFieldList) + while currentTok(p).kind != tkIndent: + if currentTok(p).kind == tkWhite: + inc p.idx + else: + let field = newRstNode(rnField) + var fieldName = "" + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof} and + currentTok(p).symbol != "=": + fieldName.add currentTok(p).symbol + inc p.idx + field.add(newRstNode(rnFieldName, @[newLeaf(fieldName)])) + if currentTok(p).kind == tkWhite: inc p.idx + let fieldBody = newRstNode(rnFieldBody) + if currentTok(p).symbol == "=": + inc p.idx + if currentTok(p).kind == tkWhite: inc p.idx + var fieldValue = "" + if currentTok(p).symbol == "\"": + while true: + fieldValue.add currentTok(p).symbol + inc p.idx + if currentTok(p).kind == tkEof: + rstMessage(p, meExpected, "\"") + elif currentTok(p).symbol == "\"": + fieldValue.add "\"" + inc p.idx + break + else: + while currentTok(p).kind notin {tkWhite, tkIndent, tkEof}: + fieldValue.add currentTok(p).symbol + inc p.idx + fieldBody.add newLeaf(fieldValue) + field.add(fieldBody) + result.add(field) + proc parseMarkdownCodeblock(p: var RstParser): PRstNode = result = newRstNodeA(p, rnCodeBlock) + let line = curLine(p) + let baseCol = currentTok(p).col + let baseSym = currentTok(p).symbol # usually just ``` + inc p.idx result.info = lineInfo(p) var args = newRstNode(rnDirArg) + var fields: PRstNode = nil if currentTok(p).kind == tkWord: args.add(newLeaf(p)) inc p.idx + fields = parseMarkdownCodeblockFields(p) else: args = nil var n = newLeaf("") while true: - case currentTok(p).kind - of tkEof: - rstMessage(p, meExpected, "```") + if currentTok(p).kind == tkEof: + rstMessage(p, meMissingClosing, + "$1 (started at line $2)" % [baseSym, $line]) break - of tkPunct, tkAdornment: - if currentTok(p).symbol == "```": - inc p.idx - break - else: - n.text.add(currentTok(p).symbol) - inc p.idx + elif nextTok(p).kind in {tkPunct, tkAdornment} and + nextTok(p).symbol[0] == baseSym[0] and + nextTok(p).symbol.len >= baseSym.len: + inc p.idx, 2 + break + elif currentTok(p).kind == tkIndent: + n.text.add "\n" + if currentTok(p).ival > baseCol: + n.text.add " ".repeat(currentTok(p).ival - baseCol) + elif currentTok(p).ival < baseCol: + rstMessage(p, mwRstStyle, + "unexpected de-indentation in Markdown code block") + inc p.idx else: n.text.add(currentTok(p).symbol) inc p.idx var lb = newRstNode(rnLiteralBlock) lb.add(n) result.add(args) - result.add(PRstNode(nil)) + result.add(fields) result.add(lb) proc parseMarkdownLink(p: var RstParser; father: PRstNode): bool = @@ -1730,6 +1801,12 @@ proc parseFootnoteName(p: var RstParser, reference: bool): PRstNode = inc i p.idx = i +proc isMarkdownCodeBlock(p: RstParser): bool = + result = (roSupportMarkdown in p.s.options and + currentTok(p).kind in {tkPunct, tkAdornment} and + currentTok(p).symbol[0] == '`' and # tilde ~ is not supported + currentTok(p).symbol.len >= 3) + proc parseInline(p: var RstParser, father: PRstNode) = var n: PRstNode # to be used in `if` condition let saveIdx = p.idx @@ -1755,8 +1832,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = addAnchorRst(p, name = linkName(n), refn = refn, reset = true, anchorType=manualInlineAnchor) father.add(n) - elif roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": - inc p.idx + elif isMarkdownCodeBlock(p): father.add(parseMarkdownCodeblock(p)) elif isInlineMarkupStart(p, "``"): var n = newRstNode(rnInlineLiteral) @@ -1816,8 +1892,7 @@ proc parseInline(p: var RstParser, father: PRstNode) = return parseWordOrRef(p, father) of tkAdornment, tkOther, tkWhite: - if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": - inc p.idx + if isMarkdownCodeBlock(p): father.add(parseMarkdownCodeblock(p)) return if roSupportSmilies in p.s.options: @@ -2194,7 +2269,7 @@ proc findPipe(p: RstParser, start: int): bool = proc whichSection(p: RstParser): RstNodeKind = if currentTok(p).kind in {tkAdornment, tkPunct}: # for punctuation sequences that can be both tkAdornment and tkPunct - if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```": + if isMarkdownCodeBlock(p): return rnCodeBlock elif currentTok(p).symbol == "::": return rnLiteralBlock @@ -2633,7 +2708,9 @@ proc parseSimpleTable(p: var RstParser): PRstNode = # fix rnTableDataCell -> rnTableHeaderCell for previous table rows: for nRow in 0 ..< result.sons.len: for nCell in 0 ..< result.sons[nRow].len: - result.sons[nRow].sons[nCell].kind = rnTableHeaderCell + template cell: PRstNode = result.sons[nRow].sons[nCell] + cell = PRstNode(kind: rnTableHeaderCell, sons: cell.sons, + span: cell.span, anchor: cell.anchor) if currentTok(p).kind == tkEof: break let tabRow = parseSimpleTableRow(p, cols, colChar) result.add tabRow @@ -2892,11 +2969,19 @@ proc parseSection(p: var RstParser, result: PRstNode) = if currInd(p) == currentTok(p).ival: inc p.idx elif currentTok(p).ival > currInd(p): - pushInd(p, currentTok(p).ival) - var a = newRstNodeA(p, rnBlockQuote) - parseSection(p, a) - result.add(a) - popInd(p) + if roPreferMarkdown in p.s.options: # Markdown => normal paragraphs + if currentTok(p).ival - currInd(p) >= 4: + rstMessage(p, mwRstStyle, + "Markdown indented code not implemented") + pushInd(p, currentTok(p).ival) + parseSection(p, result) + popInd(p) + else: # RST mode => block quotes + pushInd(p, currentTok(p).ival) + var a = newRstNodeA(p, rnBlockQuote) + parseSection(p, a) + result.add(a) + popInd(p) else: while currentTok(p).kind != tkEof and nextTok(p).kind == tkIndent: inc p.idx # skip blank lines diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim index 4113a6a80..5134215c1 100644 --- a/tests/stdlib/thighlite.nim +++ b/tests/stdlib/thighlite.nim @@ -12,6 +12,12 @@ block: # Nim tokenizing @[("\"\"\"ok1\\nok2\\nok3\"\"\"", gtLongStringLit) ]) + test "whitespace at beginning of line is preserved": + check(" discard 1".tokenize(langNim) == + @[(" ", gtWhitespace), ("discard", gtKeyword), (" ", gtWhitespace), + ("1", gtDecNumber) + ]) + block: # Cmd (shell) tokenizing test "cmd with dollar and output": check( diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 24bc03027..823697336 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -24,8 +24,11 @@ import unittest, strutils import std/private/miscdollars import os +const preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} +const preferRst = {roSupportMarkdown, roNimFile, roSandboxDisabled} + proc toAst(input: string, - rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled}, + rstOptions: RstParseOptions = preferMarkdown, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -451,7 +454,7 @@ suite "RST parsing": > - y > > Paragraph. - """.toAst == dedent""" + """.toAst(rstOptions = preferRst) == dedent""" rnMarkdownBlockQuote rnMarkdownBlockQuoteItem quotationDepth=1 rnInner @@ -468,6 +471,93 @@ suite "RST parsing": rnLeaf '.' """) + test "Markdown code blocks with more > 3 backticks": + check(dedent""" + ```` + let a = 1 + ``` + ````""".toAst == + dedent""" + rnCodeBlock + [nil] + [nil] + rnLiteralBlock + rnLeaf ' + let a = 1 + ```' + """) + + test "Markdown code blocks with Nim-specific arguments": + check(dedent""" + ```nim number-lines=1 test + let a = 1 + ```""".toAst == + dedent""" + rnCodeBlock + rnDirArg + rnLeaf 'nim' + rnFieldList + rnField + rnFieldName + rnLeaf 'number-lines' + rnFieldBody + rnLeaf '1' + rnField + rnFieldName + rnLeaf 'test' + rnFieldBody + rnLiteralBlock + rnLeaf ' + let a = 1' + """) + + check(dedent""" + ```nim test = "nim c $1" number-lines = 1 + let a = 1 + ```""".toAst == + dedent""" + rnCodeBlock + rnDirArg + rnLeaf 'nim' + rnFieldList + rnField + rnFieldName + rnLeaf 'test' + rnFieldBody + rnLeaf '"nim c $1"' + rnField + rnFieldName + rnLeaf 'number-lines' + rnFieldBody + rnLeaf '1' + rnLiteralBlock + rnLeaf ' + let a = 1' + """) + + test "additional indentation < 4 spaces is handled fine": + check(dedent""" + Indentation + + ```nim + let a = 1 + ```""".toAst == + dedent""" + rnInner + rnParagraph + rnLeaf 'Indentation' + rnParagraph + rnCodeBlock + rnDirArg + rnLeaf 'nim' + [nil] + rnLiteralBlock + rnLeaf ' + let a = 1' + """) + # | | + # | \ indentation of exactly two spaces before 'let a = 1' + test "option list has priority over definition list": check(dedent""" --defusages @@ -562,7 +652,7 @@ suite "RST parsing": notAcomment1 notAcomment2 - someParagraph""".toAst == + someParagraph""".toAst(rstOptions = preferRst) == dedent""" rnInner rnBlockQuote @@ -574,6 +664,25 @@ suite "RST parsing": rnLeaf 'someParagraph' """) + test "check that additional line right after .. ends comment (Markdown mode)": + # in Markdown small indentation does not matter so this should + # just be split to 2 paragraphs. + check(dedent""" + .. + + notAcomment1 + notAcomment2 + someParagraph""".toAst == + dedent""" + rnInner + rnInner + rnLeaf 'notAcomment1' + rnLeaf ' ' + rnLeaf 'notAcomment2' + rnParagraph + rnLeaf 'someParagraph' + """) + test "but blank lines after 2nd non-empty line don't end the comment": check(dedent""" .. @@ -592,7 +701,7 @@ suite "RST parsing": .. - someBlockQuote""".toAst == + someBlockQuote""".toAst(rstOptions = preferRst) == dedent""" rnInner rnAdmonition adType=note diff --git a/tests/stdlib/trstgen.nim b/tests/stdlib/trstgen.nim index d91b5615e..7fae4ba8b 100644 --- a/tests/stdlib/trstgen.nim +++ b/tests/stdlib/trstgen.nim @@ -9,8 +9,13 @@ import ../../lib/packages/docutils/rst import unittest, strutils, strtabs import std/private/miscdollars +const + NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} + preferMarkdown = {roPreferMarkdown, roSupportMarkdown, roNimFile} + preferRst = {roSupportMarkdown, roNimFile} + proc toHtml(input: string, - rstOptions: RstParseOptions = {roPreferMarkdown, roSupportMarkdown, roNimFile}, + rstOptions: RstParseOptions = preferMarkdown, error: ref string = nil, warnings: ref seq[string] = nil): string = ## If `error` is nil then no errors should be generated. @@ -47,9 +52,6 @@ proc optionListLabel(opt: string): string = opt & "
            " -const - NoSandboxOpts = {roPreferMarkdown, roSupportMarkdown, roNimFile, roSandboxDisabled} - suite "YAML syntax highlighting": test "Basics": @@ -1180,7 +1182,7 @@ Test1 "input(8, 4) Warning: language 'anotherLang' not supported" ]) check(output == "
            anything
            " & - "

            \nsomeCode\n

            ") + "

            \nsomeCode

            ") test "RST admonitions": # check that all admonitions are implemented @@ -1321,7 +1323,7 @@ Test1 That was a transition. """ let output1 = input1.toHtml( - NoSandboxOpts + preferRst ) doAssert "

            text""" & "\n

        Authors:Andreas Rumpf, Zahary Karadjov
        Version:|nimversion|
        ") test "Field list: body after newline": - let output = dedent """ + let output = dedent""" :field: text1""".toHtml check " Date: Fri, 15 Jul 2022 20:56:05 +0300 Subject: [PATCH 0887/1654] Implement type command (#19944) * Implement type command - this will be mapped to textDocument/typeDefinition in LSP protocol. It will be very useful for `nim` in particular because typically most of the time the type is inferred. * Update nimsuggest/nimsuggest.nim Co-authored-by: Andreas Rumpf --- compiler/options.nim | 4 +++- nimsuggest/nimsuggest.nim | 12 +++++++++- nimsuggest/tester.nim | 2 +- nimsuggest/tests/tv3_typeDefinition.nim | 32 +++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 nimsuggest/tests/tv3_typeDefinition.nim diff --git a/compiler/options.nim b/compiler/options.nim index d567927cb..9031e86c1 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -192,7 +192,7 @@ type IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod, ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols, - ideRecompile, ideChanged + ideRecompile, ideChanged, ideType Feature* = enum ## experimental features; DO NOT RENAME THESE! implicitDeref, @@ -1006,6 +1006,7 @@ proc parseIdeCmd*(s: string): IdeCmd = of "globalSymbols": ideGlobalSymbols of "recompile": ideRecompile of "changed": ideChanged + of "type": ideType else: ideNone proc `$`*(c: IdeCmd): string = @@ -1027,6 +1028,7 @@ proc `$`*(c: IdeCmd): string = of ideGlobalSymbols: "globalSymbols" of ideRecompile: "recompile" of ideChanged: "changed" + of ideType: "type" proc floatInt64Align*(conf: ConfigRef): int16 = ## Returns either 4 or 8 depending on reasons. diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index a18b2f960..6f35be925 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -457,6 +457,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = of "globalsymbols": conf.ideCmd = ideGlobalSymbols of "chkfile": conf.ideCmd = ideChkFile of "recompile": conf.ideCmd = ideRecompile + of "type": conf.ideCmd = ideType else: err() var dirtyfile = "" var orig = "" @@ -786,7 +787,7 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, graph.unmarkAllDirty() # these commands require partially compiled project - elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile} and + elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile, ideType} and (graph.needsCompilation(fileIndex) or cmd == ideSug): # for ideSug use v2 implementation if cmd == ideSug: @@ -802,6 +803,15 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, let (sym, info) = graph.findSymData(file, line, col) if sym != nil: graph.suggestResult(sym, sym.info) + of ideType: + let (sym, _) = graph.findSymData(file, line, col) + if sym != nil: + let typeSym = sym.typ.sym + if typeSym != nil: + graph.suggestResult(typeSym, typeSym.info, ideType) + elif sym.typ.len != 0: + let genericType = sym.typ[0].sym + graph.suggestResult(genericType, genericType.info, ideType) of ideUse, ideDus: let symbol = graph.findSymData(file, line, col).sym if symbol != nil: diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index fea0a8d45..6e068e067 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -282,7 +282,7 @@ proc runEpcTest(filename: string): int = socket.sendEpcStr(req) let sx = parseSexp(socket.recvEpc()) if not req.startsWith("mod "): - let answer = sexpToAnswer(sx) + let answer = if sx[2].kind == SNil: "" else: sexpToAnswer(sx) doReport(filename, answer, resp, report) socket.sendEpcStr "return arg" diff --git a/nimsuggest/tests/tv3_typeDefinition.nim b/nimsuggest/tests/tv3_typeDefinition.nim new file mode 100644 index 000000000..f86d12cc6 --- /dev/null +++ b/nimsuggest/tests/tv3_typeDefinition.nim @@ -0,0 +1,32 @@ +# tests v3 + +type + Foo* = ref object of RootObj + bar*: string + +proc test(ff: Foo) = + echo f#[!]#f.bar + +type + Fo#[!]#o2* = ref object of RootObj + +type + FooGeneric[T] = ref object of RootObj + bar*: T + +let fooGeneric = FooGeneric[string]() +echo fo#[!]#oGeneric.bar + +# bad type +echo unde#[!]#fined + +discard """ +$nimsuggest --v3 --tester $file +>type $1 +type skType tv3_typeDefinition.Foo Foo $file 4 2 "" 100 +>type $2 +type skType tv3_typeDefinition.Foo2 Foo2 $file 11 2 "" 100 +>type $3 +type skType tv3_typeDefinition.FooGeneric FooGeneric $file 14 2 "" 100 +>type $4 +""" From b0b9a3e5fade002b8f117f7ffb4d8fb93686401d Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Fri, 15 Jul 2022 20:56:33 +0300 Subject: [PATCH 0888/1654] Use module actual file instead of PSym.info (#19956) After this you can do goto module from module import --- compiler/suggest.nim | 30 ++++++++++++++++++------------ nimsuggest/tests/tv3_import.nim | 7 +++++++ 2 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 nimsuggest/tests/tv3_import.nim diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 4b48ef5bb..7e7d876fb 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -157,19 +157,25 @@ proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info result.forth = "" when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler): result.doc = extractDocComment(g, s) - let infox = - if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}: - info - else: - s.info - result.filePath = toFullPath(g.config, infox) - result.line = toLinenumber(infox) - result.column = toColumn(infox) + if s.kind == skModule and s.ast.len != 0 and section != ideHighlight: + result.filePath = toFullPath(g.config, s.ast[0].info) + result.line = 1 + result.column = 0 + result.tokenLen = 0 + else: + let infox = + if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}: + info + else: + s.info + result.filePath = toFullPath(g.config, infox) + result.line = toLinenumber(infox) + result.column = toColumn(infox) + result.tokenLen = if section != ideHighlight: + s.name.s.len + else: + getTokenLenFromSource(g.config, s.name.s, infox) result.version = g.config.suggestVersion - result.tokenLen = if section != ideHighlight: - s.name.s.len - else: - getTokenLenFromSource(g.config, s.name.s, infox) proc `$`*(suggest: Suggest): string = result = $suggest.section diff --git a/nimsuggest/tests/tv3_import.nim b/nimsuggest/tests/tv3_import.nim new file mode 100644 index 000000000..3c128f85b --- /dev/null +++ b/nimsuggest/tests/tv3_import.nim @@ -0,0 +1,7 @@ +import tv#[!]#3 + +discard """ +$nimsuggest --v3 --tester $file +>def $1 +def skModule tv3 */tv3.nim 1 0 "" 100 +""" From 60dd38c50262774cadb1d365e649696380c0595e Mon Sep 17 00:00:00 2001 From: David Krause Date: Sat, 16 Jul 2022 11:59:58 +0200 Subject: [PATCH 0889/1654] make AsyncSocket.getPeerAddr work ; fix #15022 (#20038) Signed-off-by: David Krause --- lib/pure/asyncdispatch.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 0d406b271..ed786cefb 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1733,6 +1733,8 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: DWORD, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): + const SO_UPDATE_CONNECT_CONTEXT = 0x7010 + socket.SocketHandle.setSockOptInt(SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, 1) # 15022 retFuture.complete() else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) From cf78c02b70ed105a173cce956ffab90bd57468a1 Mon Sep 17 00:00:00 2001 From: Amjad Ben Hedhili Date: Sat, 16 Jul 2022 21:51:27 +0100 Subject: [PATCH 0890/1654] Make `random.rand` work with `Ordinal` (#20043) * Make `random.rand` work with `Ordinal` * Add changelog entry * It's fine to cast to char --- changelog.md | 1 + lib/pure/random.nim | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/changelog.md b/changelog.md index 47479fc5c..c1dc6e27d 100644 --- a/changelog.md +++ b/changelog.md @@ -39,6 +39,7 @@ becomes an alias for `addr`. - `std/smtp` sends `ehlo` first. If the mail server does not understand, it sends `helo` as a fallback. - Changed mimedb to use an `OrderedTable` instead of `OrderedTableRef`, to use it in a const. - `strutils.find` now use and default to `last=-1` for whole string searches, making limiting it to just the first char (`last=0`) valid. +- `random.rand` now works with `Ordinal`s. [//]: # "Additions:" - Added `IsoWeekRange`, a range type to represent the number of weeks in an ISO week-based year. diff --git a/lib/pure/random.nim b/lib/pure/random.nim index b8aeb86e0..832d54e3d 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -353,8 +353,8 @@ proc rand*[T: Ordinal or SomeFloat](x: HSlice[T, T]): T = result = rand(state, x) -proc rand*[T: SomeInteger](t: typedesc[T]): T = - ## Returns a random integer in the range `low(T)..high(T)`. +proc rand*[T: Ordinal](t: typedesc[T]): T = + ## Returns a random Ordinal in the range `low(T)..high(T)`. ## ## If `randomize <#randomize>`_ has not been called, the sequence of random ## numbers returned from this proc will always be the same. @@ -368,13 +368,16 @@ proc rand*[T: SomeInteger](t: typedesc[T]): T = ## that accepts a slice runnableExamples: randomize(567) + type E = enum a, b, c, d if false: # implementation defined - assert rand(int8) == -42 - assert rand(uint32) == 578980729'u32 - assert rand(range[1..16]) == 11 + assert rand(E) in a..d + assert rand(char) in low(char)..high(char) + assert rand(int8) in low(int8)..high(int8) + assert rand(uint32) in low(uint32)..high(uint32) + assert rand(range[1..16]) in 1..16 # pending csources >= 1.4.0 or fixing https://github.com/timotheecour/Nim/issues/251#issuecomment-831599772, # use `runnableExamples("-r:off")` instead of `if false` - when T is range: + when T is range or T is enum or T is bool: result = rand(state, low(T)..high(T)) else: result = cast[T](state.next) From 094d86f99783f1366394fa444277c417dcbd1af3 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 16 Jul 2022 22:56:54 +0200 Subject: [PATCH 0891/1654] testament: use full test name in skips [backport] (#19937) testament: use full test name in skips --- testament/testament.nim | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/testament/testament.nim b/testament/testament.nim index d2b0e7fb4..98fc3c2aa 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -258,6 +258,15 @@ Tests failed and allowed to fail: $3 / $1
        Tests skipped: $4 / $1
        """ % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] +proc testName(test: TTest, target: TTarget, extraOptions: string, allowFailure: bool): string = + var name = test.name.replace(DirSep, '/') + name.add ' ' & $target + if allowFailure: + name.add " (allowed to fail) " + if test.options.len > 0: name.add ' ' & test.options + if extraOptions.len > 0: name.add ' ' & extraOptions + name.strip() + proc addResult(r: var TResults, test: TTest, target: TTarget, extraOptions, expected, given: string, successOrig: TResultEnum, allowFailure = false, givenSpec: ptr TSpec = nil) = @@ -265,13 +274,7 @@ proc addResult(r: var TResults, test: TTest, target: TTarget, # instead of having to pass individual fields, or abusing existing ones like expected vs given. # test.name is easier to find than test.name.extractFilename # A bit hacky but simple and works with tests/testament/tshould_not_work.nim - var name = test.name.replace(DirSep, '/') - name.add ' ' & $target - if allowFailure: - name.add " (allowed to fail) " - if test.options.len > 0: name.add ' ' & test.options - if extraOptions.len > 0: name.add ' ' & extraOptions - + let name = testName(test, target, extraOptions, allowFailure) let duration = epochTime() - test.startTime let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout else: successOrig @@ -470,6 +473,9 @@ proc equalModuloLastNewline(a, b: string): bool = proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, target: TTarget, extraOptions: string, nimcache: string) = test.startTime = epochTime() + if testName(test, target, extraOptions, false) in skips: + test.spec.err = reDisabled + if test.spec.err in {reDisabled, reJoined}: r.addResult(test, target, extraOptions, "", "", test.spec.err) inc(r.skipped) From c43a377057c56d5213eb298feba883363f2b2a26 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Sun, 17 Jul 2022 07:41:18 +1000 Subject: [PATCH 0892/1654] Make imports/exports not be a dropdown in sidebar (#19907) * Don't make a section be a dropdown if it has no child links * - Cleaned up code - Updated tests * Document what the 'if' is checking --- compiler/docgen.nim | 17 ++++++++++++++--- config/nimdoc.cfg | 8 ++++++++ nimdoc/testproject/expected/testproject.html | 7 +------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index ed5fe06ef..d728c535f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1474,10 +1474,21 @@ proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) = for item in d.tocSimple[kind].sorted(cmp): d.toc2[kind].add item.content - d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % [ - "sectionid", $ord(kind), "sectionTitle", title, - "sectionTitleID", $(ord(kind) + 50), "content", d.toc2[kind]] + let sectionValues = @[ + "sectionID", $ord(kind), "sectionTitleID", $(ord(kind) + 50), + "sectionTitle", title + ] + # Check if the toc has any children + if d.toc2[kind] != "": + # Use the dropdown version instead and store the children in the dropdown + d.toc[kind] = getConfigVar(d.conf, "doc.section.toc") % (sectionValues & @[ + "content", d.toc2[kind] + ]) + else: + # Just have the link + d.toc[kind] = getConfigVar(d.conf, "doc.section.toc_item") % sectionValues + proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): string = $relativeTo(outDir / linkto, destFile.splitFile().dir, '/') diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 3b0afe8d6..7aee50d8c 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -16,6 +16,14 @@ doc.section = """ """ +# Just a single item in the TOC (e.g. imports, exports) +doc.section.toc_item = """ +
      • + $sectionTitle +
        • +""" + +# This is a section (e.g. procs, types) in the TOC which gets turned into a drop down doc.section.toc = """
      • diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 24dfdc13f..5fb9e697c 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -51,12 +51,7 @@
          baz @@ -539,7 +540,15 @@
          -
          +
          + 
          proc bar(f: FooBuzz) {....raises: [], tags: [], forbids: [].}
          +
          + + + +
          +
          +
          proc bar[T](a, b: T): T
          diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index 4d0e63c0c..4d0172313 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -7,6 +7,7 @@ bar testproject.html#bar,T,T testproject: bar[T](a, b: T): T baz testproject.html#baz,T,T testproject: baz[T](a, b: T): T buzz testproject.html#buzz,T,T testproject: buzz[T](a, b: T): T FooBuzz testproject.html#FooBuzz testproject: FooBuzz +bar testproject.html#bar testproject: bar(f: FooBuzz) aVariable testproject.html#aVariable testproject: aVariable A testproject.html#A testproject: A B testproject.html#B testproject: B diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 402541540..9f73040f2 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -78,6 +78,8 @@
        bar:
        • testproject: bar(f: FooBuzz)
          • +
        • testproject: bar[T](a, b: T): T
        • testproject: bar(): untyped
          • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index 57960c69f..d236552ac 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -42,6 +42,11 @@ proc buzz*[T](a, b: T): T {.deprecated: "since v0.20".} = type FooBuzz* {.deprecated: "FooBuzz msg".} = int +using f: FooBuzz + +proc bar*(f) = # `f` should be expanded to `f: FooBuzz` + discard + import std/macros var aVariable*: array[1, int] From 28bed059aadf808fe56eaf2b36274bdf6baec053 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 12 Dec 2022 11:46:51 +0100 Subject: [PATCH 1450/1654] closes #20808 (#21077) --- tests/effects/tfuncs_cannot_mutate3.nim | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/effects/tfuncs_cannot_mutate3.nim diff --git a/tests/effects/tfuncs_cannot_mutate3.nim b/tests/effects/tfuncs_cannot_mutate3.nim new file mode 100644 index 000000000..029152029 --- /dev/null +++ b/tests/effects/tfuncs_cannot_mutate3.nim @@ -0,0 +1,35 @@ +discard """ + errormsg: "cannot mutate location kid.parent within a strict func" + line: 16 +""" + +{.experimental: "strictFuncs".} + +type + Node = ref object + name: string + kids: seq[Node] + parent: Node + +func initParents(tree: Node) = + for kid in tree.kids: + kid.parent = tree + initParents(kid) + +proc process(intro: Node): Node = + var tree = Node(name: "root", kids: @[ + intro, + Node(name: "one", kids: @[ + Node(name: "two"), + Node(name: "three"), + ]), + Node(name: "four"), + ]) + initParents(tree) + +proc main() = + var intro = Node(name: "intro") + var tree = process(intro) + echo intro.parent.name + +main() From 19923aa8b7bb93f687b71942b2ab3de39cba2f5a Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 21:14:24 +0800 Subject: [PATCH 1451/1654] work around chronos (#21079) * work around chronos chronos has tests against the devel branch, it will be eventually fine. * Apply suggestions from code review * Update testament/important_packages.nim --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 75283edbc..d0c87919b 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -52,7 +52,7 @@ pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true pkg "chroma" pkg "chronicles", "nimble install -y stew@#head; nim c -o:chr -r chronicles.nim" -pkg "chronos", "nimble install -y bearssl@#head stew@#head httputils@#head; nim c -r -d:release tests/testall" +pkg "chronos", "nim c -r -d:release tests/testall", url = "https://github.com/nim-lang/nim-chronos" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" From fd7da7da629f9f569503220b656047fb2f8da125 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 21:57:27 +0800 Subject: [PATCH 1452/1654] add a changelog for nimble (#21078) --- changelog.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 8efb7200c..5762d18ad 100644 --- a/changelog.md +++ b/changelog.md @@ -292,6 +292,4 @@ ## Tool changes -- Nim now supports Nimble version 0.14 which added support for lock-files. This is done by - a simple configuration change setting that you can do yourself too. In `$nim/config/nim.cfg` - replace `pkgs` by `pkgs2`. +- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`). From 584cfa6af81303caeaffe4084472167ad47216fe Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 22:59:09 +0800 Subject: [PATCH 1453/1654] remove implicitDeref feature enum (#21075) remove implicitDeref feature --- compiler/options.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index a94405e05..7e4f6898f 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -196,7 +196,6 @@ type ideRecompile, ideChanged, ideType, ideDeclaration Feature* = enum ## experimental features; DO NOT RENAME THESE! - implicitDeref, # deadcode; remains here for backwards compatibility dotOperators, callOperator, parallel, @@ -457,7 +456,7 @@ when false: fn(globalOptions) fn(selectedGC) -const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, parallel} +const oldExperimentalFeatures* = {dotOperators, callOperator, parallel} const ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, From 438863601047cc974d831c6c0dd5533e4e3f0dc5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 12 Dec 2022 23:14:44 +0800 Subject: [PATCH 1454/1654] fixes #21043; fixes a named exception in the infixAs expression which generate an implicit uninitialized let statement (#21081) * fixes #21043; fixes a named exception in the infixAs expression which generate an implicit uninitialized let statement * Update compiler/sempass2.nim Co-authored-by: Andreas Rumpf Co-authored-by: Andreas Rumpf --- compiler/sempass2.nim | 4 ++++ tests/init/tlet.nim | 23 ++++++++++++++++++++++- tests/iter/tyieldintry.nim | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 4520609e3..f13d8d8d4 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -491,6 +491,10 @@ proc trackTryStmt(tracked: PEffects, n: PNode) = let b = n[i] if b.kind == nkExceptBranch: setLen(tracked.init, oldState) + for j in 0.. Date: Tue, 13 Dec 2022 08:19:02 +1100 Subject: [PATCH 1455/1654] Nimsuggest now defines the backends symbol (#21083) * Add testcase * Define the backend symbol * Remove unneeded whitespace --- nimsuggest/nimsuggest.nim | 7 +++++-- nimsuggest/tests/t20440.nim | 7 +++++++ nimsuggest/tests/t20440.nims | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 nimsuggest/tests/t20440.nim create mode 100644 nimsuggest/tests/t20440.nims diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c4798e88c..c5e015ce9 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -27,7 +27,7 @@ import compiler / [options, commands, modules, sem, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, - pathutils] + pathutils, condsyms] when defined(nimPreviewSlimSystem): import std/typedthreads @@ -553,6 +553,7 @@ proc mainCommand(graph: ModuleGraph) = registerPass graph, verbosePass registerPass graph, semPass conf.setCmd cmdIdeTools + defineSymbol(conf.symbols, $conf.backend) wantMainModule(conf) if not fileExists(conf.projectFull): @@ -952,10 +953,12 @@ else: proc mockCommand(graph: ModuleGraph) = retval = graph let conf = graph.config + conf.setCmd cmdIdeTools + defineSymbol(conf.symbols, $conf.backend) clearPasses(graph) registerPass graph, verbosePass registerPass graph, semPass - conf.setCmd cmdIdeTools + wantMainModule(conf) if not fileExists(conf.projectFull): diff --git a/nimsuggest/tests/t20440.nim b/nimsuggest/tests/t20440.nim new file mode 100644 index 000000000..0456aa074 --- /dev/null +++ b/nimsuggest/tests/t20440.nim @@ -0,0 +1,7 @@ +when not defined(js): + {.fatal: "Crash".} +echo 4 + +discard """ +$nimsuggest --v3 --tester $file +""" diff --git a/nimsuggest/tests/t20440.nims b/nimsuggest/tests/t20440.nims new file mode 100644 index 000000000..1336be3d4 --- /dev/null +++ b/nimsuggest/tests/t20440.nims @@ -0,0 +1 @@ +switch("backend", "js") From d84f64d2e7a28d69c9a5b22b8434d1fb2f00ebf2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 13 Dec 2022 09:52:35 +0800 Subject: [PATCH 1456/1654] add CI for version-2-0 branch (#21084) --- .github/workflows/ci_packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml index 93f5bfe19..0bec4cc21 100644 --- a/.github/workflows/ci_packages.yml +++ b/.github/workflows/ci_packages.yml @@ -4,6 +4,7 @@ on: push: branches: - 'devel' + - 'version-2-0' - 'version-1-6' - 'version-1-2' From 1fefb8e92a62c647028d0f00ebd9fc04da169789 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 13 Dec 2022 15:21:51 +1100 Subject: [PATCH 1457/1654] Refactor JS sourcemap generator (#21053) * Parse the new line format * Fix pattern Didn't have space after the path * Remove duplicate sources Sources would sometimes double up with a new line at the end * Remove unused variable * Refactor sourcemap.nim Removes the multiple translations needed, now goes from single high level type to the final SourceMap Adds documentation for procs * Line numbers line up properly now Files aren't linking correctly though * Files now link up correctly Lines are sometimes off but overall seems pretty good Just need to implement parser * Add column info to output Add sourceMappingURL to rope directly to prevent copy * Properly handle columns * Remove debug lines * Add testcase * Finish testcase * Use the outdir folder instead of the folder the test is in to find the sourcemap Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- compiler/jsgen.nim | 8 +- compiler/sourcemap.nim | 540 ++++++++++++++-------------------------- tests/js/tsourcemap.nim | 96 +++++++ 3 files changed, 283 insertions(+), 361 deletions(-) create mode 100644 tests/js/tsourcemap.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index bcabc42ed..42a3dcf31 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -724,8 +724,9 @@ proc hasFrameInfo(p: PProc): bool = ((p.prc == nil) or not (sfPure in p.prc.flags)) proc lineDir(config: ConfigRef, info: TLineInfo, line: int): Rope = - ropes.`%`("/* line $2 \"$1\" */$n", - [rope(toFullPath(config, info)), rope(line)]) + "/* line $2:$3 \"$1\" */$n" % [ + rope(toFullPath(config, info)), rope(line), rope(info.toColumn) + ] proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) @@ -2885,7 +2886,8 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = # Generate an optional source map. if optSourcemap in m.config.globalOptions: var map: SourceMap - (code, map) = genSourceMap($(code), outFile.string) + map = genSourceMap($code, outFile.string) + code &= "\n//# sourceMappingURL=$#.map" % [outFile.string] writeFile(outFile.string & ".map", $(%map)) # Check if the generated JS code matches the output file, or else # write it to the file. diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim index b87de75f3..2245250ac 100644 --- a/compiler/sourcemap.nim +++ b/compiler/sourcemap.nim @@ -1,383 +1,207 @@ -import os, strformat, strutils, tables, sets, ropes, json, algorithm +import std/[strutils, strscans, parseutils, assertions] type - SourceNode* = ref object - line*: int - column*: int - source*: string - name*: string - children*: seq[Child] + Segment = object + ## Segment refers to a block of something in the JS output. + ## This could be a token or an entire line + original: int # Column in the Nim source + generated: int # Column in the generated JS + name: int # Index into names list (-1 for no name) - C = enum cSourceNode, cSourceString + Mapping = object + ## Mapping refers to a line in the JS output. + ## It is made up of segments which refer to the tokens in the line + case inSource: bool # Whether the line in JS has Nim equivilant + of true: + file: int # Index into files list + line: int # 0 indexed line of code in the Nim source + segments: seq[Segment] + else: discard - Child* = ref object - case kind*: C: - of cSourceNode: - node*: SourceNode - of cSourceString: - s*: string + SourceInfo = object + mappings: seq[Mapping] + names, files: seq[string] - SourceMap* = ref object + SourceMap* = object version*: int sources*: seq[string] names*: seq[string] mappings*: string file*: string - # sourceRoot*: string - # sourcesContent*: string - SourceMapGenerator = ref object - file: string - sourceRoot: string - skipValidation: bool - sources: seq[string] - names: seq[string] - mappings: seq[Mapping] +func addSegment(info: var SourceInfo, original, generated: int, name: string = "") {.raises: [].} = + ## Adds a new segment into the current line + assert info.mappings.len > 0, "No lines have been added yet" + var segment = Segment(original: original, generated: generated, name: -1) + if name != "": + # Make name be index into names list + segment.name = info.names.find(name) + if segment.name == -1: + segment.name = info.names.len + info.names &= name - Mapping* = ref object - source*: string - original*: tuple[line: int, column: int] - generated*: tuple[line: int, column: int] - name*: string - noSource*: bool - noName*: bool + assert info.mappings[^1].inSource, "Current line isn't in Nim source" + info.mappings[^1].segments &= segment +func newLine(info: var SourceInfo) {.raises: [].} = + ## Add new mapping which doesn't appear in the Nim source + info.mappings &= Mapping(inSource: false) -proc child*(s: string): Child = - Child(kind: cSourceString, s: s) - - -proc child*(node: SourceNode): Child = - Child(kind: cSourceNode, node: node) - - -proc newSourceNode(line: int, column: int, path: string, node: SourceNode, name: string = ""): SourceNode = - SourceNode(line: line, column: column, source: path, name: name, children: @[child(node)]) - - -proc newSourceNode(line: int, column: int, path: string, s: string, name: string = ""): SourceNode = - SourceNode(line: line, column: column, source: path, name: name, children: @[child(s)]) - - -proc newSourceNode(line: int, column: int, path: string, children: seq[Child], name: string = ""): SourceNode = - SourceNode(line: line, column: column, source: path, name: name, children: children) - - - - -# debugging - - -proc text*(sourceNode: SourceNode, depth: int): string = - let empty = " " - result = &"{repeat(empty, depth)}SourceNode({sourceNode.source}:{sourceNode.line}:{sourceNode.column}):\n" - for child in sourceNode.children: - if child.kind == cSourceString: - result.add(&"{repeat(empty, depth + 1)}{child.s}\n") - else: - result.add(child.node.text(depth + 1)) - - -proc `$`*(sourceNode: SourceNode): string = text(sourceNode, 0) +func newLine(info: var SourceInfo, file: string, line: int) {.raises: [].} = + ## Starts a new line in the mappings. Call addSegment after this to add + ## segments into the line + var mapping = Mapping(inSource: true, line: line) + # Set file to file position. Add in if needed + mapping.file = info.files.find(file) + if mapping.file == -1: + mapping.file = info.files.len + info.files &= file + info.mappings &= mapping # base64_VLQ +func encode*(values: seq[int]): string {.raises: [].} = + ## Encodes a series of integers into a VLQ base64 encoded string + # References: + # - https://www.lucidchart.com/techblog/2019/08/22/decode-encoding-base64-vlqs-source-maps/ + # - https://github.com/rails/sprockets/blob/main/guides/source_maps.md#source-map-file + const + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + shift = 5 + continueBit = 1 shl 5 + mask = continueBit - 1 + for val in values: + # Sign is stored in first bit + var newVal = abs(val) shl 1 + if val < 0: + newVal = newVal or 1 + # Now comes the variable length part + # This is how we are able to store large numbers + while true: + # We only encode 5 bits. + var masked = newVal and mask + newVal = newVal shr shift + # If there is still something left + # then signify with the continue bit that the + # decoder should keep decoding + if newVal > 0: + masked = masked or continueBit + result &= alphabet[masked] + # If the value is zero then we have nothing left to encode + if newVal == 0: + break +iterator tokenize*(line: string): (int, string) = + ## Goes through a line and splits it into Nim identifiers and + ## normal JS code. This allows us to map mangled names back to Nim names. + ## Yields (column, name). Doesn't yield anything but identifiers. + ## See mangleName in compiler/jsgen.nim for how name mangling is done + var + col = 0 + token = "" + while col < line.len: + var + token: string + name: string + # First we find the next identifier + col += line.skipWhitespace(col) + col += line.skipUntil(IdentStartChars, col) + let identStart = col + col += line.parseIdent(token, col) + # Idents will either be originalName_randomInt or HEXhexCode_randomInt + if token.startsWith("HEX"): + var hex: int + # 3 = "HEX".len and we only want to parse the two integers after it + discard token[3 ..< 5].parseHex(hex) + name = $chr(hex) + elif not token.endsWith("_Idx"): # Ignore address indexes + # It might be in the form originalName_randomInt + let lastUnderscore = token.rfind('_') + if lastUnderscore != -1: + name = token[0.. 0: - var e = n and 31 - n = n shr 5 - if n > 0: - e = e or 32 - - result.add(integers[e]) - z += 1 - - -type TokenState = enum Normal, String, Ident, Mangled - -iterator tokenize*(line: string): (bool, string) = - # result = @[] - var state = Normal - var token = "" - var isMangled = false - for z, ch in line: - if ch.isAlphaAscii: - if state == Normal: - state = Ident - if token.len > 0: - yield (isMangled, token) - token = $ch - isMangled = false - else: - token.add(ch) - elif ch == '_': - if state == Ident: - state = Mangled - isMangled = true - token.add($ch) - elif ch != '"' and not ch.isAlphaNumeric: - if state in {Ident, Mangled}: - state = Normal - if token.len > 0: - yield (isMangled, token) - token = $ch - isMangled = false - else: - token.add($ch) - elif ch == '"': - if state != String: - state = String - if token.len > 0: - yield (isMangled, token) - token = $ch - isMangled = false - else: - state = Normal - token.add($ch) - if token.len > 0: - yield (isMangled, token) - isMangled = false - token = "" +func parse*(source: string): SourceInfo = + ## Parses the JS output for embedded line info + ## So it can convert those into a series of mappings + var + skipFirstLine = true + currColumn = 0 + currLine = 0 + currFile = "" + # Add each line as a node into the output + for line in source.splitLines(): + var + lineNumber: int + linePath: string + column: int + if line.strip().scanf("/* line $i:$i \"$+\" */", lineNumber, column, linePath): + # When we reach the first line mappinsegmentg then we can assume + # we can map the rest of the JS lines to Nim lines + currColumn = column # Column is already zero indexed + currLine = lineNumber - 1 + currFile = linePath + # Lines are zero indexed + result.newLine(currFile, currLine) + # Skip whitespace to find the starting column + result.addSegment(currColumn, line.skipWhitespace()) + elif currFile != "": + result.newLine(currFile, currLine) + # There mightn't be any tokens so add a starting segment + result.addSegment(currColumn, line.skipWhitespace()) + for jsColumn, token in line.tokenize: + result.addSegment(currColumn, jsColumn, token) else: - token.add($ch) - if token.len > 0: - yield (isMangled, token) + result.newLine() -proc parse*(source: string, path: string): SourceNode = - let lines = source.splitLines() - var lastLocation: SourceNode = nil - result = newSourceNode(0, 0, path, @[]) - - # we just use one single parent and add all nim lines - # as its children, I guess in typical codegen - # that happens recursively on ast level - # we also don't have column info, but I doubt more one nim lines can compile to one js - # maybe in macros? +func toSourceMap*(info: SourceInfo, file: string): SourceMap {.raises: [].} = + ## Convert from high level SourceInfo into the required SourceMap object + # Add basic info + result.version = 3 + result.file = file + result.sources = info.files + result.names = info.names + # Convert nodes into mappings. + # Mappings are split into blocks where each block referes to a line in the outputted JS. + # Blocks can be seperated into statements which refere to tokens on the line. + # Since the mappings depend on previous values we need to + # keep track of previous file, name, etc + var + prevFile = 0 + prevLine = 0 + prevName = 0 + prevNimCol = 0 - for i, originalLine in lines: - let line = originalLine.strip - if line.len == 0: - continue - - # this shouldn't be a problem: - # jsgen doesn't generate comments - # and if you emit // line you probably know what you're doing - if line.startsWith("// line"): - if result.children.len > 0: - result.children[^1].node.children.add(child(line & "\n")) - let pos = line.find(" ", 8) - let lineNumber = line[8 .. pos - 1].parseInt - let linePath = line[pos + 2 .. ^2] # quotes - - lastLocation = newSourceNode( - lineNumber, - 0, - linePath, - @[]) - result.children.add(child(lastLocation)) - else: - var last: SourceNode - for token in line.tokenize(): - var name = "" - if token[0]: - name = token[1].split('_', 1)[0] - - - if result.children.len > 0: - result.children[^1].node.children.add( - child( - newSourceNode( - result.children[^1].node.line, - 0, - result.children[^1].node.source, - token[1], - name))) - last = result.children[^1].node.children[^1].node - else: - result.children.add( - child( - newSourceNode(i + 1, 0, path, token[1], name))) - last = result.children[^1].node - let nl = "\n" - if not last.isNil: - last.source.add(nl) + for mapping in info.mappings: + # We know need to encode segments with the following fields + # All these fields are relative to their previous values + # - 0: Column in generated code + # - 1: Index of Nim file in source list + # - 2: Line in Nim source + # - 3: Column in Nim source + # - 4: Index in names list + if mapping.inSource: + # JS Column is special in that it is reset after every line + var prevJSCol = 0 + for segment in mapping.segments: + var values = @[segment.generated - prevJSCol, mapping.file - prevFile, mapping.line - prevLine, segment.original - prevNimCol] + # Add name field if needed + if segment.name != -1: + values &= segment.name - prevName + prevName = segment.name + prevJSCol = segment.generated + prevNimCol = segment.original + prevFile = mapping.file + prevLine = mapping.line + result.mappings &= encode(values) & "," + # Remove trailing , + if mapping.segments.len > 0: + result.mappings.setLen(result.mappings.len - 1) -proc cmp(a: Mapping, b: Mapping): int = - var c = cmp(a.generated, b.generated) - if c != 0: - return c + result.mappings &= ";" - c = cmp(a.source, b.source) - if c != 0: - return c - - c = cmp(a.original, b.original) - if c != 0: - return c - - return cmp(a.name, b.name) - - -proc index*[T](elements: seq[T], element: T): int = - for z in 0 ..< elements.len: - if elements[z] == element: - return z - return -1 - - -proc serializeMappings(map: SourceMapGenerator, mappings: seq[Mapping]): string = - var previous = Mapping(generated: (line: 1, column: 0), original: (line: 0, column: 0), name: "", source: "") - var previousSourceId = 0 - var previousNameId = 0 - var next = "" - var nameId = 0 - var sourceId = 0 - result = "" - - for z, mapping in mappings: - next = "" - - if mapping.generated.line != previous.generated.line: - previous.generated.column = 0 - - while mapping.generated.line != previous.generated.line: - next.add(";") - previous.generated.line += 1 - - else: - if z > 0: - if cmp(mapping, mappings[z - 1]) == 0: - continue - next.add(",") - - next.add(encode(mapping.generated.column - previous.generated.column)) - previous.generated.column = mapping.generated.column - - if not mapping.noSource and mapping.source.len > 0: - sourceId = map.sources.index(mapping.source) - next.add(encode(sourceId - previousSourceId)) - previousSourceId = sourceId - next.add(encode(mapping.original.line - 1 - previous.original.line)) - previous.original.line = mapping.original.line - 1 - next.add(encode(mapping.original.column - previous.original.column)) - previous.original.column = mapping.original.column - - if not mapping.noName and mapping.name.len > 0: - nameId = map.names.index(mapping.name) - next.add(encode(nameId - previousNameId)) - previousNameId = nameId - - result.add(next) - - -proc gen*(map: SourceMapGenerator): SourceMap = - var mappings = map.mappings.sorted do (a: Mapping, b: Mapping) -> int: - cmp(a, b) - result = SourceMap( - file: map.file, - version: 3, - sources: map.sources[0..^1], - names: map.names[0..^1], - mappings: map.serializeMappings(mappings)) - - - -proc addMapping*(map: SourceMapGenerator, mapping: Mapping) = - if not mapping.noSource and mapping.source notin map.sources: - map.sources.add(mapping.source) - - if not mapping.noName and mapping.name.len > 0 and mapping.name notin map.names: - map.names.add(mapping.name) - - # echo "map ", mapping.source, " ", mapping.original, " ", mapping.generated, " ", mapping.name - map.mappings.add(mapping) - - -proc walk*(node: SourceNode, fn: proc(line: string, original: SourceNode)) = - for child in node.children: - if child.kind == cSourceString and child.s.len > 0: - fn(child.s, node) - else: - child.node.walk(fn) - - -proc toSourceMap*(node: SourceNode, file: string): SourceMapGenerator = - var map = SourceMapGenerator(file: file, sources: @[], names: @[], mappings: @[]) - - var generated = (line: 1, column: 0) - var sourceMappingActive = false - var lastOriginal = SourceNode(source: "", line: -1, column: 0, name: "", children: @[]) - - node.walk do (line: string, original: SourceNode): - if original.source.endsWith(".js"): - # ignore it - discard - else: - if original.line != -1: - if lastOriginal.source != original.source or - lastOriginal.line != original.line or - lastOriginal.column != original.column or - lastOriginal.name != original.name: - map.addMapping( - Mapping( - source: original.source, - original: (line: original.line, column: original.column), - generated: (line: generated.line, column: generated.column), - name: original.name)) - - lastOriginal = SourceNode( - source: original.source, - line: original.line, - column: original.column, - name: original.name, - children: lastOriginal.children) - sourceMappingActive = true - elif sourceMappingActive: - map.addMapping( - Mapping( - noSource: true, - noName: true, - generated: (line: generated.line, column: generated.column), - original: (line: -1, column: -1))) - lastOriginal.line = -1 - sourceMappingActive = false - - for z in 0 ..< line.len: - if line[z] in Newlines: - generated.line += 1 - generated.column = 0 - - if z == line.len - 1: - lastOriginal.line = -1 - sourceMappingActive = false - elif sourceMappingActive: - map.addMapping( - Mapping( - source: original.source, - original: (line: original.line, column: original.column), - generated: (line: generated.line, column: generated.column), - name: original.name)) - else: - generated.column += 1 - - map - - -proc genSourceMap*(source: string, outFile: string): (Rope, SourceMap) = - let node = parse(source, outFile) - let map = node.toSourceMap(file = outFile) - ((&"{source}\n//# sourceMappingURL={outFile}.map").rope, map.gen) +proc genSourceMap*(source: string, outFile: string): SourceMap = + let node = parse(source) + result = node.toSourceMap(outFile) diff --git a/tests/js/tsourcemap.nim b/tests/js/tsourcemap.nim new file mode 100644 index 000000000..ff6f6122f --- /dev/null +++ b/tests/js/tsourcemap.nim @@ -0,0 +1,96 @@ +discard """ + action: "run" + target: "js" + cmd: "nim js -r -d:nodejs $options --sourceMap:on $file" +""" +import std/[os, json, strutils, sequtils, algorithm, assertions, paths, compilesettings] + +# Implements a very basic sourcemap parser and then runs it on itself. +# Allows to check for basic problems such as bad counts and lines missing (e.g. issue #21052) + +type + SourceMap = object + version: int + sources: seq[string] + names: seq[string] + mappings: string + file: string + + Line = object + line, column: int + file: string + +const + flag = 1 shl 5 + signBit = 0b1 + fourBits = 0b1111 + fiveBits = 0b11111 + mask = (1 shl 5) - 1 + alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +var b64Table: seq[int] = 0.repeat(max(alphabet.mapIt(it.ord)) + 1) +for i, b in alphabet.pairs: + b64Table[b.ord] = i + +# From https://github.com/juancarlospaco/nodejs/blob/main/src/nodejs/jsfs.nim +proc importFs*() {.importjs: "var fs = require(\"fs\");".} +proc readFileSync*(path: cstring): cstring {.importjs: "(fs.$1(#).toString())".} +importFS() +# Read in needed files +let + jsFileName = string(querySetting(outDir).Path / "tsourcemap.js".Path) + mapFileName = jsFileName & ".map" + + data = parseJson($mapFileName.cstring.readFileSync()).to(SourceMap) + jsFile = $readFileSync(jsFileName.cstring) + +proc decodeVLQ(inp: string): seq[int] = + var + shift, value: int + for v in inp.mapIt(b64Table[it.ord]): + value += (v and mask) shl shift + if (v and flag) > 0: + shift += 5 + continue + result &= (value shr 1) * (if (value and 1) > 0: -1 else: 1) + shift = 0 + value = 0 + + +# Keep track of state +var + line = 0 + source = 0 + name = 0 + column = 0 + jsLine = 1 + lines: seq[Line] + +for gline in data.mappings.split(';'): + jsLine += 1 + var jsColumn = 0 + for item in gline.strip().split(','): + let value = item.decodeVLQ() + doAssert value.len in [0, 1, 4, 5] + if value.len == 0: + continue + jsColumn += value[0] + if value.len >= 4: + source += value[1] + line += value[2] + column += value[3] + lines &= Line(line: line, column: column, file: data.sources[source]) + +let jsLines = jsFile.splitLines().len +# There needs to be a mapping for every line in the JS +# If there isn't then the JS lines wont match up with Nim lines. +# Except we don't care about the final line since that doesn't need to line up +doAssert data.mappings.count(';') == jsLines - 1 + +# Check we can find this file somewhere in the source map +var foundSelf = false +for line in lines: + if "tsourcemap.nim" in line.file: + foundSelf = true + doAssert line.line in 0.. Date: Tue, 13 Dec 2022 18:43:37 +1100 Subject: [PATCH 1458/1654] JS backend properly extends string with `setLen` (#21087) * Add test case * Extend string with '0' when setting length to be longer --- compiler/jsgen.nim | 4 +++- tests/js/t20235.nim | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/js/t20235.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 42a3dcf31..d8fb6d57d 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2209,7 +2209,9 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) = if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2") else: binaryExpr(p, n, r, "subInt", "$1 = subInt($3, $2)", true) of mSetLengthStr: - binaryExpr(p, n, r, "mnewString", "($1.length = $2)") + binaryExpr(p, n, r, "mnewString", + """if ($1.length < $2) { for (var i = $3.length; i < $4; ++i) $3.push(0); } + else {$3.length = $4; }""") of mSetLengthSeq: var x, y: TCompRes gen(p, n[1], x) diff --git a/tests/js/t20235.nim b/tests/js/t20235.nim new file mode 100644 index 000000000..3a69c2bd6 --- /dev/null +++ b/tests/js/t20235.nim @@ -0,0 +1,11 @@ +discard """ + action: "run" + output: "0 4" +""" + +proc main = + var s = "" + s.setLen(4) + echo s[0].ord, " ", s.len + +main() From e4aadcf1c1b39a5fef55ce9f164b3251c44ca000 Mon Sep 17 00:00:00 2001 From: Xavier Noria Date: Tue, 13 Dec 2022 08:47:01 +0100 Subject: [PATCH 1459/1654] Document that system:pop() may raise IndexDefect (#21070) * Document system:pop() may raise IndexDefect * Add backticks to KeyError --- lib/pure/collections/sets.nim | 6 +++--- lib/system.nim | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/pure/collections/sets.nim b/lib/pure/collections/sets.nim index 09648ea65..11e324923 100644 --- a/lib/pure/collections/sets.nim +++ b/lib/pure/collections/sets.nim @@ -347,7 +347,7 @@ proc missingOrExcl*[A](s: var HashSet[A], key: A): bool = proc pop*[A](s: var HashSet[A]): A = ## Removes and returns an arbitrary element from the set `s`. ## - ## Raises KeyError if the set `s` is empty. + ## Raises `KeyError` if the set `s` is empty. ## ## See also: ## * `clear proc <#clear,HashSet[A]>`_ @@ -425,7 +425,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = assert c == toHashSet(["b"]) result = initHashSet[A](max(min(s1.data.len, s2.data.len), 2)) - + # iterate over the elements of the smaller set if s1.data.len < s2.data.len: for item in s1: @@ -433,7 +433,7 @@ proc intersection*[A](s1, s2: HashSet[A]): HashSet[A] = else: for item in s2: if item in s1: incl(result, item) - + proc difference*[A](s1, s2: HashSet[A]): HashSet[A] = ## Returns the difference of the sets `s1` and `s2`. diff --git a/lib/system.nim b/lib/system.nim index d0f78bfbd..65055f145 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1664,6 +1664,8 @@ proc contains*[T](a: openArray[T], item: T): bool {.inline.}= proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = ## Returns the last item of `s` and decreases `s.len` by one. This treats ## `s` as a stack and implements the common *pop* operation. + ## + ## Raises `IndexDefect` if `s` is empty. runnableExamples: var a = @[1, 3, 5, 7] let b = pop(a) From 2564b5c938602e4f8820d2ed2b778c6d4f1d0cd5 Mon Sep 17 00:00:00 2001 From: Aditya Siram Date: Tue, 13 Dec 2022 08:57:07 -0600 Subject: [PATCH 1460/1654] Fix #20416. Enable the recursion limit for ref/ptr types. (#21092) --- compiler/semtypinst.nim | 18 ++++++++++++++---- tests/generics/tgenerics_issues.nim | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 945667a81..0a0ac1704 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -499,10 +499,20 @@ proc propagateFieldFlags(t: PType, n: PNode) = proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = template bailout = - if t.sym != nil and sfGeneratedType in t.sym.flags: - # Only consider the recursion limit if the symbol is a type with generic - # parameters that have not been explicitly supplied, typechecking should - # terminate when generic parameters are explicitly supplied. + if (t.sym == nil) or (t.sym != nil and sfGeneratedType in t.sym.flags): + # In the first case 't.sym' can be 'nil' if the type is a ref/ptr, see + # issue https://github.com/nim-lang/Nim/issues/20416 for more details. + # Fortunately for us this works for now because partial ref/ptr types are + # not allowed in object construction, eg. + # type + # Container[T] = ... + # O = object + # val: ref Container + # + # In the second case only consider the recursion limit if the symbol is a + # type with generic parameters that have not been explicitly supplied, + # typechecking should terminate when generic parameters are explicitly + # supplied. if cl.recursionLimit > 100: # bail out, see bug #2509. But note this caching is in general wrong, # look at this example where TwoVectors should not share the generic diff --git a/tests/generics/tgenerics_issues.nim b/tests/generics/tgenerics_issues.nim index db7a16569..3068a22f2 100644 --- a/tests/generics/tgenerics_issues.nim +++ b/tests/generics/tgenerics_issues.nim @@ -872,3 +872,23 @@ block: # Ensure no segfault from constraint a = Regex[int]() b = Regex[bool]() c = MyOtherType[seq[int]]() + +block: # https://github.com/nim-lang/Nim/issues/20416 + type + Item[T] = object + link:ptr Item[T] + data:T + + KVSeq[A,B] = seq[(A,B)] + + MyTable[A,B] = object + data: KVSeq[A,B] + + Container[T] = object + a: MyTable[int,ref Item[T]] + + proc p1(sg:Container) = discard # Make sure that a non parameterized 'Container' argument still compiles + + proc p2[T](sg:Container[T]) = discard + var v : Container[int] + p2(v) From 9a50033d5b09ec3263a53bd2bb4182a8a50e6f4d Mon Sep 17 00:00:00 2001 From: metagn Date: Tue, 13 Dec 2022 23:20:55 +0300 Subject: [PATCH 1461/1654] generic `define` pragma + string alias (#20979) * generic `define` pragma + string alias * clean * add tests and document * remove char/float, minimize changelog --- changelog.md | 30 +++++++++- compiler/ast.nim | 2 +- compiler/condsyms.nim | 2 + compiler/pragmas.nim | 23 +++++--- compiler/semfold.nim | 109 ++++++++++++++++++++++++++++--------- doc/manual.md | 18 ++++++ doc/manual_experimental.md | 23 ++++++++ tests/misc/tdefine.nim | 64 ++++++++++++++++------ 8 files changed, 218 insertions(+), 53 deletions(-) diff --git a/changelog.md b/changelog.md index 5762d18ad..0181d03d2 100644 --- a/changelog.md +++ b/changelog.md @@ -221,9 +221,33 @@ - Full command syntax and block arguments i.e. `foo a, b: c` are now allowed for the right-hand side of type definitions in type sections. Previously they would error with "invalid indentation". -- `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. - In the command line, this is defined as `-d:a.b.c`. Older versions can - use accents as in ``defined(`a.b.c`)`` to access such defines. + +- Compile-time define changes: + - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use accents as in ``defined(`a.b.c`)`` to access such defines. + - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) + now support a string argument for qualified define names. + + ```nim + # -d:package.FooBar=42 + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar # 42 + ``` + + This was added to help disambiguate similar define names for different packages. + In older versions, this could only be achieved with something like the following: + + ```nim + const FooBar = block: + const `package.FooBar` {.intdefine.}: int = 5 + `package.FooBar` + ``` + - A generic `define` pragma for constants has been added that interprets + the value of the define based on the type of the constant value. + See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma) + for a list of supported types. + - [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: - Templates now accept macro pragmas. - Macro pragmas for var/let/const sections have been redesigned in a way that works diff --git a/compiler/ast.nim b/compiler/ast.nim index 198d21bea..c85a94860 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -712,7 +712,7 @@ type mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2, - mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples, + mNimvm, mIntDefine, mStrDefine, mBoolDefine, mGenericDefine, mRunnableExamples, mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf, mSymIsInstantiationOf, mNodeId, mPrivateAccess, mZeroDefault diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 182e49bf8..41b4c83aa 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -149,3 +149,5 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasOutParams") defineSymbol("nimHasSystemRaisesDefect") defineSymbol("nimHasWarnUnnamedBreak") + defineSymbol("nimHasGenericDefine") + defineSymbol("nimHasDefineAliases") diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index b60331b29..993e6edce 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -81,7 +81,8 @@ const wGuard, wGoto, wCursor, wNoalias, wAlign} constPragmas* = declPragmas + {wHeader, wMagic, wGensym, wInject, - wIntDefine, wStrDefine, wBoolDefine, wCompilerProc, wCore} + wIntDefine, wStrDefine, wBoolDefine, wDefine, + wCompilerProc, wCore} paramPragmas* = {wNoalias, wInject, wGensym} letPragmas* = varPragmas procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNoSideEffect, @@ -476,8 +477,16 @@ proc processPop(c: PContext, n: PNode) = when defined(debugOptions): echo c.config $ n.info, " POP config is now ", c.config.options -proc processDefine(c: PContext, n: PNode) = - if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent): +proc processDefineConst(c: PContext, n: PNode, sym: PSym, kind: TMagic) = + sym.magic = kind + if n.kind in nkPragmaCallKinds and n.len == 2: + # could also use TLib + n[1] = getStrLitNode(c, n) + +proc processDefine(c: PContext, n: PNode, sym: PSym) = + if sym != nil and sym.kind == skConst: + processDefineConst(c, n, sym, mGenericDefine) + elif (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent): defineSymbol(c.config.symbols, n[1].ident.s) else: invalidPragma(c, n) @@ -1066,7 +1075,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, recordPragma(c, it, "error", s) localError(c.config, it.info, errUser, s) of wFatal: fatal(c.config, it.info, expectStrLit(c, it)) - of wDefine: processDefine(c, it) + of wDefine: processDefine(c, it, sym) of wUndef: processUndef(c, it) of wCompile: processCompile(c, it) of wLink: processLink(c, it) @@ -1213,11 +1222,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, noVal(c, it) sym.flags.incl sfBase of wIntDefine: - sym.magic = mIntDefine + processDefineConst(c, n, sym, mIntDefine) of wStrDefine: - sym.magic = mStrDefine + processDefineConst(c, n, sym, mStrDefine) of wBoolDefine: - sym.magic = mBoolDefine + processDefineConst(c, n, sym, mBoolDefine) of wUsed: noVal(c, it) if sym == nil: invalidPragma(c, it) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 9e45e3b2d..7c57eb370 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -13,7 +13,7 @@ import strutils, options, ast, trees, nimsets, platform, math, msgs, idents, renderer, types, - commands, magicsys, modulegraphs, strtabs, lineinfos + commands, magicsys, modulegraphs, strtabs, lineinfos, wordrecg from system/memory import nimCStrLen @@ -371,6 +371,11 @@ proc rangeCheck(n: PNode, value: Int128; g: ModuleGraph) = localError(g.config, n.info, "cannot convert " & $value & " to " & typeToString(n.typ)) +proc floatRangeCheck(n: PNode, value: BiggestFloat; g: ModuleGraph) = + if value < firstFloat(n.typ) or value > lastFloat(n.typ): + localError(g.config, n.info, "cannot convert " & $value & + " to " & typeToString(n.typ)) + proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): PNode = let dstTyp = skipTypes(n.typ, abstractRange - {tyTypeDesc}) let srcTyp = skipTypes(a.typ, abstractRange - {tyTypeDesc}) @@ -490,6 +495,81 @@ proc newSymNodeTypeDesc*(s: PSym; idgen: IdGenerator; info: TLineInfo): PNode = else: result.typ = s.typ +proc foldDefine(m, s: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode = + var name = s.name.s + let prag = extractPragma(s) + if prag != nil: + for it in prag: + if it.kind in nkPragmaCallKinds and it.len == 2 and it[0].kind == nkIdent: + let word = whichKeyword(it[0].ident) + if word in {wStrDefine, wIntDefine, wBoolDefine, wDefine}: + # should be processed in pragmas.nim already + if it[1].kind in {nkStrLit, nkRStrLit, nkTripleStrLit}: + name = it[1].strVal + if isDefined(g.config, name): + let str = g.config.symbols[name] + case s.magic + of mIntDefine: + try: + result = newIntNodeT(toInt128(str.parseInt), n, idgen, g) + except ValueError: + localError(g.config, s.info, + "{.intdefine.} const was set to an invalid integer: '" & + str & "'") + of mStrDefine: + result = newStrNodeT(str, n, g) + of mBoolDefine: + try: + result = newIntNodeT(toInt128(str.parseBool.int), n, idgen, g) + except ValueError: + localError(g.config, s.info, + "{.booldefine.} const was set to an invalid bool: '" & + str & "'") + of mGenericDefine: + let rawTyp = s.typ + # pretend we don't support distinct types + let typ = rawTyp.skipTypes(abstractVarRange-{tyDistinct}) + try: + template intNode(value): PNode = + let val = toInt128(value) + rangeCheck(n, val, g) + newIntNodeT(val, n, idgen, g) + case typ.kind + of tyString, tyCstring: + result = newStrNodeT(str, n, g) + of tyInt..tyInt64: + result = intNode(str.parseBiggestInt) + of tyUInt..tyUInt64: + result = intNode(str.parseBiggestUInt) + of tyBool: + result = intNode(str.parseBool.int) + of tyEnum: + # compile time parseEnum + let ident = getIdent(g.cache, str) + for e in typ.n: + if e.kind != nkSym: internalError(g.config, "foldDefine for enum") + let es = e.sym + let match = + if es.ast.isNil: + es.name.id == ident.id + else: + es.ast.strVal == str + if match: + result = intNode(es.position) + break + if result.isNil: + raise newException(ValueError, "invalid enum value: " & str) + else: + localError(g.config, s.info, "unsupported type $1 for define '$2'" % + [name, typeToString(rawTyp)]) + except ValueError as e: + localError(g.config, s.info, + "could not process define '$1' of type $2; $3" % + [name, typeToString(rawTyp), e.msg]) + else: result = copyTree(s.astdef) # unreachable + else: + result = copyTree(s.astdef) + proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode = result = nil case n.kind @@ -509,31 +589,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode of mBuildOS: result = newStrNodeT(toLowerAscii(platform.OS[g.config.target.hostOS].name), n, g) of mBuildCPU: result = newStrNodeT(platform.CPU[g.config.target.hostCPU].name.toLowerAscii, n, g) of mAppType: result = getAppType(n, g) - of mIntDefine: - if isDefined(g.config, s.name.s): - try: - result = newIntNodeT(toInt128(g.config.symbols[s.name.s].parseInt), n, idgen, g) - except ValueError: - localError(g.config, s.info, - "{.intdefine.} const was set to an invalid integer: '" & - g.config.symbols[s.name.s] & "'") - else: - result = copyTree(s.astdef) - of mStrDefine: - if isDefined(g.config, s.name.s): - result = newStrNodeT(g.config.symbols[s.name.s], n, g) - else: - result = copyTree(s.astdef) - of mBoolDefine: - if isDefined(g.config, s.name.s): - try: - result = newIntNodeT(toInt128(g.config.symbols[s.name.s].parseBool.int), n, idgen, g) - except ValueError: - localError(g.config, s.info, - "{.booldefine.} const was set to an invalid bool: '" & - g.config.symbols[s.name.s] & "'") - else: - result = copyTree(s.astdef) + of mIntDefine, mStrDefine, mBoolDefine, mGenericDefine: + result = foldDefine(m, s, n, idgen, g) else: result = copyTree(s.astdef) of skProc, skFunc, skMethod: diff --git a/doc/manual.md b/doc/manual.md index bd2cf8568..0a295fdb5 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -8049,6 +8049,24 @@ used. To see if a value was provided, `defined(FooBar)` can be used. The syntax `-d:flag`:option: is actually just a shortcut for `-d:flag=true`:option:. +These pragmas also accept an optional string argument for qualified +define names. + + ```nim + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar + ``` + + ```cmd + nim c -d:package.FooBar=42 foobar.nim + ``` + +This helps disambiguate define names in different packages. + +See also the [generic `define` pragma](manual_experimental.html#generic-define-pragma) +for a version of these pragmas that detects the type of the define based on +the constant value. + User-defined pragmas ==================== diff --git a/doc/manual_experimental.md b/doc/manual_experimental.md index 6f5906019..2a56c1354 100644 --- a/doc/manual_experimental.md +++ b/doc/manual_experimental.md @@ -65,6 +65,29 @@ However, a `void` type cannot be inferred in generic code: The `void` type is only valid for parameters and return types; other symbols cannot have the type `void`. +Generic `define` pragma +======================= + +Aside the [typed define pragmas for constants](manual.html#implementation-specific-pragmas-compileminustime-define-pragmas), +there is a generic `{.define.}` pragma that interprets the value of the define +based on the type of the constant value. + + ```nim + const foo {.define: "package.foo".} = 123 + const bar {.define: "package.bar".} = false + ``` + + ```cmd + nim c -d:package.foo=456 -d:package.bar foobar.nim + ``` + +The following types are supported: + +* `string` and `cstring` +* Signed and unsigned integer types +* `bool` +* Enums + Top-down type inference ======================= diff --git a/tests/misc/tdefine.nim b/tests/misc/tdefine.nim index c4d11c941..f3fa4711f 100644 --- a/tests/misc/tdefine.nim +++ b/tests/misc/tdefine.nim @@ -1,12 +1,23 @@ discard """ joinable: false -cmd: "nim c -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -d:namespaced.define=false -d:double.namespaced.define -r $file" +cmd: "nim c $options -d:booldef -d:booldef2=false -d:intdef=2 -d:strdef=foobar -d:namespaced.define=false -d:double.namespaced.define -r $file" +matrix: "; -d:useGenericDefine" """ -const booldef {.booldefine.} = false -const booldef2 {.booldefine.} = true -const intdef {.intdefine.} = 0 -const strdef {.strdefine.} = "" +when defined(useGenericDefine): + {.pragma: booldefine2, define.} + {.pragma: intdefine2, define.} + {.pragma: strdefine2, define.} +else: + + {.pragma: booldefine2, booldefine.} + {.pragma: intdefine2, intdefine.} + {.pragma: strdefine2, strdefine.} + +const booldef {.booldefine2.} = false +const booldef2 {.booldefine2.} = true +const intdef {.intdefine2.} = 0 +const strdef {.strdefine2.} = "" doAssert defined(booldef) doAssert defined(booldef2) @@ -17,17 +28,28 @@ doAssert not booldef2 doAssert intdef == 2 doAssert strdef == "foobar" +when defined(useGenericDefine): + block: + const uintdef {.define: "intdef".}: uint = 17 + doAssert intdef == int(uintdef) + const cstrdef {.define: "strdef".}: cstring = "not strdef" + doAssert $cstrdef == strdef + type FooBar = enum foo, bar, foobar + const enumdef {.define: "strdef".} = foo + doAssert $enumdef == strdef + doAssert enumdef == foobar + # Intentionally not defined from command line -const booldef3 {.booldefine.} = true -const intdef2 {.intdefine.} = 1 -const strdef2 {.strdefine.} = "abc" +const booldef3 {.booldefine2.} = true +const intdef2 {.intdefine2.} = 1 +const strdef2 {.strdefine2.} = "abc" type T = object - when booldef3: - field1: int - when intdef2 == 1: - field2: int - when strdef2 == "abc": - field3: int + when booldef3: + field1: int + when intdef2 == 1: + field2: int + when strdef2 == "abc": + field3: int doAssert not defined(booldef3) doAssert not defined(intdef2) @@ -35,11 +57,21 @@ doAssert not defined(strdef2) discard T(field1: 1, field2: 2, field3: 3) doAssert defined(namespaced.define) -const `namespaced.define` {.booldefine.} = true +const `namespaced.define` {.booldefine2.} = true doAssert not `namespaced.define` +when defined(useGenericDefine): + const aliasToNamespacedDefine {.define: "namespaced.define".} = not `namespaced.define` +else: + const aliasToNamespacedDefine {.booldefine: "namespaced.define".} = not `namespaced.define` +doAssert aliasToNamespacedDefine == `namespaced.define` doAssert defined(double.namespaced.define) -const `double.namespaced.define` {.booldefine.} = false +const `double.namespaced.define` {.booldefine2.} = false doAssert `double.namespaced.define` +when defined(useGenericDefine): + const aliasToDoubleNamespacedDefine {.define: "double.namespaced.define".} = not `double.namespaced.define` +else: + const aliasToDoubleNamespacedDefine {.booldefine: "double.namespaced.define".} = not `double.namespaced.define` +doAssert aliasToDoubleNamespacedDefine == `double.namespaced.define` doAssert not defined(namespaced.butnotdefined) From 91ce8c385d4ccbaab8048cf0393b01cd72282272 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:45:36 +0800 Subject: [PATCH 1462/1654] fix #19580; add warning for bare except: clause (#21099) * fix #19580; add warning for bare except: clause * fixes some easy ones * Update doc/manual.md * fixes docs * Update changelog.md * addition * Apply suggestions from code review Co-authored-by: Jacek Sieka * Update doc/tut2.md Co-authored-by: Jacek Sieka --- changelog.md | 2 ++ compiler/condsyms.nim | 1 + compiler/docgen.nim | 8 ++++---- compiler/extccomp.nim | 4 ++-- compiler/lineinfos.nim | 2 ++ compiler/nim.cfg | 4 ++++ compiler/semstmts.nim | 5 ++++- compiler/types.nim | 12 ++++++++++++ doc/manual.md | 20 +++++++++----------- doc/tut2.md | 4 ++-- lib/packages/docutils/rst.nim | 6 +++--- lib/pure/htmlparser.nim | 4 ++-- lib/std/assertions.nim | 2 ++ tests/nimdoc/trunnableexamples.nim | 2 +- 14 files changed, 50 insertions(+), 26 deletions(-) diff --git a/changelog.md b/changelog.md index 0181d03d2..00bdb50c2 100644 --- a/changelog.md +++ b/changelog.md @@ -133,6 +133,8 @@ `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. +- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 41b4c83aa..9d081ef0a 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -151,3 +151,4 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasWarnUnnamedBreak") defineSymbol("nimHasGenericDefine") defineSymbol("nimHasDefineAliases") + defineSymbol("nimHasWarnBareExcept") diff --git a/compiler/docgen.nim b/compiler/docgen.nim index fc37bd596..f8a804cec 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1746,7 +1746,7 @@ proc commandJson*(cache: IdentCache, conf: ConfigRef) = let filename = getOutFile(conf, RelativeFile conf.projectName, JsonExt) try: writeFile(filename, content) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) proc commandTags*(cache: IdentCache, conf: ConfigRef) = @@ -1768,7 +1768,7 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) = let filename = getOutFile(conf, RelativeFile conf.projectName, TagsExt) try: writeFile(filename, content) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") = @@ -1789,7 +1789,7 @@ proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") try: writeFile(filename, code) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile"") = @@ -1803,5 +1803,5 @@ proc commandBuildIndexJson*(conf: ConfigRef, dir: string, outFile = RelativeFile try: writeFile(filename, $body) - except: + except IOError: rawMessage(conf, errCannotOpenFile, filename.string) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 256e02b22..66de46556 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -528,7 +528,7 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool = var exe = getConfigVar(conf, conf.cCompiler, ".exe") if exe.len == 0: exe = CC[conf.cCompiler].compilerExe # NOTE: should we need the full version, use -dumpfullversion - let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except: ("", 1) + let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1) if exitCode == 0: var major: int discard parseInt(s, major) @@ -1018,7 +1018,7 @@ proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: Absolute proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = var bcache: BuildCache try: bcache.fromJson(jsonFile.string.parseFile) - except: + except ValueError, KeyError, JsonKindError: let e = getCurrentException() conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % [e.msg, e.getStackTrace(), jsonFile.string] diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index ab9c902c3..cb60d8949 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -86,6 +86,7 @@ type warnImplicitTemplateRedefinition = "ImplicitTemplateRedefinition", warnUnnamedBreak = "UnnamedBreak", warnStmtListLambda = "StmtListLambda", + warnBareExcept = "BareExcept", warnUser = "User", # hints hintSuccess = "Success", hintSuccessX = "SuccessX", @@ -185,6 +186,7 @@ const warnImplicitTemplateRedefinition: "template '$1' is implicitly redefined; this is deprecated, add an explicit .redefine pragma", warnUnnamedBreak: "Using an unnamed break in a block is deprecated; Use a named block with a named break instead", warnStmtListLambda: "statement list expression assumed to be anonymous proc; this is deprecated, use `do (): ...` or `proc () = ...` instead", + warnBareExcept: "$1", warnUser: "$1", hintSuccess: "operation successful: $#", # keep in sync with `testament.isSuccess` diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 20ce3810f..96b47b0e6 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -38,3 +38,7 @@ define:useStdoutAsStdmsg @if nimHasWarnUnnamedBreak: warningAserror[UnnamedBreak]:on @end + +@if nimHasWarnBareExcept: + warningAserror[BareExcept]:on +@end diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d19d75611..a7a02d7fa 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -210,6 +210,8 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags; expectedType: PType = nil) isImported = true elif not isException(typ): localError(c.config, typeNode.info, errExprCannotBeRaised) + elif not isDefectOrCatchableError(typ): + message(c.config, a.info, warnBareExcept, "catch a more precise Exception deriving from CatchableError or Defect.") if containsOrIncl(check, typ.id): localError(c.config, typeNode.info, errExceptionAlreadyHandled) @@ -251,7 +253,8 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags; expectedType: PType = nil) elif a.len == 1: # count number of ``except: body`` blocks inc catchAllExcepts - + message(c.config, a.info, warnBareExcept, + "The bare except clause is deprecated; use `except CatchableError:` instead") else: # support ``except KeyError, ValueError, ... : body`` if catchAllExcepts > 0: diff --git a/compiler/types.nim b/compiler/types.nim index 4bbbaf13c..457568e32 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1721,6 +1721,18 @@ proc isDefectException*(t: PType): bool = t = skipTypes(t[0], abstractPtrs) return false +proc isDefectOrCatchableError*(t: PType): bool = + var t = t.skipTypes(abstractPtrs) + while t.kind == tyObject: + if t.sym != nil and t.sym.owner != nil and + sfSystemModule in t.sym.owner.flags and + (t.sym.name.s == "Defect" or + t.sym.name.s == "CatchableError"): + return true + if t[0] == nil: break + t = skipTypes(t[0], abstractPtrs) + return false + proc isSinkTypeForParam*(t: PType): bool = # a parameter like 'seq[owned T]' must not be used only once, but its # elements must, so we detect this case here: diff --git a/doc/manual.md b/doc/manual.md index 0a295fdb5..779e99186 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -4773,8 +4773,8 @@ Example: echo "overflow!" except ValueError, IOError: echo "catch multiple exceptions!" - except: - echo "Unknown exception!" + except CatchableError: + echo "Catchable exception!" finally: close(f) ``` @@ -4786,9 +4786,6 @@ listed in an `except` clause, the corresponding statements are executed. The statements following the `except` clauses are called `exception handlers`:idx:. -The empty `except`:idx: clause is executed if there is an exception that is -not listed otherwise. It is similar to an `else` clause in `if` statements. - If there is a `finally`:idx: clause, it is always executed after the exception handlers. @@ -4806,11 +4803,11 @@ Try can also be used as an expression; the type of the `try` branch then needs to fit the types of `except` branches, but the type of the `finally` branch always has to be `void`: - ```nim + ```nim test from std/strutils import parseInt let x = try: parseInt("133a") - except: -1 + except ValueError: -1 finally: echo "hi" ``` @@ -4818,8 +4815,9 @@ branch always has to be `void`: To prevent confusing code there is a parsing limitation; if the `try` follows a `(` it has to be written as a one liner: - ```nim - let x = (try: parseInt("133a") except: -1) + ```nim test + from std/strutils import parseInt + let x = (try: parseInt("133a") except ValueError: -1) ``` @@ -4867,7 +4865,7 @@ error message from `e`, and for such situations, it is enough to use ```nim try: # ... - except: + except CatchableError: echo getCurrentExceptionMsg() ``` @@ -5055,7 +5053,7 @@ An empty `raises` list (`raises: []`) means that no exception may be raised: try: unsafeCall() result = true - except: + except CatchableError: result = false ``` diff --git a/doc/tut2.md b/doc/tut2.md index 3c858c64e..4b9049082 100644 --- a/doc/tut2.md +++ b/doc/tut2.md @@ -393,7 +393,7 @@ The `try` statement handles exceptions: echo "could not convert string to integer" except IOError: echo "IO error!" - except: + except CatchableError: echo "Unknown exception!" # reraise the unknown exception: raise @@ -425,7 +425,7 @@ module. Example: ```nim try: doSomethingHere() - except: + except CatchableError: let e = getCurrentException() msg = getCurrentExceptionMsg() diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index f81be7a50..e9f76a26f 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -1610,7 +1610,7 @@ proc getFootnoteType(label: PRstNode): (FootnoteType, int) = elif label.len == 1 and label.sons[0].kind == rnLeaf: try: result = (fnManualNumber, parseInt(label.sons[0].text)) - except: + except ValueError: result = (fnCitation, -1) else: result = (fnCitation, -1) @@ -2899,13 +2899,13 @@ proc parseEnumList(p: var RstParser): PRstNode = let enumerator = p.tok[p.idx + 1 + wildIndex[w]].symbol # check that it's in sequence: enumerator == next(prevEnum) if "n" in wildcards[w]: # arabic numeral - let prevEnumI = try: parseInt(prevEnum) except: 1 + let prevEnumI = try: parseInt(prevEnum) except ValueError: 1 if enumerator in autoEnums: if prevAE != "" and enumerator != prevAE: break prevAE = enumerator curEnum = prevEnumI + 1 - else: curEnum = (try: parseInt(enumerator) except: 1) + else: curEnum = (try: parseInt(enumerator) except ValueError: 1) if curEnum - prevEnumI != 1: break prevEnum = enumerator diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 6b1300f11..24eab3abb 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -394,10 +394,10 @@ proc entityToRune*(entity: string): Rune = case entity[1] of '0'..'9': try: runeValue = parseInt(entity[1..^1]) - except: discard + except ValueError: discard of 'x', 'X': # not case sensitive here try: runeValue = parseHexInt(entity[2..^1]) - except: discard + except ValueError: discard else: discard # other entities are not defined with prefix `#` if runeValue notin 0..0x10FFFF: runeValue = 0 # only return legal values return Rune(runeValue) diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 03bab1b1b..5623ff8ef 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -98,6 +98,7 @@ template doAssertRaises*(exception: typedesc, code: untyped) = const begin = "expected raising '" & astToStr(exception) & "', instead" const msgEnd = " by: " & astToStr(code) template raisedForeign {.gensym.} = raiseAssert(begin & " raised foreign exception" & msgEnd) + {.warning[BareExcept]:off.} when Exception is exception: try: if true: @@ -116,5 +117,6 @@ template doAssertRaises*(exception: typedesc, code: untyped) = mixin `$` # alternatively, we could define $cstring in this module raiseAssert(begin & " raised '" & $e.name & "'" & msgEnd) except: raisedForeign() + {.warning[BareExcept]:on.} if wrong: raiseAssert(begin & " nothing was raised" & msgEnd) diff --git a/tests/nimdoc/trunnableexamples.nim b/tests/nimdoc/trunnableexamples.nim index ac7a0e26f..c88d8fa3f 100644 --- a/tests/nimdoc/trunnableexamples.nim +++ b/tests/nimdoc/trunnableexamples.nim @@ -65,7 +65,7 @@ when true: # issue #12746 runnableExamples: try: discard - except: + except CatchableError: # just the general except will work discard From d88f46df388604c1d21a2a95482c3255dc5884b6 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 16 Dec 2022 02:05:34 +1100 Subject: [PATCH 1463/1654] Make async stacktraces less verbose (#21091) * Name iterators something human readable Remove intermediate async procs from stacktraces Clean async traceback message from reraises message * Remove unused import/variable * Fix failing tests Don't add {.stackTrace: off.} to anonymous procs (They already don't appear in stacktrace) * Fix failing tests in pragma category Now check that the nim is a routine type first so we don't run into any assertion defects * Hide stack trace pragma in docs and update doc tests User doesn't need to know if something won't appear so this more becomes verbose noise If this is a bad idea we can always add a `when defined(nimdoc)` switch so we don't add {.stackTrace: off.} to the Future[T] returning proc for docs --- compiler/renderer.nim | 4 +- lib/pure/asyncfutures.nim | 16 ++----- lib/pure/asyncmacro.nim | 16 ++++--- nimdoc/testproject/expected/testproject.html | 11 ++--- tests/async/tasync_gcunsafe.nim | 2 +- tests/async/tasync_traceback.nim | 47 ++++---------------- 6 files changed, 33 insertions(+), 63 deletions(-) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index ad74cf2fe..cd0d83eb0 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -619,7 +619,9 @@ proc isHideable(config: ConfigRef, n: PNode): bool = # xxx compare `ident` directly with `getIdent(cache, wRaises)`, but # this requires a `cache`. case n.kind - of nkExprColonExpr: result = n[0].kind == nkIdent and n[0].ident.s in ["raises", "tags", "extern", "deprecated", "forbids"] + of nkExprColonExpr: + result = n[0].kind == nkIdent and + n[0].ident.s.nimIdentNormalize in ["raises", "tags", "extern", "deprecated", "forbids", "stacktrace"] of nkIdent: result = n.ident.s in ["gcsafe", "deprecated"] else: result = false diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index 035b6182d..edf5f7277 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -329,29 +329,21 @@ proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string = if leftLen > longestLeft: longestLeft = leftLen - var indent = 2 # Format the entries. for entry in entries: let (filename, procname) = getFilenameProcname(entry) - if procname == "": - if entry.line == reraisedFromBegin: - result.add(spaces(indent) & "#[\n") - indent.inc(2) - elif entry.line == reraisedFromEnd: - indent.dec(2) - result.add(spaces(indent) & "]#\n") - continue + if procname == "" and entry.line == reraisedFromBegin: + break let left = "$#($#)" % [filename, $entry.line] - result.add((spaces(indent) & "$#$# $#\n") % [ + result.add((spaces(2) & "$# $#\n") % [ left, - spaces(longestLeft - left.len + 2), procname ]) let hint = getHint(entry) if hint.len > 0: - result.add(spaces(indent+2) & "## " & hint & "\n") + result.add(spaces(4) & "## " & hint & "\n") proc injectStacktrace[T](future: Future[T]) = when not defined(release): diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim index c4f6fd80e..d80a47101 100644 --- a/lib/pure/asyncmacro.nim +++ b/lib/pure/asyncmacro.nim @@ -20,9 +20,8 @@ proc newCallWithLineInfo(fromNode: NimNode; theProc: NimNode, args: varargs[NimN template createCb(retFutureSym, iteratorNameSym, strName, identName, futureVarCompletions: untyped) = bind finished - var nameIterVar = iteratorNameSym - proc identName {.closure.} = + proc identName {.closure, stackTrace: off.} = try: if not nameIterVar.finished: var next = nameIterVar() @@ -151,6 +150,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = result[0][0] = quote do: Future[void] return result + if prc.kind in RoutineNodes and prc.name.kind != nnkEmpty: + # Only non anonymous functions need/can have stack trace disabled + prc.addPragma(nnkExprColonExpr.newTree(ident"stackTrace", ident"off")) + if prc.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.", prc) @@ -209,7 +212,7 @@ proc asyncSingleProc(prc: NimNode): NimNode = # -> {.pop.} # -> # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prcName & "Iter") + var iteratorNameSym = genSym(nskIterator, $prcName & " (Async)") var procBody = prc.body.processBody(retFutureSym, futureVarIdents) # don't do anything with forward bodies (empty) if procBody.kind != nnkEmpty: @@ -251,9 +254,10 @@ proc asyncSingleProc(prc: NimNode): NimNode = # friendlier stack traces: var cbName = genSym(nskProc, prcName & NimAsyncContinueSuffix) var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prcName), - cbName, - createFutureVarCompletions(futureVarIdents, nil)) + newStrLitNode(prcName), + cbName, + createFutureVarCompletions(futureVarIdents, nil) + ) outerProcBody.add procCb # -> return retFuture diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index b68b721ff..7d10d7900 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -503,7 +503,8 @@
          - 
          proc asyncFun1(): Future[int] {....raises: [Exception, ValueError],
+  
          proc asyncFun1(): Future[int] {....stackTrace: false,
+                                raises: [Exception, ValueError],
                                 tags: [RootEffect], forbids: [].}
          
   
          
     
@@ -515,8 +516,8 @@
          - 
          proc asyncFun2(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect],
-                                        forbids: [].}
          + 
          proc asyncFun2(): owned(Future[void]) {....stackTrace: false, raises: [Exception],
+                                        tags: [RootEffect], forbids: [].}
          @@ -527,8 +528,8 @@
          - 
          proc asyncFun3(): owned(Future[void]) {....raises: [Exception], tags: [RootEffect],
-                                        forbids: [].}
          + 
          proc asyncFun3(): owned(Future[void]) {....stackTrace: false, raises: [Exception],
+                                        tags: [RootEffect], forbids: [].}
          diff --git a/tests/async/tasync_gcunsafe.nim b/tests/async/tasync_gcunsafe.nim index 00c92b109..f3e6bc691 100644 --- a/tests/async/tasync_gcunsafe.nim +++ b/tests/async/tasync_gcunsafe.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "'anotherGCSafeAsyncProcIter' is not GC-safe as it calls 'asyncGCUnsafeProc'" + errormsg: "'anotherGCSafeAsyncProc (Async)' is not GC-safe as it calls 'asyncGCUnsafeProc'" cmd: "nim c --threads:on $file" file: "asyncmacro.nim" """ diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim index ec67d34f9..98f71b192 100644 --- a/tests/async/tasync_traceback.nim +++ b/tests/async/tasync_traceback.nim @@ -67,51 +67,22 @@ import re const expected = """ b failure Async traceback: - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncmacro\.nim\(\d+?\)\s+?a - asyncmacro\.nim\(\d+?\)\s+?aNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?aIter - asyncmacro\.nim\(\d+?\)\s+?b - asyncmacro\.nim\(\d+?\)\s+?bNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?bIter - #\[ - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncmacro\.nim\(\d+?\)\s+?a - asyncmacro\.nim\(\d+?\)\s+?aNimAsyncContinue - ## Resumes an async procedure - asyncmacro\.nim\(\d+?\)\s+?aIter - asyncfutures\.nim\(\d+?\)\s+?read - \]# + tasync_traceback\.nim\(\d+?\) tasync_traceback + tasync_traceback\.nim\(\d+?\) a \(Async\) + tasync_traceback\.nim\(\d+?\) b \(Async\) Exception message: b failure bar failure Async traceback: - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncdispatch\.nim\(\d+?\)\s+?waitFor - asyncdispatch\.nim\(\d+?\)\s+?poll + tasync_traceback\.nim\(\d+?\) tasync_traceback + asyncdispatch\.nim\(\d+?\) waitFor + asyncdispatch\.nim\(\d+?\) poll ## Processes asynchronous completion events - asyncdispatch\.nim\(\d+?\)\s+?runOnce - asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks + asyncdispatch\.nim\(\d+?\) runOnce + asyncdispatch\.nim\(\d+?\) processPendingCallbacks ## Executes pending callbacks - asyncmacro\.nim\(\d+?\)\s+?barNimAsyncContinue - ## Resumes an async procedure - tasync_traceback\.nim\(\d+?\)\s+?barIter - #\[ - tasync_traceback\.nim\(\d+?\)\s+?tasync_traceback - asyncdispatch\.nim\(\d+?\)\s+?waitFor - asyncdispatch\.nim\(\d+?\)\s+?poll - ## Processes asynchronous completion events - asyncdispatch\.nim\(\d+?\)\s+?runOnce - asyncdispatch\.nim\(\d+?\)\s+?processPendingCallbacks - ## Executes pending callbacks - asyncmacro\.nim\(\d+?\)\s+?fooNimAsyncContinue - ## Resumes an async procedure - asyncmacro\.nim\(\d+?\)\s+?fooIter - asyncfutures\.nim\(\d+?\)\s+?read - \]# + tasync_traceback\.nim\(\d+?\) bar \(Async\) Exception message: bar failure """ From 8054be6e529db9e65a18e4c4a37ec54f64f9121c Mon Sep 17 00:00:00 2001 From: Bung Date: Thu, 15 Dec 2022 23:12:17 +0800 Subject: [PATCH 1464/1654] fix #20588 (#21104) --- compiler/semexprs.nim | 2 ++ tests/arc/t20588.nim | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/arc/t20588.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 2efa1259a..8ac8f4750 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -319,6 +319,8 @@ proc semConv(c: PContext, n: PNode; expectedType: PType = nil): PNode = return evaluated else: targetType = targetType.base + of tyAnything, tyUntyped, tyTyped: + localError(c.config, n.info, "illegal type conversion to '$1'" % typeToString(targetType)) else: discard maybeLiftType(targetType, c, n[0].info) diff --git a/tests/arc/t20588.nim b/tests/arc/t20588.nim new file mode 100644 index 000000000..d747c656d --- /dev/null +++ b/tests/arc/t20588.nim @@ -0,0 +1,22 @@ +discard """ + cmd: "nim check --warnings:off --hints:off $file" + errormsg: "" + nimout: ''' +t20588.nim(20, 12) Error: illegal type conversion to 'auto' +t20588.nim(21, 14) Error: illegal type conversion to 'typed' +t20588.nim(22, 16) Error: illegal type conversion to 'untyped' +''' +""" + + + + + + + + + + +discard 0.0.auto +discard typed("abc") +discard untyped(4) From d00477dffb2c62732b87a440c61706de5b480f48 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 16 Dec 2022 05:22:45 +1100 Subject: [PATCH 1465/1654] Check file exists in `{.compile.}` pragma (#21105) * Add test * Check file exists before adding it into compilation * Make error message look like other error messages i.e. following the format `error msg: file` --- compiler/pragmas.nim | 7 +++++-- tests/pragmas/tcompile_missing_file.nim | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/pragmas/tcompile_missing_file.nim diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 993e6edce..92e3d1922 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -513,8 +513,11 @@ proc processCompile(c: PContext, n: PNode) = var cf = Cfile(nimname: splitFile(src).name, cname: src, obj: dest, flags: {CfileFlag.External}, customArgs: customArgs) - extccomp.addExternalFileToCompile(c.config, cf) - recordPragma(c, it, "compile", src.string, dest.string, customArgs) + if not fileExists(src): + localError(c.config, n.info, "cannot find: " & src.string) + else: + extccomp.addExternalFileToCompile(c.config, cf) + recordPragma(c, it, "compile", src.string, dest.string, customArgs) proc getStrLit(c: PContext, n: PNode; i: int): string = n[i] = c.semConstExpr(c, n[i]) diff --git a/tests/pragmas/tcompile_missing_file.nim b/tests/pragmas/tcompile_missing_file.nim new file mode 100644 index 000000000..fd90bd57d --- /dev/null +++ b/tests/pragmas/tcompile_missing_file.nim @@ -0,0 +1,5 @@ +discard """ + joinable: false + errormsg: "cannot find: noexist.c" +""" +{.compile: "noexist.c".} From 78b68421da0a678deb0c17cd03345d62410dbeac Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 11:36:52 +0800 Subject: [PATCH 1466/1654] fixes chronicles (#21114) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index d0c87919b..c134be6f9 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -51,7 +51,7 @@ pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true pkg "chroma" -pkg "chronicles", "nimble install -y stew@#head; nim c -o:chr -r chronicles.nim" +pkg "chronicles", "nim c -o:chr -r chronicles.nim", url = "https://github.com/nim-lang/nim-chronicles" pkg "chronos", "nim c -r -d:release tests/testall", url = "https://github.com/nim-lang/nim-chronos" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" From 644318ad9cbf079f33d8d7b6c5ead4a4429adb3e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 14:58:40 +0800 Subject: [PATCH 1467/1654] the devel branch now uses csources_v2 (#21108) * the devel branch now uses csources_v2 * fixes hash * bump csources_v2 --- .gitignore | 1 + config/build_config.txt | 6 +++--- nim.nimble | 2 +- readme.md | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d228b498c..d0c4558df 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,7 @@ testament.db /csources /csources_v1 +/csources_v2 /dist/ # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it diff --git a/config/build_config.txt b/config/build_config.txt index 15f44bb57..3ae618f00 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -1,5 +1,5 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" -nim_csourcesDir=csources_v1 -nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git +nim_csourcesDir=csources_v2 +nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=561b417c65791cd8356b5f73620914ceff845d10 +nim_csourcesHash=ffd834c20915dc04c78eb0d3b2162832e7ec9c13 diff --git a/nim.nimble b/nim.nimble index a629c0e04..5d1be49b1 100644 --- a/nim.nimble +++ b/nim.nimble @@ -5,7 +5,7 @@ license = "MIT" bin = @["compiler/nim", "nimsuggest/nimsuggest"] skipFiles = @["azure-pipelines.yml" , "build_all.bat" , "build_all.sh" , "build_nimble.bat" , "build_nimble.sh" , "changelog.md" , "koch.nim.cfg" , "nimblemeta.json" , "readme.md" , "security.md" ] -skipDirs = @["build" , "changelogs" , "ci" , "csources_v1" , "drnim" , "nimdoc", "testament"] +skipDirs = @["build" , "changelogs" , "ci" , "csources_v2" , "drnim" , "nimdoc", "testament"] before install: when defined(windows): diff --git a/readme.md b/readme.md index 3d32bcf87..a0f47eb41 100644 --- a/readme.md +++ b/readme.md @@ -47,7 +47,7 @@ Compiling the Nim compiler is quite straightforward if you follow these steps: First, the C source of an older version of the Nim compiler is needed to bootstrap the latest version because the Nim compiler itself is written in the Nim programming language. Those C sources are available within the -[``nim-lang/csources_v1``][csources-v1-repo] repository. +[``nim-lang/csources_v2``][csources-v2-repo] repository. Next, to build from source you will need: @@ -223,7 +223,7 @@ Copyright © 2006-2022 Andreas Rumpf, all rights reserved. [nimble-repo]: https://github.com/nim-lang/nimble [nimsuggest-repo]: https://github.com/nim-lang/nimsuggest [csources-repo-deprecated]: https://github.com/nim-lang/csources -[csources-v1-repo]: https://github.com/nim-lang/csources_v1 +[csources-v2-repo]: https://github.com/nim-lang/csources_v2 [badge-nim-irc]: https://img.shields.io/badge/chat-on_irc-blue.svg?style=flat-square [badge-nim-discord]: https://img.shields.io/discord/371759389889003530?color=blue&label=discord&logo=discord&logoColor=gold&style=flat-square [badge-nim-gitter]: https://img.shields.io/badge/chat-on_gitter-blue.svg?style=flat-square From 8a3b76b28733a87e5cffa3f4b7b99d76f13ee2ba Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 14:59:18 +0800 Subject: [PATCH 1468/1654] csource building prefers `bin/nim` (#21115) --- tools/niminst/niminst.nim | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index cd2e5a481..bc763a56f 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -515,6 +515,17 @@ template gatherFiles(fun, libpath, outDir) = # commenting out for now, see discussion in https://github.com/nim-lang/Nim/pull/13413 # copySrc(libpath / "lib/wrappers/linenoise/linenoise.h") +proc exe(f: string): string = + result = addFileExt(f, ExeExt) + when defined(windows): + result = result.replace('/','\\') + +proc findNim(): string = + let nim = "nim".exe + result = quoteShell("bin" / nim) + if not fileExists(result): + result = "nim" + proc srcdist(c: var ConfigData) = let cCodeDir = getOutputDir(c) / "c_code" if not dirExists(cCodeDir): createDir(cCodeDir) @@ -533,10 +544,10 @@ proc srcdist(c: var ConfigData) = var dir = getOutputDir(c) / buildDir(osA, cpuA) if dirExists(dir): removeDir(dir) createDir(dir) - var cmd = ("nim compile -f --symbolfiles:off --compileonly " & + var cmd = ("$# compile -f --symbolfiles:off --compileonly " & "--gen_mapping --cc:gcc --skipUserCfg" & " --os:$# --cpu:$# $# $#") % - [osname, cpuname, c.nimArgs, c.mainfile] + [findNim(), osname, cpuname, c.nimArgs, c.mainfile] echo(cmd) if execShellCmd(cmd) != 0: quit("Error: call to nim compiler failed") From a9bd78d579168ba50f42d3bc418cd7de70a226f9 Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 16 Dec 2022 15:01:15 +0800 Subject: [PATCH 1469/1654] fix #12122 (#21096) --- compiler/astalgo.nim | 2 +- compiler/evalffi.nim | 8 ++-- lib/core/typeinfo.nim | 4 +- lib/pure/coro.nim | 4 +- lib/pure/memfiles.nim | 8 ++-- lib/pure/oids.nim | 4 +- lib/pure/strutils.nim | 4 +- lib/std/envvars.nim | 2 +- lib/std/private/strimpl.nim | 4 +- lib/std/syncio.nim | 2 +- lib/system/alloc.nim | 70 ++++++++++++++++----------------- lib/system/assign.nim | 28 ++++++------- lib/system/channels_builtin.nim | 12 +++--- lib/system/ctypes.nim | 2 +- lib/system/deepcopy.nim | 16 ++++---- lib/system/gc.nim | 24 +++++------ lib/system/gc2.nim | 22 +++++------ lib/system/gc_common.nim | 24 +++++------ lib/system/gc_ms.nim | 18 ++++----- lib/system/osalloc.nim | 6 +-- lib/system/repr.nim | 10 ++--- lib/system/sysstr.nim | 4 +- tests/parallel/tguard1.nim | 2 +- 23 files changed, 140 insertions(+), 140 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 28c38129e..e9ed1497a 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -333,7 +333,7 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int, sonsRope = rope("null") elif containsOrIncl(marker, n.id): sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope( - strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))] + strutils.toHex(cast[int](n), sizeof(n) * 2))] else: if n.len > 0: sonsRope = rope("[") diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 37edef86c..0577619b8 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -77,7 +77,7 @@ proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode = theAddr = dllhandle.symAddr(name.cstring) if theAddr.isNil: globalError(conf, sym.info, "cannot import symbol: " & name & " from " & libPathMsg) - result.intVal = cast[ByteAddress](theAddr) + result.intVal = cast[int](theAddr) proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type = if t == nil: return addr libffi.type_void @@ -113,7 +113,7 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[] template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v template `+!`(x, y: untyped): untyped = - cast[pointer](cast[ByteAddress](x) + y) + cast[pointer](cast[int](x) + y) proc packSize(conf: ConfigRef, v: PNode, typ: PType): int = ## computes the size of the blob @@ -369,13 +369,13 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = # in their unboxed representation so nothing it to be unpacked: result = n else: - awi(nkPtrLit, cast[ByteAddress](p)) + awi(nkPtrLit, cast[int](p)) of tyPtr, tyRef, tyVar, tyLent: let p = rd(pointer, x) if p.isNil: setNil() elif n == nil or n.kind == nkPtrLit: - awi(nkPtrLit, cast[ByteAddress](p)) + awi(nkPtrLit, cast[int](p)) elif n != nil and n.len == 1: internalAssert(conf, n.kind == nkRefTy) n[0] = unpack(conf, p, typ.lastSon, n[0]) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 89b1deb31..5075c8458 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -132,12 +132,12 @@ else: proc prepareSeqAdd(len: int; p: pointer; addlen, elemSize, elemAlign: int): pointer {. importCompilerProc.} -template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b) +template `+!!`(a, b): untyped = cast[pointer](cast[int](a) + b) proc getDiscriminant(aa: pointer, n: ptr TNimNode): int = assert(n.kind == nkCase) var d: int - let a = cast[ByteAddress](aa) + let a = cast[int](aa) case n.typ.size of 1: d = int(cast[ptr uint8](a +% n.offset)[]) of 2: d = int(cast[ptr uint16](a +% n.offset)[]) diff --git a/lib/pure/coro.nim b/lib/pure/coro.nim index 47dfdd791..3b9bba59b 100644 --- a/lib/pure/coro.nim +++ b/lib/pure/coro.nim @@ -281,8 +281,8 @@ proc start*(c: proc(), stacksize: int = defaultStackSize): CoroutineRef {.discar (proc(p: pointer) {.stdcall.} = runCurrentTask()), nil) else: coro = cast[CoroutinePtr](alloc0(sizeof(Coroutine) + stacksize)) - coro.stack.top = cast[pointer](cast[ByteAddress](coro) + sizeof(Coroutine)) - coro.stack.bottom = cast[pointer](cast[ByteAddress](coro.stack.top) + stacksize) + coro.stack.top = cast[pointer](cast[int](coro) + sizeof(Coroutine)) + coro.stack.bottom = cast[pointer](cast[int](coro.stack.top) + stacksize) when coroBackend == CORO_BACKEND_UCONTEXT: discard getcontext(coro.execContext) coro.execContext.uc_stack.ss_sp = coro.stack.top diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 48c07b149..9ff8bcd74 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -495,8 +495,8 @@ proc mmsSetPosition(s: Stream, pos: int) = proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int = - let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem) - let p = cast[ByteAddress](MemMapFileStream(s).pos) + let startAddress = cast[int](MemMapFileStream(s).mf.mem) + let p = cast[int](MemMapFileStream(s).pos) let l = min(bufLen, MemMapFileStream(s).mf.size - p) moveMem(buffer, cast[pointer](startAddress + p), l) result = l @@ -511,8 +511,8 @@ proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) = let size = MemMapFileStream(s).mf.size if MemMapFileStream(s).pos + bufLen > size: raise newEIO("cannot write to stream") - let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) + - cast[ByteAddress](MemMapFileStream(s).pos) + let p = cast[int](MemMapFileStream(s).mf.mem) + + cast[int](MemMapFileStream(s).pos) moveMem(cast[pointer](p), buffer, bufLen) inc(MemMapFileStream(s).pos, bufLen) diff --git a/lib/pure/oids.nim b/lib/pure/oids.nim index e6e5e6e56..ad8eeefd7 100644 --- a/lib/pure/oids.nim +++ b/lib/pure/oids.nim @@ -44,7 +44,7 @@ proc hexbyte*(hex: char): int {.inline.} = proc parseOid*(str: cstring): Oid = ## Parses an OID. - var bytes = cast[cstring](cast[pointer](cast[ByteAddress](addr(result.time)) + 4)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(result.time)) + 4)) var i = 0 while i < 12: bytes[i] = chr((hexbyte(str[2 * i]) shl 4) or hexbyte(str[2 * i + 1])) @@ -57,7 +57,7 @@ proc `$`*(oid: Oid): string = result.setLen 24 var o = oid - var bytes = cast[cstring](cast[pointer](cast[ByteAddress](addr(o)) + 4)) + var bytes = cast[cstring](cast[pointer](cast[int](addr(o)) + 4)) var i = 0 while i < 12: let b = bytes[i].ord diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 0367ea1e5..f628cfc2d 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1914,7 +1914,7 @@ func find*(s: string, sub: char, start: Natural = 0, last = -1): int {.rtl, if length > 0: let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](length)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + return cast[int](found) -% cast[int](s.cstring) else: findImpl() @@ -1970,7 +1970,7 @@ func find*(s, sub: string, start: Natural = 0, last = -1): int {.rtl, if last < 0 and start < s.len and subLen != 0: let found = memmem(s[start].unsafeAddr, csize_t(s.len - start), sub.cstring, csize_t(subLen)) result = if not found.isNil: - cast[ByteAddress](found) -% cast[ByteAddress](s.cstring) + cast[int](found) -% cast[int](s.cstring) else: -1 else: diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index 46595a3cf..817a155ec 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -183,7 +183,7 @@ when not defined(nimscript): let kv = $e let p = find(kv, '=') yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[ByteAddress](eend)+size) + e = cast[typ](cast[int](eend)+size) if typeof(zero)(eend[1]) == zero: break discard free_fun(env) impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) diff --git a/lib/std/private/strimpl.nim b/lib/std/private/strimpl.nim index 6a38cbfd2..f8c9236a5 100644 --- a/lib/std/private/strimpl.nim +++ b/lib/std/private/strimpl.nim @@ -93,7 +93,7 @@ func find*(s: cstring, sub: char, start: Natural = 0, last = 0): int = if L > 0: let found = c_memchr(s[start].unsafeAddr, sub, cast[csize_t](L)) if not found.isNil: - return cast[ByteAddress](found) -% cast[ByteAddress](s) + return cast[int](found) -% cast[int](s) return -1 func find*(s, sub: cstring, start: Natural = 0, last = 0): int = @@ -108,6 +108,6 @@ func find*(s, sub: cstring, start: Natural = 0, last = 0): int = if last == 0 and s.len > start: let found = c_strstr(cast[cstring](s[start].unsafeAddr), sub) if not found.isNil: - result = cast[ByteAddress](found) -% cast[ByteAddress](s) + result = cast[int](found) -% cast[int](s) else: result = -1 diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 2b5ea6a3c..46f113450 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -480,7 +480,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], let m = c_memchr(addr line[pos], '\L'.ord, cast[csize_t](sp)) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done - var last = cast[ByteAddress](m) - cast[ByteAddress](addr line[0]) + var last = cast[int](m) - cast[int](addr line[0]) if last > 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index a7db3704b..5d1548977 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -290,7 +290,7 @@ proc llAlloc(a: var MemRegion, size: int): pointer = a.llmem.size = PageSize - sizeof(LLChunk) a.llmem.acc = sizeof(LLChunk) a.llmem.next = old - result = cast[pointer](cast[ByteAddress](a.llmem) + a.llmem.acc) + result = cast[pointer](cast[int](a.llmem) + a.llmem.acc) dec(a.llmem.size, size) inc(a.llmem.acc, size) zeroMem(result, size) @@ -422,7 +422,7 @@ iterator allObjects(m: var MemRegion): pointer {.inline.} = var c = cast[PSmallChunk](c) let size = c.size - var a = cast[ByteAddress](addr(c.data)) + var a = cast[int](addr(c.data)) let limit = a + c.acc while a <% limit: yield cast[pointer](a) @@ -441,13 +441,13 @@ when not defined(gcDestructors): # ------------- chunk management ---------------------------------------------- proc pageIndex(c: PChunk): int {.inline.} = - result = cast[ByteAddress](c) shr PageShift + result = cast[int](c) shr PageShift proc pageIndex(p: pointer): int {.inline.} = - result = cast[ByteAddress](p) shr PageShift + result = cast[int](p) shr PageShift proc pageAddr(p: pointer): PChunk {.inline.} = - result = cast[PChunk](cast[ByteAddress](p) and not PageMask) + result = cast[PChunk](cast[int](p) and not PageMask) #sysAssert(Contains(allocator.chunkStarts, pageIndex(result))) when false: @@ -495,13 +495,13 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = when defined(memtracker): trackLocation(addr result.size, sizeof(int)) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "requestOsChunks 1") + sysAssert((cast[int](result) and PageMask) == 0, "requestOsChunks 1") #zeroMem(result, size) result.next = nil result.prev = nil result.size = size # update next.prevSize: - var nxt = cast[ByteAddress](result) +% size + var nxt = cast[int](result) +% size sysAssert((nxt and PageMask) == 0, "requestOsChunks 2") var next = cast[PChunk](nxt) if pageIndex(next) in a.chunkStarts: @@ -509,7 +509,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk = next.prevSize = size or (next.prevSize and 1) # set result.prevSize: var lastSize = if a.lastSize != 0: a.lastSize else: PageSize - var prv = cast[ByteAddress](result) -% lastSize + var prv = cast[int](result) -% lastSize sysAssert((nxt and PageMask) == 0, "requestOsChunks 3") var prev = cast[PChunk](prv) if pageIndex(prev) in a.chunkStarts and prev.size == lastSize: @@ -555,13 +555,13 @@ proc listRemove[T](head: var T, c: T) {.inline.} = proc updatePrevSize(a: var MemRegion, c: PBigChunk, prevSize: int) {.inline.} = - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "updatePrevSize") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "updatePrevSize") if isAccessible(a, ri): ri.prevSize = prevSize or (ri.prevSize and 1) proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = - result = cast[PBigChunk](cast[ByteAddress](c) +% size) + result = cast[PBigChunk](cast[int](c) +% size) result.size = c.size - size track("result.size", addr result.size, sizeof(int)) when not defined(nimOptimizedSplitChunk): @@ -590,8 +590,8 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = when coalescLeft: let prevSize = c.prevSize if prevSize != 0: - var le = cast[PChunk](cast[ByteAddress](c) -% prevSize) - sysAssert((cast[ByteAddress](le) and PageMask) == 0, "freeBigChunk 4") + var le = cast[PChunk](cast[int](c) -% prevSize) + sysAssert((cast[int](le) and PageMask) == 0, "freeBigChunk 4") if isAccessible(a, le) and chunkUnused(le): sysAssert(not isSmallChunk(le), "freeBigChunk 5") if not isSmallChunk(le) and le.size < MaxBigChunkSize: @@ -607,8 +607,8 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) = addChunkToMatrix(a, c) c = rest when coalescRight: - var ri = cast[PChunk](cast[ByteAddress](c) +% c.size) - sysAssert((cast[ByteAddress](ri) and PageMask) == 0, "freeBigChunk 2") + var ri = cast[PChunk](cast[int](c) +% c.size) + sysAssert((cast[int](ri) and PageMask) == 0, "freeBigChunk 2") if isAccessible(a, ri) and chunkUnused(ri): sysAssert(not isSmallChunk(ri), "freeBigChunk 3") if not isSmallChunk(ri) and c.size < MaxBigChunkSize: @@ -669,7 +669,7 @@ proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = incCurrMem(a, size) # XXX add this to the heap links. But also remove it from it later. when false: a.addHeapLink(result, size) - sysAssert((cast[ByteAddress](result) and PageMask) == 0, "getHugeChunk") + sysAssert((cast[int](result) and PageMask) == 0, "getHugeChunk") result.next = nil result.prev = nil result.size = size @@ -838,7 +838,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = c.prev = nil listAdd(a.freeSmallChunks[s], c) result = addr(c.data) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 4") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 4") else: sysAssert(allocInv(a), "rawAlloc: begin c != nil") sysAssert c.next != c, "rawAlloc 5" @@ -856,7 +856,7 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = if c.freeList == nil: sysAssert(c.acc + smallChunkOverhead() + size <= SmallChunkSize, "rawAlloc 7") - result = cast[pointer](cast[ByteAddress](addr(c.data)) +% c.acc) + result = cast[pointer](cast[int](addr(c.data)) +% c.acc) inc(c.acc, size) else: result = c.freeList @@ -864,14 +864,14 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert(c.freeList.zeroField == 0, "rawAlloc 8") c.freeList = c.freeList.next dec(c.free, size) - sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9") + sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert(allocInv(a), "rawAlloc: end c != nil") sysAssert(allocInv(a), "rawAlloc: before c.free < size") if c.free < size: sysAssert(allocInv(a), "rawAlloc: before listRemove test") listRemove(a.freeSmallChunks[s], c) sysAssert(allocInv(a), "rawAlloc: end listRemove test") - sysAssert(((cast[ByteAddress](result) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](result) and PageMask) - smallChunkOverhead()) %% size == 0, "rawAlloc 21") sysAssert(allocInv(a), "rawAlloc: end small size") inc a.occ, size @@ -893,11 +893,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer = sysAssert c.prev == nil, "rawAlloc 10" sysAssert c.next == nil, "rawAlloc 11" result = addr(c.data) - sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13") - sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") + sysAssert((cast[int](c) and (MemAlign-1)) == 0, "rawAlloc 13") + sysAssert((cast[int](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary") when not defined(gcDestructors): if a.root == nil: a.root = getBottom(a) - add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size) + add(a, a.root, cast[int](result), cast[int](result)+%size) inc a.occ, c.size trackSize(c.size) sysAssert(isAccessible(a, result), "rawAlloc 14") @@ -927,10 +927,10 @@ proc rawDealloc(a: var MemRegion, p: pointer) = dec a.occ, s untrackSize(s) sysAssert a.occ >= 0, "rawDealloc: negative occupied memory (case A)" - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 3") when not defined(gcDestructors): - #echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField))) + #echo("setting to nil: ", $cast[int](addr(f.zeroField))) sysAssert(f.zeroField != 0, "rawDealloc 1") f.zeroField = 0 f.next = c.freeList @@ -953,7 +953,7 @@ proc rawDealloc(a: var MemRegion, p: pointer) = else: when defined(gcDestructors): addToSharedFreeList(c, f) - sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %% + sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead()) %% s == 0, "rawDealloc 2") else: # set to 0xff to check for usage after free bugs: @@ -975,7 +975,7 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() result = (c.acc >% offset) and (offset %% c.size == 0) and (cast[ptr FreeCell](p).zeroField >% 1) @@ -993,12 +993,12 @@ when not defined(gcDestructors): if not chunkUnused(c): if isSmallChunk(c): var c = cast[PSmallChunk](c) - var offset = (cast[ByteAddress](p) and (PageSize-1)) -% + var offset = (cast[int](p) and (PageSize-1)) -% smallChunkOverhead() if c.acc >% offset: - sysAssert(cast[ByteAddress](addr(c.data)) +% offset == - cast[ByteAddress](p), "offset is not what you think it is") - var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +% + sysAssert(cast[int](addr(c.data)) +% offset == + cast[int](p), "offset is not what you think it is") + var d = cast[ptr FreeCell](cast[int](addr(c.data)) +% offset -% (offset %% c.size)) if d.zeroField >% 1: result = d @@ -1025,7 +1025,7 @@ when not defined(gcDestructors): proc ptrSize(p: pointer): int = when not defined(gcDestructors): - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) var c = pageAddr(p) sysAssert(not chunkUnused(c), "ptrSize") result = c.size -% sizeof(FreeCell) @@ -1043,7 +1043,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} = result = rawAlloc(allocator, size+sizeof(FreeCell)) cast[ptr FreeCell](result).zeroField = 1 # mark it as used sysAssert(not isAllocatedPtr(allocator, result), "alloc") - result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) + result = cast[pointer](cast[int](result) +% sizeof(FreeCell)) track("alloc", result, size) else: result = rawAlloc(allocator, size) @@ -1055,7 +1055,7 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer = proc dealloc(allocator: var MemRegion, p: pointer) = when not defined(gcDestructors): sysAssert(p != nil, "dealloc: p is nil") - var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell)) + var x = cast[pointer](cast[int](p) -% sizeof(FreeCell)) sysAssert(x != nil, "dealloc: x is nil") sysAssert(isAccessible(allocator, x), "is not accessible") sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted") @@ -1116,7 +1116,7 @@ template instantiateForRegion(allocator: untyped) {.dirty.} = result = interiorAllocatedPtr(allocator, p) proc isAllocatedPtr*(p: pointer): bool = - let p = cast[pointer](cast[ByteAddress](p)-%ByteAddress(sizeof(Cell))) + let p = cast[pointer](cast[int](p)-%ByteAddress(sizeof(Cell))) result = isAllocatedPtr(allocator, p) proc deallocOsPages = deallocOsPages(allocator) diff --git a/lib/system/assign.nim b/lib/system/assign.nim index 20b854107..42587929c 100644 --- a/lib/system/assign.nim +++ b/lib/system/assign.nim @@ -15,8 +15,8 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) {.benign. proc genericAssignAux(dest, src: pointer, n: ptr TNimNode, shallow: bool) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericAssignAux(cast[pointer](d +% n.offset), @@ -56,8 +56,8 @@ template deepSeqAssignImpl(operation, additionalArg) {.dirty.} = proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericAssignAux 2") case mt.kind of tyString: @@ -89,17 +89,17 @@ proc genericAssignAux(dest, src: pointer, mt: PNimType, shallow: bool) = var ss = nimNewSeqOfCap(mt, seq.len) cast[PGenericSeq](ss).len = seq.len unsureAsgnRef(x, ss) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) copyMem(cast[pointer](dst +% align(GenericSeqSize, mt.base.align)), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align)), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align)), seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericAssignAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size ), mt.base, shallow) of tyObject: var it = mt.base @@ -181,15 +181,15 @@ proc genericSeqAssign(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericAssignOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericAssign(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) proc objectInit(dest: pointer, typ: PNimType) {.compilerproc, benign.} proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "objectInitAux") of nkSlot: objectInit(cast[pointer](d +% n.offset), n.typ) @@ -203,7 +203,7 @@ proc objectInitAux(dest: pointer, n: ptr TNimNode) {.benign.} = proc objectInit(dest: pointer, typ: PNimType) = # the generic init proc that takes care of initialization of complex # objects on the stack or heap - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case typ.kind of tyObject: # iterate over any structural type @@ -226,7 +226,7 @@ proc objectInit(dest: pointer, typ: PNimType) = proc genericReset(dest: pointer, mt: PNimType) {.compilerproc, benign.} proc genericResetAux(dest: pointer, n: ptr TNimNode) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkNone: sysAssert(false, "genericResetAux") of nkSlot: genericReset(cast[pointer](d +% n.offset), n.typ) @@ -238,7 +238,7 @@ proc genericResetAux(dest: pointer, n: ptr TNimNode) = zeroMem(cast[pointer](d +% n.offset), n.typ.size) proc genericReset(dest: pointer, mt: PNimType) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) sysAssert(mt != nil, "genericReset 2") case mt.kind of tyRef: diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim index fbe3f0e98..e04f2a0db 100644 --- a/lib/system/channels_builtin.nim +++ b/lib/system/channels_builtin.nim @@ -185,8 +185,8 @@ when not usesDestructors: proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, mode: LoadStoreMode) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: storeAux(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), n.typ, t, mode) @@ -205,8 +205,8 @@ when not usesDestructors: cast[pointer](cast[int](p) +% x) var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "mt == nil") case mt.kind of tyString: @@ -245,14 +245,14 @@ when not usesDestructors: x[] = alloc0(t.region, align(GenericSeqSize, mt.base.align) +% seq.len *% mt.base.size) else: unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) var dstseq = cast[PGenericSeq](dst) dstseq.len = seq.len dstseq.reserved = seq.len for i in 0..seq.len-1: storeAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), mt.base, t, mode) if mode != mStore: dealloc(t.region, s2) diff --git a/lib/system/ctypes.nim b/lib/system/ctypes.nim index 6ba28ed49..f6a341477 100644 --- a/lib/system/ctypes.nim +++ b/lib/system/ctypes.nim @@ -67,7 +67,7 @@ type # these work for most platforms: ## This is the same as the type `unsigned long long` in *C*. type - ByteAddress* = int + ByteAddress* {.deprecated: "use `uint`".} = int ## is the signed integer type that should be used for converting ## pointers to integer addresses for readability. diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 1f30b8427..72d35f518 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -61,8 +61,8 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; tab: var PtrTable) {.benign.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) case n.kind of nkSlot: genericDeepCopyAux(cast[pointer](d +% n.offset), @@ -85,8 +85,8 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode; proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) sysAssert(mt != nil, "genericDeepCopyAux 2") case mt.kind of tyString: @@ -113,11 +113,11 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) = return sysAssert(dest != nil, "genericDeepCopyAux 3") unsureAsgnRef(x, newSeq(mt, seq.len)) - var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dst = cast[int](cast[PPointer](dest)[]) for i in 0..seq.len-1: genericDeepCopyAux( cast[pointer](dst +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), - cast[pointer](cast[ByteAddress](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), + cast[pointer](cast[int](s2) +% align(GenericSeqSize, mt.base.align) +% i *% mt.base.size), mt.base, tab) of tyObject: # we need to copy m_type field for tyObject, as it could be empty for @@ -199,8 +199,8 @@ proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerproc.} = proc genericDeepCopyOpenArray(dest, src: pointer, len: int, mt: PNimType) {.compilerproc.} = var - d = cast[ByteAddress](dest) - s = cast[ByteAddress](src) + d = cast[int](dest) + s = cast[int](src) for i in 0..len-1: genericDeepCopy(cast[pointer](d +% i *% mt.base.size), cast[pointer](s +% i *% mt.base.size), mt.base) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index b36822aad..7db2b6567 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -170,11 +170,11 @@ proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -336,7 +336,7 @@ proc cellsetReset(s: var CellSet) = {.push stacktrace:off.} proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -356,7 +356,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -382,7 +382,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -457,7 +457,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) #gcAssert typ.kind in {tyString, tySequence} or size >= typ.base.size, "size too small" - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -507,7 +507,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} = var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) sysAssert(allocInv(gch.region), "newObjRC1 after rawAlloc") - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ setFrameInfo(res) @@ -549,9 +549,9 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len * elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res) +% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res) +% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") # This can be wrong for intermediate temps that are nevertheless on the # heap because of lambda lifting: #gcAssert(res.refcount shr rcShift <=% 1, "growObj: 4") @@ -683,7 +683,7 @@ proc collectCycles(gch: var GcHeap) {.raises: [].} = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") - var c = cast[ByteAddress](p) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) @@ -848,10 +848,10 @@ when withRealTime: stack.bottomSaved = stack.bottom when stackIncreases: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) else: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) GC_step(gch, us, strongAdvice) diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 0593b396e..ed046b5fd 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -133,11 +133,11 @@ template gcAssert(cond: bool, msg: string) = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -252,7 +252,7 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerproc.} = dest[] = src proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -264,7 +264,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -290,7 +290,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = of tyRef: # common case forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -330,7 +330,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -388,9 +388,9 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") when false: # this is wrong since seqs can be shared via 'shallow': when reallyDealloc: rawDealloc(gch.region, ol) @@ -593,7 +593,7 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: sysAssert(allocInv(gch.region), "gcMark begin") var cell = usrToCell(p) - var c = cast[ByteAddress](cell) + var c = cast[int](cell) if c >% PageSize: # fast check: does it look like a cell? var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell)) @@ -697,10 +697,10 @@ when withRealTime: stack.bottomSaved = stack.bottom when stackIncreases: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) - sizeof(pointer) * 6 - stackSize) + cast[int](stack.pos) - sizeof(pointer) * 6 - stackSize) else: stack.bottom = cast[pointer]( - cast[ByteAddress](stack.pos) + sizeof(pointer) * 6 + stackSize) + cast[int](stack.pos) + sizeof(pointer) * 6 + stackSize) GC_step(gch, us, strongAdvice) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index f5f4f164f..54c51e783 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -222,9 +222,9 @@ proc stackSize(stack: ptr GcStack): int {.noinline.} = if pos != nil: when stackIncreases: - result = cast[ByteAddress](pos) -% cast[ByteAddress](stack.bottom) + result = cast[int](pos) -% cast[int](stack.bottom) else: - result = cast[ByteAddress](stack.bottom) -% cast[ByteAddress](pos) + result = cast[int](stack.bottom) -% cast[int](pos) else: result = 0 @@ -295,8 +295,8 @@ when not defined(useNimRtl): # the first init must be the one that defines the stack bottom: gch.stack.bottom = theStackBottom elif theStackBottom != gch.stack.bottom: - var a = cast[ByteAddress](theStackBottom) # and not PageMask - PageSize*2 - var b = cast[ByteAddress](gch.stack.bottom) + var a = cast[int](theStackBottom) # and not PageMask - PageSize*2 + var b = cast[int](gch.stack.bottom) #c_fprintf(stdout, "old: %p new: %p;\n",gch.stack.bottom,theStackBottom) when stackIncreases: gch.stack.bottom = cast[pointer](min(a, b)) @@ -312,11 +312,11 @@ when not defined(useNimRtl): proc isOnStack(p: pointer): bool = var stackTop {.volatile, noinit.}: pointer stackTop = addr(stackTop) - var a = cast[ByteAddress](gch.getActiveStack().bottom) - var b = cast[ByteAddress](stackTop) + var a = cast[int](gch.getActiveStack().bottom) + var b = cast[int](stackTop) when not stackIncreases: swap(a, b) - var x = cast[ByteAddress](p) + var x = cast[int](p) result = a <=% x and x <=% b when defined(sparc): # For SPARC architecture. @@ -337,7 +337,7 @@ when defined(sparc): # For SPARC architecture. # Addresses decrease as the stack grows. while sp <= max: gcMark(gch, sp[]) - sp = cast[PPointer](cast[ByteAddress](sp) +% sizeof(pointer)) + sp = cast[PPointer](cast[int](sp) +% sizeof(pointer)) elif defined(ELATE): {.error: "stack marking code is to be written for this architecture".} @@ -354,8 +354,8 @@ elif stackIncreases: template forEachStackSlotAux(gch, gcMark: untyped) {.dirty.} = for stack in gch.stack.items(): - var max = cast[ByteAddress](gch.stack.bottom) - var sp = cast[ByteAddress](addr(registers)) -% sizeof(pointer) + var max = cast[int](gch.stack.bottom) + var sp = cast[int](addr(registers)) -% sizeof(pointer) while sp >=% max: gcMark(gch, cast[PPointer](sp)[]) sp = sp -% sizeof(pointer) @@ -383,8 +383,8 @@ else: gch.getActiveStack().setPosition(addr(registers)) if c_setjmp(registers) == 0'i32: # To fill the C stack with registers. for stack in gch.stack.items(): - var max = cast[ByteAddress](stack.bottom) - var sp = cast[ByteAddress](addr(registers)) + var max = cast[int](stack.bottom) + var sp = cast[int](addr(registers)) when defined(amd64): if stack.isActiveStack(): # words within the jmp_buf structure may not be properly aligned. diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index f91b37b94..72d22f08b 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -94,11 +94,11 @@ template gcAssert(cond: bool, msg: string) = proc cellToUsr(cell: PCell): pointer {.inline.} = # convert object (=pointer to refcount) to pointer to userdata - result = cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(Cell))) + result = cast[pointer](cast[int](cell)+%ByteAddress(sizeof(Cell))) proc usrToCell(usr: pointer): PCell {.inline.} = # convert pointer to userdata to object (=pointer to refcount) - result = cast[PCell](cast[ByteAddress](usr)-%ByteAddress(sizeof(Cell))) + result = cast[PCell](cast[int](usr)-%ByteAddress(sizeof(Cell))) proc extGetCellType(c: pointer): PNimType {.compilerproc.} = # used for code generation concerning debugging @@ -217,7 +217,7 @@ proc initGC() = gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID") proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) case n.kind of nkSlot: forAllChildrenAux(cast[pointer](d +% n.offset), n.typ, op) of nkList: @@ -229,7 +229,7 @@ proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = of nkNone: sysAssert(false, "forAllSlotsAux") proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) = - var d = cast[ByteAddress](dest) + var d = cast[int](dest) if dest == nil: return # nothing to do if ntfNoRefs notin mt.flags: case mt.kind @@ -255,7 +255,7 @@ proc forAllChildren(cell: PCell, op: WalkOp) = forAllChildrenAux(cellToUsr(cell), cell.typ.base, op) of tySequence: when not defined(nimSeqsV2): - var d = cast[ByteAddress](cellToUsr(cell)) + var d = cast[int](cellToUsr(cell)) var s = cast[PGenericSeq](d) if s != nil: for i in 0..s.len-1: @@ -268,7 +268,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = gcAssert(typ.kind in {tyRef, tyString, tySequence}, "newObj: 1") collectCT(gch, size + sizeof(Cell)) var res = cast[PCell](rawAlloc(gch.region, size + sizeof(Cell))) - gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2") + gcAssert((cast[int](res) and (MemAlign-1)) == 0, "newObj: 2") # now it is buffered in the ZCT res.typ = typ when leakDetector and not hasThreadSupport: @@ -336,9 +336,9 @@ when not defined(nimSeqsV2): var oldsize = align(GenericSeqSize, elemAlign) + cast[PGenericSeq](old).len*elemSize copyMem(res, ol, oldsize + sizeof(Cell)) - zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + zeroMem(cast[pointer](cast[int](res)+% oldsize +% sizeof(Cell)), newsize-oldsize) - sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + sysAssert((cast[int](res) and (MemAlign-1)) == 0, "growObj: 3") when withBitvectors: incl(gch.allocated, res) when useCellIds: inc gch.idGenerator @@ -446,7 +446,7 @@ proc markGlobals(gch: var GcHeap) = proc gcMark(gch: var GcHeap, p: pointer) {.inline.} = # the addresses are not as cells on the stack, so turn them to cells: - var c = cast[ByteAddress](p) + var c = cast[int](p) if c >% PageSize: # fast check: does it look like a cell? var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p)) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 4817059be..925e98ed3 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -80,12 +80,12 @@ elif defined(emscripten) and not defined(StandaloneHeapSize): let pos = cast[int](result) # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + var new_pos = cast[int](pos) +% (PageSize - (pos %% PageSize)) if (new_pos-pos) < sizeof(EmscriptenMMapBlock): new_pos = new_pos +% PageSize result = cast[pointer](new_pos) - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](result) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) mmapDescr.realSize = realSize @@ -96,7 +96,7 @@ elif defined(emscripten) and not defined(StandaloneHeapSize): proc osTryAllocPages(size: int): pointer = osAllocPages(size) proc osDeallocPages(p: pointer, size: int) {.inline.} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescrPos = cast[int](p) -% sizeof(EmscriptenMMapBlock) var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) munmap(mmapDescr.realPointer, mmapDescr.realSize) diff --git a/lib/system/repr.nim b/lib/system/repr.nim index e47220656..6b6f7e340 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -155,7 +155,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..typ.size div bs - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), typ.base, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), typ.base, cl) add result, "]" when defined(nimSeqsV2): @@ -183,7 +183,7 @@ when not defined(useNimRtl): var bs = typ.base.size for i in 0..cast[PGenericSeq](p).len-1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), + reprAux(result, cast[pointer](cast[int](payloadPtr(p)) + align(payloadOffset, typ.align) + i*bs), typ.base, cl) add result, "]" @@ -194,14 +194,14 @@ when not defined(useNimRtl): of nkSlot: add result, $n.name add result, " = " - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) of nkList: for i in 0..n.len-1: if i > 0: add result, ",\n" reprRecordAux(result, p, n.sons[i], cl) of nkCase: var m = selectBranch(p, n) - reprAux(result, cast[pointer](cast[ByteAddress](p) + n.offset), n.typ, cl) + reprAux(result, cast[pointer](cast[int](p) + n.offset), n.typ, cl) if m != nil: reprRecordAux(result, p, m, cl) proc reprRecord(result: var string, p: pointer, typ: PNimType, @@ -307,7 +307,7 @@ when not defined(useNimRtl): var bs = elemtyp.size for i in 0..length - 1: if i > 0: add result, ", " - reprAux(result, cast[pointer](cast[ByteAddress](p) + i*bs), elemtyp, cl) + reprAux(result, cast[pointer](cast[int](p) + i*bs), elemtyp, cl) add result, "]" deinitReprClosure(cl) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index be32652d8..b8dc7101d 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -17,10 +17,10 @@ proc dataPointer(a: PGenericSeq, elemAlign: int): pointer = - cast[pointer](cast[ByteAddress](a) +% align(GenericSeqSize, elemAlign)) + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign)) proc dataPointer(a: PGenericSeq, elemAlign, elemSize, index: int): pointer = - cast[pointer](cast[ByteAddress](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) + cast[pointer](cast[int](a) +% align(GenericSeqSize, elemAlign) +% (index*%elemSize)) proc resize(old: int): int {.inline.} = if old <= 0: result = 4 diff --git a/tests/parallel/tguard1.nim b/tests/parallel/tguard1.nim index b1eb7e7c5..f4c92319b 100644 --- a/tests/parallel/tguard1.nim +++ b/tests/parallel/tguard1.nim @@ -5,7 +5,7 @@ output: "90" when false: template lock(a, b: ptr Lock; body: stmt) = - if cast[ByteAddress](a) < cast[ByteAddress](b): + if cast[int](a) < cast[int](b): pthread_mutex_lock(a) pthread_mutex_lock(b) else: From 84200f28a114ef9744ecf2b6060ef140e350139b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 16:42:31 +0800 Subject: [PATCH 1470/1654] bump csource_v2 hash (#21118) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 3ae618f00..0ea963c39 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v2 nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=ffd834c20915dc04c78eb0d3b2162832e7ec9c13 +nim_csourcesHash=32cf8431bb19f8803352a0ab3dbee5fec35d5ec6 From 296e7f598e58430415369a34f83552915f57f3f5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 16 Dec 2022 19:49:26 +0800 Subject: [PATCH 1471/1654] Bump the devel version to 1.9.1 (#21117) --- lib/system/compilation.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim index 6109e9874..234e8cafe 100644 --- a/lib/system/compilation.nim +++ b/lib/system/compilation.nim @@ -6,11 +6,11 @@ const ## ``` # see also std/private/since - NimMinor* {.intdefine.}: int = 7 + NimMinor* {.intdefine.}: int = 9 ## is the minor number of Nim's version. ## Odd for devel, even for releases. - NimPatch* {.intdefine.}: int = 3 + NimPatch* {.intdefine.}: int = 1 ## is the patch number of Nim's version. ## Odd for devel, even for releases. From 7fa983f5e6b5f52a20e477e2443c8fdede244290 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 00:21:07 +0800 Subject: [PATCH 1472/1654] Nim now ships nimble with 0.14.1 version (#21120) * Nimble now uses latest patch * Update koch.nim --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 84d0911ac..6b3d327f5 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "0777f33d1ddbd505b3aa7b714032125349323ceb" # master + NimbleStableCommit = "afd03bc000f4013c48f52381728327dabfccfc4c" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From d83f66a24d990f2e4a135a4a64e1c464702b953d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:10:56 +0800 Subject: [PATCH 1473/1654] fixes #21116; always mangles the param (#21121) * fixes #21116; always mangles the param * idOrSig * fixes tests * Apply suggestions from code review --- compiler/ccgtypes.nim | 6 +----- tests/ccgbugs/t21116.nim | 10 ++++++++++ tests/ccgbugs/tnoalias.nim | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 tests/ccgbugs/t21116.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index d7b56de5b..3c9f9d923 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -48,11 +48,9 @@ proc fillBackendName(m: BModule; s: PSym) = writeMangledName(m.ndi, s, m.config) proc fillParamName(m: BModule; s: PSym) = - ## we cannot use 'sigConflicts' here since we have a BModule, not a BProc. - ## Fortunately C's scoping rules are sane enough so that that doesn't - ## cause any trouble. if s.loc.r == "": var res = s.name.s.mangle + res.add idOrSig(s, res, m.sigConflicts) # Take into account if HCR is on because of the following scenario: # if a module gets imported and it has some more importc symbols in it, # some param names might receive the "_0" suffix to distinguish from what @@ -69,8 +67,6 @@ proc fillParamName(m: BModule; s: PSym) = # and a function called in main or proxy uses `socket` as a parameter name. # That would lead to either needing to reload `proxy` or to overwrite the # executable file for the main module, which is running (or both!) -> error. - if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res): - res.add "_0" s.loc.r = res.rope writeMangledName(m.ndi, s, m.config) diff --git a/tests/ccgbugs/t21116.nim b/tests/ccgbugs/t21116.nim new file mode 100644 index 000000000..6418df539 --- /dev/null +++ b/tests/ccgbugs/t21116.nim @@ -0,0 +1,10 @@ +discard """ + target: "c cpp" + disabled: windows +""" +# bug #21116 +import std/os + +proc p(glob: string) = + for _ in walkFiles(glob): discard +p("dir/*") diff --git a/tests/ccgbugs/tnoalias.nim b/tests/ccgbugs/tnoalias.nim index f200992d7..96c3d390b 100644 --- a/tests/ccgbugs/tnoalias.nim +++ b/tests/ccgbugs/tnoalias.nim @@ -1,5 +1,5 @@ discard """ - ccodecheck: "\\i@'NI* NIM_NOALIAS field;' @'NIM_CHAR* NIM_NOALIAS x,' @'void* NIM_NOALIAS q'" + ccodecheck: "\\i@'NI* NIM_NOALIAS field;' @'NIM_CHAR* NIM_NOALIAS x__0qEngDE9aYoYsF8tWnyPacw,' @'void* NIM_NOALIAS q'" """ type From ae4645e8dfec5c7b5f2d8cbdb672b7a0d55dfb37 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 16:28:59 +0800 Subject: [PATCH 1474/1654] add 2.0 changelog (#21107) [backport: 2.0] add v2.0 changelog --- changelog.md | 296 +------------------------------ changelogs/changelog_2_0_0.md | 321 ++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 293 deletions(-) create mode 100644 changelogs/changelog_2_0_0.md diff --git a/changelog.md b/changelog.md index 00bdb50c2..0389faad9 100644 --- a/changelog.md +++ b/changelog.md @@ -1,321 +1,31 @@ -# v2.0.0 - yyyy-mm-dd +# v2.2.0 - yyyy-mm-dd ## Changes affecting backward compatibility -- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. -- `addr` is now available for all addressable locations, - `unsafeAddr` is now deprecated and an alias for `addr`. - -- Certain definitions from the default `system` module have been moved to - the following new modules: - - - `std/syncio` - - `std/assertions` - - `std/formatfloat` - - `std/objectdollar` - - `std/widestrs` - - `std/typedthreads` - - `std/sysatomics` - - In the future, these definitions will be removed from the `system` module, - and their respective modules will have to be imported to use them. - Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option - may be used. - -- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated - symbols in the `system` module: - - Aliases with `Error` suffix to exception types that have a `Defect` suffix - (see [exceptions](https://nim-lang.org/docs/exceptions.html)): - `ArithmeticError`, `DivByZeroError`, `OverflowError`, - `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, - `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, - `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`, - `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`, - `DeadThreadError`, `NilAccessError` - - `addQuitProc`, replaced by `exitprocs.addExitProc` - - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32` - - `TaintedString`, formerly a distinct alias to `string` - - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to - `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64` - -- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in - in the `system` module. - -- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore. - -- The `gc:v2` option is removed. - -- The `mainmodule` and `m` options are removed. - -- The `threads:on` option is now the default. - -- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via - `experimental:flexibleOptionalParams`. - -- Automatic dereferencing (experimental feature) is removed. - -- The `Math.trunc` polyfill for targeting Internet Explorer was - previously included in most JavaScript output files. - Now, it is only included with `-d:nimJsMathTruncPolyfill`. - If you are targeting Internet Explorer, you may choose to enable this option - or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). - Nim uses `Math.trunc` for the division and modulo operators for integers. - -- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and -`sink` for optimization purposes. - -- The `nimPreviewDotLikeOps` define is going to be removed or deprecated. - -- The `{.this.}` pragma, deprecated since 0.19, has been removed. -- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. - ```nim - type Foo = distinct ptr int - - # Before: - var x: Foo = nil - # After: - var x: Foo = Foo(nil) - ``` -- Removed two type pragma syntaxes deprecated since 0.20, namely - `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`. - -- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent - with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`. - This decision was made with the assumption that the old syntax was used rarely; - if your code used the old syntax, please be aware of this change. - -- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators - are no longer experimental. - -- Removed the `nimIncrSeqV3` define. - -- `macros.getImpl` for `const` symbols now returns the full definition node - (as `nnkConstDef`) rather than the AST of the constant value. - -- Lock levels are deprecated, now a noop. - -- ORC is now the default memory management strategy. Use - `--mm:refc` for a transition period. - -- `strictEffects` are no longer experimental. - Use `legacy:laxEffects` to keep backward compatibility. - -- The `gorge`/`staticExec` calls will now return a descriptive message in the output - if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`. - -- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning. - This warning will become an error in future versions! Use a `cast` operation - like `cast[cstring](x)` instead. - -- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. - -- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details. - -- Redefining templates with the same signature was previously - allowed to support certain macro code. To do this explicitly, the - `{.redefine.}` pragma has been added. Note that this is only for templates. - Implicit redefinition of templates is now deprecated and will give an error in the future. - -- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. - -- Several Standard libraries are moved to nimble packages, use `nimble` to install them: - - `std/punycode` => `punycode` - - `std/asyncftpclient` => `asyncftpclient` - - `std/smtp` => `smtp` - - `std/db_common` => `db_connector/db_common` - - `std/db_sqlite` => `db_connector/db_sqlite` - - `std/db_mysql` => `db_connector/db_mysql` - - `std/db_postgres` => `db_connector/db_postgres` - - `std/db_odbc` => `db_connector/db_odbc` - -- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of - `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. - This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. - -- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. ## Standard library additions and changes [//]: # "Changes:" -- OpenSSL 3 is now supported. -- `macros.parseExpr` and `macros.parseStmt` now accept an optional - filename argument for more informative errors. -- Module `colors` expanded with missing colors from the CSS color standard. - `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard. -- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). -- The `md5` module now works at compile time and in JavaScript. -- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables. -- `strutils.find` now uses and defaults to `last = -1` for whole string searches, - making limiting it to just the first char (`last = 0`) valid. -- `random.rand` now works with `Ordinal`s. -- Undeprecated `os.isvalidfilename`. -- `std/oids` now uses `int64` to store time internally (before it was int32). -- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`. -- `std/uri.Uri.isIpv6` is now exported. -- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2. - - -- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`. -- `std/jsfetch.newFetchOptions` now has default values for all parameters -- `std/jsformdata` now accepts `Blob` data type. [//]: # "Additions:" -- Added ISO 8601 week date utilities in `times`: - - Added `IsoWeekRange`, a range type for weeks in a week-based year. - - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year. - - Added a `initDateTime` overload to create a datetime from an ISO week date. - - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime. - - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year. -- Added new modules which were part of `std/os`: - - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. - - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`. - - Added `std/cmdline` for reading command line parameters. -- Added `sep` parameter in `std/uri` to specify the query separator. -- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) - and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) - in `jscore` for JavaScript targets. -- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`. -- Added `complex.sgn` for obtaining the phase of complex numbers. -- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`, - `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`, - `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`, - `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`, - `toggleAttribute`, and `matches` to `std/dom`. -- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices) -- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460 -- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse. -- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse. -- Added `safe` parameter to `base64.encodeMime`. + [//]: # "Deprecations:" -- Deprecated `selfExe` for Nimscript. -- Deprecated `std/sums`. -- Deprecated `std/base64.encode` for collections of arbitrary integer element type. - Now only `byte` and `char` are supported. + [//]: # "Removals:" -- Removed deprecated module `parseopt2`. -- Removed deprecated module `sharedstrings`. -- Removed deprecated module `dom_extensions`. -- Removed deprecated module `LockFreeHash`. -- Removed deprecated module `events`. -- Removed deprecated `oids.oidToString`. -- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. -- Removed deprecated `jsre.test` and `jsre.toString`. -- Removed deprecated `math.c_frexp`. -- Removed deprecated `` httpcore.`==` ``. -- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. -- Removed deprecated `osproc.poDemon`, symbol with typo. -- Removed deprecated `tables.rightSize`. - - -- Removed deprecated `posix.CLONE_STOPPED`. ## Language changes -- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma - which can be used to disable certain effects in proc types. -- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental, - meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. -- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed - for the right-hand side of type definitions in type sections. Previously - they would error with "invalid indentation". - -- Compile-time define changes: - - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. - In the command line, this is defined as `-d:a.b.c`. Older versions can - use accents as in ``defined(`a.b.c`)`` to access such defines. - - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) - now support a string argument for qualified define names. - - ```nim - # -d:package.FooBar=42 - const FooBar {.intdefine: "package.FooBar".}: int = 5 - echo FooBar # 42 - ``` - - This was added to help disambiguate similar define names for different packages. - In older versions, this could only be achieved with something like the following: - - ```nim - const FooBar = block: - const `package.FooBar` {.intdefine.}: int = 5 - `package.FooBar` - ``` - - A generic `define` pragma for constants has been added that interprets - the value of the define based on the type of the constant value. - See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma) - for a list of supported types. - -- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: - - Templates now accept macro pragmas. - - Macro pragmas for var/let/const sections have been redesigned in a way that works - similarly to routine macro pragmas. The new behavior is documented in the - [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). - - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, - allowing multiple type definitions to be injected in place of the original type definition. - - ```nim - import macros - macro multiply(amount: static int, s: untyped): untyped = - let name = $s[0].basename - result = newNimNode(nnkTypeSection) - for i in 1 .. amount: - result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) - type - Foo = object - Bar {.multiply: 3.} = object - x, y, z: int - Baz = object - # becomes - type - Foo = object - Bar1 = object - x, y, z: int - Bar2 = object - x, y, z: int - Bar3 = object - x, y, z: int - Baz = object - ``` - -- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) - has been implemented for a variety of basic cases. For example, code like the following now compiles: - - ```nim - let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")] - ``` - -- `cstring` is now accepted as a selector in `case` statements, removing the - need to convert to `string`. On the JS backend, this is translated directly - to a `switch` statement. - -- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). -- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. ## Compiler changes -- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the - reality better. (Nim moved away from all techniques based on "tracing".) -- Defines the `gcRefc` symbol which allows writing specific code for the refc GC. - -- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, - without requiring `-d:nimVersion140` which is now a noop. - -- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package. - -- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and - related functions produced on the backend. This prevents conflicts with other Nim - static libraries. - -- When compiling for Release the flag `-fno-math-errno` is used for GCC. ## Tool changes -- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`). diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md new file mode 100644 index 000000000..00bdb50c2 --- /dev/null +++ b/changelogs/changelog_2_0_0.md @@ -0,0 +1,321 @@ +# v2.0.0 - yyyy-mm-dd + + +## Changes affecting backward compatibility +- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. + +- `addr` is now available for all addressable locations, + `unsafeAddr` is now deprecated and an alias for `addr`. + +- Certain definitions from the default `system` module have been moved to + the following new modules: + + - `std/syncio` + - `std/assertions` + - `std/formatfloat` + - `std/objectdollar` + - `std/widestrs` + - `std/typedthreads` + - `std/sysatomics` + + In the future, these definitions will be removed from the `system` module, + and their respective modules will have to be imported to use them. + Currently, to make these imports required, the `-d:nimPreviewSlimSystem` option + may be used. + +- Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated + symbols in the `system` module: + - Aliases with `Error` suffix to exception types that have a `Defect` suffix + (see [exceptions](https://nim-lang.org/docs/exceptions.html)): + `ArithmeticError`, `DivByZeroError`, `OverflowError`, + `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, + `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, + `ObjectAssignmentError`, `ObjectConversionError`, `FloatingPointError`, + `FloatOverflowError`, `FloatUnderflowError`, `FloatInexactError`, + `DeadThreadError`, `NilAccessError` + - `addQuitProc`, replaced by `exitprocs.addExitProc` + - Legacy unsigned conversion operations: `ze`, `ze64`, `toU8`, `toU16`, `toU32` + - `TaintedString`, formerly a distinct alias to `string` + - `PInt32`, `PInt64`, `PFloat32`, `PFloat64`, aliases to + `ptr int32`, `ptr int64`, `ptr float32`, `ptr float64` + +- Enabling `-d:nimPreviewSlimSystem` removes the import of `channels_builtin` in + in the `system` module. + +- Enabling `-d:nimPreviewCstringConversion`, `ptr char`, `ptr array[N, char]` and `ptr UncheckedArray[N, char]` don't support conversion to cstring anymore. + +- The `gc:v2` option is removed. + +- The `mainmodule` and `m` options are removed. + +- The `threads:on` option is now the default. + +- Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via + `experimental:flexibleOptionalParams`. + +- Automatic dereferencing (experimental feature) is removed. + +- The `Math.trunc` polyfill for targeting Internet Explorer was + previously included in most JavaScript output files. + Now, it is only included with `-d:nimJsMathTruncPolyfill`. + If you are targeting Internet Explorer, you may choose to enable this option + or define your own `Math.trunc` polyfill using the [`emit` pragma](https://nim-lang.org/docs/manual.html#implementation-specific-pragmas-emit-pragma). + Nim uses `Math.trunc` for the division and modulo operators for integers. + +- `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and +`sink` for optimization purposes. + +- The `nimPreviewDotLikeOps` define is going to be removed or deprecated. + +- The `{.this.}` pragma, deprecated since 0.19, has been removed. +- `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. + ```nim + type Foo = distinct ptr int + + # Before: + var x: Foo = nil + # After: + var x: Foo = Foo(nil) + ``` +- Removed two type pragma syntaxes deprecated since 0.20, namely + `type Foo = object {.final.}`, and `type Foo {.final.} [T] = object`. + +- `foo a = b` now means `foo(a = b)` rather than `foo(a) = b`. This is consistent + with the existing behavior of `foo a, b = c` meaning `foo(a, b = c)`. + This decision was made with the assumption that the old syntax was used rarely; + if your code used the old syntax, please be aware of this change. + +- [Overloadable enums](https://nim-lang.github.io/Nim/manual.html#overloadable-enum-value-names) and Unicode Operators + are no longer experimental. + +- Removed the `nimIncrSeqV3` define. + +- `macros.getImpl` for `const` symbols now returns the full definition node + (as `nnkConstDef`) rather than the AST of the constant value. + +- Lock levels are deprecated, now a noop. + +- ORC is now the default memory management strategy. Use + `--mm:refc` for a transition period. + +- `strictEffects` are no longer experimental. + Use `legacy:laxEffects` to keep backward compatibility. + +- The `gorge`/`staticExec` calls will now return a descriptive message in the output + if the execution fails for whatever reason. To get back legacy behaviour use `-d:nimLegacyGorgeErrors`. + +- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning. + This warning will become an error in future versions! Use a `cast` operation + like `cast[cstring](x)` instead. + +- `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. + +- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details. + +- Redefining templates with the same signature was previously + allowed to support certain macro code. To do this explicitly, the + `{.redefine.}` pragma has been added. Note that this is only for templates. + Implicit redefinition of templates is now deprecated and will give an error in the future. + +- Using an unnamed break in a block is deprecated. This warning will become an error in future versions! Use a named block with a named break instead. + +- Several Standard libraries are moved to nimble packages, use `nimble` to install them: + - `std/punycode` => `punycode` + - `std/asyncftpclient` => `asyncftpclient` + - `std/smtp` => `smtp` + - `std/db_common` => `db_connector/db_common` + - `std/db_sqlite` => `db_connector/db_sqlite` + - `std/db_mysql` => `db_connector/db_mysql` + - `std/db_postgres` => `db_connector/db_postgres` + - `std/db_odbc` => `db_connector/db_odbc` + +- Previously, calls like `foo(a, b): ...` or `foo(a, b) do: ...` where the final argument of + `foo` had type `proc ()` were assumed by the compiler to mean `foo(a, b, proc () = ...)`. + This behavior is now deprecated. Use `foo(a, b) do (): ...` or `foo(a, b, proc () = ...)` instead. + +- If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. + +## Standard library additions and changes + +[//]: # "Changes:" +- OpenSSL 3 is now supported. +- `macros.parseExpr` and `macros.parseStmt` now accept an optional + filename argument for more informative errors. +- Module `colors` expanded with missing colors from the CSS color standard. + `colPaleVioletRed` and `colMediumPurple` have also been changed to match the CSS color standard. +- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)). +- The `md5` module now works at compile time and in JavaScript. +- Changed `mimedb` to use an `OrderedTable` instead of `OrderedTableRef` to support `const` tables. +- `strutils.find` now uses and defaults to `last = -1` for whole string searches, + making limiting it to just the first char (`last = 0`) valid. +- `random.rand` now works with `Ordinal`s. +- Undeprecated `os.isvalidfilename`. +- `std/oids` now uses `int64` to store time internally (before it was int32). +- `std/uri.Uri` dollar `$` improved, precalculates the `string` result length from the `Uri`. +- `std/uri.Uri.isIpv6` is now exported. +- `std/logging.ConsoleLogger` and `FileLogger` now have a `flushThreshold` attribute to set what log message levels are automatically flushed. For Nim v1 use `-d:nimFlushAllLogs` to automatically flush all message levels. Flushing all logs is the default behavior for Nim v2. + + +- `std/net.IpAddress` dollar `$` improved, uses a fixed capacity for the `string` result based from the `IpAddressFamily`. +- `std/jsfetch.newFetchOptions` now has default values for all parameters +- `std/jsformdata` now accepts `Blob` data type. + + +[//]: # "Additions:" +- Added ISO 8601 week date utilities in `times`: + - Added `IsoWeekRange`, a range type for weeks in a week-based year. + - Added `IsoYear`, a distinct type for a week-based year in contrast to a regular year. + - Added a `initDateTime` overload to create a datetime from an ISO week date. + - Added `getIsoWeekAndYear` to get an ISO week number and week-based year from a datetime. + - Added `getIsoWeeksInYear` to return the number of weeks in a week-based year. +- Added new modules which were part of `std/os`: + - Added `std/oserrors` for OS error reporting. Added `std/envvars` for environment variables handling. + - Added `std/paths`, `std/dirs`, `std/files`, `std/symlinks` and `std/appdirs`. + - Added `std/cmdline` for reading command line parameters. +- Added `sep` parameter in `std/uri` to specify the query separator. +- Added bindings to [`Array.shift`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift) + and [`queueMicrotask`](https://developer.mozilla.org/en-US/docs/Web/API/queueMicrotask) + in `jscore` for JavaScript targets. +- Added `UppercaseLetters`, `LowercaseLetters`, `PunctuationChars`, `PrintableChars` sets to `std/strutils`. +- Added `complex.sgn` for obtaining the phase of complex numbers. +- Added `insertAdjacentText`, `insertAdjacentElement`, `insertAdjacentHTML`, + `after`, `before`, `closest`, `append`, `hasAttributeNS`, `removeAttributeNS`, + `hasPointerCapture`, `releasePointerCapture`, `requestPointerLock`, + `replaceChildren`, `replaceWith`, `scrollIntoViewIfNeeded`, `setHTML`, + `toggleAttribute`, and `matches` to `std/dom`. +- Added [`jsre.hasIndices`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/hasIndices) +- Added `capacity` for `string` and `seq` to return the current capacity, see https://github.com/nim-lang/RFCs/issues/460 +- Added `openArray[char]` overloads for `std/parseutils` allowing more code reuse. +- Added `openArray[char]` overloads for `std/unicode` allowing more code reuse. +- Added `safe` parameter to `base64.encodeMime`. + +[//]: # "Deprecations:" +- Deprecated `selfExe` for Nimscript. +- Deprecated `std/sums`. +- Deprecated `std/base64.encode` for collections of arbitrary integer element type. + Now only `byte` and `char` are supported. + +[//]: # "Removals:" +- Removed deprecated module `parseopt2`. +- Removed deprecated module `sharedstrings`. +- Removed deprecated module `dom_extensions`. +- Removed deprecated module `LockFreeHash`. +- Removed deprecated module `events`. +- Removed deprecated `oids.oidToString`. +- Removed define `nimExperimentalAsyncjsThen` for `std/asyncjs.then` and `std/jsfetch`. +- Removed deprecated `jsre.test` and `jsre.toString`. +- Removed deprecated `math.c_frexp`. +- Removed deprecated `` httpcore.`==` ``. +- Removed deprecated `std/posix.CMSG_SPACE` and `std/posix.CMSG_LEN` that takes wrong argument types. +- Removed deprecated `osproc.poDemon`, symbol with typo. +- Removed deprecated `tables.rightSize`. + + +- Removed deprecated `posix.CLONE_STOPPED`. + + +## Language changes + +- [Tag tracking](https://nim-lang.github.io/Nim/manual.html#effect-system-tag-tracking) supports the definition of forbidden tags by the `.forbids` pragma + which can be used to disable certain effects in proc types. +- [Case statement macros](https://nim-lang.github.io/Nim/manual.html#macros-case-statement-macros) are no longer experimental, + meaning you no longer need to enable the experimental switch `caseStmtMacros` to use them. +- Full command syntax and block arguments i.e. `foo a, b: c` are now allowed + for the right-hand side of type definitions in type sections. Previously + they would error with "invalid indentation". + +- Compile-time define changes: + - `defined` now accepts identifiers separated by dots, i.e. `defined(a.b.c)`. + In the command line, this is defined as `-d:a.b.c`. Older versions can + use accents as in ``defined(`a.b.c`)`` to access such defines. + - [Define pragmas for constants](https://nim-lang.github.io/Nim/manual.html#implementation-specific-pragmas-compileminustime-define-pragmas) + now support a string argument for qualified define names. + + ```nim + # -d:package.FooBar=42 + const FooBar {.intdefine: "package.FooBar".}: int = 5 + echo FooBar # 42 + ``` + + This was added to help disambiguate similar define names for different packages. + In older versions, this could only be achieved with something like the following: + + ```nim + const FooBar = block: + const `package.FooBar` {.intdefine.}: int = 5 + `package.FooBar` + ``` + - A generic `define` pragma for constants has been added that interprets + the value of the define based on the type of the constant value. + See the [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#generic-define-pragma) + for a list of supported types. + +- [Macro pragmas](https://nim-lang.github.io/Nim/manual.html#userminusdefined-pragmas-macro-pragmas) changes: + - Templates now accept macro pragmas. + - Macro pragmas for var/let/const sections have been redesigned in a way that works + similarly to routine macro pragmas. The new behavior is documented in the + [experimental manual](https://nim-lang.github.io/Nim/manual_experimental.html#extended-macro-pragmas). + - Pragma macros on type definitions can now return `nnkTypeSection` nodes as well as `nnkTypeDef`, + allowing multiple type definitions to be injected in place of the original type definition. + + ```nim + import macros + macro multiply(amount: static int, s: untyped): untyped = + let name = $s[0].basename + result = newNimNode(nnkTypeSection) + for i in 1 .. amount: + result.add(newTree(nnkTypeDef, ident(name & $i), s[1], s[2])) + type + Foo = object + Bar {.multiply: 3.} = object + x, y, z: int + Baz = object + # becomes + type + Foo = object + Bar1 = object + x, y, z: int + Bar2 = object + x, y, z: int + Bar3 = object + x, y, z: int + Baz = object + ``` + +- A new form of type inference called [top-down inference](https://nim-lang.github.io/Nim/manual_experimental.html#topminusdown-type-inference) + has been implemented for a variety of basic cases. For example, code like the following now compiles: + + ```nim + let foo: seq[(float, byte, cstring)] = @[(1, 2, "abc")] + ``` + +- `cstring` is now accepted as a selector in `case` statements, removing the + need to convert to `string`. On the JS backend, this is translated directly + to a `switch` statement. + +- Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). +- Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. + + +## Compiler changes + +- The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the + reality better. (Nim moved away from all techniques based on "tracing".) + +- Defines the `gcRefc` symbol which allows writing specific code for the refc GC. + +- `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`, + without requiring `-d:nimVersion140` which is now a noop. + +- `--styleCheck`, `--hintAsError` and `--warningAsError` now only apply to the current package. + +- The switch `--nimMainPrefix:prefix` has been added to add a prefix to the names of `NimMain` and + related functions produced on the backend. This prevents conflicts with other Nim + static libraries. + +- When compiling for Release the flag `-fno-math-errno` is used for GCC. + + +## Tool changes + +- Nim now ships Nimble version 0.14 which added support for lock-files. Libraries are stored in `$nimbleDir/pkgs2` (it was `$nimbleDir/pkgs`). From b8d200683a77487e7ef5d78abd4c1b981709bddd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 20:44:30 +0800 Subject: [PATCH 1475/1654] remove unused imports (#21126) --- lib/impure/rdstdin.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 935c83671..b0c648373 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -22,8 +22,6 @@ runnableExamples("-r:off"): if line.len > 0: echo line echo "exiting" -when defined(nimPreviewSlimSystem): - import std/syncio when defined(windows): when defined(nimPreviewSlimSystem): From 100b304ac2aaa65222d7a9043acd0c534ed484cb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 17 Dec 2022 21:42:29 +0800 Subject: [PATCH 1476/1654] fixes misplaced changelog entries; pre-existing issue (#21125) * move misplaced changelog entries to the right place * fixes misplaced changelog --- changelogs/changelog.md | 33 --------------------------------- changelogs/changelog_2_0_0.md | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 33 deletions(-) delete mode 100644 changelogs/changelog.md diff --git a/changelogs/changelog.md b/changelogs/changelog.md deleted file mode 100644 index d793c4313..000000000 --- a/changelogs/changelog.md +++ /dev/null @@ -1,33 +0,0 @@ -# v1.xx.x - yyyy-mm-dd - -## Changes affecting backward compatibility - -## Standard library additions and changes - -- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433). - -### New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall - -This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. - -On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. - -When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: -```sh -$ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details -$ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 -$ sh build.sh # per unix install instructions -$ bin/nim c koch # per unix install instructions -$ ./koch boot -d:release # per unix install instructions -$ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall -``` - -This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular). - -## Language changes - - -## Compiler changes - - -## Tool changes diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 00bdb50c2..f34336ff7 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -160,6 +160,25 @@ - `std/jsfetch.newFetchOptions` now has default values for all parameters - `std/jsformdata` now accepts `Blob` data type. +- `std/sharedlist` and `std/sharedtables` are now deprecated, see RFC [#433](https://github.com/nim-lang/RFCs/issues/433). + +- New compile flag (`-d:nimNoGetRandom`) when building `std/sysrand` to remove dependency on linux `getrandom` syscall. + + This compile flag only affects linux builds and is necessary if either compiling on a linux kernel version < 3.17, or if code built will be executing on kernel < 3.17. + + On linux kernels < 3.17 (such as kernel 3.10 in RHEL7 and CentOS7), the `getrandom` syscall was not yet introduced. Without this, the `std/sysrand` module will not build properly, and if code is built on a kernel >= 3.17 without the flag, any usage of the `std/sysrand` module will fail to execute on a kernel < 3.17 (since it attempts to perform a syscall to `getrandom`, which isn't present in the current kernel). A compile flag has been added to force the `std/sysrand` module to use /dev/urandom (available since linux kernel 1.3.30), rather than the `getrandom` syscall. This allows for use of a cryptographically secure PRNG, regardless of kernel support for the `getrandom` syscall. + + When building for RHEL7/CentOS7 for example, the entire build process for nim from a source package would then be: + ```sh + $ yum install devtoolset-8 # Install GCC version 8 vs the standard 4.8.5 on RHEL7/CentOS7. Alternatively use -d:nimEmulateOverflowChecks. See issue #13692 for details + $ scl enable devtoolset-8 bash # Run bash shell with default toolchain of gcc 8 + $ sh build.sh # per unix install instructions + $ bin/nim c koch # per unix install instructions + $ ./koch boot -d:release # per unix install instructions + $ ./koch tools -d:nimNoGetRandom # pass the nimNoGetRandom flag to compile std/sysrand without support for getrandom syscall + ``` + + This is necessary to pass when building nim on kernel versions < 3.17 in particular to avoid an error of "SYS_getrandom undeclared" during the build process for stdlib (sysrand in particular). [//]: # "Additions:" - Added ISO 8601 week date utilities in `times`: From 53eed2be4515a3ae853ae4d2fbd84daa49c1d6d6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 18 Dec 2022 22:31:13 +0800 Subject: [PATCH 1477/1654] close #11705; add a testcase (#21128) --- tests/template/t11705.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/template/t11705.nim diff --git a/tests/template/t11705.nim b/tests/template/t11705.nim new file mode 100644 index 000000000..65ddc7e6c --- /dev/null +++ b/tests/template/t11705.nim @@ -0,0 +1,17 @@ +type RefObj = ref object + +proc `[]`(val: static[int]) = # works with different name/overload or without static arg + discard + +template noRef*(T: typedesc): typedesc = # works without template indirection + typeof(default(T)[]) + +proc `=destroy`(x: var noRef(RefObj)) = + discard + +proc foo = + var x = new RefObj + doAssert $(x[]) == "()" + +# bug #11705 +foo() From 70c575095e89497c63af5d8b2c3ed7dca802179f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 19 Dec 2022 17:56:49 +0800 Subject: [PATCH 1478/1654] add changelog for the new strict function checking algorithm (#21129) add changelog for the new strict function checking --- changelogs/changelog_2_0_0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index f34336ff7..a899bb929 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -135,6 +135,8 @@ - If no exception or any exception deriving from Exception but not Defect or CatchableError given in except, a `warnBareExcept` warning will be triggered. +- The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection. + ## Standard library additions and changes [//]: # "Changes:" From e278a781fc5bfc115326ed6c1873268d51b25303 Mon Sep 17 00:00:00 2001 From: Bung Date: Mon, 19 Dec 2022 19:35:15 +0800 Subject: [PATCH 1479/1654] fix #21109 (#21127) --- compiler/semexprs.nim | 2 ++ tests/misc/t21109.nim | 13 +++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/misc/t21109.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 8ac8f4750..612af482a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1947,6 +1947,8 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = tupleConstr[i] = takeImplicitAddr(c, tupleConstr[i], e.kind == tyLent) else: localError(c.config, n[0].info, errXExpected, "tuple constructor") + elif e.kind == tyEmpty: + localError(c.config, n[0].info, errTypeExpected) else: when false: # XXX investigate what we really need here. diff --git a/tests/misc/t21109.nim b/tests/misc/t21109.nim new file mode 100644 index 000000000..0f7980896 --- /dev/null +++ b/tests/misc/t21109.nim @@ -0,0 +1,13 @@ +discard """ + action: reject + errormsg: "type expected" + file: "iterators.nim" +""" + + +template b(j: untyped) = j +template m() = discard + +b: + for t, f in @[]: + m() From 886572a5162beea672d696fb03422e7777f56cfb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 20 Dec 2022 15:04:13 +0800 Subject: [PATCH 1480/1654] disable "Warning: gc is deprecated" in compiler (#21137) * disable "Warning: gc is deprecated" in compiler * Apply @tersec 's suggestion --- compiler/vmops.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index edf457295..d4a91b22d 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -145,6 +145,7 @@ when defined(nimHasInvariant): from std / compilesettings import SingleValueSetting, MultipleValueSetting proc querySettingImpl(conf: ConfigRef, switch: BiggestInt): string = + {.push warning[Deprecated]:off.} case SingleValueSetting(switch) of arguments: result = conf.arguments of outFile: result = conf.outFile.string @@ -162,6 +163,7 @@ when defined(nimHasInvariant): of libPath: result = conf.libpath.string of gc: result = $conf.selectedGC of mm: result = $conf.selectedGC + {.pop.} proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = template copySeq(field: untyped): untyped = From 40b5c4c4c340d44ee724183306716286e40002cb Mon Sep 17 00:00:00 2001 From: Bung Date: Wed, 21 Dec 2022 03:23:48 +0800 Subject: [PATCH 1481/1654] fix #20248;fix #6215;turns into simple CT error (#21141) --- compiler/semtypes.nim | 2 ++ tests/array/t20248.nim | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/array/t20248.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index dca0d753b..534861e97 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -328,6 +328,8 @@ proc semRange(c: PContext, n: PNode, prev: PType): PType = proc semArrayIndex(c: PContext, n: PNode): PType = if isRange(n): result = semRangeAux(c, n, nil) + elif n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "..<": + result = errorType(c) else: let e = semExprWithType(c, n, {efDetermineType}) if e.typ.kind == tyFromExpr: diff --git a/tests/array/t20248.nim b/tests/array/t20248.nim new file mode 100644 index 000000000..66142548b --- /dev/null +++ b/tests/array/t20248.nim @@ -0,0 +1,14 @@ +discard """ +cmd: "nim check --hints:off $file" +errormsg: "ordinal type expected; given: Error Type" +nimout: ''' +t20248.nim(10, 36) Error: ordinal type expected; given: Error Type +t20248.nim(14, 20) Error: ordinal type expected; given: Error Type +''' +""" + +type Vec[N: static[int]] = array[0 ..< N, float] + +var v: Vec[32] + +var stuff: array[0 ..< 16, int] From 0aec095b261bed8b5841beba7ce8cd52de4d54be Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 21 Dec 2022 03:26:32 +0800 Subject: [PATCH 1482/1654] fixes #19292; fixes #21122; fixes putEnv and setEnv with vcc (#21143) * fixes #19292; fixes 21122; fixes putEnv and setEnv with vcc * add a test --- lib/std/envvars.nim | 8 ++++++-- lib/std/private/win_setenv.nim | 26 +++++++++++++------------- tests/misc/t21443.nim | 6 ++++++ 3 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 tests/misc/t21443.nim diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index 817a155ec..ce90f66ba 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -63,9 +63,13 @@ when not defined(nimscript): import winlean when defined(nimPreviewSlimSystem): import std/widestrs - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", + + type wchar_t {.importc: "wchar_t", header: "".} = int16 + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} - proc getEnvImpl(env: cstring): WideCString = c_wgetenv(env.newWideCString) + proc getEnvImpl(env: cstring): WideCString = + let r: WideCString = env.newWideCString + cast[WideCString](c_wgetenv(cast[ptr wchar_t](r))) else: proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "".} diff --git a/lib/std/private/win_setenv.nim b/lib/std/private/win_setenv.nim index 303889a40..66e199dfe 100644 --- a/lib/std/private/win_setenv.nim +++ b/lib/std/private/win_setenv.nim @@ -33,25 +33,25 @@ else: # same as winlean.setEnvironmentVariableA proc c_getenv(varname: cstring): cstring {.importc: "getenv", header: "".} - proc c_wputenv(envstring: WideCString): cint {.importc: "_wputenv", header: "".} - proc c_wgetenv(varname: WideCString): WideCString {.importc: "_wgetenv", header: "".} + proc c_wputenv(envstring: ptr wchar_t): cint {.importc: "_wputenv", header: "".} + proc c_wgetenv(varname: ptr wchar_t): ptr wchar_t {.importc: "_wgetenv", header: "".} var errno {.importc, header: "".}: cint var genviron {.importc: "_environ".}: ptr ptr char # xxx `ptr UncheckedArray[WideCString]` did not work - proc wcstombs(wcstr: ptr char, mbstr: WideCString, count: csize_t): csize_t {.importc, header: "".} + proc wcstombs(wcstr: ptr char, mbstr: ptr wchar_t, count: csize_t): csize_t {.importc, header: "".} # xxx cint vs errno_t? proc setEnvImpl*(name: string, value: string, overwrite: cint): cint = const EINVAL = cint(22) - let wideName = newWideCString(name) - if overwrite == 0 and c_wgetenv(wideName) != nil: + let wideName: WideCString = newWideCString(name) + if overwrite == 0 and c_wgetenv(cast[ptr wchar_t](wideName)) != nil: return 0 if value != "": - let envstring = name & "=" & value - let e = c_wputenv(newWideCString(envstring)) + let envstring: WideCString = newWideCString(name & "=" & value) + let e = c_wputenv(cast[ptr wchar_t](envstring)) if e != 0: errno = EINVAL return -1 @@ -62,19 +62,19 @@ else: SetEnvironmentVariableA doesn't update `_environ`, so we have to do these terrible things. ]# - let envstring = name & "= " - if c_wputenv(newWideCString(envstring)) != 0: + let envstring: WideCString = newWideCString(name & "= ") + if c_wputenv(cast[ptr wchar_t](envstring)) != 0: errno = EINVAL return -1 # Here lies the documentation we blatently ignore to make this work. - var s = c_wgetenv(wideName) + var s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) s[0] = Utf16Char('\0') #[ This would result in a double null termination, which normally signifies the end of the environment variable list, so we stick a completely empty environment variable into the list instead. ]# - s = c_wgetenv(wideName) + s = cast[WideCString](c_wgetenv(cast[ptr wchar_t](wideName))) s[1] = Utf16Char('=') #[ If genviron is null, the MBCS environment has not been initialized @@ -88,12 +88,12 @@ else: # in the current codepage. Skip updating MBCS environment in this case. # For some reason, second `wcstombs` can find non-convertible characters # that the first `wcstombs` cannot. - let requiredSizeS = wcstombs(nil, wideName, 0) + let requiredSizeS = wcstombs(nil, cast[ptr wchar_t](wideName), 0) if requiredSizeS != high(csize_t): let requiredSize = requiredSizeS.int var buf = newSeq[char](requiredSize + 1) let buf2 = buf[0].addr - if wcstombs(buf2, wideName, csize_t(requiredSize + 1)) != high(csize_t): + if wcstombs(buf2, cast[ptr wchar_t](wideName), csize_t(requiredSize + 1)) != high(csize_t): var ptrToEnv = c_getenv(cast[cstring](buf2)) ptrToEnv[0] = '\0' ptrToEnv = c_getenv(cast[cstring](buf2)) diff --git a/tests/misc/t21443.nim b/tests/misc/t21443.nim new file mode 100644 index 000000000..70413c5b3 --- /dev/null +++ b/tests/misc/t21443.nim @@ -0,0 +1,6 @@ +import std/envvars + +# bug #19292 +putEnv("NimPutEnvTest", "test") +# bug #21122 +doAssert getEnv("NimPutEnvTest") == "test" From 8a77798dac66a55b42005db8837508a5af0547cf Mon Sep 17 00:00:00 2001 From: Luca Guzzon Date: Tue, 20 Dec 2022 20:29:42 +0100 Subject: [PATCH 1483/1654] ucpu fix-up for arm64 value on macos m1 (#21142) --- tools/niminst/makefile.nimf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index 2fe89ed69..b392ab472 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -173,6 +173,9 @@ endif ifeq ($(ucpu),aarch64) mycpu = arm64 endif +ifeq ($(ucpu),arm64) + mycpu = arm64 +endif ifeq ($(ucpu),riscv64) mycpu = riscv64 endif From 81d8ea95af0cfaaedca2fd1881199e113e6f5b41 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 21 Dec 2022 17:18:47 +0800 Subject: [PATCH 1484/1654] bump csource_v2 to include fixes for macos m1 (#21147) --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 0ea963c39..286e0dc2a 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v2 nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=32cf8431bb19f8803352a0ab3dbee5fec35d5ec6 +nim_csourcesHash=b66c420697c574be18a20dd4720248a715b4287e From d0721eadf8cd6bf9436b5e5c9113c4c7bcfcc770 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 22 Dec 2022 04:46:14 +0800 Subject: [PATCH 1485/1654] fixes #21144; try expression will not match the less indentation except (#21152) fixes #21144; try expression will not match the less indent except --- compiler/parser.nim | 4 +++- tests/parser/ttry.nim | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/parser/ttry.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index 6b3cebb7f..00013c218 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1770,11 +1770,13 @@ proc parseTry(p: var Parser; isExpr: bool): PNode = #| (optInd 'except' optionalExprList colcom stmt)* #| (optInd 'finally' colcom stmt)? result = newNodeP(nkTryStmt, p) + let parentIndent = p.currInd # isExpr getTok(p) colcom(p, result) result.add(parseStmt(p)) var b: PNode = nil - while sameOrNoInd(p) or isExpr: + + while sameOrNoInd(p) or (isExpr and parentIndent <= p.tok.indent): case p.tok.tokType of tkExcept: b = newNodeP(nkExceptBranch, p) diff --git a/tests/parser/ttry.nim b/tests/parser/ttry.nim new file mode 100644 index 000000000..190b0b8dc --- /dev/null +++ b/tests/parser/ttry.nim @@ -0,0 +1,27 @@ +# bug #21144 +block: + try: + let c = try: + 10 + except ValueError as exc: + 10 + except ValueError as exc: + discard + +if true: + block: + let c = try: + 10 + except ValueError as exc: + 10 + except OSError: + 99 + + +try: + let c = try: + 10 + except ValueError as exc: + 10 +except ValueError as exc: + discard \ No newline at end of file From 613829f7a4da5506269fadb3acc78d64ee0cbddf Mon Sep 17 00:00:00 2001 From: Peter Munch-Ellingsen Date: Thu, 22 Dec 2022 04:34:36 +0100 Subject: [PATCH 1486/1654] Implement setLineInfo (#21153) * Implement setLineInfo * Add tests --- compiler/msgs.nim | 7 +++++++ compiler/vm.nim | 14 ++++++++++++-- compiler/vmdef.nim | 3 ++- compiler/vmgen.nim | 14 +++++++++++++- lib/core/macros.nim | 16 ++++++++++++++++ tests/macros/tmacros_various.nim | 23 ++++++++++++++++++++++- 6 files changed, 72 insertions(+), 5 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 2c807806e..8b44c8dc6 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -128,6 +128,13 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex = var dummy: bool result = fileInfoIdx(conf, filename, dummy) +proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile; isKnownFile: var bool): FileIndex = + fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), isKnownFile) + +proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex = + var dummy: bool + fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy) + proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo = result.fileIndex = fileInfoIdx if line < int high(uint16): diff --git a/compiler/vm.nim b/compiler/vm.nim index 53a16bcb0..61261c44c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1934,14 +1934,24 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of 1: # getLine regs[ra].node = newIntNode(nkIntLit, n.info.line.int) of 2: # getColumn - regs[ra].node = newIntNode(nkIntLit, n.info.col) + regs[ra].node = newIntNode(nkIntLit, n.info.col.int) else: internalAssert c.config, false regs[ra].node.info = n.info regs[ra].node.typ = n.typ - of opcNSetLineInfo: + of opcNCopyLineInfo: decodeB(rkNode) regs[ra].node.info = regs[rb].node.info + of opcNSetLineInfoLine: + decodeB(rkNode) + regs[ra].node.info.line = regs[rb].intVal.uint16 + of opcNSetLineInfoColumn: + decodeB(rkNode) + regs[ra].node.info.col = regs[rb].intVal.int16 + of opcNSetLineInfoFile: + decodeB(rkNode) + regs[ra].node.info.fileIndex = + fileInfoIdx(c.config, RelativeFile regs[rb].node.strVal) of opcEqIdent: decodeBC(rkInt) # aliases for shorter and easier to understand code below diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 54876f37d..114a49557 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -141,7 +141,8 @@ type opcNError, opcNWarning, opcNHint, - opcNGetLineInfo, opcNSetLineInfo, + opcNGetLineInfo, opcNCopyLineInfo, opcNSetLineInfoLine, + opcNSetLineInfoColumn, opcNSetLineInfoFile opcEqIdent, opcStrToIdent, opcGetImpl, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 1fdfea7c4..6a2ab1d11 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1335,7 +1335,19 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of "copyLineInfo": internalAssert c.config, n.len == 3 unused(c, n, dest) - genBinaryStmt(c, n, opcNSetLineInfo) + genBinaryStmt(c, n, opcNCopyLineInfo) + of "setLine": + internalAssert c.config, n.len == 3 + unused(c, n, dest) + genBinaryStmt(c, n, opcNSetLineInfoLine) + of "setColumn": + internalAssert c.config, n.len == 3 + unused(c, n, dest) + genBinaryStmt(c, n, opcNSetLineInfoColumn) + of "setFile": + internalAssert c.config, n.len == 3 + unused(c, n, dest) + genBinaryStmt(c, n, opcNSetLineInfoFile) else: internalAssert c.config, false of mNHint: unused(c, n, dest) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 18a70f20c..138ec47a0 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -533,6 +533,22 @@ proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.} proc copyLineInfo*(arg: NimNode, info: NimNode) {.magic: "NLineInfo", noSideEffect.} ## Copy lineinfo from `info`. +proc setLine(arg: NimNode, line: uint16) {.magic: "NLineInfo", noSideEffect.} +proc setColumn(arg: NimNode, column: int16) {.magic: "NLineInfo", noSideEffect.} +proc setFile(arg: NimNode, file: string) {.magic: "NLineInfo", noSideEffect.} + +proc setLineInfo*(arg: NimNode, file: string, line: int, column: int) = + ## Sets the line info on the NimNode. The file needs to exists, but can be a + ## relative path. If you want to attach line info to a block using `quote` + ## you'll need to add the line information after the quote block. + arg.setFile(file) + arg.setLine(line.uint16) + arg.setColumn(column.int16) + +proc setLineInfo*(arg: NimNode, lineInfo: LineInfo) = + ## See `setLineInfo proc<#setLineInfo,NimNode,string,int,int>`_ + setLineInfo(arg, lineInfo.filename, lineInfo.line, lineInfo.column) + proc lineInfoObj*(n: NimNode): LineInfo = ## Returns `LineInfo` of `n`, using absolute path for `filename`. result = LineInfo(filename: n.getFile, line: n.getLine, column: n.getColumn) diff --git a/tests/macros/tmacros_various.nim b/tests/macros/tmacros_various.nim index d702db56a..08be4602c 100644 --- a/tests/macros/tmacros_various.nim +++ b/tests/macros/tmacros_various.nim @@ -91,7 +91,7 @@ block tlexerex: -block tlineinfo: +block tcopylineinfo: # issue #5617, feature request type Test = object @@ -103,6 +103,27 @@ block tlineinfo: var z = mixer(Test) doAssert z +block tsetgetlineinfo: + # issue #21098, feature request + type Test = object + + macro mixer1(n: typed): untyped = + let x = newIdentNode("echo") + var lineInfo = n.lineInfoObj + x.setLineInfo lineInfo + result = newLit(x.lineInfo == n.lineInfo) + + macro mixer2(n: typed): untyped = + let x = newIdentNode("echo") + var lineInfo = n.lineInfoObj + lineInfo.line += 1 + x.setLineInfo lineInfo + result = newLit(x.lineInfo != n.lineInfo) + + doAssert mixer1(Test) + + doAssert mixer2(Test) + block tdebugstmt: From c5a72ebddd88b6d3a2712230d36367f180faa7da Mon Sep 17 00:00:00 2001 From: Bung Date: Thu, 22 Dec 2022 13:16:25 +0800 Subject: [PATCH 1487/1654] fix #16541 (#21148) --- compiler/sigmatch.nim | 2 ++ tests/misc/t16541.nim | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/misc/t16541.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 51728b103..61a4395f3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2150,6 +2150,8 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, elif arg.sym.kind in {skMacro, skTemplate}: return nil else: + if arg.sym.ast == nil: + return nil let inferred = c.semGenerateInstance(c, arg.sym, m.bindings, arg.info) result = newSymNode(inferred, arg.info) if r == isInferredConvertible: diff --git a/tests/misc/t16541.nim b/tests/misc/t16541.nim new file mode 100644 index 000000000..452327e8f --- /dev/null +++ b/tests/misc/t16541.nim @@ -0,0 +1,12 @@ +discard """ + action: "reject" + +""" + +import strutils, sugar, nre + +proc my_replace*(s: string, r: Regex, by: string | (proc (match: string): string)): string = + nre.replace(s, r, by) + +discard my_replace("abcde", re"[bcd]", match => match.to_upper) == "aBCDe" +discard my_replace("abcde", re"[bcd]", (match: string) => match.to_upper) == "aBCDe" From 70fe360456b77912ec7f5014f7815137fa089fce Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Thu, 22 Dec 2022 18:25:24 +1100 Subject: [PATCH 1488/1654] Use `ErrorColor` when a warning that is turned into an error is raised (#21131) Use ErrorColor when a warning that is turned into an error is raised --- compiler/msgs.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 8b44c8dc6..8e391f2fb 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -541,10 +541,11 @@ proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string, ignoreMsg = not conf.hasWarn(msg) if not ignoreMsg and msg in conf.warningAsErrors: title = ErrorTitle + color = ErrorColor else: title = WarningTitle + color = WarningColor if not ignoreMsg: writeContext(conf, info) - color = WarningColor inc(conf.warnCounter) of hintMin..hintMax: sev = Severity.Hint From 93b59da4902886cd68dd7df1dce09a1b455a06dc Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 22 Dec 2022 15:27:11 +0800 Subject: [PATCH 1489/1654] fixes #20244; fixes castSizes warnings (#21102) * fixes #20244; fixes castSizes warnings * fixes js * fixes js * fixes js * fixes * typo * extend using uint64 * Update lib/std/syncio.nim --- compiler/ccgutils.nim | 2 +- compiler/condsyms.nim | 1 + compiler/nim.cfg | 4 ++++ lib/pure/strutils.nim | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index e19fccfa7..d86ebe461 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -53,7 +53,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt = a = a + (a shl 3) a = a xor (a shr 11) a = a + (a shl 15) - result = cast[Hash](a) + result = cast[Hash](uint(a)) template getUniqueType*(key: PType): PType = key diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 9d081ef0a..24c6d82d5 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -146,6 +146,7 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasCallsitePragma") defineSymbol("nimHasAmbiguousEnumHint") + defineSymbol("nimHasWarnCastSizes") defineSymbol("nimHasOutParams") defineSymbol("nimHasSystemRaisesDefect") defineSymbol("nimHasWarnUnnamedBreak") diff --git a/compiler/nim.cfg b/compiler/nim.cfg index 96b47b0e6..b9c8fdc08 100644 --- a/compiler/nim.cfg +++ b/compiler/nim.cfg @@ -31,6 +31,10 @@ define:useStdoutAsStdmsg warning[ObservableStores]:off @end +@if nimHasWarnCastSizes: + warning[CastSizes]:on +@end + @if nimHasWarningAsError: warningAsError[GcUnsafe2]:on @end diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f628cfc2d..dbdd0d6a1 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -944,14 +944,26 @@ func toHex*[T: SomeInteger](x: T, len: Positive): string = doAssert b.toHex(4) == "1001" doAssert toHex(62, 3) == "03E" doAssert toHex(-8, 6) == "FFFFF8" - toHexImpl(cast[BiggestUInt](x), len, x < 0) + when defined(js): + toHexImpl(cast[BiggestUInt](x), len, x < 0) + else: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), len, x < 0) + else: + toHexImpl(BiggestUInt(x), len, x < 0) func toHex*[T: SomeInteger](x: T): string = ## Shortcut for `toHex(x, T.sizeof * 2)` runnableExamples: doAssert toHex(1984'i64) == "00000000000007C0" doAssert toHex(1984'i16) == "07C0" - toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + when defined(js): + toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0) + else: + when T is SomeSignedInt: + toHexImpl(cast[BiggestUInt](BiggestInt(x)), 2*sizeof(T), x < 0) + else: + toHexImpl(BiggestUInt(x), 2*sizeof(T), x < 0) func toHex*(s: string): string {.rtl.} = ## Converts a bytes string to its hexadecimal representation. From 7931bdac956cb3ad6734eab91c62920f4e80f919 Mon Sep 17 00:00:00 2001 From: Michael Voronin Date: Thu, 22 Dec 2022 10:32:12 +0300 Subject: [PATCH 1490/1654] Feature/xmltree additions (#20988) * [change] add/insert/delete family of xmltree expanded with several variations. Added replace methods family * [change] Lifted child limitations on insert methods (consulted with @araq) * [tests] add/insert/replace/delete of xmltree XmlNodes tests added --- lib/pure/xmltree.nim | 153 ++++++++++++++++++++++++++++++++++- tests/xml/ttree_add.nim | 51 ++++++++++++ tests/xml/ttree_add1.nim | 53 ++++++++++++ tests/xml/ttree_delete.nim | 47 +++++++++++ tests/xml/ttree_delete1.nim | 48 +++++++++++ tests/xml/ttree_insert.nim | 53 ++++++++++++ tests/xml/ttree_insert1.nim | 51 ++++++++++++ tests/xml/ttree_replace.nim | 46 +++++++++++ tests/xml/ttree_replace1.nim | 53 ++++++++++++ 9 files changed, 552 insertions(+), 3 deletions(-) create mode 100644 tests/xml/ttree_add.nim create mode 100644 tests/xml/ttree_add1.nim create mode 100644 tests/xml/ttree_delete.nim create mode 100644 tests/xml/ttree_delete1.nim create mode 100644 tests/xml/ttree_insert.nim create mode 100644 tests/xml/ttree_insert1.nim create mode 100644 tests/xml/ttree_replace.nim create mode 100644 tests/xml/ttree_replace1.nim diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 6228bd10f..7a2cebe87 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -31,7 +31,7 @@ runnableExamples: ## * `htmlgen module `_ for html code generator import std/private/since -import macros, strtabs, strutils +import macros, strtabs, strutils, sequtils when defined(nimPreviewSlimSystem): import std/assertions @@ -298,26 +298,60 @@ proc innerText*(n: XmlNode): string = proc add*(father, son: XmlNode) {.inline.} = ## Adds the child `son` to `father`. + ## `father` must be of `xnElement` type ## ## See also: + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newText("my text") f.add newElement("sonTag") f.add newEntity("my entity") assert $f == "my text&my entity;" + + assert father.k == xnElement add(father.s, son) +proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = + ## Adds the children `sons` to `father`. + ## `father` must be of `xnElement` type + ## + ## See also: + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) + assert $f == "my text&my entity;" + + assert father.k == xnElement + add(father.s, sons) + + proc insert*(father, son: XmlNode, index: int) {.inline.} = ## Inserts the child `son` to a given position in `father`. ## - ## `father` and `son` must be of `xnElement` kind. + ## `father` must be of `xnElement` kind. ## ## See also: + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -327,18 +361,52 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = """ - assert father.k == xnElement and son.k == xnElement + assert father.k == xnElement if len(father.s) > index: insert(father.s, son, index) else: insert(father.s, son, len(father.s)) +proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = + ## Inserts the children openArray[`sons`] to a given position in `father`. + ## + ## `father` must be of `xnElement` kind. + ## + ## See also: + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + assert $f == """ + + + +""" + + assert father.k == xnElement + if len(father.s) > index: + insert(father.s, sons, index) + else: + insert(father.s, sons, len(father.s)) + proc delete*(n: XmlNode, i: Natural) = ## Deletes the `i`'th child of `n`. ## ## See also: + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ runnableExamples: var f = newElement("myTag") f.add newElement("first") @@ -351,6 +419,85 @@ proc delete*(n: XmlNode, i: Natural) = assert n.k == xnElement n.s.delete(i) +proc delete*(n: XmlNode, slice: Slice[int]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## See also: + ## * `delete proc <#delete.XmlNode,int>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("third")], 0) + f.delete(0..1) + assert $f == """ + +""" + + assert n.k == xnElement + n.s.delete(slice) + +proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = + ## Replaces the `i`'th child of `n` with `replacement` openArray. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,Slice[int],openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert(newElement("second"), 0) + f.replace(0, @[newElement("third"), newElement("fourth")]) + assert $f == """ + + + +""" + + assert n.k == xnElement + n.s.delete(i) + n.s.insert(replacement, i) + +proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = + ## Deletes the items `n[slice]` of `n`. + ## + ## `n` must be of `xnElement` kind. + ## + ## See also: + ## * `replace proc <#replace.XmlNode,int,openArray[XmlNode]>`_ + ## * `add proc <#add,XmlNode,XmlNode>`_ + ## * `add proc <#add,XmlNode,openArray[XmlNode]>`_ + ## * `delete proc <#delete,XmlNode,Natural>`_ + ## * `delete proc <#delete.XmlNode,Slice[int]>`_ + ## * `insert proc <#insert,XmlNode,XmlNode,int>`_ + ## * `insert proc <#insert,XmlNode,openArray[XmlNode],int>`_ + runnableExamples: + var f = newElement("myTag") + f.add newElement("first") + f.insert([newElement("second"), newElement("fifth")], 0) + f.replace(0..1, @[newElement("third"), newElement("fourth")]) + assert $f == """ + + + +""" + + assert n.k == xnElement + n.s.delete(slice) + n.s.insert(replacement, slice.a) + proc len*(n: XmlNode): int {.inline.} = ## Returns the number of `n`'s children. runnableExamples: diff --git a/tests/xml/ttree_add.nim b/tests/xml/ttree_add.nim new file mode 100644 index 000000000..4c6ef6cf9 --- /dev/null +++ b/tests/xml/ttree_add.nim @@ -0,0 +1,51 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_add() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + for item in baseDocBodyTree.items(): + newBody.add(item) + + echo $newBody + + testDoc.add(newBody) + echo $testDoc + +test_add() diff --git a/tests/xml/ttree_add1.nim b/tests/xml/ttree_add1.nim new file mode 100644 index 000000000..30ec83c02 --- /dev/null +++ b/tests/xml/ttree_add1.nim @@ -0,0 +1,53 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_add() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + var bodyItems: seq[XmlNode] = @[] + for item in baseDocBodyTree.items(): + bodyItems.add(item) + newBody.add(bodyItems) + + echo $newBody + + testDoc.add(newBody) + echo $testDoc + +test_add() diff --git a/tests/xml/ttree_delete.nim b/tests/xml/ttree_delete.nim new file mode 100644 index 000000000..32b477839 --- /dev/null +++ b/tests/xml/ttree_delete.nim @@ -0,0 +1,47 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +let initialDocBase = """ + + +
          Some text
          +
          Some more text
          + + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          Some text in body
          +
          Some more text in body
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_delete() = + var testDoc = initialDocBaseTree + + testDoc.delete(1..2) + echo $testDoc + +test_delete() diff --git a/tests/xml/ttree_delete1.nim b/tests/xml/ttree_delete1.nim new file mode 100644 index 000000000..a8442a093 --- /dev/null +++ b/tests/xml/ttree_delete1.nim @@ -0,0 +1,48 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +let initialDocBase = """ + + +
          Some text
          +
          Some more text
          + + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          MORE TEXT
          +
          MORE TEXT Some more text
          +           + +
          Some text in body
          +
          Some more text in body
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_delete() = + var testDoc = initialDocBaseTree + + testDoc.delete(1) + testDoc.delete(1) + echo $testDoc + +test_delete() diff --git a/tests/xml/ttree_insert.nim b/tests/xml/ttree_insert.nim new file mode 100644 index 000000000..b2941395b --- /dev/null +++ b/tests/xml/ttree_insert.nim @@ -0,0 +1,53 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_insert() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + var bodyItems: seq[XmlNode] = @[] + for item in baseDocBodyTree.items(): + bodyItems.insert(item, len(bodyItems)) + newBody.insert(bodyItems, 1) + + echo $newBody + + testDoc.insert(newBody, 1) + echo $testDoc + +test_insert() diff --git a/tests/xml/ttree_insert1.nim b/tests/xml/ttree_insert1.nim new file mode 100644 index 000000000..9aa3faf69 --- /dev/null +++ b/tests/xml/ttree_insert1.nim @@ -0,0 +1,51 @@ +discard """ + output: ''' + +
          Some text in body
          +
          Some more text in body
          + + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + + +
          Some text
          +
          Some more text
          + +           +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) + +proc test_insert() = + var testDoc = baseDocHeadTree + var newBody = newElement("body") + for item in baseDocBodyTree.items(): + newBody.insert(item, len(newBody)) + + echo $newBody + + testDoc.insert(newBody, 1) + echo $testDoc + +test_insert() diff --git a/tests/xml/ttree_replace.nim b/tests/xml/ttree_replace.nim new file mode 100644 index 000000000..97d2db638 --- /dev/null +++ b/tests/xml/ttree_replace.nim @@ -0,0 +1,46 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) +let initialDocBase = """ + + +
          Some text
          +
          Some more text
          + + +
          Some text in body before replace
          +
          Some more text in body before replace
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_replace() = + var testDoc = initialDocBaseTree + + testDoc.replace(1, @[baseDocBodyTree]) + echo $testDoc + +test_replace() diff --git a/tests/xml/ttree_replace1.nim b/tests/xml/ttree_replace1.nim new file mode 100644 index 000000000..059ce2085 --- /dev/null +++ b/tests/xml/ttree_replace1.nim @@ -0,0 +1,53 @@ +discard """ + output: ''' + + +
          Some text
          +
          Some more text
          + + +
          Some text in body
          +
          Some more text in body
          + +           +''' +""" + +# Test xmltree add/insert/delete/replace operations +import xmlparser +import xmltree +var baseDocHead = """ + +
          Some text
          +
          Some more text
          + +""" +var baseDocHeadTree = parseXml(baseDocHead) +var baseDocBody = """ + +
          Some text in body
          +
          Some more text in body
          + +""" +var baseDocBodyTree = parseXml(baseDocBody) +let initialDocBase = """ + + +
          Some text before replace
          +
          Some more text before replace
          + + +
          Some text in body before replace
          +
          Some more text in body before replace
          + +           +""" +var initialDocBaseTree = parseXml(initialDocBase) + +proc test_replace() = + var testDoc = initialDocBaseTree + + testDoc.replace(0..1, @[baseDocHeadTree, baseDocBodyTree]) + echo $testDoc + +test_replace() From 314a24472e83eedf058ffa614edf432e8744d0cb Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Thu, 22 Dec 2022 09:54:17 +0100 Subject: [PATCH 1491/1654] add link to setutils in system module docs (#21130) --- lib/system_overview.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/system_overview.rst b/lib/system_overview.rst index b338845d6..cc0643bf1 100644 --- a/lib/system_overview.rst +++ b/lib/system_overview.rst @@ -99,6 +99,7 @@ Proc Usage =============================== ====================================== **See also:** +* `setutils module `_ for bit set convenience functions * `sets module `_ for hash sets * `intsets module `_ for efficient int sets From 37daed389735ac66f5c950be9ee1c6658b019016 Mon Sep 17 00:00:00 2001 From: metagn Date: Thu, 22 Dec 2022 11:54:41 +0300 Subject: [PATCH 1492/1654] remove misleading slimsystem deprecated warnings (#21156) refs #20967 --- lib/std/formatfloat.nim | 2 -- lib/system.nim | 3 --- 2 files changed, 5 deletions(-) diff --git a/lib/std/formatfloat.nim b/lib/std/formatfloat.nim index 8abbe59cd..b216d1fd0 100644 --- a/lib/std/formatfloat.nim +++ b/lib/std/formatfloat.nim @@ -11,8 +11,6 @@ when defined(nimPreviewSlimSystem): import std/assertions -else: - {.deprecated: "formatfloat is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/formatfloat`".} proc c_memcpy(a, b: pointer, size: csize_t): pointer {.importc: "memcpy", header: "", discardable.} diff --git a/lib/system.nim b/lib/system.nim index 65055f145..1b6235fb3 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1630,7 +1630,6 @@ template newException*(exceptn: typedesc, message: string; (ref exceptn)(msg: message, parent: parentException) when not defined(nimPreviewSlimSystem): - {.deprecated: "assertions is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/assertions`".} import std/assertions export assertions @@ -2068,7 +2067,6 @@ when not defined(js): when hostOS != "standalone": include system/threadimpl when not defined(nimPreviewSlimSystem): - {.deprecated: "threads is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/typedthreads`".} import std/typedthreads export typedthreads @@ -2782,7 +2780,6 @@ when notJSnotNims: releaseSys echoLock when not defined(nimPreviewSlimSystem): - {.deprecated: "io is about to move out of system; use `-d:nimPreviewSlimSystem` and import `std/syncio`".} import std/syncio export syncio From 18c115c8d0b38e6dbb7fae5bdda94bfca1a0ecef Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 23 Dec 2022 00:44:10 +1100 Subject: [PATCH 1493/1654] Don't repeat suggestions for same symbol (#21140) * Track seen module graphs so symbols from the same module aren't repeated Add test case * Track symbols instead of modules * Don't show duplicate symbols in spell checker Removes the declared location from the message. Since we don't show duplicates anymore it would be a bit misleading if we only show the location for the first declaration of the symbol --- compiler/lookups.nim | 25 ++++++++++++++++++++----- compiler/suggest.nim | 6 +++--- tests/misc/tspellsuggest.nim | 30 +++++++++++++++--------------- tests/misc/tspellsuggest2.nim | 24 ++++++++++++------------ tests/misc/tspellsuggest3.nim | 21 +++++++++++++++++++++ 5 files changed, 71 insertions(+), 35 deletions(-) create mode 100644 tests/misc/tspellsuggest3.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index df5a5333e..d50fc6e27 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -15,7 +15,7 @@ when defined(nimPreviewSlimSystem): import intsets, ast, astalgo, idents, semdata, types, msgs, options, - renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs + renderer, nimfix/prettybase, lineinfos, modulegraphs, astmsgs, sets proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope) @@ -170,6 +170,7 @@ iterator allSyms*(c: PContext): (PSym, int, bool) = # really iterate over all symbols in all the scopes. This is expensive # and only used by suggest.nim. var isLocal = true + var scopeN = 0 for scope in allScopes(c.currentScope): if scope == c.topLevelScope: isLocal = false @@ -184,6 +185,17 @@ iterator allSyms*(c: PContext): (PSym, int, bool) = assert s != nil yield (s, scopeN, isLocal) +iterator uniqueSyms*(c: PContext): (PSym, int, bool) = + ## Like [allSyms] except only returns unique symbols (Uniqueness determined by line + name) + # Track seen symbols so we don't duplicate them. + # The int is for the symbols name, and line info is + # to be able to tell apart symbols with same name but on different lines + var seen = initHashSet[(TLineInfo, int)]() + for res in allSyms(c): + if not seen.containsOrIncl((res[0].info, res[0].name.id)): + yield res + + proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym = var marked = initIntSet() var symSet = OverloadableSyms @@ -445,12 +457,13 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) = let dist = editDistance(name0, sym.name.s.nimIdentNormalize) var msg: string msg.add "\n ($1, $2): '$3'" % [$dist, $depth, sym.name.s] - addDeclaredLoc(msg, c.config, sym) # `msg` needed for deterministic ordering. list.push SpellCandidate(dist: dist, depth: depth, msg: msg, sym: sym) if list.len == 0: return let e0 = list[0] - var count = 0 + var + count = 0 + last: PIdent = nil while true: # pending https://github.com/timotheecour/Nim/issues/373 use more efficient `itemsSorted`. if list.len == 0: break @@ -466,8 +479,10 @@ proc fixSpelling(c: PContext, n: PNode, ident: PIdent, result: var string) = elif count >= c.config.spellSuggestMax: break if count == 0: result.add "\ncandidates (edit distance, scope distance); see '--spellSuggest': " - result.add e.msg - count.inc + if e.sym.name != last: + result.add e.msg + count.inc + last = e.sym.name proc errorUseQualifier(c: PContext; info: TLineInfo; s: PSym; amb: var bool): PSym = var err = "ambiguous identifier: '" & s.name.s & "'" diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 41d39ccab..e0b0fb516 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -289,7 +289,7 @@ proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var s.getQuality, pm, c.inTypeContext > 0, 0)) template wholeSymTab(cond, section: untyped) {.dirty.} = - for (item, scopeN, isLocal) in allSyms(c): + for (item, scopeN, isLocal) in uniqueSyms(c): let it = item var pm: PrefixMatch if cond: @@ -362,7 +362,7 @@ proc suggestOperations(c: PContext, n, f: PNode, typ: PType, outputs: var Sugges proc suggestEverything(c: PContext, n, f: PNode, outputs: var Suggestions) = # do not produce too many symbols: - for (it, scopeN, isLocal) in allSyms(c): + for (it, scopeN, isLocal) in uniqueSyms(c): var pm: PrefixMatch if filterSym(it, f, pm): outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, n.info, @@ -680,7 +680,7 @@ proc suggestSentinel*(c: PContext) = inc(c.compilesContextId) var outputs: Suggestions = @[] # suggest everything: - for (it, scopeN, isLocal) in allSyms(c): + for (it, scopeN, isLocal) in uniqueSyms(c): var pm: PrefixMatch if filterSymNoOpr(it, nil, pm): outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, diff --git a/tests/misc/tspellsuggest.nim b/tests/misc/tspellsuggest.nim index 033ed0afc..345458bb1 100644 --- a/tests/misc/tspellsuggest.nim +++ b/tests/misc/tspellsuggest.nim @@ -5,21 +5,21 @@ discard """ nimout: ''' tspellsuggest.nim(45, 13) Error: undeclared identifier: 'fooBar' candidates (edit distance, scope distance); see '--spellSuggest': - (1, 0): 'fooBar8' [var declared in tspellsuggest.nim(43, 9)] - (1, 1): 'fooBar7' [var declared in tspellsuggest.nim(41, 7)] - (1, 3): 'fooBar1' [var declared in tspellsuggest.nim(33, 5)] - (1, 3): 'fooBar2' [let declared in tspellsuggest.nim(34, 5)] - (1, 3): 'fooBar3' [const declared in tspellsuggest.nim(35, 7)] - (1, 3): 'fooBar4' [proc declared in tspellsuggest.nim(36, 6)] - (1, 3): 'fooBar5' [template declared in tspellsuggest.nim(37, 10)] - (1, 3): 'fooBar6' [macro declared in tspellsuggest.nim(38, 7)] - (1, 5): 'FooBar' [type declared in mspellsuggest.nim(5, 6)] - (1, 5): 'fooBar4' [proc declared in mspellsuggest.nim(1, 6)] - (1, 5): 'fooBar9' [var declared in mspellsuggest.nim(2, 5)] - (1, 5): 'fooCar' [var declared in mspellsuggest.nim(4, 5)] - (2, 5): 'FooCar' [type declared in mspellsuggest.nim(6, 6)] - (2, 5): 'GooBa' [type declared in mspellsuggest.nim(7, 6)] - (3, 0): 'fooBarBaz' [const declared in tspellsuggest.nim(44, 11)] + (1, 0): 'fooBar8' + (1, 1): 'fooBar7' + (1, 3): 'fooBar1' + (1, 3): 'fooBar2' + (1, 3): 'fooBar3' + (1, 3): 'fooBar4' + (1, 3): 'fooBar5' + (1, 3): 'fooBar6' + (1, 5): 'FooBar' + (1, 5): 'fooBar4' + (1, 5): 'fooBar9' + (1, 5): 'fooCar' + (2, 5): 'FooCar' + (2, 5): 'GooBa' + (3, 0): 'fooBarBaz' ''' """ diff --git a/tests/misc/tspellsuggest2.nim b/tests/misc/tspellsuggest2.nim index 78504c513..bf76cc208 100644 --- a/tests/misc/tspellsuggest2.nim +++ b/tests/misc/tspellsuggest2.nim @@ -5,18 +5,18 @@ discard """ nimout: ''' tspellsuggest2.nim(45, 13) Error: undeclared identifier: 'fooBar' candidates (edit distance, scope distance); see '--spellSuggest': - (1, 0): 'fooBar8' [var declared in tspellsuggest2.nim(43, 9)] - (1, 1): 'fooBar7' [var declared in tspellsuggest2.nim(41, 7)] - (1, 3): 'fooBar1' [var declared in tspellsuggest2.nim(33, 5)] - (1, 3): 'fooBar2' [let declared in tspellsuggest2.nim(34, 5)] - (1, 3): 'fooBar3' [const declared in tspellsuggest2.nim(35, 7)] - (1, 3): 'fooBar4' [proc declared in tspellsuggest2.nim(36, 6)] - (1, 3): 'fooBar5' [template declared in tspellsuggest2.nim(37, 10)] - (1, 3): 'fooBar6' [macro declared in tspellsuggest2.nim(38, 7)] - (1, 5): 'FooBar' [type declared in mspellsuggest.nim(5, 6)] - (1, 5): 'fooBar4' [proc declared in mspellsuggest.nim(1, 6)] - (1, 5): 'fooBar9' [var declared in mspellsuggest.nim(2, 5)] - (1, 5): 'fooCar' [var declared in mspellsuggest.nim(4, 5)] + (1, 0): 'fooBar8' + (1, 1): 'fooBar7' + (1, 3): 'fooBar1' + (1, 3): 'fooBar2' + (1, 3): 'fooBar3' + (1, 3): 'fooBar4' + (1, 3): 'fooBar5' + (1, 3): 'fooBar6' + (1, 5): 'FooBar' + (1, 5): 'fooBar4' + (1, 5): 'fooBar9' + (1, 5): 'fooCar' ''' """ diff --git a/tests/misc/tspellsuggest3.nim b/tests/misc/tspellsuggest3.nim new file mode 100644 index 000000000..9b0b84602 --- /dev/null +++ b/tests/misc/tspellsuggest3.nim @@ -0,0 +1,21 @@ +discard """ + # pending bug #16521 (bug 12) use `matrix` + cmd: "nim c --spellsuggest:4 --hints:off $file" + action: "reject" + nimout: ''' +tspellsuggest3.nim(21, 1) Error: undeclared identifier: 'fooBar' +candidates (edit distance, scope distance); see '--spellSuggest': + (1, 2): 'FooBar' + (1, 2): 'fooBar4' + (1, 2): 'fooBar9' + (1, 2): 'fooCar' +''' +""" + +import ./mspellsuggest +import ./mspellsuggest +import ./mspellsuggest +import ./mspellsuggest + + +fooBar From 3bba2b34fd632a4c01a6eb0e6f13d4e3a20bf104 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 22 Dec 2022 13:43:27 -0700 Subject: [PATCH 1494/1654] fix socket send for string types (#21155) * fix socket send for string types * include windows version Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- lib/pure/net.nim | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 1dfdbbbe6..be9f2e48c 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1735,15 +1735,36 @@ proc send*(socket: Socket, data: pointer, size: int): int {. result = send(socket.fd, data, size, int32(MSG_NOSIGNAL)) proc send*(socket: Socket, data: string, - flags = {SocketFlag.SafeDisconn}) {.tags: [WriteIOEffect].} = - ## sends data to a socket. - let sent = send(socket, cstring(data), data.len) - if sent < 0: - let lastError = osLastError() - socketError(socket, lastError = lastError, flags = flags) + flags = {SocketFlag.SafeDisconn}, maxRetries = 100) {.tags: [WriteIOEffect].} = + ## Sends data to a socket. Will try to send all the data by handling interrupts + ## and incomplete writes up to `maxRetries`. + var written = 0 + var attempts = 0 + while data.len - written > 0: + let sent = send(socket, cstring(data), data.len) - if sent != data.len: - raiseOSError(osLastError(), "Could not send all data.") + if sent < 0: + let lastError = osLastError() + let isBlockingErr = + when defined(nimdoc): + false + elif useWinVersion: + lastError.int32 == WSAEINTR or + lastError.int32 == WSAEWOULDBLOCK + else: + lastError.int32 == EINTR or + lastError.int32 == EWOULDBLOCK or + lastError.int32 == EAGAIN + + if not isBlockingErr: + let lastError = osLastError() + socketError(socket, lastError = lastError, flags = flags) + else: + attempts.inc() + if attempts > maxRetries: + raiseOSError(osLastError(), "Could not send all data.") + else: + written.inc(sent) template `&=`*(socket: Socket; data: typed) = ## an alias for 'send'. From 13251c2ac9c7e76fb506eeb686a5e5d19d67d0de Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 23 Dec 2022 16:47:01 +0800 Subject: [PATCH 1495/1654] fix #12946 Bad C++ codegen on distinct generics C++ types (#21157) --- compiler/ccgtypes.nim | 9 +++++---- tests/cpp/t12946.nim | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/cpp/t12946.nim diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 3c9f9d923..e7677f2f2 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -826,7 +826,8 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin m.s[cfsTypes].addf("typedef $1 $2[$3];$n", [foo, result, rope(n)]) of tyObject, tyTuple: - if isImportedCppType(t) and origTyp.kind == tyGenericInst: + let tt = origTyp.skipTypes({tyDistinct}) + if isImportedCppType(t) and tt.kind == tyGenericInst: let cppNameAsRope = getTypeName(m, t, sig) let cppName = $cppNameAsRope var i = 0 @@ -849,7 +850,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result.add cppName.substr(chunkStart, chunkEnd) chunkStart = i - let typeInSlot = resolveStarsInCppType(origTyp, idx + 1, stars) + let typeInSlot = resolveStarsInCppType(tt, idx + 1, stars) addResultType(typeInSlot) else: inc i @@ -858,9 +859,9 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result.add cppName.substr(chunkStart) else: result = cppNameAsRope & "<" - for i in 1.. 1: result.add(" COMMA ") - addResultType(origTyp[i]) + addResultType(tt[i]) result.add("> ") # always call for sideeffects: assert t.kind != tyTuple diff --git a/tests/cpp/t12946.nim b/tests/cpp/t12946.nim new file mode 100644 index 000000000..cf6bf5cfc --- /dev/null +++ b/tests/cpp/t12946.nim @@ -0,0 +1,4 @@ +import std/atomics +type Futex = distinct Atomic[int32] + +var x: Futex From ca9c74391a2169be0b0e5c7b6705eaf9560a44bd Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 23 Dec 2022 18:32:03 +0800 Subject: [PATCH 1496/1654] fix #11634 (#21146) --- compiler/semstmts.nim | 9 ++++++--- tests/misc/t11634.nim | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 tests/misc/t11634.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index a7a02d7fa..356c2d63e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -725,9 +725,12 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if v.kind == skLet and sfImportc notin v.flags and (strictDefs notin c.features or not isLocalSym(v)): localError(c.config, a.info, errLetNeedsInit) if sfCompileTime in v.flags: - var x = newNodeI(result.kind, v.info) - x.add result[i] - vm.setupCompileTimeVar(c.module, c.idgen, c.graph, x) + if a.kind != nkVarTuple: + var x = newNodeI(result.kind, v.info) + x.add result[i] + vm.setupCompileTimeVar(c.module, c.idgen, c.graph, x) + else: + localError(c.config, a.info, "cannot destructure to compile time variable") if v.flags * {sfGlobal, sfThread} == {sfGlobal}: message(c.config, v.info, hintGlobalVar) if {sfGlobal, sfPure} <= v.flags: diff --git a/tests/misc/t11634.nim b/tests/misc/t11634.nim new file mode 100644 index 000000000..4ecb6a53c --- /dev/null +++ b/tests/misc/t11634.nim @@ -0,0 +1,20 @@ +discard """ + action: reject + nimout: ''' +t11634.nim(20, 7) Error: cannot destructure to compile time variable +''' +""" + +type Foo = ref object + val: int + +proc divmod(a, b: Foo): (Foo, Foo) = + ( + Foo(val: a.val div b.val), + Foo(val: a.val mod b.val) + ) + +block: + let a {.compileTime.} = Foo(val: 2) + let b {.compileTime.} = Foo(val: 3) + let (c11634 {.compileTime.}, d11634 {.compileTime.}) = divmod(a, b) From 0b319fee3de90759987beba289df3c48ab6d3f1a Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 23 Dec 2022 19:20:25 +0800 Subject: [PATCH 1497/1654] =?UTF-8?q?fix=20#20997=20calling=20system.card[?= =?UTF-8?q?T](x:=20set[T])=20with=20T=20of=20int8=20or=20uint8=20=E2=80=A6?= =?UTF-8?q?=20(#21010)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix #20997 calling system.card[T](x: set[T]) with T of int8 or uint8 uses mismatched C array sizes * fullfil set variant --- compiler/ccgexprs.nim | 2 +- lib/system/sets.nim | 8 +++++--- tests/sets/t20997.nim | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tests/sets/t20997.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 2a4335144..d7ef024b1 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2126,7 +2126,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = of mCard: var a: TLoc initLocExpr(p, e[1], a) - putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [rdCharLoc(a), size])) + putIntoDest(p, d, e, ropecg(p.module, "#cardSet($1, $2)", [addrLoc(p.config, a), size])) of mLtSet, mLeSet: getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i) # our counter initLocExpr(p, e[1], a) diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 103c8d343..e230985e0 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -10,10 +10,9 @@ # set handling type - NimSet = array[0..4*2048-1, uint8] + NimSet = array[0..8192-1, uint8] - -proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = +proc cardSetImpl(s: openArray[uint8], len: int): int {.inline.} = var i = 0 result = 0 when defined(x86) or defined(amd64): @@ -24,3 +23,6 @@ proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = while i < len: inc(result, countBits32(uint32(s[i]))) inc(i, 1) + +proc cardSet(s: NimSet, len: int): int {.compilerproc, inline.} = + result = cardSetImpl(s, len) diff --git a/tests/sets/t20997.nim b/tests/sets/t20997.nim new file mode 100644 index 000000000..b320eee1a --- /dev/null +++ b/tests/sets/t20997.nim @@ -0,0 +1,18 @@ +discard """ + joinable: false +""" + +{.passC: "-flto".} +{.passL: "-flto".} + +template f(n: int) = discard card(default(set[range[0 .. (1 shl n) - 1]])) +f( 7) +f( 8) +f( 9) +f(10) +f(11) +f(12) +f(13) +f(14) +f(15) +f(16) From 0ffc322cf615685ae4d528e26ddd13f6ea072684 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 23 Dec 2022 22:31:53 +0800 Subject: [PATCH 1498/1654] add the cpp target (#21164) The issue is related to cpp codegen, the previous test doesn't test cpp backend, which will join into the megatest. --- tests/cpp/t12946.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cpp/t12946.nim b/tests/cpp/t12946.nim index cf6bf5cfc..79cd56251 100644 --- a/tests/cpp/t12946.nim +++ b/tests/cpp/t12946.nim @@ -1,3 +1,7 @@ +discard """ + targets: "c cpp" +""" + import std/atomics type Futex = distinct Atomic[int32] From ac5a36c0a4be3de6565f2a071c04d1fd8f1b67da Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 23 Dec 2022 23:50:07 +0800 Subject: [PATCH 1499/1654] re and nre now link to regex and tinyre (#21161) --- lib/impure/nre.nim | 8 +++++--- lib/impure/re.nim | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index e9dd49df0..2c1b1deae 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -20,15 +20,17 @@ when defined(js): ## search the internet for a wide variety of third-party documentation and ## tools. ## -## **Note**: If you love `sequtils.toSeq` we have bad news for you. This -## library doesn't work with it due to documented compiler limitations. As -## a workaround, use this: +## .. warning:: If you love `sequtils.toSeq` we have bad news for you. This +## library doesn't work with it due to documented compiler limitations. As +## a workaround, use this: runnableExamples: # either `import std/nre except toSeq` or fully qualify `sequtils.toSeq`: import std/sequtils iterator iota(n: int): int = for i in 0.. Date: Sat, 24 Dec 2022 05:20:51 +1100 Subject: [PATCH 1500/1654] Make search results be relative to documentation root (#21145) Use Nim instead of JS for searching --- tools/dochack/dochack.nim | 48 ++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 4067d2ed4..0753248bc 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,5 +1,6 @@ import dom import fuzzysearch +import std/[jsfetch, asyncjs] proc setTheme(theme: cstring) {.exportc.} = @@ -252,21 +253,7 @@ proc escapeCString(x: var cstring) = proc dosearch(value: cstring): Element = if db.len == 0: - var stuff: Element - {.emit: """ - var request = new XMLHttpRequest(); - request.open("GET", document.getElementById("indexLink").href, false); - request.send(null); - - var doc = document.implementation.createHTMLDocument("theindex"); - doc.documentElement.innerHTML = request.responseText; - - `stuff` = doc.documentElement; - """.} - db = stuff.getElementsByClass"reference" - contents = @[] - for ahref in db: - contents.add ahref.getAttribute("data-doc-search-tag") + return let ul = tree("UL") result = tree("DIV") result.setClass"search_results" @@ -293,8 +280,28 @@ proc dosearch(value: cstring): Element = result.add tree("B", text"search results") result.add ul -var oldtoc: Element -var timer: Timeout +proc loadIndex() {.async.} = + ## Loads theindex.html to enable searching + let + indexURL = document.getElementById("indexLink").getAttribute("href") + # Get root of project documentation by cutting off theindex.html from index href + rootURL = ($indexURL)[0 ..< ^"theindex.html".len] + var resp = fetch(indexURL).await().text().await() + # Convert into element so we can use DOM functions to parse the html + var indexElem = document.createElement("div") + indexElem.innerHtml = resp + # Add items into the DB/contents + for href in indexElem.getElementsByClass("reference"): + # Make links be relative to project root instead of current page + href.setAttr("href", cstring(rootURL & $href.getAttribute("href"))) + db &= href + contents &= href.getAttribute("data-doc-search-tag") + + +var + oldtoc: Element + timer: Timeout + loadIndexFut: Future[void] = nil proc search*() {.exportc.} = proc wrapper() = @@ -307,7 +314,12 @@ proc search*() {.exportc.} = replaceById("tocRoot", results) elif not oldtoc.isNil: replaceById("tocRoot", oldtoc) - + # Start loading index as soon as user starts typing. + # Will only be loaded the once anyways + if loadIndexFut == nil: + loadIndexFut = loadIndex() + # Run wrapper once loaded so we don't miss the users query + discard loadIndexFut.then(wrapper) if timer != nil: clearTimeout(timer) timer = setTimeout(wrapper, 400) From 9323cb7b2a761543df9d875c05f4963c1a8b050f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 24 Dec 2022 17:37:32 +0800 Subject: [PATCH 1501/1654] enforce void for nkWhileStmt [backport: 2.0] (#21170) enforce void for nkWhileStmt --- compiler/semstmts.nim | 1 - tests/discard/tdiscardable.nim | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 356c2d63e..96883255c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -105,7 +105,6 @@ proc semWhile(c: PContext, n: PNode; flags: TExprFlags): PNode = result.typ = n[1].typ elif implicitlyDiscardable(n[1]): result[1].typ = c.enforceVoidContext - result.typ = c.enforceVoidContext proc semProc(c: PContext, n: PNode): PNode diff --git a/tests/discard/tdiscardable.nim b/tests/discard/tdiscardable.nim index b13130a13..69cb9f6a1 100644 --- a/tests/discard/tdiscardable.nim +++ b/tests/discard/tdiscardable.nim @@ -99,3 +99,14 @@ block: # bug #13583 let t = test doAssert t() == 12 +block: + proc bar(): string {.discardable.} = + "15" + + proc foo(): int = + while true: + raise newException(ValueError, "check") + 12 + + doAssertRaises(ValueError): + doAssert foo() == 12 From b08c50bb553364eec1e20ab788d990b63f4548e0 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 26 Dec 2022 13:52:40 +0800 Subject: [PATCH 1502/1654] sync some changes from release notes (#21173) --- changelogs/changelog_2_0_0.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index a899bb929..614d97d85 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -2,7 +2,7 @@ ## Changes affecting backward compatibility -- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response, it followed Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. +- `httpclient.contentLength` default to `-1` if the Content-Length header is not set in the response. It follows Apache HttpClient(Java), http(go) and .Net HttpWebResponse(C#) behavior. Previously it raised `ValueError`. - `addr` is now available for all addressable locations, `unsafeAddr` is now deprecated and an alias for `addr`. @@ -65,7 +65,7 @@ - `shallowCopy` and `shallow` are removed for ARC/ORC. Use `move` when possible or combine assignment and `sink` for optimization purposes. -- The `nimPreviewDotLikeOps` define is going to be removed or deprecated. +- The experimental `nimPreviewDotLikeOps` switch is going to be removed or deprecated because it didn't fullfill its promises. - The `{.this.}` pragma, deprecated since 0.19, has been removed. - `nil` literals can no longer be directly assigned to variables or fields of `distinct` pointer types. They must be converted instead. @@ -110,8 +110,6 @@ - `logging` will default to flushing all log level messages. To get the legacy behaviour of only flushing Error and Fatal messages, use `-d:nimV1LogFlushBehavior`. -- Object fields now support default values, see https://nim-lang.github.io/Nim/manual.html#types-default-values-for-object-fields for details. - - Redefining templates with the same signature was previously allowed to support certain macro code. To do this explicitly, the `{.redefine.}` pragma has been added. Note that this is only for templates. From f7c203fb6c89b5cef83c4f326aeb23ef8c4a2c40 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 26 Dec 2022 20:20:05 +0800 Subject: [PATCH 1503/1654] remove legacy code (#21134) * remove legacy code * fixes --- compiler/astalgo.nim | 3 - compiler/cgen.nim | 2 - compiler/condsyms.nim | 47 +-- compiler/depends.nim | 3 - compiler/renderer.nim | 7 +- compiler/sem.nim | 3 - compiler/sigmatch.nim | 2 - compiler/vm.nim | 5 - compiler/vmops.nim | 72 +++-- lib/core/macros.nim | 9 +- lib/pure/collections/lists.nim | 3 - lib/pure/hashes.nim | 40 +-- lib/pure/json.nim | 540 ++++++++++++++++---------------- lib/pure/nimprof.nim | 3 +- lib/std/assertions.nim | 2 - lib/std/jsonutils.nim | 18 +- lib/std/private/miscdollars.nim | 16 +- lib/system.nim | 35 +-- lib/system/alloc.nim | 10 +- lib/system/arithm.nim | 425 ------------------------- lib/system/arithmetics.nim | 140 +++------ lib/system/compilation.nim | 40 +-- lib/system/fatal.nim | 5 +- lib/system/inclrtl.nim | 3 +- lib/system/iterators.nim | 2 +- lib/system/jssys.nim | 22 -- lib/system/strmantle.nim | 7 +- 27 files changed, 432 insertions(+), 1032 deletions(-) delete mode 100644 lib/system/arithm.nim diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index e9ed1497a..4e09fab02 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -20,9 +20,6 @@ import strutils except addf when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimHasCursor): - {.pragma: cursor.} - proc hashNode*(p: RootRef): Hash proc treeToYaml*(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope # Convert a tree into its YAML representation; this is used by the diff --git a/compiler/cgen.nim b/compiler/cgen.nim index e85cfd773..f6457f1e0 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1929,8 +1929,6 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = injectG() diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 24c6d82d5..c05aaf10b 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -84,10 +84,24 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasSignatureHashInMacro") # deadcode defineSymbol("nimHasDefault") # deadcode defineSymbol("nimMacrosSizealignof") # deadcode + defineSymbol("nimNoZeroExtendMagic") # deadcode + defineSymbol("nimMacrosGetNodeId") # deadcode + defineSymbol("nimFixedForwardGeneric") # deadcode + defineSymbol("nimToOpenArrayCString") # deadcode + defineSymbol("nimHasUsed") # deadcode + defineSymbol("nimnomagic64") # deadcode + defineSymbol("nimNewShiftOps") # deadcode + defineSymbol("nimHasCursor") # deadcode + defineSymbol("nimAlignPragma") # deadcode + defineSymbol("nimHasExceptionsQuery") # deadcode + defineSymbol("nimHasIsNamedTuple") # deadcode + defineSymbol("nimHashOrdinalFixed") # deadcode + defineSymbol("nimHasSinkInference") # deadcode + defineSymbol("nimNewIntegerOps") # deadcode + defineSymbol("nimHasInvariant") # deadcode + + - # > 0.20.0 - defineSymbol("nimNoZeroExtendMagic") - defineSymbol("nimMacrosGetNodeId") for f in Feature: defineSymbol("nimHas" & $f) @@ -98,31 +112,18 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimFixedOwned") defineSymbol("nimHasStyleChecks") - defineSymbol("nimToOpenArrayCString") - defineSymbol("nimHasUsed") - defineSymbol("nimFixedForwardGeneric") - defineSymbol("nimnomagic64") - defineSymbol("nimNewShiftOps") - defineSymbol("nimHasCursor") - defineSymbol("nimAlignPragma") - defineSymbol("nimHasExceptionsQuery") - defineSymbol("nimHasIsNamedTuple") - defineSymbol("nimHashOrdinalFixed") when defined(nimHasLibFFI): # Renaming as we can't conflate input vs output define flags; e.g. this # will report the right thing regardless of whether user adds # `-d:nimHasLibFFI` in his user config. - defineSymbol("nimHasLibFFIEnabled") + defineSymbol("nimHasLibFFIEnabled") # deadcode - defineSymbol("nimHasSinkInference") - defineSymbol("nimNewIntegerOps") - defineSymbol("nimHasInvariant") - defineSymbol("nimHasStacktraceMsgs") + defineSymbol("nimHasStacktraceMsgs") # deadcode defineSymbol("nimDoesntTrackDefects") - defineSymbol("nimHasLentIterators") - defineSymbol("nimHasDeclaredMagic") - defineSymbol("nimHasStacktracesModule") + defineSymbol("nimHasLentIterators") # deadcode + defineSymbol("nimHasDeclaredMagic") # deadcode + defineSymbol("nimHasStacktracesModule") # deadcode defineSymbol("nimHasEffectTraitsModule") defineSymbol("nimHasCastPragmaBlocks") defineSymbol("nimHasDeclaredLocs") @@ -133,8 +134,8 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasCustomLiterals") defineSymbol("nimHasUnifiedTuple") defineSymbol("nimHasIterable") - defineSymbol("nimHasTypeofVoid") - defineSymbol("nimHasDragonBox") + defineSymbol("nimHasTypeofVoid") # deadcode + defineSymbol("nimHasDragonBox") # deadcode defineSymbol("nimHasHintAll") defineSymbol("nimHasTrace") defineSymbol("nimHasEffectsOf") diff --git a/compiler/depends.nim b/compiler/depends.nim index 93f34dc7c..59cdc0330 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -100,9 +100,6 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) = rope(project.splitFile.name), b.dotGraph], changeFileExt(project, "dot")) -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = var g: PGen new(g) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index cd0d83eb0..87a5e6580 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -9,10 +9,9 @@ # This module implements the renderer of the standard Nim representation. -when defined(nimHasUsed): - # 'import renderer' is so useful for debugging - # that Nim shouldn't produce a warning for that: - {.used.} +# 'import renderer' is so useful for debugging +# that Nim shouldn't produce a warning for that: +{.used.} import lexer, options, idents, strutils, ast, msgs, lineinfos diff --git a/compiler/sem.nim b/compiler/sem.nim index 1f8e217b3..f5283fa3d 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -405,9 +405,6 @@ proc semExprFlagDispatched(c: PContext, n: PNode, flags: TExprFlags; expectedTyp evaluated = evalAtCompileTime(c, result) if evaluated != nil: return evaluated -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - include hlo, seminst, semcall proc resetSemFlag(n: PNode) = diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 61a4395f3..d05503c00 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2661,8 +2661,6 @@ proc argtypeMatches*(c: PContext, f, a: PType, fromHlo = false): bool = # pattern templates do not allow for conversions except from int literal res != nil and m.convMatches == 0 and m.intConvMatches in [0, 256] -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; op: TTypeAttachedOp; col: int): PSym {.nosinks.} = diff --git a/compiler/vm.nim b/compiler/vm.nim index 61261c44c..d87fa33ba 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -32,8 +32,6 @@ const when hasFFI: import evalffi -when not defined(nimHasCursor): - {.pragma: cursor.} proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = if x != nil: @@ -533,9 +531,6 @@ template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool = else: false -when not defined(nimHasSinkInference): - {.pragma: nosinks.} - template takeAddress(reg, source) = reg.nodeAddr = addr source GC_ref source diff --git a/compiler/vmops.nim b/compiler/vmops.nim index d4a91b22d..bb943b8a0 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -141,41 +141,40 @@ proc staticWalkDirImpl(path: string, relative: bool): PNode = for k, f in walkDir(path, relative): result.add toLit((k, f)) -when defined(nimHasInvariant): - from std / compilesettings import SingleValueSetting, MultipleValueSetting +from std / compilesettings import SingleValueSetting, MultipleValueSetting - proc querySettingImpl(conf: ConfigRef, switch: BiggestInt): string = - {.push warning[Deprecated]:off.} - case SingleValueSetting(switch) - of arguments: result = conf.arguments - of outFile: result = conf.outFile.string - of outDir: result = conf.outDir.string - of nimcacheDir: result = conf.getNimcacheDir().string - of projectName: result = conf.projectName - of projectPath: result = conf.projectPath.string - of projectFull: result = conf.projectFull.string - of command: result = conf.command - of commandLine: result = conf.commandLine - of linkOptions: result = conf.linkOptions - of compileOptions: result = conf.compileOptions - of ccompilerPath: result = conf.cCompilerPath - of backend: result = $conf.backend - of libPath: result = conf.libpath.string - of gc: result = $conf.selectedGC - of mm: result = $conf.selectedGC - {.pop.} +proc querySettingImpl(conf: ConfigRef, switch: BiggestInt): string = + {.push warning[Deprecated]:off.} + case SingleValueSetting(switch) + of arguments: result = conf.arguments + of outFile: result = conf.outFile.string + of outDir: result = conf.outDir.string + of nimcacheDir: result = conf.getNimcacheDir().string + of projectName: result = conf.projectName + of projectPath: result = conf.projectPath.string + of projectFull: result = conf.projectFull.string + of command: result = conf.command + of commandLine: result = conf.commandLine + of linkOptions: result = conf.linkOptions + of compileOptions: result = conf.compileOptions + of ccompilerPath: result = conf.cCompilerPath + of backend: result = $conf.backend + of libPath: result = conf.libpath.string + of gc: result = $conf.selectedGC + of mm: result = $conf.selectedGC + {.pop.} - proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = - template copySeq(field: untyped): untyped = - for i in field: result.add i.string +proc querySettingSeqImpl(conf: ConfigRef, switch: BiggestInt): seq[string] = + template copySeq(field: untyped): untyped = + for i in field: result.add i.string - case MultipleValueSetting(switch) - of nimblePaths: copySeq(conf.nimblePaths) - of searchPaths: copySeq(conf.searchPaths) - of lazyPaths: copySeq(conf.lazyPaths) - of commandArgs: result = conf.commandArgs - of cincludes: copySeq(conf.cIncludes) - of clibs: copySeq(conf.cLibs) + case MultipleValueSetting(switch) + of nimblePaths: copySeq(conf.nimblePaths) + of searchPaths: copySeq(conf.searchPaths) + of lazyPaths: copySeq(conf.lazyPaths) + of commandArgs: result = conf.commandArgs + of cincludes: copySeq(conf.cIncludes) + of clibs: copySeq(conf.cLibs) proc stackTrace2(c: PCtx, msg: string, n: PNode) = stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info) @@ -257,11 +256,10 @@ proc registerAdditionalOps*(c: PCtx) = systemop getCurrentException registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) - when defined(nimHasInvariant): - registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) = - setResult(a, querySettingImpl(c.config, getInt(a, 0))) - registerCallback c, "stdlib.compilesettings.querySettingSeq", proc (a: VmArgs) = - setResult(a, querySettingSeqImpl(c.config, getInt(a, 0))) + registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) = + setResult(a, querySettingImpl(c.config, getInt(a, 0))) + registerCallback c, "stdlib.compilesettings.querySettingSeq", proc (a: VmArgs) = + setResult(a, querySettingSeqImpl(c.config, getInt(a, 0))) if defined(nimsuggest) or c.config.cmd == cmdCheck: discard "don't run staticExec for 'nim suggest'" diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 138ec47a0..bcc16038b 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1519,11 +1519,10 @@ proc boolVal*(n: NimNode): bool {.noSideEffect.} = if n.kind == nnkIntLit: n.intVal != 0 else: n == bindSym"true" # hacky solution for now -when defined(nimMacrosGetNodeId): - proc nodeID*(n: NimNode): int {.magic: "NodeId".} - ## Returns the id of `n`, when the compiler has been compiled - ## with the flag `-d:useNodeids`, otherwise returns `-1`. This - ## proc is for the purpose to debug the compiler only. +proc nodeID*(n: NimNode): int {.magic: "NodeId".} + ## Returns the id of `n`, when the compiler has been compiled + ## with the flag `-d:useNodeids`, otherwise returns `-1`. This + ## proc is for the purpose to debug the compiler only. macro expandMacros*(body: typed): untyped = ## Expands one level of macro - useful for debugging. diff --git a/lib/pure/collections/lists.nim b/lib/pure/collections/lists.nim index 829ec2ccb..a08c62964 100644 --- a/lib/pure/collections/lists.nim +++ b/lib/pure/collections/lists.nim @@ -62,9 +62,6 @@ import std/private/since when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimHasCursor): - {.pragma: cursor.} - type DoublyLinkedNodeObj*[T] = object ## A node of a doubly linked list. diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index e88210757..09e072812 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -380,16 +380,10 @@ proc hash*(x: string): Hash = runnableExamples: doAssert hash("abracadabra") != hash("AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - for c in x: - result = result !& ord(c) - result = !$result + when nimvm: + result = hashVmImpl(x, 0, high(x)) else: - when nimvm: - result = hashVmImpl(x, 0, high(x)) - else: - result = murmurHash(toOpenArrayByte(x, 0, high(x))) + result = murmurHash(toOpenArrayByte(x, 0, high(x))) proc hash*(x: cstring): Hash = ## Efficient hashing of null-terminated strings. @@ -398,22 +392,14 @@ proc hash*(x: cstring): Hash = doAssert hash(cstring"AbracadabrA") == hash("AbracadabrA") doAssert hash(cstring"abracadabra") != hash(cstring"AbracadabrA") - when not defined(nimToOpenArrayCString): - result = 0 - var i = 0 - while x[i] != '\0': - result = result !& ord(x[i]) - inc i - result = !$result + when nimvm: + hashVmImpl(x, 0, high(x)) else: - when nimvm: - hashVmImpl(x, 0, high(x)) + when not defined(js): + murmurHash(toOpenArrayByte(x, 0, x.high)) else: - when not defined(js) and defined(nimToOpenArrayCString): - murmurHash(toOpenArrayByte(x, 0, x.high)) - else: - let xx = $x - murmurHash(toOpenArrayByte(xx, 0, high(xx))) + let xx = $x + murmurHash(toOpenArrayByte(xx, 0, high(xx))) proc hash*(sBuf: string, sPos, ePos: int): Hash = ## Efficient hashing of a string buffer, from starting @@ -424,13 +410,7 @@ proc hash*(sBuf: string, sPos, ePos: int): Hash = var a = "abracadabra" doAssert hash(a, 0, 3) == hash(a, 7, 10) - when not defined(nimToOpenArrayCString): - result = 0 - for i in sPos..ePos: - result = result !& ord(sBuf[i]) - result = !$result - else: - murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) + murmurHash(toOpenArrayByte(sBuf, sPos, ePos)) proc hashIgnoreStyle*(x: string): Hash = ## Efficient hashing of strings; style is ignored. diff --git a/lib/pure/json.nim b/lib/pure/json.nim index d0b1a4051..78d6bd5bd 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1059,310 +1059,308 @@ template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind], ] raise newException(JsonKindError, msg) -when defined(nimFixedForwardGeneric): +macro isRefSkipDistinct*(arg: typed): untyped = + ## internal only, do not use + var impl = getTypeImpl(arg) + if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): + impl = getTypeImpl(impl[1]) + while impl.kind == nnkDistinctTy: + impl = getTypeImpl(impl[0]) + result = newLit(impl.kind == nnkRefTy) - macro isRefSkipDistinct*(arg: typed): untyped = - ## internal only, do not use - var impl = getTypeImpl(arg) - if impl.kind == nnkBracketExpr and impl[0].eqIdent("typeDesc"): - impl = getTypeImpl(impl[1]) - while impl.kind == nnkDistinctTy: - impl = getTypeImpl(impl[0]) - result = newLit(impl.kind == nnkRefTy) +# The following forward declarations don't work in older versions of Nim - # The following forward declarations don't work in older versions of Nim +# forward declare all initFromJson - # forward declare all initFromJson +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[S, T](dst: var array[S, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Table[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var OrderedTable[string, T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) +# initFromJson definitions - # initFromJson definitions +proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) + # since strings don't have a nil state anymore, this mapping of + # JNull to the default string is questionable. `none(string)` and + # `some("")` have the same potentional json value `JNull`. + if jsonNode.kind == JNull: + dst = "" + else: + dst = jsonNode.str - proc initFromJson(dst: var string; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString, JNull}, jsonPath) - # since strings don't have a nil state anymore, this mapping of - # JNull to the default string is questionable. `none(string)` and - # `some("")` have the same potentional json value `JNull`. - if jsonNode.kind == JNull: - dst = "" +proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JBool}, jsonPath) + dst = jsonNode.bval + +proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode == nil: + raise newException(KeyError, "key not found: " & jsonPath) + dst = jsonNode.copy + +proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = + when T is uint|uint64 or (not defined(js) and int.sizeof == 4): + verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) + case jsonNode.kind + of JString: + let x = parseBiggestUInt(jsonNode.str) + dst = cast[T](x) else: - dst = jsonNode.str + dst = T(jsonNode.num) + else: + verifyJsonKind(jsonNode, {JInt}, jsonPath) + dst = cast[T](jsonNode.num) - proc initFromJson(dst: var bool; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JBool}, jsonPath) - dst = jsonNode.bval - - proc initFromJson(dst: var JsonNode; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode == nil: - raise newException(KeyError, "key not found: " & jsonPath) - dst = jsonNode.copy - - proc initFromJson[T: SomeInteger](dst: var T; jsonNode: JsonNode, jsonPath: var string) = - when T is uint|uint64 or (not defined(js) and int.sizeof == 4): - verifyJsonKind(jsonNode, {JInt, JString}, jsonPath) - case jsonNode.kind - of JString: - let x = parseBiggestUInt(jsonNode.str) - dst = cast[T](x) - else: - dst = T(jsonNode.num) +proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode.kind == JString: + case jsonNode.str + of "nan": + let b = NaN + dst = T(b) + # dst = NaN # would fail some tests because range conversions would cause CT error + # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. + of "inf": + let b = Inf + dst = T(b) + of "-inf": + let b = -Inf + dst = T(b) + else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) + else: + verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) + if jsonNode.kind == JFloat: + dst = T(jsonNode.fnum) else: - verifyJsonKind(jsonNode, {JInt}, jsonPath) - dst = cast[T](jsonNode.num) + dst = T(jsonNode.num) - proc initFromJson[T: SomeFloat](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode.kind == JString: - case jsonNode.str - of "nan": - let b = NaN - dst = T(b) - # dst = NaN # would fail some tests because range conversions would cause CT error - # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. - of "inf": - let b = Inf - dst = T(b) - of "-inf": - let b = -Inf - dst = T(b) - else: raise newException(JsonKindError, "expected 'nan|inf|-inf', got " & jsonNode.str) +proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JString}, jsonPath) + dst = parseEnum[T](jsonNode.getStr) + +proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + dst.setLen jsonNode.len + let orignalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i], jsonNode[i], jsonPath) + jsonPath.setLen orignalJsonPathLen + +proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JArray}, jsonPath) + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initTable[string, T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = + dst = initOrderedTable[string,T]() + verifyJsonKind(jsonNode, {JObject}, jsonPath) + let originalJsonPathLen = jsonPath.len + for key in keys(jsonNode.fields): + jsonPath.add '.' + jsonPath.add key + initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) + jsonPath.setLen originalJsonPathLen + +proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = + verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) + if jsonNode.kind == JNull: + dst = nil + else: + dst = new(T) + initFromJson(dst[], jsonNode, jsonPath) + +proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = + if jsonNode != nil and jsonNode.kind != JNull: + when T is ref: + dst = some(new(T)) else: - verifyJsonKind(jsonNode, {JInt, JFloat}, jsonPath) - if jsonNode.kind == JFloat: - dst = T(jsonNode.fnum) - else: - dst = T(jsonNode.num) + dst = some(default(T)) + initFromJson(dst.get, jsonNode, jsonPath) - proc initFromJson[T: enum](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JString}, jsonPath) - dst = parseEnum[T](jsonNode.getStr) +macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = + let typInst = getTypeInst(dst) + let typImpl = getTypeImpl(dst) + let baseTyp = typImpl[0] - proc initFromJson[T](dst: var seq[T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - dst.setLen jsonNode.len - let orignalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i], jsonNode[i], jsonPath) - jsonPath.setLen orignalJsonPathLen - - proc initFromJson[S,T](dst: var array[S,T]; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JArray}, jsonPath) - let originalJsonPathLen = jsonPath.len - for i in 0 ..< jsonNode.len: - jsonPath.add '[' - jsonPath.addInt i - jsonPath.add ']' - initFromJson(dst[i.S], jsonNode[i], jsonPath) # `.S` for enum indexed arrays - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var Table[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initTable[string, T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var OrderedTable[string,T]; jsonNode: JsonNode; jsonPath: var string) = - dst = initOrderedTable[string,T]() - verifyJsonKind(jsonNode, {JObject}, jsonPath) - let originalJsonPathLen = jsonPath.len - for key in keys(jsonNode.fields): - jsonPath.add '.' - jsonPath.add key - initFromJson(mgetOrPut(dst, key, default(T)), jsonNode[key], jsonPath) - jsonPath.setLen originalJsonPathLen - - proc initFromJson[T](dst: var ref T; jsonNode: JsonNode; jsonPath: var string) = - verifyJsonKind(jsonNode, {JObject, JNull}, jsonPath) - if jsonNode.kind == JNull: - dst = nil + result = quote do: + when nimvm: + # workaround #12282 + var tmp: `baseTyp` + initFromJson( tmp, `jsonNode`, `jsonPath`) + `dst` = `typInst`(tmp) else: - dst = new(T) - initFromJson(dst[], jsonNode, jsonPath) + initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - proc initFromJson[T](dst: var Option[T]; jsonNode: JsonNode; jsonPath: var string) = - if jsonNode != nil and jsonNode.kind != JNull: - when T is ref: - dst = some(new(T)) - else: - dst = some(default(T)) - initFromJson(dst.get, jsonNode, jsonPath) +proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignDistinctImpl(dst, jsonNode, jsonPath) - macro assignDistinctImpl[T: distinct](dst: var T;jsonNode: JsonNode; jsonPath: var string) = - let typInst = getTypeInst(dst) - let typImpl = getTypeImpl(dst) - let baseTyp = typImpl[0] +proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = + if typeExpr.kind == nnkTupleConstr: + error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - result = quote do: +proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = + case typeNode.kind + of nnkEmpty: + discard + of nnkRecList, nnkTupleTy: + for it in typeNode: + foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + + of nnkIdentDefs: + typeNode.expectLen 3 + let fieldSym = typeNode[0] + let fieldNameLit = newLit(fieldSym.strVal) + let fieldPathLit = newLit("." & fieldSym.strVal) + let fieldType = typeNode[1] + + # Detecting incompatiple tuple types in `assignObjectImpl` only + # would be much cleaner, but the ast for tuple types does not + # contain usable type information. + detectIncompatibleType(fieldType, fieldSym) + + dst.add quote do: + jsonPath.add `fieldPathLit` when nimvm: - # workaround #12282 - var tmp: `baseTyp` - initFromJson( tmp, `jsonNode`, `jsonPath`) - `dst` = `typInst`(tmp) - else: - initFromJson( `baseTyp`(`dst`), `jsonNode`, `jsonPath`) - - proc initFromJson[T: distinct](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignDistinctImpl(dst, jsonNode, jsonPath) - - proc detectIncompatibleType(typeExpr, lineinfoNode: NimNode) = - if typeExpr.kind == nnkTupleConstr: - error("Use a named tuple instead of: " & typeExpr.repr, lineinfoNode) - - proc foldObjectBody(dst, typeNode, tmpSym, jsonNode, jsonPath, originalJsonPathLen: NimNode) = - case typeNode.kind - of nnkEmpty: - discard - of nnkRecList, nnkTupleTy: - for it in typeNode: - foldObjectBody(dst, it, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - - of nnkIdentDefs: - typeNode.expectLen 3 - let fieldSym = typeNode[0] - let fieldNameLit = newLit(fieldSym.strVal) - let fieldPathLit = newLit("." & fieldSym.strVal) - let fieldType = typeNode[1] - - # Detecting incompatiple tuple types in `assignObjectImpl` only - # would be much cleaner, but the ast for tuple types does not - # contain usable type information. - detectIncompatibleType(fieldType, fieldSym) - - dst.add quote do: - jsonPath.add `fieldPathLit` - when nimvm: - when isRefSkipDistinct(`tmpSym`.`fieldSym`): - # workaround #12489 - var tmp: `fieldType` - initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - `tmpSym`.`fieldSym` = tmp - else: - initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + when isRefSkipDistinct(`tmpSym`.`fieldSym`): + # workaround #12489 + var tmp: `fieldType` + initFromJson(tmp, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + `tmpSym`.`fieldSym` = tmp else: initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) - jsonPath.setLen `originalJsonPathLen` + else: + initFromJson(`tmpSym`.`fieldSym`, getOrDefault(`jsonNode`,`fieldNameLit`), `jsonPath`) + jsonPath.setLen `originalJsonPathLen` - of nnkRecCase: - let kindSym = typeNode[0][0] - let kindNameLit = newLit(kindSym.strVal) - let kindPathLit = newLit("." & kindSym.strVal) - let kindType = typeNode[0][1] - let kindOffsetLit = newLit(uint(getOffset(kindSym))) - dst.add quote do: - var kindTmp: `kindType` - jsonPath.add `kindPathLit` - initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) - jsonPath.setLen `originalJsonPathLen` - when defined js: + of nnkRecCase: + let kindSym = typeNode[0][0] + let kindNameLit = newLit(kindSym.strVal) + let kindPathLit = newLit("." & kindSym.strVal) + let kindType = typeNode[0][1] + let kindOffsetLit = newLit(uint(getOffset(kindSym))) + dst.add quote do: + var kindTmp: `kindType` + jsonPath.add `kindPathLit` + initFromJson(kindTmp, `jsonNode`[`kindNameLit`], `jsonPath`) + jsonPath.setLen `originalJsonPathLen` + when defined js: + `tmpSym`.`kindSym` = kindTmp + else: + when nimvm: `tmpSym`.`kindSym` = kindTmp else: - when nimvm: - `tmpSym`.`kindSym` = kindTmp - else: - # fuck it, assign kind field anyway - ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp - dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) - for i in 1 ..< typeNode.len: - foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # fuck it, assign kind field anyway + ((cast[ptr `kindType`](cast[uint](`tmpSym`.addr) + `kindOffsetLit`))[]) = kindTmp + dst.add nnkCaseStmt.newTree(nnkDotExpr.newTree(tmpSym, kindSym)) + for i in 1 ..< typeNode.len: + foldObjectBody(dst, typeNode[i], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - of nnkOfBranch, nnkElse: - let ofBranch = newNimNode(typeNode.kind) - for i in 0 ..< typeNode.len-1: - ofBranch.add copyNimTree(typeNode[i]) - let dstInner = newNimNode(nnkStmtListExpr) - foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) - # resOuter now contains the inner stmtList - ofBranch.add dstInner - dst[^1].expectKind nnkCaseStmt - dst[^1].add ofBranch + of nnkOfBranch, nnkElse: + let ofBranch = newNimNode(typeNode.kind) + for i in 0 ..< typeNode.len-1: + ofBranch.add copyNimTree(typeNode[i]) + let dstInner = newNimNode(nnkStmtListExpr) + foldObjectBody(dstInner, typeNode[^1], tmpSym, jsonNode, jsonPath, originalJsonPathLen) + # resOuter now contains the inner stmtList + ofBranch.add dstInner + dst[^1].expectKind nnkCaseStmt + dst[^1].add ofBranch - of nnkObjectTy: - typeNode[0].expectKind nnkEmpty - typeNode[1].expectKind {nnkEmpty, nnkOfInherit} - if typeNode[1].kind == nnkOfInherit: - let base = typeNode[1][0] - var impl = getTypeImpl(base) - while impl.kind in {nnkRefTy, nnkPtrTy}: - impl = getTypeImpl(impl[0]) - foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - let body = typeNode[2] - foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + of nnkObjectTy: + typeNode[0].expectKind nnkEmpty + typeNode[1].expectKind {nnkEmpty, nnkOfInherit} + if typeNode[1].kind == nnkOfInherit: + let base = typeNode[1][0] + var impl = getTypeImpl(base) + while impl.kind in {nnkRefTy, nnkPtrTy}: + impl = getTypeImpl(impl[0]) + foldObjectBody(dst, impl, tmpSym, jsonNode, jsonPath, originalJsonPathLen) + let body = typeNode[2] + foldObjectBody(dst, body, tmpSym, jsonNode, jsonPath, originalJsonPathLen) - else: - error("unhandled kind: " & $typeNode.kind, typeNode) + else: + error("unhandled kind: " & $typeNode.kind, typeNode) - macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - let typeSym = getTypeInst(dst) - let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") - result = newStmtList() - result.add quote do: - let `originalJsonPathLen` = len(`jsonPath`) - if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: - # both, `dst` and `typeSym` don't have good lineinfo. But nothing - # else is available here. - detectIncompatibleType(typeSym, dst) - foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) - else: - foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) +macro assignObjectImpl[T](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + let typeSym = getTypeInst(dst) + let originalJsonPathLen = genSym(nskLet, "originalJsonPathLen") + result = newStmtList() + result.add quote do: + let `originalJsonPathLen` = len(`jsonPath`) + if typeSym.kind in {nnkTupleTy, nnkTupleConstr}: + # both, `dst` and `typeSym` don't have good lineinfo. But nothing + # else is available here. + detectIncompatibleType(typeSym, dst) + foldObjectBody(result, typeSym, dst, jsonNode, jsonPath, originalJsonPathLen) + else: + foldObjectBody(result, typeSym.getTypeImpl, dst, jsonNode, jsonPath, originalJsonPathLen) - proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = - assignObjectImpl(dst, jsonNode, jsonPath) +proc initFromJson[T: object|tuple](dst: var T; jsonNode: JsonNode; jsonPath: var string) = + assignObjectImpl(dst, jsonNode, jsonPath) - proc to*[T](node: JsonNode, t: typedesc[T]): T = - ## `Unmarshals`:idx: the specified node into the object type specified. - ## - ## Known limitations: - ## - ## * Heterogeneous arrays are not supported. - ## * Sets in object variants are not supported. - ## * Not nil annotations are not supported. - ## - runnableExamples: - let jsonNode = parseJson(""" - { - "person": { - "name": "Nimmer", - "age": 21 - }, - "list": [1, 2, 3, 4] - } - """) +proc to*[T](node: JsonNode, t: typedesc[T]): T = + ## `Unmarshals`:idx: the specified node into the object type specified. + ## + ## Known limitations: + ## + ## * Heterogeneous arrays are not supported. + ## * Sets in object variants are not supported. + ## * Not nil annotations are not supported. + ## + runnableExamples: + let jsonNode = parseJson(""" + { + "person": { + "name": "Nimmer", + "age": 21 + }, + "list": [1, 2, 3, 4] + } + """) - type - Person = object - name: string - age: int + type + Person = object + name: string + age: int - Data = object - person: Person - list: seq[int] + Data = object + person: Person + list: seq[int] - var data = to(jsonNode, Data) - doAssert data.person.name == "Nimmer" - doAssert data.person.age == 21 - doAssert data.list == @[1, 2, 3, 4] + var data = to(jsonNode, Data) + doAssert data.person.name == "Nimmer" + doAssert data.person.age == 21 + doAssert data.list == @[1, 2, 3, 4] - var jsonPath = "" - initFromJson(result, node, jsonPath) + var jsonPath = "" + initFromJson(result, node, jsonPath) when false: import os diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index 7f493418e..9849e42db 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -15,8 +15,7 @@ when not defined(profiler) and not defined(memProfiler): {.error: "Profiling support is turned off! Enable profiling by passing `--profiler:on --stackTrace:on` to the compiler (see the Nim Compiler User Guide for more options).".} -when defined(nimHasUsed): - {.used.} +{.used.} # We don't want to profile the profiling code ... {.push profiler: off.} diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 5623ff8ef..229033c58 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -23,8 +23,6 @@ proc `$`(info: InstantiationInfo): string = # --------------------------------------------------------------------------- -when not defined(nimHasSinkInference): - {.pragma: nosinks.} proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = ## Raises an `AssertionDefect` with `msg`. diff --git a/lib/std/jsonutils.nim b/lib/std/jsonutils.nim index eec8dea7d..17d08e02e 100644 --- a/lib/std/jsonutils.nim +++ b/lib/std/jsonutils.nim @@ -37,22 +37,8 @@ from typetraits import OrdinalEnum, tupleLen when defined(nimPreviewSlimSystem): import std/assertions -when not defined(nimFixedForwardGeneric): - # xxx remove pending csources_v1 update >= 1.2.0 - proc to[T](node: JsonNode, t: typedesc[T]): T = - when T is string: node.getStr - elif T is bool: node.getBool - else: static: doAssert false, $T # support as needed (only needed during bootstrap) - proc isNamedTuple(T: typedesc): bool = # old implementation - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": return compiles(t.Field0) - else: return true - return false -else: - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} + +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} type Joptions* = object # xxx rename FromJsonOptions diff --git a/lib/std/private/miscdollars.nim b/lib/std/private/miscdollars.nim index 47b788ee9..06fda6fa1 100644 --- a/lib/std/private/miscdollars.nim +++ b/lib/std/private/miscdollars.nim @@ -13,21 +13,7 @@ template toLocation*(result: var string, file: string | cstring, line: int, col: addInt(result, col) result.add ")" -when defined(nimHasIsNamedTuple): - proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} -else: - # for bootstrap; remove after release 1.2 - proc isNamedTuple(T: typedesc): bool = - # Taken from typetraits. - when T isnot tuple: result = false - else: - var t: T - for name, _ in t.fieldPairs: - when name == "Field0": - return compiles(t.Field0) - else: - return true - return false +proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".} template tupleObjectDollar*[T: tuple | object](result: var string, x: T) = result = "(" diff --git a/lib/system.nim b/lib/system.nim index 1b6235fb3..e9c9ebfeb 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -85,17 +85,12 @@ when defined(nimHasIterable): type iterable*[T] {.magic: IterableType.} ## Represents an expression that yields `T` -when defined(nimHashOrdinalFixed): - type - Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, - ## bool, character, and enumeration types - ## as well as their subtypes. See also - ## `SomeOrdinal`. -else: - # bootstrap < 1.2.0 - type - OrdinalImpl[T] {.magic: Ordinal.} - Ordinal* = OrdinalImpl | uint | uint64 +type + Ordinal*[T] {.magic: Ordinal.} ## Generic ordinal type. Includes integer, + ## bool, character, and enumeration types + ## as well as their subtypes. See also + ## `SomeOrdinal`. + proc `addr`*[T](x: T): ptr T {.magic: "Addr", noSideEffect.} = ## Builtin `addr` operator for taking the address of a memory location. @@ -451,9 +446,7 @@ type ## However, objects that have no ancestor are also allowed. RootRef* = ref RootObj ## Reference to `RootObj`. -const NimStackTraceMsgs = - when defined(nimHasStacktraceMsgs): compileOption("stacktraceMsgs") - else: false +const NimStackTraceMsgs = compileOption("stacktraceMsgs") type RootEffect* {.compilerproc.} = object of RootObj ## \ @@ -2113,10 +2106,7 @@ when notJSnotNims: # we cannot compile this with stack tracing on # as it would recurse endlessly! - when defined(nimNewIntegerOps): - include "system/integerops" - else: - include "system/arithm" + include "system/integerops" {.pop.} @@ -2660,11 +2650,10 @@ when defined(nimconfig): when not defined(js): proc toOpenArray*[T](x: ptr UncheckedArray[T]; first, last: int): openArray[T] {. magic: "Slice".} - when defined(nimToOpenArrayCString): - proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. - magic: "Slice".} - proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. - magic: "Slice".} + proc toOpenArray*(x: cstring; first, last: int): openArray[char] {. + magic: "Slice".} + proc toOpenArrayByte*(x: cstring; first, last: int): openArray[byte] {. + magic: "Slice".} proc toOpenArray*[T](x: seq[T]; first, last: int): openArray[T] {. magic: "Slice".} diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 5d1548977..cee70f677 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -95,17 +95,11 @@ type acc: int # accumulator for small object allocation when defined(gcDestructors): sharedFreeList: ptr FreeCell # make no attempt at avoiding false sharing for now for this object field - when defined(nimAlignPragma): - data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory - else: - data: UncheckedArray[byte] + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory BigChunk = object of BaseChunk # not necessarily > PageSize! next, prev: PBigChunk # chunks of the same (or bigger) size - when defined(nimAlignPragma): - data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory - else: - data: UncheckedArray[byte] + data {.align: MemAlign.}: UncheckedArray[byte] # start of usable memory HeapLinks = object len: int diff --git a/lib/system/arithm.nim b/lib/system/arithm.nim deleted file mode 100644 index 158f40177..000000000 --- a/lib/system/arithm.nim +++ /dev/null @@ -1,425 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - - -# simple integer arithmetic with overflow checking - -proc raiseOverflow {.compilerproc, noinline.} = - # a single proc to reduce code size to a minimum - sysFatal(OverflowDefect, "over- or underflow") - -proc raiseDivByZero {.compilerproc, noinline.} = - sysFatal(DivByZeroDefect, "division by zero") - -when defined(builtinOverflow): - # Builtin compiler functions for improved performance - when sizeof(clong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddl_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubl_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smull_overflow", nodecl, nosideeffect.} - - elif sizeof(clonglong) == 8: - proc addInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_saddll_overflow", nodecl, nosideeffect.} - - proc subInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_ssubll_overflow", nodecl, nosideeffect.} - - proc mulInt64Overflow[T: int64|int](a, b: T, c: var T): bool {. - importc: "__builtin_smulll_overflow", nodecl, nosideeffect.} - - when sizeof(int) == 8: - proc addIntOverflow(a, b: int, c: var int): bool {.inline.} = - addInt64Overflow(a, b, c) - - proc subIntOverflow(a, b: int, c: var int): bool {.inline.} = - subInt64Overflow(a, b, c) - - proc mulIntOverflow(a, b: int, c: var int): bool {.inline.} = - mulInt64Overflow(a, b, c) - - elif sizeof(int) == 4 and sizeof(cint) == 4: - proc addIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_sadd_overflow", nodecl, nosideeffect.} - - proc subIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_ssub_overflow", nodecl, nosideeffect.} - - proc mulIntOverflow(a, b: int, c: var int): bool {. - importc: "__builtin_smul_overflow", nodecl, nosideeffect.} - - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - if addInt64Overflow(a, b, result): - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - if subInt64Overflow(a, b, result): - raiseOverflow() - - proc mulInt64(a, b: int64): int64 {.compilerproc, inline.} = - if mulInt64Overflow(a, b, result): - raiseOverflow() -else: - proc addInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= int64(0) or (result xor b) >= int64(0): - return result - raiseOverflow() - - proc subInt64(a, b: int64): int64 {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= int64(0) or (result xor not b) >= int64(0): - return result - raiseOverflow() - - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if they're - # approximately the same. Else, because the native int product is the only - # one that can lose catastrophic amounts of information, it's the native int - # product that must have overflowed. - # - proc mulInt64(a, b: int64): int64 {.compilerproc.} = - var - resAsFloat, floatProd: float64 - result = a *% b - floatProd = toBiggestFloat(a) # conversion - floatProd = floatProd * toBiggestFloat(b) - resAsFloat = toBiggestFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -proc negInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): return -a - raiseOverflow() - -proc absInt64(a: int64): int64 {.compilerproc, inline.} = - if a != low(int64): - if a >= 0: return a - else: return -a - raiseOverflow() - -proc divInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - if a == low(int64) and b == int64(-1): - raiseOverflow() - return a div b - -proc modInt64(a, b: int64): int64 {.compilerproc, inline.} = - if b == int64(0): - raiseDivByZero() - return a mod b - -proc absInt(a: int): int {.compilerproc, inline.} = - if a != low(int): - if a >= 0: return a - else: return -a - raiseOverflow() - -const - asmVersion = defined(i386) and (defined(vcc) or defined(wcc) or - defined(dmc) or defined(gcc) or defined(llvm_gcc)) - # my Version of Borland C++Builder does not have - # tasm32, which is needed for assembler blocks - # this is why Borland is not included in the 'when' - -when asmVersion and not defined(gcc) and not defined(llvm_gcc): - # assembler optimized versions for compilers that - # have an intel syntax assembler: - proc addInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - # a in eax, and b in edx - asm """ - mov eax, ecx - add eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc subInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - sub eax, edx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc negInt(a: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - neg eax - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - - proc divInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - ret - """ - - proc modInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - test edx, edx - jne L_NOT_ZERO - call `raiseDivByZero` - L_NOT_ZERO: - cmp ecx, 0x80000000 - jne L_DO_DIV - cmp edx, -1 - jne L_DO_DIV - call `raiseOverflow` - L_DO_DIV: - mov eax, ecx - mov ecx, edx - cdq - idiv ecx - mov eax, edx - ret - """ - - proc mulInt(a, b: int): int {.compilerproc, asmNoStackFrame.} = - asm """ - mov eax, ecx - mov ecx, edx - xor edx, edx - imul ecx - jno theEnd - call `raiseOverflow` - theEnd: - ret - """ - -elif false: # asmVersion and (defined(gcc) or defined(llvm_gcc)): - proc addInt(a, b: int): int {.compilerproc, inline.} = - # don't use a pure proc here! - asm """ - "addl %%ecx, %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - #".intel_syntax noprefix" - #/* Intel syntax here */ - #".att_syntax" - - proc subInt(a, b: int): int {.compilerproc, inline.} = - asm """ "subl %%ecx,%%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - """ - - proc mulInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "imull %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc negInt(a: int): int {.compilerproc, inline.} = - asm """ "negl %%eax\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`) - """ - - proc divInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - - proc modInt(a, b: int): int {.compilerproc, inline.} = - asm """ "xorl %%edx, %%edx\n" - "idivl %%ecx\n" - "jno 1\n" - "call _raiseOverflow\n" - "1: \n" - "movl %%edx, %%eax" - :"=a"(`result`) - :"a"(`a`), "c"(`b`) - :"%edx" - """ - -when not declared(addInt) and defined(builtinOverflow): - proc addInt(a, b: int): int {.compilerproc, inline.} = - if addIntOverflow(a, b, result): - raiseOverflow() - -when not declared(subInt) and defined(builtinOverflow): - proc subInt(a, b: int): int {.compilerproc, inline.} = - if subIntOverflow(a, b, result): - raiseOverflow() - -when not declared(mulInt) and defined(builtinOverflow): - proc mulInt(a, b: int): int {.compilerproc, inline.} = - if mulIntOverflow(a, b, result): - raiseOverflow() - -# Platform independent versions of the above (slower!) -when not declared(addInt): - proc addInt(a, b: int): int {.compilerproc, inline.} = - result = a +% b - if (result xor a) >= 0 or (result xor b) >= 0: - return result - raiseOverflow() - -when not declared(subInt): - proc subInt(a, b: int): int {.compilerproc, inline.} = - result = a -% b - if (result xor a) >= 0 or (result xor not b) >= 0: - return result - raiseOverflow() - -when not declared(negInt): - proc negInt(a: int): int {.compilerproc, inline.} = - if a != low(int): return -a - raiseOverflow() - -when not declared(divInt): - proc divInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - if a == low(int) and b == -1: - raiseOverflow() - return a div b - -when not declared(modInt): - proc modInt(a, b: int): int {.compilerproc, inline.} = - if b == 0: - raiseDivByZero() - return a mod b - -when not declared(mulInt): - # - # This code has been inspired by Python's source code. - # The native int product x*y is either exactly right or *way* off, being - # just the last n bits of the true product, where n is the number of bits - # in an int (the delivered product is the true product plus i*2**n for - # some integer i). - # - # The native float64 product x*y is subject to three - # rounding errors: on a sizeof(int)==8 box, each cast to double can lose - # info, and even on a sizeof(int)==4 box, the multiplication can lose info. - # But, unlike the native int product, it's not in *range* trouble: even - # if sizeof(int)==32 (256-bit ints), the product easily fits in the - # dynamic range of a float64. So the leading 50 (or so) bits of the float64 - # product are correct. - # - # We check these two ways against each other, and declare victory if - # they're approximately the same. Else, because the native int product is - # the only one that can lose catastrophic amounts of information, it's the - # native int product that must have overflowed. - # - proc mulInt(a, b: int): int {.compilerproc.} = - var - resAsFloat, floatProd: float - - result = a *% b - floatProd = toFloat(a) * toFloat(b) - resAsFloat = toFloat(result) - - # Fast path for normal case: small multiplicands, and no info - # is lost in either method. - if resAsFloat == floatProd: return result - - # Somebody somewhere lost info. Close enough, or way off? Note - # that a != 0 and b != 0 (else resAsFloat == floatProd == 0). - # The difference either is or isn't significant compared to the - # true value (of which floatProd is a good approximation). - - # abs(diff)/abs(prod) <= 1/32 iff - # 32 * abs(diff) <= abs(prod) -- 5 good bits is "close enough" - if 32.0 * abs(resAsFloat - floatProd) <= abs(floatProd): - return result - raiseOverflow() - -# We avoid setting the FPU control word here for compatibility with libraries -# written in other languages. - -proc raiseFloatInvalidOp {.compilerproc, noinline.} = - sysFatal(FloatInvalidOpDefect, "FPU operation caused a NaN result") - -proc nanCheck(x: float64) {.compilerproc, inline.} = - if x != x: raiseFloatInvalidOp() - -proc raiseFloatOverflow(x: float64) {.compilerproc, noinline.} = - if x > 0.0: - sysFatal(FloatOverflowDefect, "FPU operation caused an overflow") - else: - sysFatal(FloatUnderflowDefect, "FPU operations caused an underflow") - -proc infCheck(x: float64) {.compilerproc, inline.} = - if x != 0.0 and x*0.5 == x: raiseFloatOverflow(x) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index d05aaaa5b..5c21e56ef 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -405,105 +405,57 @@ proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) when not defined(nimPreviewSlimSystem): - when defined(nimNoZeroExtendMagic): - proc ze*(x: int8): int {.deprecated.} = - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint8](x))) + proc ze*(x: int8): int {.deprecated.} = + ## zero extends a smaller integer type to `int`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int](uint(cast[uint8](x))) - proc ze*(x: int16): int {.deprecated.} = - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int](uint(cast[uint16](x))) + proc ze*(x: int16): int {.deprecated.} = + ## zero extends a smaller integer type to `int`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int](uint(cast[uint16](x))) - proc ze64*(x: int8): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint8](x))) + proc ze64*(x: int8): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint8](x))) - proc ze64*(x: int16): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint16](x))) + proc ze64*(x: int16): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint16](x))) - proc ze64*(x: int32): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint32](x))) + proc ze64*(x: int32): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint32](x))) - proc ze64*(x: int): int64 {.deprecated.} = - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. Does nothing if the size of an `int` is the same as `int64`. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int64](uint64(cast[uint](x))) + proc ze64*(x: int): int64 {.deprecated.} = + ## zero extends a smaller integer type to `int64`. This treats `x` as + ## unsigned. Does nothing if the size of an `int` is the same as `int64`. + ## (This is the case on 64 bit processors.) + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int64](uint64(cast[uint](x))) - proc toU8*(x: int): int8 {.deprecated.} = - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int8](x) + proc toU8*(x: int): int8 {.deprecated.} = + ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits + ## from `x`. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int8](x) - proc toU16*(x: int): int16 {.deprecated.} = - ## treats `x` as unsigned and converts it to an `int16` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int16](x) + proc toU16*(x: int): int16 {.deprecated.} = + ## treats `x` as unsigned and converts it to an `int16` by taking the last + ## 16 bits from `x`. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int16](x) - proc toU32*(x: int64): int32 {.deprecated.} = - ## treats `x` as unsigned and converts it to an `int32` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - cast[int32](x) - - elif not defined(js): - proc ze*(x: int8): int {.magic: "Ze8ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze*(x: int16): int {.magic: "Ze16ToI", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int8): int64 {.magic: "Ze8ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int16): int64 {.magic: "Ze16ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int32): int64 {.magic: "Ze32ToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc ze64*(x: int): int64 {.magic: "ZeIToI64", noSideEffect, deprecated.} - ## zero extends a smaller integer type to `int64`. This treats `x` as - ## unsigned. Does nothing if the size of an `int` is the same as `int64`. - ## (This is the case on 64 bit processors.) - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU8*(x: int): int8 {.magic: "ToU8", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to a byte by taking the last 8 bits - ## from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU16*(x: int): int16 {.magic: "ToU16", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an `int16` by taking the last - ## 16 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. - - proc toU32*(x: int64): int32 {.magic: "ToU32", noSideEffect, deprecated.} - ## treats `x` as unsigned and converts it to an `int32` by taking the - ## last 32 bits from `x`. - ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + proc toU32*(x: int64): int32 {.deprecated.} = + ## treats `x` as unsigned and converts it to an `int32` by taking the + ## last 32 bits from `x`. + ## **Deprecated since version 0.19.9**: Use unsigned integers instead. + cast[int32](x) diff --git a/lib/system/compilation.nim b/lib/system/compilation.nim index 234e8cafe..c04990277 100644 --- a/lib/system/compilation.nim +++ b/lib/system/compilation.nim @@ -51,30 +51,24 @@ proc defined*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} ## * `compileOption <#compileOption,string,string>`_ for enum options ## * `define pragmas `_ -when defined(nimHasDeclaredMagic): - proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared. `x` has to be an identifier or a qualified identifier. - ## - ## This can be used to check whether a library provides a certain - ## feature or not: - ## ``` - ## when not declared(strutils.toUpper): - ## # provide our own toUpper proc here, because strutils is - ## # missing it. - ## ``` - ## - ## See also: - ## * `declaredInScope <#declaredInScope,untyped>`_ -else: - proc declared*(x: untyped): bool {.magic: "Defined", noSideEffect, compileTime.} +proc declared*(x: untyped): bool {.magic: "Declared", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared. `x` has to be an identifier or a qualified identifier. + ## + ## This can be used to check whether a library provides a certain + ## feature or not: + ## ``` + ## when not declared(strutils.toUpper): + ## # provide our own toUpper proc here, because strutils is + ## # missing it. + ## ``` + ## + ## See also: + ## * `declaredInScope <#declaredInScope,untyped>`_ -when defined(nimHasDeclaredMagic): - proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} - ## Special compile-time procedure that checks whether `x` is - ## declared in the current scope. `x` has to be an identifier. -else: - proc declaredInScope*(x: untyped): bool {.magic: "DefinedInScope", noSideEffect, compileTime.} +proc declaredInScope*(x: untyped): bool {.magic: "DeclaredInScope", noSideEffect, compileTime.} + ## Special compile-time procedure that checks whether `x` is + ## declared in the current scope. `x` has to be an identifier. proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.} = ## Special compile-time procedure that checks whether `x` can be compiled diff --git a/lib/system/fatal.nim b/lib/system/fatal.nim index a55af2dc3..6073ee779 100644 --- a/lib/system/fatal.nim +++ b/lib/system/fatal.nim @@ -9,10 +9,7 @@ {.push profiler: off.} -when defined(nimHasExceptionsQuery): - const gotoBasedExceptions = compileOption("exceptions", "goto") -else: - const gotoBasedExceptions = false +const gotoBasedExceptions = compileOption("exceptions", "goto") when hostOS == "standalone": include "$projectpath/panicoverride" diff --git a/lib/system/inclrtl.nim b/lib/system/inclrtl.nim index 42c85ad26..3bf0b9893 100644 --- a/lib/system/inclrtl.nim +++ b/lib/system/inclrtl.nim @@ -47,5 +47,4 @@ else: {.pragma: benign, gcsafe.} -when defined(nimHasSinkInference): - {.push sinkInference: on.} +{.push sinkInference: on.} diff --git a/lib/system/iterators.nim b/lib/system/iterators.nim index b16a605b7..4bd12680f 100644 --- a/lib/system/iterators.nim +++ b/lib/system/iterators.nim @@ -3,7 +3,7 @@ when defined(nimPreviewSlimSystem): import std/assertions -when defined(nimHasLentIterators) and not defined(nimNoLentIterators): +when not defined(nimNoLentIterators): template lent2(T): untyped = lent T else: template lent2(T): untyped = T diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 9dfa80877..4f64403fe 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -503,28 +503,6 @@ proc absInt(a: int): int {.compilerproc.} = proc absInt64(a: int64): int64 {.compilerproc.} = result = if a < 0: a*(-1) else: a -when not defined(nimNoZeroExtendMagic): - proc ze*(a: int): int {.compilerproc.} = - result = a - - proc ze64*(a: int64): int64 {.compilerproc.} = - result = a - - proc toU8*(a: int): int8 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU16*(a: int): int16 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - - proc toU32*(a: int64): int32 {.asmNoStackFrame, compilerproc.} = - asm """ - return `a`; - """ - proc nimMin(a, b: int): int {.compilerproc.} = return if a <= b: a else: b proc nimMax(a, b: int): int {.compilerproc.} = return if a >= b: a else: b diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index 0b2578280..f4733ed21 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -75,8 +75,8 @@ const 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22] -when defined(nimHasInvariant): - {.push staticBoundChecks: off.} + +{.push staticBoundChecks: off.} proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, ): int {.compilerproc.} = @@ -234,8 +234,7 @@ proc nimParseBiggestFloat(s: openArray[char], number: var BiggestFloat, t[ti-3] = ('0'.ord + absExponent mod 10).char number = c_strtod(cast[cstring](addr t), nil) -when defined(nimHasInvariant): - {.pop.} # staticBoundChecks +{.pop.} # staticBoundChecks proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" From 9b4516fbcb4ad9c5db1e6f563601855f0e9982a0 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Tue, 27 Dec 2022 01:13:05 -0300 Subject: [PATCH 1504/1654] fix in msg `doAssert()` to update grammar.txt (#21179) --- compiler/parser.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 00013c218..0c8edd2a8 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -42,7 +42,7 @@ when isMainModule or defined(nimTestGrammar): proc checkSameGrammar*() = doAssert sameFileContent(newGrammarText, "doc/grammar.txt"), - "execute 'nim r compiler.nim' to keep grammar.txt up-to-date" + "execute 'nim r compiler/parser.nim' to keep grammar.txt up-to-date" else: writeGrammarFile("doc/grammar.txt") import ".." / tools / grammar_nanny From f232329bb12fc9d93fe1e09769e291eae4e2953d Mon Sep 17 00:00:00 2001 From: Bung Date: Tue, 27 Dec 2022 19:07:15 +0800 Subject: [PATCH 1505/1654] add test case for #20289 (#21180) --- tests/misc/t20289.nim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/misc/t20289.nim diff --git a/tests/misc/t20289.nim b/tests/misc/t20289.nim new file mode 100644 index 000000000..5a0a269f0 --- /dev/null +++ b/tests/misc/t20289.nim @@ -0,0 +1,15 @@ +discard """ + action: reject +""" + +type E[T] = object + v: T + +template j[T](R: type E[T], x: untyped): R = R(v: x) +template d[T](O: type E, v: T): E[T] = E[T].j(v) + +proc w[T](): E[T] = + template r(k: int): auto = default(T) + E.d r + +discard w[int]() From 82d80e6e2cb09652e15d8ec81621bba734872aaa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 27 Dec 2022 14:30:26 +0100 Subject: [PATCH 1506/1654] atlas: minor doc improvements (#21183) * closes #20808 * atlas: better docs --- tools/atlas/atlas.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/atlas/atlas.md b/tools/atlas/atlas.md index a36817dc5..ee770cd62 100644 --- a/tools/atlas/atlas.md +++ b/tools/atlas/atlas.md @@ -28,7 +28,7 @@ a `nim.cfg` file for the compiler. For example: ``` The version selection is deterministic, it picks up the *minimum* required -version. Thanks to this design, lock files are not required. +version. Thanks to this design, lock files are much less important. ## Dependencies @@ -75,3 +75,7 @@ then `clone ` is performed. Search the package index `packages.json` for a package that the given terms in its description (or name or list of tags). + +### Install + +Use the .nimble file to setup the project's dependencies. From 56840931355fdf8ce12a59ad085d90b04404f831 Mon Sep 17 00:00:00 2001 From: metagn Date: Tue, 27 Dec 2022 23:24:00 +0300 Subject: [PATCH 1507/1654] fix for bad error message with const in case statement (#21182) * preliminary fix for bad error message with const * add test case * fix tmatrixconcept and tmatrixlib --- compiler/semfold.nim | 4 ++++ tests/errmsgs/tconstinfo.nim | 7 +++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/errmsgs/tconstinfo.nim diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 7c57eb370..7f966c6fd 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -569,6 +569,8 @@ proc foldDefine(m, s: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode else: result = copyTree(s.astdef) # unreachable else: result = copyTree(s.astdef) + if result != nil: + result.info = n.info proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode = result = nil @@ -593,6 +595,8 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode result = foldDefine(m, s, n, idgen, g) else: result = copyTree(s.astdef) + if result != nil: + result.info = n.info of skProc, skFunc, skMethod: result = n of skParam: diff --git a/tests/errmsgs/tconstinfo.nim b/tests/errmsgs/tconstinfo.nim new file mode 100644 index 000000000..170972b9f --- /dev/null +++ b/tests/errmsgs/tconstinfo.nim @@ -0,0 +1,7 @@ +# https://forum.nim-lang.org/t/9762 + +const foo = "abc" +case 'a' +of foo: echo "should error" #[tt.Error + ^ type mismatch: got but expected 'char']# +else: discard From 9e35631191ad67994bb45cfad4d9452f790534e0 Mon Sep 17 00:00:00 2001 From: Phil Krylov Date: Wed, 28 Dec 2022 03:26:52 +0100 Subject: [PATCH 1508/1654] std/encodings: Fix open() documentation in regard to exception raised (#21187) --- lib/pure/encodings.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 26b6e1f22..0d7e878e5 100644 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -339,7 +339,7 @@ proc getCurrentEncoding*(uiApp = false): string = proc open*(destEncoding = "UTF-8", srcEncoding = "CP1252"): EncodingConverter = ## Opens a converter that can convert from `srcEncoding` to `destEncoding`. - ## Raises `IOError` if it cannot fulfill the request. + ## Raises `EncodingError` if it cannot fulfill the request. when not defined(windows): result = iconvOpen(destEncoding, srcEncoding) if result == nil: From 9efa56a8bbacab2075a4c2fe4c5424d57b6eab46 Mon Sep 17 00:00:00 2001 From: rockcavera Date: Tue, 27 Dec 2022 23:40:17 -0300 Subject: [PATCH 1509/1654] [backport: 2.0] prevents the jsonscript command from exceeding the maximum length of a command line during linking (#21186) --- compiler/extccomp.nim | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 66de46556..6515f6968 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -851,6 +851,17 @@ proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = else: result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name) +proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) = + # Prevent linkcmd from exceeding the maximum command line length. + # Windows's command line limit is about 8K (8191 characters) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + const MaxCmdLen = when defined(windows): 8_000 else: 32_000 + if linkCmd.len > MaxCmdLen: + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) + proc callCCompiler*(conf: ConfigRef) = var linkCmd: string @@ -927,14 +938,7 @@ proc callCCompiler*(conf: ConfigRef) = linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true) extraCmds = getExtraCmds(conf, mainOutput) if optCompileOnly notin conf.globalOptions: - const MaxCmdLen = when defined(windows): 8_000 else: 32_000 - if linkCmd.len > MaxCmdLen: - # Windows's command line limit is about 8K (don't laugh...) so C compilers on - # Windows support a feature where the command line can be passed via ``@linkcmd`` - # to them. - linkViaResponseFile(conf, linkCmd) - else: - execLinkCmd(conf, linkCmd) + preventLinkCmdMaxCmdLen(conf, linkCmd) for cmd in extraCmds: execExternalProgram(conf, cmd, hintExecuting) else: @@ -1035,7 +1039,7 @@ proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = cmds.add cmd prettyCmds.add displayProgressCC(conf, name, cmd) execCmdsInParallel(conf, cmds, prettyCb) - execLinkCmd(conf, bcache.linkcmd) + preventLinkCmdMaxCmdLen(conf, bcache.linkcmd) for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = From 3d5dbf8f96a5106a3b817a45a926303bc5623786 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Wed, 28 Dec 2022 17:35:11 +1100 Subject: [PATCH 1510/1654] Fix nimsuggest not suggesting fields in when theres static parameters (#21189) Don't check against empty arguments Add test case --- compiler/sigmatch.nim | 2 +- nimsuggest/tests/t21185.nim | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 nimsuggest/tests/t21185.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index d05503c00..68068cd26 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2055,7 +2055,7 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType, a.n == nil and tfGenericTypeParam notin a.flags: return newNodeIT(nkType, argOrig.info, makeTypeFromExpr(c, arg)) - else: + elif arg.kind != nkEmpty: var evaluated = c.semTryConstExpr(c, arg) if evaluated != nil: # Don't build the type in-place because `evaluated` and `arg` may point diff --git a/nimsuggest/tests/t21185.nim b/nimsuggest/tests/t21185.nim new file mode 100644 index 000000000..bf5a0e3cc --- /dev/null +++ b/nimsuggest/tests/t21185.nim @@ -0,0 +1,18 @@ + +# Reduced case of 21185. Issue was first parameter being static +proc foo(x: static[int]) = discard + +type + Person = object + name: string + age: int + +let p = Person() +p.#[!]# + +discard """ +$nimsuggest --tester --v3 --maxresults:2 $file +>sug $1 +sug;;skField;;age;;int;;$file;;8;;4;;"";;100;;None +sug;;skField;;name;;string;;$file;;7;;4;;"";;100;;None +""" From 7a74c2dc3a82488671237555faa95fb38ef31bd5 Mon Sep 17 00:00:00 2001 From: Bung Date: Wed, 28 Dec 2022 15:28:48 +0800 Subject: [PATCH 1511/1654] fix #14667 (#21190) --- compiler/ccgexprs.nim | 4 ++++ compiler/cgendata.nim | 1 + tests/misc/t14667.nim | 12 ++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/misc/t14667.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index d7ef024b1..e2885a294 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3200,6 +3200,8 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Rope) = result.add "}" of tyTuple: result.add "{" + if p.vccAndC and t.len == 0: + result.add "0" for i in 0.. 0: result.add ", " getDefaultValue(p, t[i], info, result) @@ -3338,6 +3340,8 @@ proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Rope) = proc genConstTuple(p: BProc, n: PNode; isConst: bool; tup: PType; result: var Rope) = result.add "{" + if p.vccAndC and n.len == 0: + result.add "0" for i in 0.. 0: result.add ",\n" diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 2daa8da7c..f75598652 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -172,6 +172,7 @@ type template config*(m: BModule): ConfigRef = m.g.config template config*(p: BProc): ConfigRef = p.module.g.config +template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC proc includeHeader*(this: BModule; header: string) = if not this.headerFiles.contains header: diff --git a/tests/misc/t14667.nim b/tests/misc/t14667.nim new file mode 100644 index 000000000..3034e2841 --- /dev/null +++ b/tests/misc/t14667.nim @@ -0,0 +1,12 @@ +discard """ + matrix: "--cc:vcc" + disabled: "linux" + disabled: "bsd" + disabled: "osx" + disabled: "unix" + disabled: "posix" +""" + +type A = tuple +discard () +discard default(A) From 4b63ac4b87c54a1b31d60b7555e2c60d232817c8 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 28 Dec 2022 23:23:37 +0800 Subject: [PATCH 1512/1654] fixes #21171; dynamic acyclic refs need to use dyn decRef (#21184) * fixes #21171; dyn destructors for acyclic inherited refs * add a test * Update compiler/liftdestructors.nim --- compiler/liftdestructors.nim | 7 ++++ lib/system/orc.nim | 13 ++++++ tests/arc/t21184.nim | 77 ++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 tests/arc/t21184.nim diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 5174a908f..2b27a58fc 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -609,6 +609,11 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, elemType, c.info, c.idgen) let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType) + let isInheritableAcyclicRef = c.g.config.selectedGC == gcOrc and + (not isPureObject(elemType)) and + tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags + # dynamic Acyclic refs need to use dyn decRef + let tmp = if isCyclic and c.kind in {attachedAsgn, attachedSink}: declareTempOf(c, body, x) @@ -632,6 +637,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo) else: cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp) + elif isInheritableAcyclicRef: + cond = callCodegenProc(c.g, "nimDecRefIsLastDyn", c.info, x) else: cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x) cond.typ = getSysType(c.g, x.info, tyBool) diff --git a/lib/system/orc.nim b/lib/system/orc.nim index 83b983ee1..a56a0c057 100644 --- a/lib/system/orc.nim +++ b/lib/system/orc.nim @@ -485,6 +485,19 @@ proc nimDecRefIsLastCyclicDyn(p: pointer): bool {.compilerRtl, inl.} = #if cell.color == colPurple: rememberCycle(result, cell, cast[ptr PNimTypeV2](p)[]) +proc nimDecRefIsLastDyn(p: pointer): bool {.compilerRtl, inl.} = + if p != nil: + var cell = head(p) + if (cell.rc and not rcMask) == 0: + result = true + #cprintf("[DESTROY] %p\n", p) + else: + dec cell.rc, rcIncrement + #if cell.color == colPurple: + if result: + if cell.rootIdx > 0: + unregisterCycle(cell) + proc nimDecRefIsLastCyclicStatic(p: pointer; desc: PNimTypeV2): bool {.compilerRtl, inl.} = if p != nil: var cell = head(p) diff --git a/tests/arc/t21184.nim b/tests/arc/t21184.nim new file mode 100644 index 000000000..91d0c42c7 --- /dev/null +++ b/tests/arc/t21184.nim @@ -0,0 +1,77 @@ +discard """ + matrix: "--mm:orc" +""" + +import std/[with] + +type + Node* {.acyclic.} = ref object of RootObj + name: string + data: pointer + children: seq[Node] + TextNode = ref object of Node + text: string + +proc fakeEcho(s: string) = + if s.len < 0: + echo s + +proc newNode[T: Node](parent: Node): T = + new result + result.data = alloc0(250) + parent.children.add(result) + +proc newRootNode(): Node = + new result + result.data = alloc0(250) + +method printNode(node: Node) {.base.} = + fakeEcho node.name + +method printNode(node: TextNode) = + procCall printNode(Node(node)) + fakeEcho node.text + +proc printChildren(node: Node) = + for child in node.children: + child.printNode() + printChildren(child) + +proc free(node: Node) = + for child in node.children: + free(child) + dealloc(node.data) + +template node(parent: Node, body: untyped): untyped = + var node = newNode[Node](parent) + with node: + body + +proc textNode(parent: Node, text: string) = + var node = newNode[TextNode](parent) + node.text = text + +template withRootNode(body: untyped): untyped = + var root = newRootNode() + root.name = "root" + with root: + body + root.printNode() + printChildren(root) + root.free() + +proc doTest() = + withRootNode: + node: + name = "child1" + node: + name = "child2" + node: + name = "child3" + textNode "Hello, world!" + + +# bug #21171 +if isMainModule: + for i in 0..100000: + doTest() From 646932b3f316d6b8e3ceb6b94b9d01232d5cdb33 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 29 Dec 2022 03:41:27 +0800 Subject: [PATCH 1513/1654] fixes warnings when building csources (#21194) * replace `symbolfiles` with `incremental`; fixes warnings when build csources * fixes self conversion warnings --- tools/niminst/niminst.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index bc763a56f..81b46653d 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -168,11 +168,11 @@ proc parseCmdLine(c: var ConfigData) = next(p) var kind = p.kind var key = p.key - var val = p.val.string + var val = p.val case kind of cmdArgument: if c.actions == {}: - for a in split(normalize(key.string), {';', ','}): + for a in split(normalize(key), {';', ','}): case a of "csource": incl(c.actions, actionCSource) of "scripts": incl(c.actions, actionScripts) @@ -183,11 +183,11 @@ proc parseCmdLine(c: var ConfigData) = of "deb": incl(c.actions, actionDeb) else: quit(Usage) else: - c.infile = addFileExt(key.string, "ini") - c.nimArgs = cmdLineRest(p).string + c.infile = addFileExt(key, "ini") + c.nimArgs = cmdLineRest(p) break of cmdLongOption, cmdShortOption: - case normalize(key.string) + case normalize(key) of "help", "h": stdout.write(Usage) quit(0) @@ -544,7 +544,7 @@ proc srcdist(c: var ConfigData) = var dir = getOutputDir(c) / buildDir(osA, cpuA) if dirExists(dir): removeDir(dir) createDir(dir) - var cmd = ("$# compile -f --symbolfiles:off --compileonly " & + var cmd = ("$# compile -f --incremental:off --compileonly " & "--gen_mapping --cc:gcc --skipUserCfg" & " --os:$# --cpu:$# $# $#") % [findNim(), osname, cpuname, c.nimArgs, c.mainfile] From 761c5a0830a18cb0db721d27aff53a07ca30b1c3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 29 Dec 2022 03:54:15 +0800 Subject: [PATCH 1514/1654] less verbose type mismatch messages (#21191) * less verbose type mismatch messages * Update compiler/types.nim * fixes i386 * fixes i386 --- changelogs/changelog_2_0_0.md | 2 ++ compiler/options.nim | 1 + compiler/semcall.nim | 44 +++++++++++++++++++------ compiler/sigmatch.nim | 40 ++++++++++++---------- tests/config.nims | 1 + tests/errmsgs/tconcisetypemismatch.nim | 23 +++++++++++++ tests/errmsgs/tconcisetypemismatch.nims | 21 ++++++++++++ 7 files changed, 105 insertions(+), 27 deletions(-) create mode 100644 tests/errmsgs/tconcisetypemismatch.nim create mode 100644 tests/errmsgs/tconcisetypemismatch.nims diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 614d97d85..29d906d62 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -135,6 +135,8 @@ - The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection. +- - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. + ## Standard library additions and changes [//]: # "Changes:" diff --git a/compiler/options.nim b/compiler/options.nim index 7e4f6898f..ded084ae4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -231,6 +231,7 @@ type ## are not anymore. laxEffects ## Lax effects system prior to Nim 2.0. + verboseTypeMismatch SymbolFilesOption* = enum disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 31b490781..2147a9645 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -10,7 +10,8 @@ ## This module implements semantic checking for calls. # included from sem.nim -from algorithm import sort +from std/algorithm import sort + proc sameMethodDispatcher(a, b: PSym): bool = result = false @@ -192,7 +193,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): # argument in order to remove plenty of candidates. This is # comparable to what C# does and C# is doing fine. var filterOnlyFirst = false - if optShowAllMismatches notin c.config.globalOptions: + if optShowAllMismatches notin c.config.globalOptions and verboseTypeMismatch in c.config.legacyFeatures: for err in errors: if err.firstMismatch.arg > 1: filterOnlyFirst = true @@ -208,6 +209,10 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): if filterOnlyFirst and err.firstMismatch.arg == 1: inc skipped continue + + if verboseTypeMismatch notin c.config.legacyFeatures: + candidates.add "[" & $err.firstMismatch.arg & "] " + if err.sym.kind in routineKinds and err.sym.ast != nil: candidates.add(renderTree(err.sym.ast, {renderNoBody, renderNoComments, renderNoPragmas})) @@ -217,7 +222,7 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): candidates.add("\n") let nArg = if err.firstMismatch.arg < n.len: n[err.firstMismatch.arg] else: nil let nameParam = if err.firstMismatch.formal != nil: err.firstMismatch.formal.name.s else: "" - if n.len > 1: + if n.len > 1 and verboseTypeMismatch in c.config.legacyFeatures: candidates.add(" first type mismatch at position: " & $err.firstMismatch.arg) # candidates.add "\n reason: " & $err.firstMismatch.kind # for debugging case err.firstMismatch.kind @@ -274,11 +279,28 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors): const errTypeMismatch = "type mismatch: got <" errButExpected = "but expected one of:" + errExpectedPosition = "Expected one of (first mismatch at position [#]):" errUndeclaredField = "undeclared field: '$1'" errUndeclaredRoutine = "attempting to call undeclared routine: '$1'" errBadRoutine = "attempting to call routine: '$1'$2" errAmbiguousCallXYZ = "ambiguous call; both $1 and $2 match for: $3" +proc describeParamList(c: PContext, n: PNode, startIdx = 1; prefer = preferName): string = + result = "Expression: " & $n + for i in startIdx..') + if candidates != "": + result.add("\n" & errButExpected & "\n" & candidates) + localError(c.config, n.info, result & "\nexpression: " & $n) + proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = # Gives a detailed error message; this is separated from semOverloadedCall, # as semOverloadedCall is already pretty slow (and we need this information @@ -306,13 +328,15 @@ proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = localError(c.config, n.info, "expression '$1' cannot be called" % n[0].renderTree) return - let (prefer, candidates) = presentFailedCandidates(c, n, errors) - var result = errTypeMismatch - result.add(describeArgs(c, n, 1, prefer)) - result.add('>') - if candidates != "": - result.add("\n" & errButExpected & "\n" & candidates) - localError(c.config, n.info, result & "\nexpression: " & $n) + if verboseTypeMismatch in c.config.legacyFeatures: + legacynotFoundError(c, n, errors) + else: + let (prefer, candidates) = presentFailedCandidates(c, n, errors) + var result = "type mismatch\n" + result.add describeParamList(c, n, 1, prefer) + if candidates != "": + result.add("\n" & errExpectedPosition & "\n" & candidates) + localError(c.config, n.info, result) proc bracketNotFoundError(c: PContext; n: PNode) = var errors: CandidateErrors = @[] diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 68068cd26..75e3e5428 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -322,26 +322,32 @@ proc argTypeToString(arg: PNode; prefer: TPreferedDesc): string = else: result = arg.typ.typeToString(prefer) +template describeArgImpl(c: PContext, n: PNode, i: int, startIdx = 1; prefer = preferName) = + var arg = n[i] + if n[i].kind == nkExprEqExpr: + result.add renderTree(n[i][0]) + result.add ": " + if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo}: + # XXX we really need to 'tryExpr' here! + arg = c.semOperand(c, n[i][1]) + n[i].typ = arg.typ + n[i][1] = arg + else: + if arg.typ.isNil and arg.kind notin {nkStmtList, nkDo, nkElse, + nkOfBranch, nkElifBranch, + nkExceptBranch}: + arg = c.semOperand(c, n[i]) + n[i] = arg + if arg.typ != nil and arg.typ.kind == tyError: return + result.add argTypeToString(arg, prefer) + +proc describeArg*(c: PContext, n: PNode, i: int, startIdx = 1; prefer = preferName): string = + describeArgImpl(c, n, i, startIdx, prefer) + proc describeArgs*(c: PContext, n: PNode, startIdx = 1; prefer = preferName): string = result = "" for i in startIdx..= (1,5,1): + # to make it easier to test against older nim versions, (best effort only) + switch("filenames", "legacyRelProj") + switch("spellSuggest", "0") + +# for std/unittest +switch("define", "nimUnittestOutputLevel:PRINT_FAILURES") +switch("define", "nimUnittestColor:off") + +hint("Processing", off) From e602ebeb66986e3de65324c81253513b139c7f0e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 29 Dec 2022 15:29:40 +0800 Subject: [PATCH 1515/1654] fixes #19997; ships changelogs directory for a release (#21200) fixes #19997; ship changelogs directory for a release --- compiler/installer.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/installer.ini b/compiler/installer.ini index 094b92a68..1e57455c6 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -70,6 +70,7 @@ Files: "nimpretty" Files: "testament" Files: "nimsuggest" Files: "nimsuggest/tests/*.nim" +Files: "changelogs/*.md" [Lib] Files: "lib" From 705da9d452d19536689a32a8d4378bcce2ec320a Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 30 Dec 2022 00:06:57 +0800 Subject: [PATCH 1516/1654] fixes #14444; add `genLineDir` before assignment (#21201) * fixes #14444; add `genLineDir` before raises * add a test case * fixes differently --- compiler/ccgstmts.nim | 1 + tests/errmsgs/t14444.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/errmsgs/t14444.nim diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index fbdb9a347..9d6640fe2 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1620,6 +1620,7 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) = initLoc(a, locNone, le, OnUnknown) a.flags.incl(lfEnforceDeref) a.flags.incl(lfPrepareForMutation) + genLineDir(p, le) # it can be a nkBracketExpr, which may raise expr(p, le, a) a.flags.excl(lfPrepareForMutation) if fastAsgn: incl(a.flags, lfNoDeepCopy) diff --git a/tests/errmsgs/t14444.nim b/tests/errmsgs/t14444.nim new file mode 100644 index 000000000..143b4542e --- /dev/null +++ b/tests/errmsgs/t14444.nim @@ -0,0 +1,14 @@ +discard """ + matrix: "--hints:off" + exitcode: "1" + output: ''' +t14444.nim(13) t14444 +fatal.nim(51) sysFatal +Error: unhandled exception: index out of bounds, the container is empty [IndexDefect] +''' +""" + +when true: # bug #14444 + var i: string + i[10] = 'j' + echo i \ No newline at end of file From ebd1c678be5e0acee4fca67e1b7060234821ccf6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 30 Dec 2022 03:15:08 +0800 Subject: [PATCH 1517/1654] fixes #21195; `std/assertions` continue to use `sysFatal` when `nimPreviewSlimSystem` is not defined (#21196) * fixes #21195; `std/assertions` continue to use `sysFatal` * try includes * make `std/assertions` self-contained * fixes tests * fixes tests --- lib/std/assertions.nim | 9 ++++++++- lib/system.nim | 28 +++------------------------- lib/system/rawquits.nim | 27 +++++++++++++++++++++++++++ tests/assert/panicoverride.nim | 15 +++++++++++++++ tests/assert/t21195.nim | 6 ++++++ tests/assert/tassert_c.nim | 4 ++-- tests/errmsgs/t9768.nim | 4 ++-- 7 files changed, 63 insertions(+), 30 deletions(-) create mode 100644 lib/system/rawquits.nim create mode 100644 tests/assert/panicoverride.nim create mode 100644 tests/assert/t21195.nim diff --git a/lib/std/assertions.nim b/lib/std/assertions.nim index 229033c58..a249d7751 100644 --- a/lib/std/assertions.nim +++ b/lib/std/assertions.nim @@ -7,6 +7,10 @@ # distribution, for details about the copyright. # +when not defined(nimPreviewSlimSystem) and not declared(sysFatal): + include "system/rawquits" + include "system/fatal" + ## This module implements assertion handling. import std/private/miscdollars @@ -26,7 +30,10 @@ proc `$`(info: InstantiationInfo): string = proc raiseAssert*(msg: string) {.noinline, noreturn, nosinks.} = ## Raises an `AssertionDefect` with `msg`. - raise newException(AssertionDefect, msg) + when defined(nimPreviewSlimSystem): + raise newException(AssertionDefect, msg) + else: + sysFatal(AssertionDefect, msg) proc failedAssertImpl*(msg: string) {.raises: [], tags: [].} = ## Raises an `AssertionDefect` with `msg`, but this is hidden diff --git a/lib/system.nim b/lib/system.nim index e9c9ebfeb..d8fafc0ef 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1081,31 +1081,9 @@ proc align(address, alignment: int): int = else: result = (address + (alignment - 1)) and not (alignment - 1) -when defined(nimNoQuit): - proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" - -elif defined(genode): - import genode/env - - var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr - - type GenodeEnv* = GenodeEnvPtr - ## Opaque type representing Genode environment. - - proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, - importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "".} - - proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = - systemEnv.rawQuit(errorcode) - - -elif defined(js) and defined(nodejs) and not defined(nimscript): - proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", - importc: "process.exit", noreturn.} - -else: - proc rawQuit(errorcode: cint) {. - magic: "Exit", importc: "exit", header: "", noreturn.} +include system/rawquits +when defined(genode): + export GenodeEnv template sysAssert(cond: bool, msg: string) = when defined(useSysAssert): diff --git a/lib/system/rawquits.nim b/lib/system/rawquits.nim new file mode 100644 index 000000000..f0ead10c6 --- /dev/null +++ b/lib/system/rawquits.nim @@ -0,0 +1,27 @@ +import system/ctypes + +when defined(nimNoQuit): + proc rawQuit(errorcode: int = QuitSuccess) = discard "ignoring quit" + +elif defined(genode): + import genode/env + + var systemEnv {.exportc: runtimeEnvSym.}: GenodeEnvPtr + + type GenodeEnv = GenodeEnvPtr + ## Opaque type representing Genode environment. + + proc rawQuit(env: GenodeEnv; errorcode: int) {.magic: "Exit", noreturn, + importcpp: "#->parent().exit(@); Genode::sleep_forever()", header: "".} + + proc rawQuit(errorcode: int = QuitSuccess) {.inline, noreturn.} = + systemEnv.rawQuit(errorcode) + + +elif defined(js) and defined(nodejs) and not defined(nimscript): + proc rawQuit(errorcode: int = QuitSuccess) {.magic: "Exit", + importc: "process.exit", noreturn.} + +else: + proc rawQuit(errorcode: cint) {. + magic: "Exit", importc: "exit", header: "", noreturn.} \ No newline at end of file diff --git a/tests/assert/panicoverride.nim b/tests/assert/panicoverride.nim new file mode 100644 index 000000000..53ad64215 --- /dev/null +++ b/tests/assert/panicoverride.nim @@ -0,0 +1,15 @@ +# panicoverride.nim + +proc printf(fmt: cstring) {.varargs, importc, header:"stdio.h".} +proc exit(code: cint) {.importc, header:"stdlib.h".} + +{.push stack_trace: off, profiler:off.} + +proc rawoutput(s: cstring) = + printf("RAW: %s\n", s) + +proc panic(s: cstring) {.noreturn.} = + printf("PANIC: %s\n", s) + exit(0) + +{.pop.} \ No newline at end of file diff --git a/tests/assert/t21195.nim b/tests/assert/t21195.nim new file mode 100644 index 000000000..b690d22a0 --- /dev/null +++ b/tests/assert/t21195.nim @@ -0,0 +1,6 @@ +discard """ + matrix: "--verbosity:0 --os:standalone --mm:none" +""" +# bug #21195 +var n = 11 +assert(n == 12) diff --git a/tests/assert/tassert_c.nim b/tests/assert/tassert_c.nim index 110a9aabf..4d49a6035 100644 --- a/tests/assert/tassert_c.nim +++ b/tests/assert/tassert_c.nim @@ -1,8 +1,8 @@ discard """ - cmd: "nim $target $options --excessiveStackTrace:off $file" + cmd: "nim $target $options -d:nimPreviewSlimSystem --excessiveStackTrace:off $file" output: '''true''' """ - +import std/assertions const expected = """ tassert_c.nim(35) tassert_c tassert_c.nim(34) foo diff --git a/tests/errmsgs/t9768.nim b/tests/errmsgs/t9768.nim index 058d297b3..b5ff58367 100644 --- a/tests/errmsgs/t9768.nim +++ b/tests/errmsgs/t9768.nim @@ -1,14 +1,14 @@ discard """ errormsg: "unhandled exception: t9768.nim(24, 3) `a < 4` [AssertionDefect]" file: "std/assertions.nim" + matrix: "-d:nimPreviewSlimSystem" nimout: ''' stack trace: (most recent call last) t9768.nim(29, 33) main t9768.nim(24, 11) foo1 ''' """ - - +import std/assertions From c598d0b6eccd3133bfca710fc5fcc4576501862c Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 30 Dec 2022 07:50:12 +0800 Subject: [PATCH 1518/1654] fix #15117 zero size array cause incorrect codegen for VCC compiler (#21197) fix #15117 --- compiler/ccgexprs.nim | 2 ++ tests/array/t15117.nim | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/array/t15117.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index e2885a294..844a37997 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3331,6 +3331,8 @@ proc genConstObjConstr(p: BProc; n: PNode; isConst: bool; result: var Rope) = proc genConstSimpleList(p: BProc, n: PNode; isConst: bool; result: var Rope) = result.add "{" + if p.vccAndC and n.len == 0 and n.typ.kind == tyArray: + getDefaultValue(p, n.typ[1], n.info, result) for i in 0.. 0: result.add ",\n" diff --git a/tests/array/t15117.nim b/tests/array/t15117.nim new file mode 100644 index 000000000..157b04bee --- /dev/null +++ b/tests/array/t15117.nim @@ -0,0 +1,27 @@ +discard """ + matrix: "--cc:vcc" + disabled: "linux" + disabled: "bsd" + disabled: "osx" + disabled: "unix" + disabled: "posix" +""" +{.experimental: "views".} + +let a: array[0, byte] = [] +discard a + +type B = object + a:int +let b: array[0, B] = [] +let c: array[0, ptr B] = [] +let d: array[0, ref B] = [] +discard b +discard c +discard d + +discard default(array[0, B]) + +type + View1 = openArray[byte] +discard default(View1) From e5639408d55c3b9164790bee37580a74a15b666f Mon Sep 17 00:00:00 2001 From: rockcavera Date: Fri, 30 Dec 2022 05:20:32 -0300 Subject: [PATCH 1519/1654] avoiding unnecessary allocation for dollar IpAddress (#21199) * avoiding allocating an unnecessary byte for IPv4 The inet.h file uses 16 as the string in C needs the last null byte https://github.com/torvalds/linux/blob/1b929c02afd37871d5afb9d498426f83432e71c2/include/linux/inet.h#L49 However, strings in Nim do not need this. So one byte is being allocated unnecessary and will never be used. * avoid unnecessary allocation in IPv6 dollar It is currently allocating 48 bytes. However, the Nim implementation for IPv6 will print a maximum of 39 characters. Nim does not implement IPv6 "0000:0000:0000:0000:0000:ffff:255.255.255.255" (45 characters) nor "0000:0000:0000:0000:0000:ffff:255.255.255.255%3" (47 characters). The indication in inet.h for 48 is due to the maximum use of 47 characters of a C string that needs a null byte at the end. So 48. https://github.com/torvalds/linux/blob/1b929c02afd37871d5afb9d498426f83432e71c2/include/linux/inet.h#L50 --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index be9f2e48c..76a39226f 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1921,7 +1921,7 @@ proc `$`*(address: IpAddress): string = ## Converts an IpAddress into the textual representation case address.family of IpAddressFamily.IPv4: - result = newStringOfCap(16) + result = newStringOfCap(15) result.addInt address.address_v4[0] result.add '.' result.addInt address.address_v4[1] @@ -1930,7 +1930,7 @@ proc `$`*(address: IpAddress): string = result.add '.' result.addInt address.address_v4[3] of IpAddressFamily.IPv6: - result = newStringOfCap(48) + result = newStringOfCap(39) var currentZeroStart = -1 currentZeroCount = 0 From 4032eb4baafcfb4a1cb8fd41a474daac70103856 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 1 Jan 2023 16:51:58 +0800 Subject: [PATCH 1520/1654] fixes #20906; update copyright year [backport 1.6] (#21210) --- compiler/options.nim | 2 +- copying.txt | 2 +- readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index ded084ae4..47e8155f8 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -24,7 +24,7 @@ const useEffectSystem* = true useWriteTracking* = false hasFFI* = defined(nimHasLibFFI) - copyrightYear* = "2022" + copyrightYear* = "2023" nimEnableCovariance* = defined(nimEnableCovariance) diff --git a/copying.txt b/copying.txt index 60b6a0217..ae3a162a7 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ ===================================================== Nim -- a Compiler for Nim. https://nim-lang.org/ -Copyright (C) 2006-2022 Andreas Rumpf. All rights reserved. +Copyright (C) 2006-2023 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index a0f47eb41..325e8fff9 100644 --- a/readme.md +++ b/readme.md @@ -203,7 +203,7 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2022 Andreas Rumpf, all rights reserved. +Copyright © 2006-2023 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org From 5b20f0685c71e68ab08f7a1941f7e160d91e538f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 1 Jan 2023 17:25:04 +0800 Subject: [PATCH 1521/1654] fixes pragmas reorder (#21205) --- compiler/reorder.nim | 1 + tests/pragmas/tpragmas_reorder.nim | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/pragmas/tpragmas_reorder.nim diff --git a/compiler/reorder.nim b/compiler/reorder.nim index 4fabf9041..4ad3f1219 100644 --- a/compiler/reorder.nim +++ b/compiler/reorder.nim @@ -106,6 +106,7 @@ proc computeDeps(cache: IdentCache; n: PNode, declares, uses: var IntSet; topLev if a.kind == nkExprColonExpr and a[0].kind == nkIdent and a[0].ident.s == "pragma": # user defined pragma decl(a[1]) + for i in 1.. Date: Mon, 2 Jan 2023 08:16:18 +0000 Subject: [PATCH 1522/1654] Update nim.cfg (#21204) tlsEmulation:on under NetBSD-10Beta and NetBSD-current produces an executable which crashes immediately as follows: Core was generated by `koch'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x000000000047b4c2 in nimZeroMem () (gdb) bt #0 0x000000000047b4c2 in nimZeroMem () #1 0x00000000004897b2 in threadVarAlloc__system_2162 () #2 0x000000000048980e in initThreadVarsEmulation () #3 0x0000000000489848 in PreMain () #4 0x000000000048986a in NimMain () #5 0x00000000004898a9 in main () I can't speak about the other BSDs. --- config/nim.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/nim.cfg b/config/nim.cfg index 1601ebe08..f7b7c119a 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -181,10 +181,9 @@ nimblepath="$home/.nimble/pkgs/" gcc.maxerrorsimpl = "-fmax-errors=3" -@if freebsd: +@if freebsd or netbsd: tlsEmulation:off @elif bsd: - # at least NetBSD has problems with thread local storage: tlsEmulation:on @end From cf1b16ef8ba1c9f8411e551d317784c5ffaece0c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 2 Jan 2023 10:39:17 +0100 Subject: [PATCH 1523/1654] minor code cleanups (#21215) --- lib/posix/posix.nim | 8 ++++---- lib/posix/posix_utils.nim | 6 +++--- lib/std/enumerate.nim | 2 +- lib/std/enumutils.nim | 2 +- lib/std/envvars.nim | 35 +++++++++++++++++------------------ lib/std/oserrors.nim | 15 ++++++--------- 6 files changed, 32 insertions(+), 36 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index ddf00cbb7..43346608d 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1140,12 +1140,12 @@ type ## The getrlimit() and setrlimit() system calls get and set resource limits respectively. ## Each resource has an associated soft and hard limit, as defined by the RLimit structure -proc setrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "setrlimit",header: "".} +proc setrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "setrlimit", header: "".} ## The setrlimit() system calls sets resource limits. -proc getrlimit*(resource: cint, rlp: var RLimit): cint - {.importc: "getrlimit",header: "".} +proc getrlimit*(resource: cint, rlp: var RLimit): cint {. + importc: "getrlimit", header: "".} ## The getrlimit() system call gets resource limits. when defined(nimHasStyleChecks): diff --git a/lib/posix/posix_utils.nim b/lib/posix/posix_utils.nim index 92fe0940d..7e4a2eeda 100644 --- a/lib/posix/posix_utils.nim +++ b/lib/posix/posix_utils.nim @@ -43,8 +43,8 @@ proc uname*(): Uname = result.machine = charArrayToString u.machine proc fsync*(fd: int) = - ## synchronize a file's buffer cache to the storage device - if fsync(fd.cint) != 0: + ## synchronize a file's buffer cache to the storage device + if fsync(fd.cint) != 0: raise newException(OSError, $strerror(errno)) proc stat*(path: string): Stat = @@ -90,7 +90,7 @@ proc mkstemp*(prefix: string, suffix=""): (string, File) = ## Returns the filename and a file opened in r/w mode. var tmpl = cstring(prefix & "XXXXXX" & suffix) let fd = - if len(suffix)==0: + if len(suffix) == 0: when declared(mkostemp): mkostemp(tmpl, O_CLOEXEC) else: diff --git a/lib/std/enumerate.nim b/lib/std/enumerate.nim index a8f0e1ba7..4f0161b7c 100644 --- a/lib/std/enumerate.nim +++ b/lib/std/enumerate.nim @@ -21,7 +21,7 @@ macro enumerate*(x: ForLoopStmt): untyped {.since: (1, 3).} = ## The default starting count `0` can be manually overridden if needed. runnableExamples: let a = [10, 20, 30] - var b: seq[(int, int)] + var b: seq[(int, int)] = @[] for i, x in enumerate(a): b.add((i, x)) assert b == @[(0, 10), (1, 20), (2, 30)] diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 9d4ff1bcf..59fb112ed 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -112,9 +112,9 @@ const invalidSlot = uint8.high proc genLookup[T: typedesc[HoleyEnum]](_: T): auto = const n = span(T) - var ret: array[n, uint8] var i = 0 assert n <= invalidSlot.int + var ret {.noinit.}: array[n, uint8] for ai in mitems(ret): ai = invalidSlot for ai in items(T): ret[ai.ord - T.low.ord] = uint8(i) diff --git a/lib/std/envvars.nim b/lib/std/envvars.nim index ce90f66ba..ab5c9f06e 100644 --- a/lib/std/envvars.nim +++ b/lib/std/envvars.nim @@ -94,8 +94,10 @@ when not defined(nimscript): assert getEnv("unknownEnv", "doesn't exist") == "doesn't exist" let env = getEnvImpl(key) - if env == nil: return default - result = $env + if env == nil: + result = default + else: + result = $env proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} = ## Checks whether the environment variable named `key` exists. @@ -109,7 +111,7 @@ when not defined(nimscript): runnableExamples: assert not existsEnv("unknownEnv") - return getEnvImpl(key) != nil + result = getEnvImpl(key) != nil proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} = ## Sets the value of the `environment variable`:idx: named `key` to `val`. @@ -140,7 +142,7 @@ when not defined(nimscript): ## * `envPairs iterator`_ template bail = raiseOSError(osLastError(), key) when defined(windows): - #[ + #[ # https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/putenv-s-wputenv-s?view=msvc-160 > You can remove a variable from the environment by specifying an empty string (that is, "") for value_string note that nil is not legal @@ -177,20 +179,17 @@ when not defined(nimscript): iterator envPairsImpl(): tuple[key, value: string] {.tags: [ReadEnvEffect].} = when defined(windows): - block implBlock: - template impl(get_fun, typ, size, zero, free_fun) = - let env = get_fun() - var e = env - if e == nil: break implBlock - while true: - let eend = strEnd(e) - let kv = $e - let p = find(kv, '=') - yield (substr(kv, 0, p-1), substr(kv, p+1)) - e = cast[typ](cast[int](eend)+size) - if typeof(zero)(eend[1]) == zero: break - discard free_fun(env) - impl(getEnvironmentStringsW, WideCString, 2, 0, freeEnvironmentStringsW) + let env = getEnvironmentStringsW() + var e = env + if e != nil: + while true: + let eend = strEnd(e) + let kv = $e + let p = find(kv, '=') + yield (substr(kv, 0, p-1), substr(kv, p+1)) + e = cast[WideCString](cast[ByteAddress](eend)+2) + if int(eend[1]) == 0: break + discard freeEnvironmentStringsW(env) else: var i = 0 when defined(macosx) and not defined(ios) and not defined(emscripten): diff --git a/lib/std/oserrors.nim b/lib/std/oserrors.nim index 8d06c41da..a641a7f47 100644 --- a/lib/std/oserrors.nim +++ b/lib/std/oserrors.nim @@ -75,17 +75,14 @@ proc newOSError*( ## See also: ## * `osErrorMsg proc`_ ## * `osLastError proc`_ - var e: owned(ref OSError); new(e) - e.errorCode = errorCode.int32 - e.msg = osErrorMsg(errorCode) + result = (ref OSError)(errorCode: errorCode.int32, msg: osErrorMsg(errorCode)) if additionalInfo.len > 0: - if e.msg.len > 0 and e.msg[^1] != '\n': e.msg.add '\n' - e.msg.add "Additional info: " - e.msg.add additionalInfo + if result.msg.len > 0 and result.msg[^1] != '\n': result.msg.add '\n' + result.msg.add "Additional info: " + result.msg.add additionalInfo # don't add trailing `.` etc, which negatively impacts "jump to file" in IDEs. - if e.msg == "": - e.msg = "unknown OS error" - return e + if result.msg == "": + result.msg = "unknown OS error" proc raiseOSError*(errorCode: OSErrorCode, additionalInfo = "") {.noinline.} = ## Raises an `OSError exception `_. From 3b965f463b41eace340393af636e4abcd9c223d6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:10:26 +0800 Subject: [PATCH 1524/1654] bump csource_v2 to include fix for linking parameter maximum length (#21216) > If accepted, I believe it should be used in csources_v2 and also backport 2.0 ref https://github.com/nim-lang/Nim/pull/21186 --- config/build_config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/build_config.txt b/config/build_config.txt index 286e0dc2a..66390e695 100644 --- a/config/build_config.txt +++ b/config/build_config.txt @@ -2,4 +2,4 @@ nim_comment="key-value pairs for windows/posix bootstrapping build scripts" nim_csourcesDir=csources_v2 nim_csourcesUrl=https://github.com/nim-lang/csources_v2.git nim_csourcesBranch=master -nim_csourcesHash=b66c420697c574be18a20dd4720248a715b4287e +nim_csourcesHash=86742fb02c6606ab01a532a0085784effb2e753e From 81b7f9108f139bbf237f3e121fdbe9ff32a7193f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:15:10 +0800 Subject: [PATCH 1525/1654] fixes #21207; reports redefinition error in the definition of enums (#21217) * fixes #21207; reports redefinition in the enums * add a test --- compiler/semtypes.nim | 4 ++-- tests/enum/tredefinition.nim | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 tests/enum/tredefinition.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 534861e97..a8b147eef 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -73,7 +73,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType = rawAddSon(result, base) let isPure = result.sym != nil and sfPure in result.sym.flags var symbols: TStrTable - if isPure: initStrTable(symbols) + initStrTable(symbols) var hasNull = false for i in 1.. Date: Wed, 4 Jan 2023 12:10:44 -0800 Subject: [PATCH 1526/1654] make it printer friendly (#21218) --- doc/nimdoc.css | 9 +++++++++ nimdoc/testproject/expected/nimdoc.out.css | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/doc/nimdoc.css b/doc/nimdoc.css index f924f5a36..1417d9eff 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -147,6 +147,15 @@ body { box-sizing: border-box; margin-left: 1%; } +@media print { + #global-links, .link-seesrc, .theme-switch-wrapper, #searchInputDiv, .search-groupby { + display:none; + } + .columns { + width:100% !important; + } +} + .column:first-child, .columns:first-child { margin-left: 0; } diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index f924f5a36..1417d9eff 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -147,6 +147,15 @@ body { box-sizing: border-box; margin-left: 1%; } +@media print { + #global-links, .link-seesrc, .theme-switch-wrapper, #searchInputDiv, .search-groupby { + display:none; + } + .columns { + width:100% !important; + } +} + .column:first-child, .columns:first-child { margin-left: 0; } From 2620da9bf9994d09da3d5b31b29c95855fb79a41 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 4 Jan 2023 23:19:01 +0300 Subject: [PATCH 1527/1654] docgen: implement cross-document links (#20990) * docgen: implement cross-document links Fully implements https://github.com/nim-lang/RFCs/issues/125 Follow-up of: https://github.com/nim-lang/Nim/pull/18642 (for internal links) and https://github.com/nim-lang/Nim/issues/20127. Overview -------- Explicit import-like directive is required, called `.. importdoc::`. (the syntax is % RST, Markdown will use it for a while). Then one can reference any symbols/headings/anchors, as if they were in the local file (but they will be prefixed with a module name or markup document in link text). It's possible to reference anything from anywhere (any direction in `.nim`/`.md`/`.rst` files). See `doc/docgen.md` for full description. Working is based on `.idx` files, hence one needs to generate all `.idx` beforehand. A dedicated option `--index:only` is introduced (and a separate stage for `--index:only` is added to `kochdocs.nim`). Performance note ---------------- Full run for `./koch docs` now takes 185% of the time before this PR. (After: 315 s, before: 170 s on my PC). All the time seems to be spent on `--index:only` run, which takes almost as much (85%) of normal doc run -- it seems that most time is spent on file parsing, turning off HTML generation phase has not helped much. (One could avoid it by specifying list of files that can be referenced and pre-processing only them. But it can become error-prone and I assume that these linke will be **everywhere** in the repository anyway, especially considering https://github.com/nim-lang/RFCs/issues/478. So every `.nim`/`.md` file is processed for `.idx` first). But that's all without significant part of repository converted to cross-module auto links. To estimate impact I checked the time for `doc`ing a few files (after all indexes have been generated), and everywhere difference was **negligible**. E.g. for `lib/std/private/osfiles.nim` that `importdoc`s large `os.idx` and hence should have been a case with relatively large performance impact, but: * After: 0.59 s. * Before: 0.59 s. So Nim compiler works so slow that doc part basically does not matter :-) Testing ------- 1) added `extlinks` test to `nimdoc/` 2) checked that `theindex.html` is still correct 2) fixed broken auto-links for modules that were derived from `os.nim` by adding appropriate ``importdoc`` Implementation note ------------------- Parsing and formating of `.idx` entries is moved into a dedicated `rstidx.nim` module from `rstgen.nim`. `.idx` file format changed: * fields are not escaped in most cases because we need original strings for referencing, not HTML ones (the exception is linkTitle for titles and headings). Escaping happens later -- on the stage of `rstgen` buildIndex, etc. * all lines have fixed number of columns 6 * added discriminator tag as a first column, it always allows distinguish Nim/markup entries, titles/headings, etc. `rstgen` does not rely any more (in most cases) on ad-hoc logic to determine what type each entry is. * there is now always a title entry added at the first line. * add a line number as 6th column * linkTitle (4th) column has a different format: before it was like `module: funcName()`, now it's `proc funcName()`. (This format is also propagated to `theindex.html` and search results, I kept it that way since I like it more though it's discussible.) This column is what used for Nim symbols resolution. * also changed details on column format for headings and titles: "keyword" is original, "linkTitle" is HTML one * fix paths on Windows + more clear code * Update compiler/docgen.nim Co-authored-by: Andreas Rumpf * Handle .md and .nim paths uniformly in findRefFile * handle titles better + more comments * don't allow markup overwrite index title for .nim files Co-authored-by: Andreas Rumpf --- compiler/commands.nim | 8 +- compiler/docgen.nim | 200 ++++++++---- compiler/lineinfos.nim | 2 + compiler/options.nim | 2 + doc/advopt.txt | 4 +- doc/docgen.md | 309 ++++++++++++++++-- doc/markdown_rst.md | 84 +++++ lib/core/macros.nim | 2 + lib/packages/docutils/dochelpers.nim | 27 +- lib/packages/docutils/rst.nim | 263 +++++++++++++-- lib/packages/docutils/rstgen.nim | 200 ++++-------- lib/packages/docutils/rstidx.nim | 138 ++++++++ lib/pure/os.nim | 2 + lib/std/appdirs.nim | 2 + lib/std/private/osappdirs.nim | 2 + lib/std/private/oscommon.nim | 2 + lib/std/private/osdirs.nim | 2 + lib/std/private/osfiles.nim | 3 +- lib/std/private/ospaths2.nim | 2 + lib/std/private/osseps.nim | 2 + lib/std/private/ossymlinks.nim | 1 + lib/std/symlinks.nim | 3 +- nimdoc/extlinks/project/doc/manual.md | 17 + .../extlinks/project/expected/_._/util.html | 104 ++++++ nimdoc/extlinks/project/expected/_._/util.idx | 2 + .../extlinks/project/expected/doc/manual.html | 44 +++ .../extlinks/project/expected/doc/manual.idx | 3 + nimdoc/extlinks/project/expected/main.html | 144 ++++++++ nimdoc/extlinks/project/expected/main.idx | 4 + .../project/expected/sub/submodule.html | 130 ++++++++ .../project/expected/sub/submodule.idx | 3 + .../extlinks/project/expected/theindex.html | 58 ++++ nimdoc/extlinks/project/main.nim | 23 ++ nimdoc/extlinks/project/sub/submodule.nim | 13 + nimdoc/extlinks/util.nim | 2 + .../test_out_index_dot_html/expected/foo.idx | 3 +- .../expected/theindex.html | 2 +- nimdoc/tester.nim | 63 ++++ .../expected/subdir/subdir_b/utils.html | 6 +- .../expected/subdir/subdir_b/utils.idx | 87 ++--- nimdoc/testproject/expected/testproject.idx | 131 ++++---- nimdoc/testproject/expected/theindex.html | 196 +++++------ tests/stdlib/tdochelpers.nim | 16 + tests/stdlib/trst.nim | 2 +- tools/kochdocs.nim | 41 ++- 45 files changed, 1863 insertions(+), 491 deletions(-) create mode 100644 lib/packages/docutils/rstidx.nim create mode 100644 nimdoc/extlinks/project/doc/manual.md create mode 100644 nimdoc/extlinks/project/expected/_._/util.html create mode 100644 nimdoc/extlinks/project/expected/_._/util.idx create mode 100644 nimdoc/extlinks/project/expected/doc/manual.html create mode 100644 nimdoc/extlinks/project/expected/doc/manual.idx create mode 100644 nimdoc/extlinks/project/expected/main.html create mode 100644 nimdoc/extlinks/project/expected/main.idx create mode 100644 nimdoc/extlinks/project/expected/sub/submodule.html create mode 100644 nimdoc/extlinks/project/expected/sub/submodule.idx create mode 100644 nimdoc/extlinks/project/expected/theindex.html create mode 100644 nimdoc/extlinks/project/main.nim create mode 100644 nimdoc/extlinks/project/sub/submodule.nim create mode 100644 nimdoc/extlinks/util.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index a252f505f..73140036a 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -820,7 +820,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if conf != nil: conf.headerFile = arg incl(conf.globalOptions, optGenIndex) of "index": - processOnOffSwitchG(conf, {optGenIndex}, arg, pass, info) + case arg.normalize + of "", "on": conf.globalOptions.incl {optGenIndex} + of "only": conf.globalOptions.incl {optGenIndexOnly, optGenIndex} + of "off": conf.globalOptions.excl {optGenIndex, optGenIndexOnly} + else: localError(conf, info, errOnOrOffExpectedButXFound % arg) + of "noimportdoc": + processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info) of "import": expectArg(conf, switch, arg, pass, info) if pass in {passCmd2, passPP}: diff --git a/compiler/docgen.nim b/compiler/docgen.nim index f8a804cec..cab334e7a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -7,13 +7,17 @@ # distribution, for details about the copyright. # -# This is the documentation generator. Cross-references are generated -# by knowing how the anchors are going to be named. +## This is the Nim documentation generator. Cross-references are generated +## by knowing how the anchors are going to be named. +## +## .. importdoc:: ../docgen.md +## +## For corresponding users' documentation see [Nim DocGen Tools Guide]. import ast, strutils, strtabs, algorithm, sequtils, options, msgs, os, idents, wordrecg, syntaxes, renderer, lexer, - packages/docutils/[rst, rstgen, dochelpers], + packages/docutils/[rst, rstidx, rstgen, dochelpers], json, xmltree, trees, types, typesrenderer, astalgo, lineinfos, intsets, pathutils, tables, nimpaths, renderverbatim, osproc, packages @@ -91,7 +95,7 @@ type jEntriesFinal: JsonNode # final JSON after RST pass 2 and rendering types: TStrTable sharedState: PRstSharedState - standaloneDoc: bool + standaloneDoc: bool # is markup (.rst/.md) document? conf*: ConfigRef cache*: IdentCache exampleCounter: int @@ -225,7 +229,7 @@ proc attachToType(d: PDoc; p: PSym): PSym = if params.len > 0: check(0) for i in 2.."), "itemSymOrIDEnc", symbolOrIdEnc]) - # Ironically for types the complexSymbol is *cleaner* than the plainName - # because it doesn't include object fields or documentation comments. So we - # use the plain one for callable elements, and the complex for the rest. - var linkTitle = changeFileExt(extractFilename(d.filename), "") & ": " - if n.kind in routineDefs: linkTitle.add(xmltree.escape(plainName.strip)) - else: linkTitle.add(xmltree.escape(complexSymbol.strip)) - - setIndexTerm(d[], external, symbolOrId, name, linkTitle, - xmltree.escape(plainDocstring.docstringSummary)) + setIndexTerm(d[], ieNim, htmlFile = external, id = symbolOrId, term = name, + linkTitle = detailedName, + linkDesc = xmltree.escape(plainDocstring.docstringSummary), + line = n.info.line.int) if k == skType and nameNode.kind == nkSym: d.types.strTableAdd nameNode.sym proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonItem = if not isVisible(d, nameNode): return var - name = getName(d, nameNode) + name = getNameEsc(d, nameNode) comm = genRecComment(d, n) r: TSrcGen initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments, renderExpandUsing}) @@ -1337,9 +1369,31 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = proc overloadGroupName(s: string, k: TSymKind): string = ## Turns a name like `f` into anchor `f-procs-all` - #s & " " & k.toHumanStr & "s all" s & "-" & k.toHumanStr & "s-all" +proc setIndexTitle(d: PDoc, useMetaTitle: bool) = + let titleKind = if d.standaloneDoc: ieMarkupTitle else: ieNimTitle + let external = AbsoluteFile(d.destFile) + .relativeTo(d.conf.outDir, '/') + .changeFileExt(HtmlExt) + .string + var term, linkTitle: string + if useMetaTitle and d.meta[metaTitle].len != 0: + term = d.meta[metaTitleRaw] + linkTitle = d.meta[metaTitleRaw] + else: + let filename = extractFilename(d.filename) + term = + if d.standaloneDoc: filename # keep .rst/.md extension + else: changeFileExt(filename, "") # rm .nim extension + linkTitle = + if d.standaloneDoc: term # keep .rst/.md extension + else: canonicalImport(d.conf, AbsoluteFile d.filename) + if not d.standaloneDoc: + linkTitle = "module " & linkTitle + setIndexTerm(d[], titleKind, htmlFile = external, id = "", + term = term, linkTitle = linkTitle) + proc finishGenerateDoc*(d: var PDoc) = ## Perform 2nd RST pass for resolution of links/footnotes/headings... # copy file map `filenames` to ``rstgen.nim`` for its warnings @@ -1352,7 +1406,24 @@ proc finishGenerateDoc*(d: var PDoc) = firstRst = fragment.rst break d.hasToc = d.hasToc or d.sharedState.hasToc - preparePass2(d.sharedState, firstRst) + # in --index:only mode we do NOT want to load other .idx, only write ours: + let importdoc = optGenIndexOnly notin d.conf.globalOptions and + optNoImportdoc notin d.conf.globalOptions + preparePass2(d.sharedState, firstRst, importdoc) + + if optGenIndexOnly in d.conf.globalOptions: + # Top-level doc.comments may contain titles and :idx: statements: + for fragment in d.modDescPre: + if fragment.isRst: + traverseForIndex(d[], fragment.rst) + setIndexTitle(d, useMetaTitle = d.standaloneDoc) + # Symbol-associated doc.comments may contain :idx: statements: + for k in TSymKind: + for _, overloadChoices in d.section[k].secItems: + for item in overloadChoices: + for fragment in item.descRst: + if fragment.isRst: + traverseForIndex(d[], fragment.rst) # add anchors to overload groups before RST resolution for k in TSymKind: @@ -1362,14 +1433,25 @@ proc finishGenerateDoc*(d: var PDoc) = let refn = overloadGroupName(plainName, k) let tooltip = "$1 ($2 overloads)" % [ k.toHumanStr & " " & plainName, $overloadChoices.len] - addAnchorNim(d.sharedState, refn, tooltip, + let name = nimIdentBackticksNormalize(plainName) + # save overload group to ``.idx`` + let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/'). + changeFileExt(HtmlExt).string + setIndexTerm(d[], ieNimGroup, htmlFile = external, id = refn, + term = name, linkTitle = k.toHumanStr, + linkDesc = "", line = overloadChoices[0].info.line.int) + if optGenIndexOnly in d.conf.globalOptions: continue + addAnchorNim(d.sharedState, external=false, refn, tooltip, LangSymbol(symKind: k.toHumanStr, - name: nimIdentBackticksNormalize(plainName), + name: name, isGroup: true), priority = symbolPriority(k), # select index `0` just to have any meaningful warning: info = overloadChoices[0].info) + if optGenIndexOnly in d.conf.globalOptions: + return + # Finalize fragments of ``.nim`` or ``.rst`` file proc renderItemPre(d: PDoc, fragments: ItemPre, result: var string) = for f in fragments: @@ -1421,6 +1503,9 @@ proc finishGenerateDoc*(d: var PDoc) = d.jEntriesFinal.add entry.json # generates docs + setIndexTitle(d, useMetaTitle = d.standaloneDoc) + completePass2(d.sharedState) + proc add(d: PDoc; j: JsonItem) = if j.json != nil or j.rst != nil: d.jEntriesPre.add j @@ -1467,7 +1552,7 @@ proc generateJson*(d: PDoc, n: PNode, includeComments: bool = true) = else: discard proc genTagsItem(d: PDoc, n, nameNode: PNode, k: TSymKind): string = - result = getName(d, nameNode) & "\n" + result = getNameEsc(d, nameNode) & "\n" proc generateTags*(d: PDoc, n: PNode, r: var string) = case n.kind @@ -1574,13 +1659,7 @@ proc genOutFile(d: PDoc, groupedToc = false): string = # Extract the title. Non API modules generate an entry in the index table. if d.meta[metaTitle].len != 0: title = d.meta[metaTitle] - let external = AbsoluteFile(d.destFile) - .relativeTo(d.conf.outDir, '/') - .changeFileExt(HtmlExt) - .string - setIndexTerm(d[], external, "", title) else: - # Modules get an automatic title for the HTML, but no entry in the index. title = canonicalImport(d.conf, AbsoluteFile d.filename) title = esc(d.target, title) var subtitle = "" @@ -1619,11 +1698,17 @@ proc genOutFile(d: PDoc, groupedToc = false): string = code = content result = code +proc indexFile(d: PDoc): AbsoluteFile = + let dir = d.conf.outDir + result = dir / changeFileExt(presentationPath(d.conf, + AbsoluteFile d.filename), + IndexExt) + let (finalDir, _, _) = result.string.splitFile + createDir(finalDir) + proc generateIndex*(d: PDoc) = if optGenIndex in d.conf.globalOptions: - let dir = d.conf.outDir - createDir(dir) - let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt) + let dest = indexFile(d) writeIndexFile(d[], dest.string) proc updateOutfile(d: PDoc, outfile: AbsoluteFile) = @@ -1634,6 +1719,9 @@ proc updateOutfile(d: PDoc, outfile: AbsoluteFile) = d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) = + if optGenIndexOnly in d.conf.globalOptions: + d.conf.outFile = indexFile(d).relativeTo(d.conf.outDir) # just for display + return runAllExamples(d) var content = genOutFile(d, groupedToc) if optStdout in d.conf.globalOptions: @@ -1772,6 +1860,8 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) = rawMessage(conf, errCannotOpenFile, filename.string) proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") = + if optGenIndexOnly in conf.globalOptions: + return var content = mergeIndexes(dir) var outFile = outFile diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index cb60d8949..89aff57df 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -57,6 +57,7 @@ type warnRstBrokenLink = "BrokenLink", warnRstLanguageXNotSupported = "LanguageXNotSupported", warnRstFieldXNotSupported = "FieldXNotSupported", + warnRstUnusedImportdoc = "UnusedImportdoc", warnRstStyle = "warnRstStyle", warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam", @@ -142,6 +143,7 @@ const warnRstBrokenLink: "broken link '$1'", warnRstLanguageXNotSupported: "language '$1' not supported", warnRstFieldXNotSupported: "field '$1' not supported", + warnRstUnusedImportdoc: "importdoc for '$1' is not used", warnRstStyle: "RST style: $1", warnCommentXIgnored: "comment '$1' ignored", warnTypelessParam: "", # deadcode diff --git a/compiler/options.nim b/compiler/options.nim index 47e8155f8..fc970e420 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -78,6 +78,8 @@ type # please make sure we have under 32 options optThreadAnalysis, # thread analysis pass optTlsEmulation, # thread var emulation turned on optGenIndex # generate index file for documentation; + optGenIndexOnly # generate only index file for documentation + optNoImportdoc # disable loading external documentation files optEmbedOrigSrc # embed the original source in the generated code # also: generate header file optIdeDebug # idetools: debug mode diff --git a/doc/advopt.txt b/doc/advopt.txt index 3f439fdab..efff428e8 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -130,7 +130,9 @@ Advanced options: select which memory management to use; default is 'orc' --exceptions:setjmp|cpp|goto|quirky select the exception handling implementation - --index:on|off turn index file generation on|off + --index:on|off|only docgen: turn index file generation? (`only` means + not generate output files like HTML) + --noImportdoc:on|off disable loading documentation ``.idx`` files? --putenv:key=value set an environment variable --NimblePath:PATH add a path for Nimble support --noNimblePath deactivate the Nimble path diff --git a/doc/docgen.md b/doc/docgen.md index e4b040064..70b41f17d 100644 --- a/doc/docgen.md +++ b/doc/docgen.md @@ -9,6 +9,7 @@ .. include:: rstcommon.rst .. contents:: +.. importdoc:: markdown_rst.md, compiler/docgen.nim Introduction ============ @@ -103,6 +104,64 @@ won't influence RST formatting. ## Paragraph. ``` +Structuring output directories +------------------------------ + +Basic directory for output is set by `--outdir:OUTDIR`:option: switch, +by default `OUTDIR` is ``htmldocs`` sub-directory in the directory of +the processed file. + +There are 2 basic options as to how generated HTML output files are stored: + +1) complex hierarchy when docgen-compiling with `--project`:option:, + which follows directory structure of the project itself. + So `nim doc`:cmd: replicates project's directory structure + inside `--outdir:OUTDIR`:option: directory. + `--project`:option: is well suited for projects that have 1 main module. + File name clashes are impossible in this case. + +2) flattened structure, where user-provided script goes through all + needed input files and calls commands like `nim doc`:cmd: + with `--outdir:OUTDIR`:option: switch, thus putting all HTML (and + ``.idx``) files into 1 directory. + + .. Important:: Make sure that you don't have files with same base name + like ``x.nim`` and ``x.md`` in the same package, otherwise you'll + have name conflict for ``x.html``. + + .. Tip:: To structure your output directories and avoid file name + clashes you can split your project into + different *packages* -- parts of your repository that are + docgen-compiled with different `--outdir:OUTDIR`:option: options. + + An example of such strategy is Nim repository itself which has: + + * its stdlib ``.nim`` files from different directories and ``.md`` + documentation from ``doc/`` directory are all docgen-compiled + into `--outdir:web/upload//`:option: directory + * its ``.nim`` files from ``compiler/`` directory are docgen-compiled + into `--outdir:web/upload//compiler/`:option: directory. + Interestingly, it's compiled with complex hierarchy using + `--project`:option: switch. + + Contents of ``web/upload/`` are then deployed into Nim's + Web server. + + This output directory structure allows to work correctly with files like + ``compiler/docgen.nim`` (implementation) and ``doc/docgen.md`` (user + documentation) in 1 repository. + + +Index files +----------- + +Index (``.idx``) files are used for 2 different purposes: + +1. easy cross-referencing between different ``.nim`` and/or ``.md`` / ``.rst`` + files described in [Nim external referencing] +2. creating a whole-project index for searching of symbols and keywords, + see [Buildindex command]. + Document Types ============== @@ -226,13 +285,46 @@ Note that the `jsondoc`:option: command outputs its JSON without pretty-printing while `jsondoc0`:option: outputs pretty-printed JSON. -Referencing Nim symbols: simple documentation links -=================================================== +Simple documentation links +========================== -You can reference Nim identifiers from Nim documentation comments, currently -only inside their ``.nim`` file (or inside a ``.rst`` file included from -a ``.nim``). The point is that such links will be resolved automatically -by `nim doc`:cmd: (or `nim jsondoc`:cmd: or `nim doc2tex`:cmd:). +It's possible to use normal Markdown/RST syntax to *manually* +reference Nim symbols using HTML anchors, however Nim has an *automatic* +facility that makes referencing inside ``.nim`` and ``.md/.rst`` files and +between them easy and seamless. +The point is that such links will be resolved automatically +by `nim doc`:cmd: (or `md2html`:option:, or `jsondoc`:option:, +or `doc2tex`:option:, ...). And, unlike manual links, such automatic +links **check** that their target exists -- a warning is emitted for +any broken link, so you avoid broken links in your project. + +Nim treats both ``.md/.rst`` files and ``.nim`` modules (their doc comment +part) as *documents* uniformly. +Hence all directions of referencing are equally possible having the same syntax: + +1. ``.md/rst`` -> itself (internal). See [Markup local referencing]. +2. ``.md/rst`` -> external ``.md/rst``. See [Markup external referencing]. + To summarize, referencing in `.md`/`.rst` files was already described in + [Nim-flavored Markdown and reStructuredText] + (particularly it described usage of index files for referencing), + while in this document we focus on Nim-specific details. +3. ``.md/rst`` -> external ``.nim``. See [Nim external referencing]. +4. ``.nim`` -> itself (internal). See [Nim local referencing]. +5. ``.nim`` -> external ``.md/rst``. See [Markup external referencing]. +6. ``.nim`` -> external ``.nim``. See [Nim external referencing]. + +To put it shortly, local referencing always works out of the box, +external referencing requires to use ``.. importdoc:: `` +directive to import `file` and to ensure that the corresponding +``.idx`` file was generated. + + +Nim local referencing +--------------------- + +You can reference Nim identifiers from Nim documentation comments +inside their ``.nim`` file (or inside a ``.rst`` file included from +a ``.nim``). This pertains to any exported symbol like `proc`, `const`, `iterator`, etc. Syntax for referencing is basically a normal RST one: addition of underscore `_` to a *link text*. @@ -405,6 +497,143 @@ recognized fine: ... ## Ref. `CopyFlag enum`_ +Nim external referencing +------------------------ + +Just like for [Markup external referencing], which saves markup anchors, +the Nim symbols are also saved in ``.idx`` files, so one needs +to generate them beforehand, and they should be loaded by +an ``.. importdoc::`` directive. Arguments to ``.. importdoc::`` is a +comma-separated list of Nim modules or Markdown/RST documents. + +`--index:only`:option: tells Nim to only generate ``.idx`` file and +do **not** attempt to generate HTML/LaTeX output. +For ``.nim`` modules there are 2 alternatives to work with ``.idx`` files: + +1. using [Project switch] implies generation of ``.idx`` files, + however, if ``importdoc`` is called on upper modules as its arguments, + their ``.idx`` are not yet created. Thus one should generate **all** + required ``.idx`` first: + ```cmd + nim doc --project --index:only
          .nim + nim doc --project
          .nim + ``` +2. or run `nim doc --index:only `:cmd: command for **all** (used) + Nim modules in your project. Then run `nim doc ` on them for + output HTML generation. + + .. Warning:: A mere `nim doc --index:on`:cmd: may fail on an attempt to do + ``importdoc`` from another module (for which ``.idx`` was not yet + generated), that's why `--index:only`:option: shall be used instead. + + For ``.md``/``.rst`` markup documents point 2 is the only option. + +Then, you can freely use something like this in ``your_module.nim``: + + ```nim + ## .. importdoc:: user_manual.md, another_module.nim + + ... + ## Ref. [some section from User Manual]. + + ... + ## Ref. [proc f] + ## (assuming you have a proc `f` in ``another_module``). + ``` + +and compile it by `nim doc`:cmd:. Note that link text will +be automatically prefixed by the module name of symbol, +so you will see something like "Ref. [another_module: proc f](#)" +in the generated output. + +It's also possible to reference a whole module by prefixing or +suffixing full canonical module name with "module": + + Ref. [module subdir/name] or [subdir/name module]. + +Markup documents as a whole can be referenced just by their title +(or by their file name if the title was not set) without any prefix. + +.. Tip:: During development process the stage of ``.idx`` files generation + can be done only *once*, after that you use already generated ``.idx`` + files while working with a document *being developed* (unless you do + incompatible changes to *referenced* documents). + +.. Hint:: After changing a *referenced* document file one may need + to regenerate its corresponding ``.idx`` file to get correct results. + Of course, when referencing *internally* inside any given ``.nim`` file, + it's not needed, one can even immediately use any freshly added anchor + (a document's own ``.idx`` file is not used for resolving its internal links). + +If an ``importdoc`` directive fails to find a ``.idx``, then an error +is emitted. + +In case of such compilation failures please note that: + +* **all** relative paths, given to ``importdoc``, relate to insides of + ``OUTDIR``, and **not** project's directory structure. + +* ``importdoc`` searches for ``.idx`` in `--outdir:OUTDIR`:option: directory + (``htmldocs`` by default) and **not** around original modules, so: + + .. Tip:: look into ``OUTDIR`` to understand what's going on. + +* also keep in mind that ``.html`` and ``.idx`` files should always be + output to the same directory, so check this and, if it's not true, check + that both runs *with* and *without* `--index:only`:option: have all + other options the same. + +To summarize, for 2 basic options of [Structuring output directories] +compilation options are different: + +1) complex hierarchy with `--project`:option: switch. + + As the **original** project's directory structure is replicated in + `OUTDIR`, all passed paths are related to this structure also. + + E.g. if a module ``path1/module.nim`` does + ``.. importdoc:: path2/another.nim`` then docgen tries to load file + ``OUTDIR/path1/path2/another.idx``. + + .. Note:: markup documents are just placed into the specified directory + `OUTDIR`:option: by default (i.e. they are **not** affected by + `--project`:option:), so if you have ``PROJECT/doc/manual.md`` + document and want to use complex hirearchy (with ``doc/``), + compile it with `--docroot`:option:\: + ```cmd + # 1st stage + nim md2html --outdir:OUTDIR --docroot:/absolute/path/to/PROJECT \ + --index:only PROJECT/doc/manual.md + ... + # 2nd stage + nim md2html --outdir:OUTDIR --docroot:/absolute/path/to/PROJECT \ + PROJECT/doc/manual.md + ``` + + Then the output file will be placed as ``OUTDIR/doc/manual.idx``. + So if you have ``PROJECT/path1/module.nim``, then ``manual.md`` can + be referenced as ``../doc/manual.md``. + +2) flattened structure. + + E.g. if a module ``path1/module.nim`` does + ``.. importdoc:: path2/another.nim`` then docgen tries to load + ``OUTDIR/path2/another.idx``, so the path ``path1`` + does not matter and providing ``path2`` can be useful only + in the case it contains another package that was placed there + using `--outdir:OUTDIR/path2`:option:. + + The links' text will be prefixed as ``another: ...`` in both cases. + + .. Warning:: Again, the same `--outdir:OUTDIR`:option: option should + be provided to both `doc --index:only`:option: / + `md2html --index:only`:option: and final generation by + `doc`:option:/`md2html`:option: inside 1 package. + +To temporarily disable ``importdoc``, e.g. if you don't need +correct link resolution at the moment, use a `--noImportdoc`:option: switch +(only warnings about unresolved links will be generated for external references). + Related Options =============== @@ -434,10 +663,21 @@ index file is line-oriented (newlines have to be escaped). Each line represents a tab-separated record of several columns, the first two mandatory, the rest optional. See the [Index (idx) file format] section for details. +.. Note:: `--index`:option: switch only affects creation of ``.idx`` + index files, while user-searchable Index HTML file is created by + `buildIndex`:option: commmand. + +Buildindex command +------------------ + Once index files have been generated for one or more modules, the Nim -compiler command `buildIndex directory` can be run to go over all the index +compiler command `nim buildIndex directory`:cmd: can be run to go over all the index files in the specified directory to generate a [theindex.html](theindex.html) -file. +file: + + ```cmd + nim buildIndex -o:path/to/htmldocs/theindex.html path/to/htmldocs + ``` See source switch ----------------- @@ -568,10 +808,22 @@ references so they can be later concatenated into a big index file with the file format in detail. Index files are line-oriented and tab-separated (newline and tab characters -have to be escaped). Each line represents a record with at least two fields -but can have up to four (additional columns are ignored). The content of these -columns is: +have to be escaped). Each line represents a record with 6 fields. +The content of these columns is: +0. Discriminator tag denoting type of the index entry, allowed values are: + `markupTitle` + : a title for ``.md``/``.rst`` document + `nimTitle` + : a title of ``.nim`` module + `heading` + : heading of sections, can be both in Nim and markup files + `idx` + : terms marked with :idx: role + `nim` + : a Nim symbol + `nimgrp` + : a Nim group for overloadable symbols like `proc`s 1. Mandatory term being indexed. Terms can include quoting according to Nim's rules (e.g. \`^\`). 2. Base filename plus anchor hyperlink (e.g. ``algorithm.html#*,int,SortOrder``). @@ -581,29 +833,20 @@ columns is: not for an API symbol but for a TOC entry. 4. Optional title or description of the hyperlink. Browsers usually display this as a tooltip after hovering a moment over the hyperlink. +5. A line number of file where the entry was defined. -The index generation tools try to differentiate between documentation -generated from ``.nim`` files and documentation generated from ``.txt`` or -``.rst`` files. The former are always closely related to source code and -consist mainly of API entries. The latter are generic documents meant for -human reading. +The index generation tools differentiate between documentation +generated from ``.nim`` files and documentation generated from ``.md`` or +``.rst`` files by tag `nimTitle` or `markupTitle` in the 1st line of +the ``.idx`` file. -To differentiate both types (documents and APIs), the index generator will add -to the index of documents an entry with the title of the document. Since the -title is the topmost element, it will be added with a second field containing -just the filename without any HTML anchor. By convention, this entry without -anchor is the *title entry*, and since entries in the index file are added as -they are scanned, the title entry will be the first line. The title for APIs -is not present because it can be generated concatenating the name of the file -to the word **Module**. - -Normal symbols are added to the index with surrounding whitespaces removed. An -exception to this are the table of content (TOC) entries. TOC entries are added to -the index file with their third column having as much prefix spaces as their -level is in the TOC (at least 1 character). The prefix whitespace helps to -filter TOC entries from API or text symbols. This is important because the -amount of spaces is used to replicate the hierarchy for document TOCs in the -final index, and TOC entries found in ``.nim`` files are discarded. +.. TODO Normal symbols are added to the index with surrounding whitespaces removed. An + exception to this are the table of content (TOC) entries. TOC entries are added to + the index file with their third column having as much prefix spaces as their + level is in the TOC (at least 1 character). The prefix whitespace helps to + filter TOC entries from API or text symbols. This is important because the + amount of spaces is used to replicate the hierarchy for document TOCs in the + final index, and TOC entries found in ``.nim`` files are discarded. Additional resources @@ -615,6 +858,8 @@ Additional resources [Markdown and RST markup languages](markdown_rst.html), which also contains the list of implemented features of these markup languages. +* the implementation is in [module compiler/docgen]. + The output for HTML and LaTeX comes from the ``config/nimdoc.cfg`` and ``config/nimdoc.tex.cfg`` configuration files. You can add and modify these files to your project to change the look of the docgen output. diff --git a/doc/markdown_rst.md b/doc/markdown_rst.md index 5fb3caefc..a374749cb 100644 --- a/doc/markdown_rst.md +++ b/doc/markdown_rst.md @@ -9,6 +9,8 @@ Nim-flavored Markdown and reStructuredText .. include:: rstcommon.rst .. contents:: +.. importdoc:: docgen.md + Both `Markdown`:idx: (md) and `reStructuredText`:idx: (RST) are markup languages whose goal is to typeset texts with complex structure, formatting and references using simple plaintext representation. @@ -110,6 +112,8 @@ Supported standard RST features: Additional Nim-specific features -------------------------------- +* referencing to definitions in external files, see + [Markup external referencing] section * directives: ``code-block`` \[cmp:Sphinx], ``title``, ``index`` \[cmp:Sphinx] * predefined roles @@ -170,6 +174,86 @@ Optional additional features, by default turned on: .. warning:: Using Nim-specific features can cause other RST implementations to fail on your document. +Referencing +=========== + +To be able to copy and share links Nim generates anchors for all +main document elements: + +* headlines (including document title) +* footnotes +* explicitly set anchors: RST internal cross-references and + inline internal targets +* Nim symbols (external referencing), see [Nim DocGen Tools Guide] for details. + +But direct use of those anchors have 2 problems: + +1. the anchors are usually mangled (e.g. spaces substituted to minus + signs, etc). +2. manual usage of anchors is not checked, so it's easy to get broken + links inside your project if e.g. spelling has changed for a heading + or you use a wrong relative path to your document. + +That's why Nim implementation has syntax for using +*original* labels for referencing. +Such referencing can be either local/internal or external: + +* Local referencing (inside any given file) is defined by + RST standard or Pandoc Markdown User guide. +* External (cross-document) referencing is a Nim-specific feature, + though it's not really different from local referencing by its syntax. + +Markup local referencing +------------------------ + +There are 2 syntax option available for referencing to objects +inside any given file, e.g. for headlines: + + Markdown RST + + Some headline Some headline + ============= ============= + + Ref. [Some headline] Ref. `Some headline`_ + + +Markup external referencing +--------------------------- + +The syntax is the same as for local referencing, but the anchors are +saved in ``.idx`` files, so one needs to generate them beforehand, +and they should be loaded by an `.. importdoc::` directive. +E.g. if we want to reference section "Some headline" in ``file1.md`` +from ``file2.md``, then ``file2.md`` may look like: + +``` +.. importdoc:: file1.md + +Ref. [Some headline] +``` + +```cmd +nim md2html --index:only file1.md # creates ``htmldocs/file1.idx`` +nim md2html file2.md # creates ``htmldocs/file2.html`` +``` + +To allow cross-references between any files in any order (especially, if +circular references are present), it's strongly reccommended +to make a run for creating all the indexes first: + +```cmd +nim md2html --index:only file1.md # creates ``htmldocs/file1.idx`` +nim md2html --index:only file2.md # creates ``htmldocs/file2.idx`` +nim md2html file1.md # creates ``htmldocs/file1.html`` +nim md2html file2.md # creates ``htmldocs/file2.html`` +``` + +and then one can freely reference any objects as if these 2 documents +are actually 1 file. + +Other +===== + Idiosyncrasies -------------- diff --git a/lib/core/macros.nim b/lib/core/macros.nim index bcc16038b..beb3aa695 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -23,6 +23,8 @@ when defined(nimPreviewSlimSystem): ## .. include:: ../../doc/astspec.txt +## .. importdoc:: system.nim + # If you look for the implementation of the magic symbol # ``{.magic: "Foo".}``, search for `mFoo` and `opcFoo`. diff --git a/lib/packages/docutils/dochelpers.nim b/lib/packages/docutils/dochelpers.nim index c7f7f73f5..b7cdd16d2 100644 --- a/lib/packages/docutils/dochelpers.nim +++ b/lib/packages/docutils/dochelpers.nim @@ -13,7 +13,7 @@ ## `type LangSymbol`_ in ``rst.nim``, while `match(generated, docLink)`_ ## matches it with `generated`, produced from `PNode` by ``docgen.rst``. -import rstast +import rstast, strutils when defined(nimPreviewSlimSystem): import std/[assertions, syncio] @@ -35,6 +35,12 @@ type ## name-type seq, e.g. for proc outType*: string ## result type, e.g. for proc +proc `$`*(s: LangSymbol): string = # for debug + ("(symkind=$1, symTypeKind=$2, name=$3, generics=$4, isGroup=$5, " & + "parametersProvided=$6, parameters=$7, outType=$8)") % [ + s.symKind, s.symTypeKind , s.name, s.generics, $s.isGroup, + $s.parametersProvided, $s.parameters, s.outType] + func nimIdentBackticksNormalize*(s: string): string = ## Normalizes the string `s` as a Nim identifier. ## @@ -71,6 +77,12 @@ func nimIdentBackticksNormalize*(s: string): string = else: discard # just omit '`' or ' ' if j != s.len: setLen(result, j) +proc langSymbolGroup*(kind: string, name: string): LangSymbol = + if kind notin ["proc", "func", "macro", "method", "iterator", + "template", "converter"]: + raise newException(ValueError, "unknown symbol kind $1" % [kind]) + result = LangSymbol(symKind: kind, name: name, isGroup: true) + proc toLangSymbol*(linkText: PRstNode): LangSymbol = ## Parses `linkText` into a more structured form using a state machine. ## @@ -82,11 +94,14 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = ## ## This proc should be kept in sync with the `renderTypes` proc from ## ``compiler/typesrenderer.nim``. - assert linkText.kind in {rnRstRef, rnInner} + template fail(msg: string) = + raise newException(ValueError, msg) + if linkText.kind notin {rnRstRef, rnInner}: + fail("toLangSymbol: wrong input kind " & $linkText.kind) const NimDefs = ["proc", "func", "macro", "method", "iterator", "template", "converter", "const", "type", "var", - "enum", "object", "tuple"] + "enum", "object", "tuple", "module"] template resolveSymKind(x: string) = if x in ["enum", "object", "tuple"]: result.symKind = "type" @@ -109,11 +124,11 @@ proc toLangSymbol*(linkText: PRstNode): LangSymbol = template flushIdent() = if curIdent != "": case state - of inBeginning: doAssert false, "incorrect state inBeginning" + of inBeginning: fail("incorrect state inBeginning") of afterSymKind: resolveSymKind curIdent - of beforeSymbolName: doAssert false, "incorrect state beforeSymbolName" + of beforeSymbolName: fail("incorrect state beforeSymbolName") of atSymbolName: result.name = curIdent.nimIdentBackticksNormalize - of afterSymbolName: doAssert false, "incorrect state afterSymbolName" + of afterSymbolName: fail("incorrect state afterSymbolName") of genericsPar: result.generics = curIdent of parameterName: result.parameters.add (curIdent, "") of parameterType: diff --git a/lib/packages/docutils/rst.nim b/lib/packages/docutils/rst.nim index e9f76a26f..b3b1fb9c0 100644 --- a/lib/packages/docutils/rst.nim +++ b/lib/packages/docutils/rst.nim @@ -22,7 +22,7 @@ import os, strutils, rstast, dochelpers, std/enumutils, algorithm, lists, sequtils, - std/private/miscdollars, tables, strscans + std/private/miscdollars, tables, strscans, rstidx from highlite import SourceLanguage, getSourceLanguage when defined(nimPreviewSlimSystem): @@ -40,7 +40,7 @@ type roNimFile ## set for Nim files where default interpreted ## text role should be :nim: roSandboxDisabled ## this option enables certain options - ## (e.g. raw, include) + ## (e.g. raw, include, importdoc) ## which are disabled by default as they can ## enable users to read arbitrary data and ## perform XSS if the parser is used in a web @@ -73,11 +73,17 @@ type mwUnsupportedLanguage = "language '$1' not supported", mwUnsupportedField = "field '$1' not supported", mwRstStyle = "RST style: $1", + mwUnusedImportdoc = "importdoc for '$1' is not used", meSandboxedDirective = "disabled directive: '$1'", MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind, arg: string) {.closure, gcsafe.} ## what to do in case of an error FindFileHandler* = proc (filename: string): string {.closure, gcsafe.} + FindRefFileHandler* = + proc (targetRelPath: string): + tuple[targetPath: string, linkRelPath: string] {.closure, gcsafe.} + ## returns where .html or .idx file should be found by its relative path; + ## `linkRelPath` is a prefix to be added before a link anchor from such file proc rstnodeToRefname*(n: PRstNode): string proc addNodes*(n: PRstNode): string @@ -333,7 +339,8 @@ type arInternalRst, ## For automatically generated RST anchors (from ## headings, footnotes, inline internal targets): ## case-insensitive, 1-space-significant (by RST spec) - arNim ## For anchors generated by ``docgen.rst``: Nim-style case + arExternalRst, ## For external .nim doc comments or .rst/.md + arNim ## For anchors generated by ``docgen.nim``: Nim-style case ## sensitivity, etc. (see `proc normalizeNimName`_ for details) arHyperlink, ## For links with manually set anchors in ## form `text `_ @@ -349,11 +356,15 @@ type of arInternalRst: anchorType: RstAnchorKind target: PRstNode + of arExternalRst: + anchorTypeExt: RstAnchorKind + refnameExt: string of arNim: tooltip: string # displayed tooltip for Nim-generated anchors langSym: LangSymbol refname: string # A reference name that will be inserted directly # into HTML/Latex. + external: bool AnchorSubstTable = Table[string, seq[AnchorSubst]] # use `seq` to account for duplicate anchors FootnoteType = enum @@ -372,6 +383,12 @@ type RstFileTable* = object filenameToIdx*: Table[string, FileIndex] idxToFilename*: seq[string] + ImportdocInfo = object + used: bool # was this import used? + fromInfo: TLineInfo # place of `.. importdoc::` directive + idxPath: string # full path to ``.idx`` file + linkRelPath: string # prefix before target anchor + title: string # document title obtained from ``.idx`` RstSharedState = object options*: RstParseOptions # parsing options hLevels: LevelMap # hierarchy of heading styles @@ -393,12 +410,17 @@ type footnotes: seq[FootnoteSubst] # correspondence b/w footnote label, # number, order of occurrence msgHandler: MsgHandler # How to handle errors. - findFile: FindFileHandler # How to find files. + findFile: FindFileHandler # How to find files for include. + findRefFile: FindRefFileHandler + # How to find files imported by importdoc. filenames*: RstFileTable # map file name <-> FileIndex (for storing # file names for warnings after 1st stage) currFileIdx*: FileIndex # current index in `filenames` tocPart*: seq[PRstNode] # all the headings of a document hasToc*: bool + idxImports*: Table[string, ImportdocInfo] + # map `importdoc`ed filename -> it's info + nimFileImported*: bool # Was any ``.nim`` module `importdoc`ed ? PRstSharedState* = ref RstSharedState ManualAnchor = object @@ -452,6 +474,9 @@ proc defaultFindFile*(filename: string): string = if fileExists(filename): result = filename else: result = "" +proc defaultFindRefFile*(filename: string): (string, string) = + (filename, "") + proc defaultRole(options: RstParseOptions): string = if roNimFile in options: "nim" else: "literal" @@ -492,12 +517,16 @@ proc getFilename(filenames: RstFileTable, fid: FileIndex): string = $fid.int, $(filenames.len - 1)]) result = filenames.idxToFilename[fid.int] +proc getFilename(s: PRstSharedState, subst: AnchorSubst): string = + getFilename(s.filenames, subst.info.fileIndex) + proc currFilename(s: PRstSharedState): string = getFilename(s.filenames, s.currFileIdx) proc newRstSharedState*(options: RstParseOptions, filename: string, findFile: FindFileHandler, + findRefFile: FindRefFileHandler, msgHandler: MsgHandler, hasToc: bool): PRstSharedState = let r = defaultRole(options) @@ -507,6 +536,9 @@ proc newRstSharedState*(options: RstParseOptions, options: options, msgHandler: if not isNil(msgHandler): msgHandler else: defaultMsgHandler, findFile: if not isNil(findFile): findFile else: defaultFindFile, + findRefFile: + if not isNil(findRefFile): findRefFile + else: defaultFindRefFile, hasToc: hasToc ) setCurrFilename(result, filename) @@ -525,6 +557,14 @@ proc rstMessage(p: RstParser, msgKind: MsgKind, arg: string) = proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string) = s.msgHandler(s.currFilename, LineRstInit, ColRstInit, msgKind, arg) +proc rstMessage(s: PRstSharedState, msgKind: MsgKind, arg: string; + line, col: int) = + s.msgHandler(s.currFilename, line, col, msgKind, arg) + +proc rstMessage(s: PRstSharedState, filename: string, msgKind: MsgKind, + arg: string) = + s.msgHandler(filename, LineRstInit, ColRstInit, msgKind, arg) + proc rstMessage*(filenames: RstFileTable, f: MsgHandler, info: TLineInfo, msgKind: MsgKind, arg: string) = ## Print warnings using `info`, i.e. in 2nd-pass warnings for @@ -756,6 +796,14 @@ proc internalRefPriority(k: RstAnchorKind): int = of footnoteAnchor: result = 4 of headlineAnchor: result = 3 +proc `$`(subst: AnchorSubst): string = # for debug + let s = + case subst.kind + of arInternalRst: "type=" & $subst.anchorType + of arExternalRst: "type=" & $subst.anchorTypeExt + of arNim: "langsym=" & $subst.langSym + result = "(kind=$1, priority=$2, $3)" % [$subst.kind, $subst.priority, s] + proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, anchorType: RstAnchorKind) = ## Associates node `target` (which has field `anchor`) with an @@ -771,31 +819,49 @@ proc addAnchorRst(p: var RstParser, name: string, target: PRstNode, info: prevLineInfo(p), anchorType: anchorType)) p.curAnchors.setLen 0 -proc addAnchorNim*(s: var PRstSharedState, refn: string, tooltip: string, +proc addAnchorExtRst(s: var PRstSharedState, key: string, refn: string, + anchorType: RstAnchorKind, info: TLineInfo) = + let name = key.toLowerAscii + let prio = internalRefPriority(anchorType) + s.anchors.mgetOrPut(name, newSeq[AnchorSubst]()).add( + AnchorSubst(kind: arExternalRst, refnameExt: refn, priority: prio, + info: info, + anchorTypeExt: anchorType)) + +proc addAnchorNim*(s: var PRstSharedState, external: bool, refn: string, tooltip: string, langSym: LangSymbol, priority: int, info: TLineInfo) = ## Adds an anchor `refn`, which follows ## the rule `arNim` (i.e. a symbol in ``*.nim`` file) s.anchors.mgetOrPut(langSym.name, newSeq[AnchorSubst]()).add( - AnchorSubst(kind: arNim, refname: refn, langSym: langSym, + AnchorSubst(kind: arNim, external: external, refname: refn, langSym: langSym, tooltip: tooltip, priority: priority, info: info)) proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, info: TLineInfo): seq[AnchorSubst] = - let langSym = toLangSymbol(signature) + var langSym: LangSymbol + try: + langSym = toLangSymbol(signature) + except ValueError: # parsing failed, not a Nim symbol + return let substitutions = s.anchors.getOrDefault(langSym.name, newSeq[AnchorSubst]()) if substitutions.len == 0: return - # map symKind (like "proc") -> found symbols/groups: - var found: Table[string, seq[AnchorSubst]] - for s in substitutions: - if s.kind == arNim: - if match(s.langSym, langSym): - found.mgetOrPut(s.langSym.symKind, newSeq[AnchorSubst]()).add s - for symKind, sList in found: + # logic to select only groups instead of concrete symbols + # with overloads, note that the same symbol can be defined + # in multiple modules and `importdoc`ed: + type GroupKey = tuple[symKind: string, origModule: string] + # map (symKind, file) (like "proc", "os.nim") -> found symbols/groups: + var found: Table[GroupKey, seq[AnchorSubst]] + for subst in substitutions: + if subst.kind == arNim: + if match(subst.langSym, langSym): + let key: GroupKey = (subst.langSym.symKind, getFilename(s, subst)) + found.mgetOrPut(key, newSeq[AnchorSubst]()).add subst + for key, sList in found: if sList.len == 1: result.add sList[0] else: # > 1, there are overloads, potential ambiguity in this `symKind` @@ -812,14 +878,16 @@ proc findMainAnchorNim(s: PRstSharedState, signature: PRstNode, result.add s foundGroup = true break - doAssert foundGroup, "docgen has not generated the group" + doAssert(foundGroup, + "docgen has not generated the group for $1 (file $2)" % [ + langSym.name, getFilename(s, sList[0]) ]) proc findMainAnchorRst(s: PRstSharedState, linkText: string, info: TLineInfo): seq[AnchorSubst] = let name = linkText.toLowerAscii let substitutions = s.anchors.getOrDefault(name, newSeq[AnchorSubst]()) for s in substitutions: - if s.kind == arInternalRst: + if s.kind in {arInternalRst, arExternalRst}: result.add s proc addFootnoteNumManual(p: var RstParser, num: int) = @@ -3251,6 +3319,15 @@ proc dirRaw(p: var RstParser): PRstNode = else: dirRawAux(p, result, rnRaw, parseSectionWrapper) +proc dirImportdoc(p: var RstParser): PRstNode = + result = parseDirective(p, rnDirective, {}, parseLiteralBlock) + assert result.sons[2].kind == rnLiteralBlock + assert result.sons[2].sons[0].kind == rnLeaf + let filenames: seq[string] = split(result.sons[2].sons[0].text, seps = {','}) + proc rmSpaces(s: string): string = s.split.join("") + for origFilename in filenames: + p.s.idxImports[origFilename.rmSpaces] = ImportdocInfo(fromInfo: lineInfo(p)) + proc selectDir(p: var RstParser, d: string): PRstNode = result = nil let tok = p.tok[p.idx-2] # report on directive in ".. directive::" @@ -3271,6 +3348,7 @@ proc selectDir(p: var RstParser, d: string): PRstNode = of "hint": result = dirAdmonition(p, d) of "image": result = dirImage(p) of "important": result = dirAdmonition(p, d) + of "importdoc": result = dirImportdoc(p) of "include": result = dirInclude(p) of "index": result = dirIndex(p) of "note": result = dirAdmonition(p, d) @@ -3401,11 +3479,90 @@ proc rstParsePass1*(fragment: string, getTokens(fragment, p.tok) result = parseDoc(p) -proc preparePass2*(s: PRstSharedState, mainNode: PRstNode) = +proc extractLinkEnd(x: string): string = + ## From links like `path/to/file.html#/%` extract `file.html#/%`. + let i = find(x, '#') + let last = + if i >= 0: i + else: x.len - 1 + let j = rfind(x, '/', start=0, last=last) + if j >= 0: + result = x[j+1 .. ^1] + else: + result = x + +proc loadIdxFile(s: var PRstSharedState, origFilename: string) = + doAssert roSandboxDisabled in s.options + var info: TLineInfo + info.fileIndex = addFilename(s, origFilename) + var (dir, basename, ext) = origFilename.splitFile + if ext notin [".md", ".rst", ".nim", ""]: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, origFilename & ": unknown extension") + let idxFilename = dir / basename & ".idx" + let (idxPath, linkRelPath) = s.findRefFile(idxFilename) + s.idxImports[origFilename].linkRelPath = linkRelPath + var + fileEntries: seq[IndexEntry] + title: IndexEntry + try: + (fileEntries, title) = parseIdxFile(idxPath) + except IOError: + rstMessage(s.filenames, s.msgHandler, s.idxImports[origFilename].fromInfo, + meCannotOpenFile, idxPath) + except ValueError as e: + s.msgHandler(idxPath, LineRstInit, ColRstInit, meInvalidField, e.msg) + + var isMarkup = false # for sanity check to avoid mixing .md <-> .nim + for entry in fileEntries: + # Though target .idx already has inside it the path to HTML relative + # project's root, we won't rely on it and use `linkRelPath` instead. + let refn = extractLinkEnd(entry.link) + # select either markup (rst/md) or Nim cases: + if entry.kind in {ieMarkupTitle, ieNimTitle}: + s.idxImports[origFilename].title = entry.keyword + case entry.kind + of ieIdxRole, ieHeading, ieMarkupTitle: + if ext == ".nim" and entry.kind == ieMarkupTitle: + rstMessage(s, idxPath, meInvalidField, + $ieMarkupTitle & " in supposedly .nim-derived file") + if entry.kind == ieMarkupTitle: + isMarkup = true + info.line = entry.line.uint16 + addAnchorExtRst(s, key = entry.keyword, refn = refn, + anchorType = headlineAnchor, info=info) + of ieNim, ieNimGroup, ieNimTitle: + if ext in [".md", ".rst"] or isMarkup: + rstMessage(s, idxPath, meInvalidField, + $entry.kind & " in supposedly markup-derived file") + s.nimFileImported = true + var langSym: LangSymbol + if entry.kind in {ieNim, ieNimTitle}: + var q: RstParser + initParser(q, s) + info.line = entry.line.uint16 + setLen(q.tok, 0) + q.idx = 0 + getTokens(entry.linkTitle, q.tok) + var sons = newSeq[PRstNode](q.tok.len) + for i in 0 ..< q.tok.len: sons[i] = newLeaf(q.tok[i].symbol) + let linkTitle = newRstNode(rnInner, sons) + langSym = linkTitle.toLangSymbol + else: # entry.kind == ieNimGroup + langSym = langSymbolGroup(kind=entry.linkTitle, name=entry.keyword) + addAnchorNim(s, external = true, refn = refn, tooltip = entry.linkDesc, + langSym = langSym, priority = -4, # lowest + info=info) + doAssert s.idxImports[origFilename].title != "" + +proc preparePass2*(s: var PRstSharedState, mainNode: PRstNode, importdoc = true) = ## Records titles in node `mainNode` and orders footnotes. countTitles(s, mainNode) fixHeadlines(s) orderFootnotes(s) + if importdoc: + for origFilename in s.idxImports.keys: + loadIdxFile(s, origFilename) proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = # Associate this link alias with its target and change node kind to @@ -3423,6 +3580,9 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = tooltip: string target: PRstNode info: TLineInfo + externFilename: string + # when external anchor: origin filename where anchor was defined + isTitle: bool proc cmp(x, y: LinkDef): int = result = cmp(x.priority, y.priority) if result == 0: @@ -3435,26 +3595,67 @@ proc resolveLink(s: PRstSharedState, n: PRstNode) : PRstNode = target: y.value, info: y.info, tooltip: "(" & $y.kind & ")") let substRst = findMainAnchorRst(s, alias.addNodes, n.info) + template getExternFilename(subst: AnchorSubst): string = + if subst.kind == arExternalRst or + (subst.kind == arNim and subst.external): + getFilename(s, subst) + else: "" for subst in substRst: - foundLinks.add LinkDef(ar: arInternalRst, priority: subst.priority, - target: newLeaf(subst.target.anchor), + var refname, fullRefname: string + if subst.kind == arInternalRst: + refname = subst.target.anchor + fullRefname = refname + else: # arExternalRst + refname = subst.refnameExt + fullRefname = s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & refname + let anchorType = + if subst.kind == arInternalRst: subst.anchorType + else: subst.anchorTypeExt # arExternalRst + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), info: subst.info, - tooltip: "(" & $subst.anchorType & ")") + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(refname), + tooltip: "(" & $anchorType & ")") # find anchors automatically generated from Nim symbols - if roNimFile in s.options: + if roNimFile in s.options or s.nimFileImported: let substNim = findMainAnchorNim(s, signature=alias, n.info) for subst in substNim: - foundLinks.add LinkDef(ar: arNim, priority: subst.priority, - target: newLeaf(subst.refname), + let fullRefname = + if subst.external: + s.idxImports[getFilename(s, subst)].linkRelPath & + "/" & subst.refname + else: subst.refname + foundLinks.add LinkDef(ar: subst.kind, priority: subst.priority, + target: newLeaf(fullRefname), + externFilename: getExternFilename(subst), + isTitle: isDocumentationTitle(subst.refname), info: subst.info, tooltip: subst.tooltip) foundLinks.sort(cmp = cmp, order = Descending) let aliasStr = addNodes(alias) if foundLinks.len >= 1: - let kind = if foundLinks[0].ar == arHyperlink: rnHyperlink - elif foundLinks[0].ar == arNim: rnNimdocRef + if foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].used = true + let kind = if foundLinks[0].ar in {arHyperlink, arExternalRst}: rnHyperlink + elif foundLinks[0].ar == arNim: + if foundLinks[0].externFilename == "": rnNimdocRef + else: rnHyperlink else: rnInternalRef result = newRstNode(kind) - result.sons = @[newRstNode(rnInner, desc.sons), foundLinks[0].target] + let documentName = # filename without ext for `.nim`, title for `.md` + if foundLinks[0].ar == arNim: + changeFileExt(foundLinks[0].externFilename.extractFilename, "") + elif foundLinks[0].externFilename != "": + s.idxImports[foundLinks[0].externFilename].title + else: foundLinks[0].externFilename.extractFilename + let linkText = + if foundLinks[0].externFilename != "": + if foundLinks[0].isTitle: newLeaf(addNodes(desc)) + else: newLeaf(documentName & ": " & addNodes(desc)) + else: + newRstNode(rnInner, desc.sons) + result.sons = @[linkText, foundLinks[0].target] if kind == rnNimdocRef: result.tooltip = foundLinks[0].tooltip if foundLinks.len > 1: # report ambiguous link var targets = newSeq[string]() @@ -3568,20 +3769,28 @@ proc resolveSubs*(s: PRstSharedState, n: PRstNode): PRstNode = inc i result.sons = newSons +proc completePass2*(s: PRstSharedState) = + for (filename, importdocInfo) in s.idxImports.pairs: + if not importdocInfo.used: + rstMessage(s.filenames, s.msgHandler, importdocInfo.fromInfo, + mwUnusedImportdoc, filename) + proc rstParse*(text, filename: string, line, column: int, options: RstParseOptions, findFile: FindFileHandler = nil, + findRefFile: FindRefFileHandler = nil, msgHandler: MsgHandler = nil): tuple[node: PRstNode, filenames: RstFileTable, hasToc: bool] = ## Parses the whole `text`. The result is ready for `rstgen.renderRstToOut`, ## note that 2nd tuple element should be fed to `initRstGenerator` ## argument `filenames` (it is being filled here at least with `filename` ## and possibly with other files from RST ``.. include::`` statement). - var sharedState = newRstSharedState(options, filename, findFile, + var sharedState = newRstSharedState(options, filename, findFile, findRefFile, msgHandler, hasToc=false) let unresolved = rstParsePass1(text, line, column, sharedState) preparePass2(sharedState, unresolved) result.node = resolveSubs(sharedState, unresolved) + completePass2(sharedState) result.filenames = sharedState.filenames result.hasToc = sharedState.hasToc diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 597ee1553..57bc00fcb 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -39,7 +39,8 @@ ## No backreferences are generated since finding all references of a footnote ## can be done by simply searching for ``[footnoteName]``. -import strutils, os, hashes, strtabs, rstast, rst, highlite, tables, sequtils, +import strutils, os, hashes, strtabs, rstast, rst, rstidx, + highlite, tables, sequtils, algorithm, parseutils, std/strbasics @@ -59,7 +60,7 @@ type outLatex # output is Latex MetaEnum* = enum - metaNone, metaTitle, metaSubtitle, metaAuthor, metaVersion + metaNone, metaTitleRaw, metaTitle, metaSubtitle, metaAuthor, metaVersion EscapeMode* = enum # in Latex text inside options [] and URLs is # escaped slightly differently than in normal text @@ -321,31 +322,8 @@ proc renderAux(d: PDoc, n: PRstNode, html, tex: string, result: var string) = # ---------------- index handling -------------------------------------------- -proc quoteIndexColumn(text: string): string = - ## Returns a safe version of `text` for serialization to the ``.idx`` file. - ## - ## The returned version can be put without worries in a line based tab - ## separated column text file. The following character sequence replacements - ## will be performed for that goal: - ## - ## * ``"\\"`` => ``"\\\\"`` - ## * ``"\n"`` => ``"\\n"`` - ## * ``"\t"`` => ``"\\t"`` - result = newStringOfCap(text.len + 3) - for c in text: - case c - of '\\': result.add "\\" - of '\L': result.add "\\n" - of '\C': discard - of '\t': result.add "\\t" - else: result.add c - -proc unquoteIndexColumn(text: string): string = - ## Returns the unquoted version generated by ``quoteIndexColumn``. - result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) - -proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string, - linkTitle, linkDesc = "") = +proc setIndexTerm*(d: var RstGenerator; k: IndexEntryKind, htmlFile, id, term: string, + linkTitle, linkDesc = "", line = 0) = ## Adds a `term` to the index using the specified hyperlink identifier. ## ## A new entry will be added to the index using the format @@ -368,21 +346,8 @@ proc setIndexTerm*(d: var RstGenerator, htmlFile, id, term: string, ## <#writeIndexFile,RstGenerator,string>`_. The purpose of the index is ## documented in the `docgen tools guide ## `_. - var - entry = term - isTitle = false - entry.add('\t') - entry.add(htmlFile) - if id.len > 0: - entry.add('#') - entry.add(id) - else: - isTitle = true - if linkTitle.len > 0 or linkDesc.len > 0: - entry.add('\t' & linkTitle.quoteIndexColumn) - entry.add('\t' & linkDesc.quoteIndexColumn) - entry.add("\n") - + let (entry, isTitle) = formatIndexEntry(k, htmlFile, id, term, + linkTitle, linkDesc, line) if isTitle: d.theIndex.insert(entry) else: d.theIndex.add(entry) @@ -395,6 +360,15 @@ proc hash(n: PRstNode): int = result = result !& hash(n.sons[i]) result = !$result +proc htmlFileRelPath(d: PDoc): string = + if d.outDir.len == 0: + # /foo/bar/zoo.nim -> zoo.html + changeFileExt(extractFilename(d.filename), HtmlExt) + else: # d is initialized in docgen.nim + # outDir = /foo -\ + # destFile = /foo/bar/zoo.html -|-> bar/zoo.html + d.destFile.relativePath(d.outDir, '/') + proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = ## Renders the string decorated within \`foobar\`\:idx\: markers. ## @@ -411,17 +385,12 @@ proc renderIndexTerm*(d: PDoc, n: PRstNode, result: var string) = var term = "" renderAux(d, n, term) - setIndexTerm(d, changeFileExt(extractFilename(d.filename), HtmlExt), id, term, d.currentSection) + setIndexTerm(d, ieIdxRole, + htmlFileRelPath(d), id, term, d.currentSection) dispA(d.target, result, "$2", "\\nimindexterm{$1}{$2}", [id, term]) type - IndexEntry* = object - keyword*: string - link*: string - linkTitle*: string ## contains a prettier text for the href - linkDesc*: string ## the title attribute of the final href - IndexedDocs* = Table[IndexEntry, seq[IndexEntry]] ## \ ## Contains the index sequences for doc types. ## @@ -432,21 +401,6 @@ type ## The value indexed by this IndexEntry is a sequence with the real index ## entries found in the ``.idx`` file. -proc cmp(a, b: IndexEntry): int = - ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. - result = cmpIgnoreStyle(a.keyword, b.keyword) - if result == 0: - result = cmpIgnoreStyle(a.link, b.link) - -proc hash(x: IndexEntry): Hash = - ## Returns the hash for the combined fields of the type. - ## - ## The hash is computed as the chained hash of the individual string hashes. - result = x.keyword.hash !& x.link.hash - result = result !& x.linkTitle.hash - result = result !& x.linkDesc.hash - result = !$result - when defined(gcDestructors): template `<-`(a, b: var IndexEntry) = a = move(b) else: @@ -455,6 +409,7 @@ else: shallowCopy a.link, b.link shallowCopy a.linkTitle, b.linkTitle shallowCopy a.linkDesc, b.linkDesc + shallowCopy a.module, b.module proc sortIndex(a: var openArray[IndexEntry]) = # we use shellsort here; fast and simple @@ -494,16 +449,20 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = result = "
          " var i = 0 while i < symbols.len: - let keyword = symbols[i].keyword + let keyword = esc(outHtml, symbols[i].keyword) let cleanedKeyword = keyword.escapeLink result.addf("
          $1:
            \n", [keyword, cleanedKeyword]) var j = i - while j < symbols.len and keyword == symbols[j].keyword: + while j < symbols.len and symbols[i].keyword == symbols[j].keyword: let url = symbols[j].link.escapeLink - text = if symbols[j].linkTitle.len > 0: symbols[j].linkTitle else: url - desc = if symbols[j].linkDesc.len > 0: symbols[j].linkDesc else: "" + module = symbols[j].module + text = + if symbols[j].linkTitle.len > 0: + esc(outHtml, module & ": " & symbols[j].linkTitle) + else: url + desc = symbols[j].linkDesc if desc.len > 0: result.addf("""
          • $2
            • @@ -517,13 +476,6 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = i = j result.add("
          ") -proc isDocumentationTitle(hyperlink: string): bool = - ## Returns true if the hyperlink is actually a documentation title. - ## - ## Documentation titles lack the hash. See `mergeIndexes() - ## <#mergeIndexes,string>`_ for a more detailed explanation. - result = hyperlink.find('#') < 0 - proc stripTocLevel(s: string): tuple[level: int, text: string] = ## Returns the *level* of the toc along with the text without it. for c in 0 ..< s.len: @@ -557,17 +509,15 @@ proc generateDocumentationToc(entries: seq[IndexEntry]): string = level = 1 levels.newSeq(entries.len) for entry in entries: - let (rawLevel, rawText) = stripTocLevel(entry.linkTitle or entry.keyword) + let (rawLevel, rawText) = stripTocLevel(entry.linkTitle) if rawLevel < 1: # This is a normal symbol, push it *inside* one level from the last one. levels[L].level = level + 1 - # Also, ignore the linkTitle and use directly the keyword. - levels[L].text = entry.keyword else: # The level did change, update the level indicator. level = rawLevel levels[L].level = rawLevel - levels[L].text = rawText + levels[L].text = rawText inc L # Now generate hierarchical lists based on the precalculated levels. @@ -598,7 +548,7 @@ proc generateDocumentationIndex(docs: IndexedDocs): string = for title in titles: let tocList = generateDocumentationToc(docs.getOrDefault(title)) result.add("\n") + title.link & "\">" & title.linkTitle & "\n" & tocList & "
        \n") proc generateDocumentationJumps(docs: IndexedDocs): string = ## Returns a plain list of hyperlinks to documentation TOCs in HTML. @@ -610,7 +560,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string = var chunks: seq[string] = @[] for title in titles: - chunks.add("" & title.keyword & "") + chunks.add("" & title.linkTitle & "") result.add(chunks.join(", ") & ".
        ") @@ -639,39 +589,11 @@ proc readIndexDir*(dir: string): # Scan index files and build the list of symbols. for path in walkDirRec(dir): if path.endsWith(IndexExt): - var - fileEntries: seq[IndexEntry] - title: IndexEntry - f = 0 - newSeq(fileEntries, 500) - setLen(fileEntries, 0) - for line in lines(path): - let s = line.find('\t') - if s < 0: continue - setLen(fileEntries, f+1) - fileEntries[f].keyword = line.substr(0, s-1) - fileEntries[f].link = line.substr(s+1) - # See if we detect a title, a link without a `#foobar` trailing part. - if title.keyword.len == 0 and fileEntries[f].link.isDocumentationTitle: - title.keyword = fileEntries[f].keyword - title.link = fileEntries[f].link - - if fileEntries[f].link.find('\t') > 0: - let extraCols = fileEntries[f].link.split('\t') - fileEntries[f].link = extraCols[0] - assert extraCols.len == 3 - fileEntries[f].linkTitle = extraCols[1].unquoteIndexColumn - fileEntries[f].linkDesc = extraCols[2].unquoteIndexColumn - else: - fileEntries[f].linkTitle = "" - fileEntries[f].linkDesc = "" - inc f + var (fileEntries, title) = parseIdxFile(path) # Depending on type add this to the list of symbols or table of APIs. - if title.keyword.len == 0: - for i in 0 ..< f: - # Don't add to symbols TOC entries (they start with a whitespace). - let toc = fileEntries[i].linkTitle - if toc.len > 0 and toc[0] == ' ': + if title.kind == ieNimTitle: + for i in 0 ..< fileEntries.len: + if fileEntries[i].kind != ieNim: continue # Ok, non TOC entry, add it. setLen(result.symbols, L + 1) @@ -687,7 +609,7 @@ proc readIndexDir*(dir: string): result.modules.add(x.changeFileExt("")) else: # Generate the symbolic anchor for index quickjumps. - title.linkTitle = "doc_toc_" & $result.docs.len + title.aux = "doc_toc_" & $result.docs.len result.docs[title] = fileEntries proc mergeIndexes*(dir: string): string = @@ -747,24 +669,6 @@ proc mergeIndexes*(dir: string): string = # ---------------------------------------------------------------------------- -proc stripTocHtml(s: string): string = - ## Ugly quick hack to remove HTML tags from TOC titles. - ## - ## A TocEntry.header field already contains rendered HTML tags. Instead of - ## implementing a proper version of renderRstToOut() which recursively - ## renders an rst tree to plain text, we simply remove text found between - ## angled brackets. Given the limited possibilities of rst inside TOC titles - ## this should be enough. - result = s - var first = result.find('<') - while first >= 0: - let last = result.find('>', first) - if last < 0: - # Abort, since we didn't found a closing angled bracket. - return - result.delete(first..last) - first = result.find('<', first) - proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = var tmp = "" for i in countup(0, len(n) - 1): renderRstToOut(d, n.sons[i], tmp) @@ -785,19 +689,12 @@ proc renderHeadline(d: PDoc, n: PRstNode, result: var string) = # Generate index entry using spaces to indicate TOC level for the output HTML. assert n.level >= 0 - let - htmlFileRelPath = if d.outDir.len == 0: - # /foo/bar/zoo.nim -> zoo.html - changeFileExt(extractFilename(d.filename), HtmlExt) - else: # d is initialized in docgen.nim - # outDir = /foo -\ - # destFile = /foo/bar/zoo.html -|-> bar/zoo.html - d.destFile.relativePath(d.outDir, '/') - setIndexTerm(d, htmlFileRelPath, n.anchor, tmp.stripTocHtml, - spaces(max(0, n.level)) & tmp) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) proc renderOverline(d: PDoc, n: PRstNode, result: var string) = if n.level == 0 and d.meta[metaTitle].len == 0: + d.meta[metaTitleRaw] = n.addNodes for i in countup(0, len(n)-1): renderRstToOut(d, n.sons[i], d.meta[metaTitle]) d.currentSection = d.meta[metaTitle] @@ -813,6 +710,8 @@ proc renderOverline(d: PDoc, n: PRstNode, result: var string) = dispA(d.target, result, "
        $3
        ", "\\rstov$4[$5]{$3}$2\n", [$n.level, n.anchor.idS, tmp, $chr(n.level - 1 + ord('A')), tocName]) + setIndexTerm(d, ieHeading, htmlFile = d.htmlFileRelPath, id = n.anchor, + term = n.addNodes, linkTitle = spaces(max(0, n.level)) & tmp) proc renderTocEntry(d: PDoc, n: PRstNode, result: var string) = var header = "" @@ -1197,6 +1096,18 @@ proc renderHyperlink(d: PDoc, text, link: PRstNode, result: var string, "\\hyperlink{$2}{$1} (p.~\\pageref{$2})", [textStr, linkStr, nimDocStr, tooltipStr]) +proc traverseForIndex*(d: PDoc, n: PRstNode) = + ## A version of [renderRstToOut] that only fills entries for ``.idx`` files. + var discarded: string + if n == nil: return + case n.kind + of rnIdx: renderIndexTerm(d, n, discarded) + of rnHeadline, rnMarkdownHeadline: renderHeadline(d, n, discarded) + of rnOverline: renderOverline(d, n, discarded) + else: + for i in 0 ..< len(n): + traverseForIndex(d, n.sons[i]) + proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = if n == nil: return case n.kind @@ -1451,6 +1362,7 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTitle: d.meta[metaTitle] = "" renderRstToOut(d, n.sons[0], d.meta[metaTitle]) + d.meta[metaTitleRaw] = n.sons[0].addNodes # ----------------------------------------------------------------------------- @@ -1616,11 +1528,13 @@ proc rstToHtml*(s: string, options: RstParseOptions, proc myFindFile(filename: string): string = # we don't find any files in online mode: result = "" + proc myFindRefFile(filename: string): (string, string) = + result = ("", "") const filen = "input" let (rst, filenames, t) = rstParse(s, filen, line=LineRstInit, column=ColRstInit, - options, myFindFile, msgHandler) + options, myFindFile, myFindRefFile, msgHandler) var d: RstGenerator initRstGenerator(d, outHtml, config, filen, myFindFile, msgHandler, filenames, hasToc = t) diff --git a/lib/packages/docutils/rstidx.nim b/lib/packages/docutils/rstidx.nim new file mode 100644 index 000000000..c109636d7 --- /dev/null +++ b/lib/packages/docutils/rstidx.nim @@ -0,0 +1,138 @@ +# +# Nim's Runtime Library +# (c) Copyright 2022 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## Nim `idx`:idx: file format related definitions. + +import strutils, std/syncio, hashes +from os import splitFile + +type + IndexEntryKind* = enum ## discriminator tag + ieMarkupTitle = "markupTitle" + ## RST/Markdown title, text in `keyword` + + ## HTML text in `linkTitle` + ieNimTitle = "nimTitle" + ## Nim title + ieHeading = "heading" ## RST/Markdown markup heading, escaped + ieIdxRole = "idx" ## RST :idx: definition, escaped + ieNim = "nim" ## Nim symbol, unescaped + ieNimGroup = "nimgrp" ## Nim overload group, unescaped + IndexEntry* = object + kind*: IndexEntryKind ## 0. + keyword*: string ## 1. + link*: string ## 2. + linkTitle*: string ## 3. contains a prettier text for the href + linkDesc*: string ## 4. the title attribute of the final href + line*: int ## 5. + module*: string ## origin file, NOT a field in ``.idx`` file + aux*: string ## auxuliary field, NOT a field in ``.idx`` file + +proc isDocumentationTitle*(hyperlink: string): bool = + ## Returns true if the hyperlink is actually a documentation title. + ## + ## Documentation titles lack the hash. See `mergeIndexes() + ## <#mergeIndexes,string>`_ for a more detailed explanation. + result = hyperlink.find('#') < 0 + +proc `$`*(e: IndexEntry): string = + """("$1", "$2", "$3", "$4", $5)""" % [ + e.keyword, e.link, e.linkTitle, e.linkDesc, $e.line] + +proc quoteIndexColumn(text: string): string = + ## Returns a safe version of `text` for serialization to the ``.idx`` file. + ## + ## The returned version can be put without worries in a line based tab + ## separated column text file. The following character sequence replacements + ## will be performed for that goal: + ## + ## * ``"\\"`` => ``"\\\\"`` + ## * ``"\n"`` => ``"\\n"`` + ## * ``"\t"`` => ``"\\t"`` + result = newStringOfCap(text.len + 3) + for c in text: + case c + of '\\': result.add "\\" + of '\L': result.add "\\n" + of '\C': discard + of '\t': result.add "\\t" + else: result.add c + +proc unquoteIndexColumn*(text: string): string = + ## Returns the unquoted version generated by ``quoteIndexColumn``. + result = text.multiReplace(("\\t", "\t"), ("\\n", "\n"), ("\\\\", "\\")) + +proc formatIndexEntry*(kind: IndexEntryKind; htmlFile, id, term, linkTitle, + linkDesc: string, line: int): + tuple[entry: string, isTitle: bool] = + result.entry = $kind + result.entry.add('\t') + result.entry.add term + result.entry.add('\t') + result.entry.add(htmlFile) + if id.len > 0: + result.entry.add('#') + result.entry.add(id) + result.isTitle = false + else: + result.isTitle = true + result.entry.add('\t' & linkTitle.quoteIndexColumn) + result.entry.add('\t' & linkDesc.quoteIndexColumn) + result.entry.add('\t' & $line) + result.entry.add("\n") + +proc parseIndexEntryKind(s: string): IndexEntryKind = + result = case s: + of "nim": ieNim + of "nimgrp": ieNimGroup + of "heading": ieHeading + of "idx": ieIdxRole + of "nimTitle": ieNimTitle + of "markupTitle": ieMarkupTitle + else: raise newException(ValueError, "unknown index entry value $1" % [s]) + +proc parseIdxFile*(path: string): + tuple[fileEntries: seq[IndexEntry], title: IndexEntry] = + var + f = 0 + newSeq(result.fileEntries, 500) + setLen(result.fileEntries, 0) + let (_, base, _) = path.splitFile + for line in lines(path): + let s = line.find('\t') + if s < 0: continue + setLen(result.fileEntries, f+1) + let cols = line.split('\t') + result.fileEntries[f].kind = parseIndexEntryKind(cols[0]) + result.fileEntries[f].keyword = cols[1] + result.fileEntries[f].link = cols[2] + if result.title.keyword.len == 0: + result.fileEntries[f].module = base + else: + result.fileEntries[f].module = result.title.keyword + + result.fileEntries[f].linkTitle = cols[3].unquoteIndexColumn + result.fileEntries[f].linkDesc = cols[4].unquoteIndexColumn + result.fileEntries[f].line = parseInt(cols[5]) + + if result.fileEntries[f].kind in {ieNimTitle, ieMarkupTitle}: + result.title = result.fileEntries[f] + inc f + +proc cmp*(a, b: IndexEntry): int = + ## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`. + result = cmpIgnoreStyle(a.keyword, b.keyword) + if result == 0: + result = cmpIgnoreStyle(a.link, b.link) + +proc hash*(x: IndexEntry): Hash = + ## Returns the hash for the combined fields of the type. + ## + ## The hash is computed as the chained hash of the individual string hashes. + result = x.keyword.hash !& x.link.hash + result = result !& x.linkTitle.hash + result = result !& x.linkDesc.hash + result = !$result diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 5f8f116d3..ecceed671 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -11,6 +11,8 @@ ## retrieving environment variables, working with directories, ## running shell commands, etc. +## .. importdoc:: symlinks.nim, appdirs.nim, dirs.nim, ospaths2.nim + runnableExamples: let myFile = "/path/to/my/file.nim" assert splitPath(myFile) == (head: "/path/to/my", tail: "file.nim") diff --git a/lib/std/appdirs.nim b/lib/std/appdirs.nim index e648fe0c1..c945ba8ec 100644 --- a/lib/std/appdirs.nim +++ b/lib/std/appdirs.nim @@ -1,5 +1,7 @@ ## This module implements helpers for determining special directories used by apps. +## .. importdoc:: paths.nim + from std/private/osappdirs import nil import std/paths import std/envvars diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim index 8e6ae00f8..581b511ff 100644 --- a/lib/std/private/osappdirs.nim +++ b/lib/std/private/osappdirs.nim @@ -1,3 +1,5 @@ +## .. importdoc:: paths.nim, dirs.nim + include system/inclrtl import std/envvars import std/private/ospaths2 diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim index 5b4e1d7e3..3aac58636 100644 --- a/lib/std/private/oscommon.nim +++ b/lib/std/private/oscommon.nim @@ -5,6 +5,8 @@ import std/[oserrors] when defined(nimPreviewSlimSystem): import std/[syncio, assertions, widestrs] +## .. importdoc:: osdirs.nim, os.nim + const weirdTarget* = defined(nimscript) or defined(js) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim index add9ed424..4af418ead 100644 --- a/lib/std/private/osdirs.nim +++ b/lib/std/private/osdirs.nim @@ -1,3 +1,5 @@ +## .. importdoc:: osfiles.nim, appdirs.nim, paths.nim + include system/inclrtl import std/oserrors diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim index 301f14600..a7a595d97 100644 --- a/lib/std/private/osfiles.nim +++ b/lib/std/private/osfiles.nim @@ -7,6 +7,7 @@ export fileExists import ospaths2, ossymlinks +## .. importdoc:: osdirs.nim, os.nim when defined(nimPreviewSlimSystem): import std/[syncio, assertions, widestrs] @@ -420,4 +421,4 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", removeFile(source) except: discard tryRemoveFile(dest) - raise \ No newline at end of file + raise diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 75c34ecf5..5e3bece68 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -10,6 +10,8 @@ export ReadDirEffect, WriteDirEffect when defined(nimPreviewSlimSystem): import std/[syncio, assertions, widestrs] +## .. importdoc:: osappdirs.nim, osdirs.nim, osseps.nim, os.nim + const weirdTarget = defined(nimscript) or defined(js) when weirdTarget: diff --git a/lib/std/private/osseps.nim b/lib/std/private/osseps.nim index 1ea587e3c..f2d49d886 100644 --- a/lib/std/private/osseps.nim +++ b/lib/std/private/osseps.nim @@ -3,6 +3,8 @@ # Improved based on info in 'compiler/platform.nim' +## .. importdoc:: ospaths2.nim + const doslikeFileSystem* = defined(windows) or defined(OS2) or defined(DOS) diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim index cb6287bde..53f59f939 100644 --- a/lib/std/private/ossymlinks.nim +++ b/lib/std/private/ossymlinks.nim @@ -31,6 +31,7 @@ elif defined(js): else: {.pragma: noNimJs.} +## .. importdoc:: os.nim proc createSymlink*(src, dest: string) {.noWeirdTarget.} = ## Create a symbolic link at `dest` which points to the item specified diff --git a/lib/std/symlinks.nim b/lib/std/symlinks.nim index 9e77bbe2a..54ab7b677 100644 --- a/lib/std/symlinks.nim +++ b/lib/std/symlinks.nim @@ -1,10 +1,11 @@ ## This module implements symlink (symbolic link) handling. +## .. importdoc:: os.nim + from paths import Path, ReadDirEffect from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink - proc symlinkExists*(link: Path): bool {.inline, tags: [ReadDirEffect].} = ## Returns true if the symlink `link` exists. Will return true ## regardless of whether the link points to a directory or file. diff --git a/nimdoc/extlinks/project/doc/manual.md b/nimdoc/extlinks/project/doc/manual.md new file mode 100644 index 000000000..d44b5ca39 --- /dev/null +++ b/nimdoc/extlinks/project/doc/manual.md @@ -0,0 +1,17 @@ +=================== +Nothing User Manual +=================== + +.. importdoc:: ../main.nim, .. / sub / submodule.nim, ../../util.nim + +First section +============= + +Second *section* & +================== + +Ref. [``] or [submoduleInt] from [module nimdoc/extlinks/project/sub/submodule]. + +Ref. [proc mainfunction*(): int]. + +Ref. [utilfunction(x: int)]. diff --git a/nimdoc/extlinks/project/expected/_._/util.html b/nimdoc/extlinks/project/expected/_._/util.html new file mode 100644 index 000000000..9b9b29a4f --- /dev/null +++ b/nimdoc/extlinks/project/expected/_._/util.html @@ -0,0 +1,104 @@ + + + + + + + +nimdoc/extlinks/util + + + + + + + + + + + + +
        +
        +

        nimdoc/extlinks/util

        +
        +
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + + +
        +
        + +
        + +

        +
        +

        Procs

        +
        +
        +
        + 
        proc utilfunction(x: int): int {....raises: [], tags: [], forbids: [].}
        +
        + + + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/_._/util.idx b/nimdoc/extlinks/project/expected/_._/util.idx new file mode 100644 index 000000000..d83d8c97d --- /dev/null +++ b/nimdoc/extlinks/project/expected/_._/util.idx @@ -0,0 +1,2 @@ +nimTitle util _._/util.html module nimdoc/extlinks/util 0 +nim utilfunction _._/util.html#utilfunction,int proc utilfunction(x: int): int 1 diff --git a/nimdoc/extlinks/project/expected/doc/manual.html b/nimdoc/extlinks/project/expected/doc/manual.html new file mode 100644 index 000000000..7b964f4ac --- /dev/null +++ b/nimdoc/extlinks/project/expected/doc/manual.html @@ -0,0 +1,44 @@ + + + + + + + +Nothing User Manual + + + + + + + + + + + + +
        +
        +

        Nothing User Manual

        + +

        First section

        +

        Second section &

        Ref. submodule: `</a>` or submodule: submoduleInt from module nimdoc/extlinks/project/sub/submodule.

        +

        Ref. main: proc mainfunction*(): int.

        +

        Ref. util: utilfunction(x: int).

        + + + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/doc/manual.idx b/nimdoc/extlinks/project/expected/doc/manual.idx new file mode 100644 index 000000000..158a758f0 --- /dev/null +++ b/nimdoc/extlinks/project/expected/doc/manual.idx @@ -0,0 +1,3 @@ +markupTitle Nothing User Manual doc/manual.html Nothing User Manual 0 +heading First section doc/manual.html#first-section First section 0 +heading Second section & doc/manual.html#second-section-amp Second section & 0 diff --git a/nimdoc/extlinks/project/expected/main.html b/nimdoc/extlinks/project/expected/main.html new file mode 100644 index 000000000..5facedb3b --- /dev/null +++ b/nimdoc/extlinks/project/expected/main.html @@ -0,0 +1,144 @@ + + + + + + + +nimdoc/extlinks/project/main + + + + + + + + + + + + +
        +
        +

        nimdoc/extlinks/project/main

        +
        +
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + + +
        +
        + +
        + +

        +

        my heading

        See also module nimdoc/extlinks/util or nimdoc/extlinks/project/sub/submodule module.

        +

        Ref. submodule: `</a>` proc.

        +

        Ref. Nothing User Manual: First section or Nothing User Manual: Second section & from Nothing User Manual.

        +

        + +
        +

        Types

        +
        +
        + 
        A = object
+  x: int
+
        +
        + + + +
        +
        + +
        +
        +
        +

        Procs

        +
        +
        +
        + 
        proc mainfunction(): int {....raises: [], tags: [], forbids: [].}
        +
        + + + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/main.idx b/nimdoc/extlinks/project/expected/main.idx new file mode 100644 index 000000000..d01f2e4c5 --- /dev/null +++ b/nimdoc/extlinks/project/expected/main.idx @@ -0,0 +1,4 @@ +nimTitle main main.html module nimdoc/extlinks/project/main 0 +nim A main.html#A object A 17 +nim mainfunction main.html#mainfunction proc mainfunction(): int 20 +heading my heading main.html#my-heading my heading 0 diff --git a/nimdoc/extlinks/project/expected/sub/submodule.html b/nimdoc/extlinks/project/expected/sub/submodule.html new file mode 100644 index 000000000..913138d6e --- /dev/null +++ b/nimdoc/extlinks/project/expected/sub/submodule.html @@ -0,0 +1,130 @@ + + + + + + + +nimdoc/extlinks/project/sub/submodule + + + + + + + + + + + + +
        +
        +

        nimdoc/extlinks/project/sub/submodule

        +
        +
        +
        + + +
        + +
        + Search: +
        +
        + Group by: + +
        + + +
        +
        + +
        + +

        Ref. main: object A from module nimdoc/extlinks/project/main.

        +

        Ref. util: utilfunction(x: int).

        +

        Ref. Nothing User Manual: Second section & from Nothing User Manual.

        +

        +
        +

        Types

        +
        +
        + 
        submoduleInt = distinct int
        +
        + + + +
        +
        + +
        +
        +
        +

        Procs

        +
        +
        +
        + 
        proc `</a>`(x, y: int): bool {....raises: [], tags: [], forbids: [].}
        +
        + + Attempt to break HTML formatting. + +
        +
        + +
        + +
        +
        + +
        +
        + +
        + +
        + Made with Nim. Generated: 1970-01-02 03:46:40 UTC +
        +
        +
        + + + + + + diff --git a/nimdoc/extlinks/project/expected/sub/submodule.idx b/nimdoc/extlinks/project/expected/sub/submodule.idx new file mode 100644 index 000000000..2b02c889e --- /dev/null +++ b/nimdoc/extlinks/project/expected/sub/submodule.idx @@ -0,0 +1,3 @@ +nimTitle submodule sub/submodule.html module nimdoc/extlinks/project/sub/submodule 0 +nim `` sub/submodule.html#,int,int proc ``(x, y: int): bool 9 +nim submoduleInt sub/submodule.html#submoduleInt type submoduleInt 13 diff --git a/nimdoc/extlinks/project/expected/theindex.html b/nimdoc/extlinks/project/expected/theindex.html new file mode 100644 index 000000000..9d2f6723c --- /dev/null +++ b/nimdoc/extlinks/project/expected/theindex.html @@ -0,0 +1,58 @@ + + + + + + + +Index + + + + + + + + + + + + + + + + + + + diff --git a/nimdoc/extlinks/project/main.nim b/nimdoc/extlinks/project/main.nim new file mode 100644 index 000000000..36b778af6 --- /dev/null +++ b/nimdoc/extlinks/project/main.nim @@ -0,0 +1,23 @@ +## my heading +## ========== +## +## .. importdoc:: sub/submodule.nim, ../util.nim, doc/manual.md +## +## .. See also [Second&&&] and particularly [first section] and [Second section &]. +## +## See also [module nimdoc/extlinks/util] or [nimdoc/extlinks/project/sub/submodule module]. +## +## Ref. [`` proc]. +## +## Ref. [First section] or [Second section &] from [Nothing User Manual]. + + +import ../util, sub/submodule + +type A* = object + x: int + +proc mainfunction*(): int = + # just to suppress "not used" warnings: + if ``(1, 2): + result = utilfunction(0) diff --git a/nimdoc/extlinks/project/sub/submodule.nim b/nimdoc/extlinks/project/sub/submodule.nim new file mode 100644 index 000000000..876e00684 --- /dev/null +++ b/nimdoc/extlinks/project/sub/submodule.nim @@ -0,0 +1,13 @@ +## .. importdoc:: ../../util.nim, ../main.nim, ../doc/manual.md +## +## Ref. [object A] from [module nimdoc/extlinks/project/main]. +## +## Ref. [utilfunction(x: int)]. +## +## Ref. [Second section &] from [Nothing User Manual]. + +proc ``*(x, y: int): bool = + ## Attempt to break HTML formatting. + result = x < y + +type submoduleInt* = distinct int diff --git a/nimdoc/extlinks/util.nim b/nimdoc/extlinks/util.nim new file mode 100644 index 000000000..f208f98c1 --- /dev/null +++ b/nimdoc/extlinks/util.nim @@ -0,0 +1,2 @@ +proc utilfunction*(x: int): int = + x + 42 diff --git a/nimdoc/test_out_index_dot_html/expected/foo.idx b/nimdoc/test_out_index_dot_html/expected/foo.idx index a8dabb67e..ac76aa532 100644 --- a/nimdoc/test_out_index_dot_html/expected/foo.idx +++ b/nimdoc/test_out_index_dot_html/expected/foo.idx @@ -1 +1,2 @@ -foo index.html#foo foo: foo() +nimTitle foo index.html module nimdoc/test_out_index_dot_html/foo 0 +nim foo index.html#foo proc foo() 1 diff --git a/nimdoc/test_out_index_dot_html/expected/theindex.html b/nimdoc/test_out_index_dot_html/expected/theindex.html index 2a0ea7900..ea3486d4b 100644 --- a/nimdoc/test_out_index_dot_html/expected/theindex.html +++ b/nimdoc/test_out_index_dot_html/expected/theindex.html @@ -24,7 +24,7 @@ Modules: index.

        API symbols

        foo:
        • foo: foo()
          • + data-doc-search-tag="foo: proc foo()" href="index.html#foo">foo: proc foo()
        diff --git a/nimdoc/tester.nim b/nimdoc/tester.nim index ef82ae1b9..e94caae7c 100644 --- a/nimdoc/tester.nim +++ b/nimdoc/tester.nim @@ -12,11 +12,16 @@ var const baseDir = "nimdoc" +let + baseDirAbs = getCurrentDir() / baseDir type NimSwitches = object doc: seq[string] + docStage2: seq[string] buildIndex: seq[string] + md2html: seq[string] + md2htmlStage2: seq[string] proc exec(cmd: string) = if execShellCmd(cmd) != 0: @@ -25,6 +30,9 @@ proc exec(cmd: string) = proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = let nimDocSwitches = switches.doc.join(" ") + nimDocStage2Switches = switches.docStage2.join(" ") + nimMd2HtmlSwitches = switches.md2html.join(" ") + nimMd2HtmlStage2Switches = switches.md2htmlStage2.join(" ") nimBuildIndexSwitches = switches.buildIndex.join(" ") putEnv("SOURCE_DATE_EPOCH", "100000") @@ -32,9 +40,23 @@ proc testNimDoc(prjDir, docsDir: string; switches: NimSwitches; fixup = false) = if nimDocSwitches != "": exec("$1 doc $2" % [nimExe, nimDocSwitches]) + echo("$1 doc $2" % [nimExe, nimDocSwitches]) + + if nimMd2HtmlSwitches != "": + exec("$1 md2html $2" % [nimExe, nimMd2HtmlSwitches]) + echo("$1 md2html $2" % [nimExe, nimMd2HtmlSwitches]) + + if nimDocStage2Switches != "": + exec("$1 doc $2" % [nimExe, nimDocStage2Switches]) + echo("$1 doc $2" % [nimExe, nimDocStage2Switches]) + + if nimMd2HtmlStage2Switches != "": + exec("$1 md2html $2" % [nimExe, nimMd2HtmlStage2Switches]) + echo("$1 md2html $2" % [nimExe, nimMd2HtmlStage2Switches]) if nimBuildIndexSwitches != "": exec("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches]) + echo("$1 buildIndex $2" % [nimExe, nimBuildIndexSwitches]) for expected in walkDirRec(prjDir / "expected/", checkDir=true): let produced = expected.replace('\\', '/').replace("/expected/", "/$1/" % [docsDir]) @@ -88,5 +110,46 @@ let test3Switches = NimSwitches(doc: @["$1/$2.nim" % [test3Dir, test3PrjName]]) testNimDoc(test3Dir, test3DocsDir, test3Switches, fixup) + +# Test concise external links (RFC#125) that work with `.idx` files. +# extlinks +# ├── project +# │   ├── main.nim +# │   ├── manual.md +# │   └── sub +# │   └── submodule.nim +# └── util.nim +# +# `main.nim` imports `submodule.nim` and `../utils.nim`. +# `main.nim`, `submodule.nim`, `manual.md` do importdoc and reference each other. +let + test4PrjName = "extlinks/project" + test4Dir = baseDir / test4PrjName + test4DirAbs = baseDirAbs / test4PrjName + test4MainModule = "main" + test4MarkupDoc = "doc" / "manual.md" + test4DocsDir = "htmldocs" + # 1st stage is with --index:only, 2nd is final + test4Switches = NimSwitches( + doc: @["--project", + "--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--index:only", + "$1/$2.nim" % [test4Dir, test4MainModule]], + md2html: + @["--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--docroot:$1" % [test4DirAbs], + "--index:only", + "$1/$2" % [test4Dir, test4MarkupDoc]], + docStage2: + @["--project", + "--outdir:$1/$2" % [test4Dir, test4DocsDir], + "$1/$2.nim" % [test4Dir, test4MainModule]], + md2htmlStage2: + @["--outdir:$1/$2" % [test4Dir, test4DocsDir], + "--docroot:$1" % [test4DirAbs], + "$1/$2" % [test4Dir, test4MarkupDoc]], + ) +testNimDoc(test4Dir, test4DocsDir, test4Switches, fixup) + if failures > 0: quit "$# failures occurred; see note in nimdoc/tester.nim regarding -d:nimTestsNimdocFixup" % $failures diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index 7d59e9e86..f0f30536f 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -226,7 +226,7 @@ cmp: proc (x: T; y: K): int {.closure.}): int" href="#binarySearch,openArray[T],K,proc(T,K)">binary search.

        Note that proc can be used in postfix form: binarySearch proc.

        -

        Ref. type like G and type G and G[T] and type G*[T].

        +

        Ref. type like G and type G and G[T] and type G*[T].

        Group ref. with capital letters works: fN11 or fn11

        Ref. [] is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): TRef. []= aka `[]=`(G[T], int, T).Ref. $ aka proc $ or proc `$`.Ref. $(a: ref SomeType).Ref. foo_bar aka iterator foo_bar_.Ref. fn[T; U,V: SomeFloat]().Ref. 'big or func `'big` or `'big`(string).

        Pandoc Markdown

        Now repeat all the auto links of above in Pandoc Markdown Syntax.

        @@ -240,11 +240,11 @@ Ref. binary search.

        Note that proc can be used in postfix form: binarySearch proc.

        -

        Ref. type like G and type G and G[T] and type G*[T].

        +

        Ref. type like G and type G and G[T] and type G*[T].

        Group ref. with capital letters works: fN11 or fn11

        Ref. `[]` is the same as proc `[]`(G[T]) because there are no overloads. The full form: proc `[]`*[T](x: G[T]): T Ref. `[]=` aka `[]=`(G[T], int, T). Ref. $ aka proc $ or proc `$`. Ref. $(a: ref SomeType). Ref. foo_bar aka iterator foo_bar_. Ref. fn[T; U,V: SomeFloat](). Ref. 'big or func `'big` or `'big`(string).

        -

        Link name syntax

        Pandoc Markdown has synax for changing text of links: Ref. this proc or another symbol.

        +

        Link name syntax

        Pandoc Markdown has synax for changing text of links: Ref. this proc or another symbol.

        Symbols documentation

        Let us repeat auto links from symbols section below:

        There is also variant f(G[string]). See also f(G[int]).

        diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx index 007101b37..c2f148a92 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.idx +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.idx @@ -1,41 +1,46 @@ -funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U utils: funWithGenerics[T, U: SomeFloat](a: T; b: U) -enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA -enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB -enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC -SomeType subdir/subdir_b/utils.html#SomeType utils: SomeType -G subdir/subdir_b/utils.html#G utils: G -someType subdir/subdir_b/utils.html#someType_2 utils: someType(): SomeType -fn2 subdir/subdir_b/utils.html#fn2 utils: fn2() -fn2 subdir/subdir_b/utils.html#fn2,int utils: fn2(x: int) -fn2 subdir/subdir_b/utils.html#fn2,int,float utils: fn2(x: int; y: float) -binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) utils: binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int -fn3 subdir/subdir_b/utils.html#fn3 utils: fn3(): auto -fn4 subdir/subdir_b/utils.html#fn4 utils: fn4(): auto -fn5 subdir/subdir_b/utils.html#fn5 utils: fn5() -fn6 subdir/subdir_b/utils.html#fn6 utils: fn6() -fn7 subdir/subdir_b/utils.html#fn7 utils: fn7() -fn8 subdir/subdir_b/utils.html#fn8 utils: fn8(): auto -fn9 subdir/subdir_b/utils.html#fn9,int utils: fn9(a: int): int -fn10 subdir/subdir_b/utils.html#fn10,int utils: fn10(a: int): int -fN11 subdir/subdir_b/utils.html#fN11 utils: fN11() -fN11 subdir/subdir_b/utils.html#fN11,int utils: fN11(x: int) -aEnum subdir/subdir_b/utils.html#aEnum.t utils: aEnum(): untyped -bEnum subdir/subdir_b/utils.html#bEnum.t utils: bEnum(): untyped -fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t utils: fromUtilsGen(): untyped -f subdir/subdir_b/utils.html#f,G[int] utils: f(x: G[int]) -f subdir/subdir_b/utils.html#f,G[string] utils: f(x: G[string]) -`[]` subdir/subdir_b/utils.html#[],G[T] utils: `[]`[T](x: G[T]): T -`[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T utils: `[]=`[T](a: var G[T]; index: int; value: T) -`$` subdir/subdir_b/utils.html#$,G[T] utils: `$`[T](a: G[T]): string -`$` subdir/subdir_b/utils.html#$,ref.SomeType utils: `$`[T](a: ref SomeType): string -fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] utils: fooBar(a: seq[SomeType]): int -fn subdir/subdir_b/utils.html#fn utils: fn[T; U, V: SomeFloat]() -`'big` subdir/subdir_b/utils.html#'big,string utils: `'big`(a: string): SomeType -This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header -Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header -And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on -More headers subdir/subdir_b/utils.html#more-headers More headers -Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 -Pandoc Markdown subdir/subdir_b/utils.html#pandoc-markdown Pandoc Markdown -Link name syntax subdir/subdir_b/utils.html#pandoc-markdown-link-name-syntax Link name syntax -Symbols documentation subdir/subdir_b/utils.html#pandoc-markdown-symbols-documentation Symbols documentation +nimTitle utils subdir/subdir_b/utils.html module subdir/subdir_b/utils 0 +nim funWithGenerics subdir/subdir_b/utils.html#funWithGenerics,T,U proc funWithGenerics[T, U: SomeFloat](a: T; b: U) 1 +nim enumValueA subdir/subdir_b/utils.html#enumValueA SomeType.enumValueA 45 +nim enumValueB subdir/subdir_b/utils.html#enumValueB SomeType.enumValueB 45 +nim enumValueC subdir/subdir_b/utils.html#enumValueC SomeType.enumValueC 45 +nim SomeType subdir/subdir_b/utils.html#SomeType enum SomeType 45 +nim G subdir/subdir_b/utils.html#G object G 49 +nim someType subdir/subdir_b/utils.html#someType_2 proc someType(): SomeType 52 +nim fn2 subdir/subdir_b/utils.html#fn2 proc fn2() 57 +nim fn2 subdir/subdir_b/utils.html#fn2,int proc fn2(x: int) 58 +nim fn2 subdir/subdir_b/utils.html#fn2,int,float proc fn2(x: int; y: float) 61 +nim binarySearch subdir/subdir_b/utils.html#binarySearch,openArray[T],K,proc(T,K) proc binarySearch[T, K](a: openArray[T]; key: K;\n cmp: proc (x: T; y: K): int {.closure.}): int 63 +nim fn3 subdir/subdir_b/utils.html#fn3 proc fn3(): auto 66 +nim fn4 subdir/subdir_b/utils.html#fn4 proc fn4(): auto 67 +nim fn5 subdir/subdir_b/utils.html#fn5 proc fn5() 69 +nim fn6 subdir/subdir_b/utils.html#fn6 proc fn6() 70 +nim fn7 subdir/subdir_b/utils.html#fn7 proc fn7() 72 +nim fn8 subdir/subdir_b/utils.html#fn8 proc fn8(): auto 75 +nim fn9 subdir/subdir_b/utils.html#fn9,int proc fn9(a: int): int 78 +nim fn10 subdir/subdir_b/utils.html#fn10,int proc fn10(a: int): int 79 +nim fN11 subdir/subdir_b/utils.html#fN11 proc fN11() 85 +nim fN11 subdir/subdir_b/utils.html#fN11,int proc fN11(x: int) 86 +nim aEnum subdir/subdir_b/utils.html#aEnum.t template aEnum(): untyped 90 +nim bEnum subdir/subdir_b/utils.html#bEnum.t template bEnum(): untyped 95 +nim fromUtilsGen subdir/subdir_b/utils.html#fromUtilsGen.t template fromUtilsGen(): untyped 106 +nim f subdir/subdir_b/utils.html#f,G[int] proc f(x: G[int]) 130 +nim f subdir/subdir_b/utils.html#f,G[string] proc f(x: G[string]) 133 +nim `[]` subdir/subdir_b/utils.html#[],G[T] proc `[]`[T](x: G[T]): T 140 +nim `[]=` subdir/subdir_b/utils.html#[]=,G[T],int,T proc `[]=`[T](a: var G[T]; index: int; value: T) 144 +nim `$` subdir/subdir_b/utils.html#$,G[T] proc `$`[T](a: G[T]): string 148 +nim `$` subdir/subdir_b/utils.html#$,ref.SomeType proc `$`[T](a: ref SomeType): string 152 +nim fooBar subdir/subdir_b/utils.html#fooBar.i,seq[SomeType] iterator fooBar(a: seq[SomeType]): int 156 +nim fn subdir/subdir_b/utils.html#fn proc fn[T; U, V: SomeFloat]() 160 +nim `'big` subdir/subdir_b/utils.html#'big,string proc `'big`(a: string): SomeType 164 +nimgrp $ subdir/subdir_b/utils.html#$-procs-all proc 148 +nimgrp fn11 subdir/subdir_b/utils.html#fN11-procs-all proc 85 +nimgrp fn2 subdir/subdir_b/utils.html#fn2-procs-all proc 57 +nimgrp f subdir/subdir_b/utils.html#f-procs-all proc 130 +heading This is now a header subdir/subdir_b/utils.html#this-is-now-a-header This is now a header 0 +heading Next header subdir/subdir_b/utils.html#this-is-now-a-header-next-header Next header 0 +heading And so on subdir/subdir_b/utils.html#next-header-and-so-on And so on 0 +heading More headers subdir/subdir_b/utils.html#more-headers More headers 0 +heading Up to level 6 subdir/subdir_b/utils.html#more-headers-up-to-level-6 Up to level 6 0 +heading Pandoc Markdown subdir/subdir_b/utils.html#pandoc-markdown Pandoc Markdown 0 +heading Link name syntax subdir/subdir_b/utils.html#pandoc-markdown-link-name-syntax Link name syntax 0 +heading Symbols documentation subdir/subdir_b/utils.html#pandoc-markdown-symbols-documentation Symbols documentation 0 diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index 4d0172313..e69bedf33 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -1,64 +1,67 @@ -someVariable testproject.html#someVariable testproject: someVariable -C_A testproject.html#C_A testproject: C_A -C_B testproject.html#C_B testproject: C_B -C_C testproject.html#C_C testproject: C_C -C_D testproject.html#C_D testproject: C_D -bar testproject.html#bar,T,T testproject: bar[T](a, b: T): T -baz testproject.html#baz,T,T testproject: baz[T](a, b: T): T -buzz testproject.html#buzz,T,T testproject: buzz[T](a, b: T): T -FooBuzz testproject.html#FooBuzz testproject: FooBuzz -bar testproject.html#bar testproject: bar(f: FooBuzz) -aVariable testproject.html#aVariable testproject: aVariable -A testproject.html#A testproject: A -B testproject.html#B testproject: B -someFunc testproject.html#someFunc testproject: someFunc() -fromUtils1 testproject.html#fromUtils1.i testproject: fromUtils1(): int -fromUtils2 testproject.html#fromUtils2.t testproject: fromUtils2() -fromUtils3 testproject.html#fromUtils3 testproject: fromUtils3() -isValid testproject.html#isValid,T testproject: isValid[T](x: T): bool -enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 -Foo testproject.html#Foo testproject: Foo -z1 testproject.html#z1 testproject: z1(): Foo -z2 testproject.html#z2 testproject: z2() -z3 testproject.html#z3 testproject: z3() -z4 testproject.html#z4 testproject: z4() -z5 testproject.html#z5 testproject: z5(): int -z6 testproject.html#z6 testproject: z6(): int -z6t testproject.html#z6t.t testproject: z6t(): int -z7 testproject.html#z7 testproject: z7(): int -z8 testproject.html#z8 testproject: z8(): int -z9 testproject.html#z9 testproject: z9() -z10 testproject.html#z10 testproject: z10() -z11 testproject.html#z11 testproject: z11() -z12 testproject.html#z12 testproject: z12(): int -z13 testproject.html#z13 testproject: z13() -baz testproject.html#baz testproject: baz() -z17 testproject.html#z17 testproject: z17() -p1 testproject.html#p1 testproject: p1() -addfBug14485 testproject.html#addfBug14485 testproject: addfBug14485() -c_printf testproject.html#c_printf,cstring testproject: c_printf(frmt: cstring): cint -c_nonexistent testproject.html#c_nonexistent,cstring testproject: c_nonexistent(frmt: cstring): cint -low testproject.html#low,T testproject: low[T: Ordinal | enum | range](x: T): T -low2 testproject.html#low2,T testproject: low2[T: Ordinal | enum | range](x: T): T -tripleStrLitTest testproject.html#tripleStrLitTest testproject: tripleStrLitTest() -method1 testproject.html#method1.e,Moo testproject: method1(self: Moo) -method2 testproject.html#method2.e,Moo testproject: method2(self: Moo): int -method3 testproject.html#method3.e,Moo testproject: method3(self: Moo): int -iter1 testproject.html#iter1.i,int testproject: iter1(n: int): int -iter2 testproject.html#iter2.i,int testproject: iter2(n: int): int -bar testproject.html#bar.m testproject: bar(): untyped -z16 testproject.html#z16.m testproject: z16() -z18 testproject.html#z18.m testproject: z18(): int -foo testproject.html#foo.t,SomeType,SomeType testproject: foo(a, b: SomeType) -myfn testproject.html#myfn.t testproject: myfn() -z14 testproject.html#z14.t testproject: z14() -z15 testproject.html#z15.t testproject: z15() -asyncFun1 testproject.html#asyncFun1 testproject: asyncFun1(): Future[int] -asyncFun2 testproject.html#asyncFun2 testproject: asyncFun2(): owned(Future[void]) -asyncFun3 testproject.html#asyncFun3 testproject: asyncFun3(): owned(Future[void]) -testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t testproject: testNimDocTrailingExample() -Circle testproject.html#Circle Shapes.Circle -Triangle testproject.html#Triangle Shapes.Triangle -Rectangle testproject.html#Rectangle Shapes.Rectangle -Shapes testproject.html#Shapes testproject: Shapes -anything testproject.html#anything testproject: anything() +nimTitle testproject testproject.html module testproject 0 +nim someVariable testproject.html#someVariable var someVariable 13 +nim C_A testproject.html#C_A const C_A 26 +nim C_B testproject.html#C_B const C_B 27 +nim C_C testproject.html#C_C const C_C 28 +nim C_D testproject.html#C_D const C_D 29 +nim bar testproject.html#bar,T,T proc bar[T](a, b: T): T 31 +nim baz testproject.html#baz,T,T proc baz[T](a, b: T): T 34 +nim buzz testproject.html#buzz,T,T proc buzz[T](a, b: T): T 38 +nim FooBuzz testproject.html#FooBuzz type FooBuzz 43 +nim bar testproject.html#bar proc bar(f: FooBuzz) 47 +nim aVariable testproject.html#aVariable var aVariable 52 +nim A testproject.html#A enum A 92 +nim B testproject.html#B enum B 97 +nim someFunc testproject.html#someFunc proc someFunc() 56 +nim fromUtils1 testproject.html#fromUtils1.i iterator fromUtils1(): int 112 +nim fromUtils2 testproject.html#fromUtils2.t template fromUtils2() 119 +nim fromUtils3 testproject.html#fromUtils3 proc fromUtils3() 57 +nim isValid testproject.html#isValid,T proc isValid[T](x: T): bool 59 +nim enumValueA2 testproject.html#enumValueA2 Foo.enumValueA2 66 +nim Foo testproject.html#Foo enum Foo 66 +nim z1 testproject.html#z1 proc z1(): Foo 69 +nim z2 testproject.html#z2 proc z2() 73 +nim z3 testproject.html#z3 proc z3() 78 +nim z4 testproject.html#z4 proc z4() 81 +nim z5 testproject.html#z5 proc z5(): int 87 +nim z6 testproject.html#z6 proc z6(): int 91 +nim z6t testproject.html#z6t.t template z6t(): int 95 +nim z7 testproject.html#z7 proc z7(): int 99 +nim z8 testproject.html#z8 proc z8(): int 103 +nim z9 testproject.html#z9 proc z9() 111 +nim z10 testproject.html#z10 proc z10() 114 +nim z11 testproject.html#z11 proc z11() 119 +nim z12 testproject.html#z12 proc z12(): int 124 +nim z13 testproject.html#z13 proc z13() 129 +nim baz testproject.html#baz proc baz() 134 +nim z17 testproject.html#z17 proc z17() 144 +nim p1 testproject.html#p1 proc p1() 156 +nim addfBug14485 testproject.html#addfBug14485 proc addfBug14485() 177 +nim c_printf testproject.html#c_printf,cstring proc c_printf(frmt: cstring): cint 193 +nim c_nonexistent testproject.html#c_nonexistent,cstring proc c_nonexistent(frmt: cstring): cint 197 +nim low testproject.html#low,T proc low[T: Ordinal | enum | range](x: T): T 200 +nim low2 testproject.html#low2,T proc low2[T: Ordinal | enum | range](x: T): T 210 +nim tripleStrLitTest testproject.html#tripleStrLitTest proc tripleStrLitTest() 223 +nim method1 testproject.html#method1.e,Moo method method1(self: Moo) 264 +nim method2 testproject.html#method2.e,Moo method method2(self: Moo): int 266 +nim method3 testproject.html#method3.e,Moo method method3(self: Moo): int 269 +nim iter1 testproject.html#iter1.i,int iterator iter1(n: int): int 274 +nim iter2 testproject.html#iter2.i,int iterator iter2(n: int): int 278 +nim bar testproject.html#bar.m macro bar(): untyped 285 +nim z16 testproject.html#z16.m macro z16() 288 +nim z18 testproject.html#z18.m macro z18(): int 297 +nim foo testproject.html#foo.t,SomeType,SomeType template foo(a, b: SomeType) 302 +nim myfn testproject.html#myfn.t template myfn() 307 +nim z14 testproject.html#z14.t template z14() 328 +nim z15 testproject.html#z15.t template z15() 333 +nim asyncFun1 testproject.html#asyncFun1 proc asyncFun1(): Future[int] 358 +nim asyncFun2 testproject.html#asyncFun2 proc asyncFun2(): owned(Future[void]) 361 +nim asyncFun3 testproject.html#asyncFun3 proc asyncFun3(): owned(Future[void]) 362 +nim testNimDocTrailingExample testproject.html#testNimDocTrailingExample.t template testNimDocTrailingExample() 371 +nim Circle testproject.html#Circle Shapes.Circle 380 +nim Triangle testproject.html#Triangle Shapes.Triangle 380 +nim Rectangle testproject.html#Rectangle Shapes.Rectangle 380 +nim Shapes testproject.html#Shapes enum Shapes 380 +nim anything testproject.html#anything proc anything() 387 +nimgrp bar testproject.html#bar-procs-all proc 31 +nimgrp baz testproject.html#baz-procs-all proc 34 diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 9f73040f2..7b4a95561 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -24,377 +24,377 @@ Modules: subdir/subdir_b/utils, testproject.

        API symbols

        `$`:
        • utils: `$`[T](a: G[T]): string
          • + data-doc-search-tag="utils: proc `$`[T](a: G[T]): string" href="subdir/subdir_b/utils.html#%24%2CG%5BT%5D">utils: proc `$`[T](a: G[T]): string
        • utils: `$`[T](a: ref SomeType): string
          • + data-doc-search-tag="utils: proc `$`[T](a: ref SomeType): string" href="subdir/subdir_b/utils.html#%24%2Cref.SomeType">utils: proc `$`[T](a: ref SomeType): string
        `'big`:
        • utils: `'big`(a: string): SomeType
          • + data-doc-search-tag="utils: proc `'big`(a: string): SomeType" href="subdir/subdir_b/utils.html#%27big%2Cstring">utils: proc `'big`(a: string): SomeType
        `[]=`:
        • utils: `[]=`[T](a: var G[T]; index: int; value: T)
          • + data-doc-search-tag="utils: proc `[]=`[T](a: var G[T]; index: int; value: T)" href="subdir/subdir_b/utils.html#%5B%5D%3D%2CG%5BT%5D%2Cint%2CT">utils: proc `[]=`[T](a: var G[T]; index: int; value: T)
        `[]`:
        • utils: `[]`[T](x: G[T]): T
          • + data-doc-search-tag="utils: proc `[]`[T](x: G[T]): T" href="subdir/subdir_b/utils.html#%5B%5D%2CG%5BT%5D">utils: proc `[]`[T](x: G[T]): T
        A:
        • testproject: A
          • + data-doc-search-tag="testproject: enum A" href="testproject.html#A">testproject: enum A
        addfBug14485:
        • testproject: addfBug14485()
          • + data-doc-search-tag="testproject: proc addfBug14485()" href="testproject.html#addfBug14485">testproject: proc addfBug14485()
        aEnum:
        • utils: aEnum(): untyped
          • + data-doc-search-tag="utils: template aEnum(): untyped" href="subdir/subdir_b/utils.html#aEnum.t">utils: template aEnum(): untyped
        anything:
        • testproject: anything()
          • + data-doc-search-tag="testproject: proc anything()" href="testproject.html#anything">testproject: proc anything()
        asyncFun1:
        asyncFun2:
        asyncFun3:
        aVariable:
        • testproject: aVariable
          • + data-doc-search-tag="testproject: var aVariable" href="testproject.html#aVariable">testproject: var aVariable
        B:
        • testproject: B
          • + data-doc-search-tag="testproject: enum B" href="testproject.html#B">testproject: enum B
        bar:
        • testproject: bar(f: FooBuzz)
          • + data-doc-search-tag="testproject: proc bar(f: FooBuzz)" href="testproject.html#bar">testproject: proc bar(f: FooBuzz)
        • testproject: bar[T](a, b: T): T
          • + data-doc-search-tag="testproject: proc bar[T](a, b: T): T" href="testproject.html#bar%2CT%2CT">testproject: proc bar[T](a, b: T): T
        • testproject: bar(): untyped
          • + data-doc-search-tag="testproject: macro bar(): untyped" href="testproject.html#bar.m">testproject: macro bar(): untyped
        baz:
        • testproject: baz()
          • + data-doc-search-tag="testproject: proc baz()" href="testproject.html#baz">testproject: proc baz()
        • testproject: baz[T](a, b: T): T
          • + data-doc-search-tag="testproject: proc baz[T](a, b: T): T" href="testproject.html#baz%2CT%2CT">testproject: proc baz[T](a, b: T): T
        bEnum:
        • utils: bEnum(): untyped
          • + data-doc-search-tag="utils: template bEnum(): untyped" href="subdir/subdir_b/utils.html#bEnum.t">utils: template bEnum(): untyped
        binarySearch:
        buzz:
        • testproject: buzz[T](a, b: T): T
          • + data-doc-search-tag="testproject: proc buzz[T](a, b: T): T" href="testproject.html#buzz%2CT%2CT">testproject: proc buzz[T](a, b: T): T
        C_A:
        • testproject: C_A
          • + data-doc-search-tag="testproject: const C_A" href="testproject.html#C_A">testproject: const C_A
        C_B:
        • testproject: C_B
          • + data-doc-search-tag="testproject: const C_B" href="testproject.html#C_B">testproject: const C_B
        C_C:
        • testproject: C_C
          • + data-doc-search-tag="testproject: const C_C" href="testproject.html#C_C">testproject: const C_C
        C_D:
        • testproject: C_D
          • + data-doc-search-tag="testproject: const C_D" href="testproject.html#C_D">testproject: const C_D
        Circle:
        • Shapes.Circle
          • + data-doc-search-tag="testproject: Shapes.Circle" href="testproject.html#Circle">testproject: Shapes.Circle
        c_nonexistent:
        c_printf:
        enumValueA:
        • SomeType.enumValueA
          • + data-doc-search-tag="utils: SomeType.enumValueA" href="subdir/subdir_b/utils.html#enumValueA">utils: SomeType.enumValueA
        enumValueA2:
        • Foo.enumValueA2
          • + data-doc-search-tag="testproject: Foo.enumValueA2" href="testproject.html#enumValueA2">testproject: Foo.enumValueA2
        enumValueB:
        • SomeType.enumValueB
          • + data-doc-search-tag="utils: SomeType.enumValueB" href="subdir/subdir_b/utils.html#enumValueB">utils: SomeType.enumValueB
        enumValueC:
        • SomeType.enumValueC
          • + data-doc-search-tag="utils: SomeType.enumValueC" href="subdir/subdir_b/utils.html#enumValueC">utils: SomeType.enumValueC
        f:
        • utils: f(x: G[int])
          • + data-doc-search-tag="utils: proc f(x: G[int])" href="subdir/subdir_b/utils.html#f%2CG%5Bint%5D">utils: proc f(x: G[int])
        • utils: f(x: G[string])
          • + data-doc-search-tag="utils: proc f(x: G[string])" href="subdir/subdir_b/utils.html#f%2CG%5Bstring%5D">utils: proc f(x: G[string])
        fn:
        • utils: fn[T; U, V: SomeFloat]()
          • + data-doc-search-tag="utils: proc fn[T; U, V: SomeFloat]()" href="subdir/subdir_b/utils.html#fn">utils: proc fn[T; U, V: SomeFloat]()
        fn10:
        • utils: fn10(a: int): int
          • + data-doc-search-tag="utils: proc fn10(a: int): int" href="subdir/subdir_b/utils.html#fn10%2Cint">utils: proc fn10(a: int): int
        fN11:
        • utils: fN11()
          • + data-doc-search-tag="utils: proc fN11()" href="subdir/subdir_b/utils.html#fN11">utils: proc fN11()
        • utils: fN11(x: int)
          • + data-doc-search-tag="utils: proc fN11(x: int)" href="subdir/subdir_b/utils.html#fN11%2Cint">utils: proc fN11(x: int)
        fn2:
        • utils: fn2()
          • + data-doc-search-tag="utils: proc fn2()" href="subdir/subdir_b/utils.html#fn2">utils: proc fn2()
        • utils: fn2(x: int)
          • + data-doc-search-tag="utils: proc fn2(x: int)" href="subdir/subdir_b/utils.html#fn2%2Cint">utils: proc fn2(x: int)
        • utils: fn2(x: int; y: float)
          • + data-doc-search-tag="utils: proc fn2(x: int; y: float)" href="subdir/subdir_b/utils.html#fn2%2Cint%2Cfloat">utils: proc fn2(x: int; y: float)
        fn3:
        • utils: fn3(): auto
          • + data-doc-search-tag="utils: proc fn3(): auto" href="subdir/subdir_b/utils.html#fn3">utils: proc fn3(): auto
        fn4:
        • utils: fn4(): auto
          • + data-doc-search-tag="utils: proc fn4(): auto" href="subdir/subdir_b/utils.html#fn4">utils: proc fn4(): auto
        fn5:
        • utils: fn5()
          • + data-doc-search-tag="utils: proc fn5()" href="subdir/subdir_b/utils.html#fn5">utils: proc fn5()
        fn6:
        • utils: fn6()
          • + data-doc-search-tag="utils: proc fn6()" href="subdir/subdir_b/utils.html#fn6">utils: proc fn6()
        fn7:
        • utils: fn7()
          • + data-doc-search-tag="utils: proc fn7()" href="subdir/subdir_b/utils.html#fn7">utils: proc fn7()
        fn8:
        • utils: fn8(): auto
          • + data-doc-search-tag="utils: proc fn8(): auto" href="subdir/subdir_b/utils.html#fn8">utils: proc fn8(): auto
        fn9:
        • utils: fn9(a: int): int
          • + data-doc-search-tag="utils: proc fn9(a: int): int" href="subdir/subdir_b/utils.html#fn9%2Cint">utils: proc fn9(a: int): int
        Foo:
        • testproject: Foo
          • + data-doc-search-tag="testproject: enum Foo" href="testproject.html#Foo">testproject: enum Foo
        foo:
        • testproject: foo(a, b: SomeType)
          • + data-doc-search-tag="testproject: template foo(a, b: SomeType)" href="testproject.html#foo.t%2CSomeType%2CSomeType">testproject: template foo(a, b: SomeType)
        fooBar:
        • utils: fooBar(a: seq[SomeType]): int
          • + data-doc-search-tag="utils: iterator fooBar(a: seq[SomeType]): int" href="subdir/subdir_b/utils.html#fooBar.i%2Cseq%5BSomeType%5D">utils: iterator fooBar(a: seq[SomeType]): int
        FooBuzz:
        • testproject: FooBuzz
          • + data-doc-search-tag="testproject: type FooBuzz" href="testproject.html#FooBuzz">testproject: type FooBuzz
        fromUtils1:
        • testproject: fromUtils1(): int
          • + data-doc-search-tag="testproject: iterator fromUtils1(): int" href="testproject.html#fromUtils1.i">testproject: iterator fromUtils1(): int
        fromUtils2:
        • testproject: fromUtils2()
          • + data-doc-search-tag="testproject: template fromUtils2()" href="testproject.html#fromUtils2.t">testproject: template fromUtils2()
        fromUtils3:
        • testproject: fromUtils3()
          • + data-doc-search-tag="testproject: proc fromUtils3()" href="testproject.html#fromUtils3">testproject: proc fromUtils3()
        fromUtilsGen:
        • utils: fromUtilsGen(): untyped
          • + data-doc-search-tag="utils: template fromUtilsGen(): untyped" href="subdir/subdir_b/utils.html#fromUtilsGen.t">utils: template fromUtilsGen(): untyped
        funWithGenerics:
        G:
        • utils: G
          • + data-doc-search-tag="utils: object G" href="subdir/subdir_b/utils.html#G">utils: object G
        isValid:
        • testproject: isValid[T](x: T): bool
          • + data-doc-search-tag="testproject: proc isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: proc isValid[T](x: T): bool
        iter1:
        • testproject: iter1(n: int): int
          • + data-doc-search-tag="testproject: iterator iter1(n: int): int" href="testproject.html#iter1.i%2Cint">testproject: iterator iter1(n: int): int
        iter2:
        • testproject: iter2(n: int): int
          • + data-doc-search-tag="testproject: iterator iter2(n: int): int" href="testproject.html#iter2.i%2Cint">testproject: iterator iter2(n: int): int
        low:
        low2:
        method1:
        • testproject: method1(self: Moo)
          • + data-doc-search-tag="testproject: method method1(self: Moo)" href="testproject.html#method1.e%2CMoo">testproject: method method1(self: Moo)
        method2:
        • testproject: method2(self: Moo): int
          • + data-doc-search-tag="testproject: method method2(self: Moo): int" href="testproject.html#method2.e%2CMoo">testproject: method method2(self: Moo): int
        method3:
        • testproject: method3(self: Moo): int
          • + data-doc-search-tag="testproject: method method3(self: Moo): int" href="testproject.html#method3.e%2CMoo">testproject: method method3(self: Moo): int
        myfn:
        • testproject: myfn()
          • + data-doc-search-tag="testproject: template myfn()" href="testproject.html#myfn.t">testproject: template myfn()
        p1:
        • testproject: p1()
          • + data-doc-search-tag="testproject: proc p1()" href="testproject.html#p1">testproject: proc p1()
        Rectangle:
        • Shapes.Rectangle
          • + data-doc-search-tag="testproject: Shapes.Rectangle" href="testproject.html#Rectangle">testproject: Shapes.Rectangle
        Shapes:
        • testproject: Shapes
          • + data-doc-search-tag="testproject: enum Shapes" href="testproject.html#Shapes">testproject: enum Shapes
        someFunc:
        • testproject: someFunc()
          • + data-doc-search-tag="testproject: proc someFunc()" href="testproject.html#someFunc">testproject: proc someFunc()
        SomeType:
        • utils: SomeType
          • + data-doc-search-tag="utils: enum SomeType" href="subdir/subdir_b/utils.html#SomeType">utils: enum SomeType
        someType:
        • utils: someType(): SomeType
          • + data-doc-search-tag="utils: proc someType(): SomeType" href="subdir/subdir_b/utils.html#someType_2">utils: proc someType(): SomeType
        someVariable:
        • testproject: someVariable
          • + data-doc-search-tag="testproject: var someVariable" href="testproject.html#someVariable">testproject: var someVariable
        testNimDocTrailingExample:
        • testproject: testNimDocTrailingExample()
          • + data-doc-search-tag="testproject: template testNimDocTrailingExample()" href="testproject.html#testNimDocTrailingExample.t">testproject: template testNimDocTrailingExample()
        Triangle:
        • Shapes.Triangle
          • + data-doc-search-tag="testproject: Shapes.Triangle" href="testproject.html#Triangle">testproject: Shapes.Triangle
        tripleStrLitTest:
        • testproject: tripleStrLitTest()
          • + data-doc-search-tag="testproject: proc tripleStrLitTest()" href="testproject.html#tripleStrLitTest">testproject: proc tripleStrLitTest()
        z1:
        • testproject: z1(): Foo
          • + data-doc-search-tag="testproject: proc z1(): Foo" href="testproject.html#z1">testproject: proc z1(): Foo
        z10:
        • testproject: z10()
          • + data-doc-search-tag="testproject: proc z10()" href="testproject.html#z10">testproject: proc z10()
        z11:
        • testproject: z11()
          • + data-doc-search-tag="testproject: proc z11()" href="testproject.html#z11">testproject: proc z11()
        z12:
        • testproject: z12(): int
          • + data-doc-search-tag="testproject: proc z12(): int" href="testproject.html#z12">testproject: proc z12(): int
        z13:
        • testproject: z13()
          • + data-doc-search-tag="testproject: proc z13()" href="testproject.html#z13">testproject: proc z13()
        z14:
        • testproject: z14()
          • + data-doc-search-tag="testproject: template z14()" href="testproject.html#z14.t">testproject: template z14()
        z15:
        • testproject: z15()
          • + data-doc-search-tag="testproject: template z15()" href="testproject.html#z15.t">testproject: template z15()
        z16:
        • testproject: z16()
          • + data-doc-search-tag="testproject: macro z16()" href="testproject.html#z16.m">testproject: macro z16()
        z17:
        • testproject: z17()
          • + data-doc-search-tag="testproject: proc z17()" href="testproject.html#z17">testproject: proc z17()
        z18:
        • testproject: z18(): int
          • + data-doc-search-tag="testproject: macro z18(): int" href="testproject.html#z18.m">testproject: macro z18(): int
        z2:
        • testproject: z2()
          • + data-doc-search-tag="testproject: proc z2()" href="testproject.html#z2">testproject: proc z2()
        z3:
        • testproject: z3()
          • + data-doc-search-tag="testproject: proc z3()" href="testproject.html#z3">testproject: proc z3()
        z4:
        • testproject: z4()
          • + data-doc-search-tag="testproject: proc z4()" href="testproject.html#z4">testproject: proc z4()
        z5:
        • testproject: z5(): int
          • + data-doc-search-tag="testproject: proc z5(): int" href="testproject.html#z5">testproject: proc z5(): int
        z6:
        • testproject: z6(): int
          • + data-doc-search-tag="testproject: proc z6(): int" href="testproject.html#z6">testproject: proc z6(): int
        z6t:
        • testproject: z6t(): int
          • + data-doc-search-tag="testproject: template z6t(): int" href="testproject.html#z6t.t">testproject: template z6t(): int
        z7:
        • testproject: z7(): int
          • + data-doc-search-tag="testproject: proc z7(): int" href="testproject.html#z7">testproject: proc z7(): int
        z8:
        • testproject: z8(): int
          • + data-doc-search-tag="testproject: proc z8(): int" href="testproject.html#z8">testproject: proc z8(): int
        z9:
        • testproject: z9()
          • + data-doc-search-tag="testproject: proc z9()" href="testproject.html#z9">testproject: proc z9()
        diff --git a/tests/stdlib/tdochelpers.nim b/tests/stdlib/tdochelpers.nim index 15d538891..300ce76dc 100644 --- a/tests/stdlib/tdochelpers.nim +++ b/tests/stdlib/tdochelpers.nim @@ -202,3 +202,19 @@ suite "Integration with Nim": name: "Copyflag") check inputRst.fromRst == expected check inputMd.fromMd == expected + + test "prefixed module": + let inputRst = "`module std / paths`_" + let inputMd = "[module std / paths]" + let expected = LangSymbol(symKind: "module", + name: "std/paths") + check inputRst.fromRst == expected + check inputMd.fromMd == expected + + test "postfixed module": + let inputRst = "`std / paths module`_" + let inputMd = "[std / paths module]" + let expected = LangSymbol(symKind: "module", + name: "std/paths") + check inputRst.fromRst == expected + check inputMd.fromMd == expected diff --git a/tests/stdlib/trst.nim b/tests/stdlib/trst.nim index 3575d17aa..da787d274 100644 --- a/tests/stdlib/trst.nim +++ b/tests/stdlib/trst.nim @@ -60,7 +60,7 @@ proc toAst(input: string, result = "" var (rst, _, _) = rstParse(input, filen, line=LineRstInit, column=ColRstInit, - rstOptions, myFindFile, testMsgHandler) + rstOptions, myFindFile, nil, testMsgHandler) result = treeRepr(rst) except EParseError as e: if e.msg != "": diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 4fffc7533..f63aea030 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -240,27 +240,28 @@ proc buildDocSamples(nimArgs, destPath: string) = exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDocPackages(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = # compiler docs; later, other packages (perhaps tools, testament etc) let nim = findNim().quoteShell() # to avoid broken links to manual from compiler dir, but a multi-package # structure could be supported later proc docProject(outdir, options, mainproj: string) = - exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [ + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ "nim", nim, "outdir", outdir, "nimArgs", nimArgs, "gitUrl", gitUrl, "options", options, "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" ]) let extra = "-u:boot" # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") -proc buildDoc(nimArgs, destPath: string) = +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: let rst2html = getMd2html() var @@ -268,17 +269,19 @@ proc buildDoc(nimArgs, destPath: string) = officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " md2html $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): let extra = if isJsOnly(d): "--backend:js" else: "" var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): doAssert false - commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" % - [extra, nimArgs2, gitUrl, destPath, d] + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % @@ -300,12 +303,6 @@ proc buildDoc(nimArgs, destPath: string) = i.inc mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) - # caveat: this works so long it's called before `buildDocPackages` which - # populates `compiler/` with unrelated idx files that shouldn't be in index, - # so should work in CI but you may need to remove your generated html files - # locally after calling `./koch docs`. The clean fix would be for `idx` files - # to be transient with `--project` (eg all in memory). proc nim2pdf(src: string, dst: string, nimArgs: string) = # xxx expose as a `nim` command or in some other reusable way. @@ -351,9 +348,23 @@ proc buildDocsDir*(args: string, dir: string) = gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector"]) createDir(dir) buildDocSamples(args, dir) - buildDoc(args, dir) # bottleneck + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + copyFile(dir / "overview.html", dir / "index.html") - buildDocPackages(args, dir) copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) proc buildDocs*(args: string, localOnly = false, localOutDir = "") = From b34412ff0fb450278d8c93f6e5d7bf0e8d3f0a0b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 6 Jan 2023 13:24:58 +0800 Subject: [PATCH 1528/1654] adds changelog for more architectures support and fixes a link (#21229) --- changelogs/changelog_2_0_0.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 29d906d62..9ff493b54 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -26,7 +26,7 @@ - Enabling `-d:nimPreviewSlimSystem` also removes the following deprecated symbols in the `system` module: - Aliases with `Error` suffix to exception types that have a `Defect` suffix - (see [exceptions](https://nim-lang.org/docs/exceptions.html)): + (see [exceptions](https://nim-lang.github.io/Nim/exceptions.html)): `ArithmeticError`, `DivByZeroError`, `OverflowError`, `AccessViolationError`, `AssertionError`, `OutOfMemError`, `IndexError`, `FieldError`, `RangeError`, `StackOverflowError`, `ReraiseError`, @@ -317,6 +317,7 @@ - Nim now supports `out` parameters and ["strict definitions"](https://nim-lang.github.io/Nim/manual_experimental.html#strict-definitions-and-nimout-parameters). - Nim now offers a [strict mode](https://nim-lang.github.io/Nim/manual_experimental.html#strict-case-objects) for `case objects`. +- IBM Z architecture and macOS m1 arm64 architecture are supported. ## Compiler changes From 05c6db6500de0551882a2d6cc612c9afb6ca674c Mon Sep 17 00:00:00 2001 From: metagn Date: Sun, 8 Jan 2023 08:49:49 +0300 Subject: [PATCH 1529/1654] underscores for routine parameters (#21192) * underscores for routine parameters fixes #13443, fixes #13804, refs #21121 * add changelog + more tests * support generics and ensure inferred lambdas work --- changelogs/changelog_2_0_0.md | 15 +++++ compiler/lookups.nim | 1 + compiler/semtempl.nim | 5 +- compiler/semtypes.nim | 4 +- tests/proc/tunderscoreparam.nim | 109 ++++++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 tests/proc/tunderscoreparam.nim diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index 9ff493b54..c8427cc2a 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -135,6 +135,21 @@ - The experimental strictFuncs feature now disallows a store to the heap via a `ref` or `ptr` indirection. +- Underscores (`_`) as routine parameters are now ignored and cannot be used in the routine body. + The following code now does not compile: + + ```nim + proc foo(_: int): int = _ + 1 + echo foo(1) + ``` + + Instead, the following code now compiles: + + ```nim + proc foo(_, _: int): int = 123 + echo foo(1, 2) + ``` + - - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. ## Standard library additions and changes diff --git a/compiler/lookups.nim b/compiler/lookups.nim index d50fc6e27..3f4fcb4d0 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -323,6 +323,7 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string; # xxx pending bootstrap >= 1.4, replace all those overloads with a single one: # proc addDecl*(c: PContext, sym: PSym, info = sym.info, scope = c.currentScope) {.inline.} = proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = + if sym.name.s == "_": return let conflict = scope.addUniqueSym(sym) if conflict != nil: if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 0016e9edf..3b2f2dd9e 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -642,8 +642,9 @@ proc semTemplateDef(c: PContext, n: PNode): PNode = # body by the absence of the sfGenSym flag: for i in 1.. Date: Sun, 8 Jan 2023 16:37:40 +0800 Subject: [PATCH 1530/1654] disable bump (#21241) --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index c134be6f9..77d89a420 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -46,7 +46,7 @@ pkg "BipBuffer" pkg "blscurve", allowFailure = true # pending https://github.com/status-im/nim-blscurve/issues/39 pkg "bncurve" pkg "brainfuck", "nim c -d:release -r tests/compile.nim" -pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump" +pkg "bump", "nim c --gc:arc --path:. -r tests/tbump.nim", "https://github.com/disruptek/bump", allowFailure = true pkg "c2nim", "nim c testsuite/tester.nim" pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true From 76e4f8afb1423eb04f26059f1ed01ca0375bee63 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:39:25 +0800 Subject: [PATCH 1531/1654] fixes #21209; exclude threads options for JS (#21233) --- compiler/commands.nim | 2 ++ tests/js/t21209.nim | 6 ++++++ tests/js/t21209.nims | 1 + 3 files changed, 9 insertions(+) create mode 100644 tests/js/t21209.nim create mode 100644 tests/js/t21209.nims diff --git a/compiler/commands.nim b/compiler/commands.nim index 73140036a..59b221647 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -649,6 +649,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "backend", "b": let backend = parseEnum(arg.normalize, TBackend.default) if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg) + if backend == backendJs: # bug #21209 + conf.globalOptions.excl {optThreadAnalysis, optThreads} conf.backend = backend of "doccmd": conf.docCmd = arg of "define", "d": diff --git a/tests/js/t21209.nim b/tests/js/t21209.nim new file mode 100644 index 000000000..4de34f035 --- /dev/null +++ b/tests/js/t21209.nim @@ -0,0 +1,6 @@ +discard """ + action: "compile" + cmd: "nim check --warning[UnusedImport]:off $file" +""" + +import std/times diff --git a/tests/js/t21209.nims b/tests/js/t21209.nims new file mode 100644 index 000000000..318e28f97 --- /dev/null +++ b/tests/js/t21209.nims @@ -0,0 +1 @@ +--b:js \ No newline at end of file From a55604a3d8605e398f27df364cc215b7114395eb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 8 Jan 2023 22:40:27 +0800 Subject: [PATCH 1532/1654] fixes #21232; highlite.getNextToken() returns gtOperator for '/' in C (#21234) --- lib/packages/docutils/highlite.nim | 3 +++ tests/stdlib/thighlite.nim | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 3f65c8cc4..c3caf31b4 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -498,6 +498,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], of '\0': break else: inc(pos) + else: + g.kind = gtOperator + while g.buf[pos] in OpChars: inc(pos) of '#': inc(pos) if hasPreprocessor in flags: diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim index 5134215c1..d88bc7fec 100644 --- a/tests/stdlib/thighlite.nim +++ b/tests/stdlib/thighlite.nim @@ -1,6 +1,7 @@ import unittest, strutils import ../../lib/packages/docutils/highlite +import std/objectdollar block: # Nim tokenizing test "string literals and escape seq": @@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing ("file.nim", gtIdentifier), ("\n", gtWhitespace), ("out: file [SuccessX]", gtProgramOutput) ]) + +block: # bug #21232 + let code = "/" + var toknizr: GeneralTokenizer + + initGeneralTokenizer(toknizr, code) + + getNextToken(toknizr, langC) + check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)""" From b68b28fd2448b15e989b9e8b07e33eee4f9b8822 Mon Sep 17 00:00:00 2001 From: Bung Date: Sun, 8 Jan 2023 22:34:51 -0800 Subject: [PATCH 1533/1654] fix typeRel of typeDesc (#21227) --- compiler/sigmatch.nim | 8 ++++++-- tests/typerel/tint.nim | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 tests/typerel/tint.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 75e3e5428..0dbd5e5f1 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1840,7 +1840,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, elif f.base.kind == tyNone: result = isGeneric else: - result = typeRel(c, f.base, a.base, flags) + let r = typeRel(c, f.base, a.base, flags) + if r >= isIntConv: + result = r if result != isNone: put(c, f, a) @@ -1848,7 +1850,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, if tfUnresolved in f.flags: result = typeRel(c, prev.base, a, flags) elif a.kind == tyTypeDesc: - result = typeRel(c, prev.base, a.base, flags) + let r = typeRel(c, prev.base, a.base, flags) + if r >= isIntConv: + result = r else: result = isNone diff --git a/tests/typerel/tint.nim b/tests/typerel/tint.nim new file mode 100644 index 000000000..de0e78a81 --- /dev/null +++ b/tests/typerel/tint.nim @@ -0,0 +1,4 @@ + +template a(T: type int32) = discard +template a(T: type int64) = discard +a(int) From 8be46f3d16fc4c3b05d927aa1e9738ac865a266e Mon Sep 17 00:00:00 2001 From: Russell Brinson <57331546+rbrins@users.noreply.github.com> Date: Wed, 11 Jan 2023 12:20:39 -0500 Subject: [PATCH 1534/1654] parseopt.nim documentation clarity - default values & cmdEnd in getopt (#21047) parseopt.nim documentation clarity Added example for default values to cmd line parameters. Additionally, added lines in getopt documentation about case switching still requiring the `cmdEnd` kind. Hopefully this clears up any vagueness for those following along in the example but omitting the `cmdEnd` in the case because the documentation said it wasn't needed. --- lib/pure/parseopt.nim | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index f8d03e092..9052067db 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -74,7 +74,28 @@ ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. +## +## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. +## Assign the default value to a variable before parsing. +## Then set the variable to the new value while parsing. ## +## Here is an example: +## .. code-block:: +## import std/parseopt +## +## var varName: string = "defaultValue" +## +## for kind, key, val in getopt(): +## case kind +## of cmdArgument: +## discard +## of cmdLongOption, cmdShortOption: +## case key: +## of "varName": # --varName: in the console when executing +## varName = val # do input sanitization in production systems +## of cmdEnd: +## discard +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -210,6 +231,8 @@ proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## + ## This does not provide a way of passing default values to arguments. + ## ## See also: ## * `getopt iterator<#getopt.i,OptParser>`_ runnableExamples: @@ -403,7 +426,8 @@ iterator getopt*(p: var OptParser): tuple[kind: CmdLineKind, key, ## Convenience iterator for iterating over the given ## `OptParser<#OptParser>`_. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,string,set[char],seq[string]>`_ @@ -451,7 +475,8 @@ iterator getopt*(cmdline: seq[string] = @[], ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## with case switching, checking for `cmdEnd` is required. ## ## See also: ## * `initOptParser proc<#initOptParser,seq[string],set[char],seq[string]>`_ From e4e947232b9e054cd55c9fefe3ea97f47619a5ca Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 11 Jan 2023 22:23:43 +0500 Subject: [PATCH 1535/1654] testament: make valgrind optional (#21238) Add a simple switch to enable/disable valgrind support. On by default --- doc/testament.md | 1 + testament/testament.nim | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/testament.md b/doc/testament.md index ee57be4f1..08c3cd835 100644 --- a/doc/testament.md +++ b/doc/testament.md @@ -50,6 +50,7 @@ Options --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. +--valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored diff --git a/testament/testament.nim b/testament/testament.nim index ac99a2373..296c313aa 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -29,6 +29,7 @@ var backendLogging = true var simulate = false var optVerbose = false var useMegatest = true +var valgrindEnabled = true proc verboseCmd(cmd: string) = if optVerbose: @@ -60,6 +61,7 @@ Options: --colors:on|off Turn messages coloring on|off. --backendLogging:on|off Disable or enable backend logging. By default turned on. --megatest:on|off Enable or disable megatest. Default is on. + --valgrind:on|off Enable or disable valgrind support. Default is on. --skipFrom:file Read tests to skip from `file` - one test per line, # comments ignored On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API @@ -487,7 +489,7 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, args = @["--unhandled-rejections=strict", exeFile] & args else: exeCmd = exeFile.dup(normalizeExe) - if expected.useValgrind != disabled: + if valgrindEnabled and expected.useValgrind != disabled: var valgrindOptions = @["--error-exitcode=1"] if expected.useValgrind != leaking: valgrindOptions.add "--leak-check=yes" @@ -664,6 +666,14 @@ proc main() = useMegatest = false else: quit Usage + of "valgrind": + case p.val: + of "on": + valgrindEnabled = true + of "off": + valgrindEnabled = false + else: + quit Usage of "backendlogging": case p.val: of "on": From 1e52423774e81f71d10a712575fd65231705362e Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Thu, 12 Jan 2023 12:44:33 +1100 Subject: [PATCH 1536/1654] Fix getting custom pragma from generic object (#20481) * Merge devel Add another test case * Fix test Use getCustomPragmaVal instead of hasCustomPragma Co-authored-by: Clay Sweetser --- lib/core/macros.nim | 2 +- tests/stdlib/tmacros.nim | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index beb3aa695..9cb694bd8 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -1591,7 +1591,7 @@ proc customPragmaNode(n: NimNode): NimNode = elif impl.kind in {nnkIdentDefs, nnkConstDef} and impl[0].kind == nnkPragmaExpr: return impl[0][1] else: - let timpl = typ.getImpl() + let timpl = getImpl(if typ.kind == nnkBracketExpr: typ[0] else: typ) if timpl.len>0 and timpl[0].len>1: return timpl[0][1] else: diff --git a/tests/stdlib/tmacros.nim b/tests/stdlib/tmacros.nim index 7ec2fed9f..c054b66c7 100644 --- a/tests/stdlib/tmacros.nim +++ b/tests/stdlib/tmacros.nim @@ -157,3 +157,21 @@ block: # bug #19020 static: doAssert $bar.getCustomPragmaVal(typ) == "foo" doAssert $bar.getCustomPragmaVal(typ) == "foo" + +block hasCustomPragmaGeneric: + template examplePragma() {.pragma.} + type + Foo[T] {.examplePragma.} = object + x {.examplePragma.}: T + var f: Foo[string] + doAssert f.hasCustomPragma(examplePragma) + doAssert f.x.hasCustomPragma(examplePragma) + +block getCustomPragmaValGeneric: + template examplePragma(x: int) {.pragma.} + type + Foo[T] {.examplePragma(42).} = object + x {.examplePragma(25).}: T + var f: Foo[string] + doAssert f.getCustomPragmaVal(examplePragma) == 42 + doAssert f.x.getCustomPragmaVal(examplePragma) == 25 From 5f41be3b4503a76fb68018c3ff5b05b35d6d68c1 Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 13 Jan 2023 06:36:13 -0800 Subject: [PATCH 1537/1654] fix #20253 (#21174) * fix #20253 * change NimbleStableCommit * Update koch.nim Co-authored-by: Andreas Rumpf --- compiler/sempass2.nim | 5 ++++- tests/misc/t20253.nim | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/misc/t20253.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index f13d8d8d4..747540c78 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1488,7 +1488,10 @@ proc trackProc*(c: PContext; s: PSym, body: PNode) = s.kind in {skProc, skFunc, skConverter, skMethod} and s.magic == mNone: var res = s.ast[resultPos].sym # get result symbol if res.id notin t.init: - message(g.config, body.info, warnProveInit, "result") + if tfRequiresInit in s.typ[0].flags: + localError(g.config, body.info, "'$1' requires explicit initialization" % "result") + else: + message(g.config, body.info, warnProveInit, "result") let p = s.ast[pragmasPos] let raisesSpec = effectSpec(p, wRaises) if not isNil(raisesSpec): diff --git a/tests/misc/t20253.nim b/tests/misc/t20253.nim new file mode 100644 index 000000000..d47c36c55 --- /dev/null +++ b/tests/misc/t20253.nim @@ -0,0 +1,10 @@ +discard """ + errormsg: "'result' requires explicit initialization" + line: 10 +""" + +type Meow {.requiresInit.} = object + init: bool + +proc initMeow(): Meow = + discard From 71c60bf6f760d3a1dfc12135866cd46f817ee439 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 14 Jan 2023 19:30:02 +0800 Subject: [PATCH 1538/1654] bump nimble (#21259) --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index 6b3d327f5..27953e05a 100644 --- a/koch.nim +++ b/koch.nim @@ -10,7 +10,7 @@ # const - NimbleStableCommit = "afd03bc000f4013c48f52381728327dabfccfc4c" # master + NimbleStableCommit = "7efb226ef908297e8791cade20d991784b4e8bfc" # master # examples of possible values: #head, #ea82b54, 1.2.3 FusionStableHash = "#372ee4313827ef9f2ea388840f7d6b46c2b1b014" HeadHash = "#head" From 30da566d9dc81feab9ba3015d293d93b5c6785af Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 17 Jan 2023 12:09:26 +0800 Subject: [PATCH 1539/1654] switch to official DelaunayNim URL (#21269) follow up https://github.com/Nycto/DelaunayNim/pull/4#event-8240077930 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 77d89a420..d9835ef23 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -60,7 +60,7 @@ pkg "comprehension", "nimble test", "https://github.com/alehander92/comprehensio pkg "criterion", allowFailure = true # pending https://github.com/disruptek/criterion/issues/3 (wrongly closed) pkg "datamancer" pkg "dashing", "nim c tests/functional.nim" -pkg "delaunay", url = "https://github.com/nim-lang/DelaunayNim", useHead = true +pkg "delaunay" pkg "docopt" pkg "easygl", "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl" pkg "elvis" From cc08a9015e446471be8b734c2895d99c91c59cb3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 18 Jan 2023 06:15:23 +0800 Subject: [PATCH 1540/1654] fixes #21263; consider all candidates for concept matches (#21265) --- compiler/concepts.nim | 2 +- compiler/lookups.nim | 17 +++++++++++++++++ tests/concepts/tconcepts_issues.nim | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/compiler/concepts.nim b/compiler/concepts.nim index a810b3642..bcf2d4d0e 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -274,7 +274,7 @@ proc matchSym(c: PContext; candidate: PSym, n: PNode; m: var MatchCon): bool = proc matchSyms(c: PContext, n: PNode; kinds: set[TSymKind]; m: var MatchCon): bool = ## Walk the current scope, extract candidates which the same name as 'n[namePos]', ## 'n' is the nkProcDef or similar from the concept that we try to match. - let candidates = searchInScopesFilterBy(c, n[namePos].sym.name, kinds) + let candidates = searchInScopesAllCandidatesFilterBy(c, n[namePos].sym.name, kinds) for candidate in candidates: #echo "considering ", typeToString(candidate.typ), " ", candidate.magic m.magic = candidate.magic diff --git a/compiler/lookups.nim b/compiler/lookups.nim index 3f4fcb4d0..e7bca08bc 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -228,6 +228,23 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} = if i == limit: return inc i +proc searchInScopesAllCandidatesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] = + result = @[] + for scope in allScopes(c.currentScope): + var ti: TIdentIter + var candidate = initIdentIter(ti, scope.symbols, s) + while candidate != nil: + if candidate.kind in filter: + result.add candidate + candidate = nextIdentIter(ti, scope.symbols) + + if result.len == 0: + var marked = initIntSet() + for im in c.imports.mitems: + for s in symbols(im, marked, s, c.graph): + if s.kind in filter: + result.add s + proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSym] = result = @[] block outer: diff --git a/tests/concepts/tconcepts_issues.nim b/tests/concepts/tconcepts_issues.nim index 802582f57..3efb9964a 100644 --- a/tests/concepts/tconcepts_issues.nim +++ b/tests/concepts/tconcepts_issues.nim @@ -27,6 +27,8 @@ false true -1 Meow +10 0.0 +1 2.0 ''' joinable: false """ @@ -500,3 +502,26 @@ var r, b: Fp2[6, uint64] prod(r, b) + +block: # bug #21263 + type + DateDayFraction = concept # no T, an atom + proc date(a: Self): int + proc fraction(b: Self): float + Date = distinct int + DateDayFractionImpl = object + date : int + fraction : float + + proc date(a: Date): int = a.int + proc fraction(a:Date): float = 0.0 + + proc date(a: DateDayFractionImpl): int = a.date + proc fraction(b: DateDayFractionImpl): float = b.fraction + + + proc print(a: DateDayFraction) = + echo a.date, " ", a.fraction + + print(10.Date) # ok + print(DateDayFractionImpl(date: 1, fraction: 2)) # error From c4035d7f7c424caa81ee449d2032ecd8e8967ebd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 18 Jan 2023 16:03:03 +0800 Subject: [PATCH 1541/1654] fixes unittest warnings (#21271) --- lib/pure/unittest.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index c87fe38f1..9dafa8f03 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -544,11 +544,14 @@ template test*(name, body) {.dirty.} = for formatter in formatters: formatter.testStarted(name) + {.warning[BareExcept]:off.} try: when declared(testSetupIMPLFlag): testSetupIMPL() when declared(testTeardownIMPLFlag): defer: testTeardownIMPL() + {.warning[BareExcept]:on.} body + {.warning[BareExcept]:off.} except: let e = getCurrentException() @@ -570,6 +573,7 @@ template test*(name, body) {.dirty.} = ) testEnded(testResult) checkpoints = @[] + {.warning[BareExcept]:on.} proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all From fc35f83eee55610af3931f95771b6d1bce1fc845 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 18 Jan 2023 18:52:18 +0800 Subject: [PATCH 1542/1654] fixes #21260; add check for illegal recursion for defaults (#21270) * fixes #21260; add check for illegal recursion for defaults * fixes differently --- compiler/sem.nim | 36 ++++++++++++++---------------------- compiler/semdata.nim | 1 + compiler/semmagic.nim | 3 +-- compiler/semobjconstr.nim | 15 +++++++++------ tests/types/t21260.nim | 13 +++++++++++++ 5 files changed, 38 insertions(+), 30 deletions(-) create mode 100644 tests/types/t21260.nim diff --git a/compiler/sem.nim b/compiler/sem.nim index f5283fa3d..c1bb89fc8 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -550,18 +550,17 @@ proc pickCaseBranchIndex(caseExpr, matched: PNode): int = if endsWithElse: return caseExpr.len - 1 -proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSet): seq[PNode] -proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode -proc defaultNodeField(c: PContext, a: PNode): PNode +proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode): seq[PNode] proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode +proc defaultNodeField(c: PContext, a: PNode): PNode const defaultFieldsSkipTypes = {tyGenericInst, tyAlias, tySink} -proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefault: var bool): seq[PNode] = +proc defaultFieldsForTuple(c: PContext, recNode: PNode, hasDefault: var bool): seq[PNode] = case recNode.kind of nkRecList: for field in recNode: - result.add defaultFieldsForTuple(c, field, id, hasDefault) + result.add defaultFieldsForTuple(c, field, hasDefault) of nkSym: let field = recNode.sym let recType = recNode.typ.skipTypes(defaultFieldsSkipTypes) @@ -570,7 +569,7 @@ proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefau result.add newTree(nkExprColonExpr, recNode, field.ast) else: if recType.kind in {tyObject, tyArray, tyTuple}: - let asgnExpr = defaultNodeField(c, recNode, recNode.typ, id) + let asgnExpr = defaultNodeField(c, recNode, recNode.typ) if asgnExpr != nil: hasDefault = true asgnExpr.flags.incl nfUseDefaultField @@ -589,11 +588,11 @@ proc defaultFieldsForTuple(c: PContext, recNode: PNode, id: var IntSet, hasDefau else: doAssert false -proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSet): seq[PNode] = +proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode): seq[PNode] = case recNode.kind of nkRecList: for field in recNode: - result.add defaultFieldsForTheUninitialized(c, field, id) + result.add defaultFieldsForTheUninitialized(c, field) of nkRecCase: let discriminator = recNode[0] var selectedBranch: int @@ -607,14 +606,14 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSe selectedBranch = recNode.pickCaseBranchIndex defaultValue defaultValue.flags.incl nfUseDefaultField result.add newTree(nkExprColonExpr, discriminator, defaultValue) - result.add defaultFieldsForTheUninitialized(c, recNode[selectedBranch][^1], id) + result.add defaultFieldsForTheUninitialized(c, recNode[selectedBranch][^1]) of nkSym: let field = recNode.sym let recType = recNode.typ.skipTypes(defaultFieldsSkipTypes) if field.ast != nil: #Try to use default value result.add newTree(nkExprColonExpr, recNode, field.ast) elif recType.kind in {tyObject, tyArray, tyTuple}: - let asgnExpr = defaultNodeField(c, recNode, recType, id) + let asgnExpr = defaultNodeField(c, recNode, recType) if asgnExpr != nil: asgnExpr.typ = recType asgnExpr.flags.incl nfUseDefaultField @@ -622,19 +621,17 @@ proc defaultFieldsForTheUninitialized(c: PContext, recNode: PNode, id: var IntSe else: doAssert false -proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode = +proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode = let aTypSkip = aTyp.skipTypes(defaultFieldsSkipTypes) if aTypSkip.kind == tyObject: - if id.containsOrIncl(aTypSkip.id): - return - let child = defaultFieldsForTheUninitialized(c, aTypSkip.n, id) + let child = defaultFieldsForTheUninitialized(c, aTypSkip.n) if child.len > 0: var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, a.info, aTypSkip)) asgnExpr.typ = aTypSkip asgnExpr.sons.add child result = semExpr(c, asgnExpr) elif aTypSkip.kind == tyArray: - let child = defaultNodeField(c, a, aTypSkip[1], id) + let child = defaultNodeField(c, a, aTypSkip[1]) if child != nil: let node = newNode(nkIntLit) @@ -647,20 +644,15 @@ proc defaultNodeField(c: PContext, a: PNode, aTyp: PType, id: var IntSet): PNode elif aTypSkip.kind == tyTuple: var hasDefault = false if aTypSkip.n != nil: - let children = defaultFieldsForTuple(c, aTypSkip.n, id, hasDefault) + let children = defaultFieldsForTuple(c, aTypSkip.n, hasDefault) if hasDefault and children.len > 0: result = newNodeI(nkTupleConstr, a.info) result.typ = aTyp result.sons.add children result = semExpr(c, result) -proc defaultNodeField(c: PContext, a: PNode, aTyp: PType): PNode = - var s = initIntSet() - defaultNodeField(c, a, aTyp, s) - proc defaultNodeField(c: PContext, a: PNode): PNode = - var s = initIntSet() - result = defaultNodeField(c, a, a.typ, s) + result = defaultNodeField(c, a, a.typ) include semtempl, semgnrc, semstmts, semexprs diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 29c2c139f..5b94cc770 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -75,6 +75,7 @@ type # overload resolution. efNoDiagnostics, efTypeAllowed # typeAllowed will be called after + efWantNoDefaults TExprFlags* = set[TExprFlag] diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 7025dc867..751ca8fe7 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -20,9 +20,8 @@ proc addDefaultFieldForNew(c: PContext, n: PNode): PNode = var asgnExpr = newTree(nkObjConstr, newNodeIT(nkType, result[1].info, typ)) asgnExpr.typ = typ var t = typ.skipTypes({tyGenericInst, tyAlias, tySink})[0] - var id = initIntSet() while true: - asgnExpr.sons.add defaultFieldsForTheUninitialized(c, t.n, id) + asgnExpr.sons.add defaultFieldsForTheUninitialized(c, t.n) let base = t[0] if base == nil: break diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 206602677..15e53a639 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -326,10 +326,13 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, result.status = initUnknown result.defaults.add newTree(nkExprColonExpr, n, field.ast) else: - let defaultExpr = defaultNodeField(c, n) - if defaultExpr != nil: - result.status = initUnknown - result.defaults.add newTree(nkExprColonExpr, n, defaultExpr) + if efWantNoDefaults notin flags: # cannot compute defaults at the typeRightPass + let defaultExpr = defaultNodeField(c, n) + if defaultExpr != nil: + result.status = initUnknown + result.defaults.add newTree(nkExprColonExpr, n, defaultExpr) + else: + result.status = initNone else: result.status = initNone else: @@ -364,7 +367,7 @@ proc initConstrContext(t: PType, initExpr: PNode): ObjConstrContext = proc computeRequiresInit(c: PContext, t: PType): bool = assert t.kind == tyObject var constrCtx = initConstrContext(t, newNode(nkObjConstr)) - let initResult = semConstructTypeAux(c, constrCtx, {}) + let initResult = semConstructTypeAux(c, constrCtx, {efWantNoDefaults}) constrCtx.missingFields.len > 0 proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = @@ -374,7 +377,7 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) = assert objType != nil if objType.kind == tyObject: var constrCtx = initConstrContext(objType, newNodeI(nkObjConstr, info)) - let initResult = semConstructTypeAux(c, constrCtx, {}) + let initResult = semConstructTypeAux(c, constrCtx, {efWantNoDefaults}) if constrCtx.missingFields.len > 0: localError(c.config, info, "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)]) diff --git a/tests/types/t21260.nim b/tests/types/t21260.nim new file mode 100644 index 000000000..90d6613c1 --- /dev/null +++ b/tests/types/t21260.nim @@ -0,0 +1,13 @@ +discard """ + errormsg: "illegal recursion in type 'Foo'" + line: 8 +""" + +type + Kind = enum kA, kB + Foo = object + case k: Kind: + of kA: + foo: Foo + of kB: + discard From fd207827c5929ba5eb244dd2442457944a55b7d0 Mon Sep 17 00:00:00 2001 From: PhilippMDoerner Date: Wed, 18 Jan 2023 19:10:05 +0100 Subject: [PATCH 1543/1654] Add docs informing about gotcha of clients (#21262) * Add docs informing about gotcha of clients It is a hidden problem that AsyncHttpClient-instances *can not* deal with multiple requests at once. Similar to normal HttpClients they can only deal with sending one request at a time. This is not told anywhere in the documentation, but critical information that should be available to the user. * Update lib/pure/httpclient.nim fixes typo Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- lib/pure/httpclient.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index c832d4285..d2cf64149 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -46,6 +46,10 @@ ## **Note:** You need to run asynchronous examples in an async proc ## otherwise you will get an `Undeclared identifier: 'await'` error. ## +## **Note:** An asynchronous client instance can only deal with one +## request at a time. To send multiple requests in parallel, use +## multiple client instances. +## ## Using HTTP POST ## =============== ## From fc5b7a01eccfbfeba6ded8765ca956eab0952db5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 19 Jan 2023 02:10:34 +0800 Subject: [PATCH 1544/1654] fixes #21243; ships `build_all.sh` on Unix [backport] (#21264) fixes #21243; ships `build_all.sh` on Unix --- compiler/installer.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/installer.ini b/compiler/installer.ini index 1e57455c6..f67ca5c25 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -121,6 +121,7 @@ Files: "bin/nim" InstallScript: "yes" UninstallScript: "yes" Files: "bin/nim-gdb" +Files: "build_all.sh" [InnoSetup] From f04f3e8bf78ab4187a395a024c94cd545c275720 Mon Sep 17 00:00:00 2001 From: Phil Krylov Date: Wed, 18 Jan 2023 19:11:15 +0100 Subject: [PATCH 1545/1654] std/parsesql: Add OFFSET keyword support. (#21255) Co-authored-by: Clay Sweetser --- lib/pure/parsesql.nim | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index 3af840e29..074faf64c 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -60,7 +60,7 @@ const reservedKeywords = @[ # statements - "select", "from", "where", "group", "limit", "having", + "select", "from", "where", "group", "limit", "offset", "having", # functions "count", ] @@ -509,6 +509,7 @@ type nkFromItemPair, nkGroup, nkLimit, + nkOffset, nkHaving, nkOrder, nkJoin, @@ -1126,6 +1127,11 @@ proc parseSelect(p: var SqlParser): SqlNode = var l = newNode(nkLimit) l.add(parseExpr(p)) result.add(l) + if isKeyw(p, "offset"): + getTok(p) + var o = newNode(nkOffset) + o.add(parseExpr(p)) + result.add(o) proc parseStmt(p: var SqlParser; parent: SqlNode) = if isKeyw(p, "create"): @@ -1388,6 +1394,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkLimit: s.addKeyw("limit") s.addMulti(n) + of nkOffset: + s.addKeyw("offset") + s.addMulti(n) of nkHaving: s.addKeyw("having") s.addMulti(n) From 00ef27f4d18bcd5f56132d9ac5dfab840ef4e6fe Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 18 Jan 2023 21:37:26 +0100 Subject: [PATCH 1546/1654] minor parseopt.nim improvements (#21256) * minor parseopt.nim improvements * attempt to make CI happy Co-authored-by: Clay Sweetser --- lib/pure/json.nim | 1 + lib/pure/parseopt.nim | 72 ++++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 78d6bd5bd..e881fbff7 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -1360,6 +1360,7 @@ proc to*[T](node: JsonNode, t: typedesc[T]): T = doAssert data.list == @[1, 2, 3, 4] var jsonPath = "" + result = default(T) initFromJson(result, node, jsonPath) when false: diff --git a/lib/pure/parseopt.nim b/lib/pure/parseopt.nim index 9052067db..059f8e0f5 100644 --- a/lib/pure/parseopt.nim +++ b/lib/pure/parseopt.nim @@ -74,9 +74,9 @@ ## ## The `getopt iterator<#getopt.i,OptParser>`_, which is provided for ## convenience, can be used to iterate through all command line options as well. -## +## ## To set a default value for a variable assigned through `getopt` and accept arguments from the cmd line. -## Assign the default value to a variable before parsing. +## Assign the default value to a variable before parsing. ## Then set the variable to the new value while parsing. ## ## Here is an example: @@ -84,7 +84,7 @@ ## import std/parseopt ## ## var varName: string = "defaultValue" -## +## ## for kind, key, val in getopt(): ## case kind ## of cmdArgument: @@ -95,7 +95,7 @@ ## varName = val # do input sanitization in production systems ## of cmdEnd: ## discard -## +## ## `shortNoVal` and `longNoVal` ## ============================ ## @@ -213,36 +213,6 @@ proc parseWord(s: string, i: int, w: var string, add(w, s[result]) inc(result) -proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser - -proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, - longNoVal: seq[string] = @[]; - allowWhitespaceAfterColon = true): OptParser = - ## Initializes the command line parser. - ## - ## If `cmdline == ""`, the real command line as provided by the - ## `os` module is retrieved instead if it is available. If the - ## command line is not available, a `ValueError` will be raised. - ## - ## `shortNoVal` and `longNoVal` are used to specify which options - ## do not take values. See the `documentation about these - ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on - ## how this affects parsing. - ## - ## This does not provide a way of passing default values to arguments. - ## - ## See also: - ## * `getopt iterator<#getopt.i,OptParser>`_ - runnableExamples: - var p = initOptParser() - p = initOptParser("--left --debug:3 -l -r:2") - p = initOptParser("--left --debug:3 -l -r:2", - shortNoVal = {'l'}, longNoVal = @["left"]) - - initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) - proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, longNoVal: seq[string] = @[]; allowWhitespaceAfterColon = true): OptParser = @@ -287,6 +257,32 @@ proc initOptParser*(cmdline: seq[string], shortNoVal: set[char] = {}, result.key = "" result.val = "" +proc initOptParser*(cmdline = "", shortNoVal: set[char] = {}, + longNoVal: seq[string] = @[]; + allowWhitespaceAfterColon = true): OptParser = + ## Initializes the command line parser. + ## + ## If `cmdline == ""`, the real command line as provided by the + ## `os` module is retrieved instead if it is available. If the + ## command line is not available, a `ValueError` will be raised. + ## + ## `shortNoVal` and `longNoVal` are used to specify which options + ## do not take values. See the `documentation about these + ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on + ## how this affects parsing. + ## + ## This does not provide a way of passing default values to arguments. + ## + ## See also: + ## * `getopt iterator<#getopt.i,OptParser>`_ + runnableExamples: + var p = initOptParser() + p = initOptParser("--left --debug:3 -l -r:2") + p = initOptParser("--left --debug:3 -l -r:2", + shortNoVal = {'l'}, longNoVal = @["left"]) + + initOptParser(parseCmdLine(cmdline), shortNoVal, longNoVal, allowWhitespaceAfterColon) + proc handleShortOption(p: var OptParser; cmd: string) = var i = p.pos p.kind = cmdShortOption @@ -379,7 +375,7 @@ proc next*(p: var OptParser) {.rtl, extern: "npo$1".} = handleShortOption(p, p.cmds[p.idx]) else: p.kind = cmdArgument - p.key = p.cmds[p.idx] + p.key = p.cmds[p.idx] inc p.idx p.pos = 0 @@ -398,7 +394,6 @@ when declared(quoteShellCommand): ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.cmdLineRest == "foo.txt bar.txt" result = p.cmds[p.idx .. ^1].quoteShellCommand @@ -416,7 +411,6 @@ proc remainingArgs*(p: OptParser): seq[string] {.rtl, extern: "npo$1".} = ## p.next() ## if p.kind == cmdLongOption and p.key == "": # Look for "--" ## break - ## else: continue ## doAssert p.remainingArgs == @["foo.txt", "bar.txt"] result = @[] for i in p.idx..`_. ## - ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` ## with case switching, checking for `cmdEnd` is required. ## ## See also: @@ -475,7 +469,7 @@ iterator getopt*(cmdline: seq[string] = @[], ## parameters<#nimshortnoval-and-nimlongnoval>`_ for more information on ## how this affects parsing. ## - ## There is no need to check for `cmdEnd` while iterating. If using `getopt` + ## There is no need to check for `cmdEnd` while iterating. If using `getopt` ## with case switching, checking for `cmdEnd` is required. ## ## See also: From 7c6dcfd968867ec9b2fee5ca3ef8f251a6faa350 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Wed, 18 Jan 2023 22:38:48 +0200 Subject: [PATCH 1547/1654] Implicitly set noNimblePath when nimble.lock is present (#21266) Fixes https://github.com/nim-lang/nimble/issues/1004 --- compiler/nim.nim | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/compiler/nim.nim b/compiler/nim.nim index c2b70dda2..420579b97 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -41,6 +41,15 @@ when defined(profiler) or defined(memProfiler): {.hint: "Profiling support is turned on!".} import nimprof +proc nimbleLockExists(config: ConfigRef): bool = + const nimbleLock = "nimble.lock" + let pd = if not config.projectPath.isEmpty: config.projectPath else: AbsoluteDir(getCurrentDir()) + if optSkipParentConfigFiles notin config.globalOptions: + for dir in parentDirs(pd.string, fromRoot=true, inclusive=false): + if fileExists(dir / nimbleLock): + return true + return fileExists(pd.string / nimbleLock) + proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = var p = parseopt.initOptParser(cmd) var argsCount = 0 @@ -74,6 +83,11 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) = config.arguments.len > 0 and config.cmd notin {cmdTcc, cmdNimscript, cmdCrun}: rawMessage(config, errGenerated, errArgsNeedRunOption) + if config.nimbleLockExists: + # disable nimble path if nimble.lock is present. + # see https://github.com/nim-lang/nimble/issues/1004 + disableNimblePath(config) + proc getNimRunExe(conf: ConfigRef): string = # xxx consider defining `conf.getConfigVar("nimrun.exe")` to allow users to # customize the binary to run the command with, e.g. for custom `nodejs` or `wine`. From ac982d8a96d4602838d46abc61a8f5385f8c766d Mon Sep 17 00:00:00 2001 From: metagn Date: Wed, 18 Jan 2023 23:40:00 +0300 Subject: [PATCH 1548/1654] close #21257 (#21275) * close #21257 * fix generics --- compiler/semtypes.nim | 4 +++- tests/errmsgs/t21257.nim | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/errmsgs/t21257.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 844bc2070..aaffa1ea7 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1310,13 +1310,15 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, if hasDefault: def = a[^1] block determineType: + var defTyp = typ if genericParams != nil and genericParams.len > 0: + defTyp = nil def = semGenericStmt(c, def) if hasUnresolvedArgs(c, def): def.typ = makeTypeFromExpr(c, def.copyTree) break determineType - def = semExprWithType(c, def, {efDetermineType}) + def = semExprWithType(c, def, {efDetermineType}, defTyp) if def.referencesAnotherParam(getCurrOwner(c)): def.flags.incl nfDefaultRefsParam diff --git a/tests/errmsgs/t21257.nim b/tests/errmsgs/t21257.nim new file mode 100644 index 000000000..eecdb1dfe --- /dev/null +++ b/tests/errmsgs/t21257.nim @@ -0,0 +1,20 @@ +discard """ + action: compile + cmd: "nim check $file" +""" + +type AC_WINCTRL_Fields* = distinct uint8 + +type AC_STATUSA_WSTATE0* {.pure.} = enum + ABOVE = 0x0, + INSIDE = 0x1, + BELOW = 0x2, + +type AC_WINCTRL_WINTSEL0* {.pure.} = enum + ABOVE = 0x0, + INSIDE = 0x1, + BELOW = 0x2, + OUTSIDE = 0x3, + +proc write*(WINTSEL0: AC_WINCTRL_WINTSEL0 = ABOVE) = + discard From 9afb466d733bc16f5ac0fef7477c2ac919046eb2 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Thu, 19 Jan 2023 11:46:41 +0100 Subject: [PATCH 1549/1654] Force lambda lifting for getImplTransformed. Hacky. Fixes #19818 (#21031) --- compiler/lambdalifting.nim | 4 ++-- compiler/transf.nim | 6 +++--- compiler/vm.nim | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 9790eefd8..f6bc21c01 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -859,7 +859,7 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType; fn.typ.callConv = oldCC proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool; - idgen: IdGenerator): PNode = + idgen: IdGenerator, force: bool): PNode = # XXX backend == backendJs does not suffice! The compiletime stuff needs # the transformation even when compiling to JS ... @@ -868,7 +868,7 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool; if body.kind == nkEmpty or ( g.config.backend == backendJs and not isCompileTime) or - fn.skipGenericOwner.kind != skModule: + (fn.skipGenericOwner.kind != skModule and not force): # ignore forward declaration: result = body diff --git a/compiler/transf.nim b/compiler/transf.nim index 30d64282f..7277e6898 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -31,7 +31,7 @@ type TransformBodyFlag* = enum dontUseCache, useCache -proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, flag: TransformBodyFlag): PNode +proc transformBody*(g: ModuleGraph; idgen: IdGenerator, prc: PSym, flag: TransformBodyFlag, force = false): PNode import closureiters, lambdalifting @@ -1145,7 +1145,7 @@ template liftDefer(c, root) = if c.deferDetected: liftDeferAux(root) -proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flag: TransformBodyFlag): PNode = +proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flag: TransformBodyFlag, force = false): PNode = assert prc.kind in routineKinds if prc.transformedBody != nil: @@ -1155,7 +1155,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flag: Transfo else: prc.transformedBody = newNode(nkEmpty) # protects from recursion var c = openTransf(g, prc.getModule, "", idgen) - result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen) + result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen, force) result = processTransf(c, result, prc) liftDefer(c, result) result = liftLocalsIfRequested(prc, result, g.cache, g.config, c.idgen) diff --git a/compiler/vm.nim b/compiler/vm.nim index d87fa33ba..b8c23290e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1271,7 +1271,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let ast = a.sym.ast.shallowCopy for i in 0.. Date: Fri, 20 Jan 2023 12:48:12 +0800 Subject: [PATCH 1550/1654] switch to official chronos URL (#21283) ref https://github.com/status-im/nim-chronos/pull/348 --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index d9835ef23..90d12df4e 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -52,7 +52,7 @@ pkg "cascade" pkg "cello", url = "https://github.com/nim-lang/cello", useHead = true pkg "chroma" pkg "chronicles", "nim c -o:chr -r chronicles.nim", url = "https://github.com/nim-lang/nim-chronicles" -pkg "chronos", "nim c -r -d:release tests/testall", url = "https://github.com/nim-lang/nim-chronos" +pkg "chronos", "nim c -r -d:release tests/testall" pkg "cligen", "nim c --path:. -r cligen.nim" pkg "combparser", "nimble test --gc:orc" pkg "compactdict" From b82b5d44afbd860075f806f71a62d595f2376d6e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 20 Jan 2023 18:43:45 +0800 Subject: [PATCH 1551/1654] fixes #21278; `deques.shrink` off by one bug (#21284) fixes #21278; deques.shrink off ny one bug --- lib/pure/collections/deques.nim | 2 +- tests/stdlib/tdeques.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim index 7e9ed32f3..60e0fd351 100644 --- a/lib/pure/collections/deques.nim +++ b/lib/pure/collections/deques.nim @@ -440,7 +440,7 @@ proc shrink*[T](deq: var Deque[T], fromFirst = 0, fromLast = 0) = deq.head = (deq.head + 1) and deq.mask for i in 0 ..< fromLast: - destroy(deq.data[deq.tail]) + destroy(deq.data[(deq.tail - 1) and deq.mask]) deq.tail = (deq.tail - 1) and deq.mask dec deq.count, fromFirst + fromLast diff --git a/tests/stdlib/tdeques.nim b/tests/stdlib/tdeques.nim index 8a788d337..fcfc278a9 100644 --- a/tests/stdlib/tdeques.nim +++ b/tests/stdlib/tdeques.nim @@ -183,6 +183,12 @@ proc main() = clear(a) doAssert len(a) == 0 + block: # bug #21278 + var a = [10, 20, 30, 40].toDeque + + a.shrink(fromFirst = 0, fromLast = 1) + doAssert $a == "[10, 20, 30]" + static: main() main() From f5ec4fbde3d8fb5250e2944d397448219e9053bc Mon Sep 17 00:00:00 2001 From: ghais Date: Sat, 21 Jan 2023 05:07:40 -0500 Subject: [PATCH 1552/1654] Update the reference to parseSpec (#21289) The current reference pointed to the wrong location --- tests/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/readme.md b/tests/readme.md index fe59a7757..f638ddc10 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -9,7 +9,7 @@ top-level tests like `tests/tbar.nim`. Each test can contain a spec in a ``discard """ ... """`` block. -**Check out the [``parseSpec`` procedure](https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L158) in the ``specs`` module for a full and reliable reference** +**Check out the [``parseSpec`` procedure](https://github.com/nim-lang/Nim/blob/devel/testament/specs.nim#L315) in the ``specs`` module for a full and reliable reference** ## action From 36e489e69b08ac6a34dfd5c22abc3873c7d86838 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 24 Jan 2023 20:33:13 +0800 Subject: [PATCH 1553/1654] add `md2html`, `md2tex` to cmd helpers (#21295) --- doc/advopt.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/advopt.txt b/doc/advopt.txt index efff428e8..122a7c6ea 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -4,8 +4,11 @@ Advanced commands: //compileToOC, objc compile project to Objective C code //js compile project to Javascript //e run a Nimscript file + //md2html convert a Markdown file to HTML + use `--docCmd:skip` to skip compiling snippets //rst2html convert a reStructuredText file to HTML use `--docCmd:skip` to skip compiling snippets + //md2tex convert a Markdown file to LaTeX //rst2tex convert a reStructuredText file to LaTeX //doc2tex extract the documentation to a LaTeX file //jsondoc extract the documentation to a json file From 5e7f7109e19b33bc142aa7b06e9044420232a578 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 24 Jan 2023 20:35:19 +0800 Subject: [PATCH 1554/1654] fixes #21290; deindent if the last same level is a text node (#21293) * fixes #21290; deindent if the last same level is a text node * add one more test --- lib/pure/xmltree.nim | 37 +++++++++++++++++++++---------------- tests/stdlib/txmltree.nim | 18 +++++++++++++++++- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index 7a2cebe87..c3923ca50 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -723,20 +723,8 @@ proc addIndent(result: var string, indent: int, addNewLines: bool) = for i in 1 .. indent: result.add(' ') -proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, - addNewLines = true) = - ## Adds the textual representation of `n` to string `result`. - runnableExamples: - var - a = newElement("firstTag") - b = newText("my text") - c = newComment("my comment") - s = "" - s.add(c) - s.add(a) - s.add(b) - assert s == "my text" - +proc addImpl(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true, lastNodeIsText = false) = proc noWhitespace(n: XmlNode): bool = for i in 0 ..< n.len: if n[i].kind in {xnText, xnEntity}: return true @@ -756,7 +744,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, case n.k of xnElement: - if indent > 0: + if indent > 0 and not lastNodeIsText: result.addIndent(indent, addNewLines) let @@ -785,8 +773,10 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, else: indent+indWidth result.add('>') + var lastNodeIsText = false for i in 0 ..< n.len: - result.add(n[i], indentNext, indWidth, addNewLines) + result.addImpl(n[i], indentNext, indWidth, addNewLines, lastNodeIsText) + lastNodeIsText = n[i].kind == xnText if not n.noWhitespace(): result.addIndent(indent, addNewLines) @@ -811,6 +801,21 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, result.add(n.fText) result.add(';') +proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2, + addNewLines = true) {.inline.} = + ## Adds the textual representation of `n` to string `result`. + runnableExamples: + var + a = newElement("firstTag") + b = newText("my text") + c = newComment("my comment") + s = "" + s.add(c) + s.add(a) + s.add(b) + assert s == "my text" + result.addImpl(n, indent, indWidth, addNewLines) + proc `$`*(n: XmlNode): string = ## Converts `n` into its string representation. ## diff --git a/tests/stdlib/txmltree.nim b/tests/stdlib/txmltree.nim index f6b7c62e8..4362f5ec3 100644 --- a/tests/stdlib/txmltree.nim +++ b/tests/stdlib/txmltree.nim @@ -1,4 +1,4 @@ -import std/[xmltree, assertions] +import std/[xmltree, assertions, xmlparser] block: @@ -83,3 +83,19 @@ block: x.add newElement("sonTag") x.add newEntity("my entity") doAssert $x == "my text&my entity;" + +block: # bug #21290 + let x = newXmlTree("foo",[ + newXmlTree("bar",[ + newText("Hola"), + newXmlTree("qux",[ + newXmlTree("plugh",[]) + ]) + ]) + ]) + + let s = $x + doAssert $parseXml(s) == s + doAssert s == """ + Hola +""" From d54a7f078d29445bc79ec5f117a2ceb73761b4c4 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 24 Jan 2023 22:37:22 +0800 Subject: [PATCH 1555/1654] fixes #21261; always checking `nimTestErrorFlag` in the main module (#21288) * fixes #21261; always checking nimTestErrorFlag in the main module * add a test --- compiler/cgen.nim | 3 +++ tests/exception/m21261.nim | 1 + tests/exception/t21261.nim | 9 +++++++++ 3 files changed, 13 insertions(+) create mode 100644 tests/exception/m21261.nim create mode 100644 tests/exception/t21261.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f6457f1e0..5bf9400c5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1801,6 +1801,9 @@ proc genInitCode(m: BModule) = if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: prc.add(deinitFrame(m.initProc)) + elif sfMainModule in m.module.flags and m.config.exc == excGoto: + if getCompilerProc(m.g.graph, "nimTestErrorFlag") != nil: + m.appcg(prc, "\t#nimTestErrorFlag();$n", []) prc.addf("}$N", []) diff --git a/tests/exception/m21261.nim b/tests/exception/m21261.nim new file mode 100644 index 000000000..11b12fb5b --- /dev/null +++ b/tests/exception/m21261.nim @@ -0,0 +1 @@ +raise (ref Exception)(msg: "something") \ No newline at end of file diff --git a/tests/exception/t21261.nim b/tests/exception/t21261.nim new file mode 100644 index 000000000..84817d854 --- /dev/null +++ b/tests/exception/t21261.nim @@ -0,0 +1,9 @@ +discard """ + exitcode: 1 + outputsub: ''' +m21261.nim(1) m21261 +Error: unhandled exception: something [Exception] +''' +""" + +import m21261 \ No newline at end of file From c4d3d650baa88820dadeafa81ca75b6e495084f2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 26 Jan 2023 03:56:19 +0800 Subject: [PATCH 1556/1654] fixes #21273; fixes an io.readLine off by one bug [backport 1.0] (#21276) fixes #21273; io.readLine off by one --- lib/std/syncio.nim | 7 ++++--- tests/stdlib/tio.nim | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 46f113450..83b2d6d13 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -484,11 +484,12 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], if last > 0 and line[last-1] == '\c': line.setLen(last-1) return last > 1 or fgetsSuccess - # We have to distinguish between two possible cases: + elif last > 0 and line[last-1] == '\0': + # We have to distinguish among three possible cases: # \0\l\0 => line ending in a null character. # \0\l\l => last line without newline, null was put there by fgets. - elif last > 0 and line[last-1] == '\0': - if last < pos + sp - 1 and line[last+1] != '\0': + # \0\l => last line without newline, null was put there by fgets. + if last >= pos + sp - 1 or line[last+1] != '\0': # bug #21273 dec last line.setLen(last) return last > 0 or fgetsSuccess diff --git a/tests/stdlib/tio.nim b/tests/stdlib/tio.nim index 0e20d6495..ca411379c 100644 --- a/tests/stdlib/tio.nim +++ b/tests/stdlib/tio.nim @@ -36,3 +36,21 @@ block: # readChars break doAssert n2s == @[2,2,2,1,0] doAssert s2 == s + + +import std/strutils + +block: # bug #21273 + let FILE = buildDir / "D20220119T134305.txt" + + let hex = "313632313920313632343720313632353920313632363020313632393020323035363520323037323120323131353020323239393820323331303520323332313020323332343820323332363820" + + + writeFile FILE, parseHexStr(hex) + + doAssert readFile(FILE).toHex == hex + + let f = open(FILE) + var s = newString(80) + while f.readLine(s): + doAssert s.toHex == hex From af0f3639ec6fabc00bf8357615b62641ee0b4508 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 26 Jan 2023 03:58:52 +0800 Subject: [PATCH 1557/1654] remove dead code in VM (#21296) remove deadcode in VM --- compiler/ast.nim | 2 +- compiler/vm.nim | 6 ------ compiler/vmdef.nim | 2 +- compiler/vmgen.nim | 3 --- 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index c85a94860..aa0ca98c9 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -706,7 +706,7 @@ type mNctPut, mNctLen, mNctGet, mNctHasNext, mNctNext, mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, - mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, + mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetStrVal, mNLineInfo, mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNSizeOf, mNBindSym, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, diff --git a/compiler/vm.nim b/compiler/vm.nim index b8c23290e..c6ca228ff 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2071,12 +2071,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = dest.ident = regs[rb].node.ident else: stackTrace(c, tos, pc, errFieldXNotFound & "ident") - of opcNSetType: - decodeB(rkNode) - let b = regs[rb].node - internalAssert c.config, b.kind == nkSym and b.sym.kind == skType - internalAssert c.config, regs[ra].node != nil - regs[ra].node.typ = b.sym.typ of opcNSetStrVal: decodeB(rkNode) var dest = regs[ra].node diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 114a49557..0af15e5c9 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -127,7 +127,7 @@ type opcNGetSize, opcNSetIntVal, - opcNSetFloatVal, opcNSetSymbol, opcNSetIdent, opcNSetType, opcNSetStrVal, + opcNSetFloatVal, opcNSetSymbol, opcNSetIdent, opcNSetStrVal, opcNNewNimNode, opcNCopyNimNode, opcNCopyNimTree, opcNDel, opcGenSym, opcNccValue, opcNccInc, opcNcsAdd, opcNcsIncl, opcNcsLen, opcNcsAt, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 6a2ab1d11..ba3a05481 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1313,9 +1313,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = of mNSetIdent: unused(c, n, dest) genBinaryStmt(c, n, opcNSetIdent) - of mNSetType: - unused(c, n, dest) - genBinaryStmt(c, n, opcNSetType) of mNSetStrVal: unused(c, n, dest) genBinaryStmt(c, n, opcNSetStrVal) From 23bd812b0c531dbb6a9854a81030468bbae29cd8 Mon Sep 17 00:00:00 2001 From: Heechul Ryu Date: Thu, 26 Jan 2023 03:43:55 -0800 Subject: [PATCH 1558/1654] Update nims.md to use `nim e` instead of `nim` in advanced shebang (#21301) Update nims.md - suggest a better shebang that works with any filename - related to https://github.com/nim-lang/Nim/issues/17190 --- doc/nims.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/nims.md b/doc/nims.md index a69ea1789..2409bb577 100644 --- a/doc/nims.md +++ b/doc/nims.md @@ -211,7 +211,7 @@ ends with ``.nims``: echo "hello world" ``` -Use `#!/usr/bin/env -S nim --hints:off` to disable hints. +Use `#!/usr/bin/env -S nim e --hints:off` to disable hints and relax the file extension constraint. Benefits From fc068ee06d058b35d2d5890cf1fa3c356e549d1f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 27 Jan 2023 06:03:59 +0800 Subject: [PATCH 1559/1654] fixes #18134; registers `formatBiggestFloat` in VM (#21299) fixes #18134; registers formatBiggestFloat in vmops strformat supports float format in VM --- compiler/vmops.nim | 8 ++- lib/pure/strutils.nim | 107 ++++++++++++++++++------------------ tests/stdlib/tstrformat.nim | 19 ++++--- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index bb943b8a0..2b5b2c938 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -36,7 +36,9 @@ from std/osproc import nil when defined(nimPreviewSlimSystem): import std/syncio else: - from std/formatfloat import addFloatRoundtrip, addFloatSprintf + from std/formatfloat import addFloatRoundtrip, addFloatSprintf + +from std/strutils import formatBiggestFloat, FloatFormatMode # There are some useful procs in vmconv. import vmconv, vmmarshal @@ -374,6 +376,10 @@ proc registerAdditionalOps*(c: PCtx) = let x = a.getFloat(1) addFloatSprintf(p.strVal, x) + registerCallback c, "stdlib.strutils.formatBiggestFloat", proc(a: VmArgs) = + setResult(a, formatBiggestFloat(a.getFloat(0), FloatFormatMode(a.getInt(1)), + a.getInt(2), chr(a.getInt(3)))) + wrapIterator("stdlib.envvars.envPairsImplSeq"): envPairs() registerCallback c, "stdlib.marshal.toVM", proc(a: VmArgs) = diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index dbdd0d6a1..c458d605e 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2413,60 +2413,63 @@ func formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault, doAssert x.formatBiggestFloat() == "123.4560000000000" doAssert x.formatBiggestFloat(ffDecimal, 4) == "123.4560" doAssert x.formatBiggestFloat(ffScientific, 2) == "1.23e+02" - when defined(js): - var precision = precision - if precision == -1: - # use the same default precision as c_sprintf - precision = 6 - var res: cstring - case format - of ffDefault: - {.emit: "`res` = `f`.toString();".} - of ffDecimal: - {.emit: "`res` = `f`.toFixed(`precision`);".} - of ffScientific: - {.emit: "`res` = `f`.toExponential(`precision`);".} - result = $res - if 1.0 / f == -Inf: - # JavaScript removes the "-" from negative Zero, add it back here - result = "-" & $res - for i in 0 ..< result.len: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if result[i] in {'.', ','}: result[i] = decimalSep + when nimvm: + discard "implemented in the vmops" else: - const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] - var - frmtstr {.noinit.}: array[0..5, char] - buf {.noinit.}: array[0..2500, char] - L: cint - frmtstr[0] = '%' - if precision >= 0: - frmtstr[1] = '#' - frmtstr[2] = '.' - frmtstr[3] = '*' - frmtstr[4] = floatFormatToChar[format] - frmtstr[5] = '\0' - L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), precision, f) + when defined(js): + var precision = precision + if precision == -1: + # use the same default precision as c_sprintf + precision = 6 + var res: cstring + case format + of ffDefault: + {.emit: "`res` = `f`.toString();".} + of ffDecimal: + {.emit: "`res` = `f`.toFixed(`precision`);".} + of ffScientific: + {.emit: "`res` = `f`.toExponential(`precision`);".} + result = $res + if 1.0 / f == -Inf: + # JavaScript removes the "-" from negative Zero, add it back here + result = "-" & $res + for i in 0 ..< result.len: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if result[i] in {'.', ','}: result[i] = decimalSep else: - frmtstr[1] = floatFormatToChar[format] - frmtstr[2] = '\0' - L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), f) - result = newString(L) - for i in 0 ..< L: - # Depending on the locale either dot or comma is produced, - # but nothing else is possible: - if buf[i] in {'.', ','}: result[i] = decimalSep - else: result[i] = buf[i] - when defined(windows): - # VS pre 2015 violates the C standard: "The exponent always contains at - # least two digits, and only as many more digits as necessary to - # represent the exponent." [C11 §7.21.6.1] - # The following post-processing fixes this behavior. - if result.len > 4 and result[^4] == '+' and result[^3] == '0': - result[^3] = result[^2] - result[^2] = result[^1] - result.setLen(result.len - 1) + const floatFormatToChar: array[FloatFormatMode, char] = ['g', 'f', 'e'] + var + frmtstr {.noinit.}: array[0..5, char] + buf {.noinit.}: array[0..2500, char] + L: cint + frmtstr[0] = '%' + if precision >= 0: + frmtstr[1] = '#' + frmtstr[2] = '.' + frmtstr[3] = '*' + frmtstr[4] = floatFormatToChar[format] + frmtstr[5] = '\0' + L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), precision, f) + else: + frmtstr[1] = floatFormatToChar[format] + frmtstr[2] = '\0' + L = c_sprintf(cast[cstring](addr buf), cast[cstring](addr frmtstr), f) + result = newString(L) + for i in 0 ..< L: + # Depending on the locale either dot or comma is produced, + # but nothing else is possible: + if buf[i] in {'.', ','}: result[i] = decimalSep + else: result[i] = buf[i] + when defined(windows): + # VS pre 2015 violates the C standard: "The exponent always contains at + # least two digits, and only as many more digits as necessary to + # represent the exponent." [C11 §7.21.6.1] + # The following post-processing fixes this behavior. + if result.len > 4 and result[^4] == '+' and result[^3] == '0': + result[^3] = result[^2] + result[^2] = result[^1] + result.setLen(result.len - 1) func formatFloat*(f: float, format: FloatFormatMode = ffDefault, precision: range[-1..32] = 16; decimalSep = '.'): string {. diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim index b44a11e68..d332e8375 100644 --- a/tests/stdlib/tstrformat.nim +++ b/tests/stdlib/tstrformat.nim @@ -475,15 +475,17 @@ proc main() = # Note: times.format adheres to the format protocol. Test that this # works: + when nimvm: + discard + else: + var dt = dateTime(2000, mJan, 01, 00, 00, 00) + check &"{dt:yyyy-MM-dd}", "2000-01-01" - var dt = initDateTime(01, mJan, 2000, 00, 00, 00) - check &"{dt:yyyy-MM-dd}", "2000-01-01" + var tm = fromUnix(0) + discard &"{tm}" - var tm = fromUnix(0) - discard &"{tm}" - - var noww = now() - check &"{noww}", $noww + var noww = now() + check &"{noww}", $noww # Unicode string tests check &"""{"αβγ"}""", "αβγ" @@ -558,5 +560,6 @@ proc main() = doAssert &"""{(if true: "'" & "'" & ')' else: "")}""" == "'')" doAssert &"{(if true: \"\'\" & \"'\" & ')' else: \"\")}" == "'')" doAssert fmt"""{(if true: "'" & ')' else: "")}""" == "')" -# xxx static: main() + +static: main() main() From 4647c7b59634e0d8bd24d815337e7a7a1f0830bd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 27 Jan 2023 06:49:04 +0800 Subject: [PATCH 1560/1654] fixes #14409; fixes #10674 VM callbacks switch to table-index seqs (#21297) * fixes #14409; fixes#10674 VM callbacks switch to table-index seqs * fixes package name * reduce runtime cost --- compiler/vm.nim | 4 ++-- compiler/vmdef.nim | 18 ++++++++++++---- compiler/vmgen.nim | 34 +++++++++++++++++------------- compiler/vmops.nim | 2 +- tests/compilerapi/tcompilerapi.nim | 2 +- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index c6ca228ff..4f5f8485b 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1351,7 +1351,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let prc = if not isClosure: bb.sym else: bb[0].sym if prc.offset < -1: # it's a callback: - c.callbacks[-prc.offset-2].value( + c.callbacks[-prc.offset-2]( VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]), currentException: c.currentExceptionA, currentLineInfo: c.debug[pc]) @@ -1674,7 +1674,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = rb = instr.regB rc = instr.regC idx = int(regs[rb+rc-1].intVal) - callback = c.callbacks[idx].value + callback = c.callbacks[idx] args = VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]), currentException: c.currentExceptionA, currentLineInfo: c.debug[pc]) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 0af15e5c9..f369908ba 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -10,7 +10,7 @@ ## This module contains the type definitions for the new evaluation engine. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import tables +import std/[tables, strutils] import ast, idents, options, modulegraphs, lineinfos @@ -259,7 +259,8 @@ type traceActive*: bool loopIterations*: int comesFromHeuristic*: TLineInfo # Heuristic for better macro stack traces - callbacks*: seq[tuple[key: string, value: VmCallback]] + callbacks*: seq[VmCallback] + callbackIndex*: Table[string, int] errorFlag*: string cache*: IdentCache config*: ConfigRef @@ -292,7 +293,7 @@ proc newCtx*(module: PSym; cache: IdentCache; g: ModuleGraph; idgen: IdGenerator PCtx(code: @[], debug: @[], globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[], prc: PProc(blocks: @[]), module: module, loopIterations: g.config.maxLoopIterationsVM, - comesFromHeuristic: unknownLineInfo, callbacks: @[], errorFlag: "", + comesFromHeuristic: unknownLineInfo, callbacks: @[], callbackIndex: initTable[string, int](), errorFlag: "", cache: cache, config: g.config, graph: g, idgen: idgen) proc refresh*(c: PCtx, module: PSym; idgen: IdGenerator) = @@ -301,9 +302,18 @@ proc refresh*(c: PCtx, module: PSym; idgen: IdGenerator) = c.loopIterations = c.config.maxLoopIterationsVM c.idgen = idgen +proc reverseName(s: string): string = + result = newStringOfCap(s.len) + let y = s.split('.') + for i in 1..y.len: + result.add y[^i] + if i != y.len: + result.add '.' + proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.discardable.} = result = c.callbacks.len - c.callbacks.add((name, callback)) + c.callbacks.add(callback) + c.callbackIndex[reverseName(name)] = result const firstABxInstr* = opcTJmp diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index ba3a05481..bca961a63 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -2011,25 +2011,29 @@ proc genTupleConstr(c: PCtx, n: PNode, dest: var TDest) = proc genProc*(c: PCtx; s: PSym): int -proc matches(s: PSym; x: string): bool = - let y = x.split('.') +proc toKey(s: PSym): string = var s = s - for i in 1..y.len: - if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"): - return false - s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner - while s != nil and s.kind == skPackage and s.owner != nil: s = s.owner - result = true + while s != nil: + result.add s.name.s + if s.owner != nil: + if sfFromGeneric in s.flags: + s = s.owner.owner + else: + s = s.owner + result.add "." + else: + break proc procIsCallback(c: PCtx; s: PSym): bool = if s.offset < -1: return true - var i = -2 - for key, value in items(c.callbacks): - if s.matches(key): - doAssert s.offset == -1 - s.offset = i - return true - dec i + let key = toKey(s) + if c.callbackIndex.contains(key): + let index = c.callbackIndex[key] + doAssert s.offset == -1 + s.offset = -2 - index + result = true + else: + result = false proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = when defined(nimCompilerStacktraceHints): diff --git a/compiler/vmops.nim b/compiler/vmops.nim index 2b5b2c938..f1a2e1e69 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -256,7 +256,7 @@ proc registerAdditionalOps*(c: PCtx) = wrap2si(readLines, ioop) systemop getCurrentExceptionMsg systemop getCurrentException - registerCallback c, "stdlib.*.staticWalkDir", proc (a: VmArgs) {.nimcall.} = + registerCallback c, "stdlib.osdirs.staticWalkDir", proc (a: VmArgs) {.nimcall.} = setResult(a, staticWalkDirImpl(getString(a, 0), getBool(a, 1))) registerCallback c, "stdlib.compilesettings.querySetting", proc (a: VmArgs) = setResult(a, querySettingImpl(c.config, getInt(a, 0))) diff --git a/tests/compilerapi/tcompilerapi.nim b/tests/compilerapi/tcompilerapi.nim index ab995cb6d..828dcd0d9 100644 --- a/tests/compilerapi/tcompilerapi.nim +++ b/tests/compilerapi/tcompilerapi.nim @@ -23,7 +23,7 @@ proc initInterpreter(script: string): Interpreter = proc main() = let i = initInterpreter("myscript.nim") - i.implementRoutine("*", "exposed", "addFloats", proc (a: VmArgs) = + i.implementRoutine("nim", "exposed", "addFloats", proc (a: VmArgs) = setResult(a, getFloat(a, 0) + getFloat(a, 1) + getFloat(a, 2)) ) i.evalScript() From 7031ea65cd220360b8e9f566fd28f01bc0bf53c4 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Fri, 27 Jan 2023 08:11:30 +0200 Subject: [PATCH 1561/1654] Implemented basic macro expand functionality (#20579) * Implemented level based macro expand functionality - it can handle single macro call or expand whole function/proc/etc and it - In addition, I have altered the parser to provide the endInfo for the node. The usefulness of the `endInfo` is not limited to the `expandMacro` functionality but also it is useful for `ideOutline` functionality and I have altered the ideOutline functionality to use `endInfo`. Note `endInfo` most of the time is lost during the AST transformation thus in `nimsuggest.nim` I am using freshly parsed tree to get the location information. * Make sure we stop expanding correctly * Test CI * Fix tv3_outline.nim --- compiler/ast.nim | 8 ++ compiler/lexer.nim | 6 + compiler/options.nim | 14 ++- compiler/parser.nim | 70 ++++++++++- compiler/semexprs.nim | 30 ++++- compiler/sempass2.nim | 3 + compiler/semstmts.nim | 2 +- compiler/suggest.nim | 12 +- compiler/vm.nim | 3 + nimsuggest/nimsuggest.nim | 193 ++++++++++++++++++++++++------- nimsuggest/tester.nim | 15 ++- nimsuggest/tests/tv3.nim | 4 - nimsuggest/tests/tv3_outline.nim | 45 +++++++ 13 files changed, 345 insertions(+), 60 deletions(-) create mode 100644 nimsuggest/tests/tv3_outline.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index aa0ca98c9..aa70560ed 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -796,6 +796,8 @@ type ident*: PIdent else: sons*: TNodeSeq + when defined(nimsuggest): + endInfo*: TLineInfo TStrTable* = object # a table[PIdent] of PSym counter*: int @@ -892,6 +894,8 @@ type typ*: PType name*: PIdent info*: TLineInfo + when defined(nimsuggest): + endInfo*: TLineInfo owner*: PSym flags*: TSymFlags ast*: PNode # syntax tree of proc, iterator, etc.: @@ -1690,6 +1694,8 @@ proc copyNode*(src: PNode): PNode = of nkIdent: result.ident = src.ident of nkStrLit..nkTripleStrLit: result.strVal = src.strVal else: discard + when defined(nimsuggest): + result.endInfo = src.endInfo template transitionNodeKindCommon(k: TNodeKind) = let obj {.inject.} = n[] @@ -1742,6 +1748,8 @@ template copyNodeImpl(dst, src, processSonsStmt) = if src == nil: return dst = newNode(src.kind) dst.info = src.info + when defined(nimsuggest): + result.endInfo = src.endInfo dst.typ = src.typ dst.flags = src.flags * PersistentNodeFlags dst.comment = src.comment diff --git a/compiler/lexer.nim b/compiler/lexer.nim index ec27add8a..a62d40e54 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -127,6 +127,8 @@ type cache*: IdentCache when defined(nimsuggest): previousToken: TLineInfo + tokenEnd*: TLineInfo + previousTokenEnd*: TLineInfo config*: ConfigRef proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} = @@ -1224,6 +1226,10 @@ proc skip(L: var Lexer, tok: var Token) = proc rawGetTok*(L: var Lexer, tok: var Token) = template atTokenEnd() {.dirty.} = when defined(nimsuggest): + L.previousTokenEnd.line = L.tokenEnd.line + L.previousTokenEnd.col = L.tokenEnd.col + L.tokenEnd.line = tok.line.uint16 + L.tokenEnd.col = getColNumber(L, L.bufpos).int16 # we attach the cursor to the last *strong* token if tok.tokType notin weakTokens: L.previousToken.line = tok.line.uint16 diff --git a/compiler/options.nim b/compiler/options.nim index fc970e420..8c477f0a5 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -195,7 +195,7 @@ type IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod, ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols, - ideRecompile, ideChanged, ideType, ideDeclaration + ideRecompile, ideChanged, ideType, ideDeclaration, ideExpand Feature* = enum ## experimental features; DO NOT RENAME THESE! dotOperators, @@ -278,6 +278,9 @@ type scope*, localUsages*, globalUsages*: int # more usages is better tokenLen*: int version*: int + endLine*: uint16 + endCol*: int + Suggestions* = seq[Suggest] ProfileInfo* = object @@ -408,6 +411,11 @@ type nimMainPrefix*: string vmProfileData*: ProfileData + expandProgress*: bool + expandLevels*: int + expandNodeResult*: string + expandPosition*: TLineInfo + proc parseNimVersion*(a: string): NimVer = # could be moved somewhere reusable if a.len > 0: @@ -996,6 +1004,9 @@ proc isDynlibOverride*(conf: ConfigRef; lib: string): bool = result = optDynlibOverrideAll in conf.globalOptions or conf.dllOverrides.hasKey(lib.canonDynlibName) +proc expandDone*(conf: ConfigRef): bool = + result = conf.ideCmd == ideExpand and conf.expandLevels == 0 and conf.expandProgress + proc parseIdeCmd*(s: string): IdeCmd = case s: of "sug": ideSug @@ -1035,6 +1046,7 @@ proc `$`*(c: IdeCmd): string = of ideProject: "project" of ideGlobalSymbols: "globalSymbols" of ideDeclaration: "declaration" + of ideExpand: "expand" of ideRecompile: "recompile" of ideChanged: "changed" of ideType: "type" diff --git a/compiler/parser.nim b/compiler/parser.nim index 0c8edd2a8..14882d415 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -354,6 +354,12 @@ proc colcom(p: var Parser, n: PNode) = const tkBuiltInMagics = {tkType, tkStatic, tkAddr} +template setEndInfo() = + when defined(nimsuggest): + result.endInfo = TLineInfo(fileIndex: p.lex.fileIdx, + line: p.lex.previousTokenEnd.line, + col: p.lex.previousTokenEnd.col) + proc parseSymbol(p: var Parser, mode = smNormal): PNode = #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' #| | IDENT | KEYW @@ -406,6 +412,7 @@ proc parseSymbol(p: var Parser, mode = smNormal): PNode = # if it is a keyword: #if not isKeyword(p.tok.tokType): getTok(p) result = p.emptyNode + setEndInfo() proc equals(p: var Parser, a: PNode): PNode = if p.tok.tokType == tkEquals: @@ -577,6 +584,7 @@ proc parseCast(p: var Parser): PNode = result.add(exprColonEqExpr(p)) optPar(p) eat(p, tkParRi) + setEndInfo() proc setBaseFlags(n: PNode, base: NumericalBase) = case base @@ -599,6 +607,7 @@ proc parseGStrLit(p: var Parser, a: PNode): PNode = getTok(p) else: result = a + setEndInfo() proc complexOrSimpleStmt(p: var Parser): PNode proc simpleExpr(p: var Parser, mode = pmNormal): PNode @@ -703,6 +712,7 @@ proc parsePar(p: var Parser): PNode = skipComment(p, a) optPar(p) eat(p, tkParRi) + setEndInfo() proc identOrLiteral(p: var Parser, mode: PrimaryMode): PNode = #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT @@ -941,6 +951,7 @@ proc parseOperators(p: var Parser, headNode: PNode, a.add(b) result = a opPrec = getPrecedence(p.tok) + setEndInfo() proc simpleExprAux(p: var Parser, limit: int, mode: PrimaryMode): PNode = var mode = mode @@ -990,6 +1001,7 @@ proc parsePragma(p: var Parser): PNode = when defined(nimpretty): dec p.em.doIndentMore dec p.em.keepIndents + setEndInfo() proc identVis(p: var Parser; allowDot=false): PNode = #| identVis = symbol OPR? # postfix position @@ -1058,6 +1070,7 @@ proc parseIdentColonEquals(p: var Parser, flags: DeclaredIdentFlags): PNode = result.add(parseExpr(p)) else: result.add(newNodeP(nkEmpty, p)) + setEndInfo() proc parseTuple(p: var Parser, indentAllowed = false): PNode = #| tupleTypeBracket = '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' @@ -1102,6 +1115,7 @@ proc parseTuple(p: var Parser, indentAllowed = false): PNode = parMessage(p, errGenerated, "the syntax for tuple types is 'tuple[...]', not 'tuple(...)'") else: result = newNodeP(nkTupleClassTy, p) + setEndInfo() proc parseParamList(p: var Parser, retColon = true): PNode = #| paramList = '(' declColonEquals ^* (comma/semicolon) ')' @@ -1150,6 +1164,7 @@ proc parseParamList(p: var Parser, retColon = true): PNode = when defined(nimpretty): dec p.em.doIndentMore dec p.em.keepIndents + setEndInfo() proc optPragmas(p: var Parser): PNode = if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)): @@ -1170,6 +1185,7 @@ proc parseDoBlock(p: var Parser; info: TLineInfo): PNode = result = newProcNode(nkDo, info, body = result, params = params, name = p.emptyNode, pattern = p.emptyNode, genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) + setEndInfo() proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = #| routineExpr = ('proc' | 'func' | 'iterator') paramListColon pragma? ('=' COMMENT? stmt)? @@ -1192,6 +1208,7 @@ proc parseProcExpr(p: var Parser; isExpr: bool; kind: TNodeKind): PNode = if kind == nkFuncDef: parMessage(p, "func keyword is not allowed in type descriptions, use proc with {.noSideEffect.} pragma instead") result.add(pragmas) + setEndInfo() proc isExprStart(p: Parser): bool = case p.tok.tokType @@ -1211,6 +1228,7 @@ proc parseSymbolList(p: var Parser, result: PNode) = if p.tok.tokType != tkComma: break getTok(p) optInd(p, s) + setEndInfo() proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, mode: PrimaryMode): PNode = @@ -1239,6 +1257,7 @@ proc parseTypeDescKAux(p: var Parser, kind: TNodeKind, parseSymbolList(p, list) if mode == pmTypeDef and not isTypedef: result = parseOperators(p, result, -1, mode) + setEndInfo() proc parseVarTuple(p: var Parser): PNode @@ -1264,6 +1283,7 @@ proc parseFor(p: var Parser): PNode = result.add(parseExpr(p)) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() template nimprettyDontTouch(body) = when defined(nimpretty): @@ -1302,6 +1322,7 @@ proc parseExpr(p: var Parser): PNode = nimprettyDontTouch: result = parseTry(p, isExpr=true) else: result = simpleExpr(p) + setEndInfo() proc parseEnum(p: var Parser): PNode proc parseObject(p: var Parser): PNode @@ -1411,6 +1432,7 @@ proc parseTypeDesc(p: var Parser, fullExpr = false): PNode = else: result = simpleExpr(p, pmTypeDesc) result = binaryNot(p, result) + setEndInfo() proc parseTypeDefValue(p: var Parser): PNode = #| typeDefValue = ((tupleDecl | enumDecl | objectDecl | conceptDecl | @@ -1441,6 +1463,7 @@ proc parseTypeDefValue(p: var Parser): PNode = result.add(commandParam(p, isFirstParam, pmTypeDef)) result = postExprBlocks(p, result) result = binaryNot(p, result) + setEndInfo() proc makeCall(n: PNode): PNode = ## Creates a call if the given node isn't already a call. @@ -1561,6 +1584,7 @@ proc parseExprStmt(p: var Parser): PNode = else: result = a result = postExprBlocks(p, result) + setEndInfo() proc parseModuleName(p: var Parser, kind: TNodeKind): PNode = result = parseExpr(p) @@ -1572,6 +1596,7 @@ proc parseModuleName(p: var Parser, kind: TNodeKind): PNode = getTok(p) result.add(a) result.add(parseExpr(p)) + setEndInfo() proc parseImport(p: var Parser, kind: TNodeKind): PNode = #| importStmt = 'import' optInd expr @@ -1600,6 +1625,7 @@ proc parseImport(p: var Parser, kind: TNodeKind): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseIncludeStmt(p: var Parser): PNode = #| includeStmt = 'include' optInd expr ^+ comma @@ -1616,6 +1642,7 @@ proc parseIncludeStmt(p: var Parser): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseFromStmt(p: var Parser): PNode = #| fromStmt = 'from' expr 'import' optInd expr (comma expr)* @@ -1636,6 +1663,7 @@ proc parseFromStmt(p: var Parser): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = #| returnStmt = 'return' optInd expr? @@ -1657,6 +1685,7 @@ proc parseReturnOrRaise(p: var Parser, kind: TNodeKind): PNode = var e = parseExpr(p) e = postExprBlocks(p, e) result.add(e) + setEndInfo() proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = #| condStmt = expr colcom stmt COMMENT? @@ -1681,6 +1710,7 @@ proc parseIfOrWhen(p: var Parser, kind: TNodeKind): PNode = colcom(p, branch) branch.add(parseStmt(p)) result.add(branch) + setEndInfo() proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = #| condExpr = expr colcom expr optInd @@ -1705,6 +1735,7 @@ proc parseIfOrWhenExpr(p: var Parser, kind: TNodeKind): PNode = colcom(p, branch) branch.add(parseStmt(p)) result.add(branch) + setEndInfo() proc parseWhile(p: var Parser): PNode = #| whileStmt = 'while' expr colcom stmt @@ -1714,6 +1745,7 @@ proc parseWhile(p: var Parser): PNode = result.add(parseExpr(p)) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseCase(p: var Parser): PNode = #| ofBranch = 'of' exprList colcom stmt @@ -1761,6 +1793,7 @@ proc parseCase(p: var Parser): PNode = if wasIndented: p.currInd = oldInd + setEndInfo() proc parseTry(p: var Parser; isExpr: bool): PNode = #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') @@ -1789,12 +1822,14 @@ proc parseTry(p: var Parser; isExpr: bool): PNode = b.add(parseStmt(p)) result.add(b) if b == nil: parMessage(p, "expected 'except'") + setEndInfo() proc parseExceptBlock(p: var Parser, kind: TNodeKind): PNode = result = newNodeP(kind, p) getTok(p) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseBlock(p: var Parser): PNode = #| blockStmt = 'block' symbol? colcom stmt @@ -1805,6 +1840,7 @@ proc parseBlock(p: var Parser): PNode = else: result.add(parseSymbol(p)) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode = #| staticStmt = 'static' colcom stmt @@ -1813,6 +1849,7 @@ proc parseStaticOrDefer(p: var Parser; k: TNodeKind): PNode = getTok(p) colcom(p, result) result.add(parseStmt(p)) + setEndInfo() proc parseAsm(p: var Parser): PNode = #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT) @@ -1829,6 +1866,7 @@ proc parseAsm(p: var Parser): PNode = result.add(p.emptyNode) return getTok(p) + setEndInfo() proc parseGenericParam(p: var Parser): PNode = #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)? @@ -1864,6 +1902,7 @@ proc parseGenericParam(p: var Parser): PNode = result.add(parseExpr(p)) else: result.add(p.emptyNode) + setEndInfo() proc parseGenericParamList(p: var Parser): PNode = #| genericParamList = '[' optInd @@ -1882,12 +1921,14 @@ proc parseGenericParamList(p: var Parser): PNode = skipComment(p, a) optPar(p) eat(p, tkBracketRi) + setEndInfo() proc parsePattern(p: var Parser): PNode = #| pattern = '{' stmt '}' eat(p, tkCurlyLe) result = parseStmt(p) eat(p, tkCurlyRi) + setEndInfo() proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = #| indAndComment = (IND{>} COMMENT)? | COMMENT? @@ -1932,6 +1973,7 @@ proc parseRoutine(p: var Parser, kind: TNodeKind): PNode = #else: # assert false, p.lex.config$body.info # avoids hard to track bugs, fail early. # Yeah, that worked so well. There IS a bug in this logic, now what? + setEndInfo() proc newCommentStmt(p: var Parser): PNode = #| commentStmt = COMMENT @@ -1967,6 +2009,7 @@ proc parseSection(p: var Parser, kind: TNodeKind, result.add(defparser(p)) else: parMessage(p, errIdentifierExpected, p.tok) + setEndInfo() proc parseEnum(p: var Parser): PNode = #| enumDecl = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+ @@ -2013,6 +2056,7 @@ proc parseEnum(p: var Parser): PNode = break if result.len <= 1: parMessage(p, errIdentifierExpected, p.tok) + setEndInfo() proc parseObjectPart(p: var Parser): PNode proc parseObjectWhen(p: var Parser): PNode = @@ -2038,6 +2082,7 @@ proc parseObjectWhen(p: var Parser): PNode = branch.add(parseObjectPart(p)) flexComment(p, branch) result.add(branch) + setEndInfo() proc parseObjectCase(p: var Parser): PNode = #| objectBranch = 'of' exprList colcom objectPart @@ -2079,6 +2124,7 @@ proc parseObjectCase(p: var Parser): PNode = if b.kind == nkElse: break if wasIndented: p.currInd = oldInd + setEndInfo() proc parseObjectPart(p: var Parser): PNode = #| objectPart = IND{>} objectPart^+IND{=} DED @@ -2111,6 +2157,7 @@ proc parseObjectPart(p: var Parser): PNode = result = p.emptyNode else: result = p.emptyNode + setEndInfo() proc parseObject(p: var Parser): PNode = #| objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart @@ -2131,6 +2178,7 @@ proc parseObject(p: var Parser): PNode = result.add(p.emptyNode) else: result.add(parseObjectPart(p)) + setEndInfo() proc parseTypeClassParam(p: var Parser): PNode = let modifier = @@ -2148,6 +2196,7 @@ proc parseTypeClassParam(p: var Parser): PNode = result.add(p.parseSymbol) else: result = p.parseSymbol + setEndInfo() proc parseTypeClass(p: var Parser): PNode = #| conceptParam = ('var' | 'out')? symbol @@ -2191,6 +2240,7 @@ proc parseTypeClass(p: var Parser): PNode = result.add(p.emptyNode) else: result.add(parseStmt(p)) + setEndInfo() proc parseTypeDef(p: var Parser): PNode = #| @@ -2224,6 +2274,7 @@ proc parseTypeDef(p: var Parser): PNode = else: result.add(p.emptyNode) indAndComment(p, result) # special extension! + setEndInfo() proc parseVarTuple(p: var Parser): PNode = #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr @@ -2240,6 +2291,7 @@ proc parseVarTuple(p: var Parser): PNode = result.add(p.emptyNode) # no type desc optPar(p) eat(p, tkParRi) + setEndInfo() proc parseVariable(p: var Parser): PNode = #| colonBody = colcom stmt postExprBlocks? @@ -2252,6 +2304,7 @@ proc parseVariable(p: var Parser): PNode = else: result = parseIdentColonEquals(p, {withPragma, withDot}) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) + setEndInfo() proc parseConstant(p: var Parser): PNode = #| constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment @@ -2271,6 +2324,7 @@ proc parseConstant(p: var Parser): PNode = result.add(parseExpr(p)) result[^1] = postExprBlocks(p, result[^1]) indAndComment(p, result) + setEndInfo() proc parseBind(p: var Parser, k: TNodeKind): PNode = #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma @@ -2286,6 +2340,7 @@ proc parseBind(p: var Parser, k: TNodeKind): PNode = getTok(p) optInd(p, a) #expectNl(p) + setEndInfo() proc parseStmtPragma(p: var Parser): PNode = #| pragmaStmt = pragma (':' COMMENT? stmt)? @@ -2297,6 +2352,7 @@ proc parseStmtPragma(p: var Parser): PNode = skipComment(p, result) result.add a result.add parseStmt(p) + setEndInfo() proc simpleStmt(p: var Parser): PNode = #| simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt @@ -2439,6 +2495,7 @@ proc parseStmt(p: var Parser): PNode = if p.tok.tokType != tkSemiColon: break getTok(p) if err and p.tok.tokType == tkEof: break + setEndInfo() proc parseAll(p: var Parser): PNode = ## Parses the rest of the input stream held by the parser into a PNode. @@ -2454,6 +2511,7 @@ proc parseAll(p: var Parser): PNode = getTok(p) if p.tok.indent != 0: parMessage(p, errInvalidIndentation) + setEndInfo() proc checkFirstLineIndentation*(p: var Parser) = if p.tok.indent != 0 and p.tok.strongSpaceA: @@ -2487,6 +2545,7 @@ proc parseTopLevelStmt(p: var Parser): PNode = result = complexOrSimpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break + setEndInfo() proc parseString*(s: string; cache: IdentCache; config: ConfigRef; filename: string = ""; line: int = 0; @@ -2498,9 +2557,10 @@ proc parseString*(s: string; cache: IdentCache; config: ConfigRef; var stream = llStreamOpen(s) stream.lineOffset = line - var parser: Parser - parser.lex.errorHandler = errorHandler - openParser(parser, AbsoluteFile filename, stream, cache, config) + var p: Parser + p.lex.errorHandler = errorHandler + openParser(p, AbsoluteFile filename, stream, cache, config) - result = parser.parseAll - closeParser(parser) + result = p.parseAll + closeParser(p) + setEndInfo() diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 612af482a..7454c7f8e 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -980,6 +980,14 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy return errorNode(c, n) result = n + + when defined(nimsuggest): + if c.config.expandProgress: + if c.config.expandLevels == 0: + return n + else: + c.config.expandLevels -= 1 + let callee = result[0].sym case callee.kind of skMacro: result = semMacroExpr(c, result, orig, callee, flags, expectedType) @@ -1890,6 +1898,9 @@ proc semReturn(c: PContext, n: PNode): PNode = localError(c.config, n.info, "'return' not allowed here") proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode = + when defined(nimsuggest): + if c.graph.config.expandDone(): + return n openScope(c) result = semExpr(c, n, expectedType = expectedType) if c.p.resultSym != nil and not isEmptyType(result.typ): @@ -2895,7 +2906,6 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType defer: if isCompilerDebug(): echo ("<", c.config$n.info, n, ?.result.typ) - template directLiteral(typeKind: TTypeKind) = if result.typ == nil: if expectedType != nil and ( @@ -2907,6 +2917,19 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType result.typ = getSysType(c.graph, n.info, typeKind) result = n + when defined(nimsuggest): + var expandStarted = false + if c.config.ideCmd == ideExpand and not c.config.expandProgress and + ((n.kind in {nkFuncDef, nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkConverterDef} and + n.info.exactEquals(c.config.expandPosition)) or + (n.kind in {nkCall, nkCommand} and + n[0].info.exactEquals(c.config.expandPosition))): + expandStarted = true + c.config.expandProgress = true + if c.config.expandLevels == 0: + c.config.expandNodeResult = $n + suggestQuit() + if c.config.cmd == cmdIdeTools: suggestExpr(c, n) if nfSem in n.flags: return case n.kind @@ -3234,3 +3257,8 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType localError(c.config, n.info, "invalid expression: " & renderTree(n, {renderNoComments})) if result != nil: incl(result.flags, nfSem) + + when defined(nimsuggest): + if expandStarted: + c.config.expandNodeResult = $result + suggestQuit() diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 747540c78..e989eb4a6 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1450,6 +1450,9 @@ proc hasRealBody(s: PSym): bool = proc trackProc*(c: PContext; s: PSym, body: PNode) = let g = c.graph + when defined(nimsuggest): + if g.config.expandDone(): + return var effects = s.typ.n[0] if effects.kind != nkEffectList: return # effects already computed? diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 96883255c..c7e48db4c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2156,7 +2156,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if s.kind notin {skMacro, skTemplate} and s.magic == mNone: paramsTypeCheck(c, s.typ) maybeAddResult(c, s, n) - let resultType = + let resultType = if s.kind == skMacro: sysTypeFromName(c.graph, n.info, "NimNode") elif not isInlineIterator(s.typ): diff --git a/compiler/suggest.nim b/compiler/suggest.nim index e0b0fb516..637010ad5 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -120,7 +120,9 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo; quality: range[0..100]; prefix: PrefixMatch; inTypeContext: bool; scope: int; - useSuppliedInfo = false): Suggest = + useSuppliedInfo = false, + endLine: uint16 = 0, + endCol = 0): Suggest = new(result) result.section = section result.quality = quality @@ -176,6 +178,8 @@ proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info else: getTokenLenFromSource(g.config, s.name.s, infox) result.version = g.config.suggestVersion + result.endLine = endLine + result.endCol = endCol proc `$`*(suggest: Suggest): string = result = $suggest.section @@ -216,6 +220,12 @@ proc `$`*(suggest: Suggest): string = result.add(sep) result.add($suggest.prefix) + if (suggest.version == 3 and suggest.section in {ideOutline, ideExpand}): + result.add(sep) + result.add($suggest.endLine) + result.add(sep) + result.add($suggest.endCol) + proc suggestResult*(conf: ConfigRef; s: Suggest) = if not isNil(conf.suggestionResultHook): conf.suggestionResultHook(s) diff --git a/compiler/vm.nim b/compiler/vm.nim index 4f5f8485b..424685e40 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -2333,6 +2333,9 @@ const evalPass* = makePass(myOpen, myProcess, myClose) proc evalConstExprAux(module: PSym; idgen: IdGenerator; g: ModuleGraph; prc: PSym, n: PNode, mode: TEvalMode): PNode = + when defined(nimsuggest): + if g.config.expandDone(): + return n #if g.config.errorCounter > 0: return n let n = transformExpr(g, idgen, module, n) setupGlobalCtx(module, g, idgen) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c5e015ce9..a0a3b8e82 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -27,7 +27,7 @@ import compiler / [options, commands, modules, sem, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, - pathutils, condsyms] + pathutils, condsyms, syntaxes] when defined(nimPreviewSlimSystem): import std/typedthreads @@ -88,7 +88,7 @@ var requests: Channel[string] results: Channel[Suggest] -proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; +proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph); proc writelnToChannel(line: string) = @@ -140,6 +140,9 @@ proc sexp(s: Suggest): SexpNode = ]) if s.section == ideSug: result.add convertSexp(s.prefix) + if s.section in {ideOutline, ideExpand} and s.version == 3: + result.add convertSexp(s.endLine.int) + result.add convertSexp(s.endCol) proc sexp(s: seq[Suggest]): SexpNode = result = newSList() @@ -175,12 +178,23 @@ proc symFromInfo(graph: ModuleGraph; trackPos: TLineInfo): PSym = if m != nil and m.ast != nil: result = findNode(m.ast, trackPos) -proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; +template benchmark(benchmarkName: untyped, code: untyped) = + block: + myLog "Started [" & benchmarkName & "]..." + let t0 = epochTime() + code + let elapsed = epochTime() - t0 + let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) + myLog "CPU Time [" & benchmarkName & "] " & elapsedStr & "s" + +proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, tag: string, graph: ModuleGraph) = let conf = graph.config if conf.suggestVersion == 3: - executeNoHooksV3(cmd, file, dirtyfile, line, col, graph) + let command = fmt "cmd = {cmd} {file}:{line}:{col}" + benchmark command: + executeNoHooksV3(cmd, file, dirtyfile, line, col, tag, graph) return myLog("cmd: " & $cmd & ", file: " & file.string & @@ -219,7 +233,10 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; else: localError(conf, conf.m.trackPos, "found no symbol at this position " & (conf $ conf.m.trackPos)) -proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; +proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int, graph: ModuleGraph) = + executeNoHooks(cmd, file, dirtyfile, line, col, graph) + +proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph) = if cmd == ideChk: graph.config.structuredErrorHook = errorHook @@ -227,7 +244,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int; else: graph.config.structuredErrorHook = nil graph.config.writelnHook = myLog - executeNoHooks(cmd, file, dirtyfile, line, col, graph) + executeNoHooks(cmd, file, dirtyfile, line, col, tag, graph) proc executeEpc(cmd: IdeCmd, args: SexpNode; graph: ModuleGraph) = @@ -238,7 +255,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode; var dirtyfile = AbsoluteFile"" if len(args) > 3: dirtyfile = AbsoluteFile args[3].getStr("") - execute(cmd, file, dirtyfile, int(line), int(column), graph) + execute(cmd, file, dirtyfile, int(line), int(column), args[3].getStr, graph) proc returnEpc(socket: Socket, uid: BiggestInt, s: SexpNode|string, returnSymbol = "return") = @@ -459,6 +476,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = of "changed": conf.ideCmd = ideChanged of "globalsymbols": conf.ideCmd = ideGlobalSymbols of "declaration": conf.ideCmd = ideDeclaration + of "expand": conf.ideCmd = ideExpand of "chkfile": conf.ideCmd = ideChkFile of "recompile": conf.ideCmd = ideRecompile of "type": conf.ideCmd = ideType @@ -478,6 +496,7 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = i += parseInt(cmd, line, i) i += skipWhile(cmd, seps, i) i += parseInt(cmd, col, i) + let tag = substr(cmd, i) if conf.ideCmd == ideKnown: results.send(Suggest(section: ideKnown, quality: ord(fileInfoKnown(conf, AbsoluteFile orig)))) @@ -486,18 +505,9 @@ proc execCmd(cmd: string; graph: ModuleGraph; cachedMsgs: CachedMsgs) = else: if conf.ideCmd == ideChk: for cm in cachedMsgs: errorHook(conf, cm.info, cm.msg, cm.sev) - execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, graph) + execute(conf.ideCmd, AbsoluteFile orig, AbsoluteFile dirtyfile, line, col, tag, graph) sentinel() -template benchmark(benchmarkName: string, code: untyped) = - block: - myLog "Started [" & benchmarkName & "]..." - let t0 = epochTime() - code - let elapsed = epochTime() - t0 - let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 3) - myLog "CPU Time [" & benchmarkName & "] " & elapsedStr & "s" - proc recompileFullProject(graph: ModuleGraph) = benchmark "Recompilation(clean)": graph.resetForBackend() @@ -509,9 +519,9 @@ proc recompileFullProject(graph: ModuleGraph) = proc mainThread(graph: ModuleGraph) = let conf = graph.config - if gLogging: - for it in conf.searchPaths: - log(it.string) + myLog "searchPaths: " + for it in conf.searchPaths: + myLog(" " & it.string) proc wrHook(line: string) {.closure.} = if gMode == mepc: @@ -732,16 +742,20 @@ func deduplicateSymInfoPair[SymInfoPair](xs: seq[SymInfoPair]): seq[SymInfoPair] result.add(itm) result.reverse() +proc findSymData(graph: ModuleGraph, trackPos: TLineInfo): + ref SymInfoPair = + for s in graph.fileSymbols(trackPos.fileIndex).deduplicateSymInfoPair: + if isTracked(s.info, trackPos, s.sym.name.s.len): + new(result) + result[] = s + break + proc findSymData(graph: ModuleGraph, file: AbsoluteFile; line, col: int): ref SymInfoPair = let fileIdx = fileInfoIdx(graph.config, file) trackPos = newLineInfo(fileIdx, line, col) - for s in graph.fileSymbols(fileIdx).deduplicateSymInfoPair: - if isTracked(s.info, trackPos, s.sym.name.s.len): - new(result) - result[] = s - break + result = findSymData(graph, trackPos) proc markDirtyIfNeeded(graph: ModuleGraph, file: string, originalFileIdx: FileIndex) = let sha = $sha1.secureHashFile(file) @@ -752,7 +766,8 @@ proc markDirtyIfNeeded(graph: ModuleGraph, file: string, originalFileIdx: FileIn else: myLog fmt "No changes in file {file} compared to last compilation" -proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, defaultSection = ideNone) = +proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, + defaultSection = ideNone, endLine: uint16 = 0, endCol = 0) = let section = if defaultSection != ideNone: defaultSection elif sym.info.exactEquals(info): @@ -760,7 +775,8 @@ proc suggestResult(graph: ModuleGraph, sym: PSym, info: TLineInfo, defaultSectio else: ideUse let suggest = symToSuggest(graph, sym, isLocal=false, section, - info, 100, PrefixMatch.None, false, 0) + info, 100, PrefixMatch.None, false, 0, + endLine = endLine, endCol = endCol) suggestResult(graph.config, suggest) const @@ -771,7 +787,75 @@ proc symbolEqual(left, right: PSym): bool = # More relaxed symbol comparison return left.info.exactEquals(right.info) and left.name == right.name -proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; +proc findDef(n: PNode, line: uint16, col: int16): PNode = + if n.kind in {nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkMacroDef}: + if n.info.line == line: + return n + else: + for i in 0 ..< safeLen(n): + let res = findDef(n[i], line, col) + if res != nil: return res + +proc findByTLineInfo(trackPos: TLineInfo, infoPairs: seq[SymInfoPair]): + ref SymInfoPair = + for s in infoPairs: + if s.info.exactEquals trackPos: + new(result) + result[] = s + break + +proc outlineNode(graph: ModuleGraph, n: PNode, endInfo: TLineInfo, infoPairs: seq[SymInfoPair]): bool = + proc checkSymbol(sym: PSym, info: TLineInfo): bool = + result = (sym.owner.kind in {skModule, skType} or sym.kind in {skProc, skMethod, skIterator, skTemplate, skType}) + + if n.kind == nkSym and n.sym.checkSymbol(n.info): + graph.suggestResult(n.sym, n.sym.info, ideOutline, endInfo.line, endInfo.col) + return true + elif n.kind == nkIdent: + let symData = findByTLineInfo(n.info, infoPairs) + if symData != nil and symData.sym.checkSymbol(symData.info): + let sym = symData.sym + graph.suggestResult(sym, sym.info, ideOutline, endInfo.line, endInfo.col) + return true + +proc handleIdentOrSym(graph: ModuleGraph, n: PNode, endInfo: TLineInfo, infoPairs: seq[SymInfoPair]): bool = + for child in n: + if child.kind in {nkIdent, nkSym}: + if graph.outlineNode(child, endInfo, infoPairs): + return true + elif child.kind == nkPostfix: + if graph.handleIdentOrSym(child, endInfo, infoPairs): + return true + +proc iterateOutlineNodes(graph: ModuleGraph, n: PNode, infoPairs: seq[SymInfoPair]) = + var matched = true + if n.kind == nkIdent: + let symData = findByTLineInfo(n.info, infoPairs) + if symData != nil and symData.sym.kind == skEnumField and symData.info.exactEquals(symData.sym.info): + let sym = symData.sym + graph.suggestResult(sym, sym.info, ideOutline, n.endInfo.line, n.endInfo.col) + elif (n.kind in {nkFuncDef, nkProcDef, nkTypeDef, nkMacroDef, nkTemplateDef, nkConverterDef, nkEnumFieldDef, nkConstDef}): + matched = handleIdentOrSym(graph, n, n.endInfo, infoPairs) + else: + matched = false + + if n.kind != nkFormalParams: + for child in n: + graph.iterateOutlineNodes(child, infoPairs) + +proc calculateExpandRange(n: PNode, info: TLineInfo): TLineInfo = + if ((n.kind in {nkFuncDef, nkProcDef, nkIteratorDef, nkTemplateDef, nkMethodDef, nkConverterDef} and + n.info.exactEquals(info)) or + (n.kind in {nkCall, nkCommand} and n[0].info.exactEquals(info))): + result = n.endInfo + else: + for child in n: + result = child.calculateExpandRange(info) + if result != unknownLineInfo: + return result + result = unknownLineInfo + +proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, line, col: int; tag: string, graph: ModuleGraph) = let conf = graph.config conf.writelnHook = proc (s: string) = discard @@ -783,7 +867,7 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, conf.ideCmd = cmd - myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}" + myLog fmt "cmd: {cmd}, file: {file}[{line}:{col}], dirtyFile: {dirtyfile}, tag: {tag}" var fileIndex: FileIndex @@ -811,7 +895,7 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, graph.unmarkAllDirty() # these commands require partially compiled project - elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration} and + elif cmd in {ideSug, ideOutline, ideHighlight, ideDef, ideChkFile, ideType, ideDeclaration, ideExpand} and (graph.needsCompilation(fileIndex) or cmd == ideSug): # for ideSug use v2 implementation if cmd == ideSug: @@ -861,16 +945,8 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, # future calls. graph.markDirtyIfNeeded(file.string, fileIndex) of ideOutline: - let - module = graph.getModule fileIndex - symbols = graph.fileSymbols(fileIndex) - .deduplicateSymInfoPair - .filterIt(it.sym.info.exactEquals(it.info) and - (it.sym.owner == module or - it.sym.kind in searchableSymKinds)) - - for s in symbols: - graph.suggestResult(s.sym, s.info, ideOutline) + let n = parseFile(fileIndex, graph.cache, graph.config) + graph.iterateOutlineNodes(n, graph.fileSymbols(fileIndex).deduplicateSymInfoPair) of ideChk: myLog fmt "Reporting errors for {graph.suggestErrors.len} file(s)" for sug in graph.suggestErrorsIter: @@ -933,6 +1009,39 @@ proc executeNoHooksV3(cmd: IdeCmd, file: AbsoluteFile, dirtyfile: AbsoluteFile, else: # we are on definition or usage, look for declaration graph.suggestResult(first.sym, first.info, ideDeclaration) + of ideExpand: + var level: int = high(int) + let index = skipWhitespace(tag, 0); + let trimmed = substr(tag, index) + if not (trimmed == "" or trimmed == "all"): + discard parseInt(trimmed, level, 0) + + conf.expandPosition = newLineInfo(fileIndex, line, col) + conf.expandLevels = level + conf.expandProgress = false + conf.expandNodeResult = "" + + graph.markDirty fileIndex + graph.markClientsDirty fileIndex + graph.recompilePartially() + var suggest = Suggest() + suggest.section = ideExpand + suggest.version = 3 + suggest.line = line + suggest.column = col + suggest.doc = graph.config.expandNodeResult + if suggest.doc != "": + let + n = parseFile(fileIndex, graph.cache, graph.config) + endInfo = n.calculateExpandRange(conf.expandPosition) + + suggest.endLine = endInfo.line + suggest.endCol = endInfo.col + + suggestResult(graph.config, suggest) + + graph.markDirty fileIndex + graph.markClientsDirty fileIndex else: myLog fmt "Discarding {cmd}" @@ -1018,9 +1127,9 @@ else: if self.loadConfigsAndProcessCmdLine(cache, conf, graph): mockCommand(graph) if gLogging: - log("Search paths:") + myLog("Search paths:") for it in conf.searchPaths: - log(" " & it.string) + myLog(" " & it.string) retval.doStopCompile = proc (): bool = false return NimSuggest(graph: retval, idle: 0, cachedMsgs: @[]) diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 6e068e067..060335959 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -66,7 +66,7 @@ proc parseTest(filename: string; epcMode=false): Test = elif x.startsWith(">"): # since 'markers' here are not complete yet, we do the $substitutions # afterwards - result.script.add((x.substr(1).replaceWord("$path", tpath), "")) + result.script.add((x.substr(1).replaceWord("$path", tpath).replaceWord("$file", filename), "")) elif x.len > 0: # expected output line: let x = x % ["file", filename, "lib", libpath] @@ -218,7 +218,12 @@ proc sexpToAnswer(s: SexpNode): string = result.add doc result.add '\t' result.addInt a[8].getNum - if a.len >= 10: + if a.len >= 11: + result.add '\t' + result.addInt a[9].getNum + result.add '\t' + result.addInt a[10].getNum + elif a.len >= 10: result.add '\t' result.add a[9].getStr result.add '\L' @@ -229,8 +234,8 @@ proc doReport(filename, answer, resp: string; report: var string) = var hasDiff = false for i in 0..min(resp.len-1, answer.len-1): if resp[i] != answer[i]: - report.add "\n Expected: " & resp.substr(i, i+200) - report.add "\n But got: " & answer.substr(i, i+200) + report.add "\n Expected:\n" & resp + report.add "\n But got:\n" & answer hasDiff = true break if not hasDiff: @@ -342,8 +347,8 @@ proc main() = if os.paramCount() > 0: let x = os.paramStr(1) let xx = expandFilename x + # run only stdio when running single test failures += runTest(xx) - failures += runEpcTest(xx) else: let files = toSeq(walkFiles(tpath / "t*.nim")) for i, x in files: diff --git a/nimsuggest/tests/tv3.nim b/nimsuggest/tests/tv3.nim index 9d8b1ef2d..fd736a1d8 100644 --- a/nimsuggest/tests/tv3.nim +++ b/nimsuggest/tests/tv3.nim @@ -16,10 +16,6 @@ def skField tv3.Foo.bar string $file 5 4 "" 100 use skField tv3.Foo.bar string $file 8 9 "" 100 >def $1 def skField tv3.Foo.bar string $file 5 4 "" 100 ->outline $1 -outline skType tv3.Foo Foo $file 4 2 "" 100 -outline skField tv3.Foo.bar string $file 5 4 "" 100 -outline skProc tv3.test proc (f: Foo){.gcsafe.} $file 7 5 "" 100 >sug $1 sug skField bar string $file 5 4 "" 100 Prefix >globalSymbols test diff --git a/nimsuggest/tests/tv3_outline.nim b/nimsuggest/tests/tv3_outline.nim new file mode 100644 index 000000000..6370948d9 --- /dev/null +++ b/nimsuggest/tests/tv3_outline.nim @@ -0,0 +1,45 @@ +# tests v3 outline + +type + Foo* = ref object of RootObj + bar*: string + FooEnum = enum value1, value2 + FooPrivate = ref object of RootObj + barPrivate: string + +macro m(arg: untyped): untyped = discard +template t(arg: untyped): untyped = discard +proc p(): void = discard +iterator i(): int = discard +converter c(s: string): int = discard +method m(f: Foo): void = discard +func f(): void = discard + +let a = 1 +var b = 2 +const con = 2 + +proc outer(): void = + proc inner() = discard + +proc procWithLocal(): void = + let local = 10 + +discard """ +$nimsuggest --v3 --tester $file +>outline $file +outline skType tv3_outline.Foo Foo $file 4 2 "" 100 5 16 +outline skType tv3_outline.FooEnum FooEnum $file 6 2 "" 100 6 31 +outline skEnumField tv3_outline.FooEnum.value1 FooEnum $file 6 17 "" 100 6 23 +outline skEnumField tv3_outline.FooEnum.value2 FooEnum $file 6 25 "" 100 6 31 +outline skType tv3_outline.FooPrivate FooPrivate $file 7 2 "" 100 8 22 +outline skMacro tv3_outline.m macro (arg: untyped): untyped{.noSideEffect, gcsafe.} $file 10 6 "" 100 10 40 +outline skTemplate tv3_outline.t template (arg: untyped): untyped $file 11 9 "" 100 11 43 +outline skProc tv3_outline.p proc (){.noSideEffect, gcsafe.} $file 12 5 "" 100 12 24 +outline skConverter tv3_outline.c converter (s: string): int{.noSideEffect, gcsafe.} $file 14 10 "" 100 14 37 +outline skFunc tv3_outline.f proc (){.noSideEffect, gcsafe.} $file 16 5 "" 100 16 24 +outline skConst tv3_outline.con int literal(2) $file 20 6 "" 100 20 13 +outline skProc tv3_outline.outer proc (){.noSideEffect, gcsafe.} $file 22 5 "" 100 23 24 +outline skProc tv3_outline.outer.inner proc (){.noSideEffect, gcsafe.} $file 23 7 "" 100 23 24 +outline skProc tv3_outline.procWithLocal proc (){.noSideEffect, gcsafe.} $file 25 5 "" 100 26 16 +""" From a7bae919adb952362cb53206140872d2b7424b47 Mon Sep 17 00:00:00 2001 From: Bung Date: Fri, 27 Jan 2023 00:50:44 -0800 Subject: [PATCH 1562/1654] Fix #13093 C++ Atomics: operator= is implicitly deleted because the default definition would be ill-formed (#21169) * add test * fix #17982 Invalid C++ code generation when returning discardable var T * fix #13093 * cpp atomic good example * clearify the condition --- compiler/cgen.nim | 4 ++++ compiler/semstmts.nim | 6 +++++- lib/pure/concurrency/atomics.nim | 2 +- tests/cpp/t13093.nim | 24 ++++++++++++++++++++++++ tests/cpp/t17982.nim | 20 ++++++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 tests/cpp/t13093.nim create mode 100644 tests/cpp/t17982.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5bf9400c5..869c1bfdd 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -491,6 +491,10 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = nilLoc.r = rope("NIM_NIL") genRefAssign(p, loc, nilLoc) else: + if typ.kind == tyVar: + let tt = typ.skipTypes({tyVar}) + if isImportedType(tt) and tfRequiresInit in tt.flags: + return linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) else: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c7e48db4c..12a346f63 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -717,7 +717,11 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = tyUserTypeClassInst}) if actualType.kind in {tyObject, tyDistinct} and actualType.requiresInit: - defaultConstructionError(c, v.typ, v.info) + # imported type use requiresInit pragma prevent implicit initialization + if (tfRequiresInit in actualType.flags and sfImportc in actualType.sym.flags): + discard + else: + defaultConstructionError(c, v.typ, v.info) else: checkNilable(c, v) # allow let to not be initialised if imported from C: diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index c7b881bc5..49204bd4d 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -89,7 +89,7 @@ when defined(cpp) or defined(nimdoc): ## with other moSequentiallyConsistent operations. type - Atomic*[T] {.importcpp: "std::atomic", completeStruct.} = object + Atomic*[T] {.importcpp: "std::atomic", completeStruct, requiresInit.} = object ## An atomic object with underlying type `T`. raw: T diff --git a/tests/cpp/t13093.nim b/tests/cpp/t13093.nim new file mode 100644 index 000000000..17c730d16 --- /dev/null +++ b/tests/cpp/t13093.nim @@ -0,0 +1,24 @@ +discard """ + targets: "cpp" + action: reject + errormsg: "The PledgeObj type requires the following fields to be initialized: refCount" +""" + +import atomics + +type + Pledge* = object + p: PledgePtr + + PledgePtr = ptr PledgeObj + PledgeObj = object + refCount: Atomic[int32] + +proc main() = + var pledge: Pledge + pledge.p = createShared(PledgeObj) + let tmp = PledgeObj() # <---- not allowed: atomics are not copyable + + pledge.p[] = tmp + +main() diff --git a/tests/cpp/t17982.nim b/tests/cpp/t17982.nim new file mode 100644 index 000000000..5413f06ff --- /dev/null +++ b/tests/cpp/t17982.nim @@ -0,0 +1,20 @@ +discard """ + targets: "cpp" + action: "compile" +""" + +type + String* {.importcpp: "std::string", header: "string".} = object + +proc initString*(): String + {.importcpp: "std::string()", header: "string".} + +proc append*(this: var String, str: String): String + # bug seems to trigger when `#`, `@`, or `$1` is used inside `importcpp` + {.importcpp: "#.append(@)", header: "string", discardable.} # <- changed from `importcpp: "append"` + +var + s1 = initString() + s2 = initString() + +s1.append s2 From 1431f90d8c3d3ed0afcaee62073d5808cea227ff Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 28 Jan 2023 04:41:59 +0800 Subject: [PATCH 1563/1654] Revert "Fix #13093 C++ Atomics: operator= is implicitly deleted because the default definition would be ill-formed " (#21307) Revert "Fix #13093 C++ Atomics: operator= is implicitly deleted because the default definition would be ill-formed (#21169)" This reverts commit a7bae919adb952362cb53206140872d2b7424b47. --- compiler/cgen.nim | 4 ---- compiler/semstmts.nim | 6 +----- lib/pure/concurrency/atomics.nim | 2 +- tests/cpp/t13093.nim | 24 ------------------------ tests/cpp/t17982.nim | 20 -------------------- 5 files changed, 2 insertions(+), 54 deletions(-) delete mode 100644 tests/cpp/t13093.nim delete mode 100644 tests/cpp/t17982.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 869c1bfdd..5bf9400c5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -491,10 +491,6 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = nilLoc.r = rope("NIM_NIL") genRefAssign(p, loc, nilLoc) else: - if typ.kind == tyVar: - let tt = typ.skipTypes({tyVar}) - if isImportedType(tt) and tfRequiresInit in tt.flags: - return linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) else: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 12a346f63..c7e48db4c 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -717,11 +717,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = tyUserTypeClassInst}) if actualType.kind in {tyObject, tyDistinct} and actualType.requiresInit: - # imported type use requiresInit pragma prevent implicit initialization - if (tfRequiresInit in actualType.flags and sfImportc in actualType.sym.flags): - discard - else: - defaultConstructionError(c, v.typ, v.info) + defaultConstructionError(c, v.typ, v.info) else: checkNilable(c, v) # allow let to not be initialised if imported from C: diff --git a/lib/pure/concurrency/atomics.nim b/lib/pure/concurrency/atomics.nim index 49204bd4d..c7b881bc5 100644 --- a/lib/pure/concurrency/atomics.nim +++ b/lib/pure/concurrency/atomics.nim @@ -89,7 +89,7 @@ when defined(cpp) or defined(nimdoc): ## with other moSequentiallyConsistent operations. type - Atomic*[T] {.importcpp: "std::atomic", completeStruct, requiresInit.} = object + Atomic*[T] {.importcpp: "std::atomic", completeStruct.} = object ## An atomic object with underlying type `T`. raw: T diff --git a/tests/cpp/t13093.nim b/tests/cpp/t13093.nim deleted file mode 100644 index 17c730d16..000000000 --- a/tests/cpp/t13093.nim +++ /dev/null @@ -1,24 +0,0 @@ -discard """ - targets: "cpp" - action: reject - errormsg: "The PledgeObj type requires the following fields to be initialized: refCount" -""" - -import atomics - -type - Pledge* = object - p: PledgePtr - - PledgePtr = ptr PledgeObj - PledgeObj = object - refCount: Atomic[int32] - -proc main() = - var pledge: Pledge - pledge.p = createShared(PledgeObj) - let tmp = PledgeObj() # <---- not allowed: atomics are not copyable - - pledge.p[] = tmp - -main() diff --git a/tests/cpp/t17982.nim b/tests/cpp/t17982.nim deleted file mode 100644 index 5413f06ff..000000000 --- a/tests/cpp/t17982.nim +++ /dev/null @@ -1,20 +0,0 @@ -discard """ - targets: "cpp" - action: "compile" -""" - -type - String* {.importcpp: "std::string", header: "string".} = object - -proc initString*(): String - {.importcpp: "std::string()", header: "string".} - -proc append*(this: var String, str: String): String - # bug seems to trigger when `#`, `@`, or `$1` is used inside `importcpp` - {.importcpp: "#.append(@)", header: "string", discardable.} # <- changed from `importcpp: "append"` - -var - s1 = initString() - s2 = initString() - -s1.append s2 From 4459c272ecb8ddb964fa2d34a4dcef3368d807b5 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 29 Jan 2023 22:11:24 +0800 Subject: [PATCH 1564/1654] remove `enumUtilsExist` defines (#21309) --- compiler/wordrecg.nim | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index a9ef31dc2..29a50ac52 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -130,20 +130,7 @@ const wFor, wIf, wReturn, wStatic, wTemplate, wTry, wWhile, wUsing} -const enumUtilsExist = compiles: - import std/enumutils - -when enumUtilsExist: - from std/enumutils import genEnumCaseStmt - from strutils import normalize - proc findStr*[T: enum](a, b: static[T], s: string, default: T): T = - genEnumCaseStmt(T, s, default, ord(a), ord(b), normalize) - -else: - from strutils import cmpIgnoreStyle - proc findStr*[T: enum](a, b: static[T], s: string, default: T): T {.deprecated.} = - # used for compiler bootstrapping only - for i in a..b: - if cmpIgnoreStyle($i, s) == 0: - return i - result = default +from std/enumutils import genEnumCaseStmt +from strutils import normalize +proc findStr*[T: enum](a, b: static[T], s: string, default: T): T = + genEnumCaseStmt(T, s, default, ord(a), ord(b), normalize) From ebaa07b955182dd98a1bfe06de4195fe1d306c5e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 31 Jan 2023 23:11:44 +0800 Subject: [PATCH 1565/1654] remove md5_old (#21312) --- compiler/md5_old.nim | 297 -------------------------------------- compiler/modulegraphs.nim | 2 +- compiler/sighashes.nim | 2 +- 3 files changed, 2 insertions(+), 299 deletions(-) delete mode 100644 compiler/md5_old.nim diff --git a/compiler/md5_old.nim b/compiler/md5_old.nim deleted file mode 100644 index da4a51198..000000000 --- a/compiler/md5_old.nim +++ /dev/null @@ -1,297 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2010 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# `std/md5` without VM and JavaScript support, to circumvent a bug with -# openarrays on Nim < 1.4. - -when defined(nimHasStyleChecks): - {.push styleChecks: off.} - -type - MD5State = array[0..3, uint32] - MD5Block = array[0..15, uint32] - MD5CBits = array[0..7, uint8] - MD5Digest* = array[0..15, uint8] - ## MD5 checksum of a string, obtained with the `toMD5 proc <#toMD5,string>`_. - MD5Buffer = array[0..63, uint8] - MD5Context* {.final.} = object - state: MD5State - count: array[0..1, uint32] - buffer: MD5Buffer - -const - padding: array[0..63, uint8] = [ - 0x80'u8, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0 - ] - -proc F(x, y, z: uint32): uint32 {.inline.} = - result = (x and y) or ((not x) and z) - -proc G(x, y, z: uint32): uint32 {.inline.} = - result = (x and z) or (y and (not z)) - -proc H(x, y, z: uint32): uint32 {.inline.} = - result = x xor y xor z - -proc I(x, y, z: uint32): uint32 {.inline.} = - result = y xor (x or (not z)) - -proc rot(x: var uint32, n: uint8) {.inline.} = - x = (x shl n) or (x shr (32'u32 - n)) - -proc FF(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + F(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc GG(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + G(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc HH(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + H(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc II(a: var uint32, b, c, d, x: uint32, s: uint8, ac: uint32) = - a = a + I(b, c, d) + x + ac - rot(a, s) - a = a + b - -proc encode(dest: var MD5Block, src: openArray[uint8]) = - var j = 0 - for i in 0..high(dest): - dest[i] = uint32(ord(src[j])) or - uint32(ord(src[j+1])) shl 8 or - uint32(ord(src[j+2])) shl 16 or - uint32(ord(src[j+3])) shl 24 - inc(j, 4) - -proc decode(dest: var openArray[uint8], src: openArray[uint32]) = - var i = 0 - for j in 0..high(src): - dest[i] = uint8(src[j] and 0xff'u32) - dest[i+1] = uint8(src[j] shr 8 and 0xff'u32) - dest[i+2] = uint8(src[j] shr 16 and 0xff'u32) - dest[i+3] = uint8(src[j] shr 24 and 0xff'u32) - inc(i, 4) - -template slice(s: string | cstring, a, b): openArray[uint8] = - s.toOpenArrayByte(a, b) - -template slice(s: openArray[uint8], a, b): openArray[uint8] = - s.toOpenArray(a, b) - -proc transform(buffer: openArray[uint8], state: var MD5State) = - var - myBlock: MD5Block - encode(myBlock, buffer) - var a = state[0] - var b = state[1] - var c = state[2] - var d = state[3] - FF(a, b, c, d, myBlock[0], 7'u8, 0xD76AA478'u32) - FF(d, a, b, c, myBlock[1], 12'u8, 0xE8C7B756'u32) - FF(c, d, a, b, myBlock[2], 17'u8, 0x242070DB'u32) - FF(b, c, d, a, myBlock[3], 22'u8, 0xC1BDCEEE'u32) - FF(a, b, c, d, myBlock[4], 7'u8, 0xF57C0FAF'u32) - FF(d, a, b, c, myBlock[5], 12'u8, 0x4787C62A'u32) - FF(c, d, a, b, myBlock[6], 17'u8, 0xA8304613'u32) - FF(b, c, d, a, myBlock[7], 22'u8, 0xFD469501'u32) - FF(a, b, c, d, myBlock[8], 7'u8, 0x698098D8'u32) - FF(d, a, b, c, myBlock[9], 12'u8, 0x8B44F7AF'u32) - FF(c, d, a, b, myBlock[10], 17'u8, 0xFFFF5BB1'u32) - FF(b, c, d, a, myBlock[11], 22'u8, 0x895CD7BE'u32) - FF(a, b, c, d, myBlock[12], 7'u8, 0x6B901122'u32) - FF(d, a, b, c, myBlock[13], 12'u8, 0xFD987193'u32) - FF(c, d, a, b, myBlock[14], 17'u8, 0xA679438E'u32) - FF(b, c, d, a, myBlock[15], 22'u8, 0x49B40821'u32) - GG(a, b, c, d, myBlock[1], 5'u8, 0xF61E2562'u32) - GG(d, a, b, c, myBlock[6], 9'u8, 0xC040B340'u32) - GG(c, d, a, b, myBlock[11], 14'u8, 0x265E5A51'u32) - GG(b, c, d, a, myBlock[0], 20'u8, 0xE9B6C7AA'u32) - GG(a, b, c, d, myBlock[5], 5'u8, 0xD62F105D'u32) - GG(d, a, b, c, myBlock[10], 9'u8, 0x02441453'u32) - GG(c, d, a, b, myBlock[15], 14'u8, 0xD8A1E681'u32) - GG(b, c, d, a, myBlock[4], 20'u8, 0xE7D3FBC8'u32) - GG(a, b, c, d, myBlock[9], 5'u8, 0x21E1CDE6'u32) - GG(d, a, b, c, myBlock[14], 9'u8, 0xC33707D6'u32) - GG(c, d, a, b, myBlock[3], 14'u8, 0xF4D50D87'u32) - GG(b, c, d, a, myBlock[8], 20'u8, 0x455A14ED'u32) - GG(a, b, c, d, myBlock[13], 5'u8, 0xA9E3E905'u32) - GG(d, a, b, c, myBlock[2], 9'u8, 0xFCEFA3F8'u32) - GG(c, d, a, b, myBlock[7], 14'u8, 0x676F02D9'u32) - GG(b, c, d, a, myBlock[12], 20'u8, 0x8D2A4C8A'u32) - HH(a, b, c, d, myBlock[5], 4'u8, 0xFFFA3942'u32) - HH(d, a, b, c, myBlock[8], 11'u8, 0x8771F681'u32) - HH(c, d, a, b, myBlock[11], 16'u8, 0x6D9D6122'u32) - HH(b, c, d, a, myBlock[14], 23'u8, 0xFDE5380C'u32) - HH(a, b, c, d, myBlock[1], 4'u8, 0xA4BEEA44'u32) - HH(d, a, b, c, myBlock[4], 11'u8, 0x4BDECFA9'u32) - HH(c, d, a, b, myBlock[7], 16'u8, 0xF6BB4B60'u32) - HH(b, c, d, a, myBlock[10], 23'u8, 0xBEBFBC70'u32) - HH(a, b, c, d, myBlock[13], 4'u8, 0x289B7EC6'u32) - HH(d, a, b, c, myBlock[0], 11'u8, 0xEAA127FA'u32) - HH(c, d, a, b, myBlock[3], 16'u8, 0xD4EF3085'u32) - HH(b, c, d, a, myBlock[6], 23'u8, 0x04881D05'u32) - HH(a, b, c, d, myBlock[9], 4'u8, 0xD9D4D039'u32) - HH(d, a, b, c, myBlock[12], 11'u8, 0xE6DB99E5'u32) - HH(c, d, a, b, myBlock[15], 16'u8, 0x1FA27CF8'u32) - HH(b, c, d, a, myBlock[2], 23'u8, 0xC4AC5665'u32) - II(a, b, c, d, myBlock[0], 6'u8, 0xF4292244'u32) - II(d, a, b, c, myBlock[7], 10'u8, 0x432AFF97'u32) - II(c, d, a, b, myBlock[14], 15'u8, 0xAB9423A7'u32) - II(b, c, d, a, myBlock[5], 21'u8, 0xFC93A039'u32) - II(a, b, c, d, myBlock[12], 6'u8, 0x655B59C3'u32) - II(d, a, b, c, myBlock[3], 10'u8, 0x8F0CCC92'u32) - II(c, d, a, b, myBlock[10], 15'u8, 0xFFEFF47D'u32) - II(b, c, d, a, myBlock[1], 21'u8, 0x85845DD1'u32) - II(a, b, c, d, myBlock[8], 6'u8, 0x6FA87E4F'u32) - II(d, a, b, c, myBlock[15], 10'u8, 0xFE2CE6E0'u32) - II(c, d, a, b, myBlock[6], 15'u8, 0xA3014314'u32) - II(b, c, d, a, myBlock[13], 21'u8, 0x4E0811A1'u32) - II(a, b, c, d, myBlock[4], 6'u8, 0xF7537E82'u32) - II(d, a, b, c, myBlock[11], 10'u8, 0xBD3AF235'u32) - II(c, d, a, b, myBlock[2], 15'u8, 0x2AD7D2BB'u32) - II(b, c, d, a, myBlock[9], 21'u8, 0xEB86D391'u32) - state[0] = state[0] + a - state[1] = state[1] + b - state[2] = state[2] + c - state[3] = state[3] + d - -proc md5Init*(c: var MD5Context) {.raises: [], tags: [], gcsafe.} -proc md5Update*(c: var MD5Context, input: openArray[uint8]) {.raises: [], - tags: [], gcsafe.} -proc md5Final*(c: var MD5Context, digest: var MD5Digest) {.raises: [], tags: [], gcsafe.} - -proc md5Update*(c: var MD5Context, input: cstring, len: int) {.raises: [], - tags: [], gcsafe.} = - ## Updates the `MD5Context` with the `input` data of length `len`. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - md5Update(c, input.slice(0, len - 1)) - - -proc toMD5*(s: string): MD5Digest = - ## Computes the `MD5Digest` value for a string `s`. - ## - ## **See also:** - ## * `getMD5 proc <#getMD5,string>`_ which returns a string representation - ## of the `MD5Digest` - ## * `$ proc <#$,MD5Digest>`_ for converting MD5Digest to string - runnableExamples: - assert $toMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" - - var c: MD5Context - md5Init(c) - md5Update(c, s.slice(0, s.len - 1)) - md5Final(c, result) - -proc `$`*(d: MD5Digest): string = - ## Converts a `MD5Digest` value into its string representation. - const digits = "0123456789abcdef" - result = "" - for i in 0..15: - add(result, digits[(d[i].int shr 4) and 0xF]) - add(result, digits[d[i].int and 0xF]) - -proc getMD5*(s: string): string = - ## Computes an MD5 value of `s` and returns its string representation. - ## - ## **See also:** - ## * `toMD5 proc <#toMD5,string>`_ which returns the `MD5Digest` of a string - runnableExamples: - assert getMD5("abc") == "900150983cd24fb0d6963f7d28e17f72" - - var - c: MD5Context - d: MD5Digest - md5Init(c) - md5Update(c, s.slice(0, s.len - 1)) - md5Final(c, d) - result = $d - -proc `==`*(D1, D2: MD5Digest): bool = - ## Checks if two `MD5Digest` values are identical. - for i in 0..15: - if D1[i] != D2[i]: return false - return true - - -proc clearBuffer(c: var MD5Context) {.inline.} = - zeroMem(addr(c.buffer), sizeof(MD5Buffer)) - -proc md5Init*(c: var MD5Context) = - ## Initializes an `MD5Context`. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - c.state[0] = 0x67452301'u32 - c.state[1] = 0xEFCDAB89'u32 - c.state[2] = 0x98BADCFE'u32 - c.state[3] = 0x10325476'u32 - c.count[0] = 0'u32 - c.count[1] = 0'u32 - clearBuffer(c) - -proc writeBuffer(c: var MD5Context, index: int, - input: openArray[uint8], inputIndex, len: int) {.inline.} = - copyMem(addr(c.buffer[index]), unsafeAddr(input[inputIndex]), len) - -proc md5Update*(c: var MD5Context, input: openArray[uint8]) = - ## Updates the `MD5Context` with the `input` data. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - var Index = int((c.count[0] shr 3) and 0x3F) - c.count[0] = c.count[0] + (uint32(input.len) shl 3) - if c.count[0] < (uint32(input.len) shl 3): c.count[1] = c.count[1] + 1'u32 - c.count[1] = c.count[1] + (uint32(input.len) shr 29) - var PartLen = 64 - Index - if input.len >= PartLen: - writeBuffer(c, Index, input, 0, PartLen) - transform(c.buffer, c.state) - var i = PartLen - while i + 63 < input.len: - transform(input.slice(i, i + 63), c.state) - inc(i, 64) - if i < input.len: - writeBuffer(c, 0, input, i, input.len - i) - elif input.len > 0: - writeBuffer(c, Index, input, 0, input.len) - -proc md5Final*(c: var MD5Context, digest: var MD5Digest) = - ## Finishes the `MD5Context` and stores the result in `digest`. - ## - ## If you use the `toMD5 proc <#toMD5,string>`_, there's no need to call this - ## function explicitly. - var - Bits: MD5CBits - PadLen: int - decode(Bits, c.count) - var Index = int((c.count[0] shr 3) and 0x3F) - if Index < 56: PadLen = 56 - Index - else: PadLen = 120 - Index - md5Update(c, padding.slice(0, PadLen - 1)) - md5Update(c, Bits) - decode(digest, c.state) - clearBuffer(c) - - -when defined(nimHasStyleChecks): - {.pop.} #{.push styleChecks: off.} diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index e8f77a491..598959a37 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import intsets, tables, hashes, md5_old +import intsets, tables, hashes, md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages import ic / [packed_ast, ic] diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index a642d5cb3..3508a146a 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -9,7 +9,7 @@ ## Computes hash values for routine (proc, method etc) signatures. -import ast, tables, ropes, md5_old, modulegraphs +import ast, tables, ropes, md5, modulegraphs from hashes import Hash import types From b5f64f55d02a8ba980244596dcf310dd76b48fd8 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 1 Feb 2023 02:22:10 +0800 Subject: [PATCH 1566/1654] fixes #16790; fixes #19075; put big arrays on the constant seqs; don't inline them in the VM; big performance boost (#21318) * don't inline arrays in VM * add a test for #19075 --- compiler/vmgen.nim | 7 +++++-- tests/vm/t19075.nim | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/vm/t19075.nim diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bca961a63..bdf7b679e 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -32,7 +32,7 @@ when defined(nimPreviewSlimSystem): import std/assertions import - strutils, ast, types, msgs, renderer, vmdef, + strutils, ast, types, msgs, renderer, vmdef, trees, intsets, magicsys, options, lowerings, lineinfos, transf, astmsgs from modulegraphs import getBody @@ -2054,7 +2054,10 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = genLit(c, n, dest) of skConst: let constVal = if s.astdef != nil: s.astdef else: s.typ.n - gen(c, constVal, dest) + if dontInlineConstant(n, constVal): + genLit(c, constVal, dest) + else: + gen(c, constVal, dest) of skEnumField: # we never reach this case - as of the time of this comment, # skEnumField is folded to an int in semfold.nim, but this code diff --git a/tests/vm/t19075.nim b/tests/vm/t19075.nim new file mode 100644 index 000000000..89ca9cb19 --- /dev/null +++ b/tests/vm/t19075.nim @@ -0,0 +1,19 @@ +discard """ + timeout: 10 + joinable: false +""" + +# bug #19075 +const size = 50_000 + +const stuff = block: + var a: array[size, int] + a + +const zeugs = block: + var zeugs: array[size, int] + for i in 0.. Date: Wed, 1 Feb 2023 14:58:12 +0800 Subject: [PATCH 1567/1654] fixes #21317; 1.6.4 regression; etyBaseIndex should return fat pointers [backport 1.6] (#21320) fixes #21317; regression; etyBaseIndex should return fat pointers --- compiler/jsgen.nim | 7 ++++--- tests/system/trefs.nim | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/system/trefs.nim diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index d8fb6d57d..60399d3cd 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2484,17 +2484,18 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = if prc.typ[0] != nil and sfPure notin prc.flags: resultSym = prc.ast[resultPos].sym let mname = mangleName(p.module, resultSym) - let returnAddress = not isIndirect(resultSym) and + # otherwise uses "fat pointers" + let useRawPointer = not isIndirect(resultSym) and resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and mapType(p, resultSym.typ) == etyBaseIndex - if returnAddress: + if useRawPointer: resultAsgn = p.indentLine(("var $# = null;$n") % [mname]) resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname]) else: let resVar = createVar(p, resultSym.typ, isIndirect(resultSym)) resultAsgn = p.indentLine(("var $# = $#;$n") % [mname, resVar]) gen(p, prc.ast[resultPos], a) - if returnAddress: + if mapType(p, resultSym.typ) == etyBaseIndex: returnStmt = "return [$#, $#];$n" % [a.address, a.res] else: returnStmt = "return $#;$n" % [a.res] diff --git a/tests/system/trefs.nim b/tests/system/trefs.nim new file mode 100644 index 000000000..8c36aec52 --- /dev/null +++ b/tests/system/trefs.nim @@ -0,0 +1,15 @@ +discard """ + targets: "c js" +""" + +# bug #21317 +proc parseHook*(v: var ref int) = + var a: ref int + new(a) + a[] = 123 + v = a + +proc fromJson2*(): ref int = + parseHook(result) + +doAssert fromJson2()[] == 123 From ff8ab06720976f18d75257a22bf8e23873514c67 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:13:40 +0800 Subject: [PATCH 1568/1654] fixes #19396; Nimdoc hide nonexported fields (#21305) * suppresses non-exported fields of types and adds command-line option to re-enable this if desired * corrected the doctest that produced a CI error * an embarrassingly bad error in reasoning * modified a nimdoc test to reflect updated behavior * needed another change to bring utils.html doctest in sync with update * add info * fix nimdoc * lint * render postfix * fixes a problem * fixes nimdoc * fix nimdoc --------- Co-authored-by: johnperry-math Co-authored-by: johnperry-math --- compiler/commands.nim | 3 +++ compiler/docgen.nim | 22 +++++++++++-------- compiler/docgen2.nim | 2 +- compiler/options.nim | 4 ++++ compiler/renderer.nim | 13 ++++++----- compiler/vm.nim | 2 +- doc/docgen.md | 1 + nimdoc/extlinks/project/expected/main.html | 6 ++--- .../expected/subdir/subdir_b/utils.html | 6 ++--- nimdoc/testproject/expected/testproject.html | 12 ++++++++++ nimdoc/testproject/expected/testproject.idx | 1 + nimdoc/testproject/expected/theindex.html | 4 ++++ nimdoc/testproject/testproject.nim | 4 ++++ 13 files changed, 56 insertions(+), 24 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 59b221647..7c0f7ab0a 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1008,6 +1008,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; expectNoArg(conf, switch, arg, pass, info) conf.exc = low(ExceptionSystem) defineSymbol(conf.symbols, "noCppExceptions") + of "shownonexports": + expectNoArg(conf, switch, arg, pass, info) + showNonExportedFields(conf) of "exceptions": case arg.normalize of "cpp": conf.exc = excCpp diff --git a/compiler/docgen.nim b/compiler/docgen.nim index cab334e7a..8105cc807 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -1008,7 +1008,7 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol = if k == skType: result.symTypeKind = getTypeKind(n) -proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = +proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonExports: bool = false) = if (docFlags != kForceExport) and not isVisible(d, nameNode): return let name = getName(nameNode) @@ -1062,8 +1062,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) = tooltip = detailedName, langSym = rstLangSymbol, priority = symbolPriority(k), info = lineinfo) - nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments, - renderDocComments, renderSyms, renderExpandUsing}, symbolOrIdEnc) + let renderFlags = + if nonExports: {renderNoBody, renderNoComments, renderDocComments, renderSyms, + renderExpandUsing, renderNonExportedFields} + else: {renderNoBody, renderNoComments, renderDocComments, renderSyms, renderExpandUsing} + nodeToHighlightedHtml(d, n, result, renderFlags, symbolOrIdEnc) let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int) @@ -1311,12 +1314,13 @@ proc documentRaises*(cache: IdentCache; n: PNode) = if p5 != nil: n[pragmasPos].add p5 if p6 != nil: n[pragmasPos].add p6 -proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = +proc generateDoc*(d: PDoc, n, orig: PNode, config: ConfigRef, docFlags: DocFlags = kDefault) = ## Goes through nim nodes recursively and collects doc comments. ## Main function for `doc`:option: command, ## which is implemented in ``docgen2.nim``. template genItemAux(skind) = genItem(d, n, n[namePos], skind, docFlags) + let showNonExports = optShowNonExportedFields in config.globalOptions case n.kind of nkPragma: let pragmaNode = findPragma(n, wDeprecated) @@ -1343,20 +1347,20 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) = if n[i].kind != nkCommentStmt: # order is always 'type var let const': genItem(d, n[i], n[i][0], - succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags) + succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags, showNonExports) of nkStmtList: - for i in 0.. Types
        @@ -97,8 +96,7 @@ 
        A = object
-  x: int
-
        +
        diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html index f0f30536f..ee2618ab3 100644 --- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html +++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html @@ -59,8 +59,7 @@
        Types
          -
        • G
          • +
        • G
        • SomeType
          • @@ -254,8 +253,7 @@ Ref. 
          G[T] = object
-  val: T
-
          +
          diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 7d10d7900..45a53bb93 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -65,6 +65,8 @@ Circle, ## A circle Triangle, ## A three-sided shape Rectangle ## A four-sided shape">Shapes +
        • T19396
        @@ -399,6 +401,16 @@ Some shapes. +
        +
        +
        + 
        T19396 = object
+  a*: int
+
        +
        + + +
        diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index e69bedf33..46ffaee72 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -63,5 +63,6 @@ nim Triangle testproject.html#Triangle Shapes.Triangle 380 nim Rectangle testproject.html#Rectangle Shapes.Rectangle 380 nim Shapes testproject.html#Shapes enum Shapes 380 nim anything testproject.html#anything proc anything() 387 +nim T19396 testproject.html#T19396 object T19396 392 nimgrp bar testproject.html#bar-procs-all proc 31 nimgrp baz testproject.html#baz-procs-all proc 34 diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 7b4a95561..f9488eb2d 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -308,6 +308,10 @@
      • testproject: var someVariable
        • +
        T19396:
        testNimDocTrailingExample:
        • testproject: template testNimDocTrailingExample()
          • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index d236552ac..b5fa2ac37 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -388,3 +388,7 @@ when true: # issue #15184 ## ## There is no block quote after blank lines at the beginning. discard + +type T19396* = object # bug #19396 + a*: int + b: float From 900fe8f501838077a5a1ef3d6793a1e901083bf8 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Wed, 1 Feb 2023 20:00:10 +1100 Subject: [PATCH 1569/1654] Add `contains` to `std/macrocache` (#21304) * Add test cases * Implement contains for CacheSeq * Implement contains for CacheTable * Fix implementation of hasKey * Remove contains for CacheSeq Fix runnable examples I was accidently using --doccmd:skip so I didn't spot the failure locally * Implement hasKey as a VM callback instead of magic * Implement suggestions from PR Co-Authored-By: ringabout * Update lib/core/macrocache.nim --------- Co-authored-by: ringabout Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- compiler/vmops.nim | 6 ++++++ lib/core/macrocache.nim | 26 ++++++++++++++++++++++++++ tests/macros/tmacros_various.nim | 7 +++++++ 3 files changed, 39 insertions(+) diff --git a/compiler/vmops.nim b/compiler/vmops.nim index f1a2e1e69..f51280ec5 100644 --- a/compiler/vmops.nim +++ b/compiler/vmops.nim @@ -284,6 +284,12 @@ proc registerAdditionalOps*(c: PCtx) = stackTrace2(c, "isExported() requires a symbol. '$#' is of kind '$#'" % [$n, $n.kind], n) setResult(a, sfExported in n.sym.flags) + registerCallback c, "stdlib.macrocache.hasKey", proc (a: VmArgs) = + let + table = getString(a, 0) + key = getString(a, 1) + setResult(a, table in c.graph.cacheTables and key in c.graph.cacheTables[table]) + registerCallback c, "stdlib.vmutils.vmTrace", proc (a: VmArgs) = c.config.isVmTrace = getBool(a, 0) diff --git a/lib/core/macrocache.nim b/lib/core/macrocache.nim index d4b103793..ffd07abb1 100644 --- a/lib/core/macrocache.nim +++ b/lib/core/macrocache.nim @@ -181,6 +181,32 @@ proc `[]`*(t: CacheTable; key: string): NimNode {.magic: "NctGet".} = # get the NimNode back assert mcTable["toAdd"].kind == nnkStmtList +proc hasKey*(t: CacheTable; key: string): bool = + ## Returns true if `key` is in the table `t`. + ## + ## See also: + ## * [contains proc][contains(CacheTable, string)] for use with the `in` operator + runnableExamples: + import std/macros + const mcTable = CacheTable"hasKeyEx" + static: + assert not mcTable.hasKey("foo") + mcTable["foo"] = newEmptyNode() + # Will now be true since we inserted a value + assert mcTable.hasKey("foo") + discard "Implemented in vmops" + +proc contains*(t: CacheTable; key: string): bool {.inline.} = + ## Alias of [hasKey][hasKey(CacheTable, string)] for use with the `in` operator. + runnableExamples: + import std/macros + const mcTable = CacheTable"containsEx" + static: + mcTable["foo"] = newEmptyNode() + # Will be true since we gave it a value before + assert "foo" in mcTable + t.hasKey(key) + proc hasNext(t: CacheTable; iter: int): bool {.magic: "NctHasNext".} proc next(t: CacheTable; iter: int): (string, NimNode, int) {.magic: "NctNext".} diff --git a/tests/macros/tmacros_various.nim b/tests/macros/tmacros_various.nim index 08be4602c..2446912ea 100644 --- a/tests/macros/tmacros_various.nim +++ b/tests/macros/tmacros_various.nim @@ -351,3 +351,10 @@ block: # bug #15118 block: flop("b") + +static: + block: + const containsTable = CacheTable"containsTable" + doAssert "foo" notin containsTable + containsTable["foo"] = newLit 42 + doAssert "foo" in containsTable From 43b1b9d077c6a96580f202101e643fdcd34e30fb Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Wed, 1 Feb 2023 11:38:22 +0200 Subject: [PATCH 1570/1654] Fix the nimble build on Windows (#21314) Fix the build on Windows - `nimble install` fails on Windows, the `./` is not needed. --- nim.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nim.nimble b/nim.nimble index 5d1be49b1..713fc9256 100644 --- a/nim.nimble +++ b/nim.nimble @@ -9,6 +9,6 @@ skipDirs = @["build" , "changelogs" , "ci" , "csources_v2" , "drnim" , "nimdoc", before install: when defined(windows): - exec "./build_all.bat" + exec "build_all.bat" else: exec "./build_all.sh" From 17115cbc7375cdd47affd06dded62591d887441e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Feb 2023 23:44:14 +0800 Subject: [PATCH 1571/1654] fixes SSL version check logic [backport] (#21324) * fixed version check logic [backport] * add ciphersuites * debug nimble * fixes returns omission * finally * remove debug message * add ciphersuites --------- Co-authored-by: Araq --- lib/pure/asynchttpserver.nim | 4 ++-- lib/pure/net.nim | 6 +++--- lib/pure/unicode.nim | 2 +- lib/wrappers/openssl.nim | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 6694c4bc2..845875a1e 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -158,7 +158,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = proc sendStatus(client: AsyncSocket, status: string): Future[void] = client.send("HTTP/1.1 " & status & "\c\L\c\L") -func hasChunkedEncoding(request: Request): bool = +func hasChunkedEncoding(request: Request): bool = ## Searches for a chunked transfer encoding const transferEncoding = "Transfer-Encoding" @@ -300,7 +300,7 @@ proc processRequest( while true: lineFut.mget.setLen(0) lineFut.clean() - + # The encoding format alternates between specifying a number of bytes to read # and the data to be read, of the previously specified size if sizeOrData mod 2 == 0: diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 76a39226f..1dfef3b30 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -621,7 +621,7 @@ when defineSsl: proc newContext*(protVersion = protSSLv23, verifyMode = CVerifyPeer, certFile = "", keyFile = "", cipherList = CiphersIntermediate, - caDir = "", caFile = ""): SslContext = + caDir = "", caFile = "", ciphersuites = CiphersModern): SslContext = ## Creates an SSL context. ## ## Protocol version is currently ignored by default and TLS is used. @@ -675,10 +675,10 @@ when defineSsl: raiseSSLError() when not defined(openssl10) and not defined(libressl): let sslVersion = getOpenSSLVersion() - if sslVersion >= 0x010101000 and not sslVersion == 0x020000000: + if sslVersion >= 0x010101000 and sslVersion != 0x020000000: # In OpenSSL >= 1.1.1, TLSv1.3 cipher suites can only be configured via # this API. - if newCTX.SSL_CTX_set_ciphersuites(cipherList) != 1: + if newCTX.SSL_CTX_set_ciphersuites(ciphersuites) != 1: raiseSSLError() # Automatically the best ECDH curve for client exchange. Without this, ECDH # ciphers will be ignored by the server. diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 09233035f..c8d18831a 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -887,7 +887,7 @@ proc reversed*(s: openArray[char]): string = proc graphemeLen*(s: openArray[char]; i: Natural): Natural = ## The number of bytes belonging to byte index ``s[i]``, - ## including following combining code unit. + ## including following combining code units. runnableExamples: let a = "añyóng" doAssert a.graphemeLen(1) == 2 ## ñ diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index b3fe8a608..fcf52a8d9 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -467,10 +467,10 @@ else: raiseInvalidLibrary MainProc proc SSL_CTX_set_ciphersuites*(ctx: SslCtx, str: cstring): cint = - var theProc {.global.}: proc(ctx: SslCtx, str: cstring) {.cdecl, gcsafe.} + var theProc {.global.}: proc(ctx: SslCtx, str: cstring): cint {.cdecl, gcsafe.} if theProc.isNil: theProc = cast[typeof(theProc)](sslSymThrows("SSL_CTX_set_ciphersuites")) - theProc(ctx, str) + result = theProc(ctx, str) proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.} proc SSL_free*(ssl: SslPtr){.cdecl, dynlib: DLLSSLName, importc.} From 8e53fba083a7450b8c2e9771cba8d477468a520e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 3 Feb 2023 02:11:20 +0800 Subject: [PATCH 1572/1654] replaces `pairs` with `keys` and `items`; saves 8 bytes (#21319) replace `pairs` with `keys` and `items` --- compiler/condsyms.nim | 2 +- compiler/semfold.nim | 6 +++--- compiler/seminst.nim | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index c05aaf10b..a499b7142 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -25,7 +25,7 @@ proc undefSymbol*(symbols: StringTableRef; symbol: string) = # result = if isDefined(symbol): gSymbols[symbol] else: nil iterator definedSymbolNames*(symbols: StringTableRef): string = - for key, val in pairs(symbols): + for key in keys(symbols): yield key proc countDefinedSymbols*(symbols: StringTableRef): int = diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 7f966c6fd..07c3b3ae4 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -685,7 +685,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode n[0] = a of nkBracket, nkCurly: result = copyNode(n) - for i, son in n.pairs: + for son in n.items: var a = getConstExpr(m, son, idgen, g) if a == nil: return nil result.add a @@ -709,7 +709,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode # tuple constructor result = copyNode(n) if (n.len > 0) and (n[0].kind == nkExprColonExpr): - for i, expr in n.pairs: + for expr in n.items: let exprNew = copyNode(expr) # nkExprColonExpr exprNew.add expr[0] let a = getConstExpr(m, expr[1], idgen, g) @@ -717,7 +717,7 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode exprNew.add a result.add exprNew else: - for i, expr in n.pairs: + for expr in n.items: let a = getConstExpr(m, expr, idgen, g) if a == nil: return nil result.add a diff --git a/compiler/seminst.nim b/compiler/seminst.nim index bd5eb1ec3..80d455d1e 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -40,7 +40,7 @@ const iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym = internalAssert c.config, n.kind == nkGenericParams - for i, a in n.pairs: + for a in n.items: internalAssert c.config, a.kind == nkSym var q = a.sym if q.typ.kind in {tyTypeDesc, tyGenericParam, tyStatic, tyConcept}+tyTypeClasses: From 9f651f05d56407f63b9334917fecad975bdd49b4 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Tue, 7 Feb 2023 17:15:44 +0900 Subject: [PATCH 1573/1654] nimgrep: fix typo (#21337) occurences -> occurrences --- doc/nimgrep.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/nimgrep.md b/doc/nimgrep.md index 8fb86a9d3..63f760051 100644 --- a/doc/nimgrep.md +++ b/doc/nimgrep.md @@ -77,7 +77,7 @@ That means you can always use only 1 such an option with logical OR, e.g. .. Note:: If you want logical AND on patterns you should compose 1 appropriate pattern, possibly combined with multi-line mode `(?s)`:literal:. - E.g. to require that multi-line context of matches has occurences of + E.g. to require that multi-line context of matches has occurrences of **both** PAT1 and PAT2 use positive lookaheads (`(?=PAT)`:literal:): ```cmd nimgrep --inContext:'(?s)(?=.*PAT1)(?=.*PAT2)' From cbb6001d1e2e484986387e815a61339cd3dc916f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:27:44 +0800 Subject: [PATCH 1574/1654] fixes backticks in the documentation (#21340) --- lib/pure/collections/tables.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index cd08dcb9a..57a0107fd 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -2393,7 +2393,7 @@ proc contains*[A](t: CountTable[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTable[A], key: A; default: int = 0): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: @@ -2713,7 +2713,7 @@ proc contains*[A](t: CountTableRef[A], key: A): bool = return hasKey[A](t, key) proc getOrDefault*[A](t: CountTableRef[A], key: A, default: int): int = - ## Retrieves the value at `t[key]` if`key` is in `t`. Otherwise, the + ## Retrieves the value at `t[key]` if `key` is in `t`. Otherwise, the ## integer value of `default` is returned. ## ## See also: From 9fb4c2b3c765f993ba0055b281de19786a484c61 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 8 Feb 2023 20:55:01 +0100 Subject: [PATCH 1575/1654] fixes #21333; bad codegen for the at operator; [backport:1.6] (#21344) --- compiler/ccgexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 844a37997..fe7cb252e 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1647,6 +1647,7 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) = return if d.k == locNone: getTemp(p, n.typ, d) + initLocExpr(p, n[1], a) # generate call to newSeq before adding the elements per hand: let L = toInt(lengthOrd(p.config, n[1].typ)) if optSeqDestructors in p.config.globalOptions: @@ -1658,7 +1659,6 @@ proc genArrToSeq(p: BProc, n: PNode, d: var TLoc) = var lit = newRopeAppender() intLiteral(L, lit) genNewSeqAux(p, d, lit, L == 0) - initLocExpr(p, n[1], a) # bug #5007; do not produce excessive C source code: if L < 10: for i in 0.. Date: Thu, 9 Feb 2023 20:00:56 +0800 Subject: [PATCH 1576/1654] add `nimib` to important packages (#20697) r Co-authored-by: Clay Sweetser --- testament/important_packages.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index 90d12df4e..b3a4bfb82 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -105,6 +105,7 @@ pkg "nimfp", "nim c -o:nfp -r src/fp.nim" pkg "nimgame2", "nim c --mm:refc nimgame2/nimgame.nim" # XXX Doesn't work with deprecated 'randomize', will create a PR. pkg "nimgen", "nim c -o:nimgenn -r src/nimgen/runcfg.nim" +pkg "nimib" pkg "nimlsp" pkg "nimly", "nim c -r tests/test_readme_example.nim" pkg "nimongo", "nimble test_ci", allowFailure = true From 51f410e1d5c19e42e97057f0b88e87056b7c43e2 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 10 Feb 2023 05:14:39 +0800 Subject: [PATCH 1577/1654] megatest now checks refc too (#21341) * megatest now checks refc too * fixes refc --- testament/categories.nim | 2 ++ tests/enum/tenum.nim | 3 ++- tests/misc/t18079.nim | 4 ++++ tests/misc/tunsignedconv.nim | 3 ++- tests/typerel/ttypedesc_as_genericparam1_orc.nim | 6 +++++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index e8b13746a..cae699327 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -718,6 +718,8 @@ proc processCategory(r: var TResults, cat: Category, case cat2 of "megatest": runJoinedTest(r, cat, testsDir, options) + if isNimRepoTests(): + runJoinedTest(r, cat, testsDir, options & " --mm:refc") else: var testsRun = 0 var files: seq[string] diff --git a/tests/enum/tenum.nim b/tests/enum/tenum.nim index b11b02ec6..88d85ddcc 100644 --- a/tests/enum/tenum.nim +++ b/tests/enum/tenum.nim @@ -174,4 +174,5 @@ block: # bug #12589 proc typ(): OGRwkbGeometryType = return wkbPoint25D - doAssert $typ() == "wkbPoint25D" + when not defined(gcRefc): + doAssert $typ() == "wkbPoint25D" diff --git a/tests/misc/t18079.nim b/tests/misc/t18079.nim index 352348daf..ae64bbff9 100644 --- a/tests/misc/t18079.nim +++ b/tests/misc/t18079.nim @@ -1,3 +1,7 @@ +discard """ + matrix: "--mm:orc" +""" + type Foo = object y: int diff --git a/tests/misc/tunsignedconv.nim b/tests/misc/tunsignedconv.nim index 989f39277..c32f85b4d 100644 --- a/tests/misc/tunsignedconv.nim +++ b/tests/misc/tunsignedconv.nim @@ -66,7 +66,8 @@ let limit = 1'u64 let rangeVar = 0'u64 ..< limit -doAssert repr(rangeVar) == """0 .. 0""", repr(rangeVar) +when not defined(gcRefc): + doAssert repr(rangeVar) == """0 .. 0""", repr(rangeVar) # bug #15210 diff --git a/tests/typerel/ttypedesc_as_genericparam1_orc.nim b/tests/typerel/ttypedesc_as_genericparam1_orc.nim index 0ee4d8f92..d528a7421 100644 --- a/tests/typerel/ttypedesc_as_genericparam1_orc.nim +++ b/tests/typerel/ttypedesc_as_genericparam1_orc.nim @@ -1 +1,5 @@ -doAssert repr(int) == "int" \ No newline at end of file +discard """ + matrix: "--mm:orc" +""" + +doAssert repr(int) == "int" From 642136ec4f2cd93cdd753bf16fa4aec89b8fee82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20M=20G=C3=B3mez?= Date: Thu, 9 Feb 2023 21:19:05 +0000 Subject: [PATCH 1578/1654] Adds an extra optional argument to vcc: `vctoolset` Fixes #21335 (#21336) --- tools/vccexe/vccexe.nim | 15 +++++++++++---- tools/vccexe/vcvarsall.nim | 7 ++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/vccexe/vccexe.nim b/tools/vccexe/vccexe.nim index abe68c0a0..d548666b9 100644 --- a/tools/vccexe/vccexe.nim +++ b/tools/vccexe/vccexe.nim @@ -41,6 +41,7 @@ const platformPrefix = "--platform" sdktypePrefix = "--sdktype" sdkversionPrefix = "--sdkversion" + vctoolsetPrefix = "--vctoolset" verbosePrefix = "--verbose" vccversionSepIdx = vccversionPrefix.len @@ -49,6 +50,7 @@ const platformSepIdx = platformPrefix.len sdktypeSepIdx = sdktypePrefix.len sdkversionSepIdx = sdkversionPrefix.len + vctoolsetSepIdx = vctoolsetPrefix.len vcvarsallDefaultPath = "vcvarsall.bat" @@ -97,6 +99,8 @@ Options: "8.1" to use the windows 8.1 SDK --verbose Echoes the command line for loading the Developer Command Prompt and the command line passed on to the secondary command. + --vctoolset Optionally specifies the Visual Studio compiler toolset to use. + By default, the environment is set to use the current Visual Studio compiler toolset. Other command line arguments are passed on to the secondary command specified by --command or to the @@ -108,7 +112,7 @@ proc parseVccexeCmdLine(argseq: seq[string], vccversionArg: var seq[string], printPathArg: var bool, vcvarsallArg: var string, commandArg: var string, noCommandArg: var bool, platformArg: var VccArch, sdkTypeArg: var VccPlatformType, - sdkVersionArg: var string, verboseArg: var bool, + sdkVersionArg: var string, vctoolsetArg: var string, verboseArg: var bool, clArgs: var seq[string]) = ## Cannot use usual command-line argument parser here ## Since vccexe command-line arguments are intermingled @@ -125,7 +129,7 @@ proc parseVccexeCmdLine(argseq: seq[string], responseargs = parseCmdLine(responsecontent) parseVccexeCmdLine(responseargs, vccversionArg, printPathArg, vcvarsallArg, commandArg, noCommandArg, platformArg, sdkTypeArg, - sdkVersionArg, verboseArg, clArgs) + sdkVersionArg, vctoolsetArg, verboseArg, clArgs) elif wargv.startsWith(vccversionPrefix): # Check for vccversion vccversionArg.add(wargv.substr(vccversionSepIdx + 1)) elif wargv.cmpIgnoreCase(printPathPrefix) == 0: # Check for printPath @@ -142,6 +146,8 @@ proc parseVccexeCmdLine(argseq: seq[string], sdkTypeArg = parseEnum[VccPlatformType](wargv.substr(sdktypeSepIdx + 1)) elif wargv.startsWith(sdkversionPrefix): # Check for sdkversion sdkVersionArg = wargv.substr(sdkversionSepIdx + 1) + elif wargv.startsWith(vctoolsetPrefix): # Check for vctoolset + vctoolsetArg = wargv.substr(vctoolsetSepIdx + 1) elif wargv.startsWith(verbosePrefix): verboseArg = true else: # Regular cl.exe argument -> store for final cl.exe invocation @@ -158,13 +164,14 @@ when isMainModule: var platformArg: VccArch var sdkTypeArg: VccPlatformType var sdkVersionArg: string + var vctoolsetArg: string var verboseArg: bool = false var clArgs: seq[string] = @[] let wrapperArgs = commandLineParams() parseVccexeCmdLine(wrapperArgs, vccversionArg, printPathArg, vcvarsallArg, - commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, + commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg, clArgs) @@ -195,7 +202,7 @@ when isMainModule: echo "$1: $2" % [head, vcvarsallArg] # Call vcvarsall to get the appropriate VCC process environment - var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, verboseArg) + var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg) if vcvars != nil: for vccEnvKey, vccEnvVal in vcvars: putEnv(vccEnvKey, vccEnvVal) diff --git a/tools/vccexe/vcvarsall.nim b/tools/vccexe/vcvarsall.nim index 29d13cc7e..73b103e3c 100644 --- a/tools/vccexe/vcvarsall.nim +++ b/tools/vccexe/vcvarsall.nim @@ -33,7 +33,7 @@ type vccplatUWP = "uwp", ## Universal Windows Platform (UWP) Application vccplatOneCore = "onecore" # Undocumented platform type in the Windows SDK, probably XBox One SDK platform type. -proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version: string = "", verbose: bool = false): StringTableRef = +proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version, vctoolset: string = "", verbose: bool = false): StringTableRef = ## Returns a string table containing the proper process environment to successfully execute VCC compile commands for the specified SDK version, CPU architecture and platform type. ## ## path @@ -44,6 +44,8 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type ## The compile target Platform Type. Defaults to the Windows Desktop platform, i.e. a regular Windows executable binary. ## sdk_version ## The Windows SDK version to use. + ## vctoolset + ## Visual Studio compiler toolset to use. ## verbose ## Echo the command-line passed on to the system to load the VCC environment. Defaults to `false`. @@ -63,6 +65,9 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type if sdk_version.len > 0: args.add(sdk_version) + + if vctoolset.len > 0: + args.add("-vcvars_ver="&vctoolset) let argStr = args.join " " From 6c15958a835b645f3acdc3d1d012b10d0df43454 Mon Sep 17 00:00:00 2001 From: Gabriel Huber Date: Sun, 12 Feb 2023 16:41:27 +0100 Subject: [PATCH 1579/1654] Change the default stdlib location for Linux packaging (#21328) * Correctly evaluate the Nim prefix on Posix * Document new packaging layout --- compiler/options.nim | 22 ++++++++++++---------- doc/packaging.md | 7 +++++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 833d86a16..862b42446 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -717,22 +717,24 @@ proc getPrefixDir*(conf: ConfigRef): AbsoluteDir = ## clone or using installed nim, so that these exist: `result/doc/advopt.txt` ## and `result/lib/system.nim` if not conf.prefixDir.isEmpty: result = conf.prefixDir - else: result = AbsoluteDir splitPath(getAppDir()).head + else: + let binParent = AbsoluteDir splitPath(getAppDir()).head + when defined(posix): + if binParent == AbsoluteDir"/usr": + result = AbsoluteDir"/usr/lib/nim" + elif binParent == AbsoluteDir"/usr/local": + result = AbsoluteDir"/usr/local/lib/nim" + else: + result = binParent + else: + result = binParent proc setDefaultLibpath*(conf: ConfigRef) = # set default value (can be overwritten): if conf.libpath.isEmpty: # choose default libpath: var prefix = getPrefixDir(conf) - when defined(posix): - if prefix == AbsoluteDir"/usr": - conf.libpath = AbsoluteDir"/usr/lib/nim" - elif prefix == AbsoluteDir"/usr/local": - conf.libpath = AbsoluteDir"/usr/local/lib/nim" - else: - conf.libpath = prefix / RelativeDir"lib" - else: - conf.libpath = prefix / RelativeDir"lib" + conf.libpath = prefix / RelativeDir"lib" # Special rule to support other tools (nimble) which import the compiler # modules and make use of them. diff --git a/doc/packaging.md b/doc/packaging.md index 9a1cc0f6e..522ca6d44 100644 --- a/doc/packaging.md +++ b/doc/packaging.md @@ -69,8 +69,11 @@ Hints on the build process: What to install: -- The expected stdlib location is /usr/lib/nim -- Global configuration files under /etc/nim +- The expected stdlib location is `/usr/lib/nim/lib`, previously it was just `/usr/lib/nim` +- `nimdoc.css` and `nimdoc.cls` from the `doc` folder should go into `/usr/lib/nim/doc/` +- `tools/nim-gdb.py` should go into `/usr/lib/nim/tools/` +- `tools/dochack/dochack.js` should be installed to `/usr/lib/nim/tools/dochack/` +- Global configuration files under `/etc/nim` - Optionally: manpages, documentation, shell completion - When installing documentation, .idx files are not required - The "compiler" directory contains compiler sources and should not be part of the compiler binary package From 684c3b3aeb90ae549382ed0f9c3286464c72809f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:00:29 +0800 Subject: [PATCH 1580/1654] fixes comments about type bound operations (#21365) --- compiler/liftdestructors.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 2b27a58fc..738f659b7 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -8,7 +8,7 @@ # ## This module implements lifting for type-bound operations -## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``). +## (``=sink``, ``=copy``, ``=destroy``, ``=deepCopy``). import modulegraphs, lineinfos, idents, ast, renderer, semdata, sighashes, lowerings, options, types, msgs, magicsys, tables, ccgutils From 406d3021314c66fdf4c715efeeaea3c8f478fcaa Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:29:30 +0800 Subject: [PATCH 1581/1654] allow omitting stmts using `finally` as post expr blocks; make it consistent with `else`, `except` etc. (#21361) allow omitting stmts using `finally` as post expr blocks --- compiler/parser.nim | 2 +- tests/parser/tpostexprblocks.nim | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 14882d415..abfc7b323 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -1496,7 +1496,7 @@ proc postExprBlocks(p: var Parser, x: PNode): PNode = result = makeCall(result) getTok(p) skipComment(p, result) - if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}: + if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept, tkFinally}: var stmtList = newNodeP(nkStmtList, p) stmtList.add parseStmt(p) # to keep backwards compatibility (see tests/vm/tstringnil) diff --git a/tests/parser/tpostexprblocks.nim b/tests/parser/tpostexprblocks.nim index d272c712f..6cd4a8350 100644 --- a/tests/parser/tpostexprblocks.nim +++ b/tests/parser/tpostexprblocks.nim @@ -498,6 +498,13 @@ StmtList StmtList DiscardStmt Empty + + Call + Ident "foo" + Finally + StmtList + DiscardStmt + Empty ''' """ @@ -655,3 +662,7 @@ dumpTree: discard finally: discard + + foo: + finally: + discard From 1d06c2b6cf84bcf65b67e88667dba4fd5e8ba519 Mon Sep 17 00:00:00 2001 From: c-blake Date: Tue, 14 Feb 2023 02:00:30 -0500 Subject: [PATCH 1582/1654] This adds `parseutils.parseSize`, an inverse to `strutils.formatSize` (#21349) * This adds `parseutils.parseSize`, an inverse to `strutils.formatSize` which has existed since 2017. It is useful for parsing the compiler's own output logs (like SuccessX) or many other scenarios where "human readable" units have been chosen. The doc comment and tests explain accepted syntax in detail. Big units lead to small numbers, often with a fractional part, but we parse into an `int64` since that is what `formatSize` stringifies and this is an inverse over partial function slots. Although metric prefixes z & y for zettabyte & yottabyte are accepted, these will saturate the result at `int64.high` unless the qualified number is a small fraction. This should not be much of a problem until such sizes are common (at which point another overload with the parse result either `float64` or `int128` could be added). Tests avoids `test()` because of a weakly related static: test() failure as mentioned in https://github.com/nim-lang/Nim/pull/21325. This is a more elemental VM failure. As such, it needs its own failure exhibition issue that is a smaller test case. (I am working on that, but unless there is a burning need to `parseSize` at compile-time before run-time it need not hold up this PR.) * This worked with `int` but fails with `int64`. Try for green tests. * Lift 2-result matching into a `checkParseSize` template and format as a table of input & 2 expected outputs which seems nicer and to address https://github.com/nim-lang/Nim/pull/21349#pullrequestreview-1294407679 * Fix (probably) the i386 trouble by using `int64` consistently. * Improve documentation by mentioning saturation. * Improve documentation with `runnableExamples` and a little more detail in the main doc comment based on excellent code review by @juancarlospaco: https://github.com/nim-lang/Nim/pull/21349#pullrequestreview-1294564155 * Address some more @juancarlospaco code review concerns. * Remove a stray space. * Mention milli-bytes in docs to maybe help clarify why wild conventions are so prone to going case-insensitive-metric. * Add some parens. --- changelog.md | 2 +- lib/pure/parseutils.nim | 65 ++++++++++++++++++++++++++++++++++++ tests/stdlib/tparseutils.nim | 41 +++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 0389faad9..2db577d0b 100644 --- a/changelog.md +++ b/changelog.md @@ -10,7 +10,7 @@ [//]: # "Additions:" - +- Added `parseutils.parseSize` - inverse to `strutils.formatSize` - to parse human readable sizes. [//]: # "Deprecations:" diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 2bb23c626..c4d59d892 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -592,6 +592,71 @@ proc parseFloat*(s: openArray[char], number: var float): int {. if result != 0: number = bf +func toLowerAscii(c: char): char = + if c in {'A'..'Z'}: char(uint8(c) xor 0b0010_0000'u8) else: c + +func parseSize*(s: openArray[char], size: var int64, alwaysBin=false): int = + ## Parse a size qualified by binary or metric units into `size`. This format + ## is often called "human readable". Result is the number of processed chars + ## or 0 on parse errors and size is rounded to the nearest integer. Trailing + ## garbage like "/s" in "1k/s" is allowed and detected by `result < s.len`. + ## + ## To simplify use, following non-rare wild conventions, and since fractional + ## data like milli-bytes is so rare, unit matching is case-insensitive but for + ## the 'i' distinguishing binary-metric from metric (which cannot be 'I'). + ## + ## An optional trailing 'B|b' is ignored but processed. I.e., you must still + ## know if units are bytes | bits or infer this fact via the case of s[^1] (if + ## users can even be relied upon to use 'B' for byte and 'b' for bit or have + ## that be s[^1]). + ## + ## If `alwaysBin==true` then scales are always binary-metric, but e.g. "KiB" + ## is still accepted for clarity. If the value would exceed the range of + ## `int64`, `size` saturates to `int64.high`. Supported metric prefix chars + ## include k, m, g, t, p, e, z, y (but z & y saturate unless the number is a + ## small fraction). + ## + ## **See also:** + ## * https://en.wikipedia.org/wiki/Binary_prefix + ## * `formatSize module`_ for formatting + runnableExamples: + var res: int64 # caller must still know if 'b' refers to bytes|bits + doAssert parseSize("10.5 MB", res) == 7 + doAssert res == 10_500_000 # decimal metric Mega prefix + doAssert parseSize("64 mib", res) == 6 + doAssert res == 67108864 # 64 shl 20 + doAssert parseSize("1G/h", res, true) == 2 # '/' stops parse + doAssert res == 1073741824 # 1 shl 30, forced binary metric + const prefix = "b" & "kmgtpezy" # byte|bit & lowCase metric-ish prefixes + const scaleM = [1.0, 1e3, 1e6, 1e9, 1e12, 1e15, 1e18, 1e21, 1e24] # 10^(3*idx) + const scaleB = [1.0, 1024, 1048576, 1073741824, 1099511627776.0, # 2^(10*idx) + 1125899906842624.0, 1152921504606846976.0, # ldexp? + 1.180591620717411303424e21, 1.208925819614629174706176e24] + var number: float + var scale = 1.0 + result = parseFloat(s, number) + if number < 0: # While parseFloat accepts negatives .. + result = 0 #.. we do not since sizes cannot be < 0 + if result > 0: + let start = result # Save spot to maybe unwind white to EOS + while result < s.len and s[result] in Whitespace: + inc result + if result < s.len: # Illegal starting char => unity + if (let si = prefix.find(s[result].toLowerAscii); si >= 0): + inc result # Now parse the scale + scale = if alwaysBin: scaleB[si] else: scaleM[si] + if result < s.len and s[result] == 'i': + scale = scaleB[si] # Switch from default to binary-metric + inc result + if result < s.len and s[result].toLowerAscii == 'b': + inc result # Skip optional '[bB]' + else: # Unwind result advancement when there.. + result = start #..is no unit to the end of `s`. + var sizeF = number * scale + 0.5 # Saturate to int64.high when too big + size = if sizeF > 9223372036854774784.0: int64.high else: sizeF.int64 +# Above constant=2^63-1024 avoids C UB; github.com/nim-lang/Nim/issues/20102 or +# stackoverflow.com/questions/20923556/math-pow2-63-1-math-pow2-63-512-is-true + type InterpolatedKind* = enum ## Describes for `interpolatedFragments` ## which part of the interpolated string is diff --git a/tests/stdlib/tparseutils.nim b/tests/stdlib/tparseutils.nim index 102e74067..218dd08f6 100644 --- a/tests/stdlib/tparseutils.nim +++ b/tests/stdlib/tparseutils.nim @@ -55,5 +55,46 @@ proc test() = doAssert res == @[(17, "9.123456789012344"), (18, "11.123456789012344"), (17, "9.123456789012344"), (17, "8.123456789012344"), (16, "9.12345678901234"), (17, "9.123456789012344")] + test() static: test() + +block: # With this included, static: test() crashes the compiler (from a + # VM problem with parseSize calling parseFloat). + var sz: int64 + template checkParseSize(s, expectLen, expectVal) = + if (let got = parseSize(s, sz); got != expectLen): + raise newException(IOError, "got len " & $got & " != " & $expectLen) + if sz != expectVal: + raise newException(IOError, "got sz " & $sz & " != " & $expectVal) + # STRING LEN SZ + # Good, complete parses + checkParseSize "1 b" , 4, 1 + checkParseSize "1 B" , 4, 1 + checkParseSize "1k" , 2, 1000 + checkParseSize "1 kib" , 5, 1024 + checkParseSize "1 ki" , 4, 1024 + checkParseSize "1mi" , 3, 1048576 + checkParseSize "1 mi" , 4, 1048576 + checkParseSize "1 mib" , 5, 1048576 + checkParseSize "1 Mib" , 5, 1048576 + checkParseSize "1 MiB" , 5, 1048576 + checkParseSize "1.23GiB", 7, 1320702444 # 1320702443.52 rounded + checkParseSize "0.001k" , 6, 1 + checkParseSize "0.0004k", 7, 0 + checkParseSize "0.0006k", 7, 1 + # Incomplete parses + checkParseSize "1 " , 1, 1 # Trailing white IGNORED + checkParseSize "1 B " , 4, 1 # Trailing white IGNORED + checkParseSize "1 B/s" , 4, 1 # Trailing junk IGNORED + checkParseSize "1 kX" , 3, 1000 + checkParseSize "1 kiX" , 4, 1024 + checkParseSize "1j" , 1, 1 # Unknown prefix IGNORED + checkParseSize "1 jib" , 2, 1 # Unknown prefix post space + checkParseSize "1 ji" , 3, 1 + # Bad parses; `sz` should stay last good|incomplete value + checkParseSize "-1b" , 0, 1 # Negative numbers + checkParseSize "abc" , 0, 1 # Non-numeric + checkParseSize " 12" , 0, 1 # Leading white + # Value Edge cases + checkParseSize "9223372036854775807", 19, int64.high From cac49694c011e851707cb3a86a3263c741559b3d Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 14 Feb 2023 18:14:19 +1100 Subject: [PATCH 1583/1654] `std/asyncjs` allow transforming proc types (#21356) * Add test case * Implement JS async transform for nnkProcTy --- lib/js/asyncjs.nim | 10 +++++++++- tests/js/tasyncjs.nim | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/js/asyncjs.nim b/lib/js/asyncjs.nim index 364f7a21a..8e2f85156 100644 --- a/lib/js/asyncjs.nim +++ b/lib/js/asyncjs.nim @@ -100,10 +100,18 @@ proc isFutureVoid(node: NimNode): bool = node[1].kind == nnkIdent and $node[1] == "void" proc generateJsasync(arg: NimNode): NimNode = - if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo}: + if arg.kind notin {nnkProcDef, nnkLambda, nnkMethodDef, nnkDo, nnkProcTy}: error("Cannot transform this node kind into an async proc." & " proc/method definition or lambda node expected.") + # Transform type X = proc (): something {.async.} + # into type X = proc (): Future[something] + if arg.kind == nnkProcTy: + result = arg + if arg[0][0].kind == nnkEmpty: + result[0][0] = quote do: Future[void] + return result + result = arg var isVoid = false let jsResolve = ident("jsResolve") diff --git a/tests/js/tasyncjs.nim b/tests/js/tasyncjs.nim index 00753a16c..3de143643 100644 --- a/tests/js/tasyncjs.nim +++ b/tests/js/tasyncjs.nim @@ -94,4 +94,9 @@ proc main() {.async.} = doAssert "foobar: 7" in $reason.message echo "done" # justified here to make sure we're running this, since it's inside `async` +block asyncPragmaInType: + type Handler = proc () {.async.} + proc foo() {.async.} = discard + var x: Handler = foo + discard main() From 56a4d246a4288a7005de0b20e312fa4e4af89534 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 14 Feb 2023 20:11:14 +0800 Subject: [PATCH 1584/1654] Refines raises list in osproc (#21323) * Remove Exception from raises in closeImpl * Update osproc.nim * refine errors * add ValueError * cast raises * refactor raises lists * Update lib/pure/osproc.nim * Update lib/pure/osproc.nim --------- Co-authored-by: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Co-authored-by: Andreas Rumpf --- lib/pure/osproc.nim | 41 ++++++++++++++++++++------------------- lib/pure/streams.nim | 2 +- lib/std/socketstreams.nim | 2 +- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 5e130b47a..9a0e5895d 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -76,7 +76,7 @@ type proc execProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut, poUsePath, poEvalCommand}): - string {.rtl, extern: "nosp$1", + string {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect].} ## A convenience procedure that executes ``command`` with ``startProcess`` ## and returns its output as a string. @@ -120,7 +120,7 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", proc startProcess*(command: string, workingDir: string = "", args: openArray[string] = [], env: StringTableRef = nil, options: set[ProcessOption] = {poStdErrToStdOut}): - owned(Process) {.rtl, extern: "nosp$1", + owned(Process) {.rtl, extern: "nosp$1", raises: [OSError, IOError], tags: [ExecIOEffect, ReadEnvEffect, RootEffect].} ## Starts a process. `Command` is the executable file, `workingDir` is the ## process's working directory. If ``workingDir == ""`` the current directory @@ -152,7 +152,7 @@ proc startProcess*(command: string, workingDir: string = "", ## <#execProcess,string,string,openArray[string],StringTableRef,set[ProcessOption]>`_ ## * `execCmd proc <#execCmd,string>`_ -proc close*(p: Process) {.rtl, extern: "nosp$1", tags: [WriteIOEffect].} +proc close*(p: Process) {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [WriteIOEffect].} ## When the process has finished executing, cleanup related handles. ## ## .. warning:: If the process has not finished executing, this will forcibly @@ -201,7 +201,7 @@ proc kill*(p: Process) {.rtl, extern: "nosp$1", tags: [].} ## * `terminate proc <#terminate,Process>`_ ## * `posix_utils.sendSignal(pid: Pid, signal: int) `_ -proc running*(p: Process): bool {.rtl, extern: "nosp$1", tags: [].} +proc running*(p: Process): bool {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Returns true if the process `p` is still running. Returns immediately. proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = @@ -212,7 +212,7 @@ proc processID*(p: Process): int {.rtl, extern: "nosp$1".} = return p.id proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, - extern: "nosp$1", tags: [].} + extern: "nosp$1", raises: [OSError, ValueError], tags: [].} ## Waits for the process to finish and returns `p`'s error code. ## ## .. warning:: Be careful when using `waitForExit` for processes created without @@ -221,7 +221,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. -proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].} +proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. ## ## On posix, if the process has exited because of a signal, 128 + signal @@ -237,7 +237,7 @@ proc inputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} ## * `outputStream proc <#outputStream,Process>`_ ## * `errorStream proc <#errorStream,Process>`_ -proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [].} +proc outputStream*(p: Process): Stream {.rtl, extern: "nosp$1", raises: [IOError, OSError], tags: [].} ## Returns ``p``'s output stream for reading from. ## ## You cannot perform peek/write/setOption operations to this stream. @@ -289,7 +289,7 @@ proc peekableErrorStream*(p: Process): Stream {.rtl, extern: "nosp$1", tags: [], ## * `errorStream proc <#errorStream,Process>`_ ## * `peekableOutputStream proc <#peekableOutputStream,Process>`_ -proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", +proc inputHandle*(p: Process): FileHandle {.rtl, raises: [], extern: "nosp$1", tags: [].} = ## Returns ``p``'s input file handle for writing to. ## @@ -302,7 +302,7 @@ proc inputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.inHandle proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s output file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -314,7 +314,7 @@ proc outputHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", result = p.outHandle proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", - tags: [].} = + raises: [], tags: [].} = ## Returns ``p``'s error file handle for reading from. ## ## .. warning:: The returned `FileHandle` should not be closed manually as @@ -325,7 +325,7 @@ proc errorHandle*(p: Process): FileHandle {.rtl, extern: "nosp$1", ## * `outputHandle proc <#outputHandle,Process>`_ result = p.errHandle -proc countProcessors*(): int {.rtl, extern: "nosp$1".} = +proc countProcessors*(): int {.rtl, extern: "nosp$1", raises: [].} = ## Returns the number of the processors/cores the machine has. ## Returns 0 if it cannot be detected. ## It is implemented just calling `cpuinfo.countProcessors`. @@ -339,6 +339,7 @@ proc execProcesses*(cmds: openArray[string], beforeRunEvent: proc(idx: int) = nil, afterRunEvent: proc(idx: int, p: Process) = nil): int {.rtl, extern: "nosp$1", + raises: [ValueError, OSError, IOError], tags: [ExecIOEffect, TimeEffect, ReadEnvEffect, RootEffect], effectsOf: [beforeRunEvent, afterRunEvent].} = ## Executes the commands `cmds` in parallel. @@ -452,7 +453,7 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: [ReadIOEffect].} = +iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), raises: [OSError, IOError, ValueError], tags: [ReadIOEffect].} = ## Convenience iterator for working with `startProcess` to read data from a ## background process. ## @@ -481,7 +482,8 @@ iterator lines*(p: Process, keepNewLines = false): string {.since: (1, 3), tags: yield line discard waitForExit(p) -proc readLines*(p: Process): (seq[string], int) {.since: (1, 3).} = +proc readLines*(p: Process): (seq[string], int) {.since: (1, 3), + raises: [OSError, IOError, ValueError], tags: [ReadIOEffect].} = ## Convenience function for working with `startProcess` to read data from a ## background process. ## @@ -747,7 +749,7 @@ when defined(windows) and not defined(useNimRtl): const errFileNotFound = 2.int if lastError.int in {errInvalidParameter, errFileNotFound}: raiseOSError(lastError, - "Requested command not found: '$1'. OS error:" % command) + "Requested command not found: '" & command & "'. OS error:") else: raiseOSError(lastError, command) result.fProcessHandle = procInfo.hProcess @@ -955,13 +957,13 @@ elif not defined(useNimRtl): not defined(useClone) and not defined(linux) when useProcessAuxSpawn: proc startProcessAuxSpawn(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} else: proc startProcessAuxFork(data: StartProcessData): Pid {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.} {.push stacktrace: off, profiler: off.} proc startProcessAfterFork(data: ptr StartProcessData) {. - tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} + raises: [OSError], tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.} {.pop.} proc startProcess(command: string, workingDir: string = "", @@ -1131,8 +1133,7 @@ elif not defined(useNimRtl): let sizeRead = read(data.pErrorPipe[readIdx], addr error, sizeof(error)) if sizeRead == sizeof(error): raiseOSError(osLastError(), - "Could not find command: '$1'. OS error: $2" % - [$data.sysCommand, $strerror(error)]) + "Could not find command: '" & $data.sysCommand & "'. OS error: " & $strerror(error)) return pid @@ -1578,7 +1579,7 @@ proc execCmdEx*(command: string, options: set[ProcessOption] = { poStdErrToStdOut, poUsePath}, env: StringTableRef = nil, workingDir = "", input = ""): tuple[ output: string, - exitCode: int] {.tags: + exitCode: int] {.raises: [OSError, IOError], tags: [ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} = ## A convenience proc that runs the `command`, and returns its `output` and ## `exitCode`. `env` and `workingDir` params behave as for `startProcess`. diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 945e85934..003f8ca4c 100644 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -115,7 +115,7 @@ type ## * That these fields here shouldn't be used directly. ## They are accessible so that a stream implementation can override them. closeImpl*: proc (s: Stream) - {.nimcall, raises: [Exception, IOError, OSError], tags: [WriteIOEffect], gcsafe.} + {.nimcall, raises: [IOError, OSError], tags: [WriteIOEffect], gcsafe.} atEndImpl*: proc (s: Stream): bool {.nimcall, raises: [Defect, IOError, OSError], tags: [], gcsafe.} setPositionImpl*: proc (s: Stream, pos: int) diff --git a/lib/std/socketstreams.nim b/lib/std/socketstreams.nim index 5c882858d..b53d6a5d3 100644 --- a/lib/std/socketstreams.nim +++ b/lib/std/socketstreams.nim @@ -146,7 +146,7 @@ proc wsFlush(s: Stream) = s.lastFlush = s.buf.len proc rsClose(s: Stream) = - {.cast(tags: []).}: + {.cast(raises: [IOError, OSError]), cast(tags: []).}: # todo fixme maybe do something? var s = ReadSocketStream(s) s.data.close() From e24d0e5faf1f6f2906262457007261e2358f1e5d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 15 Feb 2023 20:03:11 +0800 Subject: [PATCH 1585/1654] remove legacy code; the first iteration now can build Nim with cpp backend (#21373) --- koch.nim | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/koch.nim b/koch.nim index 27953e05a..fa498e017 100644 --- a/koch.nim +++ b/koch.nim @@ -295,11 +295,7 @@ proc boot(args: string) = let nimStart = findStartNim().quoteShell() for i in 0..2: - # Nim versions < (1, 1) expect Nim's exception type to have a 'raiseId' field for - # C++ interop. Later Nim versions do this differently and removed the 'raiseId' field. - # Thus we always bootstrap the first iteration with "c" and not with "cpp" as - # a workaround. - let defaultCommand = if useCpp and i > 0: "cpp" else: "c" + let defaultCommand = if useCpp: "cpp" else: "c" let bootOptions = if args.len == 0 or args.startsWith("-"): defaultCommand else: "" echo "iteration: ", i+1 var extraOption = "" From c91ef1a09f3284315e9e66c32532f5a8e3ba9297 Mon Sep 17 00:00:00 2001 From: c-blake Date: Wed, 15 Feb 2023 11:41:28 -0500 Subject: [PATCH 1586/1654] Fix `closeHandle` bug, add `setFileSize`, make `resize` work on Windows (#21375) * Add general purpose `setFileSize` (unexported for now). Use to simplify `memfiles.open` as well as make robust (via hard allocation, not merely `ftruncate` address space allocation) on systems with `posix_fallocate`. As part of this, fix a bad `closeHandle` return check bug on Windows and add `MemFile.resize` for Windows now that setFileSize makes that easier. * Adapt existing test to exercise newly portable `MemFile.resize`. * Since Apple has never provided `posix_fallocate`, provide a fallback. This is presently written in terms of `ftruncate`, but it can be improved to use `F_PREALLOCATE` instead, as mentioned in a comment. --- lib/posix/posix.nim | 11 +++- lib/pure/memfiles.nim | 123 ++++++++++++++++++++++++------------ tests/stdlib/tmemfiles2.nim | 5 +- 3 files changed, 92 insertions(+), 47 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 43346608d..aee9308a6 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -195,8 +195,14 @@ proc open*(a1: cstring, a2: cint, mode: Mode | cint = 0.Mode): cint {.inline.} = proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. importc, header: "".} -proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. - importc, header: "".} + +proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} +when defined(osx): # 2001 POSIX evidently does not concern Apple + proc posix_fallocate*(a1: cint, a2, a3: Off): cint = + ftruncate(a1, a2 + a3) # Set size to off + len, max offset +else: # TODO: Use fcntl(fd, F_PREALLOCATE, ..) above + proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. + importc, header: "".} when not defined(haiku) and not defined(openbsd): proc fmtmsg*(a1: int, a2: cstring, a3: cint, @@ -511,7 +517,6 @@ proc fpathconf*(a1, a2: cint): int {.importc, header: "".} proc fsync*(a1: cint): cint {.importc, header: "".} ## synchronize a file's buffer cache to the storage device -proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} proc getcwd*(a1: cstring, a2: int): cstring {.importc, header: "", sideEffect.} proc getuid*(): Uid {.importc, header: "", sideEffect.} ## returns the real user ID of the calling process diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 9ff8bcd74..58c4888f1 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -35,6 +35,31 @@ proc newEIO(msg: string): ref IOError = new(result) result.msg = msg +proc setFileSize(fh: FileHandle, newFileSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1. + ## Space is only allocated if that is cheaper than writing to the file. This + ## routine returns the last OSErrorCode found rather than raising to support + ## old rollback/clean-up code style. [ Should maybe move to std/osfiles. ] + if newFileSize == -1: + return + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(fh) == 0: + result = lastErr + else: + var e: cint # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e in [EINVAL, EOPNOTSUPP] and ftruncate(fh, newFileSize) == -1: + result = osLastError() # fallback arguable; Most portable, but allows SEGV + elif e != 0: + result = osLastError() + type MemFile* = object ## represents a memory mapped file mem*: pointer ## a pointer to the memory mapped file. The pointer @@ -182,17 +207,8 @@ proc open*(filename: string, mode: FileMode = fmRead, if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") - if newFileSize != -1: - var - sizeHigh = int32(newFileSize shr 32) - sizeLow = int32(newFileSize and 0xffffffff) - - var status = setFilePointer(result.fHandle, sizeLow, addr(sizeHigh), - FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - (setEndOfFile(result.fHandle) == 0): - fail(lastErr, "error setting file size") + if (let e = setFileSize(result.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") # since the strings are always 'nil', we simply always call # CreateFileMappingW which should be slightly faster anyway: @@ -226,7 +242,7 @@ proc open*(filename: string, mode: FileMode = fmRead, result.wasOpened = true if not allowRemap and result.fHandle != INVALID_HANDLE_VALUE: - if closeHandle(result.fHandle) == 0: + if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE else: @@ -249,9 +265,8 @@ proc open*(filename: string, mode: FileMode = fmRead, # Is there an exception that wraps it? fail(osLastError(), "error opening file") - if newFileSize != -1: - if ftruncate(result.handle, newFileSize) == -1: - fail(osLastError(), "error setting file size") + if (let e = setFileSize(result.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): fail(e, "error setting file size") if mappedSize != -1: result.size = mappedSize @@ -306,34 +321,58 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -when defined(posix) or defined(nimdoc): - proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## resize and re-map the file underlying an `allowRemap MemFile`. - ## **Note**: this assumes the entire file is mapped read-write at offset zero. - ## Also, the value of `.mem` will probably change. - ## **Note**: This is not (yet) available on Windows. - when defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if ftruncate(f.handle, newFileSize) == -1: - raiseOSError(osLastError()) - when defined(linux): #Maybe NetBSD, too? - #On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): - pointer {.importc: "mremap", header: "".} - let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), cint(1)) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr +proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + if (let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil); m != nil): + f.mem = m f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + if (let e = setFileSize(f.handle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr + f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim index 6fee3c1ae..c79f85ebf 100644 --- a/tests/stdlib/tmemfiles2.nim +++ b/tests/stdlib/tmemfiles2.nim @@ -15,8 +15,9 @@ var if fileExists(fn): removeFile(fn) -# Create a new file, data all zeros -mm = memfiles.open(fn, mode = fmReadWrite, newFileSize = 20) +# Create a new file, data all zeros, starting at size 10 +mm = memfiles.open(fn, mode = fmReadWrite, newFileSize = 10, allowRemap=true) +mm.resize 20 # resize up to 20 mm.close() # read, change From fc7385bfda7b3af93aa57c5cf446a90b470de348 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 16 Feb 2023 10:23:35 +0800 Subject: [PATCH 1587/1654] fixes #21360; discarding empty seqs/arrays now raises errors (#21374) * discarding empty seqs now raises errors * the same goes for sets --- compiler/semstmts.nim | 18 ++++++++++-------- tests/discard/tillegaldiscardtypes.nim | 14 ++++++++++++++ tests/typerel/ttypelessemptyset.nim | 2 +- 3 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 tests/discard/tillegaldiscardtypes.nim diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c7e48db4c..7dc976c19 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -40,6 +40,13 @@ const proc implicitlyDiscardable(n: PNode): bool +proc hasEmpty(typ: PType): bool = + if typ.kind in {tySequence, tyArray, tySet}: + result = typ.lastSon.kind == tyEmpty + elif typ.kind == tyTuple: + for s in typ.sons: + result = result or hasEmpty(s) + proc semDiscard(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1, c.config) @@ -47,7 +54,9 @@ proc semDiscard(c: PContext, n: PNode): PNode = n[0] = semExprWithType(c, n[0]) let sonType = n[0].typ let sonKind = n[0].kind - if isEmptyType(sonType) or sonType.kind in {tyNone, tyTypeDesc} or sonKind == nkTypeOfExpr: + if isEmptyType(sonType) or hasEmpty(sonType) or + sonType.kind in {tyNone, tyTypeDesc} or + sonKind == nkTypeOfExpr: localError(c.config, n.info, errInvalidDiscard) if sonType.kind == tyProc and sonKind notin nkCallKinds: # tyProc is disallowed to prevent ``discard foo`` to be valid, when ``discard foo()`` is meant. @@ -410,13 +419,6 @@ proc semUsing(c: PContext; n: PNode): PNode = if a[^1].kind != nkEmpty: localError(c.config, a.info, "'using' sections cannot contain assignments") -proc hasEmpty(typ: PType): bool = - if typ.kind in {tySequence, tyArray, tySet}: - result = typ.lastSon.kind == tyEmpty - elif typ.kind == tyTuple: - for s in typ.sons: - result = result or hasEmpty(s) - proc hasUnresolvedParams(n: PNode; flags: TExprFlags): bool = result = tfUnresolved in n.typ.flags when false: diff --git a/tests/discard/tillegaldiscardtypes.nim b/tests/discard/tillegaldiscardtypes.nim new file mode 100644 index 000000000..b7877bcd2 --- /dev/null +++ b/tests/discard/tillegaldiscardtypes.nim @@ -0,0 +1,14 @@ +discard """ + cmd: "nim check $file" + errormsg: "statement returns no value that can be discarded" + nimout: ''' +tillegaldiscardtypes.nim(11, 3) Error: statement returns no value that can be discarded +tillegaldiscardtypes.nim(12, 3) Error: statement returns no value that can be discarded +''' +""" + +proc b(v: int) = # bug #21360 + discard @[] + discard [] + +b(0) \ No newline at end of file diff --git a/tests/typerel/ttypelessemptyset.nim b/tests/typerel/ttypelessemptyset.nim index 5f49c33fd..a4b6f003a 100644 --- a/tests/typerel/ttypelessemptyset.nim +++ b/tests/typerel/ttypelessemptyset.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "internal error: invalid kind for lastOrd(tyEmpty)" + errormsg: "statement returns no value that can be discarded" """ var q = false discard (if q: {} else: {}) From 93e4f278a1b68c83944842175ab78b61fe5fb7ca Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:34:32 +0800 Subject: [PATCH 1588/1654] add deprecated warnings for `{.deadcodeelim: on.}` (#21277) --- compiler/pragmas.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 92e3d1922..6d953dee7 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -929,7 +929,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, of wThreadVar: noVal(c, it) incl(sym.flags, {sfThread, sfGlobal}) - of wDeadCodeElimUnused: discard # deprecated, dead code elim always on + of wDeadCodeElimUnused: + warningDeprecated(c.config, n.info, "'{.deadcodeelim: on.}' is deprecated, now a noop") # deprecated, dead code elim always on of wNoForward: pragmaNoForward(c, it) of wReorder: pragmaNoForward(c, it, flag = sfReorder) of wMagic: processMagic(c, it, sym) From 3b9e9fd7b29de97b394811f350876732c66cff19 Mon Sep 17 00:00:00 2001 From: Century Systems Date: Thu, 16 Feb 2023 21:47:52 +0900 Subject: [PATCH 1589/1654] Add support for NuttX RTOS. (#21372) * Add support for NuttX RTOS. Signed-off-by: Takeyoshi Kikuchi * lib: pure: asyncdispatch: assign to result. Signed-off-by: Takeyoshi Kikuchi * lib: std: typedthreads: add support for parameters to adjust Thread Stack Size. Like FreeRTOS/Zephyr, add support for following configurations. -d:nimThreadStackSize=xxxxx -d:nimThreadStackGuard=yyyy Signed-off-by: Takeyoshi Kikuchi --------- Signed-off-by: Takeyoshi Kikuchi --- compiler/options.nim | 4 +++- compiler/platform.nim | 7 ++++++- lib/posix/posix_other.nim | 5 ++++- lib/posix/posix_other_consts.nim | 13 ++++++++++++- lib/pure/asyncdispatch.nim | 5 ++++- lib/pure/asynchttpserver.nim | 5 +++-- lib/pure/asyncnet.nim | 3 ++- lib/pure/ioselects/ioselectors_select.nim | 2 +- lib/pure/nativesockets.nim | 7 ++++--- lib/pure/net.nim | 3 ++- lib/pure/selectors.nim | 9 +++++++-- lib/std/typedthreads.nim | 3 ++- lib/system.nim | 1 + lib/system/dyncalls.nim | 2 +- 14 files changed, 52 insertions(+), 17 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 862b42446..51a8ff6f0 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -621,7 +621,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = osQnx, osAtari, osAix, osHaiku, osVxWorks, osSolaris, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos, - osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr} + osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr, osNuttX} of "linux": result = conf.target.targetOS in {osLinux, osAndroid} of "bsd": @@ -643,6 +643,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = result = conf.target.targetOS == osFreeRTOS of "zephyr": result = conf.target.targetOS == osZephyr + of "nuttx": + result = conf.target.targetOS == osNuttX of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian of "cpu8": result = CPU[conf.target.targetCPU].bit == 8 diff --git a/compiler/platform.nim b/compiler/platform.nim index 4b4316bc2..b6d6a119e 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -26,7 +26,8 @@ type osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris, osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx, osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks - osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny + osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, + osNuttX, osAny type TInfoOSProp* = enum @@ -193,6 +194,10 @@ const objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", props: {ospPosix}), + (name: "NuttX", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", + objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", + scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", + props: {ospPosix}), (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/", objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/", scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".", diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim index 941e13192..1b6734b51 100644 --- a/lib/posix/posix_other.nim +++ b/lib/posix/posix_other.nim @@ -10,7 +10,7 @@ when defined(nimHasStyleChecks): {.push styleChecks: off.} -when defined(freertos) or defined(zephyr): +when defined(freertos) or defined(zephyr) or defined(nuttx): const hasSpawnH = false # should exist for every Posix system nowadays hasAioH = false @@ -640,6 +640,9 @@ when defined(linux) or defined(nimdoc): ## or UDP packets. (Requires Linux kernel > 3.9) else: const SO_REUSEPORT* = cint(15) +elif defined(nuttx): + # Not supported, use SO_REUSEADDR to avoid compilation errors. + var SO_REUSEPORT* {.importc: "SO_REUSEADDR", header: "".}: cint else: var SO_REUSEPORT* {.importc, header: "".}: cint diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 59ac6f9ed..4b1644f20 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -99,7 +99,10 @@ var EXDEV* {.importc: "EXDEV", header: "".}: cint # var F_DUPFD* {.importc: "F_DUPFD", header: "".}: cint -var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "".}: cint +when defined(nuttx): + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD_CLOEXEC", header: "".}: cint +else: + var F_DUPFD_CLOEXEC* {.importc: "F_DUPFD", header: "".}: cint var F_GETFD* {.importc: "F_GETFD", header: "".}: cint var F_SETFD* {.importc: "F_SETFD", header: "".}: cint var F_GETFL* {.importc: "F_GETFL", header: "".}: cint @@ -127,6 +130,11 @@ var O_RDONLY* {.importc: "O_RDONLY", header: "".}: cint var O_RDWR* {.importc: "O_RDWR", header: "".}: cint var O_WRONLY* {.importc: "O_WRONLY", header: "".}: cint var O_CLOEXEC* {.importc: "O_CLOEXEC", header: "".}: cint +when defined(nuttx): + var O_DIRECT* {.importc: "O_DIRECT", header: "".}: cint + var O_PATH* {.importc: "O_PATH", header: "".}: cint + var O_NOATIME* {.importc: "O_NOATIME", header: "".}: cint + var O_TMPFILE* {.importc: "O_TMPFILE", header: "".}: cint var POSIX_FADV_NORMAL* {.importc: "POSIX_FADV_NORMAL", header: "".}: cint var POSIX_FADV_SEQUENTIAL* {.importc: "POSIX_FADV_SEQUENTIAL", header: "".}: cint var POSIX_FADV_RANDOM* {.importc: "POSIX_FADV_RANDOM", header: "".}: cint @@ -465,6 +473,9 @@ var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint when defined(zephyr): # Zephyr specific hardcoded value var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint +elif defined(nuttx): + # NuttX specific user configuration value + var NACTIVESOCKETS* {.importc: "CONFIG_NET_NACTIVESOCKETS", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 214b7d12c..64100afa7 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -2024,7 +2024,7 @@ when defined(posix): import posix when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(solaris) or defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx): proc maxDescriptors*(): int {.raises: OSError.} = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is @@ -2033,6 +2033,9 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or result = 16_700_000 elif defined(zephyr) or defined(freertos): result = FD_MAX + elif defined(nuttx): + # The maximum number of concurrently active UDP and TCP ports. + result = NACTIVESOCKETS else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 845875a1e..c09fe35ca 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -387,8 +387,9 @@ proc listen*(server: AsyncHttpServer; port: Port; address = ""; domain = AF_INET server.socket = newAsyncSocket(domain) if server.reuseAddr: server.socket.setSockOpt(OptReuseAddr, true) - if server.reusePort: - server.socket.setSockOpt(OptReusePort, true) + when not defined(nuttx): + if server.reusePort: + server.socket.setSockOpt(OptReusePort, true) server.socket.bindAddr(port, address) server.socket.listen() diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 982459af7..d35dbdfaf 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -106,7 +106,8 @@ export SOBool # TODO: Remove duplication introduced by PR #4683. const defineSsl = defined(ssl) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when defineSsl: import openssl diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index 34c88d85e..a6f54e791 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -313,7 +313,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, verifySelectParams(timeout) if timeout != -1: - when defined(genode) or defined(freertos) or defined(zephyr): + when defined(genode) or defined(freertos) or defined(zephyr) or defined(nuttx): tv.tv_sec = Time(timeout div 1_000) else: tv.tv_sec = timeout.int32 div 1_000 diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 14a318412..b3465ef14 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -23,7 +23,8 @@ when hostOS == "solaris": {.passl: "-lsocket -lnsl".} const useWinVersion = defined(windows) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) when useWinVersion: import winlean @@ -304,7 +305,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, let socketPort = if sockType == SOCK_RAW: "" else: $port var gaiResult = getaddrinfo(address, socketPort.cstring, addr(hints), result) if gaiResult != 0'i32: - when useWinVersion or defined(freertos): + when useWinVersion or defined(freertos) or defined(nuttx): raiseOSError(osLastError()) else: raiseOSError(osLastError(), $gai_strerror(gaiResult)) @@ -351,7 +352,7 @@ proc getSockDomain*(socket: SocketHandle): Domain = else: raise newException(IOError, "Unknown socket family in getSockDomain") -when not useNimNetLite: +when not useNimNetLite: proc getServByName*(name, proto: string): Servent {.tags: [ReadIOEffect].} = ## Searches the database from the beginning and finds the first entry for ## which the service name specified by `name` matches the s_name member diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 1dfef3b30..d3a767eed 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -100,7 +100,8 @@ export nativesockets.Port, nativesockets.`$`, nativesockets.`==` export Domain, SockType, Protocol const useWinVersion = defined(windows) or defined(nimdoc) -const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) +const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr) or + defined(nuttx) const defineSsl = defined(ssl) or defined(nimdoc) when useWinVersion: diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index be5037fe2..ef8736cdf 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -37,7 +37,7 @@ const hasThreadSupport = compileOption("threads") and defined(threadsafe) const ioselSupportedPlatform* = defined(macosx) or defined(freebsd) or defined(netbsd) or defined(openbsd) or - defined(dragonfly) or + defined(dragonfly) or defined(nuttx) or (defined(linux) and not defined(android) and not defined(emscripten)) ## This constant is used to determine whether the destination platform is ## fully supported by `ioselectors` module. @@ -328,7 +328,7 @@ else: doAssert(timeout >= -1, "Cannot select with a negative value, got: " & $timeout) when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or - defined(solaris) or defined(zephyr) or defined(freertos): + defined(solaris) or defined(zephyr) or defined(freertos) or defined(nuttx): template maxDescriptors*(): int = ## Returns the maximum number of active file descriptors for the current ## process. This involves a system call. For now `maxDescriptors` is @@ -337,6 +337,9 @@ else: 16_700_000 elif defined(zephyr) or defined(freertos): FD_MAX + elif defined(nuttx): + # The maximum number of concurrently active UDP and TCP ports. + NACTIVESOCKETS else: var fdLim: RLimit var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) @@ -360,5 +363,7 @@ else: include ioselects/ioselectors_select elif defined(zephyr): include ioselects/ioselectors_poll + elif defined(nuttx): + include ioselects/ioselectors_epoll else: include ioselects/ioselectors_poll diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim index 970d381c3..b45ce8a61 100644 --- a/lib/std/typedthreads.nim +++ b/lib/std/typedthreads.nim @@ -46,7 +46,8 @@ when defined(genode): import genode/env -when hasAllocStack or defined(zephyr) or defined(freertos) or defined(cpu16) or defined(cpu8): +when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or + defined(cpu16) or defined(cpu8): const nimThreadStackSize {.intdefine.} = 8192 nimThreadStackGuard {.intdefine.} = 128 diff --git a/lib/system.nim b/lib/system.nim index d8fafc0ef..af7e0e4c8 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2688,6 +2688,7 @@ when notJSnotNims: not defined(nintendoswitch) and not defined(freertos) and not defined(zephyr) and + not defined(nuttx) and hostOS != "any" proc raiseEIO(msg: string) {.noinline, noreturn.} = diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index de22f7cbb..2162b234f 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -176,7 +176,7 @@ elif defined(genode): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = raiseAssert("nimGetProcAddr not implemented") -elif defined(nintendoswitch) or defined(freertos) or defined(zephyr): +elif defined(nintendoswitch) or defined(freertos) or defined(zephyr) or defined(nuttx): proc nimUnloadLibrary(lib: LibHandle) = cstderr.rawWrite("nimUnLoadLibrary not implemented") cstderr.rawWrite("\n") From 03dd853815173bcf63f80ef03b5a4db7177453ec Mon Sep 17 00:00:00 2001 From: "Qinsi (James) ZHU" Date: Fri, 17 Feb 2023 01:12:48 +0800 Subject: [PATCH 1590/1654] add .replace() with callback to jsre (#21371) --- lib/js/jsre.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/js/jsre.nim b/lib/js/jsre.nim index 19888aaa9..69bd75c3b 100644 --- a/lib/js/jsre.nim +++ b/lib/js/jsre.nim @@ -34,6 +34,9 @@ func compile*(self: RegExp; pattern: cstring; flags: cstring) {.importjs: "#.com func replace*(pattern: cstring; self: RegExp; replacement: cstring): cstring {.importjs: "#.replace(#, #)".} ## Returns a new string with some or all matches of a pattern replaced by given replacement +func replace*(pattern: cstring, self: RegExp, cb: proc (args: varargs[cstring]): cstring): cstring {.importcpp.} + ## Returns a new string with some or all matches of a pattern replaced by given callback function + func split*(pattern: cstring; self: RegExp): seq[cstring] {.importjs: "(#.split(#) || [])".} ## Divides a string into an ordered list of substrings and returns the array @@ -88,5 +91,7 @@ runnableExamples: assert "do1ne".split(jsregex) == @["do".cstring, "ne".cstring] jsregex.compile(r"[lw]", r"i") assert "hello world".replace(jsregex,"X") == "heXlo world" + jsregex.compile(r"([a-z])\1*", r"g") + assert "abbcccdddd".replace(jsregex, proc (m: varargs[cstring]): cstring = ($m[0] & $(m.len)).cstring) == "a1b2c3d4" let digitsRegex: RegExp = newRegExp(r"\d") assert "foo".match(digitsRegex) == @[] From 13711b101d5ec101b3a2e18832d80e2b71eb3c80 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 17 Feb 2023 21:24:25 +1100 Subject: [PATCH 1591/1654] `std/xmltree` Add the type of the node when the assertion fails (#21383) Print the type of the node when the assertion fails This way the user actually knows what the type was instead of just knowing it failed --- lib/pure/xmltree.nim | 52 +++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index c3923ca50..82513bc98 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -70,6 +70,14 @@ const xmlHeader* = "\n" ## Header to use for complete XML output. +template expect(node: XmlNode, kind: set[XmlNodeKind]) = + ## Check the node's kind is within a set of values + assert node.k in kind, "Got " & $node.k + +template expect(node: XmlNode, kind: XmlNodeKind) = + ## Check the node's kind equals a value + assert node.k == kind, "Got " & $node.k + proc newXmlNode(kind: XmlNodeKind): XmlNode = ## Creates a new ``XmlNode``. result = XmlNode(k: kind) @@ -182,7 +190,7 @@ proc text*(n: XmlNode): lent string {.inline.} = assert $c == "" assert c.text == "my comment" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnComment, xnCData, xnEntity} result = n.fText proc `text=`*(n: XmlNode, text: sink string) {.inline.} = @@ -200,7 +208,7 @@ proc `text=`*(n: XmlNode, text: sink string) {.inline.} = e.text = "a new entity text" assert $e == "&a new entity text;" - assert n.k in {xnText, xnComment, xnCData, xnEntity} + n.expect {xnText, xnComment, xnCData, xnEntity} n.fText = text proc tag*(n: XmlNode): lent string {.inline.} = @@ -221,7 +229,7 @@ proc tag*(n: XmlNode): lent string {.inline.} = """ assert a.tag == "firstTag" - assert n.k == xnElement + n.expect xnElement result = n.fTag proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = @@ -244,7 +252,7 @@ proc `tag=`*(n: XmlNode, tag: sink string) {.inline.} = """ - assert n.k == xnElement + n.expect xnElement n.fTag = tag proc rawText*(n: XmlNode): string {.inline.} = @@ -315,7 +323,7 @@ proc add*(father, son: XmlNode) {.inline.} = f.add newEntity("my entity") assert $f == "my text&my entity;" - assert father.k == xnElement + father.expect xnElement add(father.s, son) proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = @@ -335,7 +343,7 @@ proc add*(father: XmlNode, sons: openArray[XmlNode]) {.inline.} = f.add(@[newText("my text"), newElement("sonTag"), newEntity("my entity")]) assert $f == "my text&my entity;" - assert father.k == xnElement + father.expect xnElement add(father.s, sons) @@ -361,7 +369,7 @@ proc insert*(father, son: XmlNode, index: int) {.inline.} = """ - assert father.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, son, index) else: @@ -390,7 +398,7 @@ proc insert*(father: XmlNode, sons: openArray[XmlNode], index: int) {.inline.} = """ - assert father.k == xnElement + father.expect xnElement if len(father.s) > index: insert(father.s, sons, index) else: @@ -416,7 +424,7 @@ proc delete*(n: XmlNode, i: Natural) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(i) proc delete*(n: XmlNode, slice: Slice[int]) = @@ -439,7 +447,7 @@ proc delete*(n: XmlNode, slice: Slice[int]) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(slice) proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = @@ -466,7 +474,7 @@ proc replace*(n: XmlNode, i: Natural, replacement: openArray[XmlNode]) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(i) n.s.insert(replacement, i) @@ -494,7 +502,7 @@ proc replace*(n: XmlNode, slice: Slice[int], replacement: openArray[XmlNode]) = """ - assert n.k == xnElement + n.expect xnElement n.s.delete(slice) n.s.insert(replacement, slice.a) @@ -525,12 +533,12 @@ proc `[]`*(n: XmlNode, i: int): XmlNode {.inline.} = assert $f[1] == "" assert $f[0] == "" - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc `[]`*(n: var XmlNode, i: int): var XmlNode {.inline.} = ## Returns the `i`'th child of `n` so that it can be modified. - assert n.k == xnElement + n.expect xnElement result = n.s[i] proc clear*(n: var XmlNode) = @@ -582,12 +590,12 @@ iterator items*(n: XmlNode): XmlNode {.inline.} = # # &some entity; - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] iterator mitems*(n: var XmlNode): var XmlNode {.inline.} = ## Iterates over all direct children of `n` so that they can be modified. - assert n.k == xnElement + n.expect xnElement for i in 0 .. n.len-1: yield n[i] proc toXmlAttributes*(keyValuePairs: varargs[tuple[key, @@ -619,7 +627,7 @@ proc attrs*(n: XmlNode): XmlAttributes {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement result = n.fAttr proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = @@ -636,7 +644,7 @@ proc `attrs=`*(n: XmlNode, attr: XmlAttributes) {.inline.} = j.attrs = att assert j.attrs == att - assert n.k == xnElement + n.expect xnElement n.fAttr = attr proc attrsLen*(n: XmlNode): int {.inline.} = @@ -653,7 +661,7 @@ proc attrsLen*(n: XmlNode): int {.inline.} = j.attrs = att assert j.attrsLen == 2 - assert n.k == xnElement + n.expect xnElement if not isNil(n.fAttr): result = len(n.fAttr) proc attr*(n: XmlNode, name: string): string = @@ -671,7 +679,7 @@ proc attr*(n: XmlNode, name: string): string = assert j.attr("key1") == "first value" assert j.attr("key2") == "second value" - assert n.kind == xnElement + n.expect xnElement if n.attrs == nil: return "" return n.attrs.getOrDefault(name) @@ -834,7 +842,7 @@ proc child*(n: XmlNode, name: string): XmlNode = f.add newElement("thirdSon") assert $(f.child("secondSon")) == "" - assert n.kind == xnElement + n.expect xnElement for i in items(n): if i.kind == xnElement: if i.tag == name: @@ -870,7 +878,7 @@ proc findAll*(n: XmlNode, tag: string, result: var seq[XmlNode], a.findAll("BAD", s, caseInsensitive = true) assert $s == "@[c text, d text]" - assert n.k == xnElement + n.expect xnElement for child in n.items(): if child.k != xnElement: continue From 6b93db7070c984121e18b502d938d5e455d4ba70 Mon Sep 17 00:00:00 2001 From: Matt Haggard Date: Fri, 17 Feb 2023 05:25:15 -0500 Subject: [PATCH 1592/1654] Fix long link commands on macOS (v2) (#21382) Handle long link commands on macOS with a script, since ar does not support response files --- compiler/extccomp.nim | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 6515f6968..db97de65c 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -834,6 +834,15 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) = finally: removeFile(linkerArgs) +proc linkViaShellScript(conf: ConfigRef; cmd: string) = + let linkerScript = conf.projectName & "_" & "linkerScript.sh" + writeFile(linkerScript, cmd) + let shell = getEnv("SHELL") + try: + execLinkCmd(conf, shell & " " & linkerScript) + finally: + removeFile(linkerScript) + proc getObjFilePath(conf: ConfigRef, f: Cfile): string = if noAbsolutePaths(conf): f.obj.extractFilename else: f.obj.string @@ -856,9 +865,12 @@ proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) = # Windows's command line limit is about 8K (8191 characters) so C compilers on # Windows support a feature where the command line can be passed via ``@linkcmd`` # to them. - const MaxCmdLen = when defined(windows): 8_000 else: 32_000 + const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000 if linkCmd.len > MaxCmdLen: - linkViaResponseFile(conf, linkCmd) + when defined(macosx): + linkViaShellScript(conf, linkCmd) + else: + linkViaResponseFile(conf, linkCmd) else: execLinkCmd(conf, linkCmd) From e896977bd16b2471e4aaaa363690622420e99acc Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 19 Feb 2023 14:49:19 +0800 Subject: [PATCH 1593/1654] closes #1072; add a test case (#21396) --- tests/specialops/tcallops.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/specialops/tcallops.nim b/tests/specialops/tcallops.nim index 0508a37a1..c541a0c1d 100644 --- a/tests/specialops/tcallops.nim +++ b/tests/specialops/tcallops.nim @@ -37,3 +37,13 @@ doAssert a(b) == "(12)" doAssert a.b(c) == `()`(b, a, c) doAssert (a.b)(c) == `()`(a.b, c) doAssert `()`(a.b, c) == `()`(`()`(b, a), c) + +block: # bug #1072 + var x: int + + proc foo(some:int):int = some + proc `()`(l,r:string): string = discard + + block: + var foo = 42 + doAssert x.foo == 0 From c66dc913cefb08722c4bf9885948e26be1285c3f Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 07:19:46 +1100 Subject: [PATCH 1594/1654] Support new runtime with nim-gdb (#21400) * Add support for orc strings * Cleaned up testing script Enums now printing properly Merged both old and new strings into the one printer Moving onto sets which seem kinda difficult * Sets working Instead of trying to imitate how Nim represents enums, I just call the dollar proc for each enum value While this runs into problems if the user doesn't call the dollar proc anywhere, I believe its a decent tradeoff I've cleaned up the error message for when it cannot find dollar proc (Might add in proper message on how to fix) * Support sequences V2 runtime seems to have sequences that don't have a len (Guessing its some kind of short seq optimisation?) but I've rolled the implementation into normal sequences since the implementation is practically the same * Clean up test program so it isn't using diff Also don't redirect the first nim compile to /dev/null so that we can check for any compilation errors I found the diff to be annoying to read (Seeing as the test script already performs diffing) * Tables are now supported * Add colours to test output It was getting difficult to tell apart test output from GDB output so I added colour to better tell them apart * Both old and new runtime are working Set exit code in python test script so that this could possibly be added to the CI. Only issue is that it can be flaky (GDB crashes randomly for some reason) * Remove old code that was commented out If I need it later I'll just use git * Remove branch that never runs * Remove the old test output [skip ci] --- .../untestable/gdb/gdb_pretty_printer_test.py | 30 +- .../gdb/gdb_pretty_printer_test_output.txt | 3 - .../gdb/gdb_pretty_printer_test_program.nim | 28 +- .../gdb/gdb_pretty_printer_test_run.sh | 20 +- tools/nim-gdb.py | 386 ++++++------------ 5 files changed, 160 insertions(+), 307 deletions(-) delete mode 100644 tests/untestable/gdb/gdb_pretty_printer_test_output.txt mode change 100644 => 100755 tests/untestable/gdb/gdb_pretty_printer_test_run.sh diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index 8f0f88e85..d28d01a60 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -1,10 +1,13 @@ import gdb +import re +import sys # this test should test the gdb pretty printers of the nim # library. But be aware this test is not complete. It only tests the # command line version of gdb. It does not test anything for the # machine interface of gdb. This means if if this test passes gdb # frontends might still be broken. +gdb.execute("set python print-stack full") gdb.execute("source ../../../tools/nim-gdb.py") # debug all instances of the generic function `myDebug`, should be 14 gdb.execute("rbreak myDebug") @@ -16,7 +19,7 @@ outputs = [ '"meTwo"', '{meOne, meThree}', 'MyOtherEnum(1)', - '5', + '{MyOtherEnum(0), MyOtherEnum(2)}', 'array = {1, 2, 3, 4, 5}', 'seq(0, 0)', 'seq(0, 10)', @@ -28,13 +31,17 @@ outputs = [ '{a = 1, b = "some string"}' ] +argRegex = re.compile("^.* = (?:No suitable Nim \$ operator found for type: \w+\s*)*(.*)$") +# Remove this error message which can pop up +noSuitableRegex = re.compile("(No suitable Nim \$ operator found for type: \w+\s*)") + for i, expected in enumerate(outputs): - gdb.write(f"{i+1}) expecting: {expected}: ", gdb.STDLOG) + gdb.write(f"\x1b[38;5;105m{i+1}) expecting: {expected}: \x1b[0m", gdb.STDLOG) gdb.flush() - - functionSymbol = gdb.selected_frame().block().function - assert functionSymbol.line == 41, str(functionSymbol.line) - + currFrame = gdb.selected_frame() + functionSymbol = currFrame.block().function + assert functionSymbol.line == 24, str(functionSymbol.line) + raw = "" if i == 6: # myArray is passed as pointer to int to myDebug. I look up myArray up in the stack gdb.execute("up") @@ -44,10 +51,13 @@ for i, expected in enumerate(outputs): gdb.execute("up") raw = gdb.parse_and_eval("myOtherArray") else: - raw = gdb.parse_and_eval("arg") - + rawArg = re.sub(noSuitableRegex, "", gdb.execute("info args", to_string = True)) + raw = rawArg.split("=", 1)[-1].strip() output = str(raw) - assert output == expected, "{0} : output: ({1}) != expected: ({2})".format(i, output, expected) - gdb.write(f"passed\n", gdb.STDLOG) + if output != expected: + gdb.write(f"\x1b[38;5;196m ({output}) != expected: ({expected})\x1b[0m\n", gdb.STDERR) + gdb.execute("quit 1") + else: + gdb.write("\x1b[38;5;34mpassed\x1b[0m\n", gdb.STDLOG) gdb.execute("continue") diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_output.txt b/tests/untestable/gdb/gdb_pretty_printer_test_output.txt deleted file mode 100644 index 73d26016f..000000000 --- a/tests/untestable/gdb/gdb_pretty_printer_test_output.txt +++ /dev/null @@ -1,3 +0,0 @@ -Loading Nim Runtime support. -NimEnumPrinter: lookup global symbol 'NTI__z9cu80OJCfNgw9bUdzn5ZEzw_ failed for tyEnum_MyOtherEnum__z9cu80OJCfNgw9bUdzn5ZEzw. -8 diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim index d2acdd282..b54fc1a7f 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim +++ b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim @@ -18,23 +18,6 @@ type MyObj = object a*: int b*: string - - # MyVariant = ref object - # id*: int - # case kind*: MyEnum - # of meOne: mInt*: int - # of meTwo, meThree: discard - # of meFour: - # moInt*: int - # babies*: seq[MyVariant] - # after: float - - # MyIntVariant = ref object - # stuff*: int - # case myKind*: range[0..32766] - # of 0: mFloat*: float - # of 2: mString*: string - # else: mBabies*: seq[MyIntVariant] var counter = 0 @@ -97,16 +80,7 @@ proc testProc(): void = var obj = MyObj(a: 1, b: "some string") myDebug(obj) #15 - # var varObj = MyVariant(id: 13, kind: meFour, moInt: 94, - # babies: @[MyVariant(id: 18, kind: meOne, mInt: 7, after: 1.0), - # MyVariant(id: 21, kind: meThree, after: 2.0)], - # after: 3.0) - # myDebug(varObj) #16 - - # var varObjInt = MyIntVariant(stuff: 5, myKind: 2, mString: "this is my sweet string") - # myDebug(varObjInt) #17 - - echo(counter) + assert counter == 15 testProc() diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_run.sh b/tests/untestable/gdb/gdb_pretty_printer_test_run.sh old mode 100644 new mode 100755 index 525f54705..411c68435 --- a/tests/untestable/gdb/gdb_pretty_printer_test_run.sh +++ b/tests/untestable/gdb/gdb_pretty_printer_test_run.sh @@ -1,15 +1,13 @@ #!/usr/bin/env bash -# Exit if anything fails set -e -#!/usr/bin/env bash # Compile the test project with fresh debug information. -nim c --debugger:native gdb_pretty_printer_test_program.nim &> /dev/null +nim c --debugger:native --mm:orc --out:gdbNew gdb_pretty_printer_test_program.nim +echo "Running new runtime tests..." # 2>&1 redirects stderr to stdout (all output in stdout) -# <(...) is a bash feature that makes the output of a command into a -# file handle. -# diff compares the two files, the expected output, and the file -# handle that is created by the execution of gdb. -diff ./gdb_pretty_printer_test_output.txt <(gdb -x gdb_pretty_printer_test.py --batch-silent --args gdb_pretty_printer_test_program 2>&1) -# The exit code of diff is forwarded as the exit code of this -# script. So when the comparison fails, the exit code of this script -# won't be 0. So this script should be embeddable in a test suite. +gdb -x gdb_pretty_printer_test.py --batch-silent --args gdbNew 2>&1 + + +# Do it all again, but with old runtime +nim c --debugger:native --mm:refc --out:gdbOld gdb_pretty_printer_test_program.nim &> /dev/null +echo "Running old runtime tests" +gdb -x gdb_pretty_printer_test.py --batch-silent --args gdbOld 2>&1 diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index f35b9a033..e3af0dde6 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -16,6 +16,10 @@ def printErrorOnce(id, message): errorSet.add(id) gdb.write("printErrorOnce: " + message, gdb.STDERR) +def debugPrint(x): + gdb.write(str(x) + "\n", gdb.STDERR) + +NIM_STRING_TYPES = ["NimStringDesc", "NimStringV2"] ################################################################################ ##### Type pretty printers @@ -23,23 +27,28 @@ def printErrorOnce(id, message): type_hash_regex = re.compile("^([A-Za-z0-9]*)_([A-Za-z0-9]*)_+([A-Za-z0-9]*)$") +def getNimName(typ): + if m := type_hash_regex.match(typ): + return m.group(2) + return f"unknown <{typ}>" + def getNimRti(type_name): """ Return a ``gdb.Value`` object for the Nim Runtime Information of ``type_name``. """ # Get static const TNimType variable. This should be available for # every non trivial Nim type. m = type_hash_regex.match(type_name) - lookups = [ - "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", - "NTI" + "__" + m.group(3) + "_", - "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" - ] if m: - for l in lookups: - try: - return gdb.parse_and_eval(l) - except: - pass + lookups = [ + "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", + "NTI" + "__" + m.group(3) + "_", + "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" + ] + for l in lookups: + try: + return gdb.parse_and_eval(l) + except: + pass None def getNameFromNimRti(rti): @@ -68,7 +77,7 @@ class NimTypeRecognizer: 'NIM_BOOL': 'bool', - 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string' + 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string', 'NimStringV2': 'string' } # object_type_pattern = re.compile("^(\w*):ObjectType$") @@ -136,7 +145,7 @@ class DollarPrintFunction (gdb.Function): "Nim's equivalent of $ operator as a gdb function, available in expressions `print $dollar(myvalue)" dollar_functions = re.findall( - 'NimStringDesc \*(dollar__[A-z0-9_]+?)\(([^,)]*)\);', + '(?:NimStringDesc \*|NimStringV2)\s?(dollar__[A-z0-9_]+?)\(([^,)]*)\);', gdb.execute("info functions dollar__", True, True) ) @@ -146,12 +155,9 @@ class DollarPrintFunction (gdb.Function): @staticmethod def invoke_static(arg): - - if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name == "NimStringDesc": + if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name in NIM_STRING_TYPES: return arg - argTypeName = str(arg.type) - for func, arg_typ in DollarPrintFunction.dollar_functions: # this way of overload resolution cannot deal with type aliases, # therefore it won't find all overloads. @@ -163,7 +169,8 @@ class DollarPrintFunction (gdb.Function): func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() return func_value(arg.address) - printErrorOnce(argTypeName, "No suitable Nim $ operator found for type: " + argTypeName + "\n") + + debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") return None def invoke(self, arg): @@ -184,11 +191,11 @@ class NimStringEqFunction (gdb.Function): @staticmethod def invoke_static(arg1,arg2): - if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name == "NimStringDesc": + if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name in NIM_STRING_TYPES: str1 = NimStringPrinter(arg1).to_string() else: str1 = arg1.string() - if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name == "NimStringDesc": + if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name in NIM_STRING_TYPES: str2 = NimStringPrinter(arg1).to_string() else: str2 = arg2.string() @@ -216,7 +223,7 @@ class DollarPrintCmd (gdb.Command): strValue = DollarPrintFunction.invoke_static(param) if strValue: gdb.write( - NimStringPrinter(strValue).to_string() + "\n", + str(NimStringPrinter(strValue)) + "\n", gdb.STDOUT ) @@ -254,7 +261,6 @@ class KochCmd (gdb.Command): os.path.dirname(os.path.dirname(__file__)), "koch") def invoke(self, argument, from_tty): - import os subprocess.run([self.binary] + gdb.string_to_argv(argument)) KochCmd() @@ -308,8 +314,14 @@ class NimBoolPrinter: ################################################################################ +def strFromLazy(strVal): + if isinstance(strVal, str): + return strVal + else: + return strVal.value().string("utf-8") + class NimStringPrinter: - pattern = re.compile(r'^NimStringDesc \*$') + pattern = re.compile(r'^(NimStringDesc \*|NimStringV2)$') def __init__(self, val): self.val = val @@ -319,11 +331,19 @@ class NimStringPrinter: def to_string(self): if self.val: - l = int(self.val['Sup']['len']) - return self.val['data'].lazy_string(encoding="utf-8", length=l) + if self.val.type.name == "NimStringV2": + l = int(self.val["len"]) + data = self.val["p"]["data"] + else: + l = int(self.val['Sup']['len']) + data = self.val["data"] + return data.lazy_string(encoding="utf-8", length=l) else: return "" + def __str__(self): + return strFromLazy(self.to_string()) + class NimRopePrinter: pattern = re.compile(r'^tyObject_RopeObj__([A-Za-z0-9]*) \*$') @@ -345,39 +365,11 @@ class NimRopePrinter: ################################################################################ -# proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = -# ## Return string representation for enumeration values -# var n = typ.node -# if ntfEnumHole notin typ.flags: -# let o = e - n.sons[0].offset -# if o >= 0 and o <% typ.node.len: -# return $n.sons[o].name -# else: -# # ugh we need a slow linear search: -# var s = n.sons -# for i in 0 .. n.len-1: -# if s[i].offset == e: -# return $s[i].name -# result = $e & " (invalid data!)" - def reprEnum(e, typ): - """ this is a port of the nim runtime function `reprEnum` to python """ + # Casts the value to the enum type and then calls the enum printer e = int(e) - n = typ["node"] - flags = int(typ["flags"]) - # 1 << 6 is {ntfEnumHole} - if ((1 << 6) & flags) == 0: - o = e - int(n["sons"][0]["offset"]) - if o >= 0 and 0 < int(n["len"]): - return n["sons"][o]["name"].string("utf-8", "ignore") - else: - # ugh we need a slow linear search: - s = n["sons"] - for i in range(0, int(n["len"])): - if int(s[i]["offset"]) == e: - return s[i]["name"].string("utf-8", "ignore") - - return str(e) + " (invalid data!)" + val = gdb.Value(e).cast(typ) + return strFromLazy(NimEnumPrinter(val).to_string()) def enumNti(typeNimName, idString): typeInfoName = "NTI" + typeNimName.lower() + "__" + idString + "_" @@ -389,6 +381,7 @@ def enumNti(typeNimName, idString): class NimEnumPrinter: pattern = re.compile(r'^tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') + enumReprProc = gdb.lookup_global_symbol("reprEnum", gdb.SYMBOL_FUNCTIONS_DOMAIN) def __init__(self, val): self.val = val @@ -397,14 +390,18 @@ class NimEnumPrinter: self.typeNimName = match.group(1) typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) - if self.nti is None: - printErrorOnce(typeInfoName, f"NimEnumPrinter: lookup global symbol: '{typeInfoName}' failed for {typeName}.\n") - def to_string(self): - if self.nti: - arg0 = self.val - arg1 = self.nti.value(gdb.newest_frame()) - return reprEnum(arg0, arg1) + if NimEnumPrinter.enumReprProc and self.nti: + # Use the old runtimes enumRepr function. + # We call the Nim proc itself so that the implementation is correct + f = gdb.newest_frame() + # We need to strip the quotes so it looks like an enum instead of a string + reprProc = NimEnumPrinter.enumReprProc.value() + return str(reprProc(self.val, self.nti.value(f).address)).strip('"') + elif dollarResult := DollarPrintFunction.invoke_static(self.val): + # New runtime doesn't use enumRepr so we instead try and call the + # dollar function for it + return str(NimStringPrinter(dollarResult)) else: return self.typeNimName + "(" + str(int(self.val)) + ")" @@ -421,26 +418,20 @@ class NimSetPrinter: typeName = self.val.type.name match = self.pattern.match(typeName) self.typeNimName = match.group(1) - typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) - - if self.nti is None: - printErrorOnce(typeInfoName, f"NimSetPrinter: lookup global symbol: '{typeInfoName}' failed for {typeName}.\n") def to_string(self): - if self.nti: - nti = self.nti.value(gdb.newest_frame()) - enumStrings = [] - val = int(self.val) - i = 0 - while val > 0: - if (val & 1) == 1: - enumStrings.append(reprEnum(i, nti)) - val = val >> 1 - i += 1 + # Remove the tySet from the type name + typ = gdb.lookup_type(self.val.type.name[6:]) + enumStrings = [] + val = int(self.val) + i = 0 + while val > 0: + if (val & 1) == 1: + enumStrings.append(reprEnum(i, typ)) + val = val >> 1 + i += 1 - return '{' + ', '.join(enumStrings) + '}' - else: - return str(int(self.val)) + return '{' + ', '.join(enumStrings) + '}' ################################################################################ @@ -472,41 +463,81 @@ class NimHashSetPrinter: ################################################################################ -class NimSeqPrinter: - # the pointer is explicity part of the type. So it is part of - # ``pattern``. - pattern = re.compile(r'^tySequence_\w* \*$') +class NimSeq: + # Wrapper around sequences. + # This handles the differences between old and new runtime def __init__(self, val): self.val = val + # new runtime has sequences on stack, old has them on heap + self.new = val.type.code != gdb.TYPE_CODE_PTR + if self.new: + # Some seqs are just the content and to save repeating ourselves we do + # handle them here. Only thing that needs to check this is the len/data getters + self.isContent = val.type.name.endswith("Content") + + def __bool__(self): + if self.new: + return self.val is not None + else: + return bool(self.val) + + def __len__(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + else: + return int(self.val["len"]) + else: + return self.val["Sup"]["len"] + + @property + def data(self): + if self.new: + if self.isContent: + return self.val["data"] + elif self.val["p"]: + return self.val["p"]["data"] + else: + return self.val["data"] + + @property + def cap(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + elif self.val["p"]: + return int(self.val["p"]["cap"]) + else: + return 0 + return int(self.val['Sup']['reserved']) + +class NimSeqPrinter: + pattern = re.compile(r'^tySequence_\w*\s?\*?$') + + def __init__(self, val): + self.val = NimSeq(val) + def display_hint(self): return 'array' def to_string(self): - len = 0 - cap = 0 - if self.val: - len = int(self.val['Sup']['len']) - cap = int(self.val['Sup']['reserved']) - - return 'seq({0}, {1})'.format(len, cap) + return f'seq({len(self.val)}, {self.val.cap})' def children(self): if self.val: val = self.val - valType = val.type - length = int(val['Sup']['len']) + length = len(val) if length <= 0: return - dataType = valType['data'].type - data = val['data'] - - if self.val.type.name is None: - dataType = valType['data'].type.target().pointer() - data = val['data'].cast(dataType) + data = val.data inaccessible = False for i in range(length): @@ -585,7 +616,7 @@ class NimTablePrinter: if self.val: counter = int(self.val['counter']) if self.val['data']: - capacity = int(self.val['data']['Sup']['len']) + capacity = NimSeq(self.val["data"]).cap return 'Table({0}, {1})'.format(counter, capacity) @@ -597,163 +628,6 @@ class NimTablePrinter: yield (idxStr + '.Field1', entry['Field1']) yield (idxStr + '.Field2', entry['Field2']) -################################################################ - -# this is untested, therefore disabled - -# class NimObjectPrinter: -# pattern = re.compile(r'^tyObject_([A-Za-z0-9]+)__(_?[A-Za-z0-9]*)(:? \*)?$') - -# def __init__(self, val): -# self.val = val -# self.valType = None -# self.valTypeNimName = None - -# def display_hint(self): -# return 'object' - -# def _determineValType(self): -# if self.valType is None: -# vt = self.val.type -# if vt.name is None: -# target = vt.target() -# self.valType = target.pointer() -# self.fields = target.fields() -# self.valTypeName = target.name -# self.isPointer = True -# else: -# self.valType = vt -# self.fields = vt.fields() -# self.valTypeName = vt.name -# self.isPointer = False - -# def to_string(self): -# if self.valTypeNimName is None: -# self._determineValType() -# match = self.pattern.match(self.valTypeName) -# self.valTypeNimName = match.group(1) - -# return self.valTypeNimName - -# def children(self): -# self._determineValType() -# if self.isPointer and int(self.val) == 0: -# return -# self.baseVal = self.val.referenced_value() if self.isPointer else self.val - -# for c in self.handleFields(self.baseVal, getNimRti(self.valTypeName)): -# yield c - -# def handleFields(self, currVal, rti, fields = None): -# rtiSons = None -# discField = (0, None) -# seenSup = False -# if fields is None: -# fields = self.fields -# try: # XXX: remove try after finished debugging this method -# for (i, field) in enumerate(fields): -# if field.name == "Sup": # inherited data -# seenSup = True -# baseRef = rti['base'] -# if baseRef: -# baseRti = baseRef.referenced_value() -# baseVal = currVal['Sup'] -# baseValType = baseVal.type -# if baseValType.name is None: -# baseValType = baseValType.target().pointer() -# baseValFields = baseValType.target().fields() -# else: -# baseValFields = baseValType.fields() - -# for c in self.handleFields(baseVal, baseRti, baseValFields): -# yield c -# else: -# if field.type.code == gdb.TYPE_CODE_UNION: -# # if not rtiSons: -# rtiNode = rti['node'].referenced_value() -# rtiSons = rtiNode['sons'] - -# if not rtiSons and int(rtiNode['len']) == 0 and str(rtiNode['name']) != "0x0": -# rtiSons = [rti['node']] # sons are dereferenced by the consumer - -# if not rtiSons: -# printErrorOnce(self.valTypeName, f"NimObjectPrinter: UNION field can't be displayed without RTI {self.valTypeName}, using fallback.\n") -# # yield (field.name, self.baseVal[field]) # XXX: this fallback seems wrong -# return # XXX: this should probably continue instead? - -# if int(rtiNode['len']) != 0 and str(rtiNode['name']) != "0x0": -# gdb.write(f"wtf IT HAPPENED {self.valTypeName}\n", gdb.STDERR) - -# discNode = rtiSons[discField[0]].referenced_value() -# if not discNode: -# raise ValueError("Can't find union discriminant field in object RTI") - -# discNodeLen = int(discNode['len']) -# discFieldVal = int(currVal[discField[1].name]) - -# unionNodeRef = None -# if discFieldVal < discNodeLen: -# unionNodeRef = discNode['sons'][discFieldVal] -# if not unionNodeRef: -# unionNodeRef = discNode['sons'][discNodeLen] - -# if not unionNodeRef: -# printErrorOnce(self.valTypeName + "no union node", f"wtf is up with sons {self.valTypeName} {unionNodeRef} {rtiNode['offset']} {discNode} {discFieldVal} {discNodeLen} {discField[1].name} {field.name} {field.type}\n") -# continue - -# unionNode = unionNodeRef.referenced_value() - -# fieldName = "" if field.name == None else field.name.lower() -# unionNodeName = "" if not unionNode['name'] else unionNode['name'].string("utf-8", "ignore") -# if not unionNodeName or unionNodeName.lower() != fieldName: -# unionFieldName = f"_{discField[1].name.lower()}_{int(rti['node'].referenced_value()['len'])}" -# gdb.write(f"wtf i: {i} union: {unionFieldName} field: {fieldName} type: {field.type.name} tag: {field.type.tag}\n", gdb.STDERR) -# else: -# unionFieldName = unionNodeName - -# if discNodeLen == 0: -# yield (unionFieldName, currVal[unionFieldName]) -# else: -# unionNodeLen = int(unionNode['len']) -# if unionNodeLen > 0: -# for u in range(unionNodeLen): -# un = unionNode['sons'][u].referenced_value()['name'].string("utf-8", "ignore") -# yield (un, currVal[unionFieldName][un]) -# else: -# yield(unionNodeName, currVal[unionFieldName]) -# else: -# discIndex = i - 1 if seenSup else i -# discField = (discIndex, field) # discriminant field is the last normal field -# yield (field.name, currVal[field.name]) -# except GeneratorExit: -# raise -# except: -# gdb.write(f"wtf {self.valTypeName} {i} fn: {field.name} df: {discField} rti: {rti} rtiNode: {rti['node'].referenced_value()} rtiSons: {rtiSons} {sys.exc_info()} {traceback.format_tb(sys.exc_info()[2], limit = 10)}\n", gdb.STDERR) -# gdb.write(f"wtf {self.valTypeName} {i} {field.name}\n", gdb.STDERR) - -# # seenSup = False -# # for (i, field) in enumerate(fields): -# # # if field.name: -# # # val = currVal[field.name] -# # # else: -# # # val = None -# # rtiNode = rti['node'].referenced_value() -# # rtiLen = int(rtiNode['len']) -# # if int(rtiNode['len']) > 0: -# # sons = rtiNode['sons'] -# # elif int(rti['len']) == 0 and str(rti['name']) != "0x0": -# # sons = [rti['node']] # sons are dereferenced by the consumer -# # sonsIdx = i - 1 if seenSup else i -# # s = sons[sonsIdx].referenced_value() -# # addr = int(currVal.address) -# # off = addr + int(rtiNode['offset']) -# # seenSup = seenSup or field.name == "Sup" - -# # gdb.write(f"wtf: i: {i} sonsIdx: {sonsIdx} field: {field.name} rtiLen: {rtiLen} rti: {rti} rtiNode: {rtiNode} isUnion: {field.type.code == gdb.TYPE_CODE_UNION} s: {s}\n", gdb.STDERR) - -# raise - - ################################################################################ class NimFrameFilter: From c73c76fdc62e76a9e36e2595b9854a6f0cb9e904 Mon Sep 17 00:00:00 2001 From: Century Systems Date: Tue, 21 Feb 2023 11:38:25 +0900 Subject: [PATCH 1595/1654] NuttX added supports getrlimit(RLIMIT_NOFILE), so remove NuttX specific codes. (#21385) async: NuttX added supports getrlimit(RLIMIT_NOFILE), so remove NuttX-specific codes. Signed-off-by: Takeyoshi Kikuchi --- lib/posix/posix_other_consts.nim | 3 --- lib/pure/asyncdispatch.nim | 3 --- lib/pure/selectors.nim | 3 --- 3 files changed, 9 deletions(-) diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 4b1644f20..506c92158 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -473,9 +473,6 @@ var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "".}: cint when defined(zephyr): # Zephyr specific hardcoded value var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "".}: cint -elif defined(nuttx): - # NuttX specific user configuration value - var NACTIVESOCKETS* {.importc: "CONFIG_NET_NACTIVESOCKETS", header: "".}: cint # var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "".}: cint diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 64100afa7..2bfa8587c 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -2033,9 +2033,6 @@ when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or result = 16_700_000 elif defined(zephyr) or defined(freertos): result = FD_MAX - elif defined(nuttx): - # The maximum number of concurrently active UDP and TCP ports. - result = NACTIVESOCKETS else: var fdLim: RLimit if getrlimit(RLIMIT_NOFILE, fdLim) < 0: diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index ef8736cdf..b15a25a1d 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -337,9 +337,6 @@ else: 16_700_000 elif defined(zephyr) or defined(freertos): FD_MAX - elif defined(nuttx): - # The maximum number of concurrently active UDP and TCP ports. - NACTIVESOCKETS else: var fdLim: RLimit var res = int(getrlimit(RLIMIT_NOFILE, fdLim)) From f7bd2088aa5fe970902d4cafc94933354fdec8a7 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:27:42 +0800 Subject: [PATCH 1596/1654] corrects the linkerexe of riscv64 in config (#21292) --- config/nim.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/nim.cfg b/config/nim.cfg index f7b7c119a..973de077a 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -39,7 +39,7 @@ arm64.linux.gcc.linkerexe = "aarch64-linux-gnu-gcc" riscv32.linux.gcc.exe = "riscv64-linux-gnu-gcc" riscv32.linux.gcc.linkerexe = "riscv64-linux-gnu-gcc" riscv64.linux.gcc.exe = "riscv64-linux-gnu-gcc" -riscv64.linux.gcc.linkerexe = "arm-linux-gnueabihf-gcc" +riscv64.linux.gcc.linkerexe = "riscv64-linux-gnu-gcc" # For OpenWRT, you will also need to adjust PATH to point to your toolchain. mips.linux.gcc.exe = "mips-openwrt-linux-gcc" From ffd6bc87495ede82f9a8086d38c3348a40b11672 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:58:45 +0800 Subject: [PATCH 1597/1654] csources_v2 can build the ORC-booted compiler; building now uses strict mode (#21411) * csources_v2 can build the ORC-booted compiler; building now uses strict mode * test booting in refc --- koch.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/koch.nim b/koch.nim index fa498e017..52378e43f 100644 --- a/koch.nim +++ b/koch.nim @@ -530,8 +530,7 @@ proc runCI(cmd: string) = # boot without -d:nimHasLibFFI to make sure this still works # `--lib:lib` is needed for bootstrap on openbsd, for reasons described in # https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd). - kochExecFold("Boot in release mode", "boot -d:release --gc:refc -d:nimStrictMode --lib:lib") - kochExecFold("Boot Nim ORC", "boot -d:release --lib:lib") + kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib") when false: # debugging: when you need to run only 1 test in CI, use something like this: execFold("debugging test", "nim r tests/stdlib/tosproc.nim") @@ -587,6 +586,9 @@ proc runCI(cmd: string) = execFold("Run atlas tests", "nim c -r -d:atlasTests tools/atlas/atlas.nim clone https://github.com/disruptek/balls") + kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib") + + proc testUnixInstall(cmdLineRest: string) = csource("-d:danger" & cmdLineRest) xz(false, cmdLineRest) From bdc850916fb5d23fa4b9282309985e0fdfb0db36 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 21 Feb 2023 18:46:50 +0800 Subject: [PATCH 1598/1654] improve invalid module names errors (#21412) I have seen these questions: "Why I got an invalid module name errors?". To eliminate this kind of questions, We might improve th error messages. Though, the question might evolve into "What is a valid Nim identifier", which should be more searchable on the Internet. --- compiler/modules.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/modules.nim b/compiler/modules.nim index 7f6ff8622..838a89d83 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -38,7 +38,8 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = name: getModuleIdent(graph, filename), info: newLineInfo(fileIdx, 1, 1)) if not isNimIdentifier(result.name.s): - rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s) + rawMessage(graph.config, errGenerated, "invalid module name: '" & result.name.s & + "'; a module name must be a valid Nim identifier.") partialInitModule(result, graph, fileIdx, filename) graph.registerModule(result) From e54d3cc418f9eab750563aa771070374d559fe57 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 22:02:42 +1100 Subject: [PATCH 1599/1654] Support tuples in nim-gdb (#21410) * Support for printing tuple types * Add test of printing a tuple * Add support for printing tuples in GDB * Forgot to [skip ci] --- .../untestable/gdb/gdb_pretty_printer_test.py | 3 ++- .../gdb/gdb_pretty_printer_test_program.nim | 5 +++- tools/nim-gdb.py | 25 ++++++++++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index d28d01a60..a96df9992 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -28,7 +28,8 @@ outputs = [ 'seq(3, 3) = {"one", "two", "three"}', 'Table(3, 64) = {[4] = "four", [5] = "five", [6] = "six"}', 'Table(3, 8) = {["two"] = 2, ["three"] = 3, ["one"] = 1}', - '{a = 1, b = "some string"}' + '{a = 1, b = "some string"}', + '("hello", 42)' ] argRegex = re.compile("^.* = (?:No suitable Nim \$ operator found for type: \w+\s*)*(.*)$") diff --git a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim index b54fc1a7f..163c99860 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test_program.nim +++ b/tests/untestable/gdb/gdb_pretty_printer_test_program.nim @@ -80,7 +80,10 @@ proc testProc(): void = var obj = MyObj(a: 1, b: "some string") myDebug(obj) #15 - assert counter == 15 + var tup = ("hello", 42) + myDebug(tup) # 16 + + assert counter == 16 testProc() diff --git a/tools/nim-gdb.py b/tools/nim-gdb.py index e3af0dde6..1050197c9 100644 --- a/tools/nim-gdb.py +++ b/tools/nim-gdb.py @@ -112,6 +112,12 @@ class NimTypeRecognizer: result = self.type_map_static.get(tname, None) if result: return result + elif tname.startswith("tyEnum_"): + return getNimName(tname) + elif tname.startswith("tyTuple__"): + # We make the name be the field types (Just like in Nim) + fields = ", ".join([self.recognize(field.type) for field in type_obj.fields()]) + return f"({fields})" rti = getNimRti(tname) if rti: @@ -154,7 +160,7 @@ class DollarPrintFunction (gdb.Function): @staticmethod - def invoke_static(arg): + def invoke_static(arg, ignore_errors = False): if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name in NIM_STRING_TYPES: return arg argTypeName = str(arg.type) @@ -169,8 +175,8 @@ class DollarPrintFunction (gdb.Function): func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() return func_value(arg.address) - - debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") + if not ignore_errors: + debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") return None def invoke(self, arg): @@ -630,6 +636,19 @@ class NimTablePrinter: ################################################################################ +class NimTuplePrinter: + pattern = re.compile(r"^tyTuple__([A-Za-z0-9]*)") + + def __init__(self, val): + self.val = val + + def to_string(self): + # We don't have field names so just print out the tuple as if it was anonymous + tupleValues = [str(self.val[field.name]) for field in self.val.type.fields()] + return f"({', '.join(tupleValues)})" + +################################################################################ + class NimFrameFilter: def __init__(self): self.name = "nim-frame-filter" From 6d423f1856d086ea9e6b0995e3e5f831f4f89aa6 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 22:04:27 +1100 Subject: [PATCH 1600/1654] Make `Time` work with `std/strformat` (#21409) * Add test case * Remove formatValue template for `Time` It didn't handle empty specifier correctly which caused it to be blank with strformat --- lib/pure/times.nim | 6 +----- tests/stdlib/t21406.nim | 5 +++++ 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 tests/stdlib/t21406.nim diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 3327b2510..70af466d9 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -2109,7 +2109,7 @@ proc format*(dt: DateTime, f: static[string]): string {.raises: [].} = const f2 = initTimeFormat(f) result = dt.format(f2) -proc formatValue*(result: var string; value: DateTime, specifier: string) = +proc formatValue*(result: var string; value: DateTime | Time, specifier: string) = ## adapter for strformat. Not intended to be called directly. result.add format(value, if specifier.len == 0: "yyyy-MM-dd'T'HH:mm:sszzz" else: specifier) @@ -2133,10 +2133,6 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string const f2 = initTimeFormat(f) result = time.inZone(zone).format(f2) -template formatValue*(result: var string; value: Time, specifier: string) = - ## adapter for `strformat`. Not intended to be called directly. - result.add format(value, specifier) - proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), loc: DateTimeLocale = DefaultLocale): DateTime {.raises: [TimeParseError, Defect].} = diff --git a/tests/stdlib/t21406.nim b/tests/stdlib/t21406.nim new file mode 100644 index 000000000..5b96227ce --- /dev/null +++ b/tests/stdlib/t21406.nim @@ -0,0 +1,5 @@ +import std/[times, strformat] +import std/assertions + +doAssert fmt"{getTime()}" == $getTime() +doAssert fmt"{now()}" == $now() From de65b380edb0fe256054e217a1e6e64a6bb97527 Mon Sep 17 00:00:00 2001 From: Ecorous <49562894+Ecorous@users.noreply.github.com> Date: Tue, 21 Feb 2023 11:14:41 +0000 Subject: [PATCH 1601/1654] Add `getDataDir` proc (#21408) * Add getDataDir() * Update lib/std/private/osappdirs.nim --------- Co-authored-by: Andreas Rumpf --- lib/std/private/osappdirs.nim | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/std/private/osappdirs.nim b/lib/std/private/osappdirs.nim index 581b511ff..07a6809bb 100644 --- a/lib/std/private/osappdirs.nim +++ b/lib/std/private/osappdirs.nim @@ -12,6 +12,7 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", ## for the convenience of processing paths coming from user configuration files. ## ## See also: + ## * `getDataDir proc`_ ## * `getConfigDir proc`_ ## * `getTempDir proc`_ ## * `expandTilde proc`_ @@ -24,6 +25,30 @@ proc getHomeDir*(): string {.rtl, extern: "nos$1", when defined(windows): return getEnv("USERPROFILE") & "\\" else: return getEnv("HOME") & "/" +proc getDataDir*(): string {.rtl, extern: "nos$1" + tags: [ReadEnvEffect, ReadIOEffect].} = + ## Returns the data directory of the current user for applications. + ## + ## On non-Windows OSs, this proc conforms to the XDG Base Directory + ## spec. Thus, this proc returns the value of the `XDG_DATA_HOME` environment + ## variable if it is set, otherwise it returns the default configuration + ## directory ("~/.local/share" or "~/Library/Application Support" on macOS). + ## + ## See also: + ## * `getHomeDir proc`_ + ## * `getConfigDir proc`_ + ## * `getTempDir proc`_ + ## * `expandTilde proc`_ + ## * `getCurrentDir proc`_ + ## * `setCurrentDir proc`_ + when defined(windows): + result = getEnv("APPDATA") + elif defined(macosx): + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / "Library" / "Application Support") + else: + result = getEnv("XDG_DATA_HOME", getEnv("HOME") / ".local" / "share") + result.normalizePathEnd(trailingSep = true) + proc getConfigDir*(): string {.rtl, extern: "nos$1", tags: [ReadEnvEffect, ReadIOEffect].} = ## Returns the config directory of the current user for applications. @@ -38,6 +63,7 @@ proc getConfigDir*(): string {.rtl, extern: "nos$1", ## ## See also: ## * `getHomeDir proc`_ + ## * `getDataDir proc`_ ## * `getTempDir proc`_ ## * `expandTilde proc`_ ## * `getCurrentDir proc`_ @@ -63,6 +89,7 @@ proc getCacheDir*(): string = ## * `getHomeDir proc`_ ## * `getTempDir proc`_ ## * `getConfigDir proc`_ + ## * `getDataDir proc`_ # follows https://crates.io/crates/platform-dirs when defined(windows): result = getEnv("LOCALAPPDATA") From 0a45543cc1e8a2e8db59eaed864cd908726bf48d Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 21 Feb 2023 22:27:12 +1100 Subject: [PATCH 1602/1654] Specify that address is taken when converter takes a var parameter (#21391) * Add test case * closes #21247 Add the sfAddrTaken flag to var parameters in converters This allows the JS backend to properly pass the parameter as a fat pointer --- compiler/sigmatch.nim | 1 + tests/js/t21247.nim | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/js/t21247.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 0dbd5e5f1..f8c7fa870 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1980,6 +1980,7 @@ proc userConvMatch(c: PContext, m: var TCandidate, f, a: PType, param = implicitConv(nkHiddenSubConv, src, copyTree(arg), m, c) elif src.kind in {tyVar}: # Analyse the converter return type + arg.sym.flags.incl sfAddrTaken param = newNodeIT(nkHiddenAddr, arg.info, s.typ[1]) param.add copyTree(arg) else: diff --git a/tests/js/t21247.nim b/tests/js/t21247.nim new file mode 100644 index 000000000..5b38787b3 --- /dev/null +++ b/tests/js/t21247.nim @@ -0,0 +1,15 @@ +import std/typetraits + +type + QueryParams* = distinct seq[(string, string)] + +converter toBase*(params: var QueryParams): var seq[(string, string)] = + params.distinctBase + +proc foo(): QueryParams = + # Issue was that the implicit converter call didn't say that it took the + # address of the parameter it was converting. This led to the parameter not being + # passed as a fat pointer which toBase expected + result.add(("hello", "world")) + +assert foo().distinctBase() == @[("hello", "world")] From a3603c8a6ee4a8ae147d2f2099a18395e819a8a3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 22 Feb 2023 06:42:28 +0800 Subject: [PATCH 1603/1654] saves one unnecessary compare which is also a small regression (#21413) saves one unnecessary compare which is also a sall regression follow up https://github.com/nim-lang/Nim/commit/d30c6419a051a815e3fdb354ac79522f17e55bda --- compiler/passes.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/passes.nim b/compiler/passes.nim index a8f67300c..c20763fc3 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -143,7 +143,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; while true: openParser(p, fileIdx, s, graph.cache, graph.config) - if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"): + if (not belongsToStdlib(graph, module)) or module.name.s == "distros": # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only From 64a788cafb287ace8c63f5e86cb84c520eab3f2a Mon Sep 17 00:00:00 2001 From: Benji York Date: Wed, 22 Feb 2023 05:12:10 -0600 Subject: [PATCH 1604/1654] Fix a couple of small keyword issues. (#21416) In this section of the manual, "if" should be enclosed in backticks and "elif" should be lower case. --- doc/manual.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual.md b/doc/manual.md index 779e99186..ef790c362 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -3574,8 +3574,8 @@ Example: var y = if x > 8: 9 else: 10 ``` -An if expression always results in a value, so the `else` part is -required. `Elif` parts are also allowed. +An `if` expression always results in a value, so the `else` part is +required. `elif` parts are also allowed. When expression --------------- From ab1d4a5d58af8452b26f57f244bcd0d39b409c5e Mon Sep 17 00:00:00 2001 From: Century Systems Date: Thu, 23 Feb 2023 03:53:04 +0900 Subject: [PATCH 1605/1654] ioselectors_epoll: for NuttX, limit initial numFD to configured value. (#21421) ioselectors: ioselectors_epoll: for NuttX, limit initial numFD to configured value. In the NuttX build config, there is a setting called "FS_NEPOLL_DESCRIPTORS". -------- config FS_NEPOLL_DESCRIPTORS int "Maximum number of default epoll descriptors for epoll_create1(2)" default 8 ---help--- The maximum number of default epoll descriptors for epoll_create1(2) -------- For NuttX, change the number of fd arrays allocated by newSelector() to that value. Signed-off-by: Takeyoshi Kikuchi --- lib/posix/posix_other_consts.nim | 3 +++ lib/pure/ioselects/ioselectors_epoll.nim | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/posix/posix_other_consts.nim b/lib/posix/posix_other_consts.nim index 506c92158..08069fe9a 100644 --- a/lib/posix/posix_other_consts.nim +++ b/lib/posix/posix_other_consts.nim @@ -750,3 +750,6 @@ var SEEK_SET* {.importc: "SEEK_SET", header: "".}: cint var SEEK_CUR* {.importc: "SEEK_CUR", header: "".}: cint var SEEK_END* {.importc: "SEEK_END", header: "".}: cint +# +when defined(nuttx): + var NEPOLL_MAX* {.importc: "CONFIG_FS_NEPOLL_DESCRIPTORS", header: "".}: cint diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 8526eb8a3..08cb6ed74 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -72,11 +72,16 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = + proc initialNumFD(): int {.inline.} = + when defined(nuttx): + result = NEPOLL_MAX + else: + result = 1024 # Retrieve the maximum fd count (for current OS) via getrlimit() var maxFD = maxDescriptors() doAssert(maxFD > 0) # Start with a reasonable size, checkFd() will grow this on demand - const numFD = 1024 + let numFD = initialNumFD() var epollFD = epoll_create1(O_CLOEXEC) if epollFD < 0: From fdd75202576634f1fb8c7948d6031e34af9ea67e Mon Sep 17 00:00:00 2001 From: c-blake Date: Wed, 22 Feb 2023 13:54:04 -0500 Subject: [PATCH 1606/1654] Fix the TODO portion of recently added `posix_fallocate` on OS X. (#21387) --- lib/posix/posix.nim | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index aee9308a6..07652c64d 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -198,9 +198,24 @@ proc posix_fadvise*(a1: cint, a2, a3: Off, a4: cint): cint {. proc ftruncate*(a1: cint, a2: Off): cint {.importc, header: "".} when defined(osx): # 2001 POSIX evidently does not concern Apple + type FStore {.importc: "fstore_t", header: "", bycopy.} = object + fst_flags: uint32 ## IN: flags word + fst_posmode: cint ## IN: indicates offset field + fst_offset, ## IN: start of the region + fst_length, ## IN: size of the region + fst_bytesalloc: Off ## OUT: number of bytes allocated + var F_PEOFPOSMODE {.importc, header: "".}: cint + var F_ALLOCATEALL {.importc, header: "".}: uint32 + var F_PREALLOCATE {.importc, header: "".}: cint proc posix_fallocate*(a1: cint, a2, a3: Off): cint = - ftruncate(a1, a2 + a3) # Set size to off + len, max offset -else: # TODO: Use fcntl(fd, F_PREALLOCATE, ..) above + var fst = FStore(fst_flags: F_ALLOCATEALL, fst_posmode: F_PEOFPOSMODE, + fst_offset: a2, fst_length: a3) + # Must also call ftruncate to match what POSIX does. Unlike posix_fallocate, + # this can shrink files. Could guard w/getFileSize, but caller likely knows + # present size & has no good reason to call this unless it is growing. + if fcntl(a1, F_PREALLOCATE, fst.addr) != cint(-1): ftruncate(a1, a2 + a3) + else: cint(-1) +else: proc posix_fallocate*(a1: cint, a2, a3: Off): cint {. importc, header: "".} From 38f876dd484fc1a3ce44788b1ed9601eea8619f6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 23 Feb 2023 03:34:20 +0800 Subject: [PATCH 1607/1654] fixes #19795; fixes #11852; fixes #19974; remove parsing pipeline, Nim now parses the whole module at one time (#21379) * fixes #19795; remove parse pipeline * isScript * fixes nimscriptapi * don't touch reorder * check script * fixes tests * it seems implicit imports of system cause troubles * access the first child of `nkStmtList` * ignore comments * minor messages * perhaps increases hloLoopDetector * the module is a stmtList, which changes the errors * fixes nimdoc * fixes tlinter * fixes nim secret tests * fixes arc_misc * fixes nim secret tests again * safe; fixes one more test * GlobalError is the root cause too * fixes parsing errors * put emit types to the cfsForwardTypes section * fixes #11852; `{.push checks:off}` now works in procs * disable navigator * fixes nimdoc * add tests for JS * fixes nimsuggest --- compiler/backendpragmas.nim | 25 ++++++ compiler/ccgstmts.nim | 10 ++- compiler/cgen.nim | 2 +- compiler/cgendata.nim | 1 + compiler/jsgen.nim | 10 ++- compiler/passes.nim | 7 +- compiler/pragmas.nim | 2 +- compiler/sem.nim | 6 ++ .../expected/subdir/subdir_b/utils.idx | 2 +- nimsuggest/tests/tchk1.nim | 2 +- tests/arc/tarcmisc.nim | 4 +- .../tdont_return_unowned_from_owned.nim | 35 ++++---- tests/errmsgs/t8064.nim | 8 +- tests/misc/tnoforward.nim | 1 + tests/misc/trunner.nim | 4 +- tests/modules/t8665.nim | 1 + tests/modules/treorder.nim | 2 +- tests/navigator/tincludefile.nim | 2 +- tests/navigator/tnav1.nim | 2 +- tests/nimdoc/trunnableexamples.nim | 4 +- .../{overflw => overflow}/tdistinct_range.nim | 0 tests/overflow/toverflow.nim | 82 +++++++++++++++++++ .../toverflw2.nim => overflow/toverflow2.nim} | 0 .../toverflow_reorder.nim} | 4 +- tests/{overflw => overflow}/tovfint.nim | 0 tests/parser/t20922.nim | 29 ++----- tests/pragmas/tpush.nim | 12 +++ tests/pragmas/treorder.nim | 1 + tests/proc/t19795.nim | 19 +++++ tests/tools/tlinter.nim | 2 +- tests/trmacros/tor.nim | 20 +++-- 31 files changed, 227 insertions(+), 72 deletions(-) create mode 100644 compiler/backendpragmas.nim rename tests/{overflw => overflow}/tdistinct_range.nim (100%) create mode 100644 tests/overflow/toverflow.nim rename tests/{overflw/toverflw2.nim => overflow/toverflow2.nim} (100%) rename tests/{overflw/toverflw.nim => overflow/toverflow_reorder.nim} (96%) rename tests/{overflw => overflow}/tovfint.nim (100%) create mode 100644 tests/proc/t19795.nim diff --git a/compiler/backendpragmas.nim b/compiler/backendpragmas.nim new file mode 100644 index 000000000..b18644810 --- /dev/null +++ b/compiler/backendpragmas.nim @@ -0,0 +1,25 @@ +import pragmas, options, ast, trees + +proc pushBackendOption(optionsStack: var seq[TOptions], options: var TOptions) = + optionsStack.add options + +proc popBackendOption(optionsStack: var seq[TOptions], options: var TOptions) = + options = optionsStack[^1] + optionsStack.setLen(optionsStack.len-1) + +proc processPushBackendOption*(optionsStack: var seq[TOptions], options: var TOptions, + n: PNode, start: int) = + pushBackendOption(optionsStack, options) + for i in start..= 1 and n[0].kind in {nkStrLit..nkTripleStrLit}: let sec = n[0].strVal - if sec.startsWith("/*TYPESECTION*/"): result = cfsTypes + if sec.startsWith("/*TYPESECTION*/"): result = cfsForwardTypes # TODO WORKAROUND elif sec.startsWith("/*VARSECTION*/"): result = cfsVars elif sec.startsWith("/*INCLUDESECTION*/"): result = cfsHeaders @@ -1555,9 +1554,14 @@ proc genEmit(p: BProc, t: PNode) = line(p, cpsStmts, s) proc genPragma(p: BProc, n: PNode) = - for it in n.sons: + for i in 0.. 0 if we are within a loop splitDecls*: int # > 0 if we are in some context for C++ that diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 60399d3cd..572d19b13 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -33,7 +33,7 @@ import nversion, msgs, idents, types, ropes, passes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, - transf, injectdestructors, sourcemap, astmsgs + transf, injectdestructors, sourcemap, astmsgs, backendpragmas import json, sets, math, tables, intsets import strutils except addf @@ -98,6 +98,7 @@ type prc: PSym globals, locals, body: Rope options: TOptions + optionsStack: seq[TOptions] module: BModule g: PGlobals generatedParamCopies: IntSet @@ -2557,9 +2558,14 @@ proc genStmt(p: PProc, n: PNode) = if r.res != "": lineF(p, "$#;$n", [r.res]) proc genPragma(p: PProc, n: PNode) = - for it in n.sons: + for i in 0..> (toplevel): import(dirty): tests/tchk1.nim [Processing]";;0 chk;;skUnknown;;;;Error;;$file;;12;;0;;"identifier expected, but got \'keyword template\'";;0 chk;;skUnknown;;;;Error;;$file;;14;;0;;"nestable statement requires indentation";;0 -chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 chk;;skUnknown;;;;Error;;$file;;17;;0;;"invalid indentation";;0 +chk;;skUnknown;;;;Error;;$file;;12;;0;;"implementation of \'foo\' expected";;0 chk;;skUnknown;;;;Hint;;$file;;12;;9;;"\'foo\' is declared but not used [XDeclaredButNotUsed]";;0 chk;;skUnknown;;;;Hint;;$file;;14;;5;;"\'main\' is declared but not used [XDeclaredButNotUsed]";;0 """ diff --git a/tests/arc/tarcmisc.nim b/tests/arc/tarcmisc.nim index 6435f5e94..a03da6211 100644 --- a/tests/arc/tarcmisc.nim +++ b/tests/arc/tarcmisc.nim @@ -27,14 +27,14 @@ new line after - @['a'] finalizer aaaaa hello -ok true copying 123 42 -closed +ok destroying variable: 20 destroying variable: 10 +closed ''' cmd: "nim c --gc:arc --deepcopy:on -d:nimAllocPagesViaMalloc $file" """ diff --git a/tests/destructor/tdont_return_unowned_from_owned.nim b/tests/destructor/tdont_return_unowned_from_owned.nim index d27626dea..ffe87cd76 100644 --- a/tests/destructor/tdont_return_unowned_from_owned.nim +++ b/tests/destructor/tdont_return_unowned_from_owned.nim @@ -1,9 +1,12 @@ discard """ cmd: "nim check --newruntime --hints:off $file" nimout: ''' -tdont_return_unowned_from_owned.nim(36, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type -tdont_return_unowned_from_owned.nim(39, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type -tdont_return_unowned_from_owned.nim(42, 6) Error: type mismatch: got +tdont_return_unowned_from_owned.nim(26, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref +tdont_return_unowned_from_owned.nim(27, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref +tdont_return_unowned_from_owned.nim(31, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type +tdont_return_unowned_from_owned.nim(43, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type +tdont_return_unowned_from_owned.nim(46, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(Obj)' as the return type +tdont_return_unowned_from_owned.nim(49, 6) Error: type mismatch: got but expected one of: proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.}) first type mismatch at position: 2 @@ -11,17 +14,21 @@ proc new[T](a: var ref T; finalizer: proc (x: ref T) {.nimcall.}) 2 other mismatching symbols have been suppressed; compile with --showAllMismatches:on to see them expression: new(result) -tdont_return_unowned_from_owned.nim(42, 6) Error: illformed AST: -tdont_return_unowned_from_owned.nim(50, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref -tdont_return_unowned_from_owned.nim(51, 13) Error: assignment produces a dangling ref: the unowned ref lives longer than the owned ref -tdont_return_unowned_from_owned.nim(55, 10) Error: cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type +tdont_return_unowned_from_owned.nim(49, 6) Error: illformed AST: ''' - errormsg: "cannot return an owned pointer as an unowned pointer; use 'owned(RootRef)' as the return type" + errormsg: "illformed AST:" """ +proc testA(result: var (RootRef, RootRef)) = + let r: owned RootRef = RootRef() + result[0] = r + result[1] = RootRef() +proc testB(): RootRef = + let r: owned RootRef = RootRef() + result = r @@ -39,17 +46,11 @@ proc newObjB(): Obj = result = Obj() proc newObjC(): Obj = - new(result) + new(result) # illFormedAst raises GlobalError, + # without pipeline parsing, it needs to placed at the end + # in case that it disturbs other errors let a = newObjA() let b = newObjB() let c = newObjC() -proc testA(result: var (RootRef, RootRef)) = - let r: owned RootRef = RootRef() - result[0] = r - result[1] = RootRef() - -proc testB(): RootRef = - let r: owned RootRef = RootRef() - result = r diff --git a/tests/errmsgs/t8064.nim b/tests/errmsgs/t8064.nim index 10bb86299..e7941e36a 100644 --- a/tests/errmsgs/t8064.nim +++ b/tests/errmsgs/t8064.nim @@ -1,6 +1,8 @@ +import tables + +values + + discard """ errormsg: "expression has no type: values" """ -import tables - -values \ No newline at end of file diff --git a/tests/misc/tnoforward.nim b/tests/misc/tnoforward.nim index b6a71897a..0d9b40c83 100644 --- a/tests/misc/tnoforward.nim +++ b/tests/misc/tnoforward.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--experimental:codeReordering" output: "10" """ diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 7fb437ce3..aca19f72d 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -342,8 +342,8 @@ tests/newconfig/bar/mfoo.nims""".splitLines when not defined(windows): check3 lines.len == 5 check3 lines[0].isDots - check3 lines[1].dup(removePrefix(">>> ")) == "3" # prompt depends on `nimUseLinenoise` - check3 lines[2].isDots + # check3 lines[1].isDots # todo nim secret might use parsing pipeline + check3 lines[2].dup(removePrefix(">>> ")) == "3" # prompt depends on `nimUseLinenoise` check3 lines[3] == "ab" check3 lines[4] == "" else: diff --git a/tests/modules/t8665.nim b/tests/modules/t8665.nim index 74d31452f..7cfdbdb00 100644 --- a/tests/modules/t8665.nim +++ b/tests/modules/t8665.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--experimental:codeReordering" action: compile """ diff --git a/tests/modules/treorder.nim b/tests/modules/treorder.nim index c81715cd8..626d9684b 100644 --- a/tests/modules/treorder.nim +++ b/tests/modules/treorder.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim -d:testdef $target $file" + matrix: "--experimental:codeReordering -d:testdef" output: '''works 34 34 defined diff --git a/tests/navigator/tincludefile.nim b/tests/navigator/tincludefile.nim index f35ab2ec9..a913d0736 100644 --- a/tests/navigator/tincludefile.nim +++ b/tests/navigator/tincludefile.nim @@ -1,4 +1,5 @@ discard """ + disabled: true cmd: "nim check $options --defusages:$file,12,7 $file" nimout: '''def tincludefile_temp.nim(11, 10) usage tincludefile_temp.nim(12, 8) @@ -7,7 +8,6 @@ usage tincludefile_temp.nim(12, 8) - proc foo(x: int) = echo x diff --git a/tests/navigator/tnav1.nim b/tests/navigator/tnav1.nim index b6bbdbf19..e76c921f3 100644 --- a/tests/navigator/tnav1.nim +++ b/tests/navigator/tnav1.nim @@ -1,11 +1,11 @@ discard """ + disabled: true cmd: "nim check $options --defusages:$file,12,7 $file" nimout: '''def tnav1_temp.nim(11, 10) usage tnav1_temp.nim(12, 8) ''' """ - import std / [times] proc foo(x: int) = diff --git a/tests/nimdoc/trunnableexamples.nim b/tests/nimdoc/trunnableexamples.nim index c88d8fa3f..1188cdd1b 100644 --- a/tests/nimdoc/trunnableexamples.nim +++ b/tests/nimdoc/trunnableexamples.nim @@ -7,7 +7,6 @@ foo1 foo2 foo3 foo5 -foo6 foo7 in examplesInTemplate1 doc in outer @@ -15,6 +14,7 @@ doc in inner1 doc in inner2 foo8 foo9 +foo6 ''' joinable: false """ @@ -43,7 +43,7 @@ proc fun*() = proc fun*()=echo "foo5" fun() - runnableExamples: + runnableExamples("--experimental:codeReordering --warnings:off"): # `codeReordering` only allowed at top level {.experimental: "codeReordering".} proc fun1() = fun2() diff --git a/tests/overflw/tdistinct_range.nim b/tests/overflow/tdistinct_range.nim similarity index 100% rename from tests/overflw/tdistinct_range.nim rename to tests/overflow/tdistinct_range.nim diff --git a/tests/overflow/toverflow.nim b/tests/overflow/toverflow.nim new file mode 100644 index 000000000..972f929c6 --- /dev/null +++ b/tests/overflow/toverflow.nim @@ -0,0 +1,82 @@ +discard """ + output: "ok" + matrix: "--overflowChecks:off; --overflowChecks:off --b:js" +""" +# Tests nim's ability to detect overflows + +{.push overflowChecks: on.} + +var + a = high(int) + b = -2 + overflowDetected = false + +try: + echo(b - a) +except OverflowDefect: + overflowDetected = true + +{.pop.} # overflow check + +doAssert(overflowDetected) + +block: # Overflow checks in a proc + var + a = high(int) + b = -2 + overflowDetected = false + + {.push overflowChecks: on.} + proc foo() = + let c = b - a + {.pop.} + + try: + foo() + except OverflowDefect: + overflowDetected = true + + doAssert(overflowDetected) + +block: # Overflow checks in a forward declared proc + var + a = high(int) + b = -2 + overflowDetected = false + + proc foo() + + {.push overflowChecks: on.} + proc foo() = + let c = b - a + {.pop.} + + try: + foo() + except OverflowDefect: + overflowDetected = true + + doAssert(overflowDetected) + +block: # Overflow checks doesn't affect fwd declaration + var + a = high(int) + b = -2 + overflowDetected = false + + {.push overflowChecks: on.} + proc foo() + {.pop.} + + proc foo() = + let c = b - a + + try: + foo() + except OverflowDefect: + overflowDetected = true + + doAssert(not overflowDetected) + + +echo "ok" diff --git a/tests/overflw/toverflw2.nim b/tests/overflow/toverflow2.nim similarity index 100% rename from tests/overflw/toverflw2.nim rename to tests/overflow/toverflow2.nim diff --git a/tests/overflw/toverflw.nim b/tests/overflow/toverflow_reorder.nim similarity index 96% rename from tests/overflw/toverflw.nim rename to tests/overflow/toverflow_reorder.nim index 164e16e5c..fcf7b0c82 100644 --- a/tests/overflw/toverflw.nim +++ b/tests/overflow/toverflow_reorder.nim @@ -1,3 +1,5 @@ +{.experimental: "codeReordering".} + discard """ output: "ok" cmd: "nim $target --overflowChecks:off $options $file" @@ -12,7 +14,7 @@ var overflowDetected = false try: - writeLine(stdout, b - a) + echo b - a except OverflowDefect: overflowDetected = true diff --git a/tests/overflw/tovfint.nim b/tests/overflow/tovfint.nim similarity index 100% rename from tests/overflw/tovfint.nim rename to tests/overflow/tovfint.nim diff --git a/tests/parser/t20922.nim b/tests/parser/t20922.nim index 01af9868f..110610fb1 100644 --- a/tests/parser/t20922.nim +++ b/tests/parser/t20922.nim @@ -1,28 +1,17 @@ discard """ - cmd: "nim check $options --verbosity:0 $file" + cmd: "nim check $options --verbosity:0 --hints:off $file" action: "reject" nimout: ''' -t20922.nim(37, 5) Error: expression expected, but found ':' +t20922.nim(26, 5) Error: expression expected, but found ':' +t20922.nim(34, 7) Error: expression expected, but found ':' +t20922.nim(35, 5) Error: ':' or '=' expected, but got 'keyword of' Error: in expression ' '+'': identifier expected, but found '' -t20922.nim(37, 7) Error: attempting to call undeclared routine: '' +t20922.nim(26, 7) Error: attempting to call undeclared routine: '' Error: in expression ' '+'': identifier expected, but found '' -t20922.nim(37, 7) Error: attempting to call undeclared routine: '' -t20922.nim(37, 7) Error: expression '' cannot be called -t20922.nim(37, 7) Error: expression '' has no type (or is ambiguous) -t20922.nim(37, 7) Error: VM problem: dest register is not set -t20922.nim(45, 7) Error: expression expected, but found ':' -t20922.nim(46, 5) Error: ':' or '=' expected, but got 'keyword of' -t20922.nim(45, 9) Error: undeclared identifier: 'x' -t20922.nim(45, 9) Error: expression 'x' has no type (or is ambiguous) -Error: in expression ' x': identifier expected, but found '' -t20922.nim(45, 9) Error: attempting to call undeclared routine: '' -Error: in expression ' x': identifier expected, but found '' -t20922.nim(45, 9) Error: attempting to call undeclared routine: '' -t20922.nim(45, 9) Error: expression '' cannot be called -t20922.nim(45, 9) Error: expression '' has no type (or is ambiguous) -t20922.nim(45, 9) Error: VM problem: dest register is not set -t20922.nim(33, 6) Hint: 'mapInstrToToken' is declared but not used [XDeclaredButNotUsed] -t20922.nim(43, 3) Hint: 'Foo' is declared but not used [XDeclaredButNotUsed] +t20922.nim(26, 7) Error: attempting to call undeclared routine: '' +t20922.nim(26, 7) Error: expression '' cannot be called +t20922.nim(26, 7) Error: expression '' has no type (or is ambiguous) +t20922.nim(26, 7) Error: VM problem: dest register is not set ''' """ # original test case issue #20922 diff --git a/tests/pragmas/tpush.nim b/tests/pragmas/tpush.nim index 5ecfe9704..f2779ea70 100644 --- a/tests/pragmas/tpush.nim +++ b/tests/pragmas/tpush.nim @@ -13,3 +13,15 @@ proc WarnMe() = x: int echo(x) +# bug #11852 +proc foo(x: string, y: int, res: int) = + {.push checks: off} + var a: ptr char = unsafeAddr(x[y]) + {.pop.} + if x.len > y: + doAssert ord(a[]) == 51 + else: + doAssert x.len + 48 == res + +foo("", 0, 48) +foo("abc", 40, 51) diff --git a/tests/pragmas/treorder.nim b/tests/pragmas/treorder.nim index 09a98ef6a..72e8808b3 100644 --- a/tests/pragmas/treorder.nim +++ b/tests/pragmas/treorder.nim @@ -1,4 +1,5 @@ discard """ +matrix: "--experimental:codeReordering" output:'''0 1 2 diff --git a/tests/proc/t19795.nim b/tests/proc/t19795.nim new file mode 100644 index 000000000..677ec0a63 --- /dev/null +++ b/tests/proc/t19795.nim @@ -0,0 +1,19 @@ +discard """ + matrix: "--mm:arc" +""" + +# bug #19795 +# bug #21085 + +type Vector = seq[int] + +var vect: Vector = newSeq[int](5) +doAssert vect == @[0, 0, 0, 0, 0] + +# Needed to get the problem. Could also use "var". +let vectCopy = vect + +# Then some procedure definition is needed to get the problem. +proc p(): int = 3 + +doAssert vect == @[0, 0, 0, 0, 0] \ No newline at end of file diff --git a/tests/tools/tlinter.nim b/tests/tools/tlinter.nim index a6d45ab3b..16f67905e 100644 --- a/tests/tools/tlinter.nim +++ b/tests/tools/tlinter.nim @@ -1,10 +1,10 @@ discard """ cmd: '''nim c --styleCheck:hint $file''' nimout: ''' +tlinter.nim(25, 1) Hint: 'tyPE' should be: 'type' [Name] tlinter.nim(21, 14) Hint: 'nosideeffect' should be: 'noSideEffect' [Name] tlinter.nim(21, 28) Hint: 'myown' should be: 'myOwn' [template declared in tlinter.nim(19, 9)] [Name] tlinter.nim(21, 35) Hint: 'inLine' should be: 'inline' [Name] -tlinter.nim(25, 1) Hint: 'tyPE' should be: 'type' [Name] tlinter.nim(23, 1) Hint: 'foO' should be: 'foo' [proc declared in tlinter.nim(21, 6)] [Name] tlinter.nim(27, 14) Hint: 'Foo_bar' should be: 'FooBar' [type declared in tlinter.nim(25, 6)] [Name] tlinter.nim(29, 6) Hint: 'someVAR' should be: 'someVar' [var declared in tlinter.nim(27, 5)] [Name] diff --git a/tests/trmacros/tor.nim b/tests/trmacros/tor.nim index 087dc0d68..6b4e71216 100644 --- a/tests/trmacros/tor.nim +++ b/tests/trmacros/tor.nim @@ -1,9 +1,19 @@ discard """ - output: '''0 + output: ''' +3 +0 true -3''' +''' """ + +# bug #798 +template t012{(0|1|2){x}}(x: untyped): untyped = x+1 +let z = 1 +# outputs 3 thanks to fixpoint iteration: +echo z + + template arithOps: untyped = (`+` | `-` | `*`) template testOr{ (arithOps{f})(a, b) }(a, b, f: untyped): untyped = f(a mod 10, b) @@ -20,9 +30,3 @@ var c = false a = b and a echo a - -# bug #798 -template t012{(0|1|2){x}}(x: untyped): untyped = x+1 -let z = 1 -# outputs 3 thanks to fixpoint iteration: -echo z From 7dc80a7fefb0d2cecbdc2696c71ea2fa94d6aa76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francis=20Th=C3=A9rien?= Date: Wed, 22 Feb 2023 15:10:02 -0500 Subject: [PATCH 1608/1654] Mitigate issues related to compiler options when cross-compiling (#21330) * Document C compiler options config when cross-compiling * Allow empty string to override default --- compiler/extccomp.nim | 4 +++- doc/nimc.md | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index db97de65c..45a531852 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -328,7 +328,9 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string = platform.OS[conf.target.targetOS].name & '.' & CC[c].name & fullSuffix result = getConfigVar(conf, fullCCname) - if result.len == 0: + if existsConfigVar(conf, fullCCname): + result = getConfigVar(conf, fullCCname) + else: # not overridden for this cross compilation setting? result = getConfigVar(conf, CC[c].name & fullSuffix) else: diff --git a/doc/nimc.md b/doc/nimc.md index 81a060d8a..7b0e99e06 100644 --- a/doc/nimc.md +++ b/doc/nimc.md @@ -316,14 +316,16 @@ Another way is to make Nim invoke a cross compiler toolchain: nim c --cpu:arm --os:linux myproject.nim ``` -For cross compilation, the compiler invokes a C compiler named -like `$cpu.$os.$cc` (for example arm.linux.gcc) and the configuration -system is used to provide meaningful defaults. For example for `ARM` your +For cross compilation, the compiler invokes a C compiler named like +`$cpu.$os.$cc` (for example `arm.linux.gcc`) with options defined in +`$cpu.$os.$cc.options.always`. The configuration system is used to provide +meaningful defaults. For example, for Linux on a 32-bit ARM CPU, your configuration file should contain something like: arm.linux.gcc.path = "/usr/bin" arm.linux.gcc.exe = "arm-linux-gcc" arm.linux.gcc.linkerexe = "arm-linux-gcc" + arm.linux.gcc.options.always = "-w -fmax-errors=3" Cross-compilation for Windows ============================= From 8a19ac20708c66fb3541533d55b60624cee56ab7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 24 Feb 2023 09:02:15 +0100 Subject: [PATCH 1609/1654] fixes #21393 and misc style changes (#21419) * fixes #21393 and misc style changes * progress --------- Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- compiler/int128.nim | 39 +++++++++++++++-------------- compiler/sempass2.nim | 19 +++++++------- lib/pure/collections/tables.nim | 31 +++++++++++++++-------- lib/pure/hashes.nim | 5 +++- lib/pure/osproc.nim | 8 ++---- lib/std/private/digitsutils.nim | 4 +-- lib/std/private/ospaths2.nim | 2 +- lib/system/seqs_v2.nim | 3 ++- lib/system/strmantle.nim | 2 +- lib/windows/winlean.nim | 16 ++++++------ tests/effects/tstrictfuncs_misc.nim | 2 +- 11 files changed, 70 insertions(+), 61 deletions(-) diff --git a/compiler/int128.nim b/compiler/int128.nim index e357706b9..b0341eb37 100644 --- a/compiler/int128.nim +++ b/compiler/int128.nim @@ -33,26 +33,27 @@ template high*(t: typedesc[Int128]): Int128 = Max proc `$`*(a: Int128): string proc toInt128*[T: SomeInteger | bool](arg: T): Int128 = - when T is bool: result.sdata(0) = int32(arg) - elif T is SomeUnsignedInt: - when sizeof(arg) <= 4: - result.udata[0] = uint32(arg) + {.noSideEffect.}: + when T is bool: result.sdata(0) = int32(arg) + elif T is SomeUnsignedInt: + when sizeof(arg) <= 4: + result.udata[0] = uint32(arg) + else: + result.udata[0] = uint32(arg and T(0xffffffff)) + result.udata[1] = uint32(arg shr 32) + elif sizeof(arg) <= 4: + result.sdata(0) = int32(arg) + if arg < 0: # sign extend + result.sdata(1) = -1 + result.sdata(2) = -1 + result.sdata(3) = -1 else: - result.udata[0] = uint32(arg and T(0xffffffff)) - result.udata[1] = uint32(arg shr 32) - elif sizeof(arg) <= 4: - result.sdata(0) = int32(arg) - if arg < 0: # sign extend - result.sdata(1) = -1 - result.sdata(2) = -1 - result.sdata(3) = -1 - else: - let tmp = int64(arg) - result.udata[0] = uint32(tmp and 0xffffffff) - result.sdata(1) = int32(tmp shr 32) - if arg < 0: # sign extend - result.sdata(2) = -1 - result.sdata(3) = -1 + let tmp = int64(arg) + result.udata[0] = uint32(tmp and 0xffffffff) + result.sdata(1) = int32(tmp shr 32) + if arg < 0: # sign extend + result.sdata(2) = -1 + result.sdata(3) = -1 template isNegative(arg: Int128): bool = arg.sdata(3) < 0 diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index e989eb4a6..396ea6606 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -822,9 +822,6 @@ proc checkForSink(tracked: PEffects; n: PNode) = if tracked.inIfStmt == 0 and optSinkInference in tracked.config.options: checkForSink(tracked.config, tracked.c.idgen, tracked.owner, n) -proc strictFuncsActive(tracked: PEffects): bool {.inline.} = - sfNoSideEffect in tracked.owner.flags and strictFuncs in tracked.c.features and not tracked.inEnforcedNoSideEffects - proc trackCall(tracked: PEffects; n: PNode) = template gcsafeAndSideeffectCheck() = if notGcSafe(op) and not importedFromC(a): @@ -934,9 +931,11 @@ proc trackCall(tracked: PEffects; n: PNode) = # initialized until after the call. Since we do this after we analysed the # call, this is fine. initVar(tracked, n[i].skipAddr, false) - if tracked.strictFuncsActive and isDangerousLocation(n[i].skipAddr, tracked.owner): - localError(tracked.config, n[i].info, - "cannot pass $1 to `var T` parameter within a strict func" % renderTree(n[i])) + if strictFuncs in tracked.c.features and not tracked.inEnforcedNoSideEffects and + isDangerousLocation(n[i].skipAddr, tracked.owner): + if sfNoSideEffect in tracked.owner.flags: + localError(tracked.config, n[i].info, + "cannot pass $1 to `var T` parameter within a strict func" % renderTree(n[i])) tracked.hasSideEffect = true else: discard @@ -1090,10 +1089,12 @@ proc track(tracked: PEffects, n: PNode) = createTypeBoundOps(tracked, n[0].typ, n.info) if n[0].kind != nkSym or not isLocalSym(tracked, n[0].sym): checkForSink(tracked, n[1]) - if tracked.strictFuncsActive and isDangerousLocation(n[0], tracked.owner): + if strictFuncs in tracked.c.features and not tracked.inEnforcedNoSideEffects and + isDangerousLocation(n[0], tracked.owner): tracked.hasSideEffect = true - localError(tracked.config, n[0].info, - "cannot mutate location $1 within a strict func" % renderTree(n[0])) + if sfNoSideEffect in tracked.owner.flags: + localError(tracked.config, n[0].info, + "cannot mutate location $1 within a strict func" % renderTree(n[0])) of nkVarSection, nkLetSection: for child in n: let last = lastSon(child) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 57a0107fd..4737fa478 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -447,7 +447,7 @@ proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## ## ## Note that while the value returned is of type `var B`, - ## it is easy to accidentally create an copy of the value at `t[key]`. + ## it is easy to accidentally create a copy of the value at `t[key]`. ## Remember that seqs and strings are value types, and therefore ## cannot be copied into a separate variable for modification. ## See the example below. @@ -828,7 +828,8 @@ proc newTable*[A, B](initialSize = defaultInitialSize): TableRef[A, B] = b = newTable[char, seq[int]]() new(result) - result[] = initTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initTable[A, B](initialSize) proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = ## Creates a new ref hash table that contains the given `pairs`. @@ -844,14 +845,16 @@ proc newTable*[A, B](pairs: openArray[(A, B)]): TableRef[A, B] = assert b == {'a': 5, 'b': 9}.newTable new(result) - result[] = toTable[A, B](pairs) + {.noSideEffect.}: + result[] = toTable[A, B](pairs) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. # TODO: As soon as supported, change collection: A to collection: A[B] result = newTable[C, B]() - for item in collection: - result[index(item)] = item + {.noSideEffect.}: + for item in collection: + result[index(item)] = item proc `[]`*[A, B](t: TableRef[A, B], key: A): var B = ## Retrieves the value at `t[key]`. @@ -1825,7 +1828,8 @@ proc newOrderedTable*[A, B](initialSize = defaultInitialSize): OrderedTableRef[A a = newOrderedTable[int, string]() b = newOrderedTable[char, seq[int]]() new(result) - result[] = initOrderedTable[A, B](initialSize) + {.noSideEffect.}: + result[] = initOrderedTable[A, B](initialSize) proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## Creates a new ordered ref hash table that contains the given `pairs`. @@ -1842,7 +1846,8 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = assert b == {'a': 5, 'b': 9}.newOrderedTable result = newOrderedTable[A, B](pairs.len) - for key, val in items(pairs): result[key] = val + {.noSideEffect.}: + for key, val in items(pairs): result[key] = val proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B = @@ -2641,13 +2646,15 @@ proc newCountTable*[A](initialSize = defaultInitialSize): CountTableRef[A] = ## * `initCountTable proc<#initCountTable>`_ for creating a ## `CountTable` new(result) - result[] = initCountTable[A](initialSize) + {.noSideEffect.}: + result[] = initCountTable[A](initialSize) proc newCountTable*[A](keys: openArray[A]): CountTableRef[A] = ## Creates a new ref count table with every member of a container `keys` ## having a count of how many times it occurs in that container. result = newCountTable[A](keys.len) - for key in items(keys): result.inc(key) + {.noSideEffect.}: + for key in items(keys): result.inc(key) proc `[]`*[A](t: CountTableRef[A], key: A): int = ## Retrieves the value at `t[key]` if `key` is in `t`. @@ -2671,7 +2678,8 @@ proc `[]=`*[A](t: CountTableRef[A], key: A, val: int) = ## * `inc proc<#inc,CountTableRef[A],A,int>`_ for incrementing a ## value of a key assert val > 0 - t[][key] = val + {.noSideEffect.}: + t[][key] = val proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## Increments `t[key]` by `val` (default: 1). @@ -2680,7 +2688,8 @@ proc inc*[A](t: CountTableRef[A], key: A, val = 1) = a.inc('a') a.inc('b', 10) doAssert a == newCountTable("aaabbbbbbbbbbb") - t[].inc(key, val) + {.noSideEffect.}: + t[].inc(key, val) proc smallest*[A](t: CountTableRef[A]): tuple[key: A, val: int] = ## Returns the `(key, value)` pair with the smallest `val`. Efficiency: O(n) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index 09e072812..4ae4938c6 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -247,7 +247,7 @@ proc hash*[T](x: ptr[T]): Hash {.inline.} = when defined(nimPreviewHashRef) or defined(nimdoc): proc hash*[T](x: ref[T]): Hash {.inline.} = ## Efficient `hash` overload. - ## + ## ## .. important:: Use `-d:nimPreviewHashRef` to ## enable hashing `ref`s. It is expected that this behavior ## becomes the new default in upcoming versions. @@ -536,6 +536,7 @@ proc hash*[T: tuple | object | proc](x: T): Hash = elif T is (proc): result = hash(pointer(x)) else: + result = 0 for f in fields(x): result = result !& hash(f) result = !$result @@ -551,6 +552,7 @@ proc hash*[A](x: openArray[A]): Hash = else: result = murmurHash(toOpenArrayByte(x, 0, x.high)) else: + result = 0 for a in x: result = result !& hash(a) result = !$result @@ -583,6 +585,7 @@ proc hash*[A](aBuf: openArray[A], sPos, ePos: int): Hash = proc hash*[A](x: set[A]): Hash = ## Efficient hashing of sets. ## There must be a `hash` proc defined for the element type `A`. + result = 0 for it in items(x): result = result !& hash(it) result = !$result diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 9a0e5895d..bb6abc571 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -570,12 +570,8 @@ when defined(windows) and not defined(useNimRtl): if a == 0: raiseOSError(osLastError()) proc newFileHandleStream(handle: Handle): owned FileHandleStream = - new(result) - result.handle = handle - result.closeImpl = hsClose - result.atEndImpl = hsAtEnd - result.readDataImpl = hsReadData - result.writeDataImpl = hsWriteData + result = FileHandleStream(handle: handle, closeImpl: hsClose, atEndImpl: hsAtEnd, + readDataImpl: hsReadData, writeDataImpl: hsWriteData) proc buildCommandLine(a: string, args: openArray[string]): string = result = quoteShell(a) diff --git a/lib/std/private/digitsutils.nim b/lib/std/private/digitsutils.nim index b3dc5d14f..e051e5218 100644 --- a/lib/std/private/digitsutils.nim +++ b/lib/std/private/digitsutils.nim @@ -101,9 +101,7 @@ proc addInt*(result: var string; x: int64) {.enforceNoRaises.} = num = cast[uint64](x) else: num = uint64(-x) - let base = result.len - setLen(result, base + 1) - result[base] = '-' + result.add '-' else: num = uint64(x) addInt(result, num) diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 5e3bece68..78a806675 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -571,7 +571,7 @@ proc normExt(ext: string): string = proc searchExtPos*(path: string): int = ## Returns index of the `'.'` char in `path` if it signifies the beginning - ## of extension. Returns -1 otherwise. + ## of the file extension. Returns -1 otherwise. ## ## See also: ## * `splitFile proc`_ diff --git a/lib/system/seqs_v2.nim b/lib/system/seqs_v2.nim index 50f23111c..d7ace446d 100644 --- a/lib/system/seqs_v2.nim +++ b/lib/system/seqs_v2.nim @@ -83,7 +83,8 @@ proc shrink*[T](x: var seq[T]; newLen: Natural) {.tags: [], raises: [].} = for i in countdown(x.len - 1, newLen): reset x[i] # XXX This is wrong for const seqs that were moved into 'x'! - cast[ptr NimSeqV2[T]](addr x).len = newLen + {.noSideEffect.}: + cast[ptr NimSeqV2[T]](addr x).len = newLen proc grow*[T](x: var seq[T]; newLen: Natural; value: T) = let oldLen = x.len diff --git a/lib/system/strmantle.nim b/lib/system/strmantle.nim index f4733ed21..ab158d6b7 100644 --- a/lib/system/strmantle.nim +++ b/lib/system/strmantle.nim @@ -33,7 +33,7 @@ proc eqStrings(a, b: string): bool {.inline, compilerproc.} = proc hashString(s: string): int {.compilerproc.} = # the compiler needs exactly the same hash function! # this used to be used for efficient generation of string case statements - var h : uint = 0 + var h = 0'u for i in 0..len(s)-1: h = h + uint(s[i]) h = h + h shl 10 diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 93ab3dd2e..230f963f0 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -48,12 +48,12 @@ type HGLRC* = Handle BYTE* = uint8 - SECURITY_ATTRIBUTES* {.final, pure.} = object + SECURITY_ATTRIBUTES* = object nLength*: int32 lpSecurityDescriptor*: pointer bInheritHandle*: WINBOOL - STARTUPINFO* {.final, pure.} = object + STARTUPINFO* = object cb*: int32 lpReserved*: cstring lpDesktop*: cstring @@ -73,17 +73,17 @@ type hStdOutput*: Handle hStdError*: Handle - PROCESS_INFORMATION* {.final, pure.} = object + PROCESS_INFORMATION* = object hProcess*: Handle hThread*: Handle dwProcessId*: int32 dwThreadId*: int32 - FILETIME* {.final, pure.} = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT + FILETIME* = object ## CANNOT BE int64 BECAUSE OF ALIGNMENT dwLowDateTime*: DWORD dwHighDateTime*: DWORD - BY_HANDLE_FILE_INFORMATION* {.final, pure.} = object + BY_HANDLE_FILE_INFORMATION* = object dwFileAttributes*: DWORD ftCreationTime*: FILETIME ftLastAccessTime*: FILETIME @@ -95,7 +95,7 @@ type nFileIndexHigh*: DWORD nFileIndexLow*: DWORD - OSVERSIONINFO* {.final, pure.} = object + OSVERSIONINFO* = object dwOSVersionInfoSize*: DWORD dwMajorVersion*: DWORD dwMinorVersion*: DWORD @@ -820,7 +820,7 @@ type POVERLAPPED_COMPLETION_ROUTINE* = proc (para1: DWORD, para2: DWORD, para3: POVERLAPPED){.stdcall.} - GUID* {.final, pure.} = object + GUID* = object D1*: int32 D2*: int16 D3*: int16 @@ -1101,7 +1101,7 @@ proc wsaResetEvent*(hEvent: Handle): bool {.stdcall, importc: "WSAResetEvent", dynlib: "ws2_32.dll".} type - KEY_EVENT_RECORD* {.final, pure.} = object + KEY_EVENT_RECORD* = object eventType*: int16 bKeyDown*: WINBOOL wRepeatCount*: int16 diff --git a/tests/effects/tstrictfuncs_misc.nim b/tests/effects/tstrictfuncs_misc.nim index 59d850763..8c573bb3a 100644 --- a/tests/effects/tstrictfuncs_misc.nim +++ b/tests/effects/tstrictfuncs_misc.nim @@ -53,7 +53,7 @@ type JsonNode3 = ref object fields: MyTable -proc `[]`(t: var MyTable, key: string): var int = +proc `[]`(t: MyTable, key: string): int = result = t.data[0] proc `[]`(x: JsonNode3, key: string): int = From bbb6d2c69d1b2bca9987c708ebc6ffd2479bb822 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 24 Feb 2023 16:02:44 +0800 Subject: [PATCH 1610/1654] fixes #20695; fixes object with distinct defaults and tables (#21428) --- compiler/semtypes.nim | 4 ++- tests/objects/tobject_default_value.nim | 39 ++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index aaffa1ea7..26493703d 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -222,9 +222,11 @@ proc isRecursiveType(t: PType, cycleDetector: var IntSet): bool = proc fitDefaultNode(c: PContext, n: PNode): PType = let expectedType = if n[^2].kind != nkEmpty: semTypeNode(c, n[^2], nil) else: nil + let oldType = n[^1].typ n[^1] = semConstExpr(c, n[^1], expectedType = expectedType) + n[^1].flags.incl nfSem if n[^2].kind != nkEmpty: - if expectedType != nil: + if expectedType != nil and oldType != expectedType: n[^1] = fitNodeConsiderViewType(c, expectedType, n[^1], n[^1].info) result = n[^1].typ else: diff --git a/tests/objects/tobject_default_value.nim b/tests/objects/tobject_default_value.nim index cdc6016e0..150fb0876 100644 --- a/tests/objects/tobject_default_value.nim +++ b/tests/objects/tobject_default_value.nim @@ -3,7 +3,7 @@ discard """ targets: "c cpp js" """ -import std/[times, macros] +import std/[times, macros, tables] type Guess = object @@ -236,6 +236,7 @@ template main {.dirty.} = doAssert x.obj.name.scale == 1 when nimvm: + # todo discard "fixme" else: when defined(gcArc) or defined(gcOrc): @@ -523,5 +524,41 @@ template main {.dirty.} = discard oToEither(O()) + block: # bug #20695 + type + Default = object + tabs: Table[string, int] = initTable[string, int]() + + let d = default(Default) + doAssert d.tabs.len == 0 + + block: + type + Default = object + tabs: Table[string, int] = Table[string, int]() + + let d = default(Default) + doAssert d.tabs.len == 0 + + + block: + type DjangoDateTime = distinct DateTime + + type Default = object + data: DjangoDateTime = DjangoDateTime(DateTime()) + + let x = default(Default) + doAssert x.data is DjangoDateTime + + block: + type DjangoDateTime = distinct DateTime + + type Default = object + data = DjangoDateTime(DateTime()) + + let x = default(Default) + doAssert x.data is DjangoDateTime + + static: main() main() From 7837e57e93b1cb89f5a80a779e853bad070de777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francis=20Th=C3=A9rien?= Date: Fri, 24 Feb 2023 03:07:35 -0500 Subject: [PATCH 1611/1654] Disable threads for os:any (#21425) --- config/nim.cfg | 5 +++++ lib/std/typedthreads.nim | 2 ++ 2 files changed, 7 insertions(+) diff --git a/config/nim.cfg b/config/nim.cfg index 973de077a..ef93ec99e 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -100,6 +100,11 @@ nimblepath="$home/.nimble/pkgs/" gcc.options.always %= "${gcc.options.always} -fsanitize=null -fsanitize-undefined-trap-on-error" @end +# Turn off threads support when compiling for bare-metal targets (--os:any) +@if any: + threads:off +@end + @if unix and mingw: # Cross compile for Windows from Linux/OSX using MinGW i386.windows.gcc.exe = "i686-w64-mingw32-gcc" diff --git a/lib/std/typedthreads.nim b/lib/std/typedthreads.nim index b45ce8a61..8359ca353 100644 --- a/lib/std/typedthreads.nim +++ b/lib/std/typedthreads.nim @@ -45,6 +45,8 @@ when defined(nimPreviewSlimSystem): when defined(genode): import genode/env +when hostOS == "any": + {.error: "Threads not implemented for os:any. Please compile with --threads:off.".} when hasAllocStack or defined(zephyr) or defined(freertos) or defined(nuttx) or defined(cpu16) or defined(cpu8): From d4782c9e42ff6cee9f674a376b1595583e08c2a3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 25 Feb 2023 00:53:04 +0800 Subject: [PATCH 1612/1654] closes #17864; add a test case (#21434) --- tests/vm/tmisc_vm.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/vm/tmisc_vm.nim b/tests/vm/tmisc_vm.nim index af09e8871..1ad830b5f 100644 --- a/tests/vm/tmisc_vm.nim +++ b/tests/vm/tmisc_vm.nim @@ -445,3 +445,15 @@ static: needSecondIdentifier.mgetOrPut(firstPart, @[]).add((name, pattern)) doAssert needSecondIdentifier[0] == @[("aaaa", "bbbb"), ("aaaaa", "bbbbb"), ("aaaaaa", "bbbbbb"), ("aaaaaaa", "bbbbbbb"), ("aaaaaaaa", "bbbbb")] + +# bug #17864 +macro transform*(fn: typed) = + quote do: + `fn` + +var map: Table[string, HashSet[string]] +proc publish*(): void {.transform.} = + map["k"] = init_hash_set[string]() + map["k"].incl "d" + +publish() From b2edfe7a02f49f07eb9fcf71cd500215725c217e Mon Sep 17 00:00:00 2001 From: Dmitry Arkhipenko <36101416+dkgitdev@users.noreply.github.com> Date: Sat, 25 Feb 2023 13:47:19 +0300 Subject: [PATCH 1613/1654] Fix: nintendoswitch compilation (#21368) * Fix: make nintendoswitch someGcc, remove symlink support for nintendoswitch, add getAppFilename for nintendoswitch * Fix: use getApplHeuristic on nintendoswitch --- lib/posix/posix.nim | 8 ++++++-- lib/pure/os.nim | 4 +++- lib/std/private/ossymlinks.nim | 2 +- lib/std/sysatomics.nim | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 07652c64d..12fc8fd57 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -584,7 +584,8 @@ proc pread*(a1: cint, a2: pointer, a3: int, a4: Off): int {. proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "".} -proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} +when not defined(nintendoswitch): + proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", header: "", varargs, tags: [WriteIOEffect].} ## A system call for device-specific input/output operations and other @@ -603,7 +604,10 @@ proc setsid*(): Pid {.importc, header: "".} proc setuid*(a1: Uid): cint {.importc, header: "".} proc sleep*(a1: cint): cint {.importc, header: "".} proc swab*(a1, a2: pointer, a3: int) {.importc, header: "".} -proc symlink*(a1, a2: cstring): cint {.importc, header: "".} +when not defined(nintendoswitch): + proc symlink*(a1, a2: cstring): cint {.importc, header: "".} +else: + proc symlink*(a1, a2: cstring): cint = -1 proc sync*() {.importc, header: "".} proc sysconf*(a1: cint): int {.importc, header: "".} proc tcgetpgrp*(a1: cint): Pid {.importc, header: "".} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ecceed671..3285c07df 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -254,7 +254,7 @@ proc findExe*(exe: string, followSymlinks: bool = true; for ext in extensions: var x = addFileExt(x, ext) if fileExists(x): - when not defined(windows): + when not (defined(windows) or defined(nintendoswitch)): while followSymlinks: # doubles as if here if x.symlinkExists: var r = newString(maxSymlinkLen) @@ -702,6 +702,8 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW result = getApplHaiku() elif defined(openbsd): result = getApplOpenBsd() + elif defined(nintendoswitch): + result = "" # little heuristic that may work on other POSIX-like systems: if result.len == 0: diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim index 53f59f939..6b2de6237 100644 --- a/lib/std/private/ossymlinks.nim +++ b/lib/std/private/ossymlinks.nim @@ -67,7 +67,7 @@ proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = ## ## See also: ## * `createSymlink proc`_ - when defined(windows): + when defined(windows) or defined(nintendoswitch): result = symlinkPath else: result = newString(maxSymlinkLen) diff --git a/lib/std/sysatomics.nim b/lib/std/sysatomics.nim index b7ccb4092..36a4e5537 100644 --- a/lib/std/sysatomics.nim +++ b/lib/std/sysatomics.nim @@ -15,7 +15,7 @@ when defined(nimPreviewSlimSystem): const hasThreadSupport = compileOption("threads") and not defined(nimscript) -const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) +const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) or defined(nintendoswitch) const someVcc = defined(vcc) or defined(clang_cl) type From 425225119a383511ac6f84ba5c02a691e6c51aea Mon Sep 17 00:00:00 2001 From: PhilippMDoerner Date: Sun, 26 Feb 2023 09:57:51 +0100 Subject: [PATCH 1614/1654] Add mention of breaking change about `[_]` in generics to changelog (#21437) Add mention of breaking change to changelog As described by #21435 , generic proc declarations making use of `_` as generic parameter are no longer possible. This is a change that was introduced by #21192 . During the debate over if this should be part of the intended spec in #21435 we concluded that it is and thus should be mentioned as a breaking change in the changelogs. --- changelogs/changelog_2_0_0.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index c8427cc2a..e081865be 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -149,6 +149,13 @@ proc foo(_, _: int): int = 123 echo foo(1, 2) ``` +- Underscores (`_`) as generic parameters are not supported and cannot be used. + Generics that use `_` as parameters will no longer compile requires you to replace `_` with something else: + + ```nim + proc foo[_](t: typedesc[_]): string = "BAR" # Can not compile + proc foo[T](t: typedesc[T]): string = "BAR" # Can compile + ``` - - Added the `--legacy:verboseTypeMismatch` switch to get legacy type mismatch error messages. From 4ae598762e7dac1886b481a48ae0875843e5153f Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Mon, 27 Feb 2023 02:56:43 +0300 Subject: [PATCH 1615/1654] fixes #21439; Add tyOpenArray to genTypeInfo. (#21440) * fixes #21439; Add tyOpenArray to genTypeInfo. * Add test. --- compiler/jstypes.nim | 2 +- tests/js/t21439.nim | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/js/t21439.nim diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index 56d075af7..4b4ca9fe7 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -133,7 +133,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope = "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" % [result, rope(ord(t.kind))] prepend(p.g.typeInfo, s) - of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet: + of tyVar, tyLent, tyRef, tyPtr, tySequence, tyRange, tySet, tyOpenArray: var s = "var $1 = {size: 0, kind: $2, base: null, node: null, finalizer: null};$n" % [result, rope(ord(t.kind))] diff --git a/tests/js/t21439.nim b/tests/js/t21439.nim new file mode 100644 index 000000000..972356cd0 --- /dev/null +++ b/tests/js/t21439.nim @@ -0,0 +1,14 @@ +discard """ + action: "compile" +""" + +proc test(a: openArray[string]): proc = + result = proc = + for i in a: + discard i + + +const a = ["t1", "t2"] + +discard test(a) + From 89a60939f8b0c60e8abf63b698491a8b138da772 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Mon, 27 Feb 2023 08:57:02 +0900 Subject: [PATCH 1616/1654] Fix typo in sourcemap.nim (#21438) seperated -> separated --- compiler/sourcemap.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sourcemap.nim b/compiler/sourcemap.nim index 2245250ac..2fcc50bbe 100644 --- a/compiler/sourcemap.nim +++ b/compiler/sourcemap.nim @@ -164,7 +164,7 @@ func toSourceMap*(info: SourceInfo, file: string): SourceMap {.raises: [].} = result.names = info.names # Convert nodes into mappings. # Mappings are split into blocks where each block referes to a line in the outputted JS. - # Blocks can be seperated into statements which refere to tokens on the line. + # Blocks can be separated into statements which refere to tokens on the line. # Since the mappings depend on previous values we need to # keep track of previous file, name, etc var From 6fea221d65c999ef7923bf51bbdaa8421cb2e9df Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 27 Feb 2023 07:57:36 +0800 Subject: [PATCH 1617/1654] Overrides `=copy` for `PackedSets` (#21417) --- lib/std/packedsets.nim | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/std/packedsets.nim b/lib/std/packedsets.nim index 1e2892658..e8e03af9e 100644 --- a/lib/std/packedsets.nim +++ b/lib/std/packedsets.nim @@ -12,11 +12,6 @@ ## ## Supports any Ordinal type. ## -## .. note:: Currently the assignment operator `=` for `PackedSet[A]` -## performs some rather meaningless shallow copy. Since Nim currently does -## not allow the assignment operator to be overloaded, use the `assign proc -## <#assign,PackedSet[A],PackedSet[A]>`_ to get a deep copy. -## ## See also ## ======== ## * `sets module `_ for more general hash sets @@ -412,18 +407,9 @@ proc isNil*[A](x: PackedSet[A]): bool {.inline.} = x.head.isNil and x.elems == 0 -proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = +proc `=copy`*[A](dest: var PackedSet[A], src: PackedSet[A]) = ## Copies `src` to `dest`. ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. - runnableExamples: - var - a = initPackedSet[int]() - b = initPackedSet[int]() - b.incl(5) - b.incl(7) - a.assign(b) - assert len(a) == 2 - if src.elems <= src.a.len: dest.data = @[] dest.max = 0 @@ -452,6 +438,19 @@ proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) = dest.data[h] = n it = it.next +proc assign*[A](dest: var PackedSet[A], src: PackedSet[A]) {.inline, deprecated.} = + ## Copies `src` to `dest`. + ## `dest` does not need to be initialized by the `initPackedSet proc <#initPackedSet>`_. + runnableExamples: + var + a = initPackedSet[int]() + b = initPackedSet[int]() + b.incl(5) + b.incl(7) + a.assign(b) + assert len(a) == 2 + `=copy`(dest, src) + proc union*[A](s1, s2: PackedSet[A]): PackedSet[A] = ## Returns the union of the sets `s1` and `s2`. ## From 83e33207250643dedc541e44ecf32d58f3edd097 Mon Sep 17 00:00:00 2001 From: Antonis Geralis <43617260+planetis-m@users.noreply.github.com> Date: Tue, 28 Feb 2023 15:17:21 +0200 Subject: [PATCH 1618/1654] Use a cast to suppress KeyError raises (#21451) --- lib/pure/json.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index e881fbff7..d91da3545 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -465,11 +465,12 @@ proc `==`*(a, b: JsonNode): bool {.noSideEffect.} = if a.fields.len != b.fields.len: return false for key, val in a.fields: if not b.fields.hasKey(key): return false - when defined(nimHasEffectsOf): - {.noSideEffect.}: + {.cast(raises: []).}: + when defined(nimHasEffectsOf): + {.noSideEffect.}: + if b.fields[key] != val: return false + else: if b.fields[key] != val: return false - else: - if b.fields[key] != val: return false result = true proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} From dd629c8f450aa6bc871b54e4e549ad5fea2979a3 Mon Sep 17 00:00:00 2001 From: Century Systems Date: Tue, 28 Feb 2023 22:38:50 +0900 Subject: [PATCH 1619/1654] asyncdispatch: for NuttX, add destructor to clear global dispatcher. (#21432) * asyncdispatch: for NuttX, add destructor to clear global dispatcher using atexit(). Signed-off-by: Takeyoshi Kikuchi * std: exitprocs: remove "when defined(nuttx)" block. Signed-off-by: Takeyoshi Kikuchi --------- Signed-off-by: Takeyoshi Kikuchi --- lib/pure/asyncdispatch.nim | 13 ++++++++++++- lib/std/exitprocs.nim | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 2bfa8587c..63062d8da 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1216,6 +1216,15 @@ else: var gDisp{.threadvar.}: owned PDispatcher ## Global dispatcher + when defined(nuttx): + import std/exitprocs + + proc cleanDispatcher() {.noconv.} = + gDisp = nil + + proc addFinalyzer() = + addExitProc(cleanDispatcher) + proc setGlobalDispatcher*(disp: owned PDispatcher) = if not gDisp.isNil: assert gDisp.callbacks.len == 0 @@ -1225,6 +1234,8 @@ else: proc getGlobalDispatcher*(): PDispatcher = if gDisp.isNil: setGlobalDispatcher(newDispatcher()) + when defined(nuttx): + addFinalyzer() result = gDisp proc getIoHandler*(disp: PDispatcher): Selector[AsyncData] = @@ -1591,7 +1602,7 @@ else: owned(Future[tuple[address: string, client: AsyncFD]]) = var retFuture = newFuture[tuple[address: string, client: AsyncFD]]("acceptAddr") - proc cb(sock: AsyncFD): bool = + proc cb(sock: AsyncFD): bool {.gcsafe.} = result = true var sockAddress: Sockaddr_storage var addrLen = sizeof(sockAddress).SockLen diff --git a/lib/std/exitprocs.nim b/lib/std/exitprocs.nim index d63c1abc5..48b4fca7f 100644 --- a/lib/std/exitprocs.nim +++ b/lib/std/exitprocs.nim @@ -45,6 +45,7 @@ proc callClosures() {.noconv.} = case fun.kind of kClosure: fun.fun1() of kNoconv: fun.fun2() + gFuns.setLen(0) template fun() = if gFuns.len == 0: From 39d0a93d0e0bc2ebf24a9801fe49cdb71c085527 Mon Sep 17 00:00:00 2001 From: Constantine Molchanov Date: Wed, 1 Mar 2023 05:30:38 +0300 Subject: [PATCH 1620/1654] Fix #21452; enable Norm in important packages. (#21455) * fix #21452; enable Norm in important packages. * Run test that doesn't require SQLite. --- testament/important_packages.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testament/important_packages.nim b/testament/important_packages.nim index b3a4bfb82..0af0eb93e 100644 --- a/testament/important_packages.nim +++ b/testament/important_packages.nim @@ -119,7 +119,7 @@ pkg "nimterop", "nimble minitest" pkg "nimwc", "nim c nimwc.nim", allowFailure = true pkg "nimx", "nim c --threads:on test/main.nim", allowFailure = true pkg "nitter", "nim c src/nitter.nim", "https://github.com/zedeus/nitter" -pkg "norm", "testament r tests/sqlite/trows.nim", allowFailure = true +pkg "norm", "testament r tests/common/tmodel.nim" pkg "npeg", "nimble testarc" pkg "numericalnim", "nimble nimCI" pkg "optionsutils" From c4dd0c43016faadbdaaddb2decf95dc881eb772e Mon Sep 17 00:00:00 2001 From: heterodoxic <122719743+heterodoxic@users.noreply.github.com> Date: Wed, 1 Mar 2023 17:17:05 +0100 Subject: [PATCH 1621/1654] suggestion for a simple fix for #21279 (#21378) --- compiler/cgen.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 60623b72d..c346a0b78 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -480,6 +480,9 @@ proc resetLoc(p: BProc, loc: var TLoc) = # on the bytes following the m_type field? genObjectInit(p, cpsStmts, loc.t, loc, constructObj) +proc isOrHasImportedCppType(typ: PType): bool = + searchTypeFor(typ.skipTypes({tyRef}), isImportedCppType) + proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = let typ = loc.t if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}: @@ -497,7 +500,7 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) = if not isTemp or containsGarbageCollectedRef(loc.t): # don't use nimZeroMem for temporary values for performance if we can # avoid it: - if not isImportedCppType(typ): + if not isOrHasImportedCppType(typ): linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n", [addrLoc(p.config, loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))]) genObjectInit(p, cpsStmts, loc.t, loc, constructObj) @@ -517,7 +520,10 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = inc(p.labels) result.r = "T" & rope(p.labels) & "_" - linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r]) + if p.module.compileToCpp and isOrHasImportedCppType(t): + linefmt(p, cpsLocals, "$1 $2{};$n", [getTypeDesc(p.module, t, skVar), result.r]) + else: + linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r]) result.k = locTemp result.lode = lodeTyp t result.storage = OnStack @@ -575,7 +581,7 @@ proc assignLocalVar(p: BProc, n: PNode) = # this need not be fulfilled for inline procs; they are regenerated # for each module that uses them! let nl = if optLineDir in p.config.options: "" else: "\L" - let decl = localVarDecl(p, n) & ";" & nl + let decl = localVarDecl(p, n) & (if p.module.compileToCpp and isOrHasImportedCppType(n.typ): "{};" else: ";") & nl line(p, cpsLocals, decl) include ccgthreadvars From 1b1412f3d148b02fb553f37d84505745cf3fb435 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:18:09 +0800 Subject: [PATCH 1622/1654] fixes #10938; fixes #13312; fixes #13918; fixes #20985; always initializes global variables with null values in VM (#21351) * fixes #10938; always initialize global variable in VM * fixes importc vars * there is a pre-existing issue regarding closure types in the VM * add tests --- compiler/vmgen.nim | 12 ++++++++++++ tests/vm/tvmmisc.nim | 46 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index bdf7b679e..be7938e53 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1909,6 +1909,18 @@ proc genVarSection(c: PCtx; n: PNode) = c.genAdditionalCopy(a[2], opcWrDeref, tmp, 0, val) c.freeTemp(val) c.freeTemp(tmp) + elif not importcCondVar(s) and not (s.typ.kind == tyProc and s.typ.callConv == ccClosure): # fixes #10938 + # there is a pre-existing issue with closure types in VM + # if `(var s: proc () = default(proc ()); doAssert s == nil)` works for you; + # you might remove the second condition. + # the problem is that closure types are tuples in VM, but the types of its children + # shouldn't have the same type as closure types. + let tmp = c.genx(a[0], {gfNodeAddr}) + let sa = getNullValue(s.typ, a.info, c.config) + let val = c.genx(sa) + c.genAdditionalCopy(sa, opcWrDeref, tmp, 0, val) + c.freeTemp(val) + c.freeTemp(tmp) else: setSlot(c, s) if a[2].kind == nkEmpty: diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index 2ce4703b0..673e3e965 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -564,14 +564,14 @@ block: block: static: let x = int 1 - echo x.type # Foo + doAssert $(x.type) == "Foo" # Foo block: static: let x = int 1 let y = x + 1 # Error: unhandled exception: value out of range: -8 notin 0 .. 65535 [RangeDefect] - echo y + doAssert y == 2 type Atom* = object @@ -601,3 +601,45 @@ proc foo(s: sink string) = doAssert s.len == 3 static: foo("abc") + + +static: + for i in '1' .. '2': # bug #10938 + var s: set[char] + doAssert s == {} + incl(s, i) + + for _ in 0 ..< 3: # bug #13312 + var s: string + s.add("foo") + doAssert s == "foo" + + for i in 1 .. 5: # bug #13918 + var arr: array[3, int] + var val: int + doAssert arr == [0, 0, 0] and val == 0 + for j in 0 ..< len(arr): + arr[j] = i + val = i + +# bug #20985 +let a = block: + var groups: seq[seq[int]] + for i in 0 ..< 3: + var group: seq[int] + for j in 0 ..< 3: + group.add j + groups.add group + groups + +const b = block: + var groups: seq[seq[int]] + for i in 0 ..< 3: + var group: seq[int] + for j in 0 ..< 3: + group.add j + groups.add group + groups + +doAssert a == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] +doAssert b == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] From 89d8c0b24678a0b65820692069adffc7bec38ae7 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 1 Mar 2023 22:00:58 +0500 Subject: [PATCH 1623/1654] tests/stylecheck: make sure necessary hints are enabled (#21240) --- tests/stylecheck/t20397_1.nim | 4 ++-- tests/stylecheck/treject.nim | 4 ++-- tests/stylecheck/tusages.nim | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/stylecheck/t20397_1.nim b/tests/stylecheck/t20397_1.nim index 24f5791f8..76c03dca1 100644 --- a/tests/stylecheck/t20397_1.nim +++ b/tests/stylecheck/t20397_1.nim @@ -1,8 +1,8 @@ discard """ - matrix: "--styleCheck:off" + matrix: "--styleCheck:off --hint:Name:on" """ {.hintAsError[Name]:on.} var a_b = 1 discard a_b -{.hintAsError[Name]:off.} \ No newline at end of file +{.hintAsError[Name]:off.} diff --git a/tests/stylecheck/treject.nim b/tests/stylecheck/treject.nim index b9d97a58d..458a2d039 100644 --- a/tests/stylecheck/treject.nim +++ b/tests/stylecheck/treject.nim @@ -1,7 +1,7 @@ discard """ action: reject nimout: '''treject.nim(14, 13) Error: 'iD' should be: 'id' [field declared in treject.nim(9, 5)]''' - matrix: "--styleCheck:error --styleCheck:usages" + matrix: "--styleCheck:error --styleCheck:usages --hint:Name:on" """ type @@ -14,4 +14,4 @@ template hello = echo name.iD echo iD -hello() \ No newline at end of file +hello() diff --git a/tests/stylecheck/tusages.nim b/tests/stylecheck/tusages.nim index 2f99c70c5..b689dabb3 100644 --- a/tests/stylecheck/tusages.nim +++ b/tests/stylecheck/tusages.nim @@ -1,11 +1,9 @@ discard """ action: reject - nimout: '''tusages.nim(22, 5) Error: 'BAD_STYLE' should be: 'BADSTYLE' [proc declared in tusages.nim(11, 6)]''' - matrix: "--styleCheck:error --styleCheck:usages" + nimout: '''tusages.nim(20, 5) Error: 'BAD_STYLE' should be: 'BADSTYLE' [proc declared in tusages.nim(9, 6)]''' + matrix: "--styleCheck:error --styleCheck:usages --hint:all:off --hint:Name:on" """ - - import strutils proc BADSTYLE(c: char) = discard From cf083af262d03f62af302a28920f7bfa4b0e6233 Mon Sep 17 00:00:00 2001 From: Anna Date: Wed, 1 Mar 2023 22:01:58 +0500 Subject: [PATCH 1624/1654] tests: explicitly enable stack traces where needed (#21236) * tests/assert/tassert_c.nim: explicitly enable stack traces * tests/errmsgs: explicitly enable stack traces --- tests/assert/tassert_c.nim | 2 +- tests/errmsgs/tproper_stacktrace.nim | 31 ++++++++++++++------------- tests/errmsgs/tproper_stacktrace2.nim | 3 ++- tests/errmsgs/tproper_stacktrace3.nim | 3 ++- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/assert/tassert_c.nim b/tests/assert/tassert_c.nim index 4d49a6035..e3e3b8147 100644 --- a/tests/assert/tassert_c.nim +++ b/tests/assert/tassert_c.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim $target $options -d:nimPreviewSlimSystem --excessiveStackTrace:off $file" + matrix: "-d:nimPreviewSlimSystem --stackTrace:on --excessiveStackTrace:off" output: '''true''' """ import std/assertions diff --git a/tests/errmsgs/tproper_stacktrace.nim b/tests/errmsgs/tproper_stacktrace.nim index 8617984fb..b0a008840 100644 --- a/tests/errmsgs/tproper_stacktrace.nim +++ b/tests/errmsgs/tproper_stacktrace.nim @@ -1,4 +1,5 @@ discard """ + matrix: "--stackTrace:on --hint:all:off --warnings:off" output: '''ok''' """ import strscans, strutils @@ -76,10 +77,10 @@ when true: bar() const expectedStackTrace = """ - tproper_stacktrace.nim(86) tproper_stacktrace - tproper_stacktrace.nim(76) foo - tproper_stacktrace.nim(73) bar - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(87) tproper_stacktrace + tproper_stacktrace.nim(77) foo + tproper_stacktrace.nim(74) bar + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: @@ -93,9 +94,9 @@ when true: bar(x) const expectedStackTrace = """ - tproper_stacktrace.nim(103) tproper_stacktrace - tproper_stacktrace.nim(90) bar - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(104) tproper_stacktrace + tproper_stacktrace.nim(91) bar + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: @@ -110,10 +111,10 @@ when true: bar() const expectedStackTrace = """ - tproper_stacktrace.nim(120) tproper_stacktrace - tproper_stacktrace.nim(110) foo - tproper_stacktrace.nim(107) bar - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(121) tproper_stacktrace + tproper_stacktrace.nim(111) foo + tproper_stacktrace.nim(108) bar + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: @@ -129,10 +130,10 @@ when true: bar() const expectedStackTrace = """ - tproper_stacktrace.nim(139) tproper_stacktrace - tproper_stacktrace.nim(129) foo - tproper_stacktrace.nim(125) baz - tproper_stacktrace.nim(7) raiseTestException + tproper_stacktrace.nim(140) tproper_stacktrace + tproper_stacktrace.nim(130) foo + tproper_stacktrace.nim(126) baz + tproper_stacktrace.nim(8) raiseTestException """ verifyStackTrace expectedStackTrace: diff --git a/tests/errmsgs/tproper_stacktrace2.nim b/tests/errmsgs/tproper_stacktrace2.nim index 44b208c87..5a6ca3fcb 100644 --- a/tests/errmsgs/tproper_stacktrace2.nim +++ b/tests/errmsgs/tproper_stacktrace2.nim @@ -1,5 +1,6 @@ discard """ - outputsub: '''tproper_stacktrace2.nim(20) main''' + matrix: "--stackTrace:on" + outputsub: '''tproper_stacktrace2.nim(21) main''' exitcode: 1 """ diff --git a/tests/errmsgs/tproper_stacktrace3.nim b/tests/errmsgs/tproper_stacktrace3.nim index c292fa092..97d63e6ec 100644 --- a/tests/errmsgs/tproper_stacktrace3.nim +++ b/tests/errmsgs/tproper_stacktrace3.nim @@ -1,5 +1,6 @@ discard """ - outputsub: '''tproper_stacktrace3.nim(21) main''' + matrix: "--stackTrace:on" + outputsub: '''tproper_stacktrace3.nim(22) main''' exitcode: 1 """ From 070938720ad1a06e3273c164db53edb470bdb9a8 Mon Sep 17 00:00:00 2001 From: Anna Date: Thu, 2 Mar 2023 01:29:23 +0500 Subject: [PATCH 1625/1654] tests/errmsgs/tcall_with_default_arg.nim: sync (#21237) --- tests/errmsgs/tcall_with_default_arg.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/errmsgs/tcall_with_default_arg.nim b/tests/errmsgs/tcall_with_default_arg.nim index 1cc86638f..44752f1ee 100644 --- a/tests/errmsgs/tcall_with_default_arg.nim +++ b/tests/errmsgs/tcall_with_default_arg.nim @@ -1,5 +1,5 @@ discard """ -outputsub: '''tcall_with_default_arg.nim(16) anotherFoo''' +outputsub: '''tcall_with_default_arg.nim(8) fail''' exitcode: 1 """ # issue: #5604 From 9948fed919389229a48347aa9fa5adce9b7e0a98 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Mar 2023 12:25:55 +0800 Subject: [PATCH 1626/1654] fixes #21326; fixes #7375; fixes #11986; fixes #9607; rework quote do; `getAst` uses type info to annotate the type of quoted variables; no more type erasures for quoted variables (#21433) * fixes #21326; getAst uses type info to annotateType quoted variables * simplify logics; sem types first * fixes important packages * add testcases * tiny --- compiler/semexprs.nim | 11 ++- compiler/vm.nim | 5 +- testament/important_packages.nim | 6 +- tests/stdlib/tmacros.nim | 136 +++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 8 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 7454c7f8e..9cf206b4c 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2203,10 +2203,13 @@ proc semQuoteAst(c: PContext, n: PNode): PNode = if ids.len > 0: dummyTemplate[paramsPos] = newNodeI(nkFormalParams, n.info) dummyTemplate[paramsPos].add getSysSym(c.graph, n.info, "untyped").newSymNode # return type - ids.add getSysSym(c.graph, n.info, "untyped").newSymNode # params type - ids.add c.graph.emptyNode # no default value - dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids) - + dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[0], getSysSym(c.graph, n.info, "typed").newSymNode, c.graph.emptyNode) + for i in 1.. Date: Thu, 2 Mar 2023 12:29:40 +0800 Subject: [PATCH 1627/1654] fixes #19291; implements `wasMoved` hook (#21303) * fixes #19291; implements `wasMoved` hook * basics * checkpoint * finish `wasMoved` * add a test for #19291 * add documentation and changelog * work `attachedWasMoved` with generics * fixes optimizer * register `=wasMoved` * handle wasMoved magcis * check another round * some patches * try `op == nil` * nicer * generate `wasMoved` before `destroy` * try again * fixes tests * default wasMoved * Update tests/destructor/tv2_cast.nim * Update tests/destructor/tv2_cast.nim * Update tests/arc/topt_refcursors.nim --- changelogs/changelog_2_0_0.md | 2 ++ compiler/ast.nim | 3 +- compiler/injectdestructors.nim | 17 +++++++--- compiler/liftdestructors.nim | 41 ++++++++++++++++++++--- compiler/optimizer.nim | 9 +++-- compiler/semstmts.nim | 5 ++- doc/manual.md | 2 +- tests/arc/topt_no_cursor.nim | 19 ++++++----- tests/arc/topt_refcursors.nim | 8 +++-- tests/arc/topt_wasmoved_destroy_pairs.nim | 13 ++++--- tests/destructor/tv2_cast.nim | 14 ++++---- tests/destructor/twasmoved_error.nim | 37 ++++++++++++++++++++ 12 files changed, 131 insertions(+), 39 deletions(-) create mode 100644 tests/destructor/twasmoved_error.nim diff --git a/changelogs/changelog_2_0_0.md b/changelogs/changelog_2_0_0.md index e081865be..cbe755478 100644 --- a/changelogs/changelog_2_0_0.md +++ b/changelogs/changelog_2_0_0.md @@ -341,6 +341,8 @@ - IBM Z architecture and macOS m1 arm64 architecture are supported. +- `=wasMoved` can be overridden by users. + ## Compiler changes - The `gc` switch has been renamed to `mm` ("memory management") in order to reflect the diff --git a/compiler/ast.nim b/compiler/ast.nim index aa70560ed..f93c8d910 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -935,6 +935,7 @@ type TTypeSeq* = seq[PType] TTypeAttachedOp* = enum ## as usual, order is important here + attachedWasMoved, attachedDestructor, attachedAsgn, attachedSink, @@ -1502,7 +1503,7 @@ proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode, const AttachedOpToStr*: array[TTypeAttachedOp, string] = [ - "=destroy", "=copy", "=sink", "=trace", "=deepcopy"] + "=wasMoved", "=destroy", "=copy", "=sink", "=trace", "=deepcopy"] proc `$`*(s: PSym): string = if s != nil: diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 20e790202..13141b765 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -354,11 +354,18 @@ It is best to factor out piece of object that needs custom destructor into separ result.add newTree(nkFastAsgn, le, tmp) proc genWasMoved(c: var Con, n: PNode): PNode = - result = newNodeI(nkCall, n.info) - result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved))) - result.add copyTree(n) #mWasMoved does not take the address - #if n.kind != nkSym: - # message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")") + let typ = n.typ.skipTypes({tyGenericInst, tyAlias, tySink}) + let op = getAttachedOp(c.graph, n.typ, attachedWasMoved) + if op != nil: + if sfError in op.flags: + c.checkForErrorPragma(n.typ, n, "=wasMoved") + result = genOp(c, op, n) + else: + result = newNodeI(nkCall, n.info) + result.add(newSymNode(createMagic(c.graph, c.idgen, "wasMoved", mWasMoved))) + result.add copyTree(n) #mWasMoved does not take the address + #if n.kind != nkSym: + # message(c.graph.config, n.info, warnUser, "wasMoved(" & $n & ")") proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode = result = newNodeI(nkCall, info) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 738f659b7..c7464d39e 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -88,6 +88,8 @@ proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let call = genBuiltin(c, mDefault, "default", x) call.typ = t body.add newAsgnStmt(x, call) + elif c.kind == attachedWasMoved: + body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc genAddr(c: var TLiftCtx; x: PNode): PNode = if x.kind == nkHiddenDeref: @@ -145,6 +147,11 @@ proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode = else: result = destroy +proc genWasMovedCall(c: var TLiftCtx; op: PSym; x: PNode): PNode = + result = newNodeIT(nkCall, x.info, op.typ[0]) + result.add(newSymNode(op)) + result.add genAddr(c, x) + proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) = case n.kind of nkSym: @@ -442,6 +449,20 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool = body.add newDeepCopyCall(c, op, x, y) result = true + of attachedWasMoved: + var op = getAttachedOp(c.g, t, attachedWasMoved) + if op != nil and sfOverriden in op.flags: + + if op.ast.isGenericRoutine: + # patch generic destructor: + op = instantiateGeneric(c, op, t, t.typeInst) + setAttachedOp(c.g, c.idgen.module, t, attachedWasMoved, op) + + #markUsed(c.g.config, c.info, op, c.g.usageSym) + onUse(c.info, op) + body.add genWasMovedCall(c, op, x) + result = true + proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode = var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), nextSymId(c.idgen), c.fn, c.info) temp.typ = getSysType(c.g, body.info, tyInt) @@ -524,6 +545,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if canFormAcycle(t.elemType): # follow all elements: forallElements(c, t, body, x, y) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = createTypeBoundOps(c.g, c.c, t, body.info, c.idgen) @@ -561,6 +583,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if op == nil: return # protect from recursion body.add newHookCall(c, op, x, y) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -576,6 +599,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genBuiltin(c, mDestroy, "destroy", x) of attachedTrace: discard "strings are atomic and have no inner elements that are to trace" + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc cyclicType*(t: PType): bool = case t.kind @@ -674,6 +698,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # If the ref is polymorphic we have to account for this body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(x, c.idgen), y) #echo "can follow ", elemType, " static ", isFinal(elemType) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) + proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = ## Closures are really like refs except they always use a virtual destructor @@ -722,6 +748,7 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv, c.idgen), y) + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = case c.kind @@ -746,6 +773,7 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = var actions = newNodeI(nkStmtList, c.info) @@ -771,6 +799,7 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genIf(c, x, actions) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = if c.kind == attachedDeepCopy: @@ -805,6 +834,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.sons.insert(des, 0) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = let xx = genBuiltin(c, mAccessEnv, "accessEnv", x) @@ -820,6 +850,7 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add genIf(c, xx, actions) of attachedDeepCopy: assert(false, "cannot happen") of attachedTrace: discard + of attachedWasMoved: body.add genBuiltin(c, mWasMoved, "wasMoved", x) proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) = case t.kind @@ -935,7 +966,7 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp result.typ = newProcType(info, nextTypeId(idgen), owner) result.typ.addParam dest - if kind != attachedDestructor: + if kind notin {attachedDestructor, attachedWasMoved}: result.typ.addParam src if kind == attachedAsgn and g.config.selectedGC == gcOrc and @@ -975,7 +1006,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; let dest = result.typ.n[1].sym let d = newDeref(newSymNode(dest)) - let src = if kind == attachedDestructor: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) + let src = if kind in {attachedDestructor, attachedWasMoved}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer)) else: newSymNode(result.typ.n[2].sym) # register this operation already: @@ -1103,15 +1134,15 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf # bug #15122: We need to produce all prototypes before entering the # mind boggling recursion. Hacks like these imply we should rewrite # this module. - var generics: array[attachedDestructor..attachedTrace, bool] - for k in attachedDestructor..lastAttached: + var generics: array[attachedWasMoved..attachedTrace, bool] + for k in attachedWasMoved..lastAttached: generics[k] = getAttachedOp(g, canon, k) != nil if not generics[k]: setAttachedOp(g, idgen.module, canon, k, symPrototype(g, canon, canon.owner, k, info, idgen)) # we generate the destructor first so that other operators can depend on it: - for k in attachedDestructor..lastAttached: + for k in attachedWasMoved..lastAttached: if not generics[k]: discard produceSym(g, c, canon, k, info, idgen) else: diff --git a/compiler/optimizer.nim b/compiler/optimizer.nim index 98939f80d..0b26e8d34 100644 --- a/compiler/optimizer.nim +++ b/compiler/optimizer.nim @@ -16,6 +16,8 @@ import from trees import exprStructuralEquivalent +import std/strutils + const nfMarkForDeletion = nfNone # faster than a lookup table @@ -110,16 +112,17 @@ proc analyse(c: var Con; b: var BasicBlock; n: PNode) = var reverse = false if n[0].kind == nkSym: let s = n[0].sym - if s.magic == mWasMoved: + let name = s.name.s.normalize + if s.magic == mWasMoved or name == "=wasmoved": b.wasMovedLocs.add n special = true - elif s.name.s == "=destroy": + elif name == "=destroy": if c.inFinally > 0 and (b.hasReturn or b.hasBreak): discard "cannot optimize away the destructor" else: c.wasMovedDestroyPair b, n special = true - elif s.name.s == "=sink": + elif name == "=sink": reverse = true if not special: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 7dc976c19..6237e6eb0 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1779,7 +1779,7 @@ proc whereToBindTypeHook(c: PContext; t: PType): PType = proc bindTypeHook(c: PContext; s: PSym; n: PNode; op: TTypeAttachedOp) = let t = s.typ var noError = false - let cond = if op == attachedDestructor: + let cond = if op in {attachedDestructor, attachedWasMoved}: t.len == 2 and t[0] == nil and t[1].kind == tyVar elif op == attachedTrace: t.len == 3 and t[0] == nil and t[1].kind == tyVar and t[2].kind == tyPointer @@ -1894,6 +1894,9 @@ proc semOverride(c: PContext, s: PSym, n: PNode) = of "=trace": if s.magic != mTrace: bindTypeHook(c, s, n, attachedTrace) + of "=wasmoved": + if s.magic != mWasMoved: + bindTypeHook(c, s, n, attachedWasMoved) else: if sfOverriden in s.flags: localError(c.config, n.info, errGenerated, diff --git a/doc/manual.md b/doc/manual.md index ef790c362..31c620ca7 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -4122,7 +4122,7 @@ the operator is in scope (including if it is private). ``` Type bound operators are: -`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`. +`=destroy`, `=copy`, `=sink`, `=trace`, `=deepcopy`, `=wasMoved`. These operations can be *overridden* instead of *overloaded*. This means that the implementation is automatically lifted to structured types. For instance, diff --git a/tests/arc/topt_no_cursor.nim b/tests/arc/topt_no_cursor.nim index 30d4c316c..50dfa26ac 100644 --- a/tests/arc/topt_no_cursor.nim +++ b/tests/arc/topt_no_cursor.nim @@ -1,7 +1,8 @@ discard """ nimoutFull: true cmd: '''nim c -r --warnings:off --hints:off --gc:arc --expandArc:newTarget --expandArc:delete --expandArc:p1 --expandArc:tt --hint:Performance:off --assertions:off --expandArc:extractConfig --expandArc:mergeShadowScope --expandArc:check $file''' - nimout: '''--expandArc: newTarget + nimout: ''' +--expandArc: newTarget var splat @@ -11,9 +12,9 @@ splat = splitDrive do: let blitTmp = path blitTmp :tmp = splat.drive -wasMoved(splat.drive) +`=wasMoved`(splat.drive) :tmp_1 = splat.path_1 -wasMoved(splat.path_1) +`=wasMoved`(splat.path_1) result = ( let blitTmp_1 = :tmp blitTmp_1, @@ -60,10 +61,10 @@ var try: it_cursor = x a = ( - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, it_cursor.key) :tmpD, - wasMoved(:tmpD_1) + `=wasMoved`(:tmpD_1) `=copy`(:tmpD_1, it_cursor.val) :tmpD_1) echo [ @@ -112,7 +113,7 @@ block :tmp: var :tmpD sym = shadowScope.symbols[i] addInterfaceDecl(c): - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy_1`(:tmpD, sym) :tmpD inc(i, 1) @@ -125,7 +126,7 @@ this.isValid = fileExists(this.value) if dirExists(this.value): var :tmpD par = (dir: - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, this.value) :tmpD, front: "") else: var @@ -133,10 +134,10 @@ if dirExists(this.value): :tmpD_2 :tmpD_3 par = (dir_1: parentDir(this.value), front_1: - wasMoved(:tmpD_1) + `=wasMoved`(:tmpD_1) `=copy`(:tmpD_1, :tmpD_3 = splitDrive do: - wasMoved(:tmpD_2) + `=wasMoved`(:tmpD_2) `=copy`(:tmpD_2, this.value) :tmpD_2 :tmpD_3.path) diff --git a/tests/arc/topt_refcursors.nim b/tests/arc/topt_refcursors.nim index c13d81bad..8c638a4a1 100644 --- a/tests/arc/topt_refcursors.nim +++ b/tests/arc/topt_refcursors.nim @@ -1,7 +1,8 @@ discard """ output: '''''' cmd: '''nim c --gc:arc --expandArc:traverse --hint:Performance:off $file''' - nimout: '''--expandArc: traverse + nimout: ''' +--expandArc: traverse var it_cursor @@ -22,12 +23,13 @@ try: `=copy`(ri_1, jt.ri) echo [jt.s] `=sink`(jt, ri_1) - wasMoved(ri_1) + `=wasMoved`(ri_1) finally: `=destroy`(ri_1) finally: `=destroy`(jt) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ type diff --git a/tests/arc/topt_wasmoved_destroy_pairs.nim b/tests/arc/topt_wasmoved_destroy_pairs.nim index 2f971f112..6577d6787 100644 --- a/tests/arc/topt_wasmoved_destroy_pairs.nim +++ b/tests/arc/topt_wasmoved_destroy_pairs.nim @@ -1,7 +1,8 @@ discard """ output: '''''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:tfor --hint:Performance:off $file''' - nimout: '''--expandArc: main + nimout: ''' +--expandArc: main var a @@ -29,6 +30,7 @@ try: x = f() block :tmp: var i_cursor + mixin inc var i_1 = 0 block :tmp_1: while i_1 < 4: @@ -37,25 +39,26 @@ try: if i_cursor == 2: return add(a): - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, x) :tmpD inc i_1, 1 if cond: add(a): let blitTmp = x - wasMoved(x) + `=wasMoved`(x) blitTmp else: add(b): let blitTmp_1 = x - wasMoved(x) + `=wasMoved`(x) blitTmp_1 finally: `=destroy`(x) `=destroy_1`(b) `=destroy_1`(a) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ proc f(): seq[int] = diff --git a/tests/destructor/tv2_cast.nim b/tests/destructor/tv2_cast.nim index 917cf0eb3..6fa419996 100644 --- a/tests/destructor/tv2_cast.nim +++ b/tests/destructor/tv2_cast.nim @@ -4,7 +4,8 @@ discard """ @[1953719668, 875770417] destroying O1''' cmd: '''nim c --gc:arc --expandArc:main --expandArc:main1 --expandArc:main2 --expandArc:main3 --hints:off --assertions:off $file''' - nimout: '''--expandArc: main + nimout: ''' +--expandArc: main var data @@ -12,7 +13,7 @@ var :tmpD_1 :tmpD_2 data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_2 = encode(cast[seq[byte]]( :tmpD_1 = newString(100) @@ -32,7 +33,7 @@ var :tmpD_1 s = newString(100) data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_1 = encode(toOpenArrayByte(s, 0, len(s) - 1)) :tmpD_1)) @@ -50,7 +51,7 @@ var :tmpD_1 s = newSeq(100) data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_1 = encode(s) :tmpD_1)) @@ -67,7 +68,7 @@ var :tmpD_1 :tmpD_2 data = - wasMoved(:tmpD) + `=wasMoved`(:tmpD) `=copy`(:tmpD, cast[string]( :tmpD_2 = encode do: :tmpD_1 = newSeq(100) @@ -77,7 +78,8 @@ data = `=destroy`(:tmpD_2) `=destroy`(:tmpD_1) `=destroy_1`(data) --- end of expandArc ------------------------''' +-- end of expandArc ------------------------ +''' """ func encode*(src: openArray[byte]): seq[byte] = diff --git a/tests/destructor/twasmoved_error.nim b/tests/destructor/twasmoved_error.nim new file mode 100644 index 000000000..1cd57e3df --- /dev/null +++ b/tests/destructor/twasmoved_error.nim @@ -0,0 +1,37 @@ +discard """ + cmd: '''nim c --mm:arc $file''' + errormsg: "'=wasMoved' is not available for type ; routine: main" +""" + +# bug #19291 + +const + screenWidth = 800 + screenHeight = 450 + +var + ready = false +type + Game = object + +proc `=destroy`(x: var Game) = + assert ready, "Window is already opened" + ready = false + +proc `=sink`(x: var Game; y: Game) {.error.} +proc `=copy`(x: var Game; y: Game) {.error.} +proc `=wasMoved`(x: var Game) {.error.} + +proc initGame(width, height: int32, title: string): Game = + assert not ready, "Window is already closed" + ready = true + +proc update(x: Game) = discard + +proc main = + var g = initGame(screenWidth, screenHeight, "Tetris raylib") + g.update() + var g2 = g + echo "hello" + +main() From 9b5ae2b2eb2ba5aca219de109b03f1bb5637a777 Mon Sep 17 00:00:00 2001 From: Ivan Yonchovski Date: Thu, 2 Mar 2023 06:30:55 +0200 Subject: [PATCH 1628/1654] Define the version of nim package without using system module (#21415) This is follow up from https://github.com/nim-lang/Nim/pull/21313 --- nim.nimble | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nim.nimble b/nim.nimble index 713fc9256..380ffbce8 100644 --- a/nim.nimble +++ b/nim.nimble @@ -1,4 +1,5 @@ -version = system.NimVersion +include "lib/system/compilation.nim" +version = $NimMajor & "." & $NimMinor & "." & $NimPatch author = "Andreas Rumpf" description = "Compiler package providing the compiler sources as a library." license = "MIT" From 612abda4f40b2a0fd8dd0e4ad119c4415b9c34cb Mon Sep 17 00:00:00 2001 From: tersec Date: Thu, 2 Mar 2023 04:51:54 +0000 Subject: [PATCH 1629/1654] remove decades-deprecated Win32 API *A function support (#21315) --- lib/pure/asyncfile.nim | 11 +- lib/pure/browsers.nim | 2 +- lib/pure/memfiles.nim | 7 +- lib/pure/os.nim | 93 ++++-------- lib/pure/osproc.nim | 35 ++--- lib/std/cmdline.nim | 5 +- lib/std/private/oscommon.nim | 65 +++------ lib/std/private/osdirs.nim | 17 +-- lib/std/private/osfiles.nim | 43 ++---- lib/std/private/ospaths2.nim | 35 ++--- lib/std/private/ossymlinks.nim | 12 +- lib/std/syncio.nim | 4 +- lib/windows/winlean.nim | 258 ++++++++++----------------------- 13 files changed, 173 insertions(+), 414 deletions(-) diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 88fa90406..96d5e6d2e 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -102,14 +102,9 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL let desiredAccess = getDesiredAccess(mode) let creationDisposition = getCreationDisposition(mode, filename) - when useWinUnicode: - let fd = createFileW(newWideCString(filename), desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) - else: - let fd = createFileA(filename, desiredAccess, - FILE_SHARE_READ, - nil, creationDisposition, flags, 0) + let fd = createFileW(newWideCString(filename), desiredAccess, + FILE_SHARE_READ, + nil, creationDisposition, flags, 0) if fd == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) diff --git a/lib/pure/browsers.nim b/lib/pure/browsers.nim index b78034fe3..ab248db36 100644 --- a/lib/pure/browsers.nim +++ b/lib/pure/browsers.nim @@ -21,7 +21,7 @@ when defined(nimPreviewSlimSystem): when defined(windows): import winlean - when useWinUnicode and defined(nimPreviewSlimSystem): + when defined(nimPreviewSlimSystem): import std/widestrs from os import absolutePath else: diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 58c4888f1..74065bc2e 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -17,7 +17,7 @@ when defined(windows): import winlean - when useWinUnicode and defined(nimPreviewSlimSystem): + when defined(nimPreviewSlimSystem): import std/widestrs elif defined(posix): import posix @@ -199,10 +199,7 @@ proc open*(filename: string, mode: FileMode = fmRead, else: FILE_ATTRIBUTE_NORMAL or flags, 0) - when useWinUnicode: - result.fHandle = callCreateFile(createFileW, newWideCString(filename)) - else: - result.fHandle = callCreateFile(createFileA, filename) + result.fHandle = callCreateFile(createFileW, newWideCString(filename)) if result.fHandle == INVALID_HANDLE_VALUE: fail(osLastError(), "error opening file") diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 3285c07df..0569e6cab 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -422,32 +422,18 @@ proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", ## Raises `OSError` in case of an error. Follows symlinks. when defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var unused: WideCString = nil - var res = newWideCString("", bufsize) - while true: - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - var unused: cstring = nil - result = newString(bufsize) - while true: - var L = getFullPathNameA(filename, bufsize, result, unused) - if L == 0'i32: - raiseOSError(osLastError(), filename) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var unused: WideCString = nil + var res = newWideCString("", bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError(), filename) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break # getFullPathName doesn't do case corrections, so we have to use this convoluted # way of retrieving the true filename for x in walkFiles(result): @@ -483,14 +469,10 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = ## See also: ## * `createSymlink proc`_ when defined(windows): - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createHardLinkW(wDst, wSrc, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createHardLinkA(dest, src, nil) == 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createHardLinkW(wDst, wSrc, nil) == 0: + raiseOSError(osLastError(), $(src, dest)) else: if link(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) @@ -655,32 +637,18 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect], noW # /proc//path/a.out (complete pathname) when defined(windows): var bufsize = int32(MAX_PATH) - when useWinUnicode: - var buf = newWideCString("", bufsize) - while true: - var L = getModuleFileNameW(0, buf, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - buf = newWideCString("", L) - bufsize = L - else: - result = buf$L - break - else: - result = newString(bufsize) - while true: - var L = getModuleFileNameA(0, result, bufsize) - if L == 0'i32: - result = "" # error! - break - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var buf = newWideCString("", bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString("", L) + bufsize = L + else: + result = buf$L + break elif defined(macosx): var size = cuint32(0) getExecPath1(nil, size) @@ -977,10 +945,7 @@ proc isHidden*(path: string): bool {.noWeirdTarget.} = assert ".foo/".isHidden when defined(windows): - when useWinUnicode: - wrapUnary(attributes, getFileAttributesW, path) - else: - var attributes = getFileAttributesA(path) + wrapUnary(attributes, getFileAttributesW, path) if attributes != -1'i32: result = (attributes and FILE_ATTRIBUTE_HIDDEN) != 0'i32 else: diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index bb6abc571..4cdd98c0d 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -714,22 +714,15 @@ when defined(windows) and not defined(useNimRtl): if len(workingDir) > 0: wd = workingDir if env != nil: e = buildEnv(env) if poEchoCmd in options: echo($cmdl) - when useWinUnicode: - var tmp = newWideCString(cmdl) - var ee = - if e.str.isNil: newWideCString(cstring(nil)) - else: newWideCString(e.str, e.len) - var wwd = newWideCString(wd) - var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT - if poDaemon in options: flags = flags or CREATE_NO_WINDOW - success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, - ee, wwd, si, procInfo) - else: - var ee = - if e.str.isNil: cstring(nil) - else: cstring(e.str) - success = winlean.createProcessA(nil, - cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, ee, wd, si, procInfo) + var tmp = newWideCString(cmdl) + var ee = + if e.str.isNil: newWideCString(cstring(nil)) + else: newWideCString(e.str, e.len) + var wwd = newWideCString(wd) + var flags = NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT + if poDaemon in options: flags = flags or CREATE_NO_WINDOW + success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags, + ee, wwd, si, procInfo) let lastError = osLastError() if poParentStreams notin options: @@ -870,13 +863,9 @@ when defined(windows) and not defined(useNimRtl): si.hStdError = getStdHandle(STD_ERROR_HANDLE) si.hStdInput = getStdHandle(STD_INPUT_HANDLE) si.hStdOutput = getStdHandle(STD_OUTPUT_HANDLE) - when useWinUnicode: - var c = newWideCString(command) - var res = winlean.createProcessW(nil, c, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) - else: - var res = winlean.createProcessA(nil, command, nil, nil, 0, - NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) + var c = newWideCString(command) + var res = winlean.createProcessW(nil, c, nil, nil, 0, + NORMAL_PRIORITY_CLASS, nil, nil, si, procInfo) if res == 0: raiseOSError(osLastError()) else: diff --git a/lib/std/cmdline.nim b/lib/std/cmdline.nim index 3208da1b8..6788dacde 100644 --- a/lib/std/cmdline.nim +++ b/lib/std/cmdline.nim @@ -40,10 +40,7 @@ else: # Needed by windows in order to obtain the command line for targets # other than command line targets when defined(windows) and not weirdTarget: - when useWinUnicode: - template getCommandLine*(): untyped = getCommandLineW() - else: - template getCommandLine*(): untyped = getCommandLineA() + template getCommandLine*(): untyped = getCommandLineW() proc parseCmdLine*(c: string): seq[string] {. diff --git a/lib/std/private/oscommon.nim b/lib/std/private/oscommon.nim index 3aac58636..b747e33f1 100644 --- a/lib/std/private/oscommon.nim +++ b/lib/std/private/oscommon.nim @@ -47,23 +47,17 @@ else: when defined(windows) and not weirdTarget: - when useWinUnicode: - template wrapUnary*(varname, winApiProc, arg: untyped) = - var varname = winApiProc(newWideCString(arg)) + template wrapUnary*(varname, winApiProc, arg: untyped) = + var varname = winApiProc(newWideCString(arg)) - template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = - var varname = winApiProc(newWideCString(arg), arg2) - proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = - result = findFirstFileW(newWideCString(a), b) - template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) + template wrapBinary*(varname, winApiProc, arg, arg2: untyped) = + var varname = winApiProc(newWideCString(arg), arg2) + proc findFirstFile*(a: string, b: var WIN32_FIND_DATA): Handle = + result = findFirstFileW(newWideCString(a), b) + template findNextFile*(a, b: untyped): untyped = findNextFileW(a, b) - template getFilename*(f: untyped): untyped = - $cast[WideCString](addr(f.cFileName[0])) - else: - template findFirstFile*(a, b: untyped): untyped = findFirstFileA(a, b) - template findNextFile*(a, b: untyped): untyped = findNextFileA(a, b) - - template getFilename*(f: untyped): untyped = $cast[cstring](addr f.cFileName) + template getFilename*(f: untyped): untyped = + $cast[WideCString](addr(f.cFileName[0])) proc skipFindData*(f: WIN32_FIND_DATA): bool {.inline.} = # Note - takes advantage of null delimiter in the cstring @@ -104,12 +98,9 @@ proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} ## In case of other errors `OSError` is raised. ## Returns true in case of success. when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 - else: - result = moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 + let s = newWideCString(source) + let d = newWideCString(dest) + result = moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) != 0'i32 else: result = c_rename(source, dest) == 0'i32 @@ -137,10 +128,7 @@ proc fileExists*(filename: string): bool {.rtl, extern: "nos$1", ## * `dirExists proc`_ ## * `symlinkExists proc`_ when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, filename) - else: - var a = getFileAttributesA(filename) + wrapUnary(a, getFileAttributesW, filename) if a != -1'i32: result = (a and FILE_ATTRIBUTE_DIRECTORY) == 0'i32 else: @@ -157,10 +145,7 @@ proc dirExists*(dir: string): bool {.rtl, extern: "nos$1", tags: [ReadDirEffect] ## * `fileExists proc`_ ## * `symlinkExists proc`_ when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, dir) - else: - var a = getFileAttributesA(dir) + wrapUnary(a, getFileAttributesW, dir) if a != -1'i32: result = (a and FILE_ATTRIBUTE_DIRECTORY) != 0'i32 else: @@ -178,10 +163,7 @@ proc symlinkExists*(link: string): bool {.rtl, extern: "nos$1", ## * `fileExists proc`_ ## * `dirExists proc`_ when defined(windows): - when useWinUnicode: - wrapUnary(a, getFileAttributesW, link) - else: - var a = getFileAttributesA(link) + wrapUnary(a, getFileAttributesW, link) if a != -1'i32: # xxx see: bug #16784 (bug9); checking `IO_REPARSE_TAG_SYMLINK` # may also be needed. @@ -197,15 +179,8 @@ when defined(windows) and not weirdTarget: flags = flags or FILE_FLAG_OPEN_REPARSE_POINT let access = if writeAccess: GENERIC_WRITE else: 0'i32 - when useWinUnicode: - result = createFileW( - newWideCString(path), access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) - else: - result = createFileA( - path, access, - FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, - nil, OPEN_EXISTING, flags, 0 - ) + result = createFileW( + newWideCString(path), access, + FILE_SHARE_DELETE or FILE_SHARE_READ or FILE_SHARE_WRITE, + nil, OPEN_EXISTING, flags, 0 + ) diff --git a/lib/std/private/osdirs.nim b/lib/std/private/osdirs.nim index 4af418ead..a4318367d 100644 --- a/lib/std/private/osdirs.nim +++ b/lib/std/private/osdirs.nim @@ -328,10 +328,7 @@ iterator walkDirRec*(dir: string, proc rawRemoveDir(dir: string) {.noWeirdTarget.} = when defined(windows): - when useWinUnicode: - wrapUnary(res, removeDirectoryW, dir) - else: - var res = removeDirectoryA(dir) + wrapUnary(res, removeDirectoryW, dir) let lastError = osLastError() if res == 0'i32 and lastError.int32 != 3'i32 and lastError.int32 != 18'i32 and lastError.int32 != 2'i32: @@ -396,10 +393,7 @@ proc rawCreateDir(dir: string): bool {.noWeirdTarget.} = #echo res raiseOSError(osLastError(), dir) else: - when useWinUnicode: - wrapUnary(res, createDirectoryW, dir) - else: - let res = createDirectoryA(dir) + wrapUnary(res, createDirectoryW, dir) if res != 0'i32: result = true @@ -561,10 +555,7 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [], noWeirdTarget.} = ## * `getTempDir proc`_ ## * `getCurrentDir proc`_ when defined(windows): - when useWinUnicode: - if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: - raiseOSError(osLastError(), newDir) - else: - if setCurrentDirectoryA(newDir) == 0'i32: raiseOSError(osLastError(), newDir) + if setCurrentDirectoryW(newWideCString(newDir)) == 0'i32: + raiseOSError(osLastError(), newDir) else: if chdir(newDir) != 0'i32: raiseOSError(osLastError(), newDir) diff --git a/lib/std/private/osfiles.nim b/lib/std/private/osfiles.nim index a7a595d97..7f822ffcc 100644 --- a/lib/std/private/osfiles.nim +++ b/lib/std/private/osfiles.nim @@ -84,10 +84,7 @@ proc getFilePermissions*(filename: string): set[FilePermission] {. if (a.st_mode and S_IWOTH.Mode) != 0.Mode: result.incl(fpOthersWrite) if (a.st_mode and S_IXOTH.Mode) != 0.Mode: result.incl(fpOthersExec) else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) + wrapUnary(res, getFileAttributesW, filename) if res == -1'i32: raiseOSError(osLastError(), filename) if (res and FILE_ATTRIBUTE_READONLY) != 0'i32: result = {fpUserExec, fpUserRead, fpGroupExec, fpGroupRead, @@ -136,19 +133,13 @@ proc setFilePermissions*(filename: string, permissions: set[FilePermission], if chmod(filename, cast[Mode](p)) != 0: raiseOSError(osLastError(), $(filename, permissions)) else: - when useWinUnicode: - wrapUnary(res, getFileAttributesW, filename) - else: - var res = getFileAttributesA(filename) + wrapUnary(res, getFileAttributesW, filename) if res == -1'i32: raiseOSError(osLastError(), filename) if fpUserWrite in permissions: res = res and not FILE_ATTRIBUTE_READONLY else: res = res or FILE_ATTRIBUTE_READONLY - when useWinUnicode: - wrapBinary(res2, setFileAttributesW, filename, res) - else: - var res2 = setFileAttributesA(filename, res) + wrapBinary(res2, setFileAttributesW, filename, res) if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) @@ -221,14 +212,10 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl, if isSymlink and (cfSymlinkIgnore in options or defined(windows)): return when defined(windows): - when useWinUnicode: - let s = newWideCString(source) - let d = newWideCString(dest) - if copyFileW(s, d, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) - else: - if copyFileA(source, dest, 0'i32) == 0'i32: - raiseOSError(osLastError(), $(source, dest)) + let s = newWideCString(source) + let d = newWideCString(dest) + if copyFileW(s, d, 0'i32) == 0'i32: + raiseOSError(osLastError(), $(source, dest)) else: if isSymlink and cfSymlinkAsIs in options: createSymlink(expandSymlink(source), dest) @@ -334,14 +321,9 @@ when not declared(ENOENT) and not defined(windows): var ENOENT {.importc, header: "".}: cint when defined(windows) and not weirdTarget: - when useWinUnicode: - template deleteFile(file: untyped): untyped = deleteFileW(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesW(file, attrs) - else: - template deleteFile(file: untyped): untyped = deleteFileA(file) - template setFileAttributes(file, attrs: untyped): untyped = - setFileAttributesA(file, attrs) + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = + setFileAttributesW(file, attrs) proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirEffect], noWeirdTarget.} = ## Removes the `file`. @@ -358,10 +340,7 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE ## * `moveFile proc`_ result = true when defined(windows): - when useWinUnicode: - let f = newWideCString(file) - else: - let f = file + let f = newWideCString(file) if deleteFile(f) == 0: result = false let err = getLastError() diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 78a806675..612003023 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -849,30 +849,17 @@ when not defined(nimscript): doAssert false, "use -d:nodejs to have `getCurrentDir` defined" elif defined(windows): var bufsize = MAX_PATH.int32 - when useWinUnicode: - var res = newWideCString("", bufsize) - while true: - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - res = newWideCString("", L) - bufsize = L - else: - result = res$L - break - else: - result = newString(bufsize) - while true: - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: - raiseOSError(osLastError()) - elif L > bufsize: - result = newString(L) - bufsize = L - else: - setLen(result, L) - break + var res = newWideCString("", bufsize) + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break else: var bufsize = 1024 # should be enough result = newString(bufsize) diff --git a/lib/std/private/ossymlinks.nim b/lib/std/private/ossymlinks.nim index 6b2de6237..18737b8b5 100644 --- a/lib/std/private/ossymlinks.nim +++ b/lib/std/private/ossymlinks.nim @@ -48,14 +48,10 @@ proc createSymlink*(src, dest: string) {.noWeirdTarget.} = const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 2 # allows anyone with developer mode on to create a link let flag = dirExists(src).int32 or SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE - when useWinUnicode: - var wSrc = newWideCString(src) - var wDst = newWideCString(dest) - if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) - else: - if createSymbolicLinkA(dest, src, flag) == 0 or getLastError() != 0: - raiseOSError(osLastError(), $(src, dest)) + var wSrc = newWideCString(src) + var wDst = newWideCString(dest) + if createSymbolicLinkW(wDst, wSrc, flag) == 0 or getLastError() != 0: + raiseOSError(osLastError(), $(src, dest)) else: if symlink(src, dest) != 0: raiseOSError(osLastError(), $(src, dest)) diff --git a/lib/std/syncio.nim b/lib/std/syncio.nim index 83b2d6d13..77d4fa04c 100644 --- a/lib/std/syncio.nim +++ b/lib/std/syncio.nim @@ -390,7 +390,7 @@ proc readLine*(f: File, line: var string): bool {.tags: [ReadIOEffect], proc c_memchr(s: pointer, c: cint, n: csize_t): pointer {. importc: "memchr", header: "".} - when defined(windows) and not defined(useWinAnsi): + when defined(windows): proc readConsole(hConsoleInput: FileHandle, lpBuffer: pointer, nNumberOfCharsToRead: int32, lpNumberOfCharsRead: ptr int32, @@ -612,7 +612,7 @@ proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, # interface to the C procs: -when defined(windows) and not defined(useWinAnsi): +when defined(windows): when defined(cpp): proc wfopen(filename, mode: WideCString): pointer {. importcpp: "_wfopen((const wchar_t*)#, (const wchar_t*)#)", nodecl.} diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 230f963f0..f8c6ce021 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -21,13 +21,7 @@ when defined(nimPreviewSlimSystem): from std/syncio import FileHandle import std/widestrs -const - useWinUnicode* = not defined(useWinAnsi) - -when useWinUnicode: - type WinChar* = Utf16Char -else: - type WinChar* = char +type WinChar* = Utf16Char # See https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types type @@ -184,26 +178,14 @@ proc peekNamedPipe*(hNamedPipe: Handle, lpBuffer: pointer=nil, lpBytesLeftThisMessage: ptr int32 = nil): bool {. stdcall, dynlib: "kernel32", importc: "PeekNamedPipe".} -when useWinUnicode: - proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment, lpCurrentDirectory: WideCString, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} - -else: - proc createProcessA*(lpApplicationName, lpCommandLine: cstring, - lpProcessAttributes: ptr SECURITY_ATTRIBUTES, - lpThreadAttributes: ptr SECURITY_ATTRIBUTES, - bInheritHandles: WINBOOL, dwCreationFlags: int32, - lpEnvironment: pointer, lpCurrentDirectory: cstring, - lpStartupInfo: var STARTUPINFO, - lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. - stdcall, dynlib: "kernel32", importc: "CreateProcessA", sideEffect.} - +proc createProcessW*(lpApplicationName, lpCommandLine: WideCString, + lpProcessAttributes: ptr SECURITY_ATTRIBUTES, + lpThreadAttributes: ptr SECURITY_ATTRIBUTES, + bInheritHandles: WINBOOL, dwCreationFlags: int32, + lpEnvironment, lpCurrentDirectory: WideCString, + lpStartupInfo: var STARTUPINFO, + lpProcessInformation: var PROCESS_INFORMATION): WINBOOL{. + stdcall, dynlib: "kernel32", importc: "CreateProcessW", sideEffect.} proc suspendThread*(hThread: Handle): int32 {.stdcall, dynlib: "kernel32", importc: "SuspendThread", sideEffect.} @@ -232,67 +214,37 @@ proc getLastError*(): int32 {.importc: "GetLastError", proc setLastError*(error: int32) {.importc: "SetLastError", stdcall, dynlib: "kernel32", sideEffect.} -when useWinUnicode: - proc formatMessageW*(dwFlags: int32, lpSource: pointer, - dwMessageId, dwLanguageId: int32, - lpBuffer: pointer, nSize: int32, - arguments: pointer): int32 {. - importc: "FormatMessageW", stdcall, dynlib: "kernel32".} -else: - proc formatMessageA*(dwFlags: int32, lpSource: pointer, +proc formatMessageW*(dwFlags: int32, lpSource: pointer, dwMessageId, dwLanguageId: int32, lpBuffer: pointer, nSize: int32, arguments: pointer): int32 {. - importc: "FormatMessageA", stdcall, dynlib: "kernel32".} + importc: "FormatMessageW", stdcall, dynlib: "kernel32".} proc localFree*(p: pointer) {. importc: "LocalFree", stdcall, dynlib: "kernel32".} -when useWinUnicode: - proc getCurrentDirectoryW*(nBufferLength: int32, - lpBuffer: WideCString): int32 {. - importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. - importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. - importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc removeDirectoryW*(lpPathName: WideCString): int32 {. - importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} - proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} +proc getCurrentDirectoryW*(nBufferLength: int32, + lpBuffer: WideCString): int32 {. + importc: "GetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setCurrentDirectoryW*(lpPathName: WideCString): int32 {. + importc: "SetCurrentDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc createDirectoryW*(pathName: WideCString, security: pointer=nil): int32 {. + importc: "CreateDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc removeDirectoryW*(lpPathName: WideCString): int32 {. + importc: "RemoveDirectoryW", dynlib: "kernel32", stdcall, sideEffect.} +proc setEnvironmentVariableW*(lpName, lpValue: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableW", sideEffect.} - proc getModuleFileNameW*(handle: Handle, buf: WideCString, - size: int32): int32 {.importc: "GetModuleFileNameW", - dynlib: "kernel32", stdcall.} -else: - proc getCurrentDirectoryA*(nBufferLength: int32, lpBuffer: cstring): int32 {. - importc: "GetCurrentDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc setCurrentDirectoryA*(lpPathName: cstring): int32 {. - importc: "SetCurrentDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc createDirectoryA*(pathName: cstring, security: pointer=nil): int32 {. - importc: "CreateDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc removeDirectoryA*(lpPathName: cstring): int32 {. - importc: "RemoveDirectoryA", dynlib: "kernel32", stdcall, sideEffect.} - proc setEnvironmentVariableA*(lpName, lpValue: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "SetEnvironmentVariableA", sideEffect.} +proc getModuleFileNameW*(handle: Handle, buf: WideCString, + size: int32): int32 {.importc: "GetModuleFileNameW", + dynlib: "kernel32", stdcall.} - proc getModuleFileNameA*(handle: Handle, buf: cstring, size: int32): int32 {. - importc: "GetModuleFileNameA", dynlib: "kernel32", stdcall.} - -when useWinUnicode: - proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} - proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, - security: pointer=nil): int32 {. - importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} -else: - proc createSymbolicLinkA*(lpSymlinkFileName, lpTargetFileName: cstring, - flags: DWORD): int32 {. - importc:"CreateSymbolicLinkA", dynlib: "kernel32", stdcall, sideEffect.} - proc createHardLinkA*(lpFileName, lpExistingFileName: cstring, - security: pointer=nil): int32 {. - importc:"CreateHardLinkA", dynlib: "kernel32", stdcall, sideEffect.} +proc createSymbolicLinkW*(lpSymlinkFileName, lpTargetFileName: WideCString, + flags: DWORD): int32 {. + importc:"CreateSymbolicLinkW", dynlib: "kernel32", stdcall, sideEffect.} +proc createHardLinkW*(lpFileName, lpExistingFileName: WideCString, + security: pointer=nil): int32 {. + importc:"CreateHardLinkW", dynlib: "kernel32", stdcall, sideEffect.} const FILE_ATTRIBUTE_READONLY* = 0x00000001'i32 @@ -343,84 +295,45 @@ type cFileName*: array[0..(MAX_PATH) - 1, WinChar] cAlternateFileName*: array[0..13, WinChar] -when useWinUnicode: - proc findFirstFileW*(lpFileName: WideCString, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} - proc findNextFileW*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} -else: - proc findFirstFileA*(lpFileName: cstring, - lpFindFileData: var WIN32_FIND_DATA): Handle {. - stdcall, dynlib: "kernel32", importc: "FindFirstFileA", sideEffect.} - proc findNextFileA*(hFindFile: Handle, - lpFindFileData: var WIN32_FIND_DATA): int32 {. - stdcall, dynlib: "kernel32", importc: "FindNextFileA", sideEffect.} +proc findFirstFileW*(lpFileName: WideCString, + lpFindFileData: var WIN32_FIND_DATA): Handle {. + stdcall, dynlib: "kernel32", importc: "FindFirstFileW", sideEffect.} +proc findNextFileW*(hFindFile: Handle, + lpFindFileData: var WIN32_FIND_DATA): int32 {. + stdcall, dynlib: "kernel32", importc: "FindNextFileW", sideEffect.} proc findClose*(hFindFile: Handle) {.stdcall, dynlib: "kernel32", importc: "FindClose".} -when useWinUnicode: - proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, - lpBuffer: WideCString, - lpFilePart: var WideCString): int32 {. +proc getFullPathNameW*(lpFileName: WideCString, nBufferLength: int32, + lpBuffer: WideCString, + lpFilePart: var WideCString): int32 {. + stdcall, dynlib: "kernel32", + importc: "GetFullPathNameW", sideEffect.} +proc getFileAttributesW*(lpFileName: WideCString): int32 {. stdcall, dynlib: "kernel32", - importc: "GetFullPathNameW", sideEffect.} - proc getFileAttributesW*(lpFileName: WideCString): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesW", sideEffect.} - proc setFileAttributesW*(lpFileName: WideCString, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} + importc: "GetFileAttributesW", sideEffect.} +proc setFileAttributesW*(lpFileName: WideCString, + dwFileAttributes: int32): WINBOOL {. + stdcall, dynlib: "kernel32", importc: "SetFileAttributesW", sideEffect.} - proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: WINBOOL): WINBOOL {. - importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, + bFailIfExists: WINBOOL): WINBOOL {. + importc: "CopyFileW", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. - importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, - flags: DWORD): WINBOOL {. - importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. + importc: "MoveFileW", stdcall, dynlib: "kernel32", sideEffect.} +proc moveFileExW*(lpExistingFileName, lpNewFileName: WideCString, + flags: DWORD): WINBOOL {. + importc: "MoveFileExW", stdcall, dynlib: "kernel32", sideEffect.} - proc getEnvironmentStringsW*(): WideCString {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} - proc freeEnvironmentStringsW*(para1: WideCString): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} +proc getEnvironmentStringsW*(): WideCString {. + stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsW", sideEffect.} +proc freeEnvironmentStringsW*(para1: WideCString): int32 {. + stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsW", sideEffect.} - proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", - stdcall, dynlib: "kernel32", sideEffect.} - -else: - proc getFullPathNameA*(lpFileName: cstring, nBufferLength: int32, - lpBuffer: cstring, lpFilePart: var cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFullPathNameA", sideEffect.} - proc getFileAttributesA*(lpFileName: cstring): int32 {. - stdcall, dynlib: "kernel32", - importc: "GetFileAttributesA", sideEffect.} - proc setFileAttributesA*(lpFileName: cstring, - dwFileAttributes: int32): WINBOOL {. - stdcall, dynlib: "kernel32", importc: "SetFileAttributesA", sideEffect.} - - proc copyFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. - importc: "CopyFileA", stdcall, dynlib: "kernel32", sideEffect.} - - proc moveFileA*(lpExistingFileName, lpNewFileName: cstring): WINBOOL {. - importc: "MoveFileA", stdcall, dynlib: "kernel32", sideEffect.} - proc moveFileExA*(lpExistingFileName, lpNewFileName: cstring, - flags: DWORD): WINBOOL {. - importc: "MoveFileExA", stdcall, dynlib: "kernel32", sideEffect.} - - proc getEnvironmentStringsA*(): cstring {. - stdcall, dynlib: "kernel32", importc: "GetEnvironmentStringsA", sideEffect.} - proc freeEnvironmentStringsA*(para1: cstring): int32 {. - stdcall, dynlib: "kernel32", importc: "FreeEnvironmentStringsA", sideEffect.} - - proc getCommandLineA*(): cstring {. - importc: "GetCommandLineA", stdcall, dynlib: "kernel32", sideEffect.} +proc getCommandLineW*(): WideCString {.importc: "GetCommandLineW", + stdcall, dynlib: "kernel32", sideEffect.} proc rdFileTime*(f: FILETIME): int64 = result = int64(cast[uint32](f.dwLowDateTime)) or (int64(cast[uint32](f.dwHighDateTime)) shl 32) @@ -434,17 +347,10 @@ proc getSystemTimeAsFileTime*(lpSystemTimeAsFileTime: var FILETIME) {. proc sleep*(dwMilliseconds: int32){.stdcall, dynlib: "kernel32", importc: "Sleep", sideEffect.} -when useWinUnicode: - proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: WideCString, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} - -else: - proc shellExecuteA*(hwnd: Handle, lpOperation, lpFile, - lpParameters, lpDirectory: cstring, - nShowCmd: int32): Handle{. - stdcall, dynlib: "shell32.dll", importc: "ShellExecuteA", sideEffect.} +proc shellExecuteW*(hwnd: Handle, lpOperation, lpFile, + lpParameters, lpDirectory: WideCString, + nShowCmd: int32): Handle{. + stdcall, dynlib: "shell32.dll", importc: "ShellExecuteW", sideEffect.} proc getFileInformationByHandle*(hFile: Handle, lpFileInformation: ptr BY_HANDLE_FILE_INFORMATION): WINBOOL{. @@ -794,13 +700,6 @@ proc createFileMappingW*(hFile: Handle, lpName: pointer): Handle {. stdcall, dynlib: "kernel32", importc: "CreateFileMappingW".} -when not useWinUnicode: - proc createFileMappingA*(hFile: Handle, - lpFileMappingAttributes: pointer, - flProtect, dwMaximumSizeHigh: DWORD, - dwMaximumSizeLow: DWORD, lpName: cstring): Handle {. - stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".} - proc unmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall, dynlib: "kernel32", importc: "UnmapViewOfFile".} @@ -976,7 +875,7 @@ proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = var ver: OSVERSIONINFO ver.dwOSVersionInfoSize = sizeof(ver).DWORD - let res = when useWinUnicode: getVersionExW(ver.addr) else: getVersionExA(ver.addr) + let res = getVersionExW(ver.addr) if res == 0: result = nil elif ver.dwMajorVersion >= 6: @@ -1060,16 +959,10 @@ proc openProcess*(dwDesiredAccess: DWORD, bInheritHandle: WINBOOL, dwProcessId: DWORD): Handle {.stdcall, dynlib: "kernel32", importc: "OpenProcess".} -when defined(useWinAnsi): - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: cstring): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventA".} -else: - proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, - bManualReset: DWORD, bInitialState: DWORD, - lpName: ptr Utf16Char): Handle - {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} +proc createEvent*(lpEventAttributes: ptr SECURITY_ATTRIBUTES, + bManualReset: DWORD, bInitialState: DWORD, + lpName: ptr Utf16Char): Handle + {.stdcall, dynlib: "kernel32", importc: "CreateEventW".} proc setEvent*(hEvent: Handle): cint {.stdcall, dynlib: "kernel32", importc: "SetEvent".} @@ -1110,14 +1003,9 @@ type uChar*: int16 dwControlKeyState*: DWORD -when defined(useWinAnsi): - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputA".} -else: - proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, - lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} +proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, + lpNumberOfEventsRead: ptr cint): cint + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} type LPFIBER_START_ROUTINE* = proc (param: pointer) {.stdcall.} From 50baf21eacfaf1c9d6949bf2b2e9f931b0e1509c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 2 Mar 2023 08:36:02 +0100 Subject: [PATCH 1630/1654] =?UTF-8?q?fixes=20#20422;=20emit=20nimPrepareSt?= =?UTF-8?q?rMutationV2=20for=20toOpenArray=20to=20keep=20th=E2=80=A6=20(#2?= =?UTF-8?q?1459)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes #20422; emit nimPrepareStrMutationV2 for toOpenArray to keep the abstraction of mutable strings which have immutable string literals --- compiler/ccgcalls.nim | 4 +++- compiler/ccgexprs.nim | 5 ++++- tests/arc/topenarray.nim | 12 ++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 9993322fb..48e7fd290 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -156,7 +156,7 @@ proc reifiedOpenArray(n: PNode): bool {.inline.} = else: result = true -proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) = +proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType; prepareForMutation = false): (Rope, Rope) = var a, b, c: TLoc initLocExpr(p, q[1], a) initLocExpr(p, q[2], b) @@ -164,6 +164,8 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, # but first produce the required index checks: if optBoundsCheck in p.options: genBoundsCheck(p, a, b, c) + if prepareForMutation: + linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)]) let ty = skipTypes(a.t, abstractVar+{tyPtr}) let dest = getTypeDesc(p.module, destType) let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)] diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index fe7cb252e..f428324b7 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2408,7 +2408,10 @@ proc genDispose(p: BProc; n: PNode) = lineCg(p, cpsStmts, ["#nimDestroyAndDispose($#)", rdLoc(a)]) proc genSlice(p: BProc; e: PNode; d: var TLoc) = - let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon) + let (x, y) = genOpenArraySlice(p, e, e.typ, e.typ.lastSon, + prepareForMutation = e[1].kind == nkHiddenDeref and + e[1].typ.skipTypes(abstractInst).kind == tyString and + p.config.selectedGC in {gcArc, gcOrc}) if d.k == locNone: getTemp(p, e.typ, d) linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n", [rdLoc(d), x, y]) when false: diff --git a/tests/arc/topenarray.nim b/tests/arc/topenarray.nim index 47c26a11f..0e45f3ec7 100644 --- a/tests/arc/topenarray.nim +++ b/tests/arc/topenarray.nim @@ -6,6 +6,8 @@ Nim ''' matrix: "--gc:arc -d:useMalloc; --gc:arc" """ +{.experimental: "views".} + block: # bug 18627 proc setPosition(params: openArray[string]) = for i in params.toOpenArray(0, params.len - 1): @@ -38,3 +40,13 @@ block: # bug #20954 var v: seq[int] echo len(toOpenArray(v, 20, 30)) + +# bug #20422 + +proc f(a: var string) = + var v = a.toOpenArray(1, 3) + v[0] = 'a' + +var a = "Hello" +f(a) +doAssert a == "Hallo" From 38d299dfc0c9f1315cf412aa76cb041642d62f07 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 2 Mar 2023 16:16:28 +0800 Subject: [PATCH 1631/1654] fixes #20139; hash types based on its path relative to its package path (#21274) [backport:1.6] * fixes #20139; hash types based on its path relative its project * add a test case * fixes procs * better implementation and test case --------- Co-authored-by: Andreas Rumpf --- compiler/ccgexprs.nim | 4 +- compiler/ccgstmts.nim | 6 +-- compiler/ccgtypes.nim | 38 +++++++-------- compiler/cgen.nim | 2 +- compiler/injectdestructors.nim | 2 +- compiler/jsgen.nim | 4 +- compiler/liftdestructors.nim | 4 +- compiler/pathutils.nim | 51 +++++++++++++++++++- compiler/sighashes.nim | 86 +++++++++++++++++----------------- compiler/vm.nim | 2 +- tests/ccgbugs/m1/defs.nim | 4 ++ tests/ccgbugs/m2/defs.nim | 4 ++ tests/ccgbugs/t20139.nim | 10 ++++ 13 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 tests/ccgbugs/m1/defs.nim create mode 100644 tests/ccgbugs/m2/defs.nim create mode 100644 tests/ccgbugs/t20139.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f428324b7..a64a42219 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1704,7 +1704,7 @@ proc genNewFinalize(p: BProc, e: PNode) = proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo; result: var Rope) = if optTinyRtti in p.config.globalOptions: - let token = $genDisplayElem(MD5Digest(hashType(dest))) + let token = $genDisplayElem(MD5Digest(hashType(dest, p.config))) appcg(p.module, result, "#isObjDisplayCheck($#.m_type, $#, $#)", [a, getObjDepth(dest), token]) else: # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we @@ -2779,7 +2779,7 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) = rdMType(p, a, nilCheck, r) if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(dest) - let token = $genDisplayElem(MD5Digest(hashType(dest))) + let token = $genDisplayElem(MD5Digest(hashType(dest, p.config))) if nilCheck != "": linefmt(p, cpsStmts, "if ($1 && !#isObjDisplayCheck($2, $3, $4)){ #raiseObjectConversionError(); ", [nilCheck, r, checkFor, token]) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index db839b21b..7ade85b42 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1081,7 +1081,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type" if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(typeNode.typ) - appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ)))]) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(typeNode.typ, p.config)))]) else: let checkFor = genTypeInfoV1(p.module, typeNode.typ, typeNode.info) appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) @@ -1300,7 +1300,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) = let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type" if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(t[i][j].typ) - appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))]) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))]) else: let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info) appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) @@ -1445,7 +1445,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) = let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type" if optTinyRtti in p.config.globalOptions: let checkFor = $getObjDepth(t[i][j].typ) - appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ)))]) + appcg(p.module, orExpr, "#isObjDisplayCheck(#nimBorrowCurrentException()->$1, $2, $3)", [memberName, checkFor, $genDisplayElem(MD5Digest(hashType(t[i][j].typ, p.config)))]) else: let checkFor = genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info) appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor]) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index e7677f2f2..98c1fc55f 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -43,14 +43,14 @@ proc fillBackendName(m: BModule; s: PSym) = result.add rope s.itemId.item if m.hcrOn: result.add "_" - result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts)) + result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config)) s.loc.r = result writeMangledName(m.ndi, s, m.config) proc fillParamName(m: BModule; s: PSym) = if s.loc.r == "": var res = s.name.s.mangle - res.add idOrSig(s, res, m.sigConflicts) + res.add idOrSig(s, res, m.sigConflicts, m.config) # Take into account if HCR is on because of the following scenario: # if a module gets imported and it has some more importc symbols in it, # some param names might receive the "_0" suffix to distinguish from what @@ -302,7 +302,7 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = else: result = "" if result != "" and typ.isImportedType(): - let sig = hashType typ + let sig = hashType(typ, m.config) if cacheGetType(m.typeCache, sig) == "": m.typeCache[sig] = result @@ -362,10 +362,10 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): R if isImportedCppType(etB) and t.kind == tyGenericInst: result = getTypeDescAux(m, t, check, kind) else: - result = getTypeForward(m, t, hashType(t)) + result = getTypeForward(m, t, hashType(t, m.config)) pushType(m, t) of tySequence: - let sig = hashType(t) + let sig = hashType(t, m.config) if optSeqDestructors in m.config.globalOptions: if skipTypes(etB[0], typedescInst).kind == tyEmpty: internalError(m.config, "cannot map the empty seq type to a C type") @@ -398,7 +398,7 @@ proc getSeqPayloadType(m: BModule; t: PType): Rope = #result = getTypeForward(m, t, hashType(t)) & "_Content" proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) = - let sig = hashType(t) + let sig = hashType(t, m.config) let result = cacheGetType(m.typeCache, sig) if result == "": discard getTypeDescAux(m, t, check, skVar) @@ -664,7 +664,7 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType = else: result.elemType proc getOpenArrayDesc(m: BModule, t: PType, check: var IntSet; kind: TSymKind): Rope = - let sig = hashType(t) + let sig = hashType(t, m.config) if kind == skParam: result = getTypeDescWeak(m, t[0], check, kind) & "*" else: @@ -688,7 +688,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # C type generation into an analysis and a code generation phase somehow. if t.sym != nil: useHeader(m, t.sym) if t != origTyp and origTyp.sym != nil: useHeader(m, origTyp.sym) - let sig = hashType(origTyp) + let sig = hashType(origTyp, m.config) defer: # defer is the simplest in this case if isImportedType(t) and not m.typeABICache.containsOrIncl(sig): @@ -717,7 +717,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin result = getTypeDescAux(m, et, check, kind) & star else: # no restriction! We have a forward declaration for structs - let name = getTypeForward(m, et, hashType et) + let name = getTypeForward(m, et, hashType(et, m.config)) result = name & star m.typeCache[sig] = result of tySequence: @@ -726,7 +726,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin m.typeCache[sig] = result else: # no restriction! We have a forward declaration for structs - let name = getTypeForward(m, et, hashType et) + let name = getTypeForward(m, et, hashType(et, m.config)) result = name & seqStar(m) & star m.typeCache[sig] = result pushType(m, et) @@ -893,7 +893,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKin # Don't use the imported name as it may be scoped: 'Foo::SomeKind' result = rope("tySet_") t.lastSon.typeName(result) - result.add $t.lastSon.hashType + result.add $t.lastSon.hashType(m.config) m.typeCache[sig] = result if not isImportedType(t): let s = int(getSize(m.config, t)) @@ -1066,7 +1066,7 @@ proc discriminatorTableName(m: BModule, objtype: PType, d: PSym): Rope = objtype = objtype[0].skipTypes(abstractPtrs) if objtype.sym == nil: internalError(m.config, d.info, "anonymous obj with discriminator") - result = "NimDT_$1_$2" % [rope($hashType(objtype)), rope(d.name.s.mangle)] + result = "NimDT_$1_$2" % [rope($hashType(objtype, m.config)), rope(d.name.s.mangle)] proc rope(arg: Int128): Rope = rope($arg) @@ -1285,7 +1285,7 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope = result.add m.name.s & "." result.add it.sym.name.s else: - result = $hashType(it) + result = $hashType(it, m.config) result = makeCString(result) proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0 @@ -1329,14 +1329,14 @@ proc genDisplayElem(d: MD5Digest): uint32 = result += uint32(d[i]) result = result shl 8 -proc genDisplay(t: PType, depth: int): Rope = +proc genDisplay(m: BModule, t: PType, depth: int): Rope = result = Rope"{" var x = t var seqs = newSeq[string](depth+1) var i = 0 while x != nil: x = skipTypes(x, skipPtrs) - seqs[i] = $genDisplayElem(MD5Digest(hashType(x))) + seqs[i] = $genDisplayElem(MD5Digest(hashType(x, m.config))) x = x[0] inc i @@ -1376,7 +1376,7 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin [name, getTypeDesc(m, t), rope(objDepth), rope(flags)]) if objDepth >= 0: - let objDisplay = genDisplay(t, objDepth) + let objDisplay = genDisplay(m, t, objDepth) let objDisplayStore = getTempName(m) m.s[cfsVars].addf("static $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), skVar), objDisplayStore, rope(objDepth+1), objDisplay]) addf(typeEntry, "$1.display = $2;$n", [name, rope(objDisplayStore)]) @@ -1408,7 +1408,7 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn [getTypeDesc(m, t), rope(objDepth)]) if objDepth >= 0: - let objDisplay = genDisplay(t, objDepth) + let objDisplay = genDisplay(m, t, objDepth) let objDisplayStore = getTempName(m) m.s[cfsVars].addf("static NIM_CONST $1 $2[$3] = $4;$n", [getTypeDesc(m, getSysType(m.g.graph, unknownLineInfo, tyUInt32), skVar), objDisplayStore, rope(objDepth+1), objDisplay]) addf(typeEntry, ", .display = $1", [rope(objDisplayStore)]) @@ -1435,7 +1435,7 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope = let prefixTI = if m.hcrOn: "(" else: "(&" - let sig = hashType(origType) + let sig = hashType(origType, m.config) result = m.typeInfoMarkerV2.getOrDefault(sig) if result != "": return prefixTI.rope & result & ")".rope @@ -1509,7 +1509,7 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope = let prefixTI = if m.hcrOn: "(" else: "(&" - let sig = hashType(origType) + let sig = hashType(origType, m.config) result = m.typeInfoMarker.getOrDefault(sig) if result != "": return prefixTI.rope & result & ")".rope diff --git a/compiler/cgen.nim b/compiler/cgen.nim index c346a0b78..ab6e6559f 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1630,7 +1630,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = hcrModuleMeta.addf("\t\"\"};$n", []) hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", []) hcrModuleMeta.addf("$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n", - [($sigHash(m.module)).rope]) + [($sigHash(m.module, m.config)).rope]) if sfMainModule in m.module.flags: g.mainModProcs.add(hcrModuleMeta) g.mainModProcs.addf("static void* hcr_handle;$N", []) diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index 13141b765..d58f28289 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -217,7 +217,7 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode var op = getAttachedOp(c.graph, t, kind) if op == nil or op.ast.isGenericRoutine: # give up and find the canonical type instead: - let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct}) + let h = sighashes.hashType(t, c.graph.config, {CoType, CoConsiderOwned, CoDistinct}) let canon = c.graph.canonTypes.getOrDefault(h) if canon != nil: op = getAttachedOp(c.graph, canon, kind) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 572d19b13..6619963b0 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -262,7 +262,7 @@ proc mangleName(m: BModule, s: PSym): Rope = if m.config.hcrOn: # When hot reloading is enabled, we must ensure that the names # of functions and types will be preserved across rebuilds: - result.add(idOrSig(s, m.module.name.s, m.sigConflicts)) + result.add(idOrSig(s, m.module.name.s, m.sigConflicts, m.config)) else: result.add("_") result.add(rope(s.id)) @@ -2813,7 +2813,7 @@ proc genModule(p: PProc, n: PNode) = if p.config.hcrOn and n.kind == nkStmtList: let moduleSym = p.module.module var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" & - idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts) + idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts, p.config) lineF(p, "var $1;$n", [moduleLoadedVar]) var inGuardedBlock = false diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index c7464d39e..d7b1f2daa 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -553,7 +553,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) = # operation here: var t = t if t.assignment == nil or t.destructor == nil: - let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct}) + let h = sighashes.hashType(t,c.g.config, {CoType, CoConsiderOwned, CoDistinct}) let canon = c.g.canonTypes.getOrDefault(h) if canon != nil: t = canon @@ -1113,7 +1113,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink}) if isEmptyContainer(skipped) or skipped.kind == tyStatic: return - let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct}) + let h = sighashes.hashType(skipped, g.config, {CoType, CoConsiderOwned, CoDistinct}) var canon = g.canonTypes.getOrDefault(h) if canon == nil: g.canonTypes[h] = skipped diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 1ef0143d5..d9779deab 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -10,7 +10,7 @@ ## Path handling utilities for Nim. Strictly typed code in order ## to avoid the never ending time sink in getting path handling right. -import os, pathnorm +import os, pathnorm, strutils when defined(nimPreviewSlimSystem): import std/[syncio, assertions] @@ -102,3 +102,52 @@ when true: proc addFileExt*(x: RelativeFile; ext: string): RelativeFile {.borrow.} proc writeFile*(x: AbsoluteFile; content: string) {.borrow.} + +proc skipHomeDir(x: string): int = + when defined(windows): + if x.continuesWith("Users/", len("C:/")): + result = 3 + else: + result = 0 + else: + if x.startsWith("/home/") or x.startsWith("/Users/"): + result = 3 + elif x.startsWith("/mnt/") and x.continuesWith("/Users/", len("/mnt/c")): + result = 5 + else: + result = 0 + +proc relevantPart(s: string; afterSlashX: int): string = + result = newStringOfCap(s.len - 8) + var slashes = afterSlashX + for i in 0.. + # //user/ + if not isAbsolute(x): + result = customPathImpl(canonSlashes(getCurrentDir() / x)) + else: + let slashes = skipHomeDir(x) + if slashes > 0: + result = "//user/" & relevantPart(x, slashes) + else: + result = x + +proc customPath*(x: string): string = + customPathImpl canonSlashes x diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 3508a146a..7e2a0b660 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -9,10 +9,12 @@ ## Computes hash values for routine (proc, method etc) signatures. -import ast, tables, ropes, md5, modulegraphs +import ast, tables, ropes, md5, modulegraphs, options, msgs, packages, pathutils from hashes import Hash import types +import std/os + when defined(nimPreviewSlimSystem): import std/assertions @@ -42,8 +44,7 @@ type CoDistinct CoHashTypeInsideNode -proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) - +proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) proc hashSym(c: var MD5Context, s: PSym) = if sfAnon in s.flags or s.kind == skGenericParam: c &= ":anon" @@ -54,20 +55,21 @@ proc hashSym(c: var MD5Context, s: PSym) = c &= "." it = it.owner -proc hashTypeSym(c: var MD5Context, s: PSym) = +proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) = if sfAnon in s.flags or s.kind == skGenericParam: c &= ":anon" else: var it = s + c &= customPath(conf.toFullPath(s.info)) while it != nil: if sfFromGeneric in it.flags and it.kind in routineKinds and it.typ != nil: - hashType c, it.typ, {CoProc} + hashType c, it.typ, {CoProc}, conf c &= it.name.s c &= "." it = it.owner -proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = +proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]; conf: ConfigRef) = if n == nil: c &= "\255" return @@ -82,7 +84,7 @@ proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = of nkSym: hashSym(c, n.sym) if CoHashTypeInsideNode in flags and n.sym.typ != nil: - hashType(c, n.sym.typ, flags) + hashType(c, n.sym.typ, flags, conf) of nkCharLit..nkUInt64Lit: let v = n.intVal lowlevel v @@ -92,9 +94,9 @@ proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]) = of nkStrLit..nkTripleStrLit: c &= n.strVal else: - for i in 0.. 0 and t[0] != nil: - hashType c, t[0], flags + hashType c, t[0], flags, conf of tyRef, tyPtr, tyGenericBody, tyVar: c &= char(t.kind) if t.sons.len > 0: - c.hashType t.lastSon, flags + c.hashType t.lastSon, flags, conf if tfVarIsPtr in t.flags: c &= ".varisptr" of tyFromExpr: c &= char(t.kind) - c.hashTree(t.n, {}) + c.hashTree(t.n, {}, conf) of tyTuple: c &= char(t.kind) if t.n != nil and CoType notin flags: @@ -199,19 +201,19 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = assert(t.n[i].kind == nkSym) c &= t.n[i].sym.name.s c &= ':' - c.hashType(t[i], flags+{CoIgnoreRange}) + c.hashType(t[i], flags+{CoIgnoreRange}, conf) c &= ',' else: - for i in 0.. 1) order by hash; -proc hashType*(t: PType; flags: set[ConsiderFlag] = {CoType}): SigHash = +proc hashType*(t: PType; conf: ConfigRef; flags: set[ConsiderFlag] = {CoType}): SigHash = var c: MD5Context md5Init c - hashType c, t, flags+{CoOwnerSig} + hashType c, t, flags+{CoOwnerSig}, conf md5Final c, result.MD5Digest when defined(debugSigHashes): db.exec(sql"INSERT OR IGNORE INTO sighashes(type, hash) VALUES (?, ?)", typeToString(t), $result) -proc hashProc*(s: PSym): SigHash = +proc hashProc*(s: PSym; conf: ConfigRef): SigHash = var c: MD5Context md5Init c - hashType c, s.typ, {CoProc} + hashType c, s.typ, {CoProc}, conf var m = s while m.kind != skModule: m = m.owner @@ -315,9 +317,9 @@ proc hashOwner*(s: PSym): SigHash = md5Final c, result.MD5Digest -proc sigHash*(s: PSym): SigHash = +proc sigHash*(s: PSym; conf: ConfigRef): SigHash = if s.kind in routineKinds and s.typ != nil: - result = hashProc(s) + result = hashProc(s, conf) else: result = hashNonProc(s) @@ -378,7 +380,7 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash = var c: MD5Context md5Init(c) - c.hashType(sym.typ, {CoProc}) + c.hashType(sym.typ, {CoProc}, graph.config) c &= char(sym.kind) c.md5Final(result.MD5Digest) graph.symBodyHashes[sym.id] = result # protect from recursion in the body @@ -391,12 +393,12 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash = graph.symBodyHashes[sym.id] = result proc idOrSig*(s: PSym, currentModule: string, - sigCollisions: var CountTable[SigHash]): Rope = + sigCollisions: var CountTable[SigHash]; conf: ConfigRef): Rope = if s.kind in routineKinds and s.typ != nil: # signatures for exported routines are reliable enough to # produce a unique name and this means produced C++ is more stable regarding # Nim changes: - let sig = hashProc(s) + let sig = hashProc(s, conf) result = rope($sig) #let m = if s.typ.callConv != ccInline: findPendingModule(m, s) else: m let counter = sigCollisions.getOrDefault(sig) diff --git a/compiler/vm.nim b/compiler/vm.nim index 38b856277..18d840613 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1855,7 +1855,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind != nkSym: stackTrace(c, tos, pc, "node is not a symbol") else: - regs[ra].node.strVal = $sigHash(regs[rb].node.sym) + regs[ra].node.strVal = $sigHash(regs[rb].node.sym, c.config) of opcSlurp: decodeB(rkNode) createStr regs[ra] diff --git a/tests/ccgbugs/m1/defs.nim b/tests/ccgbugs/m1/defs.nim new file mode 100644 index 000000000..ed78d8b72 --- /dev/null +++ b/tests/ccgbugs/m1/defs.nim @@ -0,0 +1,4 @@ +type MyObj* = object + field1*: int + s*: string + ch*: char diff --git a/tests/ccgbugs/m2/defs.nim b/tests/ccgbugs/m2/defs.nim new file mode 100644 index 000000000..798d1fea8 --- /dev/null +++ b/tests/ccgbugs/m2/defs.nim @@ -0,0 +1,4 @@ +type MyObj* = object + s*: string + field1*: int + ch*: char diff --git a/tests/ccgbugs/t20139.nim b/tests/ccgbugs/t20139.nim new file mode 100644 index 000000000..4592b994d --- /dev/null +++ b/tests/ccgbugs/t20139.nim @@ -0,0 +1,10 @@ +discard """ + joinable: false +""" + +# bug #20139 +import m1/defs as md1 +import m2/defs as md2 + +doAssert $(md1.MyObj(field1: 1)) == """(field1: 1, s: "", ch: '\x00')""" +doAssert $(md2.MyObj(field1: 1)) == """(s: "", field1: 1, ch: '\x00')""" From d4d28f2ffe522d2509dccfa7be0eef709732d93f Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Fri, 3 Mar 2023 06:27:10 +1100 Subject: [PATCH 1632/1654] Allow `futureLogging` in release builds (#21448) * Add test case * Fixes 21447: Keeps stackTrace around when using futureLogging * Remove extra whitespace --- lib/pure/asyncfutures.nim | 2 +- tests/async/t21447.nim | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/async/t21447.nim diff --git a/lib/pure/asyncfutures.nim b/lib/pure/asyncfutures.nim index edf5f7277..51aaaca36 100644 --- a/lib/pure/asyncfutures.nim +++ b/lib/pure/asyncfutures.nim @@ -29,7 +29,7 @@ type finished: bool error*: ref Exception ## Stored exception errorStackTrace*: string - when not defined(release): + when not defined(release) or defined(futureLogging): stackTrace: seq[StackTraceEntry] ## For debugging purposes only. id: int fromProc: string diff --git a/tests/async/t21447.nim b/tests/async/t21447.nim new file mode 100644 index 000000000..e4f7ae31f --- /dev/null +++ b/tests/async/t21447.nim @@ -0,0 +1,6 @@ +discard """ + action: "compile" + cmd: "nim c -d:release -d:futureLogging $file" +""" + +import std/asyncdispatch From d51a392149df1c0783fa526eabbc904ce0cd0cbd Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 3 Mar 2023 14:36:38 +0800 Subject: [PATCH 1633/1654] replaces implicit passes array registed at runtime with explicit function calls; simplify compilation pipeline (#21444) * abolish using passes in the compiler; simplify compilation pipeline * duplicate code * Really cool to have the same signature... * haul * unify other backends * refactor process * introduce PipelinePhase * refactor compiler * fixes passes * fixes nimsuggest * add a sentinel * enable docs checkj * activate doc testing * clean up * complete cleanups --- compiler/cgen.nim | 25 +--- compiler/depends.nim | 9 +- compiler/docgen2.nim | 25 ++-- compiler/jsgen.nim | 19 ++- compiler/main.nim | 67 +++++----- compiler/modulegraphs.nim | 13 ++ compiler/modules.nim | 96 +------------- compiler/nimeval.nim | 29 +++-- compiler/passes.nim | 187 +++++++++++++++++---------- compiler/pipelines.nim | 254 +++++++++++++++++++++++++++++++++++++ compiler/pipelineutils.nim | 26 ++++ compiler/scriptconfig.nim | 15 +-- compiler/sem.nim | 63 +++++---- compiler/suggest.nim | 2 +- compiler/vm.nim | 11 +- doc/refc.md | 1 + nimsuggest/nimsuggest.nim | 2 +- 17 files changed, 531 insertions(+), 313 deletions(-) create mode 100644 compiler/pipelines.nim create mode 100644 compiler/pipelineutils.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index ab6e6559f..d051934b5 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -12,11 +12,13 @@ import ast, astalgo, hashes, trees, platform, magicsys, extccomp, options, intsets, nversion, nimsets, msgs, bitsets, idents, types, - ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth, + ccgutils, os, ropes, math, wordrecg, treetab, cgmeth, rodutils, renderer, cgendata, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf, injectdestructors, astmsgs, modulepaths, backendpragmas +import pipelineutils + when defined(nimPreviewSlimSystem): import std/assertions @@ -1938,8 +1940,7 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) - -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = injectG() result = newModule(g, module, graph.config) result.idgen = idgen @@ -2007,7 +2008,7 @@ proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) = proc genTopLevelStmt*(m: BModule; n: PNode) = ## Also called from `ic/cbackend.nim`. - if passes.skipCodegen(m.config, n): return + if pipelineutils.skipCodegen(m.config, n): return m.initProc.options = initProcOptions(m) #softRnl = if optLineDir in m.config.options: noRnl else: rnl # XXX replicate this logic! @@ -2020,12 +2021,6 @@ proc genTopLevelStmt*(m: BModule; n: PNode) = else: genProcBody(m.initProc, transformedN) -proc myProcess(b: PPassContext, n: PNode): PNode = - result = n - if b != nil: - var m = BModule(b) - genTopLevelStmt(m, n) - proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = if optForceFullMake notin m.config.globalOptions: if not moduleHasChanged(m.g.graph, m.module): @@ -2102,7 +2097,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = if {optGenStaticLib, optGenDynLib, optNoMain} * m.config.globalOptions == {}: for i in countdown(high(graph.globalDestructors), 0): n.add graph.globalDestructors[i] - if passes.skipCodegen(m.config, n): return + if pipelineutils.skipCodegen(m.config, n): return if moduleHasChanged(graph, m.module): # if the module is cached, we don't regenerate the main proc # nor the dispatchers? But if the dispatchers changed? @@ -2143,12 +2138,6 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) = let mm = m m.g.modulesClosed.add mm - -proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = - result = n - if b == nil: return - finalCodegenActions(graph, BModule(b), n) - proc genForwardedProcs(g: BModuleList) = # Forward declared proc:s lack bodies when first encountered, so they're given # a second pass here @@ -2175,5 +2164,3 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = m.writeModule(pending=true) writeMapping(config, g.mapping) if g.generatedHeader != nil: writeHeader(g.generatedHeader) - -const cgenPass* = makePass(myOpen, myProcess, myClose) diff --git a/compiler/depends.nim b/compiler/depends.nim index 59cdc0330..2087198f2 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -9,7 +9,7 @@ # This module implements a dependency file generator. -import options, ast, ropes, passes, pathutils, msgs, lineinfos +import options, ast, ropes, pathutils, msgs, lineinfos import modulegraphs @@ -79,7 +79,7 @@ proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) = let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym)) addDependencyAux(b, parent, child) -proc addDotDependency(c: PPassContext, n: PNode): PNode = +proc addDotDependency*(c: PPassContext, n: PNode): PNode = result = n let g = PGen(c) let b = Backend(g.graph.backend) @@ -100,7 +100,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) = rope(project.splitFile.name), b.dotGraph], changeFileExt(project, "dot")) -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = var g: PGen new(g) g.module = module @@ -109,6 +109,3 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext if graph.backend == nil: graph.backend = Backend(dotGraph: "") result = g - -const gendependPass* = makePass(open = myOpen, process = addDotDependency) - diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index e5669937f..0941be78a 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -11,7 +11,7 @@ # semantic checking. import - options, ast, msgs, passes, docgen, lineinfos, pathutils, packages + options, ast, msgs, docgen, lineinfos, pathutils, packages from modulegraphs import ModuleGraph, PPassContext @@ -38,21 +38,21 @@ template closeImpl(body: untyped) {.dirty.} = except IOError: discard -proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = +proc closeDoc*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = closeImpl: writeOutput(g.doc, useWarning, groupedToc) -proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = +proc closeJson*(graph: ModuleGraph; p: PPassContext, n: PNode): PNode = closeImpl: writeOutputJson(g.doc, useWarning) -proc processNode(c: PPassContext, n: PNode): PNode = +proc processNode*(c: PPassContext, n: PNode): PNode = result = n var g = PGen(c) if shouldProcess(g): generateDoc(g.doc, n, n, g.config) -proc processNodeJson(c: PPassContext, n: PNode): PNode = +proc processNodeJson*(c: PPassContext, n: PNode): PNode = result = n var g = PGen(c) if shouldProcess(g): @@ -68,20 +68,11 @@ template myOpenImpl(ext: untyped) {.dirty.} = g.doc = d result = g -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = +proc openHtml*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(HtmlExt) -proc myOpenTex(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = +proc openTex*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(TexExt) -proc myOpenJson(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = +proc openJson*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = myOpenImpl(JsonExt) - -const docgen2Pass* = makePass(open = myOpen, process = processNode, close = close) -const docgen2TexPass* = makePass(open = myOpenTex, process = processNode, - close = close) -const docgen2JsonPass* = makePass(open = myOpenJson, process = processNodeJson, - close = closeJson) - -proc finishDoc2Pass*(project: string) = - discard diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 6619963b0..5df40b996 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -31,10 +31,12 @@ implements the required case distinction. import ast, trees, magicsys, options, nversion, msgs, idents, types, - ropes, passes, ccgutils, wordrecg, renderer, + ropes, ccgutils, wordrecg, renderer, cgmeth, lowerings, sighashes, modulegraphs, lineinfos, rodutils, transf, injectdestructors, sourcemap, astmsgs, backendpragmas +import pipelineutils + import json, sets, math, tables, intsets import strutils except addf @@ -2830,11 +2832,11 @@ proc genModule(p: PProc, n: PNode) = if optStackTrace in p.options: p.body.add(frameDestroy(p)) -proc myProcess(b: PPassContext, n: PNode): PNode = +proc processJSCodeGen*(b: PPassContext, n: PNode): PNode = ## Generate JS code for a node. result = n let m = BModule(b) - if passes.skipCodegen(m.config, n): return n + if pipelineutils.skipCodegen(m.config, n): return n if m.module == nil: internalError(m.config, n.info, "myProcess") let globals = PGlobals(m.graph.backend) var p = newInitProc(globals, m) @@ -2869,7 +2871,7 @@ proc getClassName(t: PType): Rope = if s.loc.r != "": result = s.loc.r else: result = rope(s.name.s) -proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = +proc finalJSCodeGen*(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = ## Finalize JS code generation of a Nim module. ## Param `n` may contain nodes returned from the last module close call. var m = BModule(b) @@ -2879,14 +2881,14 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = for i in countdown(high(graph.globalDestructors), 0): n.add graph.globalDestructors[i] # Process any nodes left over from the last call to `myClose`. - result = myProcess(b, n) + result = processJSCodeGen(b, n) # Some codegen is different (such as no stacktraces; see `initProcOptions`) # when `std/system` is being processed. if sfSystemModule in m.module.flags: PGlobals(graph.backend).inSystem = false # Check if codegen should continue before any files are generated. # It may bail early is if too many errors have been raised. - if passes.skipCodegen(m.config, n): return n + if pipelineutils.skipCodegen(m.config, n): return n # Nim modules are compiled into a single JS file. # If this is the main module, then this is the final call to `myClose`. if sfMainModule in m.module.flags: @@ -2904,9 +2906,6 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = if not writeRope(code, outFile): rawMessage(m.config, errCannotOpenFile, outFile.string) -proc myOpen(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = - ## Create the JS backend pass context `BModule` for a Nim module. +proc setupJSgen*(graph: ModuleGraph; s: PSym; idgen: IdGenerator): PPassContext = result = newModule(graph, s) result.idgen = idgen - -const JSgenPass* = makePass(myOpen, myProcess, myClose) diff --git a/compiler/main.nim b/compiler/main.nim index b8e0e2f12..d37d26478 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -16,9 +16,9 @@ import std/[strutils, os, times, tables, sha1, with, json], llstream, ast, lexer, syntaxes, options, msgs, condsyms, - sem, idents, passes, extccomp, + idents, extccomp, cgen, nversion, - platform, nimconf, passaux, depends, vm, + platform, nimconf, depends, modules, modulegraphs, lineinfos, pathutils, vmprofiler @@ -29,12 +29,10 @@ when defined(nimPreviewSlimSystem): import ic / [cbackend, integrity, navigator] from ic / ic import rodViewer -when not defined(leanCompiler): - import jsgen, docgen, docgen2 +import pipelines -proc semanticPasses(g: ModuleGraph) = - registerPass g, verbosePass - registerPass g, semPass +when not defined(leanCompiler): + import docgen proc writeDepsFile(g: ModuleGraph) = let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps") @@ -68,9 +66,8 @@ proc writeCMakeDepsFile(conf: ConfigRef) = fl.close() proc commandGenDepend(graph: ModuleGraph) = - semanticPasses(graph) - registerPass(graph, gendependPass) - compileProject(graph) + setPipeLinePass(graph, GenDependPass) + compilePipelineProject(graph) let project = graph.config.projectFull writeDepsFile(graph) generateDot(graph, project) @@ -87,8 +84,8 @@ proc commandCheck(graph: ModuleGraph) = defineSymbol(conf.symbols, "nimconfig") elif conf.backend == backendJs: setTarget(conf.target, osJS, cpuJS) - semanticPasses(graph) # use an empty backend for semantic checking only - compileProject(graph) + setPipeLinePass(graph, SemPass) + compilePipelineProject(graph) if conf.symbolFiles != disabledSf: case conf.ideCmd @@ -102,22 +99,20 @@ when not defined(leanCompiler): proc commandDoc2(graph: ModuleGraph; ext: string) = handleDocOutputOptions graph.config graph.config.setErrorMaxHighMaybe - semanticPasses(graph) case ext: - of TexExt: registerPass(graph, docgen2TexPass) - of JsonExt: registerPass(graph, docgen2JsonPass) - of HtmlExt: registerPass(graph, docgen2Pass) + of TexExt: + setPipeLinePass(graph, Docgen2TexPass) + of JsonExt: + setPipeLinePass(graph, Docgen2JsonPass) + of HtmlExt: + setPipeLinePass(graph, Docgen2Pass) else: doAssert false, $ext - compileProject(graph) - finishDoc2Pass(graph.config.projectName) + compilePipelineProject(graph) proc commandCompileToC(graph: ModuleGraph) = let conf = graph.config extccomp.initVars(conf) - semanticPasses(graph) if conf.symbolFiles == disabledSf: - registerPass(graph, cgenPass) - if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"): if not changeDetectedViaJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile): # nothing changed @@ -127,7 +122,11 @@ proc commandCompileToC(graph: ModuleGraph) = if not extccomp.ccHasSaneOverflow(conf): conf.symbols.defineSymbol("nimEmulateOverflowChecks") - compileProject(graph) + if conf.symbolFiles == disabledSf: + setPipeLinePass(graph, CgenPass) + else: + setPipeLinePass(graph, SemPass) + compilePipelineProject(graph) if graph.config.errorCounter > 0: return # issue #9933 if conf.symbolFiles == disabledSf: @@ -160,33 +159,27 @@ proc commandCompileToJS(graph: ModuleGraph) = conf.exc = excCpp setTarget(conf.target, osJS, cpuJS) defineSymbol(conf.symbols, "ecmascript") # For backward compatibility - semanticPasses(graph) - registerPass(graph, JSgenPass) - compileProject(graph) + setPipeLinePass(graph, JSgenPass) + compilePipelineProject(graph) if optGenScript in conf.globalOptions: writeDepsFile(graph) -proc interactivePasses(graph: ModuleGraph) = +proc commandInteractive(graph: ModuleGraph) = + graph.config.setErrorMaxHighMaybe initDefines(graph.config.symbols) defineSymbol(graph.config.symbols, "nimscript") # note: seems redundant with -d:nimHasLibFFI when hasFFI: defineSymbol(graph.config.symbols, "nimffi") - registerPass(graph, verbosePass) - registerPass(graph, semPass) - registerPass(graph, evalPass) - -proc commandInteractive(graph: ModuleGraph) = - graph.config.setErrorMaxHighMaybe - interactivePasses(graph) - compileSystemModule(graph) + setPipeLinePass(graph, InterpreterPass) + compilePipelineSystemModule(graph) if graph.config.commandArgs.len > 0: - discard graph.compileModule(fileInfoIdx(graph.config, graph.config.projectFull), {}) + discard graph.compilePipelineModule(fileInfoIdx(graph.config, graph.config.projectFull), {}) else: var m = graph.makeStdinModule() incl(m.flags, sfMainModule) var idgen = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0) let s = llStreamOpenStdIn(onPrompt = proc() = flushDot(graph.config)) - processModule(graph, m, idgen, s) + discard processPipelineModule(graph, m, idgen, s) proc commandScan(cache: IdentCache, config: ConfigRef) = var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt) @@ -241,8 +234,6 @@ proc mainCommand*(graph: ModuleGraph) = let conf = graph.config let cache = graph.cache - # In "nim serve" scenario, each command must reset the registered passes - clearPasses(graph) conf.lastCmdTime = epochTime() conf.searchPaths.add(conf.libpath) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 598959a37..8ffbe20a5 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -57,6 +57,18 @@ type sym*: PSym info*: TLineInfo + PipelinePass* = enum + NonePass + SemPass + JSgenPass + CgenPass + EvalPass + InterpreterPass + GenDependPass + Docgen2TexPass + Docgen2JsonPass + Docgen2Pass + ModuleGraph* {.acyclic.} = ref object ifaces*: seq[Iface] ## indexed by int32 fileIdx packed*: PackedModuleGraph @@ -104,6 +116,7 @@ type cacheCounters*: Table[string, BiggestInt] # IC: implemented cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented passes*: seq[TPass] + pipelinePass*: PipelinePass onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.} diff --git a/compiler/modules.nim b/compiler/modules.nim index 838a89d83..0aa1c8930 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -11,13 +11,8 @@ import ast, magicsys, msgs, options, - idents, lexer, passes, syntaxes, llstream, modulegraphs, - lineinfos, pathutils, tables, packages - -when defined(nimPreviewSlimSystem): - import std/[syncio, assertions] - -import ic / replayer + idents, lexer, syntaxes, modulegraphs, + lineinfos, pathutils proc resetSystemArtifacts*(g: ModuleGraph) = magicsys.resetSysTypes(g) @@ -25,12 +20,12 @@ proc resetSystemArtifacts*(g: ModuleGraph) = template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent = getIdent(graph.cache, splitFile(filename).name) -proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = +proc partialInitModule*(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) = let packSym = getPackage(graph, fileIdx) result.owner = packSym result.position = int fileIdx -proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = +proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym = let filename = AbsoluteFile toFullPath(graph.config, fileIdx) # We cannot call ``newSym`` here, because we have to circumvent the ID # mechanism, which we do in order to assign each module a persistent ID. @@ -43,99 +38,16 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym = partialInitModule(result, graph, fileIdx, filename) graph.registerModule(result) -proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = - var flags = flags - if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule - result = graph.getModule(fileIdx) - - template processModuleAux(moduleStatus) = - onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) - var s: PLLStream - if sfMainModule in flags: - if graph.config.projectIsStdin: s = stdin.llStreamOpen - elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) - discard processModule(graph, result, idGeneratorFromModule(result), s) - if result == nil: - var cachedModules: seq[FileIndex] - result = moduleFromRodFile(graph, fileIdx, cachedModules) - let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - if result == nil: - result = newModule(graph, fileIdx) - result.flags.incl flags - registerModule(graph, result) - processModuleAux("import") - else: - if sfSystemModule in flags: - graph.systemModule = result - partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - registerModuleById(graph, m) - replayStateChanges(graph.packed[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) - elif graph.isDirty(result): - result.flags.excl sfDirty - # reset module fields: - initStrTables(graph, result) - result.ast = nil - processModuleAux("import(dirty)") - graph.markClientsDirty(fileIdx) - -proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = - # this is called by the semantic checking phase - assert graph.config != nil - result = compileModule(graph, fileIdx, {}, s) - graph.addDep(s, fileIdx) - # keep track of import relationships - if graph.config.hcrOn: - graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) - #if sfSystemModule in result.flags: - # localError(result.info, errAttemptToRedefine, result.name.s) - # restore the notes for outer module: - graph.config.notes = - if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes - else: graph.config.foreignPackageNotes - proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode = result = syntaxes.parseFile(fileIdx, graph.cache, graph.config) graph.addDep(s, fileIdx) graph.addIncludeDep(s.position.FileIndex, fileIdx) -proc connectCallbacks*(graph: ModuleGraph) = - graph.includeFileCallback = includeModule - graph.importModuleCallback = importModule - -proc compileSystemModule*(graph: ModuleGraph) = - if graph.systemModule == nil: - connectCallbacks(graph) - graph.config.m.systemFileIdx = fileInfoIdx(graph.config, - graph.config.libpath / RelativeFile"system.nim") - discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule}) - proc wantMainModule*(conf: ConfigRef) = if conf.projectFull.isEmpty: fatal(conf, gCmdLineInfo, "command expects a filename") conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt)) -proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = - connectCallbacks(graph) - let conf = graph.config - wantMainModule(conf) - configComplete(graph) - - let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") - let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx - conf.projectMainIdx2 = projectFile - - let packSym = getPackage(graph, projectFile) - graph.config.mainPackageId = packSym.getPackageId - graph.importStack.add projectFile - - if projectFile == systemFileIdx: - discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule}) - else: - graph.compileSystemModule() - discard graph.compileModule(projectFile, {sfMainModule}) - proc makeModule*(graph: ModuleGraph; filename: AbsoluteFile): PSym = result = graph.newModule(fileInfoIdx(graph.config, filename)) registerModule(graph, result) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 82e2f0812..8e8f4ac7b 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -9,10 +9,16 @@ ## exposes the Nim VM to clients. import - ast, astalgo, modules, passes, condsyms, - options, sem, llstream, lineinfos, vm, + ast, modules, condsyms, + options, llstream, lineinfos, vm, vmdef, modulegraphs, idents, os, pathutils, - passaux, scriptconfig, std/compilesettings + scriptconfig, std/compilesettings + +import pipelines + + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] type Interpreter* = ref object ## Use Nim as an interpreter with this object @@ -76,7 +82,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) = let s = if scriptStream != nil: scriptStream else: llStreamOpen(findFile(i.graph.config, i.scriptName), fmRead) - processModule(i.graph, i.mainModule, i.idgen, s) + discard processPipelineModule(i.graph, i.mainModule, i.idgen, s) proc findNimStdLib*(): string = ## Tries to find a path to a valid "system.nim" file. @@ -109,12 +115,10 @@ proc createInterpreter*(scriptName: string; var conf = newConfigRef() var cache = newIdentCache() var graph = newModuleGraph(cache, conf) - connectCallbacks(graph) + connectPipelineCallbacks(graph) initDefines(conf.symbols) for define in defines: defineSymbol(conf.symbols, define[0], define[1]) - registerPass(graph, semPass) - registerPass(graph, evalPass) for p in searchPaths: conf.searchPaths.add(AbsoluteDir p) @@ -129,7 +133,8 @@ proc createInterpreter*(scriptName: string; if registerOps: vm.registerAdditionalOps() # Required to register parts of stdlib modules graph.vm = vm - graph.compileSystemModule() + setPipeLinePass(graph, EvalPass) + graph.compilePipelineSystemModule() result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName, idgen: idgen) proc destroyInterpreter*(i: Interpreter) = @@ -159,13 +164,11 @@ proc runRepl*(r: TLLRepl; defineSymbol(conf.symbols, "nimscript") if supportNimscript: defineSymbol(conf.symbols, "nimconfig") when hasFFI: defineSymbol(graph.config.symbols, "nimffi") - registerPass(graph, verbosePass) - registerPass(graph, semPass) - registerPass(graph, evalPass) var m = graph.makeStdinModule() incl(m.flags, sfMainModule) var idgen = idGeneratorFromModule(m) if supportNimscript: graph.vm = setupVM(m, cache, "stdin", graph, idgen) - graph.compileSystemModule() - processModule(graph, m, idgen, llStreamOpenStdIn(r)) + setPipeLinePass(graph, InterpreterPass) + graph.compilePipelineSystemModule() + discard processPipelineModule(graph, m, idgen, llStreamOpenStdIn(r)) diff --git a/compiler/passes.nim b/compiler/passes.nim index 74c5ff77c..38c133d69 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -14,13 +14,22 @@ import options, ast, llstream, msgs, idents, syntaxes, modulegraphs, reorder, - lineinfos, pathutils, packages + lineinfos, + pipelineutils, + modules, pathutils, packages, + sem, semdata + +import ic/replayer + +export skipCodegen, resolveMod, prepareConfigNotes when defined(nimsuggest): import std/sha1 when defined(nimPreviewSlimSystem): - import std/syncio + import std/[syncio, assertions] + +import std/tables type TPassData* = tuple[input: PNode, closeOutput: PNode] @@ -38,12 +47,6 @@ proc makePass*(open: TPassOpen = nil, result.process = process result.isFrontend = isFrontend -proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} = - # can be used by codegen passes to determine whether they should do - # something with `n`. Currently, this ignores `n` and uses the global - # error count instead. - result = config.errorCounter > 0 - const maxPasses = 10 @@ -80,13 +83,6 @@ proc processTopLevelStmt(graph: ModuleGraph, n: PNode, a: var TPassContextArray) if isNil(m): return false result = true -proc resolveMod(conf: ConfigRef; module, relativeTo: string): FileIndex = - let fullPath = findModule(conf, module, relativeTo) - if fullPath.isEmpty: - result = InvalidFileIdx - else: - result = fileInfoIdx(conf, fullPath) - proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind, a: var TPassContextArray; m: PSym) = # XXX fixme this should actually be relative to the config file! @@ -100,23 +96,6 @@ proc processImplicits(graph: ModuleGraph; implicits: seq[string], nodeKind: TNod importStmt.add str if not processTopLevelStmt(graph, importStmt, a): break -const - imperativeCode = {low(TNodeKind)..high(TNodeKind)} - {nkTemplateDef, nkProcDef, nkMethodDef, - nkMacroDef, nkConverterDef, nkIteratorDef, nkFuncDef, nkPragma, - nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt} - -proc prepareConfigNotes(graph: ModuleGraph; module: PSym) = - # don't be verbose unless the module belongs to the main package: - if graph.config.belongsToProjectPackage(module): - graph.config.notes = graph.config.mainPackageNotes - else: - if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes - graph.config.notes = graph.config.foreignPackageNotes - -proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = - result = true - #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") - proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; stream: PLLStream): bool {.discardable.} = if graph.stopCompile(): return true @@ -153,42 +132,22 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; processImplicits graph, graph.config.implicitIncludes, nkIncludeStmt, a, module checkFirstLineIndentation(p) - while true: - if graph.stopCompile(): break - var n = parseTopLevelStmt(p) # todo merge it - if n.kind == nkEmpty: break + block processCode: + if graph.stopCompile(): break processCode + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break processCode - if true: - # read everything, no streaming possible - var sl = newNodeI(nkStmtList, n.info) + # read everything, no streaming possible + var sl = newNodeI(nkStmtList, n.info) + sl.add n + while true: + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break sl.add n - while true: - var n = parseTopLevelStmt(p) - if n.kind == nkEmpty: break - sl.add n - if sfReorder in module.flags or codeReordering in graph.config.features: - sl = reorder(graph, sl, module) - discard processTopLevelStmt(graph, sl, a) - break - elif n.kind in imperativeCode: - # read everything until the next proc declaration etc. - var sl = newNodeI(nkStmtList, n.info) - sl.add n - var rest: PNode = nil - while true: - var n = parseTopLevelStmt(p) - if n.kind == nkEmpty or n.kind notin imperativeCode: - rest = n - break - sl.add n - #echo "-----\n", sl - if not processTopLevelStmt(graph, sl, a): break - if rest != nil: - #echo "-----\n", rest - if not processTopLevelStmt(graph, rest, a): break - else: - #echo "----- single\n", n - if not processTopLevelStmt(graph, n, a): break + if sfReorder in module.flags or codeReordering in graph.config.features: + sl = reorder(graph, sl, module) + discard processTopLevelStmt(graph, sl, a) + closeParser(p) if s.kind != llsStdIn: break closePasses(graph, a) @@ -198,3 +157,99 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; # They are responsible for closing the rod files. See `cbackend.nim`. closeRodFile(graph, module) result = true + +proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = + var flags = flags + if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule + result = graph.getModule(fileIdx) + + template processModuleAux(moduleStatus) = + onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) + var s: PLLStream + if sfMainModule in flags: + if graph.config.projectIsStdin: s = stdin.llStreamOpen + elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) + discard processModule(graph, result, idGeneratorFromModule(result), s) + if result == nil: + var cachedModules: seq[FileIndex] + result = moduleFromRodFile(graph, fileIdx, cachedModules) + let filename = AbsoluteFile toFullPath(graph.config, fileIdx) + if result == nil: + result = newModule(graph, fileIdx) + result.flags.incl flags + registerModule(graph, result) + processModuleAux("import") + else: + if sfSystemModule in flags: + graph.systemModule = result + partialInitModule(result, graph, fileIdx, filename) + for m in cachedModules: + registerModuleById(graph, m) + replayStateChanges(graph.packed[m.int].module, graph) + replayGenericCacheInformation(graph, m.int) + elif graph.isDirty(result): + result.flags.excl sfDirty + # reset module fields: + initStrTables(graph, result) + result.ast = nil + processModuleAux("import(dirty)") + graph.markClientsDirty(fileIdx) + +proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = + # this is called by the semantic checking phase + assert graph.config != nil + result = compileModule(graph, fileIdx, {}, s) + graph.addDep(s, fileIdx) + # keep track of import relationships + if graph.config.hcrOn: + graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) + # restore the notes for outer module: + graph.config.notes = + if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes + else: graph.config.foreignPackageNotes + +proc connectCallbacks*(graph: ModuleGraph) = + graph.includeFileCallback = modules.includeModule + graph.importModuleCallback = importModule + +proc compileSystemModule*(graph: ModuleGraph) = + if graph.systemModule == nil: + connectCallbacks(graph) + graph.config.m.systemFileIdx = fileInfoIdx(graph.config, + graph.config.libpath / RelativeFile"system.nim") + discard graph.compileModule(graph.config.m.systemFileIdx, {sfSystemModule}) + +proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = + connectCallbacks(graph) + let conf = graph.config + wantMainModule(conf) + configComplete(graph) + + let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") + let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx + conf.projectMainIdx2 = projectFile + + let packSym = getPackage(graph, projectFile) + graph.config.mainPackageId = packSym.getPackageId + graph.importStack.add projectFile + + if projectFile == systemFileIdx: + discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule}) + else: + graph.compileSystemModule() + discard graph.compileModule(projectFile, {sfMainModule}) + +proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = + result = preparePContext(graph, module, idgen) + +proc mySemClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = + var c = PContext(context) + closePContext(graph, c, n) + +proc mySemProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = + result = semWithPContext(PContext(context), n) + +const semPass* = makePass(mySemOpen, mySemProcess, mySemClose, + isFrontend = true) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim new file mode 100644 index 000000000..b80681c4b --- /dev/null +++ b/compiler/pipelines.nim @@ -0,0 +1,254 @@ +import sem, cgen, modulegraphs, ast, llstream, parser, msgs, + lineinfos, reorder, options, semdata, cgendata, modules, pathutils, + packages, syntaxes, depends, vm + +import pipelineutils + +when not defined(leanCompiler): + import jsgen, docgen2 + +import std/[syncio, objectdollar, assertions, tables] +import renderer +import ic/replayer + + +proc setPipeLinePass*(graph: ModuleGraph; pass: PipelinePass) = + graph.pipelinePass = pass + +proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext): PNode = + case graph.pipelinePass + of CgenPass: + result = semNode + if bModule != nil: + genTopLevelStmt(BModule(bModule), result) + of JSgenPass: + when not defined(leanCompiler): + result = processJSCodeGen(bModule, semNode) + of GenDependPass: + result = addDotDependency(bModule, semNode) + of SemPass: + result = graph.emptyNode + of Docgen2Pass, Docgen2TexPass: + when not defined(leanCompiler): + result = processNode(bModule, semNode) + of Docgen2JsonPass: + when not defined(leanCompiler): + result = processNodeJson(bModule, semNode) + of EvalPass, InterpreterPass: + result = interpreterCode(bModule, semNode) + of NonePass: + doAssert false, "use setPipeLinePass to set a proper PipelinePass" + +proc processImplicitImports(graph: ModuleGraph; implicits: seq[string], nodeKind: TNodeKind, + m: PSym, ctx: PContext, bModule: PPassContext, idgen: IdGenerator, + ) = + # XXX fixme this should actually be relative to the config file! + let relativeTo = toFullPath(graph.config, m.info) + for module in items(implicits): + # implicit imports should not lead to a module importing itself + if m.position != resolveMod(graph.config, module, relativeTo).int32: + var importStmt = newNodeI(nodeKind, m.info) + var str = newStrNode(nkStrLit, module) + str.info = m.info + importStmt.add str + message(graph.config, importStmt.info, hintProcessingStmt, $idgen[]) + let semNode = semWithPContext(ctx, importStmt) + if semNode == nil or processPipeline(graph, semNode, bModule) == nil: + break + +proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; + stream: PLLStream): bool = + if graph.stopCompile(): return true + var + p: Parser + s: PLLStream + fileIdx = module.fileIdx + + prepareConfigNotes(graph, module) + let ctx = preparePContext(graph, module, idgen) + let bModule: PPassContext = + case graph.pipelinePass + of CgenPass: + setupCgen(graph, module, idgen) + of JSgenPass: + when not defined(leanCompiler): + setupJSgen(graph, module, idgen) + else: + nil + of EvalPass, InterpreterPass: + setupEvalGen(graph, module, idgen) + of GenDependPass: + setupDependPass(graph, module, idgen) + of Docgen2Pass: + when not defined(leanCompiler): + openHtml(graph, module, idgen) + else: + nil + of Docgen2TexPass: + when not defined(leanCompiler): + openTex(graph, module, idgen) + else: + nil + of Docgen2JsonPass: + when not defined(leanCompiler): + openJson(graph, module, idgen) + else: + nil + of SemPass: + nil + of NonePass: + doAssert false, "use setPipeLinePass to set a proper PipelinePass" + nil + + if stream == nil: + let filename = toFullPathConsiderDirty(graph.config, fileIdx) + s = llStreamOpen(filename, fmRead) + if s == nil: + rawMessage(graph.config, errCannotOpenFile, filename.string) + return false + else: + s = stream + + while true: + syntaxes.openParser(p, fileIdx, s, graph.cache, graph.config) + + if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"): + # XXX what about caching? no processing then? what if I change the + # modules to include between compilation runs? we'd need to track that + # in ROD files. I think we should enable this feature only + # for the interactive mode. + if module.name.s != "nimscriptapi": + processImplicitImports graph, graph.config.implicitImports, nkImportStmt, module, ctx, bModule, idgen + processImplicitImports graph, graph.config.implicitIncludes, nkIncludeStmt, module, ctx, bModule, idgen + + checkFirstLineIndentation(p) + block processCode: + if graph.stopCompile(): break processCode + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break processCode + # read everything, no streaming possible + var sl = newNodeI(nkStmtList, n.info) + sl.add n + while true: + var n = parseTopLevelStmt(p) + if n.kind == nkEmpty: break + sl.add n + if sfReorder in module.flags or codeReordering in graph.config.features: + sl = reorder(graph, sl, module) + if graph.pipelinePass != EvalPass: + message(graph.config, sl.info, hintProcessingStmt, $idgen[]) + var semNode = semWithPContext(ctx, sl) + discard processPipeline(graph, semNode, bModule) + + closeParser(p) + if s.kind != llsStdIn: break + let finalNode = closePContext(graph, ctx, nil) + case graph.pipelinePass + of CgenPass: + if bModule != nil: + finalCodegenActions(graph, BModule(bModule), finalNode) + of JSgenPass: + when not defined(leanCompiler): + discard finalJSCodeGen(graph, bModule, finalNode) + of EvalPass, InterpreterPass: + discard interpreterCode(bModule, finalNode) + of SemPass, GenDependPass: + discard + of Docgen2Pass, Docgen2TexPass: + when not defined(leanCompiler): + discard closeDoc(graph, bModule, finalNode) + of Docgen2JsonPass: + when not defined(leanCompiler): + discard closeJson(graph, bModule, finalNode) + of NonePass: + doAssert false, "use setPipeLinePass to set a proper PipelinePass" + + if graph.config.backend notin {backendC, backendCpp, backendObjc}: + # We only write rod files here if no C-like backend is active. + # The C-like backends have been patched to support the IC mechanism. + # They are responsible for closing the rod files. See `cbackend.nim`. + closeRodFile(graph, module) + result = true + +proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = + var flags = flags + if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule + result = graph.getModule(fileIdx) + + template processModuleAux(moduleStatus) = + onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule) + var s: PLLStream + if sfMainModule in flags: + if graph.config.projectIsStdin: s = stdin.llStreamOpen + elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) + discard processPipelineModule(graph, result, idGeneratorFromModule(result), s) + if result == nil: + var cachedModules: seq[FileIndex] + result = moduleFromRodFile(graph, fileIdx, cachedModules) + let filename = AbsoluteFile toFullPath(graph.config, fileIdx) + if result == nil: + result = newModule(graph, fileIdx) + result.flags.incl flags + registerModule(graph, result) + processModuleAux("import") + else: + if sfSystemModule in flags: + graph.systemModule = result + partialInitModule(result, graph, fileIdx, filename) + for m in cachedModules: + registerModuleById(graph, m) + replayStateChanges(graph.packed[m.int].module, graph) + replayGenericCacheInformation(graph, m.int) + elif graph.isDirty(result): + result.flags.excl sfDirty + # reset module fields: + initStrTables(graph, result) + result.ast = nil + processModuleAux("import(dirty)") + graph.markClientsDirty(fileIdx) + +proc importPipelineModule(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym = + # this is called by the semantic checking phase + assert graph.config != nil + result = compilePipelineModule(graph, fileIdx, {}, s) + graph.addDep(s, fileIdx) + # keep track of import relationships + if graph.config.hcrOn: + graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) + #if sfSystemModule in result.flags: + # localError(result.info, errAttemptToRedefine, result.name.s) + # restore the notes for outer module: + graph.config.notes = + if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes + else: graph.config.foreignPackageNotes + +proc connectPipelineCallbacks*(graph: ModuleGraph) = + graph.includeFileCallback = modules.includeModule + graph.importModuleCallback = importPipelineModule + +proc compilePipelineSystemModule*(graph: ModuleGraph) = + if graph.systemModule == nil: + connectPipelineCallbacks(graph) + graph.config.m.systemFileIdx = fileInfoIdx(graph.config, + graph.config.libpath / RelativeFile"system.nim") + discard graph.compilePipelineModule(graph.config.m.systemFileIdx, {sfSystemModule}) + +proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = + connectPipelineCallbacks(graph) + let conf = graph.config + wantMainModule(conf) + configComplete(graph) + + let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim") + let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx + conf.projectMainIdx2 = projectFile + + let packSym = getPackage(graph, projectFile) + graph.config.mainPackageId = packSym.getPackageId + graph.importStack.add projectFile + + if projectFile == systemFileIdx: + discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule}) + else: + graph.compilePipelineSystemModule() + discard graph.compilePipelineModule(projectFile, {sfMainModule}) diff --git a/compiler/pipelineutils.nim b/compiler/pipelineutils.nim new file mode 100644 index 000000000..75ba33f14 --- /dev/null +++ b/compiler/pipelineutils.nim @@ -0,0 +1,26 @@ +import ast, options, lineinfos, pathutils, msgs, modulegraphs, packages + +proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} = + # can be used by codegen passes to determine whether they should do + # something with `n`. Currently, this ignores `n` and uses the global + # error count instead. + result = config.errorCounter > 0 + +proc resolveMod*(conf: ConfigRef; module, relativeTo: string): FileIndex = + let fullPath = findModule(conf, module, relativeTo) + if fullPath.isEmpty: + result = InvalidFileIdx + else: + result = fileInfoIdx(conf, fullPath) + +proc prepareConfigNotes*(graph: ModuleGraph; module: PSym) = + # don't be verbose unless the module belongs to the main package: + if graph.config.belongsToProjectPackage(module): + graph.config.notes = graph.config.mainPackageNotes + else: + if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes + graph.config.notes = graph.config.foreignPackageNotes + +proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} = + result = true + #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange") diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 009e90fd5..27ea94aae 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -11,10 +11,10 @@ ## language. import - ast, modules, idents, passes, condsyms, - options, sem, llstream, vm, vmdef, commands, + ast, modules, idents, condsyms, + options, llstream, vm, vmdef, commands, os, times, osproc, wordrecg, strtabs, modulegraphs, - pathutils + pathutils, pipelines when defined(nimPreviewSlimSystem): import std/syncio @@ -197,13 +197,11 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; conf.symbolFiles = disabledSf let graph = newModuleGraph(cache, conf) - connectCallbacks(graph) + connectPipelineCallbacks(graph) if freshDefines: initDefines(conf.symbols) defineSymbol(conf.symbols, "nimscript") defineSymbol(conf.symbols, "nimconfig") - registerPass(graph, semPass) - registerPass(graph, evalPass) conf.searchPaths.add(conf.libpath) @@ -218,8 +216,9 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; var vm = setupVM(m, cache, scriptName.string, graph, idgen) graph.vm = vm - graph.compileSystemModule() - discard graph.processModule(m, vm.idgen, stream) + graph.setPipeLinePass(EvalPass) + graph.compilePipelineSystemModule() + discard graph.processPipelineModule(m, vm.idgen, stream) # watch out, "newruntime" can be set within NimScript itself and then we need # to remember this: diff --git a/compiler/sem.nim b/compiler/sem.nim index 78596d5b2..a5d06f3fe 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -13,7 +13,7 @@ import ast, strutils, options, astalgo, trees, wordrecg, ropes, msgs, idents, renderer, types, platform, math, magicsys, nversion, nimsets, semfold, modulepaths, importer, - procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch, + procfind, lookups, pragmas, semdata, semtypinst, sigmatch, intsets, transf, vmdef, vm, aliases, cgmeth, lambdalifting, evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity, lowerings, plugins/active, lineinfos, strtabs, int128, @@ -666,38 +666,37 @@ proc addCodeForGenerics(c: PContext, n: PNode) = n.add prc.ast c.lastGenericIdx = c.generics.len -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = - var c = newContext(graph, module) - c.idgen = idgen - c.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) - c.voidType = newType(tyVoid, nextTypeId(idgen), nil) +proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext {.nosinks.} = + result = newContext(graph, module) + result.idgen = idgen + result.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) + result.voidType = newType(tyVoid, nextTypeId(idgen), nil) - if c.p != nil: internalError(graph.config, module.info, "sem.myOpen") - c.semConstExpr = semConstExpr - c.semExpr = semExpr - c.semTryExpr = tryExpr - c.semTryConstExpr = tryConstExpr - c.computeRequiresInit = computeRequiresInit - c.semOperand = semOperand - c.semConstBoolExpr = semConstBoolExpr - c.semOverloadedCall = semOverloadedCall - c.semInferredLambda = semInferredLambda - c.semGenerateInstance = generateInstance - c.semTypeNode = semTypeNode - c.instTypeBoundOp = sigmatch.instTypeBoundOp - c.hasUnresolvedArgs = hasUnresolvedArgs - c.templInstCounter = new int + if result.p != nil: internalError(graph.config, module.info, "sem.preparePContext") + result.semConstExpr = semConstExpr + result.semExpr = semExpr + result.semTryExpr = tryExpr + result.semTryConstExpr = tryConstExpr + result.computeRequiresInit = computeRequiresInit + result.semOperand = semOperand + result.semConstBoolExpr = semConstBoolExpr + result.semOverloadedCall = semOverloadedCall + result.semInferredLambda = semInferredLambda + result.semGenerateInstance = generateInstance + result.semTypeNode = semTypeNode + result.instTypeBoundOp = sigmatch.instTypeBoundOp + result.hasUnresolvedArgs = hasUnresolvedArgs + result.templInstCounter = new int - pushProcCon(c, module) - pushOwner(c, c.module) + pushProcCon(result, module) + pushOwner(result, result.module) - c.moduleScope = openScope(c) - c.moduleScope.addSym(module) # a module knows itself + result.moduleScope = openScope(result) + result.moduleScope.addSym(module) # a module knows itself if sfSystemModule in module.flags: graph.systemModule = module - c.topLevelScope = openScope(c) - result = c + result.topLevelScope = openScope(result) proc isImportSystemStmt(g: ModuleGraph; n: PNode): bool = if g.systemModule == nil: return false @@ -771,8 +770,7 @@ proc recoverContext(c: PContext) = while getCurrOwner(c).kind != skModule: popOwner(c) while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next -proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = - var c = PContext(context) +proc semWithPContext*(c: PContext, n: PNode): PNode {.nosinks.} = # no need for an expensive 'try' if we stop after the first error anyway: if c.config.errorMax <= 1: result = semStmtAndGenerateGenerics(c, n) @@ -793,13 +791,13 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = #if c.config.cmd == cmdIdeTools: findSuggest(c, n) storeRodNode(c, result) + proc reportUnusedModules(c: PContext) = for i in 0..high(c.unusedImports): if sfUsed notin c.unusedImports[i][0].flags: message(c.config, c.unusedImports[i][1], warnUnusedImportX, c.unusedImports[i][0].name.s) -proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = - var c = PContext(context) +proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode = if c.config.cmd == cmdIdeTools and not c.suggestionsMade: suggestSentinel(c) closeScope(c) # close module's scope @@ -814,6 +812,3 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = popOwner(c) popProcCon(c) sealRodFile(c) - -const semPass* = makePass(myOpen, myProcess, myClose, - isFrontend = true) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 637010ad5..384ceb271 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -36,7 +36,7 @@ import algorithm, sets, prefixmatches, parseutils, tables from wordrecg import wDeprecated, wError, wAddr, wYield when defined(nimsuggest): - import passes, tables, pathutils # importer + import tables, pathutils # importer const sep = '\t' diff --git a/compiler/vm.nim b/compiler/vm.nim index 18d840613..96f05c0f9 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -13,7 +13,7 @@ import semmacrosanity import std/[strutils, tables, parseutils], - msgs, vmdef, vmgen, nimsets, types, passes, + msgs, vmdef, vmgen, nimsets, types, parser, vmdeps, idents, trees, renderer, options, transf, gorgeimpl, lineinfos, btrees, macrocacheimpl, modulegraphs, sighashes, int128, vmprofiler @@ -2309,7 +2309,7 @@ proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) = else: refresh(PCtx graph.vm, module, idgen) -proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = #var c = newEvalContext(module, emRepl) #c.features = {allowCast, allowInfiniteLoops} #pushStackFrame(c, newStackFrame()) @@ -2318,7 +2318,7 @@ proc myOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext setupGlobalCtx(module, graph, idgen) result = PCtx graph.vm -proc myProcess(c: PPassContext, n: PNode): PNode = +proc interpreterCode*(c: PPassContext, n: PNode): PNode = let c = PCtx(c) # don't eval errornous code: if c.oldErrorCount == c.config.errorCounter: @@ -2328,11 +2328,6 @@ proc myProcess(c: PPassContext, n: PNode): PNode = result = n c.oldErrorCount = c.config.errorCounter -proc myClose(graph: ModuleGraph; c: PPassContext, n: PNode): PNode = - result = myProcess(c, n) - -const evalPass* = makePass(myOpen, myProcess, myClose) - proc evalConstExprAux(module: PSym; idgen: IdGenerator; g: ModuleGraph; prc: PSym, n: PNode, mode: TEvalMode): PNode = diff --git a/doc/refc.md b/doc/refc.md index 6e4672ac5..4023748e6 100644 --- a/doc/refc.md +++ b/doc/refc.md @@ -138,6 +138,7 @@ The garbage collector won't try to free them, you need to call their respective when you are done with them or they will leak. + Heap dump ========= diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index a0a3b8e82..ce9681975 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -23,7 +23,7 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp # Do NOT import suggest. It will lead to weird bugs with # suggestionResultHook, because suggest.nim is included by sigmatch. # So we import that one instead. -import compiler / [options, commands, modules, sem, +import compiler / [options, commands, modules, passes, passaux, msgs, sigmatch, ast, idents, modulegraphs, prefixmatches, lineinfos, cmdlinehelper, From 2e2affb13ce13723e5ab1996d05f7b82c184a090 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 3 Mar 2023 23:13:27 +0800 Subject: [PATCH 1634/1654] test DLL generation with ORC (#21445) * test DLL generation with ORC * fixes * fixes refc * Update testament/categories.nim --- testament/categories.nim | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/testament/categories.nim b/testament/categories.nim index cae699327..c428ffc04 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -47,21 +47,24 @@ proc isTestFile*(file: string): bool = # --------------------- DLL generation tests ---------------------------------- -proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = +proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string, isOrc = false) = const rpath = when defined(macosx): " --passL:-rpath --passL:@loader_path" else: "" - var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) - test1.spec.action = actionCompile - testSpec c, test1 + if not defined(windows) or not isOrc: # todo fix me on windows + var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) + test1.spec.action = actionCompile + testSpec c, test1 var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 - var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) - test3.spec.action = actionCompile - testSpec c, test3 + + if not isOrc: + var test3 = makeTest("lib/nimhcr.nim", options & " --threads:off --outdir:tests/dll" & rpath, cat) + test3.spec.action = actionCompile + testSpec c, test3 var test4 = makeTest("tests/dll/visibility.nim", options & " --threads:off --app:lib" & rpath, cat) test4.spec.action = actionCompile testSpec c, test4 @@ -76,8 +79,9 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) - testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) - testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) + if not isOrc: + testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & " --threads:off" & rpath, cat) testSpec r, makeTest("tests/dll/visibility.nim", options & " --threads:off" & rpath, cat) if "boehm" notin options: @@ -96,6 +100,8 @@ proc dllTests(r: var TResults, cat: Category, options: string) = runBasicDLLTest c, r, cat, options & " --mm:refc" runBasicDLLTest c, r, cat, options & " -d:release --mm:refc" + runBasicDLLTest c, r, cat, options, isOrc = true + runBasicDLLTest c, r, cat, options & " -d:release", isOrc = true when not defined(windows): # still cannot find a recent Windows version of boehm.dll: runBasicDLLTest c, r, cat, options & " --gc:boehm" From b2c5f8a05f2f9db6ed06663a077882eb112b3e3f Mon Sep 17 00:00:00 2001 From: Arnaud Moura Date: Fri, 3 Mar 2023 23:37:12 +0100 Subject: [PATCH 1635/1654] fixes #21461 (#21463) * fixes #21461; Move nim-gdb.py and add nim-lldb.py * fixes bad path for nim-gdb.py --- bin/nim-gdb | 2 +- bin/nim-gdb.bat | 2 +- compiler/installer.ini | 2 +- doc/packaging.md | 2 +- .../untestable/gdb/gdb_pretty_printer_test.py | 2 +- tools/{ => debug}/nim-gdb.py | 0 tools/debug/nim-lldb.py | 1039 +++++++++++++++++ 7 files changed, 1044 insertions(+), 5 deletions(-) rename tools/{ => debug}/nim-gdb.py (100%) create mode 100644 tools/debug/nim-lldb.py diff --git a/bin/nim-gdb b/bin/nim-gdb index 13b361f93..d4d60e432 100755 --- a/bin/nim-gdb +++ b/bin/nim-gdb @@ -14,7 +14,7 @@ else fi # Find out where the pretty printer Python module is -GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py" +GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/debug/nim-gdb.py" # Run GDB with the additional arguments that load the pretty printers # Set the environment variable `NIM_GDB` to overwrite the call to a diff --git a/bin/nim-gdb.bat b/bin/nim-gdb.bat index e98a2063c..b5537dd9d 100644 --- a/bin/nim-gdb.bat +++ b/bin/nim-gdb.bat @@ -3,7 +3,7 @@ for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i) for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi) -set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py +set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\debug\nim-gdb.py set @NIM_GDB=gdb.exe @echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt diff --git a/compiler/installer.ini b/compiler/installer.ini index f67ca5c25..226682715 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -65,7 +65,7 @@ Files: "compiler" Files: "doc" Files: "doc/html" Files: "tools" -Files: "tools/nim-gdb.py" +Files: "tools/debug/nim-gdb.py" Files: "nimpretty" Files: "testament" Files: "nimsuggest" diff --git a/doc/packaging.md b/doc/packaging.md index 522ca6d44..b742bef28 100644 --- a/doc/packaging.md +++ b/doc/packaging.md @@ -71,7 +71,7 @@ What to install: - The expected stdlib location is `/usr/lib/nim/lib`, previously it was just `/usr/lib/nim` - `nimdoc.css` and `nimdoc.cls` from the `doc` folder should go into `/usr/lib/nim/doc/` -- `tools/nim-gdb.py` should go into `/usr/lib/nim/tools/` +- `tools/debug/nim-gdb.py` should go into `/usr/lib/nim/tools/` - `tools/dochack/dochack.js` should be installed to `/usr/lib/nim/tools/dochack/` - Global configuration files under `/etc/nim` - Optionally: manpages, documentation, shell completion diff --git a/tests/untestable/gdb/gdb_pretty_printer_test.py b/tests/untestable/gdb/gdb_pretty_printer_test.py index a96df9992..aed0cfeb0 100644 --- a/tests/untestable/gdb/gdb_pretty_printer_test.py +++ b/tests/untestable/gdb/gdb_pretty_printer_test.py @@ -8,7 +8,7 @@ import sys # frontends might still be broken. gdb.execute("set python print-stack full") -gdb.execute("source ../../../tools/nim-gdb.py") +gdb.execute("source ../../../tools/debug/nim-gdb.py") # debug all instances of the generic function `myDebug`, should be 14 gdb.execute("rbreak myDebug") gdb.execute("run") diff --git a/tools/nim-gdb.py b/tools/debug/nim-gdb.py similarity index 100% rename from tools/nim-gdb.py rename to tools/debug/nim-gdb.py diff --git a/tools/debug/nim-lldb.py b/tools/debug/nim-lldb.py new file mode 100644 index 000000000..625b75988 --- /dev/null +++ b/tools/debug/nim-lldb.py @@ -0,0 +1,1039 @@ +import lldb +from collections import OrderedDict +from typing import Union + + +def sbvaluegetitem(self: lldb.SBValue, name: Union[int, str]) -> lldb.SBValue: + if isinstance(name, str): + return self.GetChildMemberWithName(name) + else: + return self.GetChildAtIndex(name) + + +# Make this easier to work with +lldb.SBValue.__getitem__ = sbvaluegetitem + + +def colored(in_str, *args, **kwargs): + # TODO: Output in color if user is in terminal + return in_str + + +def reprEnum(val, typ): + """ + this is a port of the nim runtime function `reprEnum` to python + NOTE: DOES NOT WORK WITH ORC + """ + val = int(val) + n = typ["node"] + sons_type = n["sons"].type.GetPointeeType().GetPointeeType() + sons = n["sons"].deref.Cast(sons_type.GetPointerType().GetArrayType(3)) + flags = int(typ["flags"].unsigned) + # 1 << 6 is {ntfEnumHole} + if ((1 << 6) & flags) == 0: + offset = val - sons[0]["offset"].unsigned + if offset >= 0 and 0 < n["len"].unsigned: + return NCSTRING(sons[offset]["name"])[1:-1] + else: + # ugh we need a slow linear search: + for i in range(n["len"].unsigned): + if sons[i]["offset"].unsigned == val: + return NCSTRING(sons[i]["name"])[1:-1] + + return str(val) + " (invalid data!)" + + +def get_nti(value, nim_name=None): + """DOES NOT WORK WITH ORC""" + name_split = value.type.name.split("_") + type_nim_name = nim_name or name_split[1] + id_string = name_split[-1].split(" ")[0] + + type_info_name = "NTI" + type_nim_name.lower() + "__" + id_string + "_" + print("TYPEINFONAME: ", type_info_name) + nti = value.target.FindFirstGlobalVariable(type_info_name) + if nti is None: + type_info_name = "NTI" + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if nti is None: + print( + f"NimEnumPrinter: lookup global symbol: '{type_info_name}' failed for {value.type.name}.\n" + ) + return type_nim_name, nti + + +def enum_to_string(value): + type_nim_name, nti = get_nti(value) + if nti is None: + return type_nim_name + "(" + str(value.unsigned) + ")" + return reprEnum(value.signed, nti), nti + + +def to_string(value): + # For getting NimStringDesc * value + data = value["data"] + try: + size = int(value["Sup"]["len"].unsigned) + if size > 2**14: + return None + except TypeError: + return None + + cast = data.Cast(value.target.FindFirstType("char").GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def to_stringV2(value: lldb.SBValue): + # For getting NimStringDesc * value + data = value["p"]["data"] + try: + size = int(value["len"].signed) + if size > 2**14: + return "... (too long)" + except TypeError: + return "" + + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type.GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def NimStringDesc(value, internal_dict): + res = to_string(value) + if res: + return colored('"' + res + '"', "red") + else: + return str(value) + + +def NimStringV2(value: lldb.SBValue, internal_dict): + res = to_stringV2(value.GetNonSyntheticValue()) + if res is not None: + return colored('"' + res + '"', "red") + else: + return str(value) + + +def NCSTRING(value: lldb.SBValue, internal_dict=None): + ty = value.Dereference().type + val = value.target.CreateValueFromAddress( + value.name or "temp", lldb.SBAddress(value.unsigned, value.target), ty + ).AddressOf() + return val.summary + + +def ObjectV1(value, internal_dict): + if not value.num_children and not value.value: + return "" + + ignore_fields = set() + if "colonObjectType" in value.type.name: + value = value.Dereference() + ignore_fields.add("Sup") + + if not value.type.name: + return "" + + summary = value.summary + if summary is not None: + return summary + + if "_" in value.type.name: + obj_name = value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = value.type.name + + obj_name = colored(obj_name, "green") + + num_children = value.num_children + + fields = ", ".join( + [ + value[i].name + + ": " + + (value[i].summary or value[i].value or value[i].type.name or "not found") + for i in range(num_children) + if value[i].name not in ignore_fields + ] + ) + + res = f"{obj_name}({fields})" + return res + + +def ObjectV2(value: lldb.SBValue, internal_dict): + custom_summary = get_summary(value) + if not custom_summary is None: + return custom_summary + + orig_value = value.GetNonSyntheticValue() + while orig_value.type.is_pointer: + orig_value = orig_value.Dereference() + + if "_" in orig_value.type.name: + obj_name = orig_value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = orig_value.type.name + + num_children = value.num_children + fields = [] + + for i in range(num_children): + fields.append(f"{value[i].name}: {value[i].summary}") + + res = f"{obj_name}(" + ", ".join(fields) + ")" + return res + + +def Number(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.signed), "yellow") + + +def Float(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.value), "yellow") + + +def UnsignedNumber(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.unsigned), "yellow") + + +def Bool(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + return colored(str(value.GetValue()), "red") + + +def CharArray(value: lldb.SBValue, internal_dict): + return str([colored(f"'{char}'", "red") for char in value.uint8s]) + + +def Array(value: lldb.SBValue, internal_dict): + value = value.GetNonSyntheticValue() + return "[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +def Tuple(value: lldb.SBValue, internal_dict): + while value.type.is_pointer: + value = value.Dereference() + + num_children = value.num_children + + fields = [] + + for i in range(num_children): + key = value[i].name + val = value[i].summary + if key.startswith("Field"): + fields.append(f"{val}") + else: + fields.append(f"{key}: {val}") + + return "(" + ", ".join(fields) + f")" + + +def Enum(value, internal_dict): + tname = value.type.name.split("_")[1] + return colored(f"{tname}." + str(value.signed), "blue") + + +def EnumSet(value, internal_dict): + type_nim_name = value.type.name.split("_")[2] + # type_nim_name, nti = get_nti(value, type_nim_name) + + val = int(value.signed) + # if nti: + # enum_strings = [] + # i = 0 + # while val > 0: + # if (val & 1) == 1: + # enum_strings.append(reprEnum(i, nti)) + # val = val >> 1 + # i += 1 + + # return '{' + ', '.join(enum_strings) + '}' + return colored(f"{type_nim_name}." + str(val), "blue") + + +def Set(value, internal_dict): + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.value) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Table(value: lldb.SBValue, internal_dict): + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + table_suffix = "Table" + return "{" + ", ".join(fields) + f"}}.{table_suffix}" + + +def HashSet(value: lldb.SBValue, internal_dict): + fields = [] + + for i in range(value.num_children): + fields.append(f"{value[i].summary}") + + table_suffix = "HashSet" + + return "{" + ", ".join(fields) + f"}}.{table_suffix}" + + +def StringTable(value: lldb.SBValue, internal_dict): + table = value.GetNonSyntheticValue() + mode = table["mode"].unsigned + + table_suffix = "StringTable" + + table_mode = "" + if mode == 0: + table_mode = "Case Sensitive" + elif mode == 1: + table_mode = "Case Insensitive" + elif mode == 2: + table_mode = "Style Insensitive" + + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + return "{" + ", ".join(fields) + f"}}.{table_suffix}({table_mode})" + + +def Sequence(value: lldb.SBValue, internal_dict): + value = value.GetNonSyntheticValue() + + data_len = int(value["len"].unsigned) + data = value["p"]["data"] + base_data_type = data.type.GetArrayElementType() + + cast = data.Cast(base_data_type.GetArrayType(data_len)) + + return ( + "@[" + + ", ".join([cast[i].summary or cast[i].type.name for i in range(data_len)]) + + "]" + ) + + +class StringChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + self.count = 0 + + def num_children(self): + return self.count + + def get_child_index(self, name): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data_size + return self.first_element.CreateChildAtOffset( + "[" + str(index) + "]", offset, self.data_type + ) + + def update(self): + data = self.value["p"]["data"] + size = int(self.value["len"].unsigned) + + self.count = size + self.first_element = data + + self.data_type = data.type.GetArrayElementType().GetTypedefedType() + self.data_size = self.data_type.GetByteSize() + + def has_children(self): + return bool(self.num_children()) + + +class ArrayChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + + def num_children(self): + return self.value.num_children + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.value[index].GetByteSize() + return self.first_element.CreateChildAtOffset( + "[" + str(index) + "]", offset, self.data_type + ) + + def update(self): + if not self.has_children(): + return + + self.first_element = self.value[0] + self.data_type = self.value.type.GetArrayElementType() + + def has_children(self): + return bool(self.num_children()) + + +class SeqChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.update() + + def num_children(self): + return int(self.value["len"].unsigned) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data[index].GetByteSize() + return self.first_element.CreateChildAtOffset( + "[" + str(index) + "]", offset, self.data_type + ) + + def update(self): + if not self.has_children(): + return + + data = self.value["p"]["data"] + self.data_type = data.type.GetArrayElementType() + + self.data = data.Cast(self.data_type.GetArrayType(self.num_children())) + self.first_element = self.data + + def has_children(self): + return bool(self.num_children()) + + +class ObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.children) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def populate_children(self): + self.children.clear() + self.child_list = [] + stack = [self.value] + + index = 0 + + while stack: + cur_val = stack.pop() + while cur_val.type.is_pointer: + cur_val = cur_val.Dereference() + + if cur_val.num_children > 0 and cur_val[0].name == "m_type": + if "_" in cur_val.type.name: + tname = cur_val.type.name.split("_")[1].replace( + "colonObjectType", "" + ) + else: + tname = cur_val.type.name + if tname == "TNimTypeV2": + # We've reached the end + break + + if ( + cur_val.num_children > 0 + and cur_val[0].name == "Sup" + and cur_val[0].type.name.startswith("tyObject") + ): + stack.append(cur_val[0]) + + for i in range(cur_val.num_children): + child = cur_val[i] + if child.name == "Sup": + continue + self.children[child.name] = index + self.child_list.append(child) + index += 1 + + def update(self): + self.populate_children() + + def has_children(self): + return bool(self.num_children()) + + +class HashSetChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + data = self.value["data"] + + tuple_len = int(data["len"].unsigned) + tuple = data["p"]["data"] + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el["Field0"].unsigned) + if field0 == 0: + continue + key = el["Field1"] + child = key.CreateValueFromAddress( + f"[{str(index)}]", key.GetLoadAddress(), key.GetType() + ) + index += 1 + + self.child_list.append(child) + + def has_children(self): + return bool(self.num_children()) + + +class SetCharChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType("char") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + cur_pos = 0 + for child in self.value.children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class SetIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NI64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + bits = self.value.GetByteSize() * 8 + + cur_pos = -(bits // 2) + + if self.value.num_children > 0: + children = self.value.children + else: + children = [self.value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class SetUIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + + cur_pos = 0 + if self.value.num_children > 0: + children = self.value.children + else: + children = [self.value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class SetEnumChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + + cur_pos = 0 + if self.value.num_children > 0: + children = self.value.children + else: + children = [self.value] + + for child in children: + child_val = child.unsigned + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data( + f"[{len(self.child_list)}]", data, self.ty + ) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +class TableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + data = self.value["data"] + + tuple_len = int(data["len"].unsigned) + tuple = data["p"]["data"] + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el["Field0"].unsigned) + if field0 == 0: + continue + key = el["Field1"] + val = el["Field2"] + child = val.CreateValueFromAddress( + key.summary, val.GetLoadAddress(), val.GetType() + ) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class StringTableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.children.clear() + self.child_list = [] + data = self.value["data"] + + tuple_len = int(data["len"].unsigned) + tuple = data["p"]["data"] + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el["Field2"].unsigned) + if field0 == 0: + continue + key = el["Field0"] + val = el["Field1"] + child = val.CreateValueFromAddress( + key.summary, val.GetLoadAddress(), val.GetType() + ) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class CustomObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + print("CUSTOMOBJ: ", value.name) + self.value: lldb.SBValue = get_synthetic(value) or value + for child in self.value.children: + print(child) + + def num_children(self): + return self.value.num_children + + def get_child_index(self, name: str): + return self.value.GetIndexOfChildWithName(name) + + def get_child_at_index(self, index): + return self.value.GetChildAtIndex(index) + + def update(self): + pass + + def has_children(self): + return self.num_children() > 0 + + +def echo(debugger: lldb.SBDebugger, command: str, result, internal_dict): + debugger.HandleCommand("po " + command) + + +SUMMARY_FUNCTIONS: dict[str, lldb.SBFunction] = {} +SYNTHETIC_FUNCTIONS: dict[str, lldb.SBFunction] = {} + + +def get_summary(value: lldb.SBValue) -> Union[str, None]: + base_type = get_base_type(value.type) + + fn = SUMMARY_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + res = executeCommand( + f"{fn.name}(*({base_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + ) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_synthetic(value: lldb.SBValue) -> Union[lldb.SBValue, None]: + base_type = get_base_type(value.type) + + fn = SYNTHETIC_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + res = executeCommand( + f"{fn.name}(*({base_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + ) + + if res.error.fail: + return None + + return res + + +def get_base_type(ty: lldb.SBType) -> lldb.SBType: + temp = ty + while temp.IsPointerType(): + temp = temp.GetPointeeType() + return temp + + +def breakpoint_function_wrapper(frame: lldb.SBFrame, bp_loc, internal_dict): + """This allows function calls to Nim for custom object summaries and synthetic children""" + debugger = lldb.debugger + + global SUMMARY_FUNCTIONS + global SYNTHETIC_FUNCTIONS + for tname, fn in SYNTHETIC_FUNCTIONS.items(): + print("DELETING SYNTH: ", tname) + debugger.HandleCommand(f"type synthetic delete -w nim {tname}") + + SUMMARY_FUNCTIONS = {} + SYNTHETIC_FUNCTIONS = {} + + target: lldb.SBTarget = debugger.GetSelectedTarget() + print("BREAKPOINT") + module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + + for sym in module: + if not sym.name.startswith("lldbDebugSummary") and not sym.name.startswith( + "lldbDebugSynthetic" + ): + continue + + print("SYM: ", sym.name) + + fn_syms: lldb.SBSymbolContextList = target.FindFunctions(sym.name) + if not fn_syms.GetSize() > 0: + continue + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + print("fn found!") + + fn: lldb.SBFunction = fn_sym.function + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + + if not arg_types.GetSize() > 0: + continue + arg_type: lldb.SBType = get_base_type(arg_types.GetTypeAtIndex(0)) + + print("FIRST ARG TYPE: ", arg_type.name) + + if sym.name.startswith("lldbDebugSummary"): + SUMMARY_FUNCTIONS[arg_type.name] = fn + elif sym.name.startswith("lldbDebugSynthetic"): + SYNTHETIC_FUNCTIONS[arg_type.name] = fn + debugger.HandleCommand( + f"type synthetic add -w nim -l {__name__}.CustomObjectChildrenProvider -x {arg_type.name}$" + ) + + +def executeCommand(command, *args): + debugger = lldb.debugger + process = debugger.GetSelectedTarget().GetProcess() + frame: lldb.SBFrame = process.GetSelectedThread().GetSelectedFrame() + # module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + # for sym in module: + # print("SYM: ", sym.name) + # target = debugger.GetSelectedTarget() + + expr_options = lldb.SBExpressionOptions() + expr_options.SetIgnoreBreakpoints(False) + expr_options.SetFetchDynamicValue(lldb.eDynamicCanRunTarget) + expr_options.SetTimeoutInMicroSeconds(30 * 1000 * 1000) # 30 second timeout + expr_options.SetTryAllThreads(True) + expr_options.SetUnwindOnError(False) + expr_options.SetGenerateDebugInfo(True) + expr_options.SetLanguage(lldb.eLanguageTypeC) + expr_options.SetCoerceResultToId(True) + res = frame.EvaluateExpression(command, expr_options) + # if res.error.fail: + # print("ERROR: ", res.error.GetError()) + # return str(res.error) + return res + + +def __lldb_init_module(debugger, internal_dict): + # debugger.HandleCommand(f"type summary add -w nim -n any -F {__name__}.CatchAll -x .*") + debugger.HandleCommand(f"type summary add -w nim -n sequence -F {__name__}.Sequence -x tySequence_+[[:alnum:]]+$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SeqChildrenProvider -x tySequence_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n chararray -F {__name__}.CharArray -x char\s+[\d+]") + debugger.HandleCommand(f"type summary add -w nim -n array -F {__name__}.Array -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ArrayChildrenProvider -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n string -F {__name__}.NimStringDesc NimStringDesc") + + debugger.HandleCommand(f"type summary add -w nim -n stringv2 -F {__name__}.NimStringV2 -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringV2$") + + debugger.HandleCommand(f"type summary add -w nim -n cstring -F {__name__}.NCSTRING NCSTRING") + + debugger.HandleCommand(f"type summary add -w nim -n object -F {__name__}.ObjectV2 -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ObjectChildrenProvider -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n tframe -F {__name__}.ObjectV2 -x TFrame$") + + debugger.HandleCommand(f"type summary add -w nim -n rootobj -F {__name__}.ObjectV2 -x RootObj$") + + debugger.HandleCommand(f"type summary add -w nim -n enum -F {__name__}.Enum -x tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n hashset -F {__name__}.HashSet -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.HashSetChildrenProvider -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint -F {__name__}.Set -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetIntChildrenProvider -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setint -F {__name__}.Set -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint2 -F {__name__}.Set -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setenum -F {__name__}.EnumSet -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setchar -F {__name__}.Set -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetCharChildrenProvider -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n table -F {__name__}.Table -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.TableChildrenProvider -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n stringtable -F {__name__}.StringTable -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringTableChildrenProvider -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple2 -F {__name__}.Tuple -x tyObject_+Tuple_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple -F {__name__}.Tuple -x tyTuple_+[[:alnum:]]+") + # debugger.HandleCommand(f"type summary add -w nim -n TNimType -F {__name__}.Object TNimType") + debugger.HandleCommand(f"type summary add -w nim -n TNimTypeV2 -F {__name__}.ObjectV2 TNimTypeV2") + # debugger.HandleCommand(f"type summary add -w nim -n TNimNode -F {__name__}.Object TNimNode") + debugger.HandleCommand(f"type summary add -w nim -n float -F {__name__}.Float NF") + debugger.HandleCommand(f"type summary add -w nim -n float32 -F {__name__}.Float NF32") + debugger.HandleCommand(f"type summary add -w nim -n float64 -F {__name__}.Float NF64") + debugger.HandleCommand(f"type summary add -w nim -n integer -F {__name__}.Number -x NI") + debugger.HandleCommand(f"type summary add -w nim -n integer8 -F {__name__}.Number -x NI8") + debugger.HandleCommand(f"type summary add -w nim -n integer16 -F {__name__}.Number -x NI16") + debugger.HandleCommand(f"type summary add -w nim -n integer32 -F {__name__}.Number -x NI32") + debugger.HandleCommand(f"type summary add -w nim -n integer64 -F {__name__}.Number -x NI64") + debugger.HandleCommand(f"type summary add -w nim -n bool -F {__name__}.Bool -x bool") + debugger.HandleCommand(f"type summary add -w nim -n bool2 -F {__name__}.Bool -x NIM_BOOL") + debugger.HandleCommand(f"type summary add -w nim -n uinteger -F {__name__}.UnsignedNumber -x NU") + debugger.HandleCommand(f"type summary add -w nim -n uinteger8 -F {__name__}.UnsignedNumber -x NU8") + debugger.HandleCommand(f"type summary add -w nim -n uinteger16 -F {__name__}.UnsignedNumber -x NU16") + debugger.HandleCommand(f"type summary add -w nim -n uinteger32 -F {__name__}.UnsignedNumber -x NU32") + debugger.HandleCommand(f"type summary add -w nim -n uinteger64 -F {__name__}.UnsignedNumber -x NU64") + debugger.HandleCommand("type category enable nim") + debugger.HandleCommand(f"command script add -f {__name__}.echo echo") + debugger.HandleCommand(f"command script add -f {__name__}.handle_command ddp") + debugger.HandleCommand(f"breakpoint command add -F {__name__}.breakpoint_function_wrapper --script-type python 1") \ No newline at end of file From 6994e1b1d7962f9c61887c7e606aeccbebf9bef6 Mon Sep 17 00:00:00 2001 From: quantimnot <54247259+quantimnot@users.noreply.github.com> Date: Sat, 4 Mar 2023 01:57:30 -0500 Subject: [PATCH 1636/1654] `--embedsrc` for JavaScript (#21467) Co-authored-by: quantimnot --- compiler/jsgen.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 5df40b996..d0a143793 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -735,6 +735,8 @@ proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) if line < 0: return + if optEmbedOrigSrc in p.config.globalOptions: + lineF(p, "//$1$n", [sourceLine(p.config, n.info)]) if optLineDir in p.options or optLineDir in p.config.options: lineF(p, "$1", [lineDir(p.config, n.info, line)]) if hasFrameInfo(p): From d950e5f3a8efe0c3be8e773cce4a89807833ffea Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 4 Mar 2023 20:07:33 +0800 Subject: [PATCH 1637/1654] remove nosinks hacks from compiler (#21469) --- compiler/cgen.nim | 2 +- compiler/depends.nim | 2 +- compiler/passes.nim | 4 ++-- compiler/sem.nim | 4 ++-- compiler/semcall.nim | 2 +- compiler/seminst.nim | 2 +- compiler/semstmts.nim | 2 +- compiler/sigmatch.nim | 2 +- compiler/vm.nim | 6 +++--- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index d051934b5..e24a2cb1e 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1940,7 +1940,7 @@ template injectG() {.dirty.} = graph.backend = newModuleList(graph) let g = BModuleList(graph.backend) -proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = injectG() result = newModule(g, module, graph.config) result.idgen = idgen diff --git a/compiler/depends.nim b/compiler/depends.nim index 2087198f2..823e0f970 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -100,7 +100,7 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) = rope(project.splitFile.name), b.dotGraph], changeFileExt(project, "dot")) -proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupDependPass*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = var g: PGen new(g) g.module = module diff --git a/compiler/passes.nim b/compiler/passes.nim index 38c133d69..536a64714 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -241,14 +241,14 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = graph.compileSystemModule() discard graph.compileModule(projectFile, {sfMainModule}) -proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc mySemOpen(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = result = preparePContext(graph, module, idgen) proc mySemClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = var c = PContext(context) closePContext(graph, c, n) -proc mySemProcess(context: PPassContext, n: PNode): PNode {.nosinks.} = +proc mySemProcess(context: PPassContext, n: PNode): PNode = result = semWithPContext(PContext(context), n) const semPass* = makePass(mySemOpen, mySemProcess, mySemClose, diff --git a/compiler/sem.nim b/compiler/sem.nim index a5d06f3fe..48a7d56c8 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -666,7 +666,7 @@ proc addCodeForGenerics(c: PContext, n: PNode) = n.add prc.ast c.lastGenericIdx = c.generics.len -proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext {.nosinks.} = +proc preparePContext*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PContext = result = newContext(graph, module) result.idgen = idgen result.enforceVoidContext = newType(tyTyped, nextTypeId(idgen), nil) @@ -770,7 +770,7 @@ proc recoverContext(c: PContext) = while getCurrOwner(c).kind != skModule: popOwner(c) while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next -proc semWithPContext*(c: PContext, n: PNode): PNode {.nosinks.} = +proc semWithPContext*(c: PContext, n: PNode): PNode = # no need for an expensive 'try' if we stop after the first error anyway: if c.config.errorMax <= 1: result = semStmtAndGenerateGenerics(c, n) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 2147a9645..ba7908aa5 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -601,7 +601,7 @@ proc tryDeref(n: PNode): PNode = result.add n proc semOverloadedCall(c: PContext, n, nOrig: PNode, - filter: TSymKinds, flags: TExprFlags): PNode {.nosinks.} = + filter: TSymKinds, flags: TExprFlags): PNode = var errors: CandidateErrors = @[] # if efExplain in flags: @[] else: nil var r = resolveOverloads(c, n, nOrig, filter, flags, errors, efExplain in flags) if r.state == csMatch: diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 80d455d1e..b5fe244b0 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -314,7 +314,7 @@ proc fillMixinScope(c: PContext) = p = p.next proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, - info: TLineInfo): PSym {.nosinks.} = + info: TLineInfo): PSym = ## Generates a new instance of a generic procedure. ## The `pt` parameter is a type-unsafe mapping table used to link generic ## parameters to their concrete types within the generic instance. diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 6237e6eb0..6d1ce388d 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1692,7 +1692,7 @@ proc semProcAnnotation(c: PContext, prc: PNode; return result -proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode {.nosinks.} = +proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = ## used for resolving 'auto' in lambdas based on their callsite var n = n let original = n[namePos].sym diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index f8c7fa870..87a4fdf1c 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -2674,7 +2674,7 @@ proc argtypeMatches*(c: PContext, f, a: PType, fromHlo = false): bool = proc instTypeBoundOp*(c: PContext; dc: PSym; t: PType; info: TLineInfo; - op: TTypeAttachedOp; col: int): PSym {.nosinks.} = + op: TTypeAttachedOp; col: int): PSym = var m = newCandidate(c, dc.typ) if col >= dc.typ.len: localError(c.config, info, "cannot instantiate: '" & dc.name.s & "'") diff --git a/compiler/vm.nim b/compiler/vm.nim index 96f05c0f9..e00f0f02e 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1893,7 +1893,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, regs[rc].node.strVal, 0, - proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = + proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) if error.len > 0: @@ -1908,7 +1908,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = var error: string let ast = parseString(regs[rb].node.strVal, c.cache, c.config, regs[rc].node.strVal, 0, - proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) {.nosinks.} = + proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) = if error.len == 0 and msg <= errMax: error = formatMsg(conf, info, msg, arg)) if error.len > 0: @@ -2309,7 +2309,7 @@ proc setupGlobalCtx*(module: PSym; graph: ModuleGraph; idgen: IdGenerator) = else: refresh(PCtx graph.vm, module, idgen) -proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext {.nosinks.} = +proc setupEvalGen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = #var c = newEvalContext(module, emRepl) #c.features = {allowCast, allowInfiniteLoops} #pushStackFrame(c, newStackFrame()) From 04a494f8cfe960104150dd288d587d7e2cf3bd8a Mon Sep 17 00:00:00 2001 From: Sultan Al Isaiee Date: Sat, 4 Mar 2023 18:53:57 +0400 Subject: [PATCH 1638/1654] Add warning to specify timeout value in milliseconds, Fix #21449 (#21471) a warning message been added to the documentation to remind users that the timeout parameter is expressed in milliseconds, not seconds. to help prevent confusion and unexpected behaviours. --- lib/pure/osproc.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 4cdd98c0d..915337f12 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -220,6 +220,10 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl, ## ## On posix, if the process has exited because of a signal, 128 + signal ## number will be returned. + ## + ## .. warning:: When working with `timeout` parameters, remember that the value is + ## typically expressed in milliseconds, and ensure that the correct unit of time + ## is used to avoid unexpected behavior. proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", raises: [OSError], tags: [].} ## Return `-1` if the process is still running. Otherwise the process' exit code. From 7bde421e4dae1f078ad5940dda8199df00b725cc Mon Sep 17 00:00:00 2001 From: SirOlaf <34164198+SirOlaf@users.noreply.github.com> Date: Sun, 5 Mar 2023 11:56:51 +0100 Subject: [PATCH 1639/1654] Fix #21272: Rewrite parts of pickBestCandidate (#21465) * Make pickBestCandidate store syms * Remove useless cursor * Try making pickBestCandidate more readable * Fix advance order * Revert back to seq with lots of comments --------- Co-authored-by: SirOlaf <> --- compiler/semcall.nim | 86 +++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index ba7908aa5..54f03026f 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -43,6 +43,7 @@ proc initCandidateSymbols(c: PContext, headSymbol: PNode, best, alt: var TCandidate, o: var TOverloadIter, diagnostics: bool): seq[tuple[s: PSym, scope: int]] = + ## puts all overloads into a seq and prepares best+alt result = @[] var symx = initOverloadIter(o, c, headSymbol) while symx != nil: @@ -64,36 +65,35 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, errors: var CandidateErrors, diagnosticsFlag: bool, errorsEnabled: bool, flags: TExprFlags) = + # `matches` may find new symbols, so keep track of count + var symCount = c.currentScope.symbols.counter + var o: TOverloadIter - var sym = initOverloadIter(o, c, headSymbol) - var scope = o.lastOverloadScope - # Thanks to the lazy semchecking for operands, we need to check whether - # 'initCandidate' modifies the symbol table (via semExpr). - # This can occur in cases like 'init(a, 1, (var b = new(Type2); b))' - let counterInitial = c.currentScope.symbols.counter - var syms: seq[tuple[s: PSym, scope: int]] - var noSyms = true - var nextSymIndex = 0 - while sym != nil: - if sym.kind in filter: - # Initialise 'best' and 'alt' with the first available symbol - initCandidate(c, best, sym, initialBinding, scope, diagnosticsFlag) - initCandidate(c, alt, sym, initialBinding, scope, diagnosticsFlag) - best.state = csNoMatch - break - else: - sym = nextOverloadIter(o, c, headSymbol) - scope = o.lastOverloadScope - var z: TCandidate - while sym != nil: - if sym.kind notin filter: - sym = nextOverloadIter(o, c, headSymbol) - scope = o.lastOverloadScope - continue + # https://github.com/nim-lang/Nim/issues/21272 + # prevent mutation during iteration by storing them in a seq + # luckily `initCandidateSymbols` does just that + var syms = initCandidateSymbols(c, headSymbol, initialBinding, filter, + best, alt, o, diagnosticsFlag) + if len(syms) == 0: + return + # current overload being considered + var sym = syms[0].s + var scope = syms[0].scope + + # starts at 1 because 0 is already done with setup, only needs checking + var nextSymIndex = 1 + var z: TCandidate # current candidate + while true: determineType(c, sym) initCandidate(c, z, sym, initialBinding, scope, diagnosticsFlag) - if c.currentScope.symbols.counter == counterInitial or syms.len != 0: + + # this is kinda backwards as without a check here the described + # problems in recalc would not happen, but instead it 100% + # does check forever in some cases + if c.currentScope.symbols.counter == symCount: + # may introduce new symbols with caveats described in recalc branch matches(c, n, orig, z) + if z.state == csMatch: # little hack so that iterators are preferred over everything else: if sym.kind == skIterator: @@ -113,22 +113,36 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, firstMismatch: z.firstMismatch, diagnostics: z.diagnostics)) else: + # this branch feels like a ticking timebomb + # one of two bad things could happen + # 1) new symbols are discovered but the loop ends before we recalc + # 2) new symbols are discovered and resemmed forever + # not 100% sure if these are possible though as they would rely + # on somehow introducing a new overload during overload resolution + # Symbol table has been modified. Restart and pre-calculate all syms # before any further candidate init and compare. SLOW, but rare case. syms = initCandidateSymbols(c, headSymbol, initialBinding, filter, best, alt, o, diagnosticsFlag) - noSyms = false - if noSyms: - sym = nextOverloadIter(o, c, headSymbol) - scope = o.lastOverloadScope - elif nextSymIndex < syms.len: - # rare case: retrieve the next pre-calculated symbol - sym = syms[nextSymIndex].s - scope = syms[nextSymIndex].scope - nextSymIndex += 1 - else: + + # reset counter because syms may be in a new order + symCount = c.currentScope.symbols.counter + nextSymIndex = 0 + + # just in case, should be impossible though + if syms.len == 0: + break + + if nextSymIndex > high(syms): + # we have reached the end break + # advance to next sym + sym = syms[nextSymIndex].s + scope = syms[nextSymIndex].scope + inc(nextSymIndex) + + proc effectProblem(f, a: PType; result: var string; c: PContext) = if f.kind == tyProc and a.kind == tyProc: if tfThread in f.flags and tfThread notin a.flags: From 05e8a60bb4fe40d450aaec4603209e310ea490a4 Mon Sep 17 00:00:00 2001 From: Joey Date: Sun, 5 Mar 2023 12:14:59 -0700 Subject: [PATCH 1640/1654] Add line directives for C code variables (#21466) * Add line directives for C code variables * Refactor genCLineDir to only use toFullPath if necessary --- compiler/cgen.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index e24a2cb1e..d2095016c 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -273,7 +273,8 @@ proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) = [rope(makeSingleLineCString(filename)), rope(line)]) proc genCLineDir(r: var Rope, info: TLineInfo; conf: ConfigRef) = - genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf) + if optLineDir in conf.options: + genCLineDir(r, toFullPath(conf, info), info.safeLineNm, conf) proc freshLineInfo(p: BProc; info: TLineInfo): bool = if p.lastLineInfo.line != info.line or @@ -287,7 +288,7 @@ proc genLineDir(p: BProc, t: PNode) = if optEmbedOrigSrc in p.config.globalOptions: p.s(cpsStmts).add("//" & sourceLine(p.config, t.info) & "\L") - genCLineDir(p.s(cpsStmts), toFullPath(p.config, t.info), line, p.config) + genCLineDir(p.s(cpsStmts), t.info, p.config) if ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and (p.prc == nil or sfPure notin p.prc.flags) and t.info.fileIndex != InvalidFileIdx: if freshLineInfo(p, t.info): @@ -566,6 +567,9 @@ proc localVarDecl(p: BProc; n: PNode): Rope = if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy) if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0: result.addf("NIM_ALIGN($1) ", [rope(s.alignment)]) + + genCLineDir(result, n.info, p.config) + result.add getTypeDesc(p.module, s.typ, skVar) if s.constraint.isNil: if sfRegister in s.flags: result.add(" register") From 4d76725299e524e19d515c04d923bd408607a966 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 6 Mar 2023 14:12:47 +0800 Subject: [PATCH 1641/1654] closes #16654; add a test case (#21478) --- tests/errmsgs/t16654.nim | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/errmsgs/t16654.nim diff --git a/tests/errmsgs/t16654.nim b/tests/errmsgs/t16654.nim new file mode 100644 index 000000000..749707c06 --- /dev/null +++ b/tests/errmsgs/t16654.nim @@ -0,0 +1,12 @@ +discard """ + cmd: "nim check $options $file" + errormsg: "type mismatch: got but expected 'float'" +""" + +when true: # bug #16654 + func fn[T](a: T, op: proc(a: T): float) = discard + proc main() = + let v = 1 + proc bar(r: auto): auto = v + fn(1, bar) + main() From 2d9af2bd559d03c4f6b2a5f89be316f2465a091b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:34:41 +0800 Subject: [PATCH 1642/1654] closes #20704; add a test case (#21480) --- tests/stdlib/tsugar.nim | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/stdlib/tsugar.nim b/tests/stdlib/tsugar.nim index 6ef3ae519..1b629165a 100644 --- a/tests/stdlib/tsugar.nim +++ b/tests/stdlib/tsugar.nim @@ -3,7 +3,7 @@ discard """ x + y = 30 ''' """ -import std/[sugar, algorithm, random, sets, tables, strutils] +import std/[sugar, algorithm, random, sets, tables, strutils, sequtils] import std/[syncio, assertions] type # for capture test, ref #20679 @@ -287,6 +287,17 @@ proc mainProc() = doAssertRaises(AssertionDefect): doAssert false doAssert "except AssertionDefect" in s2 + block: # bug #20704 + proc test() = + var xs, ys: seq[int] + for i in 0..5: + xs.add(i) + + xs.apply(d => ys.add(d)) + doAssert ys == @[0, 1, 2, 3, 4, 5] + + test() + static: main() mainProc() From 64a0355f3f95c9ec8ca3597f8028819b489f08c8 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 6 Mar 2023 22:30:22 +0800 Subject: [PATCH 1643/1654] fixes #21377; fixes `@[]` and `{}` type inference as returns in generics (#21475) * fixes `@[]` type inference in generics * add issue links * fixes macros and iterators * refactor * add one more test --- compiler/semexprs.nim | 2 +- compiler/seminst.nim | 11 +++++++-- compiler/semmagic.nim | 6 ++++- tests/types/ttopdowninference.nim | 37 ++++++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 9cf206b4c..976d7c757 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -998,7 +998,7 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy fixAbstractType(c, result) analyseIfAddressTakenInCall(c, result) if callee.magic != mNone: - result = magicsAfterOverloadResolution(c, result, flags) + result = magicsAfterOverloadResolution(c, result, flags, expectedType) when false: if result.typ != nil and not (result.typ.kind == tySequence and result.typ[0].kind == tyEmpty): diff --git a/compiler/seminst.nim b/compiler/seminst.nim index b5fe244b0..25e5b267e 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -127,11 +127,18 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = if sfGenSym in param.flags: idTablePut(symMap, params[i].sym, result.typ.n[param.position+1].sym) freshGenSyms(c, b, result, orig, symMap) - + if sfBorrow notin orig.flags: # We do not want to generate a body for generic borrowed procs. # As body is a sym to the borrowed proc. - b = semProcBody(c, b) + let resultType = # todo probably refactor it into a function + if result.kind == skMacro: + sysTypeFromName(c.graph, n.info, "NimNode") + elif not isInlineIterator(result.typ): + result.typ[0] + else: + nil + b = semProcBody(c, b, resultType) result.ast[bodyPos] = hloBody(c, b) excl(result.flags, sfForward) trackProc(c, result, result.ast[bodyPos]) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 751ca8fe7..b4c6cd275 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -526,7 +526,7 @@ proc checkDefault(c: PContext, n: PNode): PNode = message(c.config, n.info, warnUnsafeDefault, typeToString(constructed)) proc magicsAfterOverloadResolution(c: PContext, n: PNode, - flags: TExprFlags): PNode = + flags: TExprFlags; expectedType: PType = nil): PNode = ## This is the preferred code point to implement magics. ## ``c`` the current module, a symbol table to a very good approximation ## ``n`` the ast like it would be passed to a real macro @@ -635,5 +635,9 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n of mPrivateAccess: result = semPrivateAccess(c, n) + of mArrToSeq: + result = n + if result.typ != nil and expectedType != nil and result.typ.kind == tySequence and expectedType.kind == tySequence and result.typ[0].kind == tyEmpty: + result.typ = expectedType # type inference for empty sequence # bug #21377 else: result = n diff --git a/tests/types/ttopdowninference.nim b/tests/types/ttopdowninference.nim index 310dbb2ad..cabc798ed 100644 --- a/tests/types/ttopdowninference.nim +++ b/tests/types/ttopdowninference.nim @@ -245,7 +245,7 @@ block: # bug #11777 var s: S = {1, 2} doAssert 1 in s -block: # regression #20807 +block: # bug #20807 var s: seq[string] template fail = s = @[] @@ -255,3 +255,38 @@ block: # regression #20807 test: fail() doAssert not (compiles do: let x: seq[int] = `@`[string]([])) + +block: # bug #21377 + proc b[T](v: T): seq[int] = + let x = 0 + @[] + + doAssert b(0) == @[] + +block: # bug #21377 + proc b[T](v: T): seq[T] = + let x = 0 + @[] + + doAssert b(0) == @[] + +block: # bug #21377 + proc b[T](v: T): set[bool] = + let x = 0 + {} + + doAssert b(0) == {} + +block: # bug #21377 + proc b[T](v: T): array[0, int] = + let x = 0 + [] + + doAssert b(0) == [] + +block: # bug #21377 + proc b[T](v: T): array[0, (string, string)] = + let x = 0 + {:} + + doAssert b(0) == {:} From 25eef64fe1969581436696556e0bf10ef25aa3ec Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Tue, 7 Mar 2023 01:31:53 +1100 Subject: [PATCH 1644/1654] Remove Defect from raises list in `std/times` (#21473) * Remove Defect from raises list Since defects aren't tracked anymore this causes a hint to pop up mentioning it * Still track Defect when getting ran with an older Nim version The raises followed a pattern so moving them into a pragma didn't seem to cause any extra problems --- lib/pure/times.nim | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 70af466d9..138f2d9ec 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -408,6 +408,16 @@ const unitWeights: array[FixedTimeUnit, int64] = [ 7 * secondsInDay * 1e9.int64, ] +when (NimMajor, NimMinor) >= (1, 4): + # Newer versions of Nim don't track defects + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError].} + {.pragma: parseRaises, raises: [TimeParseError].} +else: + # Still track when using older versions + {.pragma: parseFormatRaises, raises: [TimeParseError, TimeFormatParseError, Defect].} + {.pragma: parseRaises, raises: [TimeParseError, Defect].} + + # # Helper procs # @@ -2134,8 +2144,7 @@ proc format*(time: Time, f: static[string], zone: Timezone = local()): string result = time.inZone(zone).format(f2) proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Parses `input` as a `DateTime` using the format specified by `f`. ## If no UTC offset was parsed, then `input` is assumed to be specified in ## the `zone` timezone. If a UTC offset was parsed, the result will be @@ -2178,8 +2187,7 @@ proc parse*(input: string, f: TimeFormat, zone: Timezone = local(), result = toDateTime(parsed, zone, f, input) proc parse*(input, f: string, tz: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): DateTime - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`. ## @@ -2192,14 +2200,12 @@ proc parse*(input, f: string, tz: Timezone = local(), result = input.parse(dtFormat, tz, loc = loc) proc parse*(input: string, f: static[string], zone: Timezone = local(), - loc: DateTimeLocale = DefaultLocale): - DateTime {.raises: [TimeParseError, Defect].} = + loc: DateTimeLocale = DefaultLocale): DateTime {.parseRaises.} = ## Overload that validates `f` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone, loc = loc) -proc parseTime*(input, f: string, zone: Timezone): Time - {.raises: [TimeParseError, TimeFormatParseError, Defect].} = +proc parseTime*(input, f: string, zone: Timezone): Time {.parseFormatRaises.} = ## Shorthand for constructing a `TimeFormat` and using it to parse ## `input` as a `DateTime`, then converting it a `Time`. ## @@ -2211,7 +2217,7 @@ proc parseTime*(input, f: string, zone: Timezone): Time parse(input, f, zone).toTime() proc parseTime*(input: string, f: static[string], zone: Timezone): Time - {.raises: [TimeParseError, Defect].} = + {.parseRaises.} = ## Overload that validates `format` at compile time. const f2 = initTimeFormat(f) result = input.parse(f2, zone).toTime() From 6ef94301538f0a17461630c08c456112954e66bb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 7 Mar 2023 17:34:02 +0800 Subject: [PATCH 1645/1654] closes #6231; add a test case (#21485) --- tests/generics/tgenerics_various.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/generics/tgenerics_various.nim b/tests/generics/tgenerics_various.nim index 37b6e43a1..4cfa47809 100644 --- a/tests/generics/tgenerics_various.nim +++ b/tests/generics/tgenerics_various.nim @@ -262,3 +262,8 @@ proc doSomething[A, B](t: tuple[a: A, b: B]) = discard discard identity((c: 1, d: 2)) doSomething(identity((1, 2))) + +# bug #6231 +proc myProc[T, U](x: T or U) = discard + +myProc[int, string](x = 2) From 2f89f1eb780ab32be31dbe4074af70b587ad2f36 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 7 Mar 2023 20:17:05 +0800 Subject: [PATCH 1646/1654] closes #8295; add a test case (#21486) --- tests/generics/tgeneric0.nim | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/generics/tgeneric0.nim b/tests/generics/tgeneric0.nim index 44c34917d..b5e1c4bb4 100644 --- a/tests/generics/tgeneric0.nim +++ b/tests/generics/tgeneric0.nim @@ -153,3 +153,16 @@ proc unzip*[T,U](xs: List[tuple[t: T, u: U]]): (List[T], List[U]) = discard proc unzip2*[T,U](xs: List[(T,U)]): (List[T], List[U]) = discard +type + AtomicType = pointer|ptr|int + + Atomic[T: AtomicType] = distinct T + + Block[T: AtomicType] = object + + AtomicContainer[T: AtomicType] = object + b: Atomic[ptr Block[T]] + +# bug #8295 +var x = AtomicContainer[int]() +doAssert (ptr Block[int])(x.b) == nil From 8e9fc5e63968177c75d93af08d32717bf6c97693 Mon Sep 17 00:00:00 2001 From: ghost <97684920+sslime336@users.noreply.github.com> Date: Thu, 9 Mar 2023 05:34:59 +0800 Subject: [PATCH 1647/1654] a better message if graphviz's dot/nodejs is not found in PATH (#21488) * finish issue #21474: a better message if dot is not found locally when using gendepend * fix a typo in compiler * trim empty path reported in `findNodeJs` * compiler/main.nim: switch raise to simply quit --- compiler/main.nim | 7 +++++++ compiler/nodejs.nim | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compiler/main.nim b/compiler/main.nim index d37d26478..ff870a14a 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -71,6 +71,13 @@ proc commandGenDepend(graph: ModuleGraph) = let project = graph.config.projectFull writeDepsFile(graph) generateDot(graph, project) + + # dot in graphivz tool kit is required + let graphvizDotPath = findExe("dot") + if graphvizDotPath.len == 0: + quit("gendepend: Graphviz's tool dot is required," & + "see https://graphviz.org/download for downloading") + execExternalProgram(graph.config, "dot -Tpng -o" & changeFileExt(project, "png").string & ' ' & changeFileExt(project, "dot").string) diff --git a/compiler/nodejs.nim b/compiler/nodejs.nim index 283643e8d..c1feb196a 100644 --- a/compiler/nodejs.nim +++ b/compiler/nodejs.nim @@ -7,4 +7,4 @@ proc findNodeJs*(): string {.inline.} = result = findExe("node") if result.len == 0: echo "Please install NodeJS first, see https://nodejs.org/en/download" - raise newException(IOError, "NodeJS not found in PATH: " & result) + raise newException(IOError, "NodeJS not found in PATH") From 72e262666bdf2bb3c239183dd32b48bb05d113aa Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:09:08 +0800 Subject: [PATCH 1648/1654] fixes quoted variables with typedesc types (#21493) --- compiler/semexprs.nim | 2 +- tests/stdlib/tmacros.nim | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 976d7c757..cfa34fcdc 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2206,7 +2206,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode = dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids[0], getSysSym(c.graph, n.info, "typed").newSymNode, c.graph.emptyNode) for i in 1.. Date: Fri, 10 Mar 2023 16:28:51 +0800 Subject: [PATCH 1649/1654] fixes #21023; Segfault when mixing seqs, orc, variants and futures (#21497) * fixes #21023; Segfault when mixing seqs, orc, variants and futures * fixes none of the branches were explicitly selected * add one more test * one more test --- compiler/semobjconstr.nim | 61 +++++++++++++++++-------- tests/arc/tcaseobj.nim | 61 +++++++++++++++++++++++++ tests/objects/tobject_default_value.nim | 32 +++++++++++++ 3 files changed, 136 insertions(+), 18 deletions(-) diff --git a/compiler/semobjconstr.nim b/compiler/semobjconstr.nim index 15e53a639..29067cfa4 100644 --- a/compiler/semobjconstr.nim +++ b/compiler/semobjconstr.nim @@ -142,15 +142,45 @@ proc fieldsPresentInInitExpr(c: PContext, fieldsRecList, initExpr: PNode): strin if result.len != 0: result.add ", " result.add field.sym.name.s.quoteStr +proc locateFieldInDefaults(sym: PSym, defaults: seq[PNode]): bool = + result = false + for d in defaults: + if sym.id == d[0].sym.id: + return true + proc collectMissingFields(c: PContext, fieldsRecList: PNode, - constrCtx: var ObjConstrContext) = - for r in directFieldsInRecList(fieldsRecList): - if constrCtx.needsFullInit or - sfRequiresInit in r.sym.flags or - r.sym.typ.requiresInit: + constrCtx: var ObjConstrContext, defaults: seq[PNode] + ): seq[PSym] = + for r in directFieldsInRecList(fieldsRecList): let assignment = locateFieldInInitExpr(c, r.sym, constrCtx.initExpr) - if assignment == nil: - constrCtx.missingFields.add r.sym + if assignment == nil and not locateFieldInDefaults(r.sym, defaults): + if constrCtx.needsFullInit or + sfRequiresInit in r.sym.flags or + r.sym.typ.requiresInit: + constrCtx.missingFields.add r.sym + else: + result.add r.sym + +proc collectMissingCaseFields(c: PContext, branchNode: PNode, + constrCtx: var ObjConstrContext, defaults: seq[PNode]): seq[PSym] = + if branchNode != nil: + let fieldsRecList = branchNode[^1] + result = collectMissingFields(c, fieldsRecList, constrCtx, defaults) + +proc collectOrAddMissingCaseFields(c: PContext, branchNode: PNode, + constrCtx: var ObjConstrContext, defaults: var seq[PNode]) = + let res = collectMissingCaseFields(c, branchNode, constrCtx, defaults) + for sym in res: + let asgnType = newType(tyTypeDesc, nextTypeId(c.idgen), sym.typ.owner) + let recTyp = sym.typ.skipTypes(defaultFieldsSkipTypes) + rawAddSon(asgnType, recTyp) + let asgnExpr = newTree(nkCall, + newSymNode(getSysMagic(c.graph, constrCtx.initExpr.info, "zeroDefault", mZeroDefault)), + newNodeIT(nkType, constrCtx.initExpr.info, asgnType) + ) + asgnExpr.flags.incl nfUseDefaultField + asgnExpr.typ = recTyp + defaults.add newTree(nkExprColonExpr, newSymNode(sym), asgnExpr) proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, flags: TExprFlags): tuple[status: InitStatus, defaults: seq[PNode]] = @@ -166,11 +196,6 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, let fields = branch[^1] fieldsPresentInInitExpr(c, fields, constrCtx.initExpr) - template collectMissingFields(branchNode: PNode) = - if branchNode != nil: - let fields = branchNode[^1] - collectMissingFields(c, fields, constrCtx) - let discriminator = n[0] internalAssert c.config, discriminator.kind == nkSym var selectedBranch = -1 @@ -288,8 +313,7 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, # When a branch is selected with a partial match, some of the fields # that were not initialized may be mandatory. We must check for this: if result.status == initPartial: - collectMissingFields branchNode - + collectOrAddMissingCaseFields(c, branchNode, constrCtx, result.defaults) else: result.status = initNone let discriminatorVal = semConstrField(c, flags + {efPreferStatic}, @@ -302,7 +326,7 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, # a result: let defaultValue = newIntLit(c.graph, constrCtx.initExpr.info, 0) let matchedBranch = n.pickCaseBranch defaultValue - collectMissingFields matchedBranch + discard collectMissingCaseFields(c, matchedBranch, constrCtx, @[]) else: result.status = initPartial if discriminatorVal.kind == nkIntLit: @@ -312,11 +336,12 @@ proc semConstructFields(c: PContext, n: PNode, constrCtx: var ObjConstrContext, if matchedBranch != nil: let (_, defaults) = semConstructFields(c, matchedBranch[^1], constrCtx, flags) result.defaults.add defaults - collectMissingFields matchedBranch + collectOrAddMissingCaseFields(c, matchedBranch, constrCtx, result.defaults) else: # All bets are off. If any of the branches has a mandatory # fields we must produce an error: - for i in 1.. Date: Fri, 10 Mar 2023 21:19:31 +0800 Subject: [PATCH 1650/1654] fixes #21306; fixes #20485; don't transform yields in the var section when introducing new local vars [backport: 1.6] (#21489) * fixes #21306; don't transform yields in the var section when introducing new local vars * adds `inVarSection` so the var section in the var section is freshed * use `isIntroducingNewLocalVars` to avoid yield transformations in var sections * fixes comments --- compiler/transf.nim | 5 +- tests/iter/t21306.nim | 114 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 tests/iter/t21306.nim diff --git a/compiler/transf.nim b/compiler/transf.nim index 7277e6898..445bc1df1 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -50,6 +50,7 @@ type module: PSym transCon: PTransCon # top of a TransCon stack inlining: int # > 0 if we are in inlining context (copy vars) + isIntroducingNewLocalVars: bool # true if we are in `introducingNewLocalVars` (don't transform yields) contSyms, breakSyms: seq[PSym] # to transform 'continue' and 'break' deferDetected, tooEarly: bool graph: ModuleGraph @@ -450,7 +451,9 @@ proc transformYield(c: PTransf, n: PNode): PNode = result.add(c.transCon.forLoopBody) else: # we need to introduce new local variables: + c.isIntroducingNewLocalVars = true # don't transform yields when introducing new local vars result.add(introduceNewLocalVars(c, c.transCon.forLoopBody)) + c.isIntroducingNewLocalVars = false for idx in 0 ..< result.len: var changeNode = result[idx] @@ -1036,7 +1039,7 @@ proc transform(c: PTransf, n: PNode): PNode = else: result = transformSons(c, n) of nkYieldStmt: - if c.inlining > 0: + if c.inlining > 0 and not c.isIntroducingNewLocalVars: result = transformYield(c, n) else: result = transformSons(c, n) diff --git a/tests/iter/t21306.nim b/tests/iter/t21306.nim new file mode 100644 index 000000000..43fea9c80 --- /dev/null +++ b/tests/iter/t21306.nim @@ -0,0 +1,114 @@ +# bug #21306 +type + FutureState {.pure.} = enum + Pending, Finished, Cancelled, Failed + + FutureBase = ref object of RootObj + state: FutureState + error: ref CatchableError + id: uint + + Future[T] = ref object of FutureBase + closure: iterator(f: Future[T]): FutureBase {.raises: [Defect, CatchableError, Exception], gcsafe.} + value: T + +template setupFutureBase() = + new(result) + result.state = FutureState.Pending + +proc newFutureImpl[T](): Future[T] = + setupFutureBase() + +template newFuture[T](fromProc: static[string] = ""): Future[T] = + newFutureImpl[T]() + +proc internalRead[T](fut: Future[T]): T = + when T isnot void: + return fut.value + +template await[T](f: Future[T]): untyped = + when declared(chronosInternalRetFuture): + when not declaredInScope(chronosInternalTmpFuture): + var chronosInternalTmpFuture {.inject.}: FutureBase = f + else: + chronosInternalTmpFuture = f + + yield chronosInternalTmpFuture + + when T isnot void: + cast[type(f)](chronosInternalTmpFuture).internalRead() + +type + VerifierError {.pure.} = enum + Invalid + MissingParent + UnviableFork + Duplicate + ProcessingCallback = proc() {.gcsafe, raises: [Defect].} + BlockVerifier = + proc(signedBlock: int): + Future[VerifierError] {.gcsafe, raises: [Defect].} + + SyncQueueKind {.pure.} = enum + Forward, Backward + + SyncRequest[T] = object + kind: SyncQueueKind + index: uint64 + slot: uint64 + count: uint64 + item: T + + SyncResult[T] = object + request: SyncRequest[T] + data: seq[ref int] + + SyncQueue[T] = ref object + kind: SyncQueueKind + readyQueue: seq[SyncResult[T]] + blockVerifier: BlockVerifier + +iterator blocks[T](sq: SyncQueue[T], + sr: SyncResult[T]): ref int = + case sq.kind + of SyncQueueKind.Forward: + for i in countup(0, len(sr.data) - 1): + yield sr.data[i] + of SyncQueueKind.Backward: + for i in countdown(len(sr.data) - 1, 0): + yield sr.data[i] + +proc push[T](sq: SyncQueue[T]; sr: SyncRequest[T]; data: seq[ref int]; + processingCb: ProcessingCallback = nil): Future[void] {. + stackTrace: off, gcsafe.} = + iterator push_436208182(chronosInternalRetFuture: Future[void]): FutureBase {. + closure, gcsafe, raises: [Defect, CatchableError, Exception].} = + block: + template result(): auto {.used.} = + {.fatal: "You should not reference the `result` variable inside" & + " a void async proc".} + + let item = default(SyncResult[T]) + for blk in sq.blocks(item): + let res = await sq.blockVerifier(blk[]) + + var resultFuture = newFuture[void]("push") + resultFuture.closure = push_436208182 + return resultFuture + +type + SomeTPeer = ref object + score: int + +proc getSlice(): seq[ref int] = + discard + +template smokeTest(kkind: SyncQueueKind, start, finish: uint64, + chunkSize: uint64) = + var queue: SyncQueue[SomeTPeer] + var request: SyncRequest[SomeTPeer] + discard queue.push(request, getSlice()) + +for k in {SyncQueueKind.Forward}: + for item in [(uint64(1181), uint64(1399), 41'u64)]: + smokeTest(k, item[0], item[1], item[2]) \ No newline at end of file From 46d2161c23c2aa1905571512b9a1ef7d61ae670e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 10 Mar 2023 14:20:30 +0100 Subject: [PATCH 1651/1654] minor refactoring (#21499) --- compiler/liftdestructors.nim | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index d7b1f2daa..467f015d8 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -1085,13 +1085,7 @@ proc inst(g: ModuleGraph; c: PContext; t: PType; kind: TTypeAttachedOp; idgen: I let op = getAttachedOp(g, t, kind) if op != nil and op.ast != nil and op.ast.isGenericRoutine: if t.typeInst != nil: - var a: TLiftCtx - a.info = info - a.g = g - a.kind = kind - a.c = c - a.idgen = idgen - + var a = TLiftCtx(info: info, g: g, kind: kind, c: c, idgen: idgen) let opInst = instantiateGeneric(a, op, t, t.typeInst) if opInst.ast != nil: patchBody(g, c, opInst.ast, info, a.idgen) From af086b68f2f718a4ad20e37bf3a00d9a57abec4d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sun, 12 Mar 2023 13:01:12 +0800 Subject: [PATCH 1652/1654] fixes sinkinference documentation, which has been disabled (#21470) since https://github.com/nim-lang/Nim/pull/15105 --- doc/advopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/advopt.txt b/doc/advopt.txt index 122a7c6ea..4db89062b 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -172,6 +172,6 @@ Advanced options: --useVersion:1.0|1.2|1.6 emulate Nim version X of the Nim compiler, for testing --benchmarkVM:on|off turn benchmarking of VM code with cpuTime() on|off --profileVM:on|off turn compile time VM profiler on|off - --sinkInference:on|off turn sink parameter inference on|off (default: on) + --sinkInference:on|off turn sink parameter inference on|off (default: off) --panics:on|off turn panics into process terminations (default: off) --deepcopy:on|off enable 'system.deepCopy' for ``--mm:arc|orc`` From ffadc75afead6baed0885877edb2c3b1c9ef1b2e Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 13 Mar 2023 03:03:14 +0800 Subject: [PATCH 1653/1654] fixes #21496; Ambiguous calls compiles when module name are equal (#21500) * fixes #21496; Ambiguous calls compiles when module name are equal * add a test case --- compiler/lookups.nim | 3 ++- tests/import/buzz/m21496.nim | 1 + tests/import/fizz/m21496.nim | 1 + tests/import/t21496.nim | 9 +++++++++ 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/import/buzz/m21496.nim create mode 100644 tests/import/fizz/m21496.nim create mode 100644 tests/import/t21496.nim diff --git a/compiler/lookups.nim b/compiler/lookups.nim index e7bca08bc..3f028a52f 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -343,7 +343,8 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym, info: TLineInfo) = if sym.name.s == "_": return let conflict = scope.addUniqueSym(sym) if conflict != nil: - if sym.kind == skModule and conflict.kind == skModule and sym.owner == conflict.owner: + if sym.kind == skModule and conflict.kind == skModule and + sym.position == conflict.position: # e.g.: import foo; import foo # xxx we could refine this by issuing a different hint for the case # where a duplicate import happens inside an include. diff --git a/tests/import/buzz/m21496.nim b/tests/import/buzz/m21496.nim new file mode 100644 index 000000000..7c87e2c04 --- /dev/null +++ b/tests/import/buzz/m21496.nim @@ -0,0 +1 @@ +proc fb* = echo "buzz!" \ No newline at end of file diff --git a/tests/import/fizz/m21496.nim b/tests/import/fizz/m21496.nim new file mode 100644 index 000000000..834c11eae --- /dev/null +++ b/tests/import/fizz/m21496.nim @@ -0,0 +1 @@ +proc fb* = echo "fizz!" \ No newline at end of file diff --git a/tests/import/t21496.nim b/tests/import/t21496.nim new file mode 100644 index 000000000..568f2ac51 --- /dev/null +++ b/tests/import/t21496.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "redefinition of 'm21496'; previous declaration here: t21496.nim(5, 12)" +""" + +import fizz/m21496, buzz/m21496 + +# bug #21496 + +m21496.fb() From b2c1dcbbc9b097c9c13dda4951e824cdb5f16225 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 13 Mar 2023 03:03:46 +0800 Subject: [PATCH 1654/1654] fixes explicit globals in macros (#21502) --- compiler/vmgen.nim | 3 ++- tests/vm/tvmmisc.nim | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index be7938e53..937d4b095 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1909,7 +1909,8 @@ proc genVarSection(c: PCtx; n: PNode) = c.genAdditionalCopy(a[2], opcWrDeref, tmp, 0, val) c.freeTemp(val) c.freeTemp(tmp) - elif not importcCondVar(s) and not (s.typ.kind == tyProc and s.typ.callConv == ccClosure): # fixes #10938 + elif not importcCondVar(s) and not (s.typ.kind == tyProc and s.typ.callConv == ccClosure) and + sfPure notin s.flags: # fixes #10938 # there is a pre-existing issue with closure types in VM # if `(var s: proc () = default(proc ()); doAssert s == nil)` works for you; # you might remove the second condition. diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim index 673e3e965..14818375d 100644 --- a/tests/vm/tvmmisc.nim +++ b/tests/vm/tvmmisc.nim @@ -643,3 +643,16 @@ const b = block: doAssert a == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] doAssert b == @[@[0, 1, 2], @[0, 1, 2], @[0, 1, 2]] + +macro m1(s: string): int = + var ProcID {.global, compileTime.}: int + inc(ProcID) + result = newLit(ProcID) + +proc macroGlobal = + doAssert m1("Macro argument") == 1 + doAssert m1("Macro argument") == 2 + doAssert m1("Macro argument") == 3 + +static: macroGlobal() +macroGlobal()