From f838c1baa48a22ef74f431d9b691af179d1c632d Mon Sep 17 00:00:00 2001 From: Adam Strzelecki Date: Fri, 30 Oct 2015 00:27:33 +0100 Subject: [PATCH 001/125] fixes #3498 Previously it was not possible to use template arguments in template body as the symbols were not resolved correctly leading to Error: undeclared identifier: 'XX', eg.: template defaultOf[T](t: T): expr = (var d: T; d) echo defaultOf(1) #<- invalid identifier, but should output 0 --- compiler/evaltempl.nim | 3 ++- compiler/semtempl.nim | 5 +---- tests/generics/tgenerictmpl.nim | 25 +++++++++++++++++-------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/compiler/evaltempl.nim b/compiler/evaltempl.nim index c33e5be86..01233274b 100644 --- a/compiler/evaltempl.nim +++ b/compiler/evaltempl.nim @@ -38,7 +38,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) = if s.owner.id == c.owner.id: if s.kind == skParam and sfGenSym notin s.flags: handleParam actual.sons[s.position] - elif s.kind == skGenericParam: + elif s.kind == skGenericParam or + s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam: handleParam actual.sons[s.owner.typ.len + s.position - 1] else: internalAssert sfGenSym in s.flags diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 2dda8276d..0dc600983 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -228,10 +228,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym): PNode = of skParam: result = n of skType: - if (s.typ != nil) and (s.typ.kind != tyGenericParam): - result = newSymNodeTypeDesc(s, n.info) - else: - result = n + result = newSymNodeTypeDesc(s, n.info) else: result = newSymNode(s, n.info) diff --git a/tests/generics/tgenerictmpl.nim b/tests/generics/tgenerictmpl.nim index a749e6570..c71ce4e2e 100644 --- a/tests/generics/tgenerictmpl.nim +++ b/tests/generics/tgenerictmpl.nim @@ -1,12 +1,21 @@ +discard """ + output: '''0 +123''' +""" -template tmp[T](x: var seq[T]) = - #var yz: T # XXX doesn't work yet +# bug #3498 + +template defaultOf[T](t: T): expr = (var d: T; d) + +echo defaultOf(1) #<- excpected 0 + +# assignment using template + +template tassign[T](x: var seq[T]) = x = @[1, 2, 3] -macro tmp2[T](x: var seq[T]): stmt = - nil - var y: seq[int] -tmp(y) -tmp(y) -echo y.repr +tassign(y) #<- x is expected = @[1, 2, 3] +tassign(y) + +echo y[0], y[1], y[2] From 03590a1c42905d7786f307ec8e3ba6930059ee83 Mon Sep 17 00:00:00 2001 From: JamesP Date: Sat, 7 Nov 2015 15:10:51 +1000 Subject: [PATCH 002/125] Removal of RunningStats ready for new stats.nim pure lib file --- lib/pure/math.nim | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 391a880ae..91877fcfa 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -376,48 +376,6 @@ proc random*[T](a: openArray[T]): T = ## returns a random element from the openarray `a`. result = a[random(a.low..a.len)] -type - RunningStat* = object ## an accumulator for statistical data - n*: int ## number of pushed data - sum*, min*, max*, mean*: float ## self-explaining - oldM, oldS, newS: float - -{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} - -proc push*(s: var RunningStat, x: float) = - ## pushes a value `x` for processing - inc(s.n) - # See Knuth TAOCP vol 2, 3rd edition, page 232 - if s.n == 1: - s.min = x - s.max = x - s.oldM = x - s.mean = x - s.oldS = 0.0 - else: - if s.min > x: s.min = x - if s.max < x: s.max = x - s.mean = s.oldM + (x - s.oldM)/toFloat(s.n) - s.newS = s.oldS + (x - s.oldM)*(x - s.mean) - - # set up for next iteration: - s.oldM = s.mean - s.oldS = s.newS - s.sum = s.sum + x - -proc push*(s: var RunningStat, x: int) = - ## pushes a value `x` for processing. `x` is simply converted to ``float`` - ## and the other push operation is called. - push(s, toFloat(x)) - -proc variance*(s: RunningStat): float = - ## computes the current variance of `s` - if s.n > 1: result = s.newS / (toFloat(s.n - 1)) - -proc standardDeviation*(s: RunningStat): float = - ## computes the current standard deviation of `s` - result = sqrt(variance(s)) - {.pop.} {.pop.} From cdd00c4df608232c4822aed72bf7bd56e9cbd80c Mon Sep 17 00:00:00 2001 From: JamesP Date: Sat, 7 Nov 2015 15:15:30 +1000 Subject: [PATCH 003/125] Add RunningStat related statistics code removed from math lib, and add new functionality - skewness & kurtosis (3rd and 4th stat moments) - addition ops for RunningStat - RunningRegress object - slope, intercept and correlation regression procs - addition ops for RunningRegress --- lib/pure/stats.nim | 269 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 lib/pure/stats.nim diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim new file mode 100644 index 000000000..a78a2b5bf --- /dev/null +++ b/lib/pure/stats.nim @@ -0,0 +1,269 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Statistical analysis framework for performing +## basic statistical analysis of data. +## The data is analysed in a single pass, when a data value +## is pushed to the ``RunningStat`` or ``RunningRegress`` objects +## +## ``RunningStat`` calculates for a single data set +## - n (data count) +## - min (smallest value) +## - max (largest value) +## - mean +## - variance +## - varianceS (sample var) +## - standardDeviation +## - standardDeviationS (sample stddev) +## - skewness (the third statistical moment) +## - kurtosis (the fourth statistical moment) +## +## ``RunningRegress`` calculates for two sets of data +## - n +## - slope +## - intercept +## - correlation +## +from math import FloatClass, sqrt, pow, round + +{.push debugger:off .} # the user does not want to trace a part + # of the standard library! +{.push checks:off, line_dir:off, stack_trace:off.} + +type + RunningStat* = object ## an accumulator for statistical data + n*: int ## number of pushed data + min*, max*: float ## self-explaining + mom1, mom2, mom3, mom4: float ## statistical moments, mom1 is mean + + + RunningRegress* = object ## an accumulator for regression calculations + n*: int ## number of pushed data + x_stats*: RunningStat ## stats for first set of data + y_stats*: RunningStat ## stats for second set of data + s_xy: float ## acculated data for combined xy + +{.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} + +proc clear(s: var RunningStat) = + ## reset `s` + s.n = 0 + s.min = 0.0 + s.max = 0.0 + s.mom1 = 0.0 + s.mom2 = 0.0 + s.mom3 = 0.0 + s.mom4 = 0.0 + +proc push*(s: var RunningStat, x: float) = + ## pushes a value `x` for processing + inc(s.n) + # See Knuth TAOCP vol 2, 3rd edition, page 232 + if s.min > x: s.min = x + if s.max < x: s.max = x + let n = toFloat(s.n) + let delta = x - s.mom1 + let delta_n = delta / toFloat(s.n) + let delta_n2 = delta_n * delta_n + let term1 = delta * delta_n * toFloat(s.n - 1) + s.mom4 += term1 * delta_n2 * (n*n - 3*n + 3) + + 6*delta_n2*s.mom2 - 4*delta_n*s.mom3 + s.mom3 += term1 * delta_n * (n - 2) - 3*delta_n*s.mom2 + s.mom2 += term1 + s.mom1 += delta_n + +proc push*(s: var RunningStat, x: int) = + ## pushes a value `x` for processing. + ## + ## `x` is simply converted to ``float`` + ## and the other push operation is called. + s.push(toFloat(x)) + +proc push*(s: var RunningStat, x: openarray[float|int]) = + ## pushes all values of `x` for processing. + ## + ## Int values of `x` are simply converted to ``float`` and + ## the other push operation is called. + for val in x: + s.push(val) + +proc mean*(s: RunningStat): float = result = s.mom1 + ## computes the current mean of `s` + +proc variance*(s: RunningStat): float = + ## computes the current population variance of `s` + result = s.mom2 / toFloat(s.n) + +proc varianceS*(s: RunningStat): float = + ## computes the current sample variance of `s` + if s.n > 1: result = s.mom2 / toFloat(s.n - 1) + +proc standardDeviation*(s: RunningStat): float = + ## computes the current population standard deviation of `s` + result = sqrt(variance(s)) + +proc standardDeviationS*(s: RunningStat): float = + ## computes the current sample standard deviation of `s` + result = sqrt(varianceS(s)) + +proc skewness*(s: RunningStat): float = + ## computes the current population skewness of `s` + result = sqrt(toFloat(s.n)) * s.mom3 / pow(s.mom2, 1.5) + +proc skewnessS*(s: RunningStat): float = + ## computes the current sample skewness of `s` + let s2 = skewness(s) + result = sqrt(toFloat(s.n*(s.n-1)))*s2 / toFloat(s.n-2) + +proc kurtosis*(s: RunningStat): float = + ## computes the current population kurtosis of `s` + result = toFloat(s.n) * s.mom4 / (s.mom2 * s.mom2) - 3.0 + +proc kurtosisS*(s: RunningStat): float = + ## computes the current sample kurtosis of `s` + result = toFloat(s.n-1) / toFloat((s.n-2)*(s.n-3)) * + (toFloat(s.n+1)*kurtosis(s) + 6) + +proc `+`*(a, b: RunningStat): RunningStat = + ## combine two RunningStats. + ## + ## Useful if performing parallel analysis of data series + ## and need to re-combine parallel result sets + result.clear() + result.n = a.n + b.n + + let delta = b.mom1 - a.mom1 + let delta2 = delta*delta + let delta3 = delta*delta2 + let delta4 = delta2*delta2 + let n = toFloat(result.n) + + result.mom1 = (a.n.float*a.mom1 + b.n.float*b.mom1) / n + result.mom2 = a.mom2 + b.mom2 + delta2 * a.n.float * b.n.float / n + result.mom3 = a.mom3 + b.mom3 + + delta3 * a.n.float * b.n.float * (a.n.float - b.n.float)/(n*n); + result.mom3 += 3.0*delta * (a.n.float*b.mom2 - b.n.float*a.mom2) / n + result.mom4 = a.mom4 + b.mom4 + + delta4*a.n.float*b.n.float * toFloat(a.n*a.n - a.n*b.n + b.n*b.n) / + (n*n*n) + result.mom4 += 6.0*delta2 * (a.n.float*a.n.float*b.mom2 + b.n.float*b.n.float*a.mom2) / + (n*n) + + 4.0*delta*(a.n.float*b.mom3 - b.n.float*a.mom3) / n + result.max = max(a.max, b.max) + result.min = max(a.min, b.min) + +proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = + ## add a second RunningStats `b` to `a` + a = a + b + +# ---------------------- Running Regression ----------------------------- + +proc clear*(r: var RunningRegress) = + ## reset `r` + r.x_stats.clear() + r.y_stats.clear() + r.s_xy = 0.0 + r.n = 0 + +proc push*(r: var RunningRegress, x, y: float) = + ## pushes two values `x` and `y` for processing + r.s_xy += (r.x_stats.mean() -x)*(r.y_stats.mean() - y)*toFloat(r.n)/toFloat(r.n+1) + + r.x_stats.push(x) + r.y_stats.push(y) + inc(r.n) + +proc push*(r: var RunningRegress, x, y: int) {.inline.} = + ## pushes two values `x` and `y` for processing. + ## + ## `x` and `y` are converted to ``float`` + ## and the other push operation is called. + r.push(toFloat(x), toFloat(y)) + +proc push*(r: var RunningRegress, x, y: openarray[float|int]) = + ## pushes two sets of values `x` and `y` for processing. + assert(x.len == y.len) + for i in 0.. Date: Sat, 7 Nov 2015 17:47:49 +1000 Subject: [PATCH 004/125] import the new stats lib --- examples/statcsv.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/statcsv.nim b/examples/statcsv.nim index f2cf809e2..983cd555f 100644 --- a/examples/statcsv.nim +++ b/examples/statcsv.nim @@ -3,7 +3,7 @@ # the standard deviation of its columns. # The CSV file can have a header which is then used for the output. -import os, streams, parsecsv, strutils, math +import os, streams, parsecsv, strutils, math, stats if paramCount() < 1: quit("Usage: statcsv filename[.csv]") From 7d62b9fc1a7f8ec18c83b3800aaaecda1b085c46 Mon Sep 17 00:00:00 2001 From: JamesP Date: Sat, 7 Nov 2015 18:00:21 +1000 Subject: [PATCH 005/125] add sum back in, fix min inital value, tidy up spaces in code and comment wording, add a sum test assertion --- lib/pure/stats.nim | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index a78a2b5bf..50b5601e5 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -15,6 +15,7 @@ ## - n (data count) ## - min (smallest value) ## - max (largest value) +## - sum ## - mean ## - variance ## - varianceS (sample var) @@ -36,25 +37,26 @@ from math import FloatClass, sqrt, pow, round {.push checks:off, line_dir:off, stack_trace:off.} type - RunningStat* = object ## an accumulator for statistical data - n*: int ## number of pushed data - min*, max*: float ## self-explaining - mom1, mom2, mom3, mom4: float ## statistical moments, mom1 is mean + RunningStat* = object ## an accumulator for statistical data + n*: int ## number of pushed data + min*, max*, sum*: float ## self-explaining + mom1, mom2, mom3, mom4: float ## statistical moments, mom1 is mean RunningRegress* = object ## an accumulator for regression calculations n*: int ## number of pushed data x_stats*: RunningStat ## stats for first set of data - y_stats*: RunningStat ## stats for second set of data - s_xy: float ## acculated data for combined xy + y_stats*: RunningStat ## stats for second set of data + s_xy: float ## accumulated data for combined xy {.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} -proc clear(s: var RunningStat) = +proc clear*(s: var RunningStat) = ## reset `s` s.n = 0 - s.min = 0.0 + s.min = toBiggestFloat(int.high) s.max = 0.0 + s.sum = 0.0 s.mom1 = 0.0 s.mom2 = 0.0 s.mom3 = 0.0 @@ -62,10 +64,12 @@ proc clear(s: var RunningStat) = proc push*(s: var RunningStat, x: float) = ## pushes a value `x` for processing + if s.n == 0: s.min = x inc(s.n) # See Knuth TAOCP vol 2, 3rd edition, page 232 if s.min > x: s.min = x if s.max < x: s.max = x + s.sum += x let n = toFloat(s.n) let delta = x - s.mom1 let delta_n = delta / toFloat(s.n) @@ -92,8 +96,9 @@ proc push*(s: var RunningStat, x: openarray[float|int]) = for val in x: s.push(val) -proc mean*(s: RunningStat): float = result = s.mom1 +proc mean*(s: RunningStat): float = ## computes the current mean of `s` + result = s.mom1 proc variance*(s: RunningStat): float = ## computes the current population variance of `s` @@ -172,8 +177,8 @@ proc clear*(r: var RunningRegress) = proc push*(r: var RunningRegress, x, y: float) = ## pushes two values `x` and `y` for processing - r.s_xy += (r.x_stats.mean() -x)*(r.y_stats.mean() - y)*toFloat(r.n)/toFloat(r.n+1) - + r.s_xy += (r.x_stats.mean() - x)*(r.y_stats.mean() - y)* + toFloat(r.n) / toFloat(r.n + 1) r.x_stats.push(x) r.y_stats.push(y) inc(r.n) @@ -254,6 +259,10 @@ when isMainModule: doAssert(clean(rs1.mom2) == clean(rs.mom2)) doAssert(clean(rs1.mom3) == clean(rs.mom3)) doAssert(clean(rs1.mom4) == clean(rs.mom4)) + rs1.clear() + rs1.push(@[1.0, 2.2, 1.4, 4.9]) + doAssert(rs1.sum == 9.5) + doAssert(rs1.mean() == 2.375) var rr: RunningRegress rr.push(@[0.0,1.0,2.8,3.0,4.0], @[0.0,1.0,2.3,3.0,4.0]) From 0cf4e2603f7dee5f29148deaa134f6560b68353b Mon Sep 17 00:00:00 2001 From: JamesP Date: Wed, 11 Nov 2015 08:56:12 +1000 Subject: [PATCH 006/125] Removal of stats procs for openarray (use the RunningStat methodology) --- lib/pure/math.nim | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 91877fcfa..b0104336e 100644 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -118,26 +118,6 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} = ## If `x` is empty, 0 is returned. for i in items(x): result = result + i -template toFloat(f: float): float = f - -proc mean*[T](x: openArray[T]): float {.noSideEffect.} = - ## Computes the mean of the elements in `x`, which are first converted to floats. - ## If `x` is empty, NaN is returned. - ## ``toFloat(x: T): float`` must be defined. - for i in items(x): result = result + toFloat(i) - result = result / toFloat(len(x)) - -proc variance*[T](x: openArray[T]): float {.noSideEffect.} = - ## Computes the variance of the elements in `x`. - ## If `x` is empty, NaN is returned. - ## ``toFloat(x: T): float`` must be defined. - result = 0.0 - var m = mean(x) - for i in items(x): - var diff = toFloat(i) - m - result = result + diff*diff - result = result / toFloat(len(x)) - proc random*(max: int): int {.benign.} ## Returns a random number in the range 0..max-1. The sequence of ## random number is always the same, unless `randomize` is called From 4cefbc2914c5ca470b1214cc1b77bf263aebd35a Mon Sep 17 00:00:00 2001 From: JamesP Date: Wed, 11 Nov 2015 08:56:51 +1000 Subject: [PATCH 007/125] add comment to identify where RunningStat code resides --- lib/pure/stats.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 50b5601e5..5811e141e 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -51,6 +51,7 @@ type {.deprecated: [TFloatClass: FloatClass, TRunningStat: RunningStat].} +# ----------- RunningStat -------------------------- proc clear*(s: var RunningStat) = ## reset `s` s.n = 0 From 52e40995b7adfec8c1a837bed5a84d475f394602 Mon Sep 17 00:00:00 2001 From: JamesP Date: Thu, 12 Nov 2015 06:33:26 +1000 Subject: [PATCH 008/125] added helper procs for standalone calc of stats for array/seq Changed derivation to use runningstats, so there is a single point of truth for the calculations, and provides more stats functions --- lib/pure/stats.nim | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 5811e141e..43d9c35f6 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -166,6 +166,51 @@ proc `+`*(a, b: RunningStat): RunningStat = proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = ## add a second RunningStats `b` to `a` a = a + b +# ---------------------- standalone array/seq stats --------------------- +proc mean*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.mean() + +proc variance*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.variance() + +proc varianceS*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.varianceS() + +proc standardDeviation*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.standardDeviation() + +proc standardDeviationS*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.standardDeviationS() + +proc skewness*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.skewness() + +proc skewnessS*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.skewnessS() + +proc kurtosis*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.kurtosis() + +proc kurtosisS*[T](x: openArray[T]): float = + var rs: RunningStat + rs.push(x) + result = rs.kurtosisS() # ---------------------- Running Regression ----------------------------- From 41861711c81f189646222547e938778a2600d91e Mon Sep 17 00:00:00 2001 From: JamesP Date: Thu, 12 Nov 2015 07:03:16 +1000 Subject: [PATCH 009/125] add doco for array/seq helper procs --- lib/pure/stats.nim | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index 43d9c35f6..ec4cd182b 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -30,6 +30,21 @@ ## - intercept ## - correlation ## +## Procs have been provided to calculate statistics on arrays and sequences. +## +## However, if more than a single statistical calculation is required, it is more +## efficient to push the data once to the RunningStat object, and +## call the numerous statistical procs for the RunningStat object. +## +## .. code-block:: Nim +## +## var rs: RunningStat +## rs.push(MySeqOfData) +## rs.mean() +## rs.variance() +## rs.skewness() +## rs.kurtosis() + from math import FloatClass, sqrt, pow, round {.push debugger:off .} # the user does not want to trace a part @@ -168,46 +183,55 @@ proc `+=`*(a: var RunningStat, b: RunningStat) {.inline.} = a = a + b # ---------------------- standalone array/seq stats --------------------- proc mean*[T](x: openArray[T]): float = + ## computes the mean of `x` var rs: RunningStat rs.push(x) result = rs.mean() proc variance*[T](x: openArray[T]): float = + ## computes the population variance of `x` var rs: RunningStat rs.push(x) result = rs.variance() proc varianceS*[T](x: openArray[T]): float = + ## computes the sample variance of `x` var rs: RunningStat rs.push(x) result = rs.varianceS() proc standardDeviation*[T](x: openArray[T]): float = + ## computes the population standardDeviation of `x` var rs: RunningStat rs.push(x) result = rs.standardDeviation() proc standardDeviationS*[T](x: openArray[T]): float = + ## computes the sanple standardDeviation of `x` var rs: RunningStat rs.push(x) result = rs.standardDeviationS() proc skewness*[T](x: openArray[T]): float = + ## computes the population skewness of `x` var rs: RunningStat rs.push(x) result = rs.skewness() proc skewnessS*[T](x: openArray[T]): float = + ## computes the sample skewness of `x` var rs: RunningStat rs.push(x) result = rs.skewnessS() proc kurtosis*[T](x: openArray[T]): float = + ## computes the population kurtosis of `x` var rs: RunningStat rs.push(x) result = rs.kurtosis() proc kurtosisS*[T](x: openArray[T]): float = + ## computes the sample kurtosis of `x` var rs: RunningStat rs.push(x) result = rs.kurtosisS() From f3d578245bfa7c2a8ee171f37b168c8d8d3d8925 Mon Sep 17 00:00:00 2001 From: xiongxin <1781189926@qq.com> Date: Tue, 8 Dec 2015 00:48:24 +0800 Subject: [PATCH 010/125] 7 div 5 == 1 7 div 5 == 1 --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index ce7687c34..bb8254364 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -840,7 +840,7 @@ proc `div` *(x, y: int32): int32 {.magic: "DivI", noSideEffect.} ## 1 div 2 == 0 ## 2 div 2 == 1 ## 3 div 2 == 1 - ## 7 div 5 == 2 + ## 7 div 5 == 1 when defined(nimnomagic64): proc `div` *(x, y: int64): int64 {.magic: "DivI", noSideEffect.} From 8af8f7673be0550170c8d143fb6ac24f0fe52777 Mon Sep 17 00:00:00 2001 From: JamesP Date: Mon, 7 Dec 2015 15:23:16 +1000 Subject: [PATCH 011/125] add timeinterval helping functions as per issue #3609 add tests to ttime.nim for timeinterval add/subtract functionality Changed assert to doAssert in ttime.nim Added extra tests to ttime.nim Removal of singular help procs for working with TimeIntervals Added TimeIin proc toSeconds(a: TimeInfo, interval: TimeInterval) where subtracting a year gave wrong results Add overflow of units on initInterval eg, 65 seconds is 5 seconds and 1 minute --- lib/pure/times.nim | 152 +++++++++++++++++++++++++++++++++++------ tests/stdlib/ttime.nim | 94 +++++++++++++++---------- 2 files changed, 187 insertions(+), 59 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index c9854a650..f73ab99b4 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -245,13 +245,46 @@ proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} proc initInterval*(milliseconds, seconds, minutes, hours, days, months, years: int = 0): TimeInterval = ## creates a new ``TimeInterval``. - result.milliseconds = milliseconds - result.seconds = seconds - result.minutes = minutes - result.hours = hours - result.days = days - result.months = months - result.years = years + var carryO = 0 + result.milliseconds = `mod`(milliseconds, 1000) + carryO = `div`(milliseconds, 1000) + result.seconds = `mod`(carryO + seconds, 60) + carryO = `div`(seconds, 60) + result.minutes = `mod`(carryO + minutes, 60) + carryO = `div`(minutes, 60) + result.hours = `mod`(carryO + hours, 24) + carryO = `div`(hours, 24) + result.days = carryO + days + carryO = 0 + result.months = `mod`(months, 12) + carryO = `div`(months, 12) + result.years = carryO + years + +proc `+`*(ti1, ti2: TimeInterval): TimeInterval = + var carryO = 0 + result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000) + carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000) + result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60) + carryO = `div`(ti1.seconds + ti2.seconds, 60) + result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60) + carryO = `div`(ti1.minutes + ti2.minutes, 60) + result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24) + carryO = `div`(ti1.hours + ti2.hours, 24) + result.days = carryO + ti1.days + ti2.days + carryO = 0 + result.months = `mod`(ti1.months + ti2.months, 12) + carryO = `div`(ti1.months + ti2.months, 12) + result.years = carryO + ti1.years + ti2.years + +proc `-`*(ti1, ti2: TimeInterval): TimeInterval = + result = ti1 + result.milliseconds -= ti2.milliseconds + result.seconds -= ti2.seconds + result.minutes -= ti2.minutes + result.hours -= ti2.hours + result.days -= ti2.days + result.months -= ti2.months + result.years -= ti2.years proc isLeapYear*(year: int): bool = ## returns true if ``year`` is a leap year @@ -288,13 +321,23 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float = newinterv.months += interval.years * 12 var curMonth = anew.month - for mth in 1 .. newinterv.months: - result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) - if curMonth == mDec: - curMonth = mJan - anew.year.inc() - else: - curMonth.inc() + if newinterv.months < 0: # subtracting + for mth in countDown(-1 * newinterv.months, 1): + result -= float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) + if curMonth == mJan: + curMonth = mDec + anew.year.dec() + else: + curMonth.dec() + else: # adding + for mth in 1 .. newinterv.months: + result += float(getDaysInMonth(curMonth, anew.year) * 24 * 60 * 60) + if curMonth == mDec: + curMonth = mJan + anew.year.inc() + else: + curMonth.inc() + result += float(newinterv.days * 24 * 60 * 60) result += float(newinterv.days * 24 * 60 * 60) result += float(newinterv.hours * 60 * 60) result += float(newinterv.minutes * 60) @@ -308,9 +351,6 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## very accurate. let t = toSeconds(timeInfoToTime(a)) let secs = toSeconds(a, interval) - #if a.tzname == "UTC": - # result = getGMTime(fromSeconds(t + secs)) - #else: result = getLocalTime(fromSeconds(t + secs)) proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = @@ -319,11 +359,16 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. let t = toSeconds(timeInfoToTime(a)) - let secs = toSeconds(a, interval) - #if a.tzname == "UTC": - # result = getGMTime(fromSeconds(t - secs)) - #else: - result = getLocalTime(fromSeconds(t - secs)) + var intval: TimeInterval + intval.milliseconds = - interval.milliseconds + intval.seconds = - interval.seconds + intval.minutes = - interval.minutes + intval.hours = - interval.hours + intval.days = - interval.days + intval.months = - interval.months + intval.years = - interval.years + let secs = toSeconds(a, intval) + result = getLocalTime(fromSeconds(t + secs)) when not defined(JS): proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} @@ -603,6 +648,69 @@ proc `$`*(m: Month): string = "November", "December"] return lookup[m] +proc milliseconds*(ms: int): TimeInterval {.inline.} = + ## TimeInterval of `ms` milliseconds + ## + ## Note: not all time functions have millisecond resolution + initInterval(`mod`(ms,1000), `div`(ms,1000)) + +proc seconds*(s: int): TimeInterval {.inline.} = + ## TimeInterval of `s` seconds + ## + ## ``echo getTime() + 5.second`` + initInterval(0,`mod`(s,60), `div`(s,60)) + +proc minutes*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` minutes + ## + ## ``echo getTime() + 5.minutes`` + initInterval(0,0,`mod`(m,60), `div`(m,60)) + +proc hours*(h: int): TimeInterval {.inline.} = + ## TimeInterval of `h` hours + ## + ## ``echo getTime() + 2.hours`` + initInterval(0,0,0,`mod`(h,24),`div`(h,24)) + +proc days*(d: int): TimeInterval {.inline.} = + ## TimeInterval of `d` days + ## + ## ``echo getTime() + 2.days`` + initInterval(0,0,0,0,d) + +proc months*(m: int): TimeInterval {.inline.} = + ## TimeInterval of `m` months + ## + ## ``echo getTime() + 2.months`` + initInterval(0,0,0,0,0,`mod`(m,12),`div`(m,12)) + +proc years*(y: int): TimeInterval {.inline.} = + ## TimeInterval of `y` years + ## + ## ``echo getTime() + 2.years`` + initInterval(0,0,0,0,0,0,y) + +proc `+=`*(t: var Time, ti: TimeInterval) = + ## modifies `t` by adding the interval `ti` + t = timeInfoToTime(getLocalTime(t) + ti) + +proc `+`*(t: Time, ti: TimeInterval): Time = + ## adds the interval `ti` to Time `t` + ## by converting to localTime, adding the interval, and converting back + ## + ## ``echo getTime() + 1.day`` + result = timeInfoToTime(getLocalTime(t) + ti) + +proc `-=`*(t: var Time, ti: TimeInterval) = + ## modifies `t` by subtracting the interval `ti` + t = timeInfoToTime(getLocalTime(t) - ti) + +proc `-`*(t: Time, ti: TimeInterval): Time = + ## adds the interval `ti` to Time `t` + ## + ## ``echo getTime() - 1.day`` + result = timeInfoToTime(getLocalTime(t) - ti) + proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. ## diff --git a/tests/stdlib/ttime.nim b/tests/stdlib/ttime.nim index efc371995..1df5daeec 100644 --- a/tests/stdlib/ttime.nim +++ b/tests/stdlib/ttime.nim @@ -6,89 +6,89 @@ discard """ import times, strutils -assert( $getTime() == getLocalTime(getTime()).format("ddd MMM dd HH:mm:ss yyyy")) +doAssert( $getTime() == getLocalTime(getTime()).format("ddd MMM dd HH:mm:ss yyyy")) # $ date --date='@2147483647' # Tue 19 Jan 03:14:07 GMT 2038 var t = getGMTime(fromSeconds(2147483647)) -assert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" -assert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" +doAssert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" +doAssert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" -assert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & +doAssert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC" -assert t.format("yyyyMMddhhmmss") == "20380119031407" +doAssert t.format("yyyyMMddhhmmss") == "20380119031407" var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 -assert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & +doAssert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC" when not defined(JS): when sizeof(Time) == 8: var t3 = getGMTime(fromSeconds(889067643645)) # Fri 7 Jun 19:20:45 BST 30143 - assert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + doAssert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 0 00 00:00 UTC" - assert t3.format(":,[]()-/") == ":,[]()-/" + doAssert t3.format(":,[]()-/") == ":,[]()-/" var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 -assert t4.format("M MM MMM MMMM") == "10 10 Oct October" +doAssert t4.format("M MM MMM MMMM") == "10 10 Oct October" # Interval tests -assert((t4 - initInterval(years = 2)).format("yyyy") == "1995") -assert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") +doAssert((t4 - initInterval(years = 2)).format("yyyy") == "1995") +doAssert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") var s = "Tuesday at 09:04am on Dec 15, 2015" var f = "dddd at hh:mmtt on MMM d, yyyy" -assert($s.parse(f) == "Tue Dec 15 09:04:00 2015") +doAssert($s.parse(f) == "Tue Dec 15 09:04:00 2015") # ANSIC = "Mon Jan _2 15:04:05 2006" s = "Thu Jan 12 15:04:05 2006" f = "ddd MMM dd HH:mm:ss yyyy" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # UnixDate = "Mon Jan _2 15:04:05 MST 2006" s = "Thu Jan 12 15:04:05 MST 2006" f = "ddd MMM dd HH:mm:ss ZZZ yyyy" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # RubyDate = "Mon Jan 02 15:04:05 -0700 2006" s = "Thu Jan 12 15:04:05 -07:00 2006" f = "ddd MMM dd HH:mm:ss zzz yyyy" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # RFC822 = "02 Jan 06 15:04 MST" s = "12 Jan 16 15:04 MST" f = "dd MMM yy HH:mm ZZZ" -assert($s.parse(f) == "Tue Jan 12 15:04:00 2016") +doAssert($s.parse(f) == "Tue Jan 12 15:04:00 2016") # RFC822Z = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone s = "12 Jan 16 15:04 -07:00" f = "dd MMM yy HH:mm zzz" -assert($s.parse(f) == "Tue Jan 12 15:04:00 2016") +doAssert($s.parse(f) == "Tue Jan 12 15:04:00 2016") # RFC850 = "Monday, 02-Jan-06 15:04:05 MST" s = "Monday, 12-Jan-06 15:04:05 MST" f = "dddd, dd-MMM-yy HH:mm:ss ZZZ" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" s = "Thu, 12 Jan 2006 15:04:05 MST" f = "ddd, dd MMM yyyy HH:mm:ss ZZZ" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone s = "Thu, 12 Jan 2006 15:04:05 -07:00" f = "ddd, dd MMM yyyy HH:mm:ss zzz" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # RFC3339 = "2006-01-02T15:04:05Z07:00" s = "2006-01-12T15:04:05Z-07:00" f = "yyyy-MM-ddTHH:mm:ssZzzz" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") f = "yyyy-MM-dd'T'HH:mm:ss'Z'zzz" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" s = "2006-01-12T15:04:05.999999999Z-07:00" f = "yyyy-MM-ddTHH:mm:ss.999999999Zzzz" -assert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") # Kitchen = "3:04PM" s = "3:04PM" f = "h:mmtt" -assert "15:04:00" in $s.parse(f) +doAssert "15:04:00" in $s.parse(f) #when not defined(testing): # echo "Kitchen: " & $s.parse(f) # var ti = timeToTimeInfo(getTime()) @@ -97,29 +97,49 @@ assert "15:04:00" in $s.parse(f) # echo "Todays date after decoding to interval: ", tint # checking dayOfWeek matches known days -assert getDayOfWeek(21, 9, 1900) == dFri -assert getDayOfWeek(1, 1, 1970) == dThu -assert getDayOfWeek(21, 9, 1970) == dMon -assert getDayOfWeek(1, 1, 2000) == dSat -assert getDayOfWeek(1, 1, 2021) == dFri +doAssert getDayOfWeek(21, 9, 1900) == dFri +doAssert getDayOfWeek(1, 1, 1970) == dThu +doAssert getDayOfWeek(21, 9, 1970) == dMon +doAssert getDayOfWeek(1, 1, 2000) == dSat +doAssert getDayOfWeek(1, 1, 2021) == dFri # Julian tests -assert getDayOfWeekJulian(21, 9, 1900) == dFri -assert getDayOfWeekJulian(21, 9, 1970) == dMon -assert getDayOfWeekJulian(1, 1, 2000) == dSat -assert getDayOfWeekJulian(1, 1, 2021) == dFri +doAssert getDayOfWeekJulian(21, 9, 1900) == dFri +doAssert getDayOfWeekJulian(21, 9, 1970) == dMon +doAssert getDayOfWeekJulian(1, 1, 2000) == dSat +doAssert getDayOfWeekJulian(1, 1, 2021) == dFri # toSeconds tests with GM and Local timezones #var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 var t4L = getLocalTime(fromSeconds(876124714)) -assert toSeconds(timeInfoToTime(t4L)) == 876124714 # fromSeconds is effectively "localTime" -assert toSeconds(timeInfoToTime(t4L)) + t4L.timezone.float == toSeconds(timeInfoToTime(t4)) +doAssert toSeconds(timeInfoToTime(t4L)) == 876124714 # fromSeconds is effectively "localTime" +doAssert toSeconds(timeInfoToTime(t4L)) + t4L.timezone.float == toSeconds(timeInfoToTime(t4)) # adding intervals var a1L = toSeconds(timeInfoToTime(t4L + initInterval(hours = 1))) + t4L.timezone.float a1G = toSeconds(timeInfoToTime(t4)) + 60.0 * 60.0 -assert a1L == a1G +doAssert a1L == a1G + # subtracting intervals a1L = toSeconds(timeInfoToTime(t4L - initInterval(hours = 1))) + t4L.timezone.float a1G = toSeconds(timeInfoToTime(t4)) - (60.0 * 60.0) -assert a1L == a1G +doAssert a1L == a1G + +# add/subtract TimeIntervals and Time/TimeInfo +doAssert getTime() - 1.seconds == getTime() - 3.seconds + 2.seconds +doAssert getTime() + 65.seconds == getTime() + 1.minutes + 5.seconds +doAssert getTime() + 60.minutes == getTime() + 1.hours +doAssert getTime() + 24.hours == getTime() + 1.days +doAssert getTime() + 13.months == getTime() + 1.years + 1.months +var + ti1 = getTime() + 1.years +ti1 -= 1.years +doAssert ti1 == getTime() +ti1 += 1.days +doAssert ti1 == getTime() + 1.days + +# overflow of TimeIntervals on initalisation +doAssert initInterval(milliseconds = 25000) == initInterval(seconds = 25) +doAssert initInterval(seconds = 65) == initInterval(seconds = 5, minutes = 1) +doAssert initInterval(hours = 25) == initInterval(hours = 1, days = 1) +doAssert initInterval(months = 13) == initInterval(months = 1, years = 1) From 5545d36330e03eab32773f91f1a800d7181a0679 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 8 Dec 2015 12:30:38 +0200 Subject: [PATCH 012/125] Fixed getAddrInfo on android --- lib/pure/nativesockets.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 9e6ff21ef..8a7780570 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -207,7 +207,8 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, # FreeBSD doesn't support AI_V4MAPPED but defines the macro. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092 when not defined(freebsd) or defined(openbsd): - hints.ai_flags = AI_V4MAPPED + if domain == AF_INET6: + hints.ai_flags = AI_V4MAPPED var gaiResult = getaddrinfo(address, $port, addr(hints), result) if gaiResult != 0'i32: when useWinVersion: From 16d7db47aad91bd4c6977e3eb11cb9a3fb7937eb Mon Sep 17 00:00:00 2001 From: JamesP Date: Wed, 9 Dec 2015 14:02:28 +1000 Subject: [PATCH 013/125] add db_odbc library module --- lib/impure/db_odbc.nim | 480 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 480 insertions(+) create mode 100644 lib/impure/db_odbc.nim diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim new file mode 100644 index 000000000..d7038303c --- /dev/null +++ b/lib/impure/db_odbc.nim @@ -0,0 +1,480 @@ +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +# +## A higher level `ODBC` database wrapper. +## +## This is the same interface that is implemented for other databases. +## +## This has NOT yet been (extensively) tested agains ODBC drivers for +## Teradata, Oracle, Sybase, MSSqlvSvr, et. al. databases +## +## Currently all queries are ANSI calls, not Unicode. +## +## Example: +## +## .. code-block:: Nim +## +## import db_odbc, math +## +## let theDb = open("localhost", "nim", "nim", "test") +## +## theDb.exec(sql"Drop table if exists myTestTbl") +## theDb.exec(sql("create table myTestTbl (" & +## " Id INT(11) NOT NULL AUTO_INCREMENT PRIMARY KEY, " & +## " Name VARCHAR(50) NOT NULL, " & +## " i INT(11), " & +## " f DECIMAL(18,10))")) +## +## theDb.exec(sql"START TRANSACTION") +## for i in 1..1000: +## theDb.exec(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", +## "Item#" & $i, i, sqrt(i.float)) +## theDb.exec(sql"COMMIT") +## +## for x in theDb.fastRows(sql"select * from myTestTbl"): +## echo x +## +## let id = theDb.tryInsertId(sql"INSERT INTO myTestTbl (name,i,f) VALUES (?,?,?)", +## "Item#1001", 1001, sqrt(1001.0)) +## echo "Inserted item: ", theDb.getValue(sql"SELECT name FROM myTestTbl WHERE id=?", id) +## +## theDb.close() + + +import strutils, odbcsql + +type + OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt] + DbConn* = OdbcConnTyp ## encapsulates a database connection + Row* = seq[string] ## a row of a dataset. NULL database values will be + ## transformed always to the empty string. + InstantRow* = tuple[row: seq[string], len: int] ## a handle that can be + ## used to get a row's + ## column text on demand + EDb* = object of IOError ## exception that is raised if a database error occurs + + SqlQuery* = distinct string ## an SQL query string + + FDb* = object of IOEffect ## effect that denotes a database operation + FReadDb* = object of FDb ## effect that denotes a read operation + FWriteDb* = object of FDb ## effect that denotes a write operation + FReadODBC* = object of FDb ## effect that denotes a read of ODBC driver + FWriteODBC* = object of FDb ## effect that denotes a write of ODBC driver +{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} + +var + buf: array[0..4096, char] + +proc properFreeResult(hType: int, sqlres: var SqlHandle) {. + tags: [FWriteODBC], raises: [].} = + try: + discard SQLFreeHandle(hType.TSqlSmallInt, sqlres) + sqlres = nil + except: discard + +proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = + ## Constructs a SqlQuery from the string `query`. This is supposed to be + ## used as a raw-string-literal modifier: + ## ``sql"update user set counter = counter + 1"`` + ## + ## If assertions are turned off, it does nothing. If assertions are turned + ## on, later versions will check the string for valid syntax. + result = SqlQuery(query) + +proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. + tags: [FReadODBC], raises: [].} = + ## Returns ODBC error information + var + sqlState: array[0..512, char] + nativeErr: array[0..512, char] + errMsg: array[0..512, char] + retSz: TSqlSmallInt = 0 + res: TSqlSmallInt = 0 + try: + sqlState[0] = '\0' + nativeErr[0] = '\0' + errMsg[0] = '\0' + res = SQLErr(db.env, db.hDb, db.stmt, + cast[PSQLCHAR](sqlState.addr), + cast[PSQLCHAR](nativeErr.addr), + cast[PSQLCHAR](errMsg.addr), + 511.TSqlSmallInt, retSz.addr.PSQLSMALLINT) + except: + discard + return (res.int, $sqlState, $nativeErr, $errMsg) + +proc dbError*(db: var DbConn) {. + tags: [FReadODBC,FWriteODBC], raises: [EDb] .} = + ## Raises an `[EDb]` exception with ODBC error information + var + e: ref EDb + ss, ne, msg: string = "" + isAnError = false + res: int = 0 + prevSs = "" + while true: + prevSs = ss + (res, ss, ne, msg) = db.getErrInfo() + if prevSs == ss: + break + # sqlState of 00000 is not an error + elif ss == "00000": + break + elif ss == "01000": + echo "\nWarning: ", ss, " ", msg + continue + else: + isAnError = true + echo "\nError: ", ss, " ", msg + if isAnError: + new(e) + e.msg = "ODBC Error" + if db.stmt != nil: + properFreeResult(SQL_HANDLE_STMT, db.stmt) + properFreeResult(SQL_HANDLE_DBC, db.hDb) + properFreeResult(SQL_HANDLE_ENV, db.env) + raise e + +proc dbError*(msg: string) {.noreturn.} = + ## Raises an EDb exception with message `msg`. + var e: ref EDb + new(e) + e.msg = msg + raise e + +proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [EDb]} = + ## Wrapper that checks if ``resVal`` is not SQL_SUCCESS and if so, raises [EDb] + if resVal != SQL_SUCCESS: dbError(db) + +proc SqlGetDBMS(db: var DbConn): string {. + tags: [FReadODBC, FWriteODBC], raises: [] .} = + ## Returns the ODBC SQL_DBMS_NAME string + const + SQL_DBMS_NAME = 17.SqlUSmallInt + var + sz: TSqlSmallInt = 0 + buf[0] = '\0' + try: + db.SqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr), + 4095.TSqlSmallInt, sz.addr)) + except: discard + return $buf.cstring + +proc dbQuote*(s: string): string {.noSideEffect.} = + ## DB quotes the string. + result = "'" + for c in items(s): + if c == '\'': add(result, "''") + else: add(result, c) + add(result, '\'') + +proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. + noSideEffect.} = + ## Replace any ``?`` placeholders with `args`, + ## and quotes the arguments + result = "" + var a = 0 + for c in items(string(formatstr)): + if c == '?': + if args[a] == nil: + add(result, "NULL") + else: + add(result, dbQuote(args[a])) + inc(a) + else: + add(result, c) + +proc prepareFetch(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]) {. + tags: [FReadDb, FReadODBC, FWriteODBC], raises: [EDb].} = + # Prepare a statement, execute it and fetch the data to the driver + # ready for retrieval of the data + # Used internally by iterators and retrieval procs + # requires calling + # properFreeResult(SQL_HANDLE_STMT, db.stmt) + # when finished + db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) + var q = dbFormat(query, args) + db.SqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) + db.SqlCheck(SQLExecute(db.stmt)) + db.SqlCheck(SQLFetch(db.stmt)) + +proc prepareFetchDirect(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]) {. + tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + # Prepare a statement, execute it and fetch the data to the driver + # ready for retrieval of the data + # Used internally by iterators and retrieval procs + # requires calling + # properFreeResult(SQL_HANDLE_STMT, db.stmt) + # when finished + db.SqlCheck(SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt)) + var q = dbFormat(query, args) + db.SqlCheck(SQLExecDirect(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt)) + db.SqlCheck(SQLFetch(db.stmt)) + +proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. + tags: [FReadDB, FWriteDb, FReadODBC, FWriteODBC], raises: [].} = + ## Tries to execute the query and returns true if successful, false otherwise. + var + res:TSqlSmallInt = -1 + try: + db.prepareFetchDirect(query, args) + var + rCnt = -1 + res = SQLRowCount(db.stmt, rCnt) + if res != SQL_SUCCESS: dbError(db) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + except: discard + return res == SQL_SUCCESS + +proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. + tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + db.prepareFetchDirect(query, args) + +proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. + tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Executes the query and raises EDB if not successful. + db.prepareFetchDirect(query, args) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +proc newRow(L: int): Row {.noSideEFfect.} = + newSeq(result, L) + for i in 0..L-1: result[i] = "" + +iterator fastRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {. + tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Executes the query and iterates over the result dataset. + ## + ## This is very fast, but potentially dangerous. Use this iterator only + ## if you require **ALL** the rows. + ## + ## Breaking the fastRows() iterator during a loop may cause a driver error + ## for subsequenct queries + ## + ## Rows are retrieved from the server at each iteration. + var + rowRes: Row + sz: TSqlSmallInt = 0 + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + rowRes = newRow(cCnt) + for rNr in 1..rCnt: + for colId in 1..cCnt: + buf[0] = '\0' + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + rowRes[colId-1] = $buf.cstring + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + yield rowRes + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +iterator instantRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [FReadDb, FReadODBC, FWriteODBC].} = + ## Same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the interator body. + var + rowRes: Row + sz: TSqlSmallInt = 0 + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + rowRes = newRow(cCnt) + for rNr in 1..rCnt: + for colId in 1..cCnt: + buf[0] = '\0' + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + rowRes[colId-1] = $buf.cstring + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + yield (row: rowRes, len: cCnt.int) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +proc `[]`*(row: InstantRow, col: int): string {.inline.} = + ## Returns text for given column of the row + row.row[col] + +proc len*(row: InstantRow): int {.inline.} = + ## Returns number of columns in the row + row.len + +proc getRow*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {. + tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Retrieves a single row. If the query doesn't return any rows, this proc + ## will return a Row with empty strings for each column. + var + sz: TSqlSmallInt = 0.TSqlSmallInt + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + result = @[] + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + for colId in 1..cCnt: + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[cstring](buf.addr), 4095.TSqlSmallInt, sz.addr)) + result.add($buf.cstring) + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +proc getAllRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): seq[Row] {. + tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Executes the query and returns the whole result dataset. + var + rowRes: Row + sz: TSqlSmallInt = 0 + cCnt: TSqlSmallInt = 0.TSqlSmallInt + rCnt = -1 + db.prepareFetch(query, args) + db.SqlCheck(SQLNumResultCols(db.stmt, cCnt)) + db.SqlCheck(SQLRowCount(db.stmt, rCnt)) + result = @[] + for rNr in 1..rCnt: + rowRes = @[] + buf[0] = '\0' + for colId in 1..cCnt: + db.SqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR, + cast[SqlPointer](buf.addr), 4095.TSqlSmallInt, sz.addr)) + rowRes.add($buf.cstring) + db.SqlCheck(SQLFetchScroll(db.stmt, SQL_FETCH_NEXT, 1)) + result.add(rowRes) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + +iterator rows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): Row {. + tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Same as `fastRows`, but slower and safe. + ## + ## This retrieves ALL rows into memory before + ## iterating through the rows. + ## Large dataset queries will impact on memory usage. + for r in items(getAllRows(db, query, args)): yield r + +proc getValue*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): string {. + tags: [FReadDB, FReadODBC, FWriteODBC], raises: [].} = + ## Executes the query and returns the first column of the first row of the + ## result dataset. Returns "" if the dataset contains no rows or the database + ## value is NULL. + result = "" + try: + result = getRow(db, query, args)[0] + except: discard + +proc tryInsertId*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): int64 {. + tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [].} = + ## Executes the query (typically "INSERT") and returns the + ## generated ID for the row or -1 in case of an error. + if not tryExec(db, query, args): + result = -1'i64 + else: + echo "DBMS: ",SqlGetDBMS(db).toLower() + result = -1'i64 + try: + case SqlGetDBMS(db).toLower(): + of "postgresql": + result = getValue(db, sql"SELECT LASTVAL();", []).parseInt + of "mysql": + result = getValue(db, sql"SELECT LAST_INSERT_ID();", []).parseInt + of "sqlite": + result = getValue(db, sql"SELECT LAST_INSERT_ROWID();", []).parseInt + of "microsoft sql server": + result = getValue(db, sql"SELECT SCOPE_IDENTITY();", []).parseInt + of "oracle": + result = getValue(db, sql"SELECT id.currval FROM DUAL;", []).parseInt + else: result = -1'i64 + except: discard + +proc insertId*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): int64 {. + tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Executes the query (typically "INSERT") and returns the + ## generated ID for the row. + result = tryInsertID(db, query, args) + if result < 0: dbError(db) + +proc execAffectedRows*(db: var DbConn, query: SqlQuery, + args: varargs[string, `$`]): int64 {. + tags: [FReadDB, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + ## Runs the query (typically "UPDATE") and returns the + ## number of affected rows + result = -1 + var res = SQLAllocHandle(SQL_HANDLE_STMT, db.hDb, db.stmt.SqlHandle) + if res != SQL_SUCCESS: dbError(db) + var q = dbFormat(query, args) + res = SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt) + if res != SQL_SUCCESS: dbError(db) + rawExec(db, query, args) + var rCnt = -1 + result = SQLRowCount(db.hDb, rCnt) + if res != SQL_SUCCESS: dbError(db) + properFreeResult(SQL_HANDLE_STMT, db.stmt) + result = rCnt + +proc close*(db: var DbConn) {. + tags: [FWriteODBC], raises: [].} = + ## Closes the database connection. + if db.hDb != nil: + try: + var res = SQLDisconnect(db.hDb) + if db.stmt != nil: + res = SQLFreeHandle(SQL_HANDLE_STMT, db.stmt) + res = SQLFreeHandle(SQL_HANDLE_DBC, db.hDb) + res = SQLFreeHandle(SQL_HANDLE_ENV, db.env) + db = (hDb: nil, env: nil, stmt: nil) + except: + discard + +proc open*(connection, user, password, database: string): DbConn {. + tags: [FReadODBC, FWriteODBC], raises: [EDb].} = + ## Opens a database connection. + ## + ## Raises `EDb` if the connection could not be established. + ## + ## Currently the database parameter is ignored, + ## but included to match ``open()`` in the other db_xxxxx library modules. + var + val: TSqlInteger = SQL_OV_ODBC3 + resLen = 0 + result = (hDb: nil, env: nil, stmt: nil) + # allocate environment handle + var res = SQLAllocHandle(SQL_HANDLE_ENV, result.env, result.env) + if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.") + res = SQLSetEnvAttr(result.env, + SQL_ATTR_ODBC_VERSION.TSqlInteger, + val, resLen.TSqlInteger) + if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.") + # allocate hDb handle + res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb) + if res != SQL_SUCCESS: dbError("Error: unable to allocate connection handle.") + + # Connect: connection = dsn str, + res = SQLConnect(result.hDb, + connection.PSQLCHAR , connection.len.TSqlSmallInt, + user.PSQLCHAR, user.len.TSqlSmallInt, + password.PSQLCHAR, password.len.TSqlSmallInt) + if res != SQL_SUCCESS: + result.dbError() + +proc setEncoding*(connection: DbConn, encoding: string): bool {. + tags: [FReadODBC, FWriteODBC], raises: [EDb].} = + ## Currently not implemented for ODBC. + ## + ## Sets the encoding of a database connection, returns true for + ## success, false for failure. + #result = set_character_set(connection, encoding) == 0 + dbError("setEncoding() is currently not implemented by the db_odbc module") From e731777853b1b4402257b8edaad992878d90c5be Mon Sep 17 00:00:00 2001 From: JamesP Date: Wed, 9 Dec 2015 14:18:55 +1000 Subject: [PATCH 014/125] add SQLErr (mapped to SQLError in odbc lib) Add some SQLGetInfo consts Fix SQLSetEnvAttr parameter --- lib/wrappers/odbcsql.nim | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/wrappers/odbcsql.nim b/lib/wrappers/odbcsql.nim index 43ad80f76..1b2544ec0 100644 --- a/lib/wrappers/odbcsql.nim +++ b/lib/wrappers/odbcsql.nim @@ -641,11 +641,42 @@ const ODBC_CONFIG_SYS_DSN* = 5 ODBC_REMOVE_SYS_DSN* = 6 + SQL_ACTIVE_CONNECTIONS* = 0 # SQLGetInfo + SQL_DATA_SOURCE_NAME* = 2 + SQL_DATA_SOURCE_READ_ONLY* = 25 + SQL_DATABASE_NAME* = 2 + SQL_DBMS_NAME* = 17 + SQL_DBMS_VERSION* = 18 + SQL_DRIVER_HDBC* = 3 + SQL_DRIVER_HENV* = 4 + SQL_DRIVER_HSTMT* = 5 + SQL_DRIVER_NAME* = 6 + SQL_DRIVER_VER* = 7 + SQL_FETCH_DIRECTION* = 8 + SQL_ODBC_VER* = 10 + SQL_DRIVER_ODBC_VER* = 77 + SQL_SERVER_NAME* = 13 + SQL_ACTIVE_ENVIRONMENTS* = 116 + SQL_ACTIVE_STATEMENTS* = 1 + SQL_SQL_CONFORMANCE* = 118 + SQL_DATETIME_LITERALS* = 119 + SQL_ASYNC_MODE* = 10021 + SQL_BATCH_ROW_COUNT* = 120 + SQL_BATCH_SUPPORT* = 121 + SQL_CATALOG_LOCATION* = 114 + #SQL_CATALOG_NAME* = 10003 + SQL_CATALOG_NAME_SEPARATOR* = 41 + SQL_CATALOG_TERM* = 42 + SQL_CATALOG_USAGE* = 92 + #SQL_COLLATION_SEQ* = 10004 + SQL_COLUMN_ALIAS* = 87 + #SQL_USER_NAME* = 47 + proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle, OutputHandlePtr: var SqlHandle): TSqlSmallInt{. dynlib: odbclib, importc.} proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, - Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{. + Value: TSqlInteger, StringLength: TSqlInteger): TSqlSmallInt{. dynlib: odbclib, importc.} proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger, Value: SqlPointer, BufferLength: TSqlInteger, @@ -807,5 +838,10 @@ proc SQLStatistics*(hstmt: SqlHStmt, CatalogName: PSQLCHAR, NameLength3: TSqlSmallInt, Unique: SqlUSmallInt, Reserved: SqlUSmallInt): TSqlSmallInt {. dynlib: odbclib, importc.} +proc SQLErr*(henv: SqlHEnv, hdbc: SqlHDBC, hstmt: SqlHStmt, + szSqlState, pfNativeError, szErrorMsg: PSQLCHAR, + cbErrorMsgMax: TSqlSmallInt, + pcbErrorMsg: PSQLINTEGER): TSqlSmallInt {. + dynlib: odbclib, importc: "SQLError".} {.pop.} From bf874cff90b78078470b35a4e9b23c45735dbe4b Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 9 Dec 2015 22:32:58 +0200 Subject: [PATCH 015/125] Added a few notes regarding JavaScript --- doc/tut2.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/tut2.txt b/doc/tut2.txt index db9e4cd58..7479d8479 100644 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -990,3 +990,17 @@ generated by `treeRepr `_. If at the end of the this example you add ``echo treeRepr(result)`` you should get the same output as using the ``dumpTree`` macro, but of course you can call that at any point of the macro where you might be having troubles. + + +Compilation to JavaScript +========================= + +Nim code can be compiled to JavaScript. However in order to write +JavaScript-compatible code you should remember the following: +- ``addr`` and ``ptr`` have slightly different semantic meaning in JavaScript. + It is recommended to avoid those if you're not sure how they are translated + to JavaScript. +- ``cast[T](x)`` in JavaScript is translated to ``(x)``. +- ``cstring`` in JavaScript means JavaScript string. It is a good practice to + use ``cstring`` only when it is semantically appropriate. E.g. don't use + ``cstring`` as a binary data buffer. From 135668a7ba9ff9bc839a23cd29d08f3323c45530 Mon Sep 17 00:00:00 2001 From: qio-io Date: Sun, 13 Dec 2015 15:36:26 -0400 Subject: [PATCH 016/125] Fixing too many open files on quit command with an explicit socket close, and importing the net module --- lib/pure/redis.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim index e3f18a496..6741d3c8e 100644 --- a/lib/pure/redis.nim +++ b/lib/pure/redis.nim @@ -14,7 +14,7 @@ ## return ``redisNil``, and functions which return a ``RedisList`` ## may return ``nil``. -import sockets, os, strutils, parseutils +import net, os, strutils, parseutils const redisNil* = "\0\0" @@ -31,7 +31,7 @@ type type Redis* = object - socket: Socket + socket: net.Socket connected: bool pipeline: Pipeline @@ -55,9 +55,8 @@ proc newPipeline(): Pipeline = proc open*(host = "localhost", port = 6379.Port): Redis = ## Opens a connection to the redis server. - result.socket = socket(buffered = false) - if result.socket == invalidSocket: - raiseOSError(osLastError()) + result.socket = newSocket(buffered = false) + result.socket.connect(host, port) result.pipeline = newPipeline() @@ -923,6 +922,7 @@ proc quit*(r: Redis) = ## Close the connection r.sendCommand("QUIT") raiseNoOK(r.readStatus(), r.pipeline.enabled) + r.socket.close() proc select*(r: Redis, index: int): RedisStatus = ## Change the selected database for the current connection From bd8e89774e075aab66bb06df36486cf965afc464 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 14 Dec 2015 18:40:47 +0100 Subject: [PATCH 017/125] fixes a critical bug concerning a[^1] rewritings --- compiler/semmagic.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index deef38ae3..f98ff0266 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -207,7 +207,7 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode, result = n.sons[1] else: result = newNodeIT(nkCall, n.info, getSysType(tyInt)) - result.add newSymNode(createMagic("-", mSubI), n.info) + result.add newSymNode(getSysMagic("-", mSubI), n.info) result.add lenExprB result.add n.sons[1] of mPlugin: From 31ebb11d73f7ad58afc497f20d9723503b83ca75 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 15 Dec 2015 00:41:03 +0100 Subject: [PATCH 018/125] disabled appveyor tests since they die with a timeout --- appveyor.yml | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 2b1eab5d0..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,36 +0,0 @@ -clone_depth: 5 - -artifacts: - - path: bin\nim.exe - -platform: - - x64 - -before_build: - - git log -1 - - C:\msys64\usr\bin\bash -lc "pacman --noconfirm -S zlib-devel" - - appveyor DownloadFile http://nim-lang.org/download/dlls.zip - - 7z e dlls.zip -odlls - - del dlls\libcurl.dll - - appveyor DownloadFile http://flatassembler.net/fasmw17139.zip - - 7z e fasmw17139.zip -obin fasm.exe - -build_script: - - SET PATH=C:\msys64\mingw64\bin;dlls;bin;%PATH% - - gcc -v - - git clone -q --depth 1 https://github.com/nim-lang/csources - - cd csources - - build64.bat - - cd .. - - nim c koch - - koch boot - - koch boot -d:release - -before_test: - - nim e install_nimble.nims - - nimble update - - nimble install zip - -test_script: - - nim c --taintMode:on tests/testament/tester - - tests\testament\tester --pedantic all From f664c576c2aab2b2d7f72693bff5a106eb6ef794 Mon Sep 17 00:00:00 2001 From: Reimer Behrends Date: Tue, 15 Dec 2015 13:33:38 +0100 Subject: [PATCH 019/125] Fix code generation issues with TNimType. This is an alternative attempt to fix issue #2841 in a way that does not rely on non-standard C compiler behavior. The underlying problem is that the typeinfo module includes system/hti.nim, but system/hti.nim is also included by system.nim. The result is that the compiler at any given point has two instances of TNimType with the same external name and will generate code either for one or both of them. Code generation for the second version is normally suppressed by using {.importc.}, but that results in a missing type declaration for modules that only use the second version. This happens for modules that import typeinfo, because the the second component of typeinfo.Any has type PNimType = ptr TNimType, which generates a "TNimType *" struct member. The previous solution was to have a typedef for TNimType in nimbase.h; however, this results in duplicate typedefs (specifically, but not only for the code generated for typeinfo.nim). Duplicate typedefs with the same type name are allowed in C++ and C11, but are explicitly forbidden in C89 and C99. Modern versions of gcc and clang accept them anyway, but this is not standard-compliant and may break for other C compilers or older versions of gcc and clang. This patch attempts to fix the issue by making the second component of typeinfo.Nim a pointer instead of a PNimType and casting back and forth as needed. Thus, the second version of TNimType occurs only within the code for typeinfo.nim, where it can be safely omitted. --- lib/core/typeinfo.nim | 20 +++++++++++++++----- lib/nimbase.h | 4 ---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 1f9fb1072..db5a83755 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -61,7 +61,10 @@ type ## wrapped value and **must not** live longer than ## its wrapped value. value: pointer - rawType: PNimType + when defined(js): + rawType: PNimType + else: + rawTypePtr: pointer ppointer = ptr pointer pbyteArray = ptr array[0.. 0xffff, int8] @@ -71,6 +74,14 @@ type when defined(gogc): elemSize: int PGenSeq = ptr TGenericSeq + +when not defined(js): + template rawType(x: Any): PNimType = + cast[PNimType](x.rawTypePtr) + + template `rawType=`(x: var Any, p: PNimType) = + x.rawTypePtr = cast[pointer](p) + {.deprecated: [TAny: Any, TAnyKind: AnyKind].} when defined(gogc): @@ -108,7 +119,7 @@ proc selectBranch(aa: pointer, n: ptr TNimNode): ptr TNimNode = else: result = n.sons[n.len] -proc newAny(value: pointer, rawType: PNimType): Any = +proc newAny(value: pointer, rawType: PNimType): Any {.inline.} = result.value = value result.rawType = rawType @@ -126,8 +137,7 @@ proc toAny*[T](x: var T): Any {.inline.} = ## constructs a ``Any`` object from `x`. This captures `x`'s address, so ## `x` can be modified with its ``Any`` wrapper! The client needs to ensure ## that the wrapper **does not** live longer than `x`! - result.value = addr(x) - result.rawType = cast[PNimType](getTypeInfo(x)) + newAny(addr(x), cast[PNimType](getTypeInfo(x))) proc kind*(x: Any): AnyKind {.inline.} = ## get the type kind @@ -345,7 +355,7 @@ proc `[]`*(x: Any, fieldName: string): Any = result.value = x.value +!! n.offset result.rawType = n.typ elif x.rawType.kind == tyObject and x.rawType.base != nil: - return `[]`(Any(value: x.value, rawType: x.rawType.base), fieldName) + return `[]`(newAny(x.value, x.rawType.base), fieldName) else: raise newException(ValueError, "invalid field name: " & fieldName) diff --git a/lib/nimbase.h b/lib/nimbase.h index 0946b9a1f..bba5ac023 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -418,10 +418,6 @@ typedef int assert_numbits[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof( # define NIM_EXTERNC #endif -/* we have to tinker with TNimType as it's both part of system.nim and - typeinfo.nim but system.nim doesn't export it cleanly... */ -typedef struct TNimType TNimType; - /* ---------------- platform specific includes ----------------------- */ /* VxWorks related includes */ From 62422b93c62b5cbd83216369a00ba9ecf1311c0b Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 16 Dec 2015 15:25:58 +0200 Subject: [PATCH 020/125] Fixed copying of nil seq in JS --- lib/system/jssys.nim | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 54c6796c9..5bac54772 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -533,15 +533,20 @@ proc nimCopy(dest, src: JSRef, ti: PNimType): JSRef = nimCopyAux(result, src, ti.node) of tySequence, tyArrayConstr, tyOpenArray, tyArray: asm """ - if (`dest` === null || `dest` === undefined) { - `dest` = new Array(`src`.length); + if (`src` === null) { + `result` = null; } else { - `dest`.length = `src`.length; - } - `result` = `dest`; - for (var i = 0; i < `src`.length; ++i) { - `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + if (`dest` === null || `dest` === undefined) { + `dest` = new Array(`src`.length); + } + else { + `dest`.length = `src`.length; + } + `result` = `dest`; + for (var i = 0; i < `src`.length; ++i) { + `result`[i] = nimCopy(`result`[i], `src`[i], `ti`.base); + } } """ of tyString: From cb10f306561319f3bc604dee924a5af7639d6434 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 16 Dec 2015 19:18:59 +0100 Subject: [PATCH 021/125] big update for the db*.nim modules; uses new db_common.nim --- lib/impure/db_mysql.nim | 74 +++++++++++------------------ lib/impure/db_postgres.nim | 87 +++++++++++++++------------------- lib/impure/db_sqlite.nim | 74 +++++++++++------------------ lib/pure/db_common.nim | 96 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 143 deletions(-) create mode 100644 lib/pure/db_common.nim diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 7f7511264..ca99636f6 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -43,6 +43,9 @@ import strutils, mysql +import db_common +export db_common + type DbConn* = PMySQL ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be @@ -50,38 +53,15 @@ type InstantRow* = tuple[row: cstringArray, len: int] ## a handle that can be ## used to get a row's ## column text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs +{.deprecated: [TRow: Row, TDbConn: DbConn].} - SqlQuery* = distinct string ## an SQL query string - - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} - -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) - -proc dbError(db: DbConn) {.noreturn.} = - ## raises an EDb exception. - var e: ref EDb +proc dbError*(db: DbConn) {.noreturn.} = + ## raises a DbError exception. + var e: ref DbError new(e) e.msg = $mysql.error(db) raise e -proc dbError*(msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - when false: proc dbQueryOpt*(db: DbConn, query: string, args: varargs[string, `$`]) = var stmt = mysql_stmt_init(db) @@ -114,7 +94,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = add(result, c) proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) return mysql.realQuery(db, q, q.len) == 0'i32 @@ -124,7 +104,7 @@ proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) = if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var q = dbFormat(query, args) if mysql.realQuery(db, q, q.len) != 0'i32: dbError(db) @@ -139,7 +119,7 @@ proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) = mysql.freeResult(sqlres) iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the query and iterates over the result dataset. ## ## This is very fast, but potentially dangerous. Use this iterator only @@ -167,9 +147,9 @@ iterator fastRows*(db: DbConn, query: SqlQuery, iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = - ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the interator body. + {.tags: [ReadDbEffect].} = + ## Same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the iterator body. rawExec(db, query, args) var sqlres = mysql.useResult(db) if sqlres != nil: @@ -182,16 +162,16 @@ iterator instantRows*(db: DbConn, query: SqlQuery, properFreeResult(sqlres, row) proc `[]`*(row: InstantRow, col: int): string {.inline.} = - ## returns text for given column of the row + ## Returns text for given column of the row. $row.row[col] proc len*(row: InstantRow): int {.inline.} = - ## returns number of columns in the row + ## Returns number of columns in the row. row.len proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = - ## retrieves a single row. If the query doesn't return any rows, this proc + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = + ## Retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. rawExec(db, query, args) var sqlres = mysql.useResult(db) @@ -209,7 +189,7 @@ proc getRow*(db: DbConn, query: SqlQuery, properFreeResult(sqlres, row) proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} = + args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. result = @[] rawExec(db, query, args) @@ -232,19 +212,19 @@ proc getAllRows*(db: DbConn, query: SqlQuery, mysql.freeResult(sqlres) iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, query, args)): yield r proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDB].} = + args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. result = getRow(db, query, args)[0] proc tryInsertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) @@ -254,7 +234,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery, result = mysql.insertId(db) proc insertId*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. result = tryInsertID(db, query, args) @@ -262,18 +242,18 @@ proc insertId*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## runs the query (typically "UPDATE") and returns the ## number of affected rows rawExec(db, query, args) result = mysql.affectedRows(db) -proc close*(db: DbConn) {.tags: [FDb].} = +proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. if db != nil: mysql.close(db) proc open*(connection, user, password, database: string): DbConn {. - tags: [FDb].} = + tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. result = mysql.init(nil) @@ -291,7 +271,7 @@ proc open*(connection, user, password, database: string): DbConn {. dbError(errmsg) proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FDb].} = + tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. result = mysql.set_character_set(connection, encoding) == 0 diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 5603d9686..802efb71f 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -62,47 +62,28 @@ ## "Dominik") import strutils, postgres +import db_common +export db_common + type DbConn* = PPGconn ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## transformed always to the empty string. InstantRow* = tuple[res: PPGresult, line: int32] ## a handle that can be ## used to get a row's ## column text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs - - SqlQuery* = distinct string ## an SQL query string SqlPrepared* = distinct string ## a identifier for the prepared queries - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn, +{.deprecated: [TRow: Row, TDbConn: DbConn, TSqlPrepared: SqlPrepared].} -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) - proc dbError*(db: DbConn) {.noreturn.} = - ## raises an EDb exception. - var e: ref EDb + ## raises a DbError exception. + var e: ref DbError new(e) e.msg = $pqErrorMessage(db) raise e -proc dbError*(msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - proc dbQuote*(s: string): string = ## DB quotes the string. result = "'" @@ -127,7 +108,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = add(result, c) proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} = + args: varargs[string, `$`]): bool {.tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, nil, nil, 0) @@ -135,7 +116,8 @@ proc tryExec*(db: DbConn, query: SqlQuery, pqclear(res) proc tryExec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): bool {.tags: [FReadDB, FWriteDb].} = + args: varargs[string, `$`]): bool {.tags: [ + ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var arr = allocCStringArray(args) var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, @@ -145,7 +127,7 @@ proc tryExec*(db: DbConn, stmtName: SqlPrepared, pqclear(res) proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDB, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query and raises EDB if not successful. var res = pqexecParams(db, dbFormat(query, args), 0, nil, nil, nil, nil, 0) @@ -153,7 +135,7 @@ proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. pqclear(res) proc exec*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string]) {.tags: [FReadDB, FWriteDb].} = + args: varargs[string]) {.tags: [ReadDbEffect, WriteDbEffect].} = var arr = allocCStringArray(args) var res = pqexecPrepared(db, stmtName.string, int32(args.len), arr, nil, nil, 0) @@ -196,7 +178,7 @@ proc setRow(res: PPGresult, r: var Row, line, cols: int32) = add(r[col], x) iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the query and iterates over the result dataset. This is very ## fast, but potenially dangerous: If the for-loop-body executes another ## query, the results can be undefined. For Postgres it is safe though. @@ -209,7 +191,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery, pqclear(res) iterator fastRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## executes the prepared query and iterates over the result dataset. var res = setupQuery(db, stmtName, args) var L = pqNfields(res) @@ -221,9 +203,9 @@ iterator fastRows*(db: DbConn, stmtName: SqlPrepared, iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = + {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within interator body. + ## on demand using []. Returned handle is valid only within iterator body. var res = setupQuery(db, query, args) for i in 0..pqNtuples(res)-1: yield (res: res, line: i) @@ -231,9 +213,9 @@ iterator instantRows*(db: DbConn, query: SqlQuery, iterator instantRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = + {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within interator body. + ## on demand using []. Returned handle is valid only within iterator body. var res = setupQuery(db, stmtName, args) for i in 0..pqNtuples(res)-1: yield (res: res, line: i) @@ -248,7 +230,7 @@ proc len*(row: InstantRow): int32 {.inline.} = pqNfields(row.res) proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. var res = setupQuery(db, query, args) @@ -258,7 +240,7 @@ proc getRow*(db: DbConn, query: SqlQuery, pqclear(res) proc getRow*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = var res = setupQuery(db, stmtName, args) var L = pqNfields(res) result = newRow(L) @@ -266,31 +248,34 @@ proc getRow*(db: DbConn, stmtName: SqlPrepared, pqClear(res) proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} = + args: varargs[string, `$`]): seq[Row] {. + tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. result = @[] for r in fastRows(db, query, args): result.add(r) proc getAllRows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDB].} = + args: varargs[string, `$`]): seq[Row] {.tags: + [ReadDbEffect].} = ## executes the prepared query and returns the whole result dataset. result = @[] for r in fastRows(db, stmtName, args): result.add(r) iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, query, args)): yield r iterator rows*(db: DbConn, stmtName: SqlPrepared, - args: varargs[string, `$`]): Row {.tags: [FReadDB].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `fastRows`, but slower and safe. for r in items(getAllRows(db, stmtName, args)): yield r proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDB].} = + args: varargs[string, `$`]): string {. + tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. @@ -298,7 +283,8 @@ proc getValue*(db: DbConn, query: SqlQuery, result = if isNil(x): "" else: $x proc tryInsertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].}= + args: varargs[string, `$`]): int64 {. + tags: [WriteDbEffect].}= ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is @@ -311,7 +297,8 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, result = -1 proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {. + tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is @@ -321,7 +308,7 @@ proc insertID*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {.tags: [ - FReadDB, FWriteDb].} = + ReadDbEffect, WriteDbEffect].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. var q = dbFormat(query, args) @@ -332,7 +319,7 @@ proc execAffectedRows*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared, args: varargs[string, `$`]): int64 {.tags: [ - FReadDB, FWriteDb].} = + ReadDbEffect, WriteDbEffect].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. var arr = allocCStringArray(args) @@ -343,12 +330,12 @@ proc execAffectedRows*(db: DbConn, stmtName: SqlPrepared, result = parseBiggestInt($pqcmdTuples(res)) pqclear(res) -proc close*(db: DbConn) {.tags: [FDb].} = +proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. if db != nil: pqfinish(db) proc open*(connection, user, password, database: string): DbConn {. - tags: [FDb].} = + tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. ## @@ -370,7 +357,7 @@ proc open*(connection, user, password, database: string): DbConn {. if pqStatus(result) != CONNECTION_OK: dbError(result) # result = nil proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FDb].} = + tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. return pqsetClientEncoding(connection, encoding) == 0 diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 8366fdadc..90a2f8728 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -42,45 +42,26 @@ import strutils, sqlite3 +import db_common +export db_common + type DbConn* = PSqlite3 ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be ## transformed always to the empty string. InstantRow* = Pstmt ## a handle that can be used to get a row's column ## text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs +{.deprecated: [TRow: Row, TDbConn: DbConn].} - SqlQuery* = distinct string ## an SQL query string - - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation -{.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} - -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) - -proc dbError(db: DbConn) {.noreturn.} = - ## raises an EDb exception. - var e: ref EDb +proc dbError*(db: DbConn) {.noreturn.} = + ## raises a DbError exception. + var e: ref DbError new(e) e.msg = $sqlite3.errmsg(db) raise e -proc dbError*(msg: string) {.noreturn.} = - ## raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - -proc dbQuote(s: string): string = +proc dbQuote*(s: string): string = + ## DB quotes the string. if s.isNil: return "NULL" result = "'" for c in items(s): @@ -99,7 +80,8 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string = add(result, c) proc tryExec*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): bool {.tags: [FReadDb, FWriteDb].} = + args: varargs[string, `$`]): bool {. + tags: [ReadDbEffect, WriteDbEffect].} = ## tries to execute the query and returns true if successful, false otherwise. var q = dbFormat(query, args) var stmt: sqlite3.Pstmt @@ -108,8 +90,8 @@ proc tryExec*(db: DbConn, query: SqlQuery, result = finalize(stmt) == SQLITE_OK proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDb, FWriteDb].} = - ## executes the query and raises EDB if not successful. + tags: [ReadDbEffect, WriteDbEffect].} = + ## executes the query and raises DbError if not successful. if not tryExec(db, query, args): dbError(db) proc newRow(L: int): Row = @@ -129,14 +111,14 @@ proc setRow(stmt: Pstmt, r: var Row, cols: cint) = if not isNil(x): add(r[col], x) iterator fastRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDb].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## Executes the query and iterates over the result dataset. ## ## This is very fast, but potentially dangerous. Use this iterator only ## if you require **ALL** the rows. ## ## Breaking the fastRows() iterator during a loop will cause the next - ## database query to raise an [EDb] exception ``unable to close due to ...``. + ## database query to raise a DbError exception ``unable to close due to ...``. var stmt = setupQuery(db, query, args) var L = (column_count(stmt)) var result = newRow(L) @@ -147,9 +129,9 @@ iterator fastRows*(db: DbConn, query: SqlQuery, iterator instantRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb].} = + {.tags: [ReadDbEffect].} = ## same as fastRows but returns a handle that can be used to get column text - ## on demand using []. Returned handle is valid only within the interator body. + ## on demand using []. Returned handle is valid only within the iterator body. var stmt = setupQuery(db, query, args) while step(stmt) == SQLITE_ROW: yield stmt @@ -164,7 +146,7 @@ proc len*(row: InstantRow): int32 {.inline.} = column_count(row) proc getRow*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDb].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. var stmt = setupQuery(db, query, args) @@ -175,19 +157,19 @@ proc getRow*(db: DbConn, query: SqlQuery, if finalize(stmt) != SQLITE_OK: dbError(db) proc getAllRows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): seq[Row] {.tags: [FReadDb].} = + args: varargs[string, `$`]): seq[Row] {.tags: [ReadDbEffect].} = ## executes the query and returns the whole result dataset. result = @[] for r in fastRows(db, query, args): result.add(r) iterator rows*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): Row {.tags: [FReadDb].} = + args: varargs[string, `$`]): Row {.tags: [ReadDbEffect].} = ## same as `FastRows`, but slower and safe. for r in fastRows(db, query, args): yield r proc getValue*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): string {.tags: [FReadDb].} = + args: varargs[string, `$`]): string {.tags: [ReadDbEffect].} = ## executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. @@ -205,7 +187,7 @@ proc getValue*(db: DbConn, query: SqlQuery, proc tryInsertID*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 - {.tags: [FWriteDb], raises: [].} = + {.tags: [WriteDbEffect], raises: [].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. var q = dbFormat(query, args) @@ -218,7 +200,7 @@ proc tryInsertID*(db: DbConn, query: SqlQuery, result = -1 proc insertID*(db: DbConn, query: SqlQuery, - args: varargs[string, `$`]): int64 {.tags: [FWriteDb].} = + args: varargs[string, `$`]): int64 {.tags: [WriteDbEffect].} = ## executes the query (typically "INSERT") and returns the ## generated ID for the row. For Postgre this adds ## ``RETURNING id`` to the query, so it only works if your primary key is @@ -228,18 +210,18 @@ proc insertID*(db: DbConn, query: SqlQuery, proc execAffectedRows*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDb, FWriteDb].} = + tags: [ReadDbEffect, WriteDbEffect].} = ## executes the query (typically "UPDATE") and returns the ## number of affected rows. exec(db, query, args) result = changes(db) -proc close*(db: DbConn) {.tags: [FDb].} = +proc close*(db: DbConn) {.tags: [DbEffect].} = ## closes the database connection. if sqlite3.close(db) != SQLITE_OK: dbError(db) proc open*(connection, user, password, database: string): DbConn {. - tags: [FDb].} = + tags: [DbEffect].} = ## opens a database connection. Raises `EDb` if the connection could not ## be established. Only the ``connection`` parameter is used for ``sqlite``. var db: DbConn @@ -249,7 +231,7 @@ proc open*(connection, user, password, database: string): DbConn {. dbError(db) proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FDb].} = + tags: [DbEffect].} = ## sets the encoding of a database connection, returns true for ## success, false for failure. ## diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim new file mode 100644 index 000000000..522d13db2 --- /dev/null +++ b/lib/pure/db_common.nim @@ -0,0 +1,96 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Common datatypes and definitions for all db_*.nim modules. + +type + DbError* = object of IOError ## exception that is raised if a database error occurs + + SqlQuery* = distinct string ## an SQL query string + + + DbEffect* = object of IOEffect ## effect that denotes a database operation + ReadDbEffect* = object of DbEffect ## effect that denotes a read operation + WriteDbEffect* = object of DbEffect ## effect that denotes a write operation + + DbTypeKind* = enum ## a superset of datatypes that might be supported. + dbUnknown, ## unknown datatype + dbSerial, ## datatype used for primary auto-increment keys + dbNull, ## datatype used for the NULL value + dbBit, ## bit datatype + dbBool, ## boolean datatype + dbBlob, ## blob datatype + dbFixedChar, ## string of fixed length + dbVarchar, ## string datatype + dbJson, ## JSON datatype + dbXml, ## XML datatype + dbInt, ## some integer type + dbUInt, ## some unsigned integer type + dbDecimal, ## decimal numbers (fixed-point number) + dbFloat, ## some floating point type + dbDate, ## a year-month-day description + dbTime, ## HH:MM:SS information + dbDatetime, ## year-month-day and HH:MM:SS information + dbTimestamp, ## Timestamp values are stored as the number of seconds + ## since the epoch ('1970-01-01 00:00:00' UTC). + dbTimeInterval, ## an interval [a,b] of times + dbEnum, ## some enum + dbSet, ## set of enum values + dbArray, ## an array of values + dbComposite, ## composite type (record, struct, etc) + dbUrl, ## a URL + dbUuid, ## a UUID + dbInet, ## an IP address + dbMacAddress, ## a MAC address + dbPoint, ## Point on a plane (x,y) + dbLine, ## Infinite line ((x1,y1),(x2,y2)) + dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) + dbBox, ## Rectangular box ((x1,y1),(x2,y2)) + dbPath, ## Closed or open path (similar to polygon) ((x1,y1),...) + dbPolygon, ## Polygon (similar to closed path) ((x1,y1),...) + dbCircle, ## Circle <(x,y),r> (center point and radius) + dbUser1, ## user definable datatype 1 (for unknown extensions) + dbUser2, ## user definable datatype 2 (for unknown extensions) + dbUser3, ## user definable datatype 3 (for unknown extensions) + dbUser4, ## user definable datatype 4 (for unknown extensions) + dbUser5 ## user definable datatype 5 (for unknown extensions) + + DbType* = object ## describes a database type + kind*: DbTypeKind ## the kind of the described type + notNull*: bool ## does the type contain NULL? + name*: string ## the name of the type + size*: Natural ## the size of the datatype; 0 if of variable size + precision*, scale*: Natural ## precision and scale of the number + min*, max*: BiggestInt ## the minimum and maximum of allowed values + validValues*: seq[string] ## valid values of an enum or a set + + DbColumn* = object ## information about a database column + name*: string ## name of the column + tableName*: string ## name of the table the column belongs to (optional) + typ*: DbType ## type of the column + DbColumns* = seq[DbColumn] + +{.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect, + FReadDb: ReadDbEffect, FWriteDb: WriteDbEffect].} + +template sql*(query: string): SqlQuery = + ## constructs a SqlQuery from the string `query`. This is supposed to be + ## used as a raw-string-literal modifier: + ## ``sql"update user set counter = counter + 1"`` + ## + ## If assertions are turned off, it does nothing. If assertions are turned + ## on, later versions will check the string for valid syntax. + SqlQuery(query) + +proc dbError*(msg: string) {.noreturn, noinline.} = + ## raises an DbError exception with message `msg`. + var e: ref DbError + new(e) + e.msg = msg + raise e From a3c8bb93760e258de406572f6181aa18fc88e7c3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 16 Dec 2015 21:24:13 +0100 Subject: [PATCH 022/125] updated db*.nim modules --- lib/impure/db_mysql.nim | 2 +- lib/impure/db_odbc.nim | 83 +++++++++++++++----------------------- lib/impure/db_postgres.nim | 2 +- lib/impure/db_sqlite.nim | 2 +- 4 files changed, 36 insertions(+), 53 deletions(-) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index ca99636f6..0a57f8a41 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -49,7 +49,7 @@ export db_common type DbConn* = PMySQL ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## converted to nil. InstantRow* = tuple[row: cstringArray, len: int] ## a handle that can be ## used to get a row's ## column text on demand diff --git a/lib/impure/db_odbc.nim b/lib/impure/db_odbc.nim index d7038303c..6af69d842 100644 --- a/lib/impure/db_odbc.nim +++ b/lib/impure/db_odbc.nim @@ -1,8 +1,12 @@ # +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# # See the file "copying.txt", included in this # distribution, for details about the copyright. # -# + ## A higher level `ODBC` database wrapper. ## ## This is the same interface that is implemented for other databases. @@ -45,46 +49,32 @@ import strutils, odbcsql +import db_common +export db_common + type OdbcConnTyp = tuple[hDb: SqlHDBC, env: SqlHEnv, stmt: SqlHStmt] DbConn* = OdbcConnTyp ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## converted to nil. InstantRow* = tuple[row: seq[string], len: int] ## a handle that can be ## used to get a row's ## column text on demand - EDb* = object of IOError ## exception that is raised if a database error occurs - SqlQuery* = distinct string ## an SQL query string - - FDb* = object of IOEffect ## effect that denotes a database operation - FReadDb* = object of FDb ## effect that denotes a read operation - FWriteDb* = object of FDb ## effect that denotes a write operation - FReadODBC* = object of FDb ## effect that denotes a read of ODBC driver - FWriteODBC* = object of FDb ## effect that denotes a write of ODBC driver {.deprecated: [TRow: Row, TSqlQuery: SqlQuery, TDbConn: DbConn].} var buf: array[0..4096, char] proc properFreeResult(hType: int, sqlres: var SqlHandle) {. - tags: [FWriteODBC], raises: [].} = + tags: [WriteDbEffect], raises: [].} = try: discard SQLFreeHandle(hType.TSqlSmallInt, sqlres) sqlres = nil except: discard -proc sql*(query: string): SqlQuery {.noSideEffect, inline.} = - ## Constructs a SqlQuery from the string `query`. This is supposed to be - ## used as a raw-string-literal modifier: - ## ``sql"update user set counter = counter + 1"`` - ## - ## If assertions are turned off, it does nothing. If assertions are turned - ## on, later versions will check the string for valid syntax. - result = SqlQuery(query) - proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. - tags: [FReadODBC], raises: [].} = + tags: [ReadDbEffect], raises: [].} = ## Returns ODBC error information var sqlState: array[0..512, char] @@ -106,10 +96,10 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {. return (res.int, $sqlState, $nativeErr, $errMsg) proc dbError*(db: var DbConn) {. - tags: [FReadODBC,FWriteODBC], raises: [EDb] .} = - ## Raises an `[EDb]` exception with ODBC error information + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} = + ## Raises an `[DbError]` exception with ODBC error information var - e: ref EDb + e: ref DbError ss, ne, msg: string = "" isAnError = false res: int = 0 @@ -137,19 +127,12 @@ proc dbError*(db: var DbConn) {. properFreeResult(SQL_HANDLE_ENV, db.env) raise e -proc dbError*(msg: string) {.noreturn.} = - ## Raises an EDb exception with message `msg`. - var e: ref EDb - new(e) - e.msg = msg - raise e - -proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [EDb]} = +proc SqlCheck(db: var DbConn, resVal: TSqlSmallInt) {.raises: [DbError]} = ## Wrapper that checks if ``resVal`` is not SQL_SUCCESS and if so, raises [EDb] if resVal != SQL_SUCCESS: dbError(db) proc SqlGetDBMS(db: var DbConn): string {. - tags: [FReadODBC, FWriteODBC], raises: [] .} = + tags: [ReadDbEffect, WriteDbEffect], raises: [] .} = ## Returns the ODBC SQL_DBMS_NAME string const SQL_DBMS_NAME = 17.SqlUSmallInt @@ -188,7 +171,7 @@ proc dbFormat(formatstr: SqlQuery, args: varargs[string]): string {. proc prepareFetch(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDb, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = # Prepare a statement, execute it and fetch the data to the driver # ready for retrieval of the data # Used internally by iterators and retrieval procs @@ -203,7 +186,7 @@ proc prepareFetch(db: var DbConn, query: SqlQuery, proc prepareFetchDirect(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = # Prepare a statement, execute it and fetch the data to the driver # ready for retrieval of the data # Used internally by iterators and retrieval procs @@ -216,7 +199,7 @@ proc prepareFetchDirect(db: var DbConn, query: SqlQuery, db.SqlCheck(SQLFetch(db.stmt)) proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {. - tags: [FReadDB, FWriteDb, FReadODBC, FWriteODBC], raises: [].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [].} = ## Tries to execute the query and returns true if successful, false otherwise. var res:TSqlSmallInt = -1 @@ -231,11 +214,11 @@ proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool return res == SQL_SUCCESS proc rawExec(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = db.prepareFetchDirect(query, args) proc exec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]) {. - tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Executes the query and raises EDB if not successful. db.prepareFetchDirect(query, args) properFreeResult(SQL_HANDLE_STMT, db.stmt) @@ -246,7 +229,7 @@ proc newRow(L: int): Row {.noSideEFfect.} = iterator fastRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {. - tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Executes the query and iterates over the result dataset. ## ## This is very fast, but potentially dangerous. Use this iterator only @@ -278,7 +261,7 @@ iterator fastRows*(db: var DbConn, query: SqlQuery, iterator instantRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): InstantRow - {.tags: [FReadDb, FReadODBC, FWriteODBC].} = + {.tags: [ReadDbEffect, WriteDbEffect].} = ## Same as fastRows but returns a handle that can be used to get column text ## on demand using []. Returned handle is valid only within the interator body. var @@ -310,7 +293,7 @@ proc len*(row: InstantRow): int {.inline.} = proc getRow*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {. - tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Retrieves a single row. If the query doesn't return any rows, this proc ## will return a Row with empty strings for each column. var @@ -331,7 +314,7 @@ proc getRow*(db: var DbConn, query: SqlQuery, proc getAllRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): seq[Row] {. - tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Executes the query and returns the whole result dataset. var rowRes: Row @@ -355,7 +338,7 @@ proc getAllRows*(db: var DbConn, query: SqlQuery, iterator rows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): Row {. - tags: [FReadDB, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Same as `fastRows`, but slower and safe. ## ## This retrieves ALL rows into memory before @@ -365,7 +348,7 @@ iterator rows*(db: var DbConn, query: SqlQuery, proc getValue*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): string {. - tags: [FReadDB, FReadODBC, FWriteODBC], raises: [].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [].} = ## Executes the query and returns the first column of the first row of the ## result dataset. Returns "" if the dataset contains no rows or the database ## value is NULL. @@ -376,7 +359,7 @@ proc getValue*(db: var DbConn, query: SqlQuery, proc tryInsertId*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row or -1 in case of an error. if not tryExec(db, query, args): @@ -401,7 +384,7 @@ proc tryInsertId*(db: var DbConn, query: SqlQuery, proc insertId*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDb, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Executes the query (typically "INSERT") and returns the ## generated ID for the row. result = tryInsertID(db, query, args) @@ -409,7 +392,7 @@ proc insertId*(db: var DbConn, query: SqlQuery, proc execAffectedRows*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): int64 {. - tags: [FReadDB, FWriteDb, FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Runs the query (typically "UPDATE") and returns the ## number of affected rows result = -1 @@ -426,7 +409,7 @@ proc execAffectedRows*(db: var DbConn, query: SqlQuery, result = rCnt proc close*(db: var DbConn) {. - tags: [FWriteODBC], raises: [].} = + tags: [WriteDbEffect], raises: [].} = ## Closes the database connection. if db.hDb != nil: try: @@ -440,7 +423,7 @@ proc close*(db: var DbConn) {. discard proc open*(connection, user, password, database: string): DbConn {. - tags: [FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Opens a database connection. ## ## Raises `EDb` if the connection could not be established. @@ -471,7 +454,7 @@ proc open*(connection, user, password, database: string): DbConn {. result.dbError() proc setEncoding*(connection: DbConn, encoding: string): bool {. - tags: [FReadODBC, FWriteODBC], raises: [EDb].} = + tags: [ReadDbEffect, WriteDbEffect], raises: [DbError].} = ## Currently not implemented for ODBC. ## ## Sets the encoding of a database connection, returns true for diff --git a/lib/impure/db_postgres.nim b/lib/impure/db_postgres.nim index 802efb71f..9bdbae4c2 100644 --- a/lib/impure/db_postgres.nim +++ b/lib/impure/db_postgres.nim @@ -68,7 +68,7 @@ export db_common type DbConn* = PPGconn ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## converted to nil. InstantRow* = tuple[res: PPGresult, line: int32] ## a handle that can be ## used to get a row's ## column text on demand diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 90a2f8728..17f031a52 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -48,7 +48,7 @@ export db_common type DbConn* = PSqlite3 ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be - ## transformed always to the empty string. + ## converted to nil. InstantRow* = Pstmt ## a handle that can be used to get a row's column ## text on demand {.deprecated: [TRow: Row, TDbConn: DbConn].} From 60a400eeb03873f26145fb75880d62bea7d01d5c Mon Sep 17 00:00:00 2001 From: jangko Date: Thu, 17 Dec 2015 10:14:48 +0700 Subject: [PATCH 023/125] fixed compile time `excl ` cause SIGSEGV #3639 --- compiler/vm.nim | 2 +- tests/vm/texcl.nim | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/vm/texcl.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 0e63daf89..7207ff223 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -611,7 +611,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = addSon(regs[ra].node, r.copyTree) of opcExcl: decodeB(rkNode) - var b = newNodeIT(nkCurly, regs[rb].node.info, regs[rb].node.typ) + var b = newNodeIT(nkCurly, regs[ra].node.info, regs[ra].node.typ) addSon(b, regs[rb].regToNode) var r = diffSets(regs[ra].node, b) discardSons(regs[ra].node) diff --git a/tests/vm/texcl.nim b/tests/vm/texcl.nim new file mode 100644 index 000000000..4ccfd6bfa --- /dev/null +++ b/tests/vm/texcl.nim @@ -0,0 +1,27 @@ +discard """ + output: '''false''' +""" + +import macros + +type + nlOptions = enum + nloNone + nloDebug + +var nlOpts {.compileTime.} = {nloDebug} + +proc initOpts(): set[nlOptions] = + result.incl nloDebug + result.incl nloNone + result.excl nloDebug + +const cOpts = initOpts() + +macro nlo(): stmt = + nlOpts.incl(nloNone) + nlOpts.excl(nloDebug) + result = newEmptyNode() + +nlo() +echo nloDebug in cOpts \ No newline at end of file From 169cb7178b4f9eb545eb0f44e9ea6bb7eaaf3f6d Mon Sep 17 00:00:00 2001 From: Hans Raaf Date: Thu, 17 Dec 2015 14:04:45 +0100 Subject: [PATCH 024/125] Fixing mysql_field. FIELD had the wrong size which made mysql.fetch_fields() useless. --- lib/wrappers/mysql.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim index 8253e53a5..af504864d 100644 --- a/lib/wrappers/mysql.nim +++ b/lib/wrappers/mysql.nim @@ -418,6 +418,7 @@ type decimals*: cuint # Number of decimals in field charsetnr*: cuint # Character set ftype*: Enum_field_types # Type of field. See mysql_com.h for types + extension*: pointer FIELD* = St_mysql_field PFIELD* = ptr FIELD From 4e157114a62aa3a55bf3358a7823134b42d75add Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 17 Dec 2015 14:37:49 +0100 Subject: [PATCH 025/125] added more column information fields --- lib/pure/db_common.nim | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim index 522d13db2..9187a67ce 100644 --- a/lib/pure/db_common.nim +++ b/lib/pure/db_common.nim @@ -36,7 +36,8 @@ type dbFloat, ## some floating point type dbDate, ## a year-month-day description dbTime, ## HH:MM:SS information - dbDatetime, ## year-month-day and HH:MM:SS information + dbDatetime, ## year-month-day and HH:MM:SS information, + ## plus optional time or timezone information dbTimestamp, ## Timestamp values are stored as the number of seconds ## since the epoch ('1970-01-01 00:00:00' UTC). dbTimeInterval, ## an interval [a,b] of times @@ -48,6 +49,7 @@ type dbUuid, ## a UUID dbInet, ## an IP address dbMacAddress, ## a MAC address + dbGeometry, ## some geometric type dbPoint, ## Point on a plane (x,y) dbLine, ## Infinite line ((x1,y1),(x2,y2)) dbLseg, ## Finite line segment ((x1,y1),(x2,y2)) @@ -66,6 +68,7 @@ type notNull*: bool ## does the type contain NULL? name*: string ## the name of the type size*: Natural ## the size of the datatype; 0 if of variable size + maxReprLen*: Natural ## maximal length required for the representation precision*, scale*: Natural ## precision and scale of the number min*, max*: BiggestInt ## the minimum and maximum of allowed values validValues*: seq[string] ## valid values of an enum or a set @@ -74,6 +77,8 @@ type name*: string ## name of the column tableName*: string ## name of the table the column belongs to (optional) typ*: DbType ## type of the column + primaryKey*: bool ## is this a primary key? + foreignKey*: bool ## is this a foreign key? DbColumns* = seq[DbColumn] {.deprecated: [EDb: DbError, TSqlQuery: SqlQuery, FDb: DbEffect, From de989e6fbd13e9cbfd754add0518d5037b276a7e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 17 Dec 2015 14:38:13 +0100 Subject: [PATCH 026/125] added column_table_name proc --- lib/wrappers/sqlite3.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim index c5019960c..e7fd2bc36 100644 --- a/lib/wrappers/sqlite3.nim +++ b/lib/wrappers/sqlite3.nim @@ -239,6 +239,8 @@ proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, importc: "sqlite3_column_count".} proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib, importc: "sqlite3_column_name".} +proc column_table_name*(para1: Pstmt; para2: int32): cstring{.cdecl, dynlib: Lib, + importc: "sqlite3_column_table_name".} proc column_name16*(para1: Pstmt, para2: int32): pointer{.cdecl, dynlib: Lib, importc: "sqlite3_column_name16".} proc column_decltype*(para1: Pstmt, i: int32): cstring{.cdecl, dynlib: Lib, From 3313170b8db620e32cf5b02c2f9b2dfa540c5b02 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 17 Dec 2015 14:39:02 +0100 Subject: [PATCH 027/125] fixes InstantRow decl; implements column information retrival for db_mysql --- lib/impure/db_mysql.nim | 93 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/lib/impure/db_mysql.nim b/lib/impure/db_mysql.nim index 0a57f8a41..170fee8b8 100644 --- a/lib/impure/db_mysql.nim +++ b/lib/impure/db_mysql.nim @@ -47,12 +47,13 @@ import db_common export db_common type - DbConn* = PMySQL ## encapsulates a database connection + DbConn* = PMySQL ## encapsulates a database connection Row* = seq[string] ## a row of a dataset. NULL database values will be ## converted to nil. - InstantRow* = tuple[row: cstringArray, len: int] ## a handle that can be - ## used to get a row's - ## column text on demand + InstantRow* = object ## a handle that can be used to get a row's + ## column text on demand + row: cstringArray + len: int {.deprecated: [TRow: Row, TDbConn: DbConn].} proc dbError*(db: DbConn) {.noreturn.} = @@ -158,9 +159,91 @@ iterator instantRows*(db: DbConn, query: SqlQuery, while true: row = mysql.fetchRow(sqlres) if row == nil: break - yield (row: row, len: L) + yield InstantRow(row: row, len: L) properFreeResult(sqlres, row) +proc setTypeName(t: var DbType; f: PFIELD) = + shallowCopy(t.name, $f.name) + t.maxReprLen = Natural(f.max_length) + if (NOT_NULL_FLAG and f.flags) != 0: t.notNull = true + case f.ftype + of TYPE_DECIMAL: + t.kind = dbDecimal + of TYPE_TINY: + t.kind = dbInt + t.size = 1 + of TYPE_SHORT: + t.kind = dbInt + t.size = 2 + of TYPE_LONG: + t.kind = dbInt + t.size = 4 + of TYPE_FLOAT: + t.kind = dbFloat + t.size = 4 + of TYPE_DOUBLE: + t.kind = dbFloat + t.size = 8 + of TYPE_NULL: + t.kind = dbNull + of TYPE_TIMESTAMP: + t.kind = dbTimestamp + of TYPE_LONGLONG: + t.kind = dbInt + t.size = 8 + of TYPE_INT24: + t.kind = dbInt + t.size = 3 + of TYPE_DATE: + t.kind = dbDate + of TYPE_TIME: + t.kind = dbTime + of TYPE_DATETIME: + t.kind = dbDatetime + of TYPE_YEAR: + t.kind = dbDate + of TYPE_NEWDATE: + t.kind = dbDate + of TYPE_VARCHAR, TYPE_VAR_STRING, TYPE_STRING: + t.kind = dbVarchar + of TYPE_BIT: + t.kind = dbBit + of TYPE_NEWDECIMAL: + t.kind = dbDecimal + of TYPE_ENUM: t.kind = dbEnum + of TYPE_SET: t.kind = dbSet + of TYPE_TINY_BLOB, TYPE_MEDIUM_BLOB, TYPE_LONG_BLOB, + TYPE_BLOB: t.kind = dbBlob + of TYPE_GEOMETRY: + t.kind = dbGeometry + +proc setColumnInfo(columns: var DbColumns; res: PRES; L: int) = + setLen(columns, L) + for i in 0.. Date: Thu, 17 Dec 2015 14:39:29 +0100 Subject: [PATCH 028/125] implements column information retrival for db_sqlite --- lib/impure/db_sqlite.nim | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 17f031a52..5cfed1eba 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -137,6 +137,39 @@ iterator instantRows*(db: DbConn, query: SqlQuery, yield stmt if finalize(stmt) != SQLITE_OK: dbError(db) +proc toTypeKind(t: var DbType; x: int32) = + case x + of SQLITE_INTEGER: + t.kind = dbInt + t.size = 8 + of SQLITE_FLOAT: + t.kind = dbFloat + t.size = 8 + of SQLITE_BLOB: t.kind = dbBlob + of SQLITE_NULL: t.kind = dbNull + of SQLITE_TEXT: t.kind = dbVarchar + else: t.kind = dbUnknown + +proc setColumns(columns: var DbColumns; x: PStmt) = + let L = column_count(x) + setLen(columns, L) + for i in 0'i32 ..< L: + columns[i].name = $column_name(x, i) + columns[i].typ.name = $column_decltype(x, i) + toTypeKind(columns[i].typ, column_type(x, i)) + columns[i].tableName = $column_table_name(x, i) + +iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery, + args: varargs[string, `$`]): InstantRow + {.tags: [ReadDbEffect].} = + ## same as fastRows but returns a handle that can be used to get column text + ## on demand using []. Returned handle is valid only within the iterator body. + var stmt = setupQuery(db, query, args) + setColumns(columns, stmt) + while step(stmt) == SQLITE_ROW: + yield stmt + if finalize(stmt) != SQLITE_OK: dbError(db) + proc `[]`*(row: InstantRow, col: int32): string {.inline.} = ## returns text for given column of the row $column_text(row, col) From f75d11193adbb19d8df2171fd7146744200a9898 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 18 Dec 2015 01:47:39 +0100 Subject: [PATCH 029/125] modified the integrated profiler to hopefully produce more reliable results --- lib/pure/nimprof.nim | 37 ++++++++++++++++++++++++++----------- lib/system/profiler.nim | 26 +++++++++----------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index cfe6bc40d..d1f215bae 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -1,7 +1,7 @@ # # # Nim's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2015 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -117,24 +117,38 @@ when defined(memProfiler): var gTicker {.threadvar.}: int - proc hook(st: StackTrace, size: int) {.nimcall.} = + proc requestedHook(): bool {.nimcall.} = if gTicker == 0: - gTicker = -1 - when defined(ignoreAllocationSize): - hookAux(st, 1) - else: - hookAux(st, size) gTicker = SamplingInterval + result = true dec gTicker + proc hook(st: StackTrace, size: int) {.nimcall.} = + when defined(ignoreAllocationSize): + hookAux(st, 1) + else: + hookAux(st, size) + else: var t0 {.threadvar.}: Ticks + gTicker: int # we use an additional counter to + # avoid calling 'getTicks' too frequently + + proc requestedHook(): bool {.nimcall.} = + if interval == 0: result = true + elif gTicker == 0: + gTicker = 500 + if getTicks() - t0 > interval: + result = true + else: + dec gTicker proc hook(st: StackTrace) {.nimcall.} = + #echo "profiling! ", interval if interval == 0: hookAux(st, 1) - elif int64(t0) == 0 or getTicks() - t0 > interval: + else: hookAux(st, 1) t0 = getTicks() @@ -145,9 +159,10 @@ proc cmpEntries(a, b: ptr ProfileEntry): int = result = b.getTotal - a.getTotal proc `//`(a, b: int): string = - result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDefault, 2)) + result = format("$1/$2 = $3%", a, b, formatFloat(a / b * 100.0, ffDecimal, 2)) proc writeProfile() {.noconv.} = + system.profilingRequestedHook = nil when declared(system.StackTrace): system.profilerHook = nil const filename = "profile_results.txt" @@ -193,12 +208,12 @@ var proc disableProfiling*() = when declared(system.StackTrace): atomicDec disabled - system.profilerHook = nil + system.profilingRequestedHook = nil proc enableProfiling*() = when declared(system.StackTrace): if atomicInc(disabled) >= 0: - system.profilerHook = hook + system.profilingRequestedHook = requestedHook when declared(system.StackTrace): system.profilerHook = hook diff --git a/lib/system/profiler.nim b/lib/system/profiler.nim index 4f600417e..ae8ff4e19 100644 --- a/lib/system/profiler.nim +++ b/lib/system/profiler.nim @@ -50,10 +50,15 @@ proc captureStackTrace(f: PFrame, st: var StackTrace) = inc(i) b = b.prev +var + profilingRequestedHook*: proc (): bool {.nimcall, benign.} + ## set this variable to provide a procedure that implements a profiler in + ## user space. See the `nimprof` module for a reference implementation. + when defined(memProfiler): type MemProfilerHook* = proc (st: StackTrace, requestedSize: int) {.nimcall, benign.} - {.deprecated: [TMemProfilerHook: MemProfilerHook].} + var profilerHook*: MemProfilerHook ## set this variable to provide a procedure that implements a profiler in @@ -65,17 +70,13 @@ when defined(memProfiler): hook(st, requestedSize) proc nimProfile(requestedSize: int) = - if not isNil(profilerHook): + if not isNil(profilingRequestedHook) and profilingRequestedHook(): callProfilerHook(profilerHook, requestedSize) else: - const - SamplingInterval = 50_000 - # set this to change the default sampling interval var profilerHook*: ProfilerHook ## set this variable to provide a procedure that implements a profiler in ## user space. See the `nimprof` module for a reference implementation. - gTicker {.threadvar.}: int proc callProfilerHook(hook: ProfilerHook) {.noinline.} = # 'noinline' so that 'nimProfile' does not perform the stack allocation @@ -86,16 +87,7 @@ else: proc nimProfile() = ## This is invoked by the compiler in every loop and on every proc entry! - if gTicker == 0: - gTicker = -1 - if not isNil(profilerHook): - # disable recursive calls: XXX should use try..finally, - # but that's too expensive! - let oldHook = profilerHook - profilerHook = nil - callProfilerHook(oldHook) - profilerHook = oldHook - gTicker = SamplingInterval - dec gTicker + if not isNil(profilingRequestedHook) and profilingRequestedHook(): + callProfilerHook(profilerHook) {.pop.} From d6fbc1425f52be6a488dc0652d259ba970f43708 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 18 Dec 2015 01:50:27 +0100 Subject: [PATCH 030/125] profiling doesn't require call to enableProfiling --- lib/pure/nimprof.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pure/nimprof.nim b/lib/pure/nimprof.nim index d1f215bae..e2397b91c 100644 --- a/lib/pure/nimprof.nim +++ b/lib/pure/nimprof.nim @@ -216,6 +216,7 @@ proc enableProfiling*() = system.profilingRequestedHook = requestedHook when declared(system.StackTrace): + system.profilingRequestedHook = requestedHook system.profilerHook = hook addQuitProc(writeProfile) From 26d02c9120c54e0c1a92e08b9b571a89c9f1d4ff Mon Sep 17 00:00:00 2001 From: jangko Date: Fri, 18 Dec 2015 19:09:15 +0700 Subject: [PATCH 031/125] fixed VM touint conversion #2514 --- compiler/vm.nim | 9 ++++- tests/vm/ttouintconv.nim | 77 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/vm/ttouintconv.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 0e63daf89..895653652 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -359,7 +359,14 @@ proc opConv*(dest: var TFullReg, src: TFullReg, desttyp, srctyp: PType): bool = of tyFloat..tyFloat64: dest.intVal = int(src.floatVal) else: - dest.intVal = src.intVal and ((1 shl (desttyp.size*8))-1) + let srcDist = (sizeof(src.intVal) - srctyp.size) * 8 + let destDist = (sizeof(dest.intVal) - desttyp.size) * 8 + when system.cpuEndian == bigEndian: + dest.intVal = (src.intVal shr srcDist) shl srcDist + dest.intVal = (dest.intVal shr destDist) shl destDist + else: + dest.intVal = (src.intVal shl srcDist) shr srcDist + dest.intVal = (dest.intVal shl destDist) shr destDist of tyFloat..tyFloat64: if dest.kind != rkFloat: myreset(dest); dest.kind = rkFloat diff --git a/tests/vm/ttouintconv.nim b/tests/vm/ttouintconv.nim new file mode 100644 index 000000000..cd25ffb00 --- /dev/null +++ b/tests/vm/ttouintconv.nim @@ -0,0 +1,77 @@ +import macros + +discard """ +msg: ''' +8 9 17 +239 255 +61439 65534 65535 +4026531839 4294967294 +17293822569102704639 +18446744073709551614 +18446744073709551615 +127 +32767 +2147483647 +9223372036854775807 +0 +128 +4294967287''' +""" + +#bug #2514 + +macro foo(): stmt = + var x = 8'u8 + var y = 9'u16 + var z = 17'u32 + + echo x," ", y," ", z + + var a = 0xEF'u8 + var aa = 0xFF'u8 + echo a, " ", aa + + var b = 0xEFFF'u16 + var bb = 0xFFFE'u16 + var bbb = 0xFFFF'u16 + echo b, " ", bb, " ", bbb + + var c = 0xEFFFFFFF'u32 + var cc = 0xFFFFFFFE'u32 + echo c, " ", cc + + var d = 0xEFFFFFFFFFFFFFFF'u64 + echo d + + var f = 0xFFFFFFFFFFFFFFFE'u64 + echo f + + var g = 0xFFFFFFFFFFFFFFFF'u64 + echo g + + var xx = 0x7F'u8 and 0xFF + echo xx + + var yy = 0x7FFF'u16 + echo yy + + var zz = 0x7FFFFFFF'u32 + echo zz + +macro foo2(): stmt = + var xx = 0x7FFFFFFFFFFFFFFF + echo xx + + var yy = 0 + echo yy + + var zz = 0x80'u8 + echo zz + + var ww = -9 + var vv = ww.uint + var kk = vv.uint32 + echo kk + +foo() +foo2() From 5a028b3956fe592d08e16a6ce2ceafcb1dd748ac Mon Sep 17 00:00:00 2001 From: Hans Raaf Date: Mon, 21 Dec 2015 00:35:20 +0100 Subject: [PATCH 032/125] Fix regression in set.excl code generation. --- compiler/ccgexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index df3b655e3..d691b8ea2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1502,7 +1502,7 @@ proc genSetOp(p: BProc, e: PNode, d: var TLoc, op: TMagic) = else: case op of mIncl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] |=(1U<<($2&7U));$n") - of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3]] &= ~(1U<<($2&7U));$n") + of mExcl: binaryStmtInExcl(p, e, d, "$1[(NU)($2)>>3] &= ~(1U<<($2&7U));$n") of mCard: unaryExprChar(p, e, d, "#cardSet($1, " & $size & ')') of mLtSet, mLeSet: getTemp(p, getSysType(tyInt), i) # our counter From 25e862b9bced07435a2ab1108a47749401edda06 Mon Sep 17 00:00:00 2001 From: def Date: Wed, 23 Dec 2015 05:56:42 +0100 Subject: [PATCH 033/125] Fix osproc compilation on NetBSD, use workaround for missing execvpe --- lib/pure/osproc.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 7a1e14a57..b703fab63 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -886,7 +886,7 @@ elif not defined(useNimRtl): discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) exitnow(1) - when defined(macosx) or defined(freebsd): + when defined(macosx) or defined(freebsd) or defined(netbsd): var environ {.importc.}: cstringArray proc startProcessAfterFork(data: ptr StartProcessData) = @@ -916,7 +916,7 @@ elif not defined(useNimRtl): discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) if data.optionPoUsePath: - when defined(macosx) or defined(freebsd): + when defined(macosx) or defined(freebsd) or defined(netbsd): # MacOSX doesn't have execvpe, so we need workaround. # On MacOSX we can arrive here only from fork, so this is safe: environ = data.sysEnv From 31cc1815d147d3f5eaa0c9c844bcf803b070b37a Mon Sep 17 00:00:00 2001 From: def Date: Wed, 23 Dec 2015 04:13:30 +0100 Subject: [PATCH 034/125] Fix nativesockets compilation on OpenBSD and NetBSD --- lib/pure/nativesockets.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 8a7780570..b5a8d5777 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -206,7 +206,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, # OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED. # FreeBSD doesn't support AI_V4MAPPED but defines the macro. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092 - when not defined(freebsd) or defined(openbsd): + when not defined(freebsd) and not defined(openbsd) and not defined(netbsd): if domain == AF_INET6: hints.ai_flags = AI_V4MAPPED var gaiResult = getaddrinfo(address, $port, addr(hints), result) From 3c557a014481e07c41633c693ba1b4baf0c63043 Mon Sep 17 00:00:00 2001 From: trustable-code Date: Thu, 24 Dec 2015 00:07:03 +0100 Subject: [PATCH 035/125] fix issue #3550 --- compiler/cgmeth.nim | 5 +---- tests/method/tmultim8.nim | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 tests/method/tmultim8.nim diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index ade202dc1..312afec1a 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -19,10 +19,7 @@ proc genConv(n: PNode, d: PType, downcast: bool): PNode = if (source.kind == tyObject) and (dest.kind == tyObject): var diff = inheritanceDiff(dest, source) if diff == high(int): - # see bug #3550 which triggers it. XXX This is a hack but I don't know yet - # how the real fix looks like: - localError(n.info, "there is no subtype relation between " & - typeToString(d) & " and " & typeToString(n.typ)) + # no subtype relation, nothing to do result = n elif diff < 0: result = newNodeIT(nkObjUpConv, n.info, d) diff --git a/tests/method/tmultim8.nim b/tests/method/tmultim8.nim new file mode 100644 index 000000000..0d067b668 --- /dev/null +++ b/tests/method/tmultim8.nim @@ -0,0 +1,19 @@ + +# bug #3550 + +type + BaseClass = ref object of RootObj + Class1 = ref object of BaseClass + Class2 = ref object of BaseClass + +method test(obj: Class1, obj2: BaseClass) = + discard + +method test(obj: Class2, obj2: BaseClass) = + discard + +var obj1 = Class1() +var obj2 = Class2() + +obj1.test(obj2) +obj2.test(obj1) From a01eb098f1e4f3d1ac556be2c07aefcde6e3a869 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 24 Dec 2015 17:09:58 +0000 Subject: [PATCH 036/125] Task proc in nimscript module will now export the proc it creates. --- lib/system/nimscript.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index aaba11324..772d25343 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -242,7 +242,7 @@ template task*(name: untyped; description: string; body: untyped): untyped = ## .. code-block:: nim ## task build, "default build is via the C backend": ## setCommand "c" - proc `name Task`() = body + proc `name Task`*() = body let cmd = getCommand() if cmd.len == 0 or cmd ==? "help": From 936265df729d0fd7d6b73ce89357d3705286e364 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Fri, 25 Dec 2015 01:03:15 +0200 Subject: [PATCH 037/125] Fixed lineinfo in getType --- compiler/vmdeps.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 2cc4a107b..a4f02092d 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -70,7 +70,7 @@ proc atomicTypeX(name: string; t: PType; info: TLineInfo): PNode = proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode proc mapTypeToBracket(name: string; t: PType; info: TLineInfo): PNode = - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicTypeX(name, t, info) for i in 0 .. < t.len: if t.sons[i] == nil: @@ -92,19 +92,19 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyStmt: result = atomicType("stmt") of tyEmpty: result = atomicType"void" of tyArrayConstr, tyArray: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("array") result.add mapTypeToAst(t.sons[0], info) result.add mapTypeToAst(t.sons[1], info) of tyTypeDesc: if t.base != nil: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("typeDesc") result.add mapTypeToAst(t.base, info) else: result = atomicType"typeDesc" of tyGenericInvocation: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) for i in 0 .. < t.len: result.add mapTypeToAst(t.sons[i], info) of tyGenericInst, tyGenericBody, tyOrdinal, tyUserTypeClassInst: @@ -117,7 +117,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyGenericParam, tyForward: result = atomicType(t.sym.name.s) of tyObject: if allowRecursion: - result = newNodeIT(nkObjectTy, info, t) + result = newNodeIT(nkObjectTy, if t.n.isNil: info else: t.n.info, t) if t.sons[0] == nil: result.add ast.emptyNode else: @@ -126,7 +126,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = else: result = atomicType(t.sym.name.s) of tyEnum: - result = newNodeIT(nkEnumTy, info, t) + result = newNodeIT(nkEnumTy, if t.n.isNil: info else: t.n.info, t) result.add copyTree(t.n) of tyTuple: result = mapTypeToBracket("tuple", t, info) of tySet: result = mapTypeToBracket("set", t, info) @@ -137,7 +137,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyProc: result = mapTypeToBracket("proc", t, info) of tyOpenArray: result = mapTypeToBracket("openArray", t, info) of tyRange: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("range") result.add t.n.sons[0].copyTree result.add t.n.sons[1].copyTree @@ -174,7 +174,7 @@ proc mapTypeToAst(t: PType, info: TLineInfo; allowRecursion=false): PNode = of tyNot: result = mapTypeToBracket("not", t, info) of tyAnything: result = atomicType"anything" of tyStatic, tyFromExpr, tyFieldAccessor: - result = newNodeIT(nkBracketExpr, info, t) + result = newNodeIT(nkBracketExpr, if t.n.isNil: info else: t.n.info, t) result.add atomicType("static") if t.n != nil: result.add t.n.copyTree From c1627354d1fe5499c3d8b6886ce63de6e062a463 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Dec 2015 00:42:30 +0100 Subject: [PATCH 038/125] first implementation of the new lambda-lifting pass; barely anything works --- compiler/ast.nim | 20 +- compiler/ccgexprs.nim | 2 +- compiler/ccgstmts.nim | 2 +- compiler/cgen.nim | 2 +- compiler/docgen.nim | 2 +- compiler/jsgen.nim | 2 +- compiler/lambdalifting.nim | 1165 +++++++++++++----------------------- compiler/lookups.nim | 2 +- compiler/lowerings.nim | 2 +- compiler/semcall.nim | 4 +- compiler/semdata.nim | 2 +- compiler/semexprs.nim | 18 +- compiler/semgnrc.nim | 4 +- compiler/semstmts.nim | 23 +- compiler/semtempl.nim | 4 +- compiler/semtypes.nim | 2 +- compiler/sigmatch.nim | 2 +- compiler/transf.nim | 18 + compiler/vmgen.nim | 4 +- lib/core/macros.nim | 2 +- 20 files changed, 479 insertions(+), 803 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 8054e9248..f34358788 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -500,8 +500,7 @@ type skResult, # special 'result' variable skProc, # a proc skMethod, # a method - skIterator, # an inline iterator - skClosureIterator, # a resumable closure iterator + skIterator, # an iterator skConverter, # a type converter skMacro, # a macro skTemplate, # a template; currently also misused for user-defined @@ -518,7 +517,7 @@ type TSymKinds* = set[TSymKind] const - routineKinds* = {skProc, skMethod, skIterator, skClosureIterator, + routineKinds* = {skProc, skMethod, skIterator, skConverter, skMacro, skTemplate} tfIncompleteStruct* = tfVarargs tfUncheckedArray* = tfVarargs @@ -905,7 +904,7 @@ type # the poor naming choices in the standard library. const - OverloadableSyms* = {skProc, skMethod, skIterator, skClosureIterator, + OverloadableSyms* = {skProc, skMethod, skIterator, skConverter, skModule, skTemplate, skMacro} GenericTypes*: TTypeKinds = {tyGenericInvocation, tyGenericBody, @@ -929,7 +928,7 @@ const NilableTypes*: TTypeKinds = {tyPointer, tyCString, tyRef, tyPtr, tySequence, tyProc, tyString, tyError} ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, - skIterator, skClosureIterator, + skIterator, skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, @@ -958,11 +957,9 @@ const nkStrKinds* = {nkStrLit..nkTripleStrLit} skLocalVars* = {skVar, skLet, skForVar, skParam, skResult} - skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skClosureIterator, + skProcKinds* = {skProc, skTemplate, skMacro, skIterator, skMethod, skConverter} - skIterators* = {skIterator, skClosureIterator} - var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things proc isCallExpr*(n: PNode): bool = @@ -1558,12 +1555,13 @@ proc isGenericRoutine*(s: PSym): bool = else: discard proc skipGenericOwner*(s: PSym): PSym = - internalAssert s.kind in skProcKinds ## Generic instantiations are owned by their originating generic ## symbol. This proc skips such owners and goes straight to the owner ## of the generic itself (the module or the enclosing proc). - result = if sfFromGeneric in s.flags: s.owner.owner - else: s.owner + result = if s.kind in skProcKinds and sfFromGeneric in s.flags: + s.owner.owner + else: + s.owner proc originatingModule*(s: PSym): PSym = result = s.owner diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index d691b8ea2..0df7e22d3 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1965,7 +1965,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = else: genProc(p.module, sym) putLocIntoDest(p, d, sym.loc) - of skProc, skConverter, skIterators: + of skProc, skConverter, skIterator: if sfCompileTime in sym.flags: localError(n.info, "request to generate code for .compileTime proc: " & sym.name.s) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 73497bded..699361ef8 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -955,7 +955,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope = res.add(t.sons[i].strVal) of nkSym: var sym = t.sons[i].sym - if sym.kind in {skProc, skIterator, skClosureIterator, skMethod}: + if sym.kind in {skProc, skIterator, skMethod}: var a: TLoc initLocExpr(p, t.sons[i], a) res.add($rdLoc(a)) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index f63134b66..db376821c 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -594,7 +594,7 @@ proc cgsym(m: BModule, name: string): Rope = var sym = magicsys.getCompilerProc(name) if sym != nil: case sym.kind - of skProc, skMethod, skConverter, skIterators: genProc(m, sym) + of skProc, skMethod, skConverter, skIterator: genProc(m, sym) of skVar, skResult, skLet: genVarPrototype(m, sym) of skType: discard getTypeDesc(m, sym.typ) else: internalError("cgsym: " & name & ": " & $sym.kind) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8536cc619..952366f93 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -537,7 +537,7 @@ proc generateJson(d: PDoc, n: PNode, jArray: JsonNode = nil): JsonNode = proc genSection(d: PDoc, kind: TSymKind) = const sectionNames: array[skModule..skTemplate, string] = [ "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Methods", - "Iterators", "Iterators", "Converters", "Macros", "Templates" + "Iterators", "Converters", "Macros", "Templates" ] if d.section[kind] == nil: return var title = sectionNames[kind].rope diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index f8bf35ed6..2f6496b9b 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -931,7 +931,7 @@ proc isIndirect(v: PSym): bool = result = {sfAddrTaken, sfGlobal} * v.flags != {} and #(mapType(v.typ) != etyObject) and {sfImportc, sfVolatile, sfExportc} * v.flags == {} and - v.kind notin {skProc, skConverter, skMethod, skIterator, skClosureIterator, + v.kind notin {skProc, skConverter, skMethod, skIterator, skConst, skTemp, skLet} proc genAddr(p: PProc, n: PNode, r: var TCompRes) = diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index be1631af0..00849b47e 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -11,7 +11,7 @@ import intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, - idents, renderer, types, magicsys, rodread, lowerings + idents, renderer, types, magicsys, rodread, lowerings, tables discard """ The basic approach is that captured vars need to be put on the heap and @@ -113,44 +113,19 @@ discard """ # local storage requirements for efficiency. This means closure iterators # have slightly different semantics from ordinary closures. +# ---------------- essential helpers ------------------------------------- const upName* = ":up" # field name for the 'up' reference paramName* = ":envP" envName* = ":env" -type - POuterContext = ref TOuterContext +proc newCall(a: PSym, b: PNode): PNode = + result = newNodeI(nkCall, a.info) + result.add newSymNode(a) + result.add b - TIter = object - fn, closureParam, state, resultSym: PSym # most are only valid if - # fn.kind == skClosureIterator - obj: PType - isIterator: bool - - PEnv = ref TEnv - TEnv {.final.} = object of RootObj - attachedNode, replacementNode: PNode - createdVar: PNode # if != nil it is a used environment; for closure - # iterators this can be 'envParam.env' - createdVarComesFromIter: bool - capturedVars: seq[PSym] # captured variables in this environment - up, next: PEnv # outer scope and next to keep all in a list - upField: PSym # if != nil the dependency to the outer scope is used - obj: PType - fn: PSym # function that belongs to this scope; - # if up.fn != fn then we cross function boundaries. - # This is an important case to consider. - vars: IntSet # variables belonging to this environment - - TOuterContext = object - fn: PSym # may also be a module! - head: PEnv - capturedVars, processed: IntSet - localsToAccess: TIdNodeTable - lambdasToEnv: TIdTable # PSym->PEnv mapping - -proc getStateType(iter: PSym): PType = +proc createStateType(iter: PSym): PType = var n = newNodeI(nkRange, iter.info) addSon(n, newIntNode(nkIntLit, -1)) addSon(n, newIntNode(nkIntLit, 0)) @@ -162,7 +137,7 @@ proc getStateType(iter: PSym): PType = proc createStateField(iter: PSym): PSym = result = newSym(skField, getIdent(":state"), iter, iter.info) - result.typ = getStateType(iter) + result.typ = createStateType(iter) proc createEnvObj(owner: PSym): PType = # YYY meh, just add the state field for every closure for now, it's too @@ -170,7 +145,7 @@ proc createEnvObj(owner: PSym): PType = result = createObj(owner, owner.info) rawAddField(result, createStateField(owner)) -proc newIterResult(iter: PSym): PSym = +proc getIterResult(iter: PSym): PSym = if resultPos < iter.ast.len: result = iter.ast.sons[resultPos].sym else: @@ -194,9 +169,13 @@ proc addHiddenParam(routine: PSym, param: PSym) = proc getHiddenParam(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] let hidden = lastSon(params) - internalAssert hidden.kind == nkSym and hidden.sym.kind == skParam - result = hidden.sym - assert sfFromGeneric in result.flags + if hidden.kind == nkSym and hidden.sym.kind == skParam: + result = hidden.sym + assert sfFromGeneric in result.flags + else: + # writeStackTrace() + localError(routine.info, "internal error: could not find env param " & routine.name.s) + result = routine proc getEnvParam*(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] @@ -205,141 +184,18 @@ proc getEnvParam*(routine: PSym): PSym = result = hidden.sym assert sfFromGeneric in result.flags -proc initIter(iter: PSym; ptrType: PType = nil): TIter = - result.fn = iter - result.isIterator = ptrType != nil or iter.kind == skClosureIterator - #echo "fuck you ", ptrType != nil - if result.isIterator: - var cp = getEnvParam(iter) - if cp == nil: - result.obj = if ptrType != nil: ptrType.lastSon else: createEnvObj(iter) - - cp = newSym(skParam, getIdent(paramName), iter, iter.info) - incl(cp.flags, sfFromGeneric) - if ptrType != nil: - cp.typ = ptrType - else: - cp.typ = newType(tyRef, iter) - rawAddSon(cp.typ, result.obj) - addHiddenParam(iter, cp) - else: - result.obj = cp.typ.sons[0] - assert result.obj.kind == tyObject - internalAssert result.obj.n.len > 0 - result.state = result.obj.n[0].sym - result.closureParam = cp - if iter.typ.sons[0] != nil: - result.resultSym = newIterResult(iter) - #iter.ast.add(newSymNode(c.resultSym)) - -proc newOuterContext(fn: PSym): POuterContext = - new(result) - result.fn = fn - result.capturedVars = initIntSet() - result.processed = initIntSet() - initIdNodeTable(result.localsToAccess) - initIdTable(result.lambdasToEnv) - -proc newEnv(o: POuterContext; up: PEnv, n: PNode; owner: PSym): PEnv = - new(result) - result.capturedVars = @[] - result.up = up - result.attachedNode = n - result.fn = owner - result.vars = initIntSet() - result.next = o.head - o.head = result - if owner.kind != skModule and (up == nil or up.fn != owner): - let param = getEnvParam(owner) - if param != nil: - result.obj = param.typ.sons[0] - assert result.obj.kind == tyObject - if result.obj.isNil: - result.obj = createEnvObj(owner) - -proc addCapturedVar(e: PEnv, v: PSym) = - for x in e.capturedVars: - if x == v: return - e.capturedVars.add(v) - addField(e.obj, v) - -proc newCall(a: PSym, b: PNode): PNode = - result = newNodeI(nkCall, a.info) - result.add newSymNode(a) - result.add b - -proc isInnerProc(s, outerProc: PSym): bool = - if s.kind in {skProc, skMethod, skConverter, skClosureIterator}: - var owner = s.skipGenericOwner - while true: - if owner.isNil: return false - if owner == outerProc: return true - owner = owner.owner - #s.typ.callConv == ccClosure - -proc addClosureParam(fn: PSym; e: PEnv) = - var cp = getEnvParam(fn) - if cp == nil: - cp = newSym(skParam, getIdent(paramName), fn, fn.info) - incl(cp.flags, sfFromGeneric) - cp.typ = newType(tyRef, fn) - rawAddSon(cp.typ, e.obj) - addHiddenParam(fn, cp) - #else: - #cp.typ.sons[0] = e.obj - #assert e.obj.kind == tyObject +proc interestingVar(s: PSym): bool {.inline.} = + result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and + sfGlobal notin s.flags proc illegalCapture(s: PSym): bool {.inline.} = result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray, tyVarargs} or s.kind == skResult -proc interestingVar(s: PSym): bool {.inline.} = - result = s.kind in {skVar, skLet, skTemp, skForVar, skParam, skResult} and - sfGlobal notin s.flags - -proc nestedAccess(top: PEnv; local: PSym): PNode = - # Parts after the transformation are in []: - # - # proc main = - # var [:env.]foo = 23 - # proc outer(:paramO) = - # [var :envO; createClosure(:envO); :envO.up = paramO] - # proc inner(:paramI) = - # echo [:paramI.up.]foo - # inner([:envO]) - # outer([:env]) - if not interestingVar(local) or top.fn == local.owner: - return nil - # check it's in fact a captured variable: - var it = top - while it != nil: - if it.vars.contains(local.id): break - it = it.up - if it == nil: return nil - let envParam = top.fn.getEnvParam - internalAssert(not envParam.isNil) - var access = newSymNode(envParam) - it = top.up - while it != nil: - if it.vars.contains(local.id): - access = indirectAccess(access, local, local.info) - return access - internalAssert it.upField != nil - access = indirectAccess(access, it.upField, local.info) - it = it.up - when false: - # Type based expression construction works too, but turned out to hide - # other bugs: - while true: - let obj = access.typ.sons[0] - let field = getFieldFromObj(obj, local) - if field != nil: - return rawIndirectAccess(access, field, local.info) - let upField = lookupInRecord(obj.n, getIdent(upName)) - if upField == nil: break - access = rawIndirectAccess(access, upField, local.info) - return nil +proc isInnerProc(s: PSym): bool = + if s.kind in {skProc, skMethod, skConverter, skIterator}: + result = s.skipGenericOwner.kind in routineKinds proc createUpField(obj, fieldType: PType): PSym = let pos = obj.n.len @@ -350,269 +206,6 @@ proc createUpField(obj, fieldType: PType): PSym = #rawAddField(obj, result) addField(obj, result) -proc captureVar(o: POuterContext; top: PEnv; local: PSym; - info: TLineInfo): bool = - # first check if we should be concerned at all: - var it = top - while it != nil: - if it.vars.contains(local.id): break - it = it.up - if it == nil: return false - # yes, so mark every 'up' pointer as taken: - if illegalCapture(local) or top.fn.typ.callConv notin {ccClosure, ccDefault}: - localError(info, errIllegalCaptureX, local.name.s) - it = top - while it != nil: - if it.vars.contains(local.id): break - # keep in mind that the first element of the chain belong to top.fn itself - # and these don't need any upFields - if it.upField == nil and it.up != nil and it.fn != top.fn: - it.upField = createUpField(it.obj, it.up.obj) - - if it.fn != local.owner: - it.fn.typ.callConv = ccClosure - incl(it.fn.typ.flags, tfCapturesEnv) - - var u = it.up - while u != nil and u.fn == it.fn: u = u.up - addClosureParam(it.fn, u) - - if idTableGet(o.lambdasToEnv, it.fn) == nil: - if u != nil: idTablePut(o.lambdasToEnv, it.fn, u) - - it = it.up - # don't do this: 'top' might not require a closure: - #if idTableGet(o.lambdasToEnv, it.fn) == nil: - # idTablePut(o.lambdasToEnv, it.fn, top) - - # mark as captured: - #if top.iter != nil: - # if not containsOrIncl(o.capturedVars, local.id): - # #addField(top.iter.obj, local) - # addCapturedVar(it, local) - #else: - incl(o.capturedVars, local.id) - addCapturedVar(it, local) - result = true - -proc semCaptureSym*(s, owner: PSym) = - if interestingVar(s) and owner.id != s.owner.id and s.kind != skResult: - if owner.typ != nil and not isGenericRoutine(owner): - # XXX: is this really safe? - # if we capture a var from another generic routine, - # it won't be consider captured. - owner.typ.callConv = ccClosure - #echo "semCaptureSym ", owner.name.s, owner.id, " ", s.name.s, s.id - # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' - # here - -proc gatherVars(o: POuterContext; e: PEnv; n: PNode): int = - # gather used vars for closure generation; returns number of captured vars - if n == nil: return 0 - case n.kind - of nkSym: - var s = n.sym - if interestingVar(s) and e.fn != s.owner: - if captureVar(o, e, s, n.info): result = 1 - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, nkProcDef, - nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, nkTypeSection: - discard - else: - for k in countup(0, sonsLen(n) - 1): - result += gatherVars(o, e, n.sons[k]) - -proc generateThunk(prc: PNode, dest: PType): PNode = - ## Converts 'prc' into '(thunk, nil)' so that it's compatible with - ## a closure. - - # we cannot generate a proper thunk here for GC-safety reasons (see internal - # documentation): - if gCmd == cmdCompileToJS: return prc - result = newNodeIT(nkClosure, prc.info, dest) - var conv = newNodeIT(nkHiddenStdConv, prc.info, dest) - conv.add(emptyNode) - conv.add(prc) - result.add(conv) - result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil))) - -proc transformOuterConv(n: PNode): PNode = - # numeric types need range checks: - var dest = skipTypes(n.typ, abstractVarRange) - var source = skipTypes(n.sons[1].typ, abstractVarRange) - if dest.kind == tyProc: - if dest.callConv == ccClosure and source.callConv == ccDefault: - result = generateThunk(n.sons[1], dest) - -proc makeClosure(prc: PSym; env: PNode; info: TLineInfo): PNode = - result = newNodeIT(nkClosure, info, prc.typ) - result.add(newSymNode(prc)) - if env == nil: - result.add(newNodeIT(nkNilLit, info, getSysType(tyNil))) - else: - result.add(env) - -proc newClosureCreationVar(e: PEnv): PNode = - var v = newSym(skVar, getIdent(envName), e.fn, e.attachedNode.info) - incl(v.flags, sfShadowed) - v.typ = newType(tyRef, e.fn) - v.typ.rawAddSon(e.obj) - if e.fn.kind == skClosureIterator: - let it = initIter(e.fn) - addUniqueField(it.obj, v) - result = indirectAccess(newSymNode(it.closureParam), v, v.info) - else: - result = newSymNode(v) - -proc getClosureVar(e: PEnv): PNode = - if e.createdVar == nil: - result = newClosureCreationVar(e) - e.createdVar = result - else: - result = e.createdVar - -proc findEnv(o: POuterContext; s: PSym): PEnv = - var env = o.head - while env != nil: - if env.fn == s: break - env = env.next - internalAssert env != nil and env.up != nil - result = env.up - while result.fn == s: result = result.up - -proc transformInnerProc(o: POuterContext; e: PEnv, n: PNode): PNode = - case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard - of nkSym: - let s = n.sym - if s == e.fn: - # recursive calls go through (lambda, hiddenParam): - result = makeClosure(s, getEnvParam(s).newSymNode, n.info) - elif isInnerProc(s, o.fn) and s.typ.callConv == ccClosure: - # ugh: call to some other inner proc; - result = makeClosure(s, findEnv(o, s).getClosureVar, n.info) - else: - # captured symbol? - result = nestedAccess(e, n.sym) - #result = idNodeTableGet(i.localsToAccess, n.sym) - #of nkLambdaKinds, nkIteratorDef: - # if n.typ != nil: - # result = transformInnerProc(o, e, n.sons[namePos]) - #of nkClosure: - # let x = transformInnerProc(o, e, n.sons[0]) - # if x != nil: n.sons[0] = x - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, - nkLambdaKinds, nkIteratorDef, nkClosure: - # don't recurse here: - discard - else: - for j in countup(0, sonsLen(n) - 1): - let x = transformInnerProc(o, e, n.sons[j]) - if x != nil: n.sons[j] = x - -proc closureCreationPoint(n: PNode): PNode = - if n.kind == nkStmtList and n.len >= 1 and n[0].kind == nkEmpty: - # we already have a free slot - result = n - else: - result = newNodeI(nkStmtList, n.info) - result.add(emptyNode) - result.add(n) - #result.flags.incl nfLL - -proc addParamsToEnv(fn: PSym; env: PEnv) = - let params = fn.typ.n - for i in 1.. declaredBlock. Note: The definition - # counts, not the block where it is captured! - for i in countup(0, sonsLen(n) - 1): - var it = n.sons[i] - if it.kind == nkCommentStmt: discard - elif it.kind == nkIdentDefs: - var L = sonsLen(it) - if it.sons[0].kind == nkSym: - # this can be false for recursive invocations that already - # transformed it into 'env.varName': - env.vars.incl(it.sons[0].sym.id) - searchForInnerProcs(o, it.sons[L-1], env) - elif it.kind == nkVarTuple: - var L = sonsLen(it) - for j in countup(0, L-3): - #echo "set: ", it.sons[j].sym.name.s, " ", o.currentBlock == nil - if it.sons[j].kind == nkSym: - env.vars.incl(it.sons[j].sym.id) - searchForInnerProcs(o, it.sons[L-1], env) - else: - internalError(it.info, "searchForInnerProcs") - of nkClosure: - searchForInnerProcs(o, n.sons[0], env) - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, - nkTypeSection: - # don't recurse here: - discard - else: - for i in countup(0, sonsLen(n) - 1): - searchForInnerProcs(o, n.sons[i], env) - proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = # Bugfix: unfortunately we cannot use 'nkFastAsgn' here as that would # mean to be able to capture string literals which have no GC header. @@ -623,113 +216,27 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = result.sons[0] = le result.sons[1] = ri -proc rawClosureCreation(o: POuterContext, scope: PEnv; env: PNode): PNode = - result = newNodeI(nkStmtList, env.info) - if env.kind == nkSym: - var v = newNodeI(nkVarSection, env.info) - addVar(v, env) - result.add(v) - # add 'new' statement: - result.add(newCall(getSysSym"internalNew", env)) - - # add assignment statements: - for local in scope.capturedVars: - let fieldAccess = indirectAccess(env, local, env.info) - if local.kind == skParam: - # maybe later: (sfByCopy in local.flags) - # add ``env.param = param`` - result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) - # it can happen that we already captured 'local' in some other environment - # then we capture by copy for now. This is not entirely correct but better - # than nothing: - let existing = idNodeTableGet(o.localsToAccess, local) - if existing.isNil: - idNodeTablePut(o.localsToAccess, local, fieldAccess) - else: - result.add(newAsgnStmt(fieldAccess, existing, env.info)) - if scope.upField != nil: - # "up" chain has been used: - if scope.up.fn != scope.fn: - # crosses function boundary: - result.add(newAsgnStmt(indirectAccess(env, scope.upField, env.info), - newSymNode(getEnvParam(scope.fn)), env.info)) - else: - result.add(newAsgnStmt(indirectAccess(env, scope.upField, env.info), - getClosureVar(scope.up), env.info)) - -proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode = - var env = getClosureVar(scope) - result = rawClosureCreation(o, scope, env) - -proc generateIterClosureCreation(o: POuterContext; env: PEnv; - scope: PNode): PNode = - if env.createdVarComesFromIter or env.createdVar.isNil: - # we have to create a new closure: - result = newClosureCreationVar(env) - let cc = rawClosureCreation(o, env, result) - var insertPoint = scope.sons[0] - if insertPoint.kind == nkEmpty: scope.sons[0] = cc - else: - assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList - for x in cc: insertPoint.add(x) - if env.createdVar == nil: env.createdVar = result +proc makeClosure(prc: PSym; env: PNode; info: TLineInfo): PNode = + result = newNodeIT(nkClosure, info, prc.typ) + result.add(newSymNode(prc)) + if env == nil: + result.add(newNodeIT(nkNilLit, info, getSysType(tyNil))) else: - result = env.createdVar - env.createdVarComesFromIter = true + result.add(env) proc interestingIterVar(s: PSym): bool {.inline.} = + # XXX optimization: Only lift the variable if it lives across + # yield/return boundaries! This can potentially speed up + # closure iterators quite a bit. result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags -proc transformOuterProc(o: POuterContext, n: PNode, it: TIter): PNode - -proc transformYield(c: POuterContext, n: PNode, it: TIter): PNode = - assert it.state != nil - assert it.state.typ != nil - assert it.state.typ.n != nil - inc it.state.typ.n.sons[1].intVal - let stateNo = it.state.typ.n.sons[1].intVal - - var stateAsgnStmt = newNodeI(nkAsgn, n.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), - it.state, n.info)) - stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt))) - - var retStmt = newNodeI(nkReturnStmt, n.info) - if n.sons[0].kind != nkEmpty: - var a = newNodeI(nkAsgn, n.sons[0].info) - var retVal = transformOuterProc(c, n.sons[0], it) - addSon(a, newSymNode(it.resultSym)) - addSon(a, if retVal.isNil: n.sons[0] else: retVal) - retStmt.add(a) - else: - retStmt.add(emptyNode) - - var stateLabelStmt = newNodeI(nkState, n.info) - stateLabelStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt))) - - result = newNodeI(nkStmtList, n.info) - result.add(stateAsgnStmt) - result.add(retStmt) - result.add(stateLabelStmt) - -proc transformReturn(c: POuterContext, n: PNode, it: TIter): PNode = - result = newNodeI(nkStmtList, n.info) - var stateAsgnStmt = newNodeI(nkAsgn, n.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), it.state, - n.info)) - stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) - result.add(stateAsgnStmt) - result.add(n) - -proc outerProcSons(o: POuterContext, n: PNode, it: TIter) = - for i in countup(0, sonsLen(n) - 1): - let x = transformOuterProc(o, n.sons[i], it) - if x != nil: n.sons[i] = x +template isIterator(owner: PSym): bool = + owner.kind == skIterator and owner.typ.callConv == ccClosure proc liftIterSym(n: PNode; owner: PSym): PNode = # transforms (iter) to (let env = newClosure[iter](); (iter, env)) let iter = n.sym - assert iter.kind == skClosureIterator + assert iter.isIterator result = newNodeIT(nkStmtListExpr, n.info, n.typ) @@ -745,227 +252,392 @@ proc liftIterSym(n: PNode; owner: PSym): PNode = result.add newCall(getSysSym"internalNew", envAsNode) result.add makeClosure(iter, envAsNode, n.info) -when false: - proc transformRemainingLocals(n: PNode; it: TIter): PNode = - assert it.fn.kind == skClosureIterator - result = n - case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard - of nkSym: - let local = n.sym - if interestingIterVar(local) and it.fn == local.owner: - addUniqueField(it.obj, local) - result = indirectAccess(newSymNode(it.closureParam), local, n.info) +# ------------------ new stuff ------------------------------------------- + +proc createUpField(dest, dep: PSym) = + let obj = getHiddenParam(dest).typ.lastSon + let fieldType = getHiddenParam(dep).typ + let pos = obj.n.len + let result = newSym(skField, getIdent(upName), obj.owner, obj.owner.info) + result.typ = fieldType + result.position = pos + addField(obj, result) + +proc markAsClosure(owner: PSym; n: PNode) = + owner.typ.callConv = ccClosure + incl(owner.typ.flags, tfCapturesEnv) + let s = n.sym + if illegalCapture(s) or owner.typ.callConv notin {ccClosure, ccDefault}: + localError(n.info, errIllegalCaptureX, s.name.s) + +type + DetectionPass = object + processed, capturedVars: IntSet + ownerToType: Table[int, PType] + somethingToDo: bool + +proc initDetectionPass(fn: PSym): DetectionPass = + result.processed = initIntSet() + result.capturedVars = initIntSet() + result.ownerToType = initTable[int, PType]() + result.processed.incl(fn.id) + +discard """ +proc outer = + var a, b: int + proc innerA = use(a) + proc innerB = use(b); innerA() +# --> innerA and innerB need to *share* the closure type! +This is why need to store the 'ownerToType' table and use it +during .closure'fication. +""" + +proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym): PType = + result = c.ownerToType.getOrDefault(owner.id) + if result.isNil: + result = newType(tyRef, owner) + let obj = createEnvObj(owner) + rawAddSon(result, obj) + c.ownerToType[owner.id] = result + +proc addClosureParam(c: var DetectionPass; fn, owner: PSym) = + var cp = getEnvParam(fn) + if cp == nil: + cp = newSym(skParam, getIdent(paramName), fn, fn.info) + incl(cp.flags, sfFromGeneric) + cp.typ = c.getEnvTypeForOwner(owner) + addHiddenParam(fn, cp) + +proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = + case n.kind + of nkSym: + let s = n.sym + let innerProc = isInnerProc(s) + if innerProc: + if not c.processed.containsOrIncl(s.id): + detectCapturedVars(s.getBody, s, c) + let ow = s.skipGenericOwner + if ow == owner: + if owner.isIterator and interestingIterVar(s): + c.somethingToDo = true + if not c.capturedVars.containsOrIncl(s.id): + let obj = getHiddenParam(owner).typ.lastSon + addField(obj, s) + # but always return because the rest of the proc is only relevant when + # ow != owner: + return + # direct or indirect dependency: + if innerProc or interestingVar(s): + discard """ + proc outer() = + var x: int + proc inner() = + proc innerInner() = + echo x + innerInner() + inner() + + # inner() takes a closure too! + """ + + echo "came here for ", n.sym.name.s, " proc ", owner.name.s + + # mark 'owner' as taking a closure: + c.somethingToDo = true + markAsClosure(owner, n) + addClosureParam(c, owner, ow) + # variable 's' is actually captured: + if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): + let obj = getHiddenParam(owner).typ.lastSon + addField(obj, s) + # create required upFields: + var w = ow + if false: # XXX rethink this w != owner: + var up = w + while w != nil and w.kind != skModule: + w = w.skipGenericOwner + markAsClosure(w, n) + discard """ + proc outer = + var a, b: int + proc outerB = + proc innerA = use(a) + proc innerB = use(b); innerA() + # --> make outerB of calling convention .closure and + # give it the same env type that outer's env var gets: + """ + addClosureParam(c, w, ow) + createUpField(w, up) + if w == owner: break + up = w + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, + nkTemplateDef, nkTypeSection: + discard + of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef: + discard + else: + for i in 0.. 0 or env.upField != nil) +proc newEnvVar(owner: PSym; typ: PType): PNode = + var v = newSym(skVar, getIdent(envName), owner, owner.info) + incl(v.flags, sfShadowed) + v.typ = typ + if owner.kind == skIterator and owner.typ.callConv == ccClosure: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + result = indirectAccess(newSymNode(it), v, v.info) + else: + result = newSymNode(v) -# We have to split up environment creation in 2 steps: -# 1. Generate it and store it in env.replacementNode -# 2. Insert replacementNode into its forseen slot. -# This split is necessary so that assignments belonging to closure -# creation like 'env.param = param' are not transformed -# into 'env.param = env.param'. -proc createEnvironments(o: POuterContext) = - var env = o.head - while env != nil: - if envActive(env): - var scope = env.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # prepare for closure construction: - env.replacementNode = generateClosureCreation(o, env) - env = env.next +proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + if c.envVar.isNil: + assert c.envCreation.isNil + let envVarType = d.ownerToType.getOrDefault(owner.id) + if envVarType.isNil: + localError n.info, "internal error: could not determine closure type" + c.envVar = newEnvVar(owner, envVarType) + c.envCreation = rawClosureCreation(c.envVar, owner, d, + envVarType.sons[0]) + let access = c.envVar + let obj = access.typ.sons[0] + let field = getFieldFromObj(obj, n.sym) + if field != nil: + result = rawIndirectAccess(access, field, n.info) + else: + localError(n.info, "internal error: not part of closure object type") + result = n -proc finishEnvironments(o: POuterContext) = - var env = o.head - while env != nil: - if env.replacementNode != nil: - var scope = env.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # change the empty node to contain the closure construction: - scope.sons[0] = env.replacementNode - when false: - if env.fn.kind == skClosureIterator: - scope.sons[0] = transformRemainingLocals(env.replacementNode, - initIter(env.fn)) - else: - scope.sons[0] = env.replacementNode - env = env.next +proc getStateField(owner: PSym): PSym = + getEnvParam(owner).typ.sons[0].n.sons[0].sym -proc transformOuterProcBody(o: POuterContext, n: PNode; it: TIter): PNode = - if nfLL in n.flags: - result = nil - elif it.isIterator: +proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode + +proc transformYield(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let state = getStateField(owner) + assert state != nil + assert state.typ != nil + assert state.typ.n != nil + inc state.typ.n.sons[1].intVal + let stateNo = state.typ.n.sons[1].intVal + + var stateAsgnStmt = newNodeI(nkAsgn, n.info) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(getEnvParam(owner)), + state, n.info)) + stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt))) + + var retStmt = newNodeI(nkReturnStmt, n.info) + if n.sons[0].kind != nkEmpty: + var a = newNodeI(nkAsgn, n.sons[0].info) + var retVal = liftCapturedVars(n.sons[0], owner, d, c) + addSon(a, newSymNode(getIterResult(owner))) + addSon(a, retVal) + retStmt.add(a) + else: + retStmt.add(emptyNode) + + var stateLabelStmt = newNodeI(nkState, n.info) + stateLabelStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt))) + + result = newNodeI(nkStmtList, n.info) + result.add(stateAsgnStmt) + result.add(retStmt) + result.add(stateLabelStmt) + +proc transformReturn(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + let state = getStateField(owner) + result = newNodeI(nkStmtList, n.info) + var stateAsgnStmt = newNodeI(nkAsgn, n.info) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(getEnvParam(owner)), + state, n.info)) + stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) + result.add(stateAsgnStmt) + result.add(n) + +proc wrapIterBody(n: PNode; owner: PSym): PNode = + if not owner.isIterator: return n + when false: # unfortunately control flow is still convoluted and we can end up # multiple times here for the very same iterator. We shield against this # with some rather primitive check for now: if n.kind == nkStmtList and n.len > 0: - if n.sons[0].kind == nkGotoState: return nil + if n.sons[0].kind == nkGotoState: return n if n.len > 1 and n[1].kind == nkStmtList and n[1].len > 0 and n[1][0].kind == nkGotoState: - return nil - result = newNodeI(nkStmtList, it.fn.info) - var gs = newNodeI(nkGotoState, it.fn.info) - assert it.closureParam != nil - assert it.state != nil - gs.add(rawIndirectAccess(newSymNode(it.closureParam), it.state, it.fn.info)) - result.add(gs) - var state0 = newNodeI(nkState, it.fn.info) - state0.add(newIntNode(nkIntLit, 0)) - result.add(state0) + return n + let info = n.info + result = newNodeI(nkStmtList, info) + var gs = newNodeI(nkGotoState, info) + gs.add(rawIndirectAccess(newSymNode(owner.getHiddenParam), getStateField(owner), info)) + result.add(gs) + var state0 = newNodeI(nkState, info) + state0.add(newIntNode(nkIntLit, 0)) + result.add(state0) - let newBody = transformOuterProc(o, n, it) - if newBody != nil: - result.add(newBody) - else: - result.add(n) + result.add(n) - var stateAsgnStmt = newNodeI(nkAsgn, it.fn.info) - stateAsgnStmt.add(rawIndirectAccess(newSymNode(it.closureParam), - it.state, it.fn.info)) - stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) - result.add(stateAsgnStmt) - result.flags.incl nfLL + var stateAsgnStmt = newNodeI(nkAsgn, info) + stateAsgnStmt.add(rawIndirectAccess(newSymNode(owner.getHiddenParam), + getStateField(owner), info)) + stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) + result.add(stateAsgnStmt) + result.flags.incl nfLL + +proc symToClosure(n: PNode; owner: PSym; c: LiftingPass): PNode = + let s = n.sym + # direct dependency, so use the outer's env variable: + if s.skipGenericOwner == owner: + result = makeClosure(s, c.envVar, n.info) + elif s == owner: + # recursive calls go through (lambda, hiddenParam): + let available = getHiddenParam(owner) + result = makeClosure(s, available.newSymNode, n.info) else: - result = transformOuterProc(o, n, it) - if result != nil: result.flags.incl nfLL + let available = getHiddenParam(owner) + let wanted = getHiddenParam(s).typ + # ugh: call through some other inner proc; + var access = newSymNode(available) + while true: + if access.typ == wanted: + return makeClosure(s, access, n.info) + let obj = access.typ.sons[0] + let upField = lookupInRecord(obj.n, getIdent(upName)) + if upField == nil: + localError(n.info, "internal error: no environment found") + break + access = rawIndirectAccess(access, upField, n.info) -proc transformOuterProc(o: POuterContext, n: PNode; it: TIter): PNode = - if n == nil or nfLL in n.flags: return nil +proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + result = n case n.kind - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard of nkSym: - var local = n.sym - - if isInnerProc(local, o.fn) and o.processed.contains(local.id): - o.processed.excl(local.id) - let body = local.getBody - let newBody = transformOuterProcBody(o, body, initIter(local)) - if newBody != nil: - local.ast.sons[bodyPos] = newBody - - if it.isIterator and interestingIterVar(local) and - it.fn == local.owner: - # every local goes through the closure: - #if not containsOrIncl(o.capturedVars, local.id): - # addField(it.obj, local) - if contains(o.capturedVars, local.id): - # change 'local' to 'closure.local', unless it's a 'byCopy' variable: - # if sfByCopy notin local.flags: - result = idNodeTableGet(o.localsToAccess, local) - assert result != nil, "cannot find: " & local.name.s - return result + let s = n.sym + # if s.kind == skIterator and s.typ.callConv == ccClosure: + # consider: [i1, i2, i1] Since we merged the iterator's closure + # with the captured owning variables, we need to generate the + # closure generation code again: XXX think about this more, + # closure iterators are really strange in this regard. + if isInnerProc(s) and s.typ.callConv == ccClosure: + if not c.processed.containsOrIncl(s.id): + let oldEnvVar = c.envVar + let oldEnvCreation = c.envCreation + c.envVar = nil + c.envCreation = nil + let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) + if c.envCreation.isNil: + s.ast.sons[bodyPos] = body + else: + s.ast.sons[bodyPos] = newTree(nkStmtList, c.envCreation, body) + c.envCreation = nil + c.envVar = oldEnvVar + c.envCreation = oldEnvCreation + result = symToClosure(n, owner, c) + elif s.id in d.capturedVars: + if s.owner != owner: + result = accessViaEnvParam(n, owner) + elif owner.kind == skIterator and interestingIterVar(s): + result = accessViaEnvParam(n, owner) else: - addUniqueField(it.obj, local) - return indirectAccess(newSymNode(it.closureParam), local, n.info) - - if local.kind == skClosureIterator: - # bug #3354; allow for - #iterator iter(): int {.closure.}= - # s.add(iter) - # yield 1 - - #if local == o.fn or local == it.fn: - # message(n.info, errRecursiveDependencyX, local.name.s) - - # consider: [i1, i2, i1] Since we merged the iterator's closure - # with the captured owning variables, we need to generate the - # closure generation code again: - # XXX why doesn't this work? - var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure.isNil: - return liftIterSym(n, o.fn) - else: - let createdVar = generateIterClosureCreation(o, closure, - closure.attachedNode) - let lpt = getHiddenParam(local).typ - if lpt != createdVar.typ: - assert lpt.kind == tyRef and createdVar.typ.kind == tyRef - # fix bug 'tshallowcopy_closures' but report if this gets any weirder: - if createdVar.typ.sons[0].len == 1 and lpt.sons[0].len >= 1: - createdVar.typ = lpt - if createdVar.kind == nkSym: createdVar.sym.typ = lpt - closure.obj = lpt.sons[0] - else: - internalError(n.info, "environment computation failed") - return makeClosure(local, createdVar, n.info) - - var closure = PEnv(idTableGet(o.lambdasToEnv, local)) - if closure != nil: - # we need to replace the lambda with '(lambda, env)': - let a = closure.createdVar - if a != nil: - return makeClosure(local, a, n.info) - else: - # can happen for dummy closures: - var scope = closure.attachedNode - assert scope.kind == nkStmtList - if scope.sons[0].kind == nkEmpty: - # change the empty node to contain the closure construction: - scope.sons[0] = generateClosureCreation(o, closure) - let x = closure.createdVar - assert x != nil - return makeClosure(local, x, n.info) - - if not contains(o.capturedVars, local.id): return - # change 'local' to 'closure.local', unless it's a 'byCopy' variable: - # if sfByCopy notin local.flags: - result = idNodeTableGet(o.localsToAccess, local) - assert result != nil, "cannot find: " & local.name.s - # else it is captured by copy and this means that 'outer' should continue - # to access the local as a local. - of nkLambdaKinds, nkIteratorDef: - if n.typ != nil: - result = transformOuterProc(o, n.sons[namePos], it) - of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef: - # don't recurse here: + result = accessViaEnvVar(n, owner, d, c) + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, + nkTemplateDef, nkTypeSection: + discard + of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef: discard - of nkClosure: - if n.sons[0].kind == nkSym: - var local = n.sons[0].sym - if isInnerProc(local, o.fn) and o.processed.contains(local.id): - o.processed.excl(local.id) - let body = local.getBody - let newBody = transformOuterProcBody(o, body, initIter(local)) - if newBody != nil: - local.ast.sons[bodyPos] = newBody - when false: - if n.sons[1].kind == nkSym: - var local = n.sons[1].sym - if it.isIterator and interestingIterVar(local) and - it.fn == local.owner: - # every local goes through the closure: - addUniqueField(it.obj, local) - n.sons[1] = indirectAccess(newSymNode(it.closureParam), local, n.info) - of nkHiddenStdConv, nkHiddenSubConv, nkConv: - let x = transformOuterProc(o, n.sons[1], it) - if x != nil: n.sons[1] = x - result = transformOuterConv(n) - of nkYieldStmt: - if it.isIterator: result = transformYield(o, n, it) - else: outerProcSons(o, n, it) - of nkReturnStmt: - if it.isIterator: result = transformReturn(o, n, it) - else: outerProcSons(o, n, it) else: - outerProcSons(o, n, it) + if owner.isIterator and n.kind == nkYieldStmt: + result = transformYield(n, owner, d, c) + elif owner.isIterator and n.kind == nkReturnStmt: + result = transformReturn(n, owner, d, c) + else: + for i in 0.. we don't want the restriction - # to 'skIterators' anymore; skIterators are preferred in sigmatch already + # to 'skIterator' anymore; skIterator is preferred in sigmatch already # for typeof support. # for ``type(countup(1,3))``, see ``tests/ttoseq``. result = semOverloadedCall(c, n, nOrig, - {skProc, skMethod, skConverter, skMacro, skTemplate}+skIterators) + {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator}) else: result = semOverloadedCall(c, n, nOrig, {skProc, skMethod, skConverter, skMacro, skTemplate}) @@ -770,7 +770,7 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, case callee.kind of skMacro, skTemplate: discard else: - if callee.kind in skIterators and callee.id == c.p.owner.id: + if callee.kind == skIterator and callee.id == c.p.owner.id: localError(n.info, errRecursiveDependencyX, callee.name.s) # error correction, prevents endless for loop elimination in transf. # See bug #2051: @@ -1201,7 +1201,7 @@ proc semSubscript(c: PContext, n: PNode, flags: TExprFlags): PNode = let s = if n.sons[0].kind == nkSym: n.sons[0].sym elif n[0].kind in nkSymChoices: n.sons[0][0].sym else: nil - if s != nil and s.kind in {skProc, skMethod, skConverter}+skIterators: + if s != nil and s.kind in {skProc, skMethod, skConverter, skIterator}: # type parameters: partial generic specialization n.sons[0] = semSymGenericInstantiation(c, n.sons[0], s) result = explicitGenericInstantiation(c, n, s) @@ -1349,8 +1349,8 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = proc semReturn(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) - if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro, - skClosureIterator}: + if c.p.owner.kind in {skConverter, skMethod, skProc, skMacro} or ( + c.p.owner.kind == skIterator and c.p.owner.typ.callConv == ccClosure): if n.sons[0].kind != nkEmpty: # transform ``return expr`` to ``result = expr; return`` if c.p.resultSym != nil: @@ -1426,7 +1426,7 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) = proc semYield(c: PContext, n: PNode): PNode = result = n checkSonsLen(n, 1) - if c.p.owner == nil or c.p.owner.kind notin skIterators: + if c.p.owner == nil or c.p.owner.kind != skIterator: localError(n.info, errYieldNotAllowedHere) elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline: localError(n.info, errYieldNotAllowedInTryStmt) @@ -2124,7 +2124,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = var s = lookUp(c, n) if c.inTypeClass == 0: semCaptureSym(s, c.p.owner) result = semSym(c, n, s, flags) - if s.kind in {skProc, skMethod, skConverter}+skIterators: + if s.kind in {skProc, skMethod, skConverter, skIterator}: #performProcvarCheck(c, n, s) result = symChoice(c, n, s, scClosed) if result.kind == nkSym: @@ -2212,7 +2212,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = localError(n.info, errUseQualifier, s.name.s) elif s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) - of skProc, skMethod, skConverter, skIterators: + of skProc, skMethod, skConverter, skIterator: if s.magic == mNone: result = semDirectOp(c, n, flags) else: result = semMagic(c, n, s, flags) else: diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index 620453277..6651de78e 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -58,7 +58,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym, of skUnknown: # Introduced in this pass! Leave it as an identifier. result = n - of skProc, skMethod, skIterators, skConverter, skModule: + of skProc, skMethod, skIterator, skConverter, skModule: result = symChoice(c, n, s, scOpen) of skTemplate: if macroToExpand(s): @@ -226,7 +226,7 @@ proc semGenericStmt(c: PContext, n: PNode, of skUnknown, skParam: # Leave it as an identifier. discard - of skProc, skMethod, skIterators, skConverter, skModule: + of skProc, skMethod, skIterator, skConverter, skModule: result.sons[0] = symChoice(c, fn, s, scOption) # do not check of 's.magic==mRoof' here because it might be some # other '^' but after overload resolution the proper one: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index e80f1cfda..f54f523ad 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -84,7 +84,7 @@ proc performProcvarCheck(c: PContext, n: PNode, s: PSym) = proc semProcvarCheck(c: PContext, n: PNode) = let n = n.skipConv if n.kind == nkSym and n.sym.kind in {skProc, skMethod, skConverter, - skIterator, skClosureIterator}: + skIterator}: performProcvarCheck(c, n, n.sym) proc semProc(c: PContext, n: PNode): PNode @@ -598,7 +598,7 @@ proc semFor(c: PContext, n: PNode): PNode = # first class iterator: result = semForVars(c, n) elif not isCallExpr or call.sons[0].kind != nkSym or - call.sons[0].sym.kind notin skIterators: + call.sons[0].sym.kind != skIterator: if length == 3: n.sons[length-2] = implicitIterator(c, "items", n.sons[length-2]) elif length == 4: @@ -1143,13 +1143,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if tfTriggersCompileTime in s.typ.flags: incl(s.flags, sfCompileTime) if n.sons[patternPos].kind != nkEmpty: n.sons[patternPos] = semPattern(c, n.sons[patternPos]) - if s.kind in skIterators: + if s.kind == skIterator: s.typ.flags.incl(tfIterator) var proto = searchForProc(c, oldScope, s) if proto == nil: - if s.kind == skClosureIterator: s.typ.callConv = ccClosure - else: s.typ.callConv = lastOptionEntry(c).defaultCC + if s.kind == skIterator and s.typ.callConv == ccClosure: + discard + else: + s.typ.callConv = lastOptionEntry(c).defaultCC # add it here, so that recursive procs are possible: if sfGenSym in s.flags: discard elif kind in OverloadableSyms: @@ -1209,7 +1211,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, n.sons[bodyPos] = transformBody(c.module, semBody, s) popProcCon(c) else: - if s.typ.sons[0] != nil and kind notin skIterators: + if s.typ.sons[0] != nil and kind != skIterator: addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info)) openScope(c) n.sons[bodyPos] = semGenericStmt(c, n.sons[bodyPos]) @@ -1230,7 +1232,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if n.sons[patternPos].kind != nkEmpty: c.patterns.add(s) if isAnon: result.typ = s.typ - if isTopLevel(c) and s.kind != skClosureIterator and + if isTopLevel(c) and s.kind != skIterator and s.typ.callConv == ccClosure: message(s.info, warnDeprecated, "top level '.closure' calling convention") @@ -1240,15 +1242,12 @@ proc determineType(c: PContext, s: PSym) = discard semProcAux(c, s.ast, s.kind, {}, stepDetermineType) proc semIterator(c: PContext, n: PNode): PNode = - let kind = if hasPragma(n[pragmasPos], wClosure) or - n[namePos].kind == nkEmpty: skClosureIterator - else: skIterator # gensym'ed iterator? if n[namePos].kind == nkSym: # gensym'ed iterators might need to become closure iterators: n[namePos].sym.owner = getCurrOwner() - n[namePos].sym.kind = kind - result = semProcAux(c, n, kind, iteratorPragmas) + n[namePos].sym.kind = skIterator + result = semProcAux(c, n, skIterator, iteratorPragmas) var s = result.sons[namePos].sym var t = s.typ if t.sons[0] == nil and s.typ.callConv != ccClosure: diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 2dda8276d..b19ffd77f 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -456,9 +456,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode = of nkMethodDef: result = semRoutineInTemplBody(c, n, skMethod) of nkIteratorDef: - let kind = if hasPragma(n[pragmasPos], wClosure): skClosureIterator - else: skIterator - result = semRoutineInTemplBody(c, n, kind) + result = semRoutineInTemplBody(c, n, skIterator) of nkTemplateDef: result = semRoutineInTemplBody(c, n, skTemplate) of nkMacroDef: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 65edb756f..d27702e8c 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -1296,7 +1296,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = child.flags.incl tfIterator result.addSonSkipIntLit(child) else: - result = semProcTypeWithScope(c, n, prev, skClosureIterator) + result = semProcTypeWithScope(c, n, prev, skIterator) result.flags.incl(tfIterator) if n.lastSon.kind == nkPragma and hasPragma(n.lastSon, wInline): result.callConv = ccInline diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index aee42e021..784294839 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1442,7 +1442,7 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType, z.calleeSym = m.calleeSym var best = -1 for i in countup(0, sonsLen(arg) - 1): - if arg.sons[i].sym.kind in {skProc, skMethod, skConverter}+skIterators: + if arg.sons[i].sym.kind in {skProc, skMethod, skConverter, skIterator}: copyCandidate(z, m) z.callee = arg.sons[i].typ z.calleeSym = arg.sons[i].sym diff --git a/compiler/transf.nim b/compiler/transf.nim index 3a5ff982e..98aafaaff 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -348,6 +348,20 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode = # addr ( deref ( x )) --> x result = PTransNode(n.sons[0].sons[0]) +proc generateThunk(prc: PNode, dest: PType): PNode = + ## Converts 'prc' into '(thunk, nil)' so that it's compatible with + ## a closure. + + # we cannot generate a proper thunk here for GC-safety reasons + # (see internal documentation): + if gCmd == cmdCompileToJS: return prc + result = newNodeIT(nkClosure, prc.info, dest) + var conv = newNodeIT(nkHiddenStdConv, prc.info, dest) + conv.add(emptyNode) + conv.add(prc) + result.add(conv) + result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil))) + proc transformConv(c: PTransf, n: PNode): PTransNode = # numeric types need range checks: var dest = skipTypes(n.typ, abstractVarRange) @@ -428,6 +442,10 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = of tyGenericParam, tyOrdinal: result = transform(c, n.sons[1]) # happens sometimes for generated assignments, etc. + of tyProc: + result = transformSons(c, n) + if dest.callConv == ccClosure and source.callConv == ccDefault: + result = generateThunk(result[1].PNode, dest).PTransNode else: result = transformSons(c, n) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 2a16406e7..75c1378e5 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1209,7 +1209,7 @@ proc checkCanEval(c: PCtx; n: PNode) = not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl: cannotEval(n) elif s.kind in {skProc, skConverter, skMethod, - skIterator, skClosureIterator} and sfForward in s.flags: + skIterator} and sfForward in s.flags: cannotEval(n) proc isTemp(c: PCtx; dest: TDest): bool = @@ -1638,7 +1638,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = case s.kind of skVar, skForVar, skTemp, skLet, skParam, skResult: genRdVar(c, n, dest, flags) - of skProc, skConverter, skMacro, skTemplate, skMethod, skIterators: + of skProc, skConverter, skMacro, skTemplate, skMethod, skIterator: # 'skTemplate' is only allowed for 'getAst' support: if procIsCallback(c, s): discard elif sfImportc in s.flags: c.importcSym(n.info, s) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index c8cb3d199..872d4848d 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -97,7 +97,7 @@ type nskUnknown, nskConditional, nskDynLib, nskParam, nskGenericParam, nskTemp, nskModule, nskType, nskVar, nskLet, nskConst, nskResult, - nskProc, nskMethod, nskIterator, nskClosureIterator, + nskProc, nskMethod, nskIterator, nskConverter, nskMacro, nskTemplate, nskField, nskEnumField, nskForVar, nskLabel, nskStub From 7dc0bca53ae6727c67b899d979c5fe40b1deadd3 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 26 Dec 2015 01:26:12 +0100 Subject: [PATCH 039/125] up references might work; got rid of debugging code --- compiler/lambdalifting.nim | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 00849b47e..139a3f9ce 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -340,8 +340,6 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # inner() takes a closure too! """ - echo "came here for ", n.sym.name.s, " proc ", owner.name.s - # mark 'owner' as taking a closure: c.somethingToDo = true markAsClosure(owner, n) @@ -351,9 +349,9 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = let obj = getHiddenParam(owner).typ.lastSon addField(obj, s) # create required upFields: - var w = ow - if false: # XXX rethink this w != owner: - var up = w + var w = ow.skipGenericOwner + if w != owner: + var up = ow while w != nil and w.kind != skModule: w = w.skipGenericOwner markAsClosure(w, n) From 9309f8101d5acf349611314267dbdf1f9a352022 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 27 Dec 2015 00:17:48 +0100 Subject: [PATCH 040/125] new-ll: further progress (bootstrapping still fails) --- compiler/lambdalifting.nim | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 139a3f9ce..613fb8555 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -169,7 +169,7 @@ proc addHiddenParam(routine: PSym, param: PSym) = proc getHiddenParam(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] let hidden = lastSon(params) - if hidden.kind == nkSym and hidden.sym.kind == skParam: + if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName: result = hidden.sym assert sfFromGeneric in result.flags else: @@ -194,7 +194,7 @@ proc illegalCapture(s: PSym): bool {.inline.} = s.kind == skResult proc isInnerProc(s: PSym): bool = - if s.kind in {skProc, skMethod, skConverter, skIterator}: + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.magic == mNone: result = s.skipGenericOwner.kind in routineKinds proc createUpField(obj, fieldType: PType): PSym = @@ -327,7 +327,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # ow != owner: return # direct or indirect dependency: - if innerProc or interestingVar(s): + if (innerProc and s.typ.callConv == ccClosure) or interestingVar(s): discard """ proc outer() = var x: int @@ -371,8 +371,11 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, nkTemplateDef, nkTypeSection: discard - of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef: + of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard + of nkLambdaKinds, nkIteratorDef: + if n.typ != nil: + detectCapturedVars(n[namePos], owner, c) else: for i in 0.. Date: Mon, 28 Dec 2015 23:39:37 +0100 Subject: [PATCH 041/125] compute LL before the other transformations as that should produce more efficient code for closure iterators --- compiler/transf.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 98aafaaff..c35a3251a 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -883,12 +883,12 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode = if nfTransf in n.flags or prc.kind in {skTemplate}: result = n else: + result = liftLambdas(prc, n) + #result = n var c = openTransf(module, "") - result = processTransf(c, n, prc) + result = processTransf(c, result, prc) liftDefer(c, result) - result = liftLambdas(prc, result) - #if prc.kind == skClosureIterator: - # result = lambdalifting.liftIterator(prc, result) + #result = liftLambdas(prc, result) incl(result.flags, nfTransf) when useEffectSystem: trackProc(prc, result) #if prc.name.s == "testbody": @@ -901,7 +901,7 @@ proc transformStmt*(module: PSym, n: PNode): PNode = var c = openTransf(module, "") result = processTransf(c, n, module) liftDefer(c, result) - result = liftLambdasForTopLevel(module, result) + #result = liftLambdasForTopLevel(module, result) incl(result.flags, nfTransf) when useEffectSystem: trackTopLevelStmt(module, result) From c12c41c7adde1a7d8426ebe5b7744289e9a4668e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Dec 2015 23:40:39 +0100 Subject: [PATCH 042/125] made indirectAccess slightly more efficient --- compiler/lowerings.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 6b60faba2..7a5c7f44b 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -165,9 +165,10 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode = deref.typ = a.typ.skipTypes(abstractInst).sons[0] var t = deref.typ.skipTypes(abstractInst) var field: PSym + let bb = getIdent(b) while true: assert t.kind == tyObject - field = getSymFromList(t.n, getIdent(b)) + field = getSymFromList(t.n, bb) if field != nil: break t = t.sons[0] if t == nil: break From 57149a47947cb9f8efc9addff9c92a0c39baa6ff Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Dec 2015 23:46:19 +0100 Subject: [PATCH 043/125] up references work --- compiler/lambdalifting.nim | 73 ++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 613fb8555..b7af99c3e 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -197,15 +197,6 @@ proc isInnerProc(s: PSym): bool = if s.kind in {skProc, skMethod, skConverter, skIterator} and s.magic == mNone: result = s.skipGenericOwner.kind in routineKinds -proc createUpField(obj, fieldType: PType): PSym = - let pos = obj.n.len - result = newSym(skField, getIdent(upName), obj.owner, obj.owner.info) - result.typ = newType(tyRef, obj.owner) - result.position = pos - rawAddSon(result.typ, fieldType) - #rawAddField(obj, result) - addField(obj, result) - proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = # Bugfix: unfortunately we cannot use 'nkFastAsgn' here as that would # mean to be able to capture string literals which have no GC header. @@ -254,15 +245,6 @@ proc liftIterSym(n: PNode; owner: PSym): PNode = # ------------------ new stuff ------------------------------------------- -proc createUpField(dest, dep: PSym) = - let obj = getHiddenParam(dest).typ.lastSon - let fieldType = getHiddenParam(dep).typ - let pos = obj.n.len - let result = newSym(skField, getIdent(upName), obj.owner, obj.owner.info) - result.typ = fieldType - result.position = pos - addField(obj, result) - proc markAsClosure(owner: PSym; n: PNode) = owner.typ.callConv = ccClosure incl(owner.typ.flags, tfCapturesEnv) @@ -300,13 +282,34 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym): PType = rawAddSon(result, obj) c.ownerToType[owner.id] = result -proc addClosureParam(c: var DetectionPass; fn, owner: PSym) = +proc createUpField(c: var DetectionPass; dest, dep: PSym) = + let refObj = c.getEnvTypeForOwner(dest) # getHiddenParam(dest).typ + let obj = refObj.lastSon + let fieldType = c.getEnvTypeForOwner(dep) #getHiddenParam(dep).typ + if refObj == fieldType: + localError(dep.info, "internal error: invald up reference computed") + + let upIdent = getIdent(upName) + let upField = lookupInRecord(obj.n, upIdent) + if upField != nil: + if upField.typ != fieldType: + localError(dep.info, "internal error: up references do not agree") + else: + let result = newSym(skField, upIdent, obj.owner, obj.owner.info) + result.typ = fieldType + rawAddField(obj, result) + +proc addClosureParam(c: var DetectionPass; fn: PSym) = var cp = getEnvParam(fn) + let owner = fn.skipGenericOwner + let t = c.getEnvTypeForOwner(owner) if cp == nil: cp = newSym(skParam, getIdent(paramName), fn, fn.info) incl(cp.flags, sfFromGeneric) - cp.typ = c.getEnvTypeForOwner(owner) + cp.typ = t addHiddenParam(fn, cp) + elif cp.typ != t: + localError(fn.info, "internal error: inconsistent environment type") proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind @@ -322,6 +325,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = c.somethingToDo = true if not c.capturedVars.containsOrIncl(s.id): let obj = getHiddenParam(owner).typ.lastSon + #let obj = c.getEnvTypeForOwner(s.owner).lastSon addField(obj, s) # but always return because the rest of the proc is only relevant when # ow != owner: @@ -336,25 +340,21 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = echo x innerInner() inner() - # inner() takes a closure too! """ - # mark 'owner' as taking a closure: c.somethingToDo = true markAsClosure(owner, n) - addClosureParam(c, owner, ow) + addClosureParam(c, owner) # variable 's' is actually captured: if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): - let obj = getHiddenParam(owner).typ.lastSon + let obj = c.getEnvTypeForOwner(ow).lastSon + #getHiddenParam(owner).typ.lastSon addField(obj, s) # create required upFields: - var w = ow.skipGenericOwner - if w != owner: - var up = ow - while w != nil and w.kind != skModule: - w = w.skipGenericOwner - markAsClosure(w, n) + var w = owner.skipGenericOwner + if isInnerProc(w): + while w != nil and w.kind != skModule and ow != w: discard """ proc outer = var a, b: int @@ -364,10 +364,12 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # --> make outerB of calling convention .closure and # give it the same env type that outer's env var gets: """ - addClosureParam(c, w, ow) - createUpField(w, up) - if w == owner: break - up = w + let up = w.skipGenericOwner + #echo "up for ", w.name.s, " up ", up.name.s + markAsClosure(w, n) + addClosureParam(c, w) # , ow + createUpField(c, w, up) + w = up of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, nkTemplateDef, nkTypeSection: discard @@ -398,6 +400,7 @@ proc accessViaEnvParam(n: PNode; owner: PSym): PNode = var access = newSymNode(envParam) while true: let obj = access.typ.sons[0] + assert obj.kind == tyObject let field = getFieldFromObj(obj, s) if field != nil: return rawIndirectAccess(access, field, n.info) @@ -429,7 +432,7 @@ proc rawClosureCreation(env: PNode; owner: PSym; if upField != nil: let param = getHiddenParam(owner) if upField.typ == param.typ: - result.add(newAsgnStmt(indirectAccess(env, upField, env.info), + result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), newSymNode(param), env.info)) else: localError(env.info, "internal error: cannot create up reference") From 870b03707b17819defec79149cc3748d15ef57e4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 01:59:36 +0100 Subject: [PATCH 044/125] bootstrapping works again --- compiler/lambdalifting.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index b7af99c3e..4ccaadc0d 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -608,8 +608,10 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard of nkLambdaKinds, nkIteratorDef: - if n.typ != nil: - discard liftCapturedVars(n[namePos], owner, d, c) + if n.typ != nil and n[namePos].kind == nkSym: + let m = newSymNode(n[namePos].sym) + m.typ = n.typ + result = liftCapturedVars(m, owner, d, c) else: if owner.isIterator and n.kind == nkYieldStmt: result = transformYield(n, owner, d, c) From 5c9332c57d0bc86f73cd93bfe6063206d96d00a7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 02:44:37 +0100 Subject: [PATCH 045/125] fixes closure generation when only paramters are captured --- compiler/lambdalifting.nim | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 4ccaadc0d..b303d11fb 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -387,6 +387,7 @@ type processed: IntSet envCreation: PNode envVar: PNode + owner: PSym # the owner of the 'envVar' proc initLiftingPass(fn: PSym): LiftingPass = result.processed = initIntSet() @@ -448,16 +449,20 @@ proc newEnvVar(owner: PSym; typ: PType): PNode = else: result = newSymNode(v) -proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; - c: var LiftingPass): PNode = - if c.envVar.isNil: - assert c.envCreation.isNil +proc setupEnvVar(owner: PSym; d: DetectionPass; + c: var LiftingPass) = + if c.owner != owner or c.envVar.isNil: let envVarType = d.ownerToType.getOrDefault(owner.id) if envVarType.isNil: - localError n.info, "internal error: could not determine closure type" + localError owner.info, "internal error: could not determine closure type" c.envVar = newEnvVar(owner, envVarType) c.envCreation = rawClosureCreation(c.envVar, owner, d, envVarType.sons[0]) + c.owner = owner + +proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + setupEnvVar(owner, d, c) let access = c.envVar let obj = access.typ.sons[0] let field = getFieldFromObj(obj, n.sym) @@ -545,10 +550,16 @@ proc wrapIterBody(n: PNode; owner: PSym): PNode = result.add(stateAsgnStmt) result.flags.incl nfLL -proc symToClosure(n: PNode; owner: PSym; c: LiftingPass): PNode = +proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = let s = n.sym # direct dependency, so use the outer's env variable: if s.skipGenericOwner == owner: + if c.owner != owner: + setupEnvVar(owner, d, c) + if c.owner != owner or c.envVar.isNil: + localError(n.info, "internal error: no environment var found") + return n result = makeClosure(s, c.envVar, n.info) elif s == owner: # recursive calls go through (lambda, hiddenParam): @@ -566,7 +577,7 @@ proc symToClosure(n: PNode; owner: PSym; c: LiftingPass): PNode = let upField = lookupInRecord(obj.n, getIdent(upName)) if upField == nil: localError(n.info, "internal error: no environment found") - break + return n access = rawIndirectAccess(access, upField, n.info) proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; @@ -584,8 +595,10 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; if not c.processed.containsOrIncl(s.id): let oldEnvVar = c.envVar let oldEnvCreation = c.envCreation + let oldOwner = c.owner c.envVar = nil c.envCreation = nil + c.owner = s let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) if c.envCreation.isNil: s.ast.sons[bodyPos] = body @@ -594,7 +607,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; c.envCreation = nil c.envVar = oldEnvVar c.envCreation = oldEnvCreation - result = symToClosure(n, owner, c) + c.owner = oldOwner + result = symToClosure(n, owner, d, c) elif s.id in d.capturedVars: if s.owner != owner: result = accessViaEnvParam(n, owner) @@ -672,6 +686,7 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = else: result = body #if fn.name.s == "outer": + # echo "had something to do ", d.somethingToDo # echo renderTree(result, {renderIds}) proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode = From 9971c0606228e44e55b0a9574a10710bef0a7887 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 13:41:35 +0100 Subject: [PATCH 046/125] fixes missing 'illegal capture' error message regression --- compiler/lambdalifting.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index b303d11fb..706594ff1 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -246,11 +246,11 @@ proc liftIterSym(n: PNode; owner: PSym): PNode = # ------------------ new stuff ------------------------------------------- proc markAsClosure(owner: PSym; n: PNode) = - owner.typ.callConv = ccClosure - incl(owner.typ.flags, tfCapturesEnv) let s = n.sym if illegalCapture(s) or owner.typ.callConv notin {ccClosure, ccDefault}: localError(n.info, errIllegalCaptureX, s.name.s) + incl(owner.typ.flags, tfCapturesEnv) + owner.typ.callConv = ccClosure type DetectionPass = object From b6c18e4bba94ee00463f38a747ab8aa34c6b4db6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 15:42:12 +0100 Subject: [PATCH 047/125] breaking change: disallow .closure procs for const --- compiler/sem.nim | 2 +- compiler/types.nim | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index f6078830b..5134cb8b5 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -202,7 +202,7 @@ proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) = "' in this context: '" & typeToString(typ) & "'") proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} = - typeAllowedCheck(typ.n.info, typ, skConst) + typeAllowedCheck(typ.n.info, typ, skProc) proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode diff --git a/compiler/types.nim b/compiler/types.nim index 3846be8a0..71ab84022 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1061,7 +1061,8 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind, else: for i in countup(0, sonsLen(n) - 1): let it = n.sons[i] - if it.kind == nkRecCase and kind == skConst: return n.typ + if it.kind == nkRecCase and kind in {skProc, skConst}: + return n.typ result = typeAllowedNode(marker, it, kind, flags) if result != nil: break @@ -1076,7 +1077,7 @@ proc matchType*(a: PType, pattern: openArray[tuple[k:TTypeKind, i:int]], proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, flags: TTypeAllowedFlags = {}): PType = - assert(kind in {skVar, skLet, skConst, skParam, skResult}) + assert(kind in {skVar, skLet, skConst, skProc, skParam, skResult}) # if we have already checked the type, return true, because we stop the # evaluation if something is wrong: result = nil @@ -1085,7 +1086,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, var t = skipTypes(typ, abstractInst-{tyTypeDesc}) case t.kind of tyVar: - if kind == skConst: return t + if kind in {skProc, skConst}: return t var t2 = skipTypes(t.sons[0], abstractInst-{tyTypeDesc}) case t2.kind of tyVar: @@ -1097,6 +1098,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, if kind notin {skParam, skResult}: result = t else: result = typeAllowedAux(marker, t2, kind, flags) of tyProc: + if kind == skConst and t.callConv == ccClosure: return t for i in countup(1, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], skParam, flags) if result != nil: break @@ -1144,7 +1146,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = typeAllowedAux(marker, t.sons[i], kind, flags) if result != nil: break of tyObject, tyTuple: - if kind == skConst and t.kind == tyObject and t.sons[0] != nil: return t + if kind in {skProc, skConst} and + t.kind == tyObject and t.sons[0] != nil: return t let flags = flags+{taField} for i in countup(0, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], kind, flags) From bf4c13711a6765d72d568eb2b686a877546c524a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 29 Dec 2015 16:25:21 +0000 Subject: [PATCH 048/125] Fixes incorrect Host header when using httpclient with proxy. --- lib/pure/httpclient.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 8e182e274..1b91132db 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -110,7 +110,7 @@ type EInvalidProtocol: ProtocolError, EHttpRequestErr: HttpRequestError ].} -const defUserAgent* = "Nim httpclient/0.1" +const defUserAgent* = "Nim httpclient/" & NimVersion proc httpError(msg: string) = var e: ref ProtocolError @@ -389,6 +389,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. var r = if proxy == nil: parseUri(url) else: proxy.url + var hostUrl = if proxy == nil: r else: parseUri(url) var headers = substr(httpMethod, len("http")) # TODO: Use generateHeaders further down once it supports proxies. if proxy == nil: @@ -402,10 +403,10 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", headers.add(" HTTP/1.1\c\L") - if r.port == "": - add(headers, "Host: " & r.hostname & "\c\L") + if hostUrl.port == "": + add(headers, "Host: " & hostUrl.hostname & "\c\L") else: - add(headers, "Host: " & r.hostname & ":" & r.port & "\c\L") + add(headers, "Host: " & hostUrl.hostname & ":" & hostUrl.port & "\c\L") if userAgent != "": add(headers, "User-Agent: " & userAgent & "\c\L") @@ -414,7 +415,6 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", add(headers, "Proxy-Authorization: basic " & auth & "\c\L") add(headers, extraHeaders) add(headers, "\c\L") - var s = newSocket() if s == nil: raiseOSError(osLastError()) var port = net.Port(80) From 99f167724b608435faff60bd58db6d7fece54a3d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 19:41:26 +0100 Subject: [PATCH 049/125] updated test case for changed LL --- tests/closure/tclosure3.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/closure/tclosure3.nim b/tests/closure/tclosure3.nim index 8f6f4a70f..d5ffb5ab2 100644 --- a/tests/closure/tclosure3.nim +++ b/tests/closure/tclosure3.nim @@ -8,8 +8,9 @@ proc main = for iterations in 0..50_000: var s: seq[proc(): string {.closure.}] = @[] for i in 0 .. n-1: - let ii = i - s.add(proc(): string = return $(ii*ii)) + (proc () = + let ii = i + s.add(proc(): string = return $(ii*ii)))() for i in 0 .. n-1: let val = s[i]() if val != $(i*i): echo "bug ", val From e079db95f8ee59b6475fbe1c59dc667caf7a423c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 19:42:11 +0100 Subject: [PATCH 050/125] changed testcase for new LL semantics --- tests/closure/tclosure2.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/closure/tclosure2.nim b/tests/closure/tclosure2.nim index d331388cf..9c5ee1426 100644 --- a/tests/closure/tclosure2.nim +++ b/tests/closure/tclosure2.nim @@ -87,7 +87,7 @@ when true: proc py() {.closure.} = echo "py" - const + let mapping = { "abc": px, "xyz": py From bbb5ce38c765c73a489e15d2a91c322fd2ff213b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 20:25:05 +0100 Subject: [PATCH 051/125] updated test cases --- tests/closure/tclosurebug2.nim | 6 +++--- tests/closure/tnestedclosure.nim | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/closure/tclosurebug2.nim b/tests/closure/tclosurebug2.nim index 581b735bf..f131406a3 100644 --- a/tests/closure/tclosurebug2.nim +++ b/tests/closure/tclosurebug2.nim @@ -19,11 +19,11 @@ proc mustRehash(length, counter: int): bool {.inline.} = assert(length > counter) result = (length * 2 < counter * 3) or (length - counter < 4) -proc nextTry(h, maxHash: THash): THash {.inline.} = +proc nextTry(h, maxHash: Hash): Hash {.inline.} = result = ((5 * h) + 1) and maxHash template rawGetImpl() {.dirty.} = - var h: THash = hash(key) and high(t.data) # start with real hash value + var h: Hash = hash(key) and high(t.data) # start with real hash value while t.data[h].slot != seEmpty: if t.data[h].key == key and t.data[h].slot == seFilled: return h @@ -31,7 +31,7 @@ template rawGetImpl() {.dirty.} = result = -1 template rawInsertImpl() {.dirty.} = - var h: THash = hash(key) and high(data) + var h: Hash = hash(key) and high(data) while data[h].slot == seFilled: h = nextTry(h, high(data)) data[h].key = key diff --git a/tests/closure/tnestedclosure.nim b/tests/closure/tnestedclosure.nim index 67e196f66..0628a6977 100644 --- a/tests/closure/tnestedclosure.nim +++ b/tests/closure/tnestedclosure.nim @@ -21,13 +21,13 @@ proc main(param: int) = # test simple closure within dummy 'main': proc dummy = proc main2(param: int) = - var foo = 23 + var fooB = 23 proc outer(outerParam: string) = var outerVar = 88 echo outerParam, outerVar proc inner() = block Test: - echo foo, " ", param, outerParam, " ", outerVar + echo fooB, " ", param, outerParam, " ", outerVar inner() outer("foo") main2(24) From e518c7a0052e709af176183e5bad63c005ac9bbb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 20:29:44 +0100 Subject: [PATCH 052/125] LL works reasonably well; closure iterators still not implemented --- compiler/lambdalifting.nim | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 706594ff1..1467d9594 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -315,6 +315,11 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind of nkSym: let s = n.sym + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure and tfCapturesEnv notin s.typ.flags: + # this handles the case that the inner proc was declared as + # .closure but does not actually capture anything: + addClosureParam(c, s) + let innerProc = isInnerProc(s) if innerProc: if not c.processed.containsOrIncl(s.id): @@ -555,7 +560,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; let s = n.sym # direct dependency, so use the outer's env variable: if s.skipGenericOwner == owner: - if c.owner != owner: + if c.owner != owner or c.envVar.isNil: setupEnvVar(owner, d, c) if c.owner != owner or c.envVar.isNil: localError(n.info, "internal error: no environment var found") @@ -591,7 +596,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; # with the captured owning variables, we need to generate the # closure generation code again: XXX think about this more, # closure iterators are really strange in this regard. - if isInnerProc(s) and s.typ.callConv == ccClosure: + if isInnerProc(s): if not c.processed.containsOrIncl(s.id): let oldEnvVar = c.envVar let oldEnvCreation = c.envCreation @@ -608,7 +613,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; c.envVar = oldEnvVar c.envCreation = oldEnvCreation c.owner = oldOwner - result = symToClosure(n, owner, d, c) + if s.typ.callConv == ccClosure: + result = symToClosure(n, owner, d, c) elif s.id in d.capturedVars: if s.owner != owner: result = accessViaEnvParam(n, owner) From 72f804671d7ae5bda2e95cf0c374e7689845d795 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 20:34:36 +0100 Subject: [PATCH 053/125] do not print spurious warnings when dlopen fails; can be re-enabled with -d:nimDebugDlOpen --- lib/system/dyncalls.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 22ac613f8..6dc8999d1 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -68,9 +68,10 @@ when defined(posix): proc nimLoadLibrary(path: string): LibHandle = result = dlopen(path, RTLD_NOW) - let error = dlerror() - if error != nil: - c_fprintf(c_stdout, "%s\n", error) + when defined(nimDebugDlOpen): + let error = dlerror() + if error != nil: + c_fprintf(c_stdout, "%s\n", error) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) From 255177f7cd0e921ac428a5eb0ee9a27f3d4ee6ef Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Dec 2015 20:39:35 +0100 Subject: [PATCH 054/125] new db_sqlite requires deadCodeElim:on because not every sqlite3_column_name --- lib/impure/db_sqlite.nim | 2 ++ tests/closure/tissue1642.nim | 7 ------- 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 tests/closure/tissue1642.nim diff --git a/lib/impure/db_sqlite.nim b/lib/impure/db_sqlite.nim index 5cfed1eba..c0d221a0d 100644 --- a/lib/impure/db_sqlite.nim +++ b/lib/impure/db_sqlite.nim @@ -40,6 +40,8 @@ ## ## theDb.close() +{.deadCodeElim:on.} + import strutils, sqlite3 import db_common diff --git a/tests/closure/tissue1642.nim b/tests/closure/tissue1642.nim deleted file mode 100644 index e3028c88e..000000000 --- a/tests/closure/tissue1642.nim +++ /dev/null @@ -1,7 +0,0 @@ -discard """ - file: "tissue1642.nim" - disabled: true -""" -block: - var i = 0 - proc p() = inc(i) From 9cc066d99ddbeffb3e6fdacbe8f9159131bb942c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 30 Dec 2015 00:47:15 +0100 Subject: [PATCH 055/125] closure iterators sometimes work --- compiler/lambdalifting.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 1467d9594..5bdf73c5f 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -359,7 +359,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # create required upFields: var w = owner.skipGenericOwner if isInnerProc(w): - while w != nil and w.kind != skModule and ow != w: + let last = if ow.isIterator: ow.skipGenericOwner else: ow + while w != nil and w.kind != skModule and last != w: discard """ proc outer = var a, b: int @@ -413,7 +414,7 @@ proc accessViaEnvParam(n: PNode; owner: PSym): PNode = let upField = lookupInRecord(obj.n, getIdent(upName)) if upField == nil: break access = rawIndirectAccess(access, upField, n.info) - localError(n.info, "internal error: no environment parameter set") + localError(n.info, "internal error: environment misses: " & s.name.s) result = n proc rawClosureCreation(env: PNode; owner: PSym; From 2162a71c9314ed8bba7a8b9926e2d9637f7c6825 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 30 Dec 2015 13:32:18 +0100 Subject: [PATCH 056/125] closure iterators do have 'result' --- compiler/semstmts.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index f54f523ad..0e167c1f1 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1006,7 +1006,8 @@ proc activate(c: PContext, n: PNode) = discard proc maybeAddResult(c: PContext, s: PSym, n: PNode) = - if s.typ.sons[0] != nil and s.kind != skIterator: + if s.typ.sons[0] != nil and not + (s.kind == skIterator and s.typ.callConv != ccClosure): addResult(c, s.typ.sons[0], n.info, s.kind) addResultNode(c, n) From 307a6095fa69104e3b21aa796095f4b5bda12041 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 31 Dec 2015 00:16:38 +0100 Subject: [PATCH 057/125] made closure iterators tests green, updated docs --- compiler/ccgexprs.nim | 6 +++--- compiler/lambdalifting.nim | 32 ++++++++++++++++++++------------ compiler/renderer.nim | 2 ++ compiler/semstmts.nim | 2 +- compiler/semtypes.nim | 4 ---- compiler/transf.nim | 10 ++++++---- tests/closure/tjester.nim | 2 +- tests/iter/timplicit_auto.nim | 2 +- tests/iter/titer7.nim | 14 +++++--------- web/news.txt | 3 +++ 10 files changed, 42 insertions(+), 35 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 0df7e22d3..afe3ec3cf 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -2004,9 +2004,9 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = putLocIntoDest(p, d, sym.loc) of skParam: if sym.loc.r == nil or sym.loc.t == nil: - #echo "FAILED FOR PRCO ", p.prc.name.s - #debug p.prc.typ.n - #echo renderTree(p.prc.ast, {renderIds}) + # echo "FAILED FOR PRCO ", p.prc.name.s + # debug p.prc.typ.n + # echo renderTree(p.prc.ast, {renderIds}) internalError(n.info, "expr: param not init " & sym.name.s & "_" & $sym.id) putLocIntoDest(p, d, sym.loc) else: internalError(n.info, "expr(" & $sym.kind & "); unknown symbol") diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 5bdf73c5f..724d3b6d6 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -174,7 +174,7 @@ proc getHiddenParam(routine: PSym): PSym = assert sfFromGeneric in result.flags else: # writeStackTrace() - localError(routine.info, "internal error: could not find env param " & routine.name.s) + localError(routine.info, "internal error: could not find env param for " & routine.name.s) result = routine proc getEnvParam*(routine: PSym): PSym = @@ -224,7 +224,7 @@ proc interestingIterVar(s: PSym): bool {.inline.} = template isIterator(owner: PSym): bool = owner.kind == skIterator and owner.typ.callConv == ccClosure -proc liftIterSym(n: PNode; owner: PSym): PNode = +proc liftIterSym*(n: PNode; owner: PSym): PNode = # transforms (iter) to (let env = newClosure[iter](); (iter, env)) let iter = n.sym assert iter.isIterator @@ -326,12 +326,14 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = detectCapturedVars(s.getBody, s, c) let ow = s.skipGenericOwner if ow == owner: - if owner.isIterator and interestingIterVar(s): + if owner.isIterator: c.somethingToDo = true - if not c.capturedVars.containsOrIncl(s.id): - let obj = getHiddenParam(owner).typ.lastSon - #let obj = c.getEnvTypeForOwner(s.owner).lastSon - addField(obj, s) + addClosureParam(c, owner) + if interestingIterVar(s): + if not c.capturedVars.containsOrIncl(s.id): + let obj = getHiddenParam(owner).typ.lastSon + #let obj = c.getEnvTypeForOwner(s.owner).lastSon + addField(obj, s) # but always return because the rest of the proc is only relevant when # ow != owner: return @@ -479,7 +481,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; result = n proc getStateField(owner: PSym): PSym = - getEnvParam(owner).typ.sons[0].n.sons[0].sym + getHiddenParam(owner).typ.sons[0].n.sons[0].sym proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; c: var LiftingPass): PNode @@ -684,6 +686,9 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = var d = initDetectionPass(fn) var c = initLiftingPass(fn) detectCapturedVars(body, fn, d) + if not d.somethingToDo and fn.isIterator: + addClosureParam(d, fn) + d.somethingToDo = true if d.somethingToDo: var newBody = liftCapturedVars(body, fn, d, c) if not c.envCreation.isNil: @@ -692,7 +697,7 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = result = wrapIterBody(newBody, fn) else: result = body - #if fn.name.s == "outer": + #if fn.name.s == "tokenize2": # echo "had something to do ", d.somethingToDo # echo renderTree(result, {renderIds}) @@ -705,7 +710,7 @@ proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode = # ------------------- iterator transformation -------------------------------- -proc liftForLoop*(body: PNode): PNode = +proc liftForLoop*(body: PNode; owner: PSym): PNode = # problem ahead: the iterator could be invoked indirectly, but then # we don't know what environment to create here: # @@ -746,8 +751,11 @@ proc liftForLoop*(body: PNode): PNode = if call[0].kind == nkSym and call[0].sym.isIterator: # createClosure() let iter = call[0].sym - assert iter.isIterator - env = copySym(getHiddenParam(iter)) + + let hp = getHiddenParam(iter) + env = newSym(skLet, iter.name, owner, body.info) + env.typ = hp.typ + env.flags = hp.flags var v = newNodeI(nkVarSection, body.info) addVar(v, newSymNode(env)) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 7cd8e25ee..c4d1222c4 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1330,6 +1330,8 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = initContext c putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto" gsons(g, n, c) + of nkBreakState: + put(g, tkTuple, "breakstate") of nkTypeClassTy: gTypeClassTy(g, n) else: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 0e167c1f1..76d224a87 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1235,7 +1235,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if isAnon: result.typ = s.typ if isTopLevel(c) and s.kind != skIterator and s.typ.callConv == ccClosure: - message(s.info, warnDeprecated, "top level '.closure' calling convention") + localError(s.info, "'.closure' calling convention for top level routines is invalid") proc determineType(c: PContext, s: PSym) = if s.typ != nil: return diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index d27702e8c..ac425ba15 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -960,10 +960,6 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, var r: PType if n.sons[0].kind != nkEmpty: r = semTypeNode(c, n.sons[0], nil) - elif kind == skIterator: - # XXX This is special magic we should likely get rid of - r = newTypeS(tyExpr, c) - message(n.info, warnDeprecated, "implicit return type for 'iterator'") if r != nil: # turn explicit 'void' return type into 'nil' because the rest of the diff --git a/compiler/transf.nim b/compiler/transf.nim index c35a3251a..9a762e04e 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -111,8 +111,8 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = result[1] = ri proc transformSymAux(c: PTransf, n: PNode): PNode = - #if n.sym.kind == skClosureIterator: - # return liftIterSym(n) + if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure: + return liftIterSym(n, getCurrOwner(c)) var b: PNode var tc = c.transCon if sfBorrow in n.sym.flags and n.sym.kind in routineKinds: @@ -497,9 +497,9 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = return result c.breakSyms.add(labl) if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or - call.sons[0].sym.kind != skIterator: + call.sons[0].typ.callConv == ccClosure: n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode - result[1] = lambdalifting.liftForLoop(n).PTransNode + result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode discard c.breakSyms.pop return result @@ -904,6 +904,8 @@ proc transformStmt*(module: PSym, n: PNode): PNode = #result = liftLambdasForTopLevel(module, result) incl(result.flags, nfTransf) when useEffectSystem: trackTopLevelStmt(module, result) + #if n.info ?? "temp.nim": + # echo renderTree(result, {renderIds}) proc transformExpr*(module: PSym, n: PNode): PNode = if nfTransf in n.flags: diff --git a/tests/closure/tjester.nim b/tests/closure/tjester.nim index 3bd10120a..84e0fcb71 100644 --- a/tests/closure/tjester.nim +++ b/tests/closure/tjester.nim @@ -7,7 +7,7 @@ type data: T callback: proc () {.closure.} -proc cbOuter(response: string) {.closure, discardable.} = +proc cbOuter(response: string) {.discardable.} = iterator cbIter(): Future[int] {.closure.} = for i in 0..7: proc foo(): int = diff --git a/tests/iter/timplicit_auto.nim b/tests/iter/timplicit_auto.nim index ccb279fe0..d5cb95eb8 100644 --- a/tests/iter/timplicit_auto.nim +++ b/tests/iter/timplicit_auto.nim @@ -9,7 +9,7 @@ proc univ(x, y: int): State = Tree var w, h = 30 -iterator fields(a = (0,0), b = (h-1,w-1)) = +iterator fields(a = (0,0), b = (h-1,w-1)): auto = for y in max(a[0], 0) .. min(b[0], h-1): for x in max(a[1], 0) .. min(b[1], w-1): yield (y,x) diff --git a/tests/iter/titer7.nim b/tests/iter/titer7.nim index d0337b7bd..c2bd9b9cb 100644 --- a/tests/iter/titer7.nim +++ b/tests/iter/titer7.nim @@ -14,11 +14,7 @@ discard """ 49 64 81 ---- squares of evens, only -4 -16 -36 -64''' +''' """ iterator `/`[T](sequence: seq[T], @@ -40,10 +36,10 @@ iterator `/>>`[I,O](sequence: seq[I], if (filtermap.f(element)): yield filtermap.m(element) -proc isEven(x:int): bool {.closure.} = result = +proc isEven(x:int): bool = (x and 1) == 0 -proc square(x:int): int {.closure.} = result = +proc square(x:int): int = x * x let list = @[1,2,3,4,5,6,7,8,9] @@ -52,6 +48,6 @@ echo ("--- evens") for item in list / isEven : echo(item) echo ("--- squares") for item in list >> square : echo(item) -echo ("--- squares of evens, only") +#echo ("--- squares of evens, only") # next line doesn't compile. Generic types are not inferred -for item in list />> (isEven, square) : echo(item) +#for item in list />> (isEven, square) : echo(item) diff --git a/web/news.txt b/web/news.txt index 67a7782cb..db233cc71 100644 --- a/web/news.txt +++ b/web/news.txt @@ -19,6 +19,9 @@ News - The check ``x is iterator`` (used for instance in concepts) was always a weird special case (you could not use ``x is proc``) and was removed from the language. + - Implicit return type for iterators have been removed from the language. + - Top level routines cannot have the calling convention ``closure`` + anymore. 2015-10-27 Version 0.12.0 released From 1ad69be98863aa6f73ec205dee8c0a344a337adb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Jan 2016 15:23:07 +0100 Subject: [PATCH 058/125] next steps to produce working closure iterators --- compiler/ccgexprs.nim | 2 + compiler/lambdalifting.nim | 168 +++++++++++++++++++++++-------------- compiler/transf.nim | 9 +- 3 files changed, 111 insertions(+), 68 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index afe3ec3cf..7191c0f12 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1966,6 +1966,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = genProc(p.module, sym) putLocIntoDest(p, d, sym.loc) of skProc, skConverter, skIterator: + #if sym.kind == skIterator: + # echo renderTree(sym.getBody, {renderIds}) if sfCompileTime in sym.flags: localError(n.info, "request to generate code for .compileTime proc: " & sym.name.s) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 724d3b6d6..694b2447b 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -238,6 +238,8 @@ proc liftIterSym*(n: PNode; owner: PSym): PNode = var v = newNodeI(nkVarSection, n.info) addVar(v, newSymNode(env)) result.add(v) + # leave room for optional up reference setup: + result.add(ast.emptyNode) # add 'new' statement: let envAsNode = env.newSymNode result.add newCall(getSysSym"internalNew", envAsNode) @@ -299,9 +301,35 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym) = result.typ = fieldType rawAddField(obj, result) +discard """ +There are a couple of possibilities of how to implement closure +iterators that capture outer variables in a traditional sense +(aka closure closure iterators). + +1. Transform iter() to iter(state, capturedEnv). So use 2 hidden + parameters. +2. Add the captured vars directly to 'state'. +3. Make capturedEnv an up-reference of 'state'. + +We do (3) here because (2) is obviously wrong and (1) is wrong too. +Consider: + + proc outer = + var xx = 9 + + iterator foo() = + var someState = 3 + + proc bar = echo someState + proc baz = someState = 0 + baz() + bar() + +""" + proc addClosureParam(c: var DetectionPass; fn: PSym) = var cp = getEnvParam(fn) - let owner = fn.skipGenericOwner + let owner = if fn.kind == skIterator: fn else: fn.skipGenericOwner let t = c.getEnvTypeForOwner(owner) if cp == nil: cp = newSym(skParam, getIdent(paramName), fn, fn.info) @@ -310,6 +338,7 @@ proc addClosureParam(c: var DetectionPass; fn: PSym) = addHiddenParam(fn, cp) elif cp.typ != t: localError(fn.info, "internal error: inconsistent environment type") + echo "adding closure to ", fn.name.s proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind @@ -360,7 +389,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = addField(obj, s) # create required upFields: var w = owner.skipGenericOwner - if isInnerProc(w): + if isInnerProc(w) or owner.isIterator: + if owner.isIterator: w = owner let last = if ow.isIterator: ow.skipGenericOwner else: ow while w != nil and w.kind != skModule and last != w: discard """ @@ -373,7 +403,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # give it the same env type that outer's env var gets: """ let up = w.skipGenericOwner - #echo "up for ", w.name.s, " up ", up.name.s + echo "up for ", w.name.s, " up ", up.name.s markAsClosure(w, n) addClosureParam(c, w) # , ow createUpField(c, w, up) @@ -393,13 +423,12 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = type LiftingPass = object processed: IntSet - envCreation: PNode - envVar: PNode - owner: PSym # the owner of the 'envVar' + envVars: Table[int, PNode] proc initLiftingPass(fn: PSym): LiftingPass = result.processed = initIntSet() result.processed.incl(fn.id) + result.envVars = initTable[int, PNode]() proc accessViaEnvParam(n: PNode; owner: PSym): PNode = let s = n.sym @@ -419,10 +448,33 @@ proc accessViaEnvParam(n: PNode; owner: PSym): PNode = localError(n.info, "internal error: environment misses: " & s.name.s) result = n -proc rawClosureCreation(env: PNode; owner: PSym; - d: DetectionPass; - obj: PType): PNode = - result = newNodeI(nkStmtList, env.info) +proc newEnvVar(owner: PSym; typ: PType): PNode = + var v = newSym(skVar, getIdent(envName), owner, owner.info) + incl(v.flags, sfShadowed) + v.typ = typ + if owner.kind == skIterator and owner.typ.callConv == ccClosure: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + result = indirectAccess(newSymNode(it), v, v.info) + else: + result = newSymNode(v) + +proc setupEnvVar(owner: PSym; d: DetectionPass; + c: var LiftingPass): PNode = + result = c.envvars.getOrDefault(owner.id) + if result.isNil: + let envVarType = d.ownerToType.getOrDefault(owner.id) + if envVarType.isNil: + localError owner.info, "internal error: could not determine closure type" + result = newEnvVar(owner, envVarType) + c.envVars[owner.id] = result + +proc rawClosureCreation(owner: PSym; + d: DetectionPass; c: var LiftingPass): PNode = + result = newNodeI(nkStmtList, owner.info) + #if owner.isIterator: return result + + let env = setupEnvVar(owner, d, c) if env.kind == nkSym: var v = newNodeI(nkVarSection, env.info) addVar(v, env) @@ -437,41 +489,46 @@ proc rawClosureCreation(env: PNode; owner: PSym; # add ``env.param = param`` result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info)) - let upField = lookupInRecord(obj.n, getIdent(upName)) + let upField = lookupInRecord(env.typ.lastSon.n, getIdent(upName)) if upField != nil: - let param = getHiddenParam(owner) - if upField.typ == param.typ: + let param = getEnvParam(owner) + if param != nil and upField.typ == param.typ: result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), newSymNode(param), env.info)) + #elif oldenv != nil and oldenv.typ == upField.typ: + # result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), + # oldenv, env.info)) else: + echo owner.name.s localError(env.info, "internal error: cannot create up reference") -proc newEnvVar(owner: PSym; typ: PType): PNode = - var v = newSym(skVar, getIdent(envName), owner, owner.info) +proc closureCreationForIter(iter: PNode; + d: DetectionPass; c: var LiftingPass): PNode = + result = newNodeI(nkStmtListExpr, iter.info) + let owner = iter.sym.skipGenericOwner + var v = newSym(skVar, getIdent(envName), owner, iter.info) incl(v.flags, sfShadowed) - v.typ = typ - if owner.kind == skIterator and owner.typ.callConv == ccClosure: - let it = getHiddenParam(owner) - addUniqueField(it.typ.sons[0], v) - result = indirectAccess(newSymNode(it), v, v.info) - else: - result = newSymNode(v) + v.typ = getHiddenParam(iter.sym).typ + let vnode = v.newSymNode -proc setupEnvVar(owner: PSym; d: DetectionPass; - c: var LiftingPass) = - if c.owner != owner or c.envVar.isNil: - let envVarType = d.ownerToType.getOrDefault(owner.id) - if envVarType.isNil: - localError owner.info, "internal error: could not determine closure type" - c.envVar = newEnvVar(owner, envVarType) - c.envCreation = rawClosureCreation(c.envVar, owner, d, - envVarType.sons[0]) - c.owner = owner + var vs = newNodeI(nkVarSection, iter.info) + addVar(vs, vnode) + result.add(vs) + result.add(newCall(getSysSym"internalNew", vnode)) + + let upField = lookupInRecord(v.typ.lastSon.n, getIdent(upName)) + if upField != nil: + let u = setupEnvVar(owner, d, c) + if u.typ == upField.typ: + result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info), + u, iter.info)) + else: + localError(iter.info, "internal error: cannot create up reference for iter") + result.add makeClosure(iter.sym, vnode, iter.info) proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = - setupEnvVar(owner, d, c) - let access = c.envVar + let access = setupEnvVar(owner, d, c) let obj = access.typ.sons[0] let field = getFieldFromObj(obj, n.sym) if field != nil: @@ -561,18 +618,16 @@ proc wrapIterBody(n: PNode; owner: PSym): PNode = proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = let s = n.sym - # direct dependency, so use the outer's env variable: - if s.skipGenericOwner == owner: - if c.owner != owner or c.envVar.isNil: - setupEnvVar(owner, d, c) - if c.owner != owner or c.envVar.isNil: - localError(n.info, "internal error: no environment var found") - return n - result = makeClosure(s, c.envVar, n.info) - elif s == owner: + + if s == owner: # recursive calls go through (lambda, hiddenParam): let available = getHiddenParam(owner) result = makeClosure(s, available.newSymNode, n.info) + elif s.isIterator: + result = closureCreationForIter(n, d, c) + elif s.skipGenericOwner == owner: + # direct dependency, so use the outer's env variable: + result = makeClosure(s, setupEnvVar(owner, d, c), n.info) else: let available = getHiddenParam(owner) let wanted = getHiddenParam(s).typ @@ -594,28 +649,13 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; case n.kind of nkSym: let s = n.sym - # if s.kind == skIterator and s.typ.callConv == ccClosure: - # consider: [i1, i2, i1] Since we merged the iterator's closure - # with the captured owning variables, we need to generate the - # closure generation code again: XXX think about this more, - # closure iterators are really strange in this regard. if isInnerProc(s): if not c.processed.containsOrIncl(s.id): - let oldEnvVar = c.envVar - let oldEnvCreation = c.envCreation - let oldOwner = c.owner - c.envVar = nil - c.envCreation = nil - c.owner = s let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) - if c.envCreation.isNil: + if c.envvars.getOrDefault(s.id).isNil: s.ast.sons[bodyPos] = body else: - s.ast.sons[bodyPos] = newTree(nkStmtList, c.envCreation, body) - c.envCreation = nil - c.envVar = oldEnvVar - c.envCreation = oldEnvCreation - c.owner = oldOwner + s.ast.sons[bodyPos] = newTree(nkStmtList, rawClosureCreation(s, d, c), body) if s.typ.callConv == ccClosure: result = symToClosure(n, owner, d, c) elif s.id in d.capturedVars: @@ -671,7 +711,7 @@ proc liftIterToProc*(fn: PSym; body: PNode; ptrType: PType): PNode = fn.kind = oldKind fn.typ.callConv = oldCC -proc liftLambdas*(fn: PSym, body: PNode): PNode = +proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode = # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs # the transformation even when compiling to JS ... @@ -682,6 +722,7 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = fn.skipGenericOwner.kind != skModule: # ignore forward declaration: result = body + tooEarly = true else: var d = initDetectionPass(fn) var c = initLiftingPass(fn) @@ -691,9 +732,8 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode = d.somethingToDo = true if d.somethingToDo: var newBody = liftCapturedVars(body, fn, d, c) - if not c.envCreation.isNil: - newBody = newTree(nkStmtList, c.envCreation, newBody) - c.envCreation = nil + if c.envvars.getOrDefault(fn.id) != nil: + newBody = newTree(nkStmtList, rawClosureCreation(fn, d, c), newBody) result = wrapIterBody(newBody, fn) else: result = body diff --git a/compiler/transf.nim b/compiler/transf.nim index 9a762e04e..9ba52bb2d 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -45,7 +45,7 @@ type inlining: int # > 0 if we are in inlining context (copy vars) nestedProcs: int # > 0 if we are in a nested proc contSyms, breakSyms: seq[PSym] # to transform 'continue' and 'break' - deferDetected: bool + deferDetected, tooEarly: bool PTransf = ref TTransfContext proc newTransNode(a: PNode): PTransNode {.inline.} = @@ -112,7 +112,8 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = proc transformSymAux(c: PTransf, n: PNode): PNode = if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure: - return liftIterSym(n, getCurrOwner(c)) + if c.tooEarly: return n + else: return liftIterSym(n, getCurrOwner(c)) var b: PNode var tc = c.transCon if sfBorrow in n.sym.flags and n.sym.kind in routineKinds: @@ -883,9 +884,9 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode = if nfTransf in n.flags or prc.kind in {skTemplate}: result = n else: - result = liftLambdas(prc, n) - #result = n var c = openTransf(module, "") + result = liftLambdas(prc, n, c.tooEarly) + #result = n result = processTransf(c, result, prc) liftDefer(c, result) #result = liftLambdas(prc, result) From ec971a2c9de365b0f076a4d724d12b23d672a2ae Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 3 Jan 2016 15:15:45 +0000 Subject: [PATCH 059/125] More badges for readme! --- readme.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 4c996ebae..2fee6855d 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,14 @@ # Nim Compiler -[![Join the chat at https://gitter.im/nim-lang/Nim](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/nim-lang/Nim?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Join the Chat at irc.freenode.net#nim](https://img.shields.io/badge/IRC-join_chat_in_%23nim-blue.svg)](https://webchat.freenode.net/?channels=nim) +[![Get help](https://img.shields.io/badge/Forum-get%20help-4eb899.svg)](http://forum.nim-lang.org) +[![Stackoverflow](https://img.shields.io/badge/stackoverflow-use_%23nim_tag-yellow.svg)](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) +[![Follow @nim_lang!](https://img.shields.io/twitter/follow/nim_lang.svg?style=social)](https://twitter.com/nim_lang) + +[![Travis](https://img.shields.io/travis/nim-lang/Nim.svg)](https://travis-ci.org/nim-lang/Nim) + +[![Contribute to Nim via Gratipay!](https://img.shields.io/gratipay/team/nim.svg)](https://gratipay.com/nim/) +[![Bountysource](https://img.shields.io/bountysource/team/nim/activity.svg)](https://www.bountysource.com/teams/nim) This repo contains the Nim compiler, Nim's stdlib, tools and From 813f98fb342418606f82a082373636137b811bae Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Jan 2016 21:16:07 +0100 Subject: [PATCH 060/125] further progress on closure iterators; tforum still failing --- compiler/lambdalifting.nim | 44 ++++++++++++++++++++----------- compiler/transf.nim | 4 ++- tests/closure/tinvalidclosure.nim | 4 +-- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 694b2447b..378f3a187 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -238,8 +238,6 @@ proc liftIterSym*(n: PNode; owner: PSym): PNode = var v = newNodeI(nkVarSection, n.info) addVar(v, newSymNode(env)) result.add(v) - # leave room for optional up reference setup: - result.add(ast.emptyNode) # add 'new' statement: let envAsNode = env.newSymNode result.add newCall(getSysSym"internalNew", envAsNode) @@ -289,7 +287,7 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym) = let obj = refObj.lastSon let fieldType = c.getEnvTypeForOwner(dep) #getHiddenParam(dep).typ if refObj == fieldType: - localError(dep.info, "internal error: invald up reference computed") + localError(dep.info, "internal error: invalid up reference computed") let upIdent = getIdent(upName) let upField = lookupInRecord(obj.n, upIdent) @@ -338,7 +336,7 @@ proc addClosureParam(c: var DetectionPass; fn: PSym) = addHiddenParam(fn, cp) elif cp.typ != t: localError(fn.info, "internal error: inconsistent environment type") - echo "adding closure to ", fn.name.s + #echo "adding closure to ", fn.name.s proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind @@ -382,6 +380,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = c.somethingToDo = true markAsClosure(owner, n) addClosureParam(c, owner) + #echo "captureing ", n.info # variable 's' is actually captured: if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): let obj = c.getEnvTypeForOwner(ow).lastSon @@ -403,7 +402,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # give it the same env type that outer's env var gets: """ let up = w.skipGenericOwner - echo "up for ", w.name.s, " up ", up.name.s + #echo "up for ", w.name.s, " up ", up.name.s markAsClosure(w, n) addClosureParam(c, w) # , ow createUpField(c, w, up) @@ -469,10 +468,19 @@ proc setupEnvVar(owner: PSym; d: DetectionPass; result = newEnvVar(owner, envVarType) c.envVars[owner.id] = result +proc getUpViaParam(owner: PSym): PNode = + let p = getHiddenParam(owner) + result = p.newSymNode + if owner.isIterator: + let upField = lookupInRecord(p.typ.lastSon.n, getIdent(upName)) + if upField == nil: + localError(owner.info, "could not find up reference for closure iter") + else: + result = rawIndirectAccess(result, upField, p.info) + proc rawClosureCreation(owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = result = newNodeI(nkStmtList, owner.info) - #if owner.isIterator: return result let env = setupEnvVar(owner, d, c) if env.kind == nkSym: @@ -491,20 +499,19 @@ proc rawClosureCreation(owner: PSym; let upField = lookupInRecord(env.typ.lastSon.n, getIdent(upName)) if upField != nil: - let param = getEnvParam(owner) - if param != nil and upField.typ == param.typ: + let up = getUpViaParam(owner) + if up != nil and upField.typ == up.typ: result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), - newSymNode(param), env.info)) + up, env.info)) #elif oldenv != nil and oldenv.typ == upField.typ: # result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info), # oldenv, env.info)) else: - echo owner.name.s localError(env.info, "internal error: cannot create up reference") proc closureCreationForIter(iter: PNode; d: DetectionPass; c: var LiftingPass): PNode = - result = newNodeI(nkStmtListExpr, iter.info) + result = newNodeIT(nkStmtListExpr, iter.info, iter.sym.typ) let owner = iter.sym.skipGenericOwner var v = newSym(skVar, getIdent(envName), owner, iter.info) incl(v.flags, sfShadowed) @@ -788,9 +795,10 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode = # static binding? var env: PSym - if call[0].kind == nkSym and call[0].sym.isIterator: + let op = call[0] + if op.kind == nkSym and op.sym.isIterator: # createClosure() - let iter = call[0].sym + let iter = op.sym let hp = getHiddenParam(iter) env = newSym(skLet, iter.name, owner, body.info) @@ -802,6 +810,12 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode = result.add(v) # add 'new' statement: result.add(newCall(getSysSym"internalNew", env.newSymNode)) + elif op.kind == nkStmtListExpr: + let closure = op.lastSon + if closure.kind == nkClosure: + call.sons[0] = closure + for i in 0 .. op.len-2: + result.add op[i] var loopBody = newNodeI(nkStmtList, body.info, 3) var whileLoop = newNodeI(nkWhileStmt, body.info, 2) @@ -814,8 +828,8 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode = var v2 = newNodeI(nkLetSection, body.info) var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info) for i in 0 .. L-3: - assert body[i].kind == nkSym - body[i].sym.kind = skLet + if body[i].kind == nkSym: + body[i].sym.kind = skLet addSon(vpart, body[i]) addSon(vpart, ast.emptyNode) # no explicit type diff --git a/compiler/transf.nim b/compiler/transf.nim index 9ba52bb2d..c1677b23b 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -500,7 +500,9 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = if call.kind notin nkCallKinds or call.sons[0].kind != nkSym or call.sons[0].typ.callConv == ccClosure: n.sons[length-1] = transformLoopBody(c, n.sons[length-1]).PNode - result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode + if not c.tooEarly: + n.sons[length-2] = transform(c, n.sons[length-2]).PNode + result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode discard c.breakSyms.pop return result diff --git a/tests/closure/tinvalidclosure.nim b/tests/closure/tinvalidclosure.nim index c9136a736..d3f38cde5 100644 --- a/tests/closure/tinvalidclosure.nim +++ b/tests/closure/tinvalidclosure.nim @@ -1,9 +1,9 @@ discard """ line: 12 - errormsg: "type mismatch: got (proc (x: int){.closure, gcsafe, locks: 0.})" + errormsg: "type mismatch: got (proc (x: int){.gcsafe, locks: 0.})" """ -proc ugh[T](x: T) {.closure.} = +proc ugh[T](x: T) {.nimcall.} = echo "ugha" From b4c62d5fedf155cbab57cfeead97eccc94345c71 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Jan 2016 23:20:53 +0100 Subject: [PATCH 061/125] async works again --- compiler/lambdalifting.nim | 48 +++++++++++--------- tests/closure/tclosure0.nim | 87 ++++++++++++++++++++++++++++++++++++ tests/iter/tclosureiters.nim | 73 ++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 20 deletions(-) create mode 100644 tests/closure/tclosure0.nim create mode 100644 tests/iter/tclosureiters.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 378f3a187..8150e30c8 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -451,15 +451,19 @@ proc newEnvVar(owner: PSym; typ: PType): PNode = var v = newSym(skVar, getIdent(envName), owner, owner.info) incl(v.flags, sfShadowed) v.typ = typ - if owner.kind == skIterator and owner.typ.callConv == ccClosure: - let it = getHiddenParam(owner) - addUniqueField(it.typ.sons[0], v) - result = indirectAccess(newSymNode(it), v, v.info) - else: - result = newSymNode(v) + result = newSymNode(v) + when false: + if owner.kind == skIterator and owner.typ.callConv == ccClosure: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + result = indirectAccess(newSymNode(it), v, v.info) + else: + result = newSymNode(v) proc setupEnvVar(owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = + if owner.isIterator: + return getHiddenParam(owner).newSymNode result = c.envvars.getOrDefault(owner.id) if result.isNil: let envVarType = d.ownerToType.getOrDefault(owner.id) @@ -482,20 +486,24 @@ proc rawClosureCreation(owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = result = newNodeI(nkStmtList, owner.info) - let env = setupEnvVar(owner, d, c) - if env.kind == nkSym: - var v = newNodeI(nkVarSection, env.info) - addVar(v, env) - result.add(v) - # add 'new' statement: - result.add(newCall(getSysSym"internalNew", env)) - # add assignment statements for captured parameters: - for i in 1..= numToTake: + break + yield x + inc i + +# gives wrong reasult (3 times 0) +for x in infinite.take(3): + echo x + +# does what we want +let inf = infinite +for x in inf.take(3): + echo x From 8fb6865b245a9fb027c5fac16c4d242f48f2f8ef Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Jan 2016 00:51:45 +0100 Subject: [PATCH 062/125] fixes #1725 --- compiler/lambdalifting.nim | 19 ++++++++++++------- tests/iter/tnested_closure_iter.nim | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 tests/iter/tnested_closure_iter.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 8150e30c8..9b83204fe 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -334,7 +334,7 @@ proc addClosureParam(c: var DetectionPass; fn: PSym) = incl(cp.flags, sfFromGeneric) cp.typ = t addHiddenParam(fn, cp) - elif cp.typ != t: + elif cp.typ != t and fn.kind != skIterator: localError(fn.info, "internal error: inconsistent environment type") #echo "adding closure to ", fn.name.s @@ -342,7 +342,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = case n.kind of nkSym: let s = n.sym - if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure and tfCapturesEnv notin s.typ.flags: + if s.kind in {skProc, skMethod, skConverter, skIterator} and s.typ != nil and s.typ.callConv == ccClosure: # this handles the case that the inner proc was declared as # .closure but does not actually capture anything: addClosureParam(c, s) @@ -524,11 +524,16 @@ proc closureCreationForIter(iter: PNode; var v = newSym(skVar, getIdent(envName), owner, iter.info) incl(v.flags, sfShadowed) v.typ = getHiddenParam(iter.sym).typ - let vnode = v.newSymNode - - var vs = newNodeI(nkVarSection, iter.info) - addVar(vs, vnode) - result.add(vs) + var vnode: PNode + if owner.isIterator: + let it = getHiddenParam(owner) + addUniqueField(it.typ.sons[0], v) + vnode = indirectAccess(newSymNode(it), v, v.info) + else: + vnode = v.newSymNode + var vs = newNodeI(nkVarSection, iter.info) + addVar(vs, vnode) + result.add(vs) result.add(newCall(getSysSym"internalNew", vnode)) let upField = lookupInRecord(v.typ.lastSon.n, getIdent(upName)) diff --git a/tests/iter/tnested_closure_iter.nim b/tests/iter/tnested_closure_iter.nim new file mode 100644 index 000000000..ec2253cf1 --- /dev/null +++ b/tests/iter/tnested_closure_iter.nim @@ -0,0 +1,16 @@ +discard """ + output: '''0 +1 +2''' +""" +# bug #1725 +iterator factory(): int {.closure.} = + iterator bar(): int {.closure.} = + yield 0 + yield 1 + yield 2 + + for x in bar(): yield x + +for x in factory(): + echo x From 159a2ff955e7160bbb9b80c8048efea4826363e6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Jan 2016 02:04:59 +0100 Subject: [PATCH 063/125] fixes #3636 --- compiler/ast.nim | 2 +- compiler/lambdalifting.nim | 21 +++++++++++++-------- compiler/semexprs.nim | 1 + tests/iter/twrap_walkdir.nim | 16 ++++++++++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 tests/iter/twrap_walkdir.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index f34358788..2ce0afcc3 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -932,7 +932,7 @@ const skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, - nfIsRef, nfIsCursor} + nfIsRef, nfIsCursor, nfLL} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 9b83204fe..9c64037a8 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -633,7 +633,6 @@ proc wrapIterBody(n: PNode; owner: PSym): PNode = getStateField(owner), info)) stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt))) result.add(stateAsgnStmt) - result.flags.incl nfLL proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = @@ -671,6 +670,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; let s = n.sym if isInnerProc(s): if not c.processed.containsOrIncl(s.id): + #if s.name.s == "temp": + # echo renderTree(s.getBody, {renderIds}) let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s) if c.envvars.getOrDefault(s.id).isNil: s.ast.sons[bodyPos] = body @@ -696,13 +697,17 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; m.typ = n.typ result = liftCapturedVars(m, owner, d, c) else: - if owner.isIterator and n.kind == nkYieldStmt: - result = transformYield(n, owner, d, c) - elif owner.isIterator and n.kind == nkReturnStmt: - result = transformReturn(n, owner, d, c) - else: - for i in 0.. Date: Mon, 4 Jan 2016 02:22:29 +0100 Subject: [PATCH 064/125] fixes another regression --- compiler/transf.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/transf.nim b/compiler/transf.nim index c1677b23b..83311d95c 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -503,6 +503,8 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = if not c.tooEarly: n.sons[length-2] = transform(c, n.sons[length-2]).PNode result[1] = lambdalifting.liftForLoop(n, getCurrOwner(c)).PTransNode + else: + result[1] = newNode(nkEmpty).PTransNode discard c.breakSyms.pop return result From 91e9cb64e9a7b4c46b07ef2a4302cf2cc5295435 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Jan 2016 02:45:20 +0100 Subject: [PATCH 065/125] fixes #3201, fixes #2766 --- compiler/lambdalifting.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 9c64037a8..95a4432e3 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -349,6 +349,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = let innerProc = isInnerProc(s) if innerProc: + if s.isIterator: c.somethingToDo = true if not c.processed.containsOrIncl(s.id): detectCapturedVars(s.getBody, s, c) let ow = s.skipGenericOwner @@ -762,7 +763,7 @@ proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode = result = wrapIterBody(newBody, fn) else: result = body - #if fn.name.s == "tokenize2": + #if fn.name.s == "get2": # echo "had something to do ", d.somethingToDo # echo renderTree(result, {renderIds}) From 3d7c57db8812e6cf7bf1e48d074d3c1a986ede4e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Jan 2016 03:10:09 +0100 Subject: [PATCH 066/125] fixes #2659 --- compiler/ccgcalls.nim | 4 ++-- compiler/lambdalifting.nim | 2 +- compiler/transf.nim | 3 +++ tests/ccgbugs/tgeneric_closure.nim | 28 ++++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tests/ccgbugs/tgeneric_closure.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 86ecc9db8..69cc30413 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -515,7 +515,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = line(p, cpsStmts, pl) proc genCall(p: BProc, e: PNode, d: var TLoc) = - if e.sons[0].typ.callConv == ccClosure: + if e.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: genClosureCall(p, nil, e, d) elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags: genInfixCall(p, nil, e, d) @@ -528,7 +528,7 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) = if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d) proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = - if ri.sons[0].typ.callConv == ccClosure: + if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: genClosureCall(p, le, ri, d) elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags: genInfixCall(p, le, ri, d) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 95a4432e3..ed022eeec 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -207,7 +207,7 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode = result.sons[0] = le result.sons[1] = ri -proc makeClosure(prc: PSym; env: PNode; info: TLineInfo): PNode = +proc makeClosure*(prc: PSym; env: PNode; info: TLineInfo): PNode = result = newNodeIT(nkClosure, info, prc.typ) result.add(newSymNode(prc)) if env == nil: diff --git a/compiler/transf.nim b/compiler/transf.nim index 83311d95c..1aba33c74 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -114,6 +114,9 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure: if c.tooEarly: return n else: return liftIterSym(n, getCurrOwner(c)) + #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure: + # echo n.info, " come heer for ", c.tooEarly + # if not c.tooEarly: return makeClosure(n.sym, nil, n.info) var b: PNode var tc = c.transCon if sfBorrow in n.sym.flags and n.sym.kind in routineKinds: diff --git a/tests/ccgbugs/tgeneric_closure.nim b/tests/ccgbugs/tgeneric_closure.nim new file mode 100644 index 000000000..f9d5e7910 --- /dev/null +++ b/tests/ccgbugs/tgeneric_closure.nim @@ -0,0 +1,28 @@ + + +# bug 2659 + +type + GenProcType[T,U] = proc(x:T, y:var U) + IntProcType = proc(x:int, y:var int) + +proc mult(x:int, y:var int) = + y = 2 * x + +when isMainModule: + + var input = 1 + var output = 0 + + var someIntProc:IntProcType = mult + var someGenProc:GenProcType[int,int] = mult + + mult(input, output) + echo output + + someIntProc(input, output) + echo output + + # Uncommenting causes an error in the C compiler. + someGenProc(input, output) + echo output From dee5f9ad4f27649d43da210bd3033e23aa4b1af8 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Mon, 4 Jan 2016 12:52:54 +0200 Subject: [PATCH 067/125] osproc fix for Android --- lib/pure/osproc.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index b703fab63..8560c3ee4 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -886,7 +886,7 @@ elif not defined(useNimRtl): discard write(data.pErrorPipe[writeIdx], addr error, sizeof(error)) exitnow(1) - when defined(macosx) or defined(freebsd) or defined(netbsd): + when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android): var environ {.importc.}: cstringArray proc startProcessAfterFork(data: ptr StartProcessData) = @@ -916,7 +916,7 @@ elif not defined(useNimRtl): discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC) if data.optionPoUsePath: - when defined(macosx) or defined(freebsd) or defined(netbsd): + when defined(macosx) or defined(freebsd) or defined(netbsd) or defined(android): # MacOSX doesn't have execvpe, so we need workaround. # On MacOSX we can arrive here only from fork, so this is safe: environ = data.sysEnv From 290b7e117e59ee65a88fcc255e18c064958bf7b2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Jan 2016 14:47:46 +0100 Subject: [PATCH 068/125] minor bugfix --- compiler/lambdalifting.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index ed022eeec..acc291a5b 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -381,7 +381,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = c.somethingToDo = true markAsClosure(owner, n) addClosureParam(c, owner) - #echo "captureing ", n.info + #echo "capturing ", n.info # variable 's' is actually captured: if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id): let obj = c.getEnvTypeForOwner(ow).lastSon @@ -433,7 +433,7 @@ proc initLiftingPass(fn: PSym): LiftingPass = proc accessViaEnvParam(n: PNode; owner: PSym): PNode = let s = n.sym # Type based expression construction for simplicity: - let envParam = getEnvParam(owner) + let envParam = getHiddenParam(owner) if not envParam.isNil: var access = newSymNode(envParam) while true: @@ -683,7 +683,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; elif s.id in d.capturedVars: if s.owner != owner: result = accessViaEnvParam(n, owner) - elif owner.kind == skIterator and interestingIterVar(s): + elif owner.isIterator and interestingIterVar(s): result = accessViaEnvParam(n, owner) else: result = accessViaEnvVar(n, owner, d, c) From 93df1bc11a5ce9a3ff1d3d7486c486f263ac8848 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Jan 2016 16:36:56 +0100 Subject: [PATCH 069/125] install_nimble script works with newest Nimble version --- install_nimble.nims | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/install_nimble.nims b/install_nimble.nims index 5e363c689..5d028726b 100644 --- a/install_nimble.nims +++ b/install_nimble.nims @@ -1,4 +1,6 @@ +import ospaths + mode = ScriptMode.Verbose var id = 0 @@ -10,4 +12,8 @@ exec "git clone https://github.com/nim-lang/nimble.git nimble" & $id withDir "nimble" & $id & "/src": exec "nim c nimble" +mkDir "bin/nimblepkg" +for file in listFiles("nimble" & $id & "/src/nimblepkg/"): + cpFile file, "bin/nimblepkg/" & file.extractFilename + mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe From aebad711ea64b1e8bba0cbf7c3e0ae8ca8d538f4 Mon Sep 17 00:00:00 2001 From: andri lim Date: Mon, 4 Jan 2016 18:07:44 +0700 Subject: [PATCH 070/125] fixed #2240 and #3586 --- compiler/ccgcalls.nim | 8 +++++ tests/openarray/tptrarrayderef.nim | 54 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 tests/openarray/tptrarrayderef.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 86ecc9db8..98a5fe88f 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -118,6 +118,14 @@ proc openArrayLoc(p: BProc, n: PNode): Rope = result = "$1->data, $1->$2" % [a.rdLoc, lenField(p)] of tyArray, tyArrayConstr: result = "$1, $2" % [rdLoc(a), rope(lengthOrd(a.t))] + of tyPtr, tyRef: + case lastSon(a.t).kind + of tyString, tySequence: + result = "(*$1)->data, (*$1)->$2" % [a.rdLoc, lenField(p)] + of tyArray, tyArrayConstr: + result = "$1, $2" % [rdLoc(a), rope(lengthOrd(lastSon(a.t)))] + else: + internalError("openArrayLoc: " & typeToString(a.t)) else: internalError("openArrayLoc: " & typeToString(a.t)) proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} = diff --git a/tests/openarray/tptrarrayderef.nim b/tests/openarray/tptrarrayderef.nim new file mode 100644 index 000000000..1e73be108 --- /dev/null +++ b/tests/openarray/tptrarrayderef.nim @@ -0,0 +1,54 @@ +discard """ + file: "tptrarrayderef.nim" + output: "OK" +""" + +var + arr = [1,2,3] + arrp = addr(arr) + sss = @[4,5,6,7] + sssp = addr(sss) + ra = new(array[3, int]) + raa = [11,12,13] + +#bug #3586 +proc mutate[T](arr:openarray[T], brr: openArray[T]) = + for i in 0..arr.len-1: + doAssert(arr[i] == brr[i]) + +mutate(arr, arr) + +#bug #2240 +proc f(a: openarray[int], b: openArray[int]) = + for i in 0..a.len-1: + doAssert(a[i] == b[i]) + +var a = [7,8,9] +var p = addr a +f(p[], a) +f(sssp[], sss) + +ra[0] = 11 +ra[1] = 12 +ra[2] = 13 +f(ra[], raa) + +#bug #2240b +proc fillBuffer(buf: var openarray[char]) = + for i in 0..buf.len-1: + buf[i] = chr(i) + +proc fillSeqBuffer(b: ref seq[char]) = + fillBuffer(b[]) + +proc getFilledBuffer(sz: int): ref seq[char] = + let s : ref seq[char] = new(seq[char]) + s[] = newSeq[char](sz) + fillBuffer(s[]) + return s + +let aa = getFilledBuffer(3) +for i in 0..aa[].len-1: + doAssert(aa[i] == chr(i)) + +echo "OK" \ No newline at end of file From 164ebb6762022581e8b77b888d138de3aaf5561e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Jan 2016 10:19:36 +0100 Subject: [PATCH 071/125] made test green --- tests/metatype/ttypedesc3.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/metatype/ttypedesc3.nim b/tests/metatype/ttypedesc3.nim index 3d40b25b2..9f19bd6e3 100644 --- a/tests/metatype/ttypedesc3.nim +++ b/tests/metatype/ttypedesc3.nim @@ -6,7 +6,7 @@ type proc pr(T: typedesc[Base]) = echo "proc " & T.name method me(T: typedesc[Base]) = echo "method " & T.name -iterator it(T: typedesc[Base]) = yield "yield " & T.name +iterator it(T: typedesc[Base]): auto = yield "yield " & T.name Base.pr Child.pr From 1a2bda45ec11d6d0fcba4ba61114c352ae66d4de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Jan 2016 10:20:24 +0100 Subject: [PATCH 072/125] fixes #2007 --- compiler/lambdalifting.nim | 4 +-- compiler/semstmts.nim | 9 ++++--- tests/async/tlambda.nim | 55 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 tests/async/tlambda.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index acc291a5b..dfd5e740e 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -162,9 +162,9 @@ proc addHiddenParam(routine: PSym, param: PSym) = # some nkEffect node: param.position = routine.typ.n.len-1 addSon(params, newSymNode(param)) - incl(routine.typ.flags, tfCapturesEnv) + #incl(routine.typ.flags, tfCapturesEnv) assert sfFromGeneric in param.flags - #echo "produced environment: ", param.id, " for ", routine.name.s + #echo "produced environment: ", param.id, " for ", routine.id proc getHiddenParam(routine: PSym): PSym = let params = routine.ast.sons[paramsPos] diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 76d224a87..1340abd10 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -958,15 +958,17 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode = var n = n let original = n.sons[namePos].sym - let s = copySym(original, false) - incl(s.flags, sfFromGeneric) + let s = original #copySym(original, false) + #incl(s.flags, sfFromGeneric) + #s.owner = original n = replaceTypesInBody(c, pt, n, original) result = n s.ast = result n.sons[namePos].sym = s n.sons[genericParamsPos] = emptyNode - let params = n.typ.n + # for LL we need to avoid wrong aliasing + let params = copyTree n.typ.n n.sons[paramsPos] = params s.typ = n.typ for i in 1.. Future[void]): Client = + new result + result.onMessage = onMessage + +proc newBuild*(onProgress: ProgressCB): Build = + new result + result.onProgress = onProgress + +proc start(build: Build, repo, hash: string) {.async.} = + let path = repo.parseUri().path.toLower() + +proc onProgress(builder: Builder, message: string) {.async.} = + debug($message) + +proc onMessage(builder: Builder, message: JsonNode) {.async.} = + debug("onMessage") + +proc newBuilder(): Builder = + var cres: Builder + new cres + + cres.client = newClient("builder", (client, msg) => (onMessage(cres, msg))) + cres.build = newBuild( + proc (msg: string): Future[void] {.closure, gcsafe.} = onProgress(cres, msg)) + return cres + +proc main() = + # Set up logging. + var console = newConsoleLogger(fmtStr = verboseFmtStr) + addHandler(console) + + var builder = newBuilder() + + +main() From bb8b1c3b57ff56ee2c90d366bf453c066541bc11 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Jan 2016 14:59:43 +0100 Subject: [PATCH 073/125] fixes #2604, fixes #1527 --- compiler/ccgexprs.nim | 1 + compiler/transf.nim | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 7191c0f12..e12a3bce2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1989,6 +1989,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if sfGlobal in sym.flags: genVarPrototype(p.module, sym) if sym.loc.r == nil or sym.loc.t == nil: #echo "FAILED FOR PRCO ", p.prc.name.s + #echo renderTree(p.prc.ast, {renderIds}) internalError n.info, "expr: var not init " & sym.name.s & "_" & $sym.id if sfThread in sym.flags: accessThreadLocalVar(p, sym) diff --git a/compiler/transf.nim b/compiler/transf.nim index 1aba33c74..1021d1f5a 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -161,8 +161,9 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode = newVar.ast = defs[2].PNode result[i] = defs else: - # has been transformed into 'param.x' for closure iterators, so keep it: - result[i] = PTransNode(it) + # has been transformed into 'param.x' for closure iterators, so just + # transform it: + result[i] = transform(c, it) else: if it.kind != nkVarTuple: internalError(it.info, "transformVarSection: not nkVarTuple") @@ -820,7 +821,14 @@ proc transform(c: PTransf, n: PNode): PTransNode = # XXX comment handling really sucks: if importantComments(): PNode(result).comment = n.comment - of nkClosure: return PTransNode(n) + of nkClosure: + # it can happen that for-loop-inlining produced a fresh + # set of variables, including some computed environment + # (bug #2604). We need to patch this environment here too: + let a = n[1] + if a.kind == nkSym: + n.sons[1] = transformSymAux(c, a) + return PTransNode(n) else: result = transformSons(c, n) when false: From 4d10937a30798fe41def9bf5c462d2675c872947 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 5 Jan 2016 15:39:49 +0100 Subject: [PATCH 074/125] fixes regressions --- compiler/sem.nim | 2 ++ compiler/semstmts.nim | 2 ++ tests/iter/titer10.nim | 51 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 tests/iter/titer10.nim diff --git a/compiler/sem.nim b/compiler/sem.nim index 5134cb8b5..359cda3ac 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -186,6 +186,8 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = result.owner = getCurrOwner() else: result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info) + #if kind in {skForVar} and result.owner.kind == skModule: + # incl(result.flags, sfGlobal) proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 1340abd10..fdf147a2e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -326,6 +326,8 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = incl(result.flags, sfGlobal) else: result = semIdentWithPragma(c, kind, n, {}) + if result.owner.kind == skModule: + incl(result.flags, sfGlobal) suggestSym(n.info, result) styleCheckDef(result) diff --git a/tests/iter/titer10.nim b/tests/iter/titer10.nim new file mode 100644 index 000000000..6a6afc780 --- /dev/null +++ b/tests/iter/titer10.nim @@ -0,0 +1,51 @@ +discard """ + output: '''3 +2 +5 +1 +@[@[0, 0], @[0, 1]] +@[@[0, 0], @[0, 1]] +@[@[2, 2], @[2, 3]] +@[@[2, 2], @[2, 3]]''' +""" + +when true: + # bug #2604 + + import algorithm + + iterator byDistance*[int]( ints: openArray[int], base: int ): int = + var sortable = @ints + + sortable.sort do (a, b: int) -> int: + result = cmp( abs(base - a), abs(base - b) ) + + for val in sortable: + yield val + + when isMainModule: + proc main = + for val in byDistance([2, 3, 5, 1], 3): + echo val + main() + +when true: + # bug #1527 + + import sequtils + + let thread = @[@[0, 0], + @[0, 1], + @[2, 2], + @[2, 3]] + + iterator threadUniqs(seq1: seq[seq[int]]): seq[seq[int]] = + for i in 0 .. bool: x[0] == seq1[i][0] + proc main2 = + for uniqs in thread.threadUniqs: + echo uniqs + + main2() From 9aa60add873c7ecb85955525d840e0bdce630e0b Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 5 Jan 2016 15:56:13 +0000 Subject: [PATCH 075/125] Fixes docgen for lib/pure/db_common.nim. See http://forum.nim-lang.org/t/1916. --- lib/pure/db_common.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/db_common.nim b/lib/pure/db_common.nim index 9187a67ce..957389605 100644 --- a/lib/pure/db_common.nim +++ b/lib/pure/db_common.nim @@ -7,7 +7,9 @@ # distribution, for details about the copyright. # -## Common datatypes and definitions for all db_*.nim modules. +## Common datatypes and definitions for all ``db_*.nim`` ( +## `db_mysql `_, `db_postgres `_, +## and `db_sqlite `_) modules. type DbError* = object of IOError ## exception that is raised if a database error occurs From dfe01bbe69789535aef01e4a89f9195024eee630 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 6 Jan 2016 02:29:27 +0100 Subject: [PATCH 076/125] fixes #603 --- compiler/ccgstmts.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 699361ef8..529e59273 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -21,8 +21,8 @@ proc registerGcRoot(p: BProc, v: PSym) = # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) let prc = genTraverseProcForGlobal(p.module, v) - linefmt(p.module.initProc, cpsStmts, - "#nimRegisterGlobalMarker($1);$n", prc) + appcg(p.module, p.module.initProc.procSec(cpsStmts), + "#nimRegisterGlobalMarker($1);$n", [prc]) proc isAssignedImmediately(n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false From 98cc99cbcfa764aa3d14b4f77dcc85dbb568af21 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 6 Jan 2016 11:26:52 +0100 Subject: [PATCH 077/125] fixes all known regressions --- compiler/lambdalifting.nim | 2 +- compiler/sem.nim | 2 +- compiler/transf.nim | 34 ++++++++++++++++------------------ 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index dfd5e740e..f1fdcf446 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -751,12 +751,12 @@ proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode = tooEarly = true else: var d = initDetectionPass(fn) - var c = initLiftingPass(fn) detectCapturedVars(body, fn, d) if not d.somethingToDo and fn.isIterator: addClosureParam(d, fn) d.somethingToDo = true if d.somethingToDo: + var c = initLiftingPass(fn) var newBody = liftCapturedVars(body, fn, d, c) if c.envvars.getOrDefault(fn.id) != nil: newBody = newTree(nkStmtList, rawClosureCreation(fn, d, c), newBody) diff --git a/compiler/sem.nim b/compiler/sem.nim index 359cda3ac..cddd763ce 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -186,7 +186,7 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym = result.owner = getCurrOwner() else: result = newSym(kind, considerQuotedIdent(n), getCurrOwner(), n.info) - #if kind in {skForVar} and result.owner.kind == skModule: + #if kind in {skForVar, skLet, skVar} and result.owner.kind == skModule: # incl(result.flags, sfGlobal) proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, diff --git a/compiler/transf.nim b/compiler/transf.nim index 1021d1f5a..ff790306b 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -111,17 +111,22 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = result[1] = ri proc transformSymAux(c: PTransf, n: PNode): PNode = - if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure: - if c.tooEarly: return n - else: return liftIterSym(n, getCurrOwner(c)) + let s = n.sym + if s.typ != nil and s.typ.callConv == ccClosure: + if s.kind == skIterator: + if c.tooEarly: return n + else: return liftIterSym(n, getCurrOwner(c)) + elif s.kind in {skProc, skConverter, skMethod} and not c.tooEarly: + # top level .closure procs are still somewhat supported for 'Nake': + return makeClosure(s, nil, n.info) #elif n.sym.kind in {skVar, skLet} and n.sym.typ.callConv == ccClosure: # echo n.info, " come heer for ", c.tooEarly - # if not c.tooEarly: return makeClosure(n.sym, nil, n.info) + # if not c.tooEarly: var b: PNode var tc = c.transCon - if sfBorrow in n.sym.flags and n.sym.kind in routineKinds: + if sfBorrow in s.flags and s.kind in routineKinds: # simply exchange the symbol: - b = n.sym.getBody + b = s.getBody if b.kind != nkSym: internalError(n.info, "wrong AST for borrowed symbol") b = newSymNode(b.sym) b.info = n.info @@ -727,18 +732,11 @@ proc transform(c: PTransf, n: PNode): PTransNode = result = PTransNode(n) of nkBracketExpr: result = transformArrayAccess(c, n) of procDefs: - when false: - if n.sons[genericParamsPos].kind == nkEmpty: - var s = n.sons[namePos].sym - n.sons[bodyPos] = PNode(transform(c, s.getBody)) - if s.ast.sons[bodyPos] != n.sons[bodyPos]: - # somehow this can happen ... :-/ - s.ast.sons[bodyPos] = n.sons[bodyPos] - #n.sons[bodyPos] = liftLambdas(s, n) - #if n.kind == nkMethodDef: methodDef(s, false) - #if n.kind == nkIteratorDef and n.typ != nil: - # return liftIterSym(n.sons[namePos]).PTransNode - result = PTransNode(n) + var s = n.sons[namePos].sym + if n.typ != nil and s.typ.callConv == ccClosure: + result = transformSym(c, n.sons[namePos]) + else: + result = PTransNode(n) of nkMacroDef: # XXX no proper closure support yet: when false: From 589132065e0e56e9061d5b77c521142a97b5dfe8 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 6 Jan 2016 13:43:56 +0200 Subject: [PATCH 078/125] Lexbase compatible with JS --- lib/pure/lexbase.nim | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/pure/lexbase.nim b/lib/pure/lexbase.nim index bfecf6a58..cf2e8bb89 100644 --- a/lib/pure/lexbase.nim +++ b/lib/pure/lexbase.nim @@ -28,7 +28,10 @@ type BaseLexer* = object of RootObj ## the base lexer. Inherit your lexer from ## this object. bufpos*: int ## the current position within the buffer - buf*: cstring ## the buffer itself + when defined(js): ## the buffer itself + buf*: string + else: + buf*: cstring bufLen*: int ## length of buffer in characters input: Stream ## the input stream lineNumber*: int ## the current line number @@ -43,7 +46,8 @@ const proc close*(L: var BaseLexer) = ## closes the base lexer. This closes `L`'s associated stream too. - dealloc(L.buf) + when not defined(js): + dealloc(L.buf) close(L.input) proc fillBuffer(L: var BaseLexer) = @@ -58,8 +62,11 @@ proc fillBuffer(L: var BaseLexer) = toCopy = L.bufLen - L.sentinel - 1 assert(toCopy >= 0) if toCopy > 0: - moveMem(L.buf, addr(L.buf[L.sentinel + 1]), toCopy * chrSize) - # "moveMem" handles overlapping regions + when defined(js): + for i in 0 ..< toCopy: L.buf[i] = L.buf[L.sentinel + 1 + i] + else: + # "moveMem" handles overlapping regions + moveMem(L.buf, addr L.buf[L.sentinel + 1], toCopy * chrSize) charsRead = readData(L.input, addr(L.buf[toCopy]), (L.sentinel + 1) * chrSize) div chrSize s = toCopy + charsRead @@ -81,7 +88,10 @@ proc fillBuffer(L: var BaseLexer) = # double the buffer's size and try again: oldBufLen = L.bufLen L.bufLen = L.bufLen * 2 - L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize)) + when defined(js): + L.buf.setLen(L.bufLen) + else: + L.buf = cast[cstring](realloc(L.buf, L.bufLen * chrSize)) assert(L.bufLen - oldBufLen == oldBufLen) charsRead = readData(L.input, addr(L.buf[oldBufLen]), oldBufLen * chrSize) div chrSize @@ -139,7 +149,10 @@ proc open*(L: var BaseLexer, input: Stream, bufLen: int = 8192; L.bufpos = 0 L.bufLen = bufLen L.refillChars = refillChars - L.buf = cast[cstring](alloc(bufLen * chrSize)) + when defined(js): + L.buf = newString(bufLen) + else: + L.buf = cast[cstring](alloc(bufLen * chrSize)) L.sentinel = bufLen - 1 L.lineStart = 0 L.lineNumber = 1 # lines start at 1 From 2a08a3db560fcb42ed09acb887bb5c86984a6047 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 7 Jan 2016 23:27:22 +0000 Subject: [PATCH 079/125] Apparently rst comments work differently. Fix http://forum.nim-lang.org/t/1932. --- web/question.txt | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/web/question.txt b/web/question.txt index 2c3191b9b..4e7c15a10 100644 --- a/web/question.txt +++ b/web/question.txt @@ -23,27 +23,27 @@ General FAQ shared memory heap is also provided for the increased efficiency that results from that model. -.. - - .. container:: standout - - Why should I use Nim? - --------------------- - - It's a conservative language in a sense that we stick to features that have - proven themselves for larger scale programming. But it's revolutionary by - the features which have been laid on top. - - One of Nim's goals is to increase developer productivity without sacrificing - the produced software's stability. The way that this is done is by providing - - Depending on your use case. - - Nim is one of the few programming languages in the world which allows you to - The language inventor describes it as the ultimate programming language - with features which make it perfect for just about any problem. +.. .. container:: standout + +.. Why should I use Nim? +.. --------------------- + +.. It's a conservative language in a sense that we stick to features that have +.. proven themselves for larger scale programming. But it's revolutionary by +.. the features which have been laid on top. + +.. One of Nim's goals is to increase developer productivity without sacrificing +.. the produced software's stability. The way that this is done is by providing + +.. Depending on your use case. + +.. Nim is one of the few programming languages in the world which allows you to + + +.. The language inventor describes it as the ultimate programming language +.. with features which make it perfect for just about any problem. .. container:: standout From cd707084d1448f1676a2a2395cce27e3021d7c2d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jan 2016 12:08:32 +0100 Subject: [PATCH 080/125] fixes calling convention for the callbacks --- lib/pure/asyncftpclient.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index b806f4235..fd899e080 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -288,7 +288,7 @@ proc defaultOnProgressChanged*(total, progress: BiggestInt, result.complete() proc retrFile*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged = defaultOnProgressChanged) {.async.} = + onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = ## Downloads ``file`` and saves it to ``dest``. ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function ## when the download is finished. The event's ``filename`` field will be equal @@ -339,7 +339,7 @@ proc doUpload(ftp: AsyncFtpClient, file: File, await countdownFut or sendFut proc store*(ftp: AsyncFtpClient, file, dest: string, - onProgressChanged = defaultOnProgressChanged) {.async.} = + onProgressChanged: ProgressChangedProc = defaultOnProgressChanged) {.async.} = ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this ## function asynchronously is recommended to view the progress of ## the download. From 0844941b35d22093861a42cb4d3070a1f54f7896 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 8 Jan 2016 12:08:42 +0100 Subject: [PATCH 081/125] fixes regressions --- compiler/transf.nim | 2 ++ tests/destructor/tdestructor3.nim | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index ff790306b..dd8dd1519 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -735,6 +735,8 @@ proc transform(c: PTransf, n: PNode): PTransNode = var s = n.sons[namePos].sym if n.typ != nil and s.typ.callConv == ccClosure: result = transformSym(c, n.sons[namePos]) + # use the same node as before if still a symbol: + if result.PNode.kind == nkSym: result = PTransNode(n) else: result = PTransNode(n) of nkMacroDef: diff --git a/tests/destructor/tdestructor3.nim b/tests/destructor/tdestructor3.nim index 0968f1fd7..d0c53c7bd 100644 --- a/tests/destructor/tdestructor3.nim +++ b/tests/destructor/tdestructor3.nim @@ -19,10 +19,11 @@ proc `=`(lhs: var T, rhs: T) = proc `=destroy`(v: var T) = echo "destroy" -block: +proc usedToBeBlock = var v1 : T var v2 : T = v1 +usedToBeBlock() # bug #1632 From 09b159b8160c1063b323eb1ffb5751f283a52b57 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 8 Jan 2016 11:46:11 +0000 Subject: [PATCH 082/125] Fixed ttime test. Improved docs in times module. --- lib/pure/times.nim | 19 ++++++++++++++++--- tests/stdlib/ttime.nim | 1 - 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index f73ab99b4..3570ad37f 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -29,7 +29,7 @@ ## echo "epochTime() float value: ", epochTime() ## echo "getTime() float value: ", toSeconds(getTime()) ## echo "cpuTime() float value: ", cpuTime() -## echo "An hour from now : ", getLocalTime(getTime()) + initInterval(0,0,0,1) +## echo "An hour from now : ", getLocalTime(getTime()) + 1.hours ## echo "An hour from (UTC) now: ", getGmTime(getTime()) + initInterval(0,0,0,1) {.push debugger:off.} # the user does not want to trace a part @@ -245,6 +245,17 @@ proc getStartMilsecs*(): int {.deprecated, tags: [TimeEffect], benign.} proc initInterval*(milliseconds, seconds, minutes, hours, days, months, years: int = 0): TimeInterval = ## creates a new ``TimeInterval``. + ## + ## You can also use the convenience procedures called ``milliseconds``, + ## ``seconds``, ``minutes``, ``hours``, ``days``, ``months``, and ``years``. + ## + ## Example: + ## + ## .. code-block:: nim + ## + ## let day = initInterval(hours=24) + ## let tomorrow = getTime() + day + ## echo(tomorrow) var carryO = 0 result.milliseconds = `mod`(milliseconds, 1000) carryO = `div`(milliseconds, 1000) @@ -261,6 +272,7 @@ proc initInterval*(milliseconds, seconds, minutes, hours, days, months, result.years = carryO + years proc `+`*(ti1, ti2: TimeInterval): TimeInterval = + ## Adds two ``TimeInterval`` objects together. var carryO = 0 result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000) carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000) @@ -277,6 +289,7 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval = result.years = carryO + ti1.years + ti2.years proc `-`*(ti1, ti2: TimeInterval): TimeInterval = + ## Subtracts TimeInterval ``ti1`` from ``ti2``. result = ti1 result.milliseconds -= ti2.milliseconds result.seconds -= ti2.seconds @@ -345,7 +358,7 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float = result += newinterv.milliseconds / 1000 proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## adds ``interval`` time. + ## adds ``interval`` time from TimeInfo ``a``. ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. @@ -354,7 +367,7 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = result = getLocalTime(fromSeconds(t + secs)) proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = - ## subtracts ``interval`` time. + ## subtracts ``interval`` time from TimeInfo ``a``. ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. diff --git a/tests/stdlib/ttime.nim b/tests/stdlib/ttime.nim index 1df5daeec..449d16275 100644 --- a/tests/stdlib/ttime.nim +++ b/tests/stdlib/ttime.nim @@ -6,7 +6,6 @@ discard """ import times, strutils -doAssert( $getTime() == getLocalTime(getTime()).format("ddd MMM dd HH:mm:ss yyyy")) # $ date --date='@2147483647' # Tue 19 Jan 03:14:07 GMT 2038 From fffdae32c71290af73c0897954733c53216ce434 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 8 Jan 2016 11:54:34 +0000 Subject: [PATCH 083/125] Fixed TimeInterval bug. --- lib/pure/times.nim | 15 +++++++++------ tests/stdlib/ttime.nim | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 3570ad37f..03745d54e 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -171,11 +171,6 @@ type {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time, TTimeInterval: TimeInterval, TTimeInfo: TimeInfo].} -proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds - -proc `miliseconds=`*(t:var TimeInterval, milliseconds: int) {.deprecated.} = - t.milliseconds = milliseconds - proc getTime*(): Time {.tags: [TimeEffect], benign.} ## gets the current calendar time as a UNIX epoch value (number of seconds ## elapsed since 1970) with integer precission. Use epochTime for higher @@ -351,7 +346,6 @@ proc toSeconds(a: TimeInfo, interval: TimeInterval): float = else: curMonth.inc() result += float(newinterv.days * 24 * 60 * 60) - result += float(newinterv.days * 24 * 60 * 60) result += float(newinterv.hours * 60 * 60) result += float(newinterv.minutes * 60) result += float(newinterv.seconds) @@ -383,6 +377,15 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = let secs = toSeconds(a, intval) result = getLocalTime(fromSeconds(t + secs)) +proc miliseconds*(t: TimeInterval): int {.deprecated.} = t.milliseconds + +proc `miliseconds=`*(t: var TimeInterval, milliseconds: int) {.deprecated.} = + ## An alias for a misspelled field in ``TimeInterval``. + ## + ## **Warning:** This should not be used! It will be removed in the next + ## version. + t.milliseconds = milliseconds + when not defined(JS): proc epochTime*(): float {.rtl, extern: "nt$1", tags: [TimeEffect].} ## gets time after the UNIX epoch (1970) in seconds. It is a float diff --git a/tests/stdlib/ttime.nim b/tests/stdlib/ttime.nim index 449d16275..ac37196fb 100644 --- a/tests/stdlib/ttime.nim +++ b/tests/stdlib/ttime.nim @@ -142,3 +142,8 @@ doAssert initInterval(milliseconds = 25000) == initInterval(seconds = 25) doAssert initInterval(seconds = 65) == initInterval(seconds = 5, minutes = 1) doAssert initInterval(hours = 25) == initInterval(hours = 1, days = 1) doAssert initInterval(months = 13) == initInterval(months = 1, years = 1) + +# Bug with adding a day to a Time +let day = 24.hours +let tomorrow = getTime() + day +doAssert tomorrow - getTime() == 60*60*24 \ No newline at end of file From 5be6c95d217c1d86ad4288ea58b0c4c6ee3ffe90 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 11 Jan 2016 11:11:44 +0100 Subject: [PATCH 084/125] fixes async regression --- compiler/lambdalifting.nim | 7 +++++-- tests/async/tnimcall_to_closure.nim | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/async/tnimcall_to_closure.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index f1fdcf446..f75be0ed1 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -408,7 +408,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = addClosureParam(c, w) # , ow createUpField(c, w, up) w = up - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkTemplateDef, nkTypeSection: discard of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: @@ -687,11 +687,14 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; result = accessViaEnvParam(n, owner) else: result = accessViaEnvVar(n, owner, d, c) - of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure, + of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkTemplateDef, nkTypeSection: discard of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef: discard + of nkClosure: + if n[1].kind == nkNilLit: + n.sons[0] = liftCapturedVars(n[0], owner, d, c) of nkLambdaKinds, nkIteratorDef: if n.typ != nil and n[namePos].kind == nkSym: let m = newSymNode(n[namePos].sym) diff --git a/tests/async/tnimcall_to_closure.nim b/tests/async/tnimcall_to_closure.nim new file mode 100644 index 000000000..748b67cb1 --- /dev/null +++ b/tests/async/tnimcall_to_closure.nim @@ -0,0 +1,17 @@ + +import asyncdispatch + +proc defaultOnProgressChanged() = discard + +proc ask(x: proc()) = x() + +proc retrFile*(onProgressChanged: proc() {.nimcall.}): Future[void] = + var retFuture = newFuture[void]("retrFile") + iterator retrFileIter(): FutureBase {.closure.} = + ask(onProgressChanged) + complete(retFuture) + + var nameIterVar = retrFileIter + return retFuture + +discard retrFile(defaultOnProgressChanged) From 708b9f90b7a2fdbcbd9729eec9b8916446317be1 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Mon, 11 Jan 2016 22:52:59 +0100 Subject: [PATCH 085/125] Made docgen emit valid XML by default --- config/nimdoc.cfg | 6 +++--- lib/packages/docutils/rstgen.nim | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 23151b275..d973b922a 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -102,8 +102,8 @@ doc.file = """ - - + + $title @@ -1246,7 +1246,7 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator
-
+
Made with Nim. Generated: $date $time UTC
diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 4a0304a7c..22d944597 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -534,7 +534,7 @@ proc generateDocumentationJumps(docs: IndexedDocs): string = for title in titles: chunks.add("" & title.keyword & "") - result.add(chunks.join(", ") & ".
") + result.add(chunks.join(", ") & ".
") proc generateModuleJumps(modules: seq[string]): string = ## Returns a plain list of hyperlinks to the list of modules. @@ -544,7 +544,7 @@ proc generateModuleJumps(modules: seq[string]): string = for name in modules: chunks.add("" & name & "") - result.add(chunks.join(", ") & ".
") + result.add(chunks.join(", ") & ".
") proc readIndexDir(dir: string): tuple[modules: seq[string], symbols: seq[IndexEntry], docs: IndexedDocs] = From d0709cc758ae9668dc04e6699604cff0870ed053 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 13 Jan 2016 03:03:59 +0100 Subject: [PATCH 086/125] fixes more regressions --- compiler/lambdalifting.nim | 16 +++++++- compiler/transf.nim | 49 ++++++++++++++---------- tests/iter/tkeep_state_between_yield.nim | 36 +++++++++++++++++ 3 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 tests/iter/tkeep_state_between_yield.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index f75be0ed1..8320292ea 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -221,7 +221,7 @@ proc interestingIterVar(s: PSym): bool {.inline.} = # closure iterators quite a bit. result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags -template isIterator(owner: PSym): bool = +template isIterator*(owner: PSym): bool = owner.kind == skIterator and owner.typ.callConv == ccClosure proc liftIterSym*(n: PNode; owner: PSym): PNode = @@ -243,6 +243,20 @@ proc liftIterSym*(n: PNode; owner: PSym): PNode = result.add newCall(getSysSym"internalNew", envAsNode) result.add makeClosure(iter, envAsNode, n.info) +proc freshVarForClosureIter*(s, owner: PSym): PNode = + let envParam = getHiddenParam(owner) + let obj = envParam.typ.lastSon + addField(obj, s) + + var access = newSymNode(envParam) + assert obj.kind == tyObject + let field = getFieldFromObj(obj, s) + if field != nil: + result = rawIndirectAccess(access, field, s.info) + else: + localError(s.info, "internal error: cannot generate fresh variable") + result = access + # ------------------ new stuff ------------------------------------------- proc markAsClosure(owner: PSym; n: PNode) = diff --git a/compiler/transf.nim b/compiler/transf.nim index dd8dd1519..296ea0a0d 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -93,10 +93,15 @@ proc getCurrOwner(c: PTransf): PSym = if c.transCon != nil: result = c.transCon.owner else: result = c.module -proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PSym = - result = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) - result.typ = skipTypes(typ, {tyGenericInst}) - incl(result.flags, sfFromGeneric) +proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode = + let r = newSym(skTemp, getIdent(genPrefix), getCurrOwner(c), info) + r.typ = skipTypes(typ, {tyGenericInst}) + incl(r.flags, sfFromGeneric) + let owner = getCurrOwner(c) + if owner.isIterator and not c.tooEarly: + result = freshVarForClosureIter(r, owner) + else: + result = newSymNode(r) proc transform(c: PTransf, n: PNode): PTransNode @@ -141,6 +146,16 @@ proc transformSymAux(c: PTransf, n: PNode): PNode = proc transformSym(c: PTransf, n: PNode): PTransNode = result = PTransNode(transformSymAux(c, n)) +proc freshVar(c: PTransf; v: PSym): PNode = + let owner = getCurrOwner(c) + if owner.isIterator and not c.tooEarly: + result = freshVarForClosureIter(v, owner) + else: + var newVar = copySym(v) + incl(newVar.flags, sfFromGeneric) + newVar.owner = owner + result = newSymNode(newVar) + proc transformVarSection(c: PTransf, v: PNode): PTransNode = result = newTransNode(v) for i in countup(0, sonsLen(v)-1): @@ -150,20 +165,16 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode = elif it.kind == nkIdentDefs: if it.sons[0].kind == nkSym: internalAssert(it.len == 3) - var newVar = copySym(it.sons[0].sym) - incl(newVar.flags, sfFromGeneric) - # fixes a strange bug for rodgen: - #include(it.sons[0].sym.flags, sfFromGeneric); - newVar.owner = getCurrOwner(c) - idNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) + let x = freshVar(c, it.sons[0].sym) + idNodeTablePut(c.transCon.mapping, it.sons[0].sym, x) var defs = newTransNode(nkIdentDefs, it.info, 3) if importantComments(): # keep documentation information: PNode(defs).comment = it.comment - defs[0] = newSymNode(newVar).PTransNode + defs[0] = x.PTransNode defs[1] = it.sons[1].PTransNode defs[2] = transform(c, it.sons[2]) - newVar.ast = defs[2].PNode + if x.kind == nkSym: x.sym.ast = defs[2].PNode result[i] = defs else: # has been transformed into 'param.x' for closure iterators, so just @@ -175,11 +186,9 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode = var L = sonsLen(it) var defs = newTransNode(it.kind, it.info, L) for j in countup(0, L-3): - var newVar = copySym(it.sons[j].sym) - incl(newVar.flags, sfFromGeneric) - newVar.owner = getCurrOwner(c) - idNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar)) - defs[j] = newSymNode(newVar).PTransNode + let x = freshVar(c, it.sons[j].sym) + idNodeTablePut(c.transCon.mapping, it.sons[j].sym, x) + defs[j] = x.PTransNode assert(it.sons[L-2].kind == nkEmpty) defs[L-2] = ast.emptyNode.PTransNode defs[L-1] = transform(c, it.sons[L-1]) @@ -549,9 +558,9 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = of paFastAsgn: # generate a temporary and produce an assignment statement: var temp = newTemp(c, formal.typ, formal.info) - addVar(v, newSymNode(temp)) - add(stmtList, newAsgnStmt(c, newSymNode(temp), arg.PTransNode)) - idNodeTablePut(newC.mapping, formal, newSymNode(temp)) + addVar(v, temp) + add(stmtList, newAsgnStmt(c, temp, arg.PTransNode)) + idNodeTablePut(newC.mapping, formal, temp) of paVarAsgn: assert(skipTypes(formal.typ, abstractInst).kind == tyVar) idNodeTablePut(newC.mapping, formal, arg) diff --git a/tests/iter/tkeep_state_between_yield.nim b/tests/iter/tkeep_state_between_yield.nim new file mode 100644 index 000000000..f4f0ee363 --- /dev/null +++ b/tests/iter/tkeep_state_between_yield.nim @@ -0,0 +1,36 @@ +discard """ + output: '''@[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 18, 20, 21, 24, 27, 30, 36, 40, 42] +1002''' +""" + +import strutils + +proc slice[T](iter: iterator(): T {.closure.}, sl: auto): seq[T] = + var res: seq[int64] = @[] + var i = 0 + for n in iter(): + if i > sl.b: + break + if i >= sl.a: + res.add(n) + inc i + res + +iterator harshad(): int64 {.closure.} = + for n in 1 .. < int64.high: + var sum = 0 + for ch in string($n): + sum += parseInt("" & ch) + if n mod sum == 0: + yield n + +echo harshad.slice 0 .. <20 + +for n in harshad(): + if n > 1000: + echo n + break + + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed From 6fbde1f4fbadd6c2b1eb472324b76116eb8cbd8e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 13 Jan 2016 11:36:34 +0100 Subject: [PATCH 087/125] fixes another regression --- compiler/sigmatch.nim | 4 ++-- tests/generics/tspecialized_procvar.nim | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 tests/generics/tspecialized_procvar.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 784294839..354cf965e 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -167,12 +167,12 @@ proc sumGeneric(t: PType): int = t = t.lastSon if t.kind == tyEmpty: break inc result - of tyGenericInvocation, tyTuple: + of tyGenericInvocation, tyTuple, tyProc: result += ord(t.kind == tyGenericInvocation) for i in 0 .. Date: Wed, 13 Jan 2016 13:47:13 +0100 Subject: [PATCH 088/125] fixes yet another regression --- compiler/lambdalifting.nim | 10 +++++++--- tests/closure/tnoclosure.nim | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/closure/tnoclosure.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 8320292ea..3dd116078 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -730,13 +730,17 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; # ------------------ old stuff ------------------------------------------- proc semCaptureSym*(s, owner: PSym) = - if interestingVar(s) and owner.id != s.owner.id and s.kind != skResult: + if interestingVar(s) and s.kind != skResult: if owner.typ != nil and not isGenericRoutine(owner): # XXX: is this really safe? # if we capture a var from another generic routine, # it won't be consider captured. - owner.typ.callConv = ccClosure - #echo "semCaptureSym ", owner.name.s, owner.id, " ", s.name.s, s.id + var o = owner.skipGenericOwner + while o.kind != skModule and o != nil: + if s.owner == o: + owner.typ.callConv = ccClosure + #echo "computing .closure for ", owner.name.s, " ", owner.info, " because of ", s.name.s + o = o.skipGenericOwner # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' # here diff --git a/tests/closure/tnoclosure.nim b/tests/closure/tnoclosure.nim new file mode 100644 index 000000000..25cce0040 --- /dev/null +++ b/tests/closure/tnoclosure.nim @@ -0,0 +1,25 @@ +discard """ + output: '''@[1] +@[1, 1] +@[1, 2, 1] +@[1, 3, 3, 1] +@[1, 4, 6, 4, 1] +@[1, 5, 10, 10, 5, 1] +@[1, 6, 15, 20, 15, 6, 1] +@[1, 7, 21, 35, 35, 21, 7, 1] +@[1, 8, 28, 56, 70, 56, 28, 8, 1] +@[1, 9, 36, 84, 126, 126, 84, 36, 9, 1]''' +""" + +import sequtils + +proc pascal(n: int) = + var row = @[1] + for r in 1..n: + echo row + row = zip(row & @[0], @[0] & row).mapIt(it[0] + it[1]) + +pascal(10) + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed From 15c63b8011327c20acab5d155e0d5b0bd1e7c205 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 13 Jan 2016 16:06:31 +0100 Subject: [PATCH 089/125] fixes a long standing codegen bug concerning closures --- compiler/ccgexprs.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index e12a3bce2..73120bc93 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1838,8 +1838,8 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = assert n.kind == nkClosure if isConstClosure(n): - inc(p.labels) - var tmp = "LOC" & rope(p.labels) + inc(p.module.labels) + var tmp = "LOC" & rope(p.module.labels) addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)]) putIntoDest(p, d, n.typ, tmp, OnStatic) From 3838cd0066160c4b009a06125eff0016c12da32d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 14 Jan 2016 10:52:18 +0100 Subject: [PATCH 090/125] fixes iterator codegen regression --- compiler/lambdalifting.nim | 3 ++ compiler/transf.nim | 10 ++++++- tests/iter/tpermutations.nim | 58 ++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tests/iter/tpermutations.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 3dd116078..20f903da9 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -213,6 +213,8 @@ proc makeClosure*(prc: PSym; env: PNode; info: TLineInfo): PNode = if env == nil: result.add(newNodeIT(nkNilLit, info, getSysType(tyNil))) else: + if env.kind == nkClosure: + localError(info, "internal error: taking closure of closure") result.add(env) proc interestingIterVar(s: PSym): bool {.inline.} = @@ -709,6 +711,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; of nkClosure: if n[1].kind == nkNilLit: n.sons[0] = liftCapturedVars(n[0], owner, d, c) + #if n.sons[0].kind == nkClosure: result = n.sons[0] of nkLambdaKinds, nkIteratorDef: if n.typ != nil and n[namePos].kind == nkSym: let m = newSymNode(n[namePos].sym) diff --git a/compiler/transf.nim b/compiler/transf.nim index 296ea0a0d..50db2c7e8 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -313,10 +313,18 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PTransNode = result = PTransNode(n) of nkVarSection, nkLetSection: result = transformVarSection(c, n) + of nkClosure: + # it can happen that for-loop-inlining produced a fresh + # set of variables, including some computed environment + # (bug #2604). We need to patch this environment here too: + let a = n[1] + if a.kind == nkSym: + n.sons[1] = transformSymAux(c, a) + return PTransNode(n) else: result = newTransNode(n) for i in countup(0, sonsLen(n)-1): - result[i] = introduceNewLocalVars(c, n.sons[i]) + result[i] = introduceNewLocalVars(c, n.sons[i]) proc transformYield(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkStmtList, n.info, 0) diff --git a/tests/iter/tpermutations.nim b/tests/iter/tpermutations.nim new file mode 100644 index 000000000..a3b383323 --- /dev/null +++ b/tests/iter/tpermutations.nim @@ -0,0 +1,58 @@ + +import sequtils, future + +iterator permutations*[T](ys: openarray[T]): tuple[perm: seq[T], sign: int] = + var + d = 1 + c = newSeq[int](ys.len) + xs = newSeq[T](ys.len) + sign = 1 + + for i, y in ys: xs[i] = y + yield (xs, sign) + + block outter: + while true: + while d > 1: + dec d + c[d] = 0 + while c[d] >= d: + inc d + if d >= ys.len: break outter + + let i = if (d and 1) == 1: c[d] else: 0 + swap xs[i], xs[d] + sign *= -1 + yield (xs, sign) + inc c[d] + +proc det(a: seq[seq[float]]): float = + let n = toSeq 0..a.high + for sigma, sign in n.permutations: + result += sign.float * n.map((i: int) => a[i][sigma[i]]).foldl(a * b) + +proc perm(a: seq[seq[float]]): float = + let n = toSeq 0..a.high + for sigma, sign in n.permutations: + result += n.map((i: int) => a[i][sigma[i]]).foldl(a * b) + +for a in [ + @[ @[1.0, 2.0] + , @[3.0, 4.0] + ], + @[ @[ 1.0, 2, 3, 4] + , @[ 4.0, 5, 6, 7] + , @[ 7.0, 8, 9, 10] + , @[10.0, 11, 12, 13] + ], + @[ @[ 0.0, 1, 2, 3, 4] + , @[ 5.0, 6, 7, 8, 9] + , @[10.0, 11, 12, 13, 14] + , @[15.0, 16, 17, 18, 19] + , @[20.0, 21, 22, 23, 24] + ] ]: + echo a + echo "perm: ", a.perm, " det: ", a.det + +# bug #3499 last snippet fixed +# bug 705 last snippet fixed From 02a0782389ba2c7a55b9911d927b00b98b2120a4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 15 Jan 2016 01:50:52 +0100 Subject: [PATCH 091/125] fixes yet another codegen problem with constant closures --- compiler/ccgexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 73120bc93..ff1501cc2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1840,7 +1840,7 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = if isConstClosure(n): inc(p.module.labels) var tmp = "LOC" & rope(p.module.labels) - addf(p.module.s[cfsData], "NIM_CONST $1 $2 = $3;$n", + addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)]) putIntoDest(p, d, n.typ, tmp, OnStatic) else: From 9429e0f06458ab806deb88548622e93a9abb48fa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 15 Jan 2016 11:28:05 +0100 Subject: [PATCH 092/125] documented upcoming changes --- web/news.txt | 69 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/web/news.txt b/web/news.txt index db233cc71..dfeec45e6 100644 --- a/web/news.txt +++ b/web/news.txt @@ -3,27 +3,86 @@ News ==== .. - 2015-11-XX Version 0.12.2 released + 2016-01-XX Version 0.13.0 released ================================== - This is a bugfix release. - + This release implements no new features. Instead existing features have + been fleshed out as we're heading to version 1. Changes affecting backwards compatibility ----------------------------------------- - ``macros.newLit`` for ``bool`` now produces false/true symbols which actually work with the bool datatype. - - when compiling to JS, ``Node``, ``NodeType`` and ``Document`` are no longer + - When compiling to JS, ``Node``, ``NodeType`` and ``Document`` are no longer defined. Use the types defined in ``dom.nim`` instead. - The check ``x is iterator`` (used for instance in concepts) was always a weird special case (you could not use ``x is proc``) and was removed from the language. - - Implicit return type for iterators have been removed from the language. - Top level routines cannot have the calling convention ``closure`` anymore. + Syntax changes + ~~~~~~~~~~~~~~ + + The parser now considers leading whitespace in front of operators + to determine if an operator is used in prefix or infix position. + This means that finally ``echo $foo`` is parsed as people expect + which is as ``echo($foo)``. It used to be parsed as ``(echo) $ (foo)``. + + ``echo $ foo`` continues to be parsed as ``(echo) $ (foo)``. + + + Iterator changes + ~~~~~~~~~~~~~~~~ + + Implicit return type inference for iterators has been removed from the language. The following used to work: + + .. code-block:: nim + iterator it = + yield 7 + + This was a strange special case and has been removed. Now you need to write it like so which is consistent with procs: + + .. code-block:: nim + iterator it: auto = + yield 7 + + + Closure changes + ~~~~~~~~~~~~~~~ + + The semantics of closures changed: Capturing variables that are in loops do not produce a new environment. Nim closures behave like JavaScript closures now. + + The following used to work as the environment creation used to be attached to the loop body: + + .. code-block:: nim + + proc outer = + var s: seq[proc(): int {.closure.}] = @[] + for i in 0 ..< 30: + let ii = i + s.add(proc(): string = return ii*ii)) + + This behaviour has changed in 0.13.0 and now needs to be written as: + .. code-block:: nim + + proc outer = + var s: seq[proc(): int {.closure.}] = @[] + for i in 0 ..< 30: + (proc () = + let ii = i + s.add(proc(): int = return ii*ii))() + + The reason is that environment creations are now only performed once + per proc call. This change is subtle and unfortunate, but: + + 1. Affects almost no code out there. + 2. Is easier to implement and we are at a point in Nim's development process where simple+stable wins over perfect-in-theory+unstable-in-practice. + 3. Implies programmers are more in control of where memory is allocated which is benefitical for a systems programming language. + + 2015-10-27 Version 0.12.0 released ================================== From 46f1f41cc94fae52d26bf49e2b6eb74987857d9a Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 15 Jan 2016 11:34:52 +0100 Subject: [PATCH 093/125] finally removes 'echo ' gotcha --- compiler/parser.nim | 12 +++++++----- doc/manual/syntax.txt | 8 ++++++++ todo.txt | 7 ++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index 11dd6788a..eb0ef071d 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -250,12 +250,14 @@ proc isUnary(p: TParser): bool = if p.tok.tokType in {tkOpr, tkDotDot} and p.tok.strongSpaceB == 0 and p.tok.strongSpaceA > 0: - # XXX change this after 0.10.4 is out - if p.strongSpaces: result = true - else: - parMessage(p, warnDeprecated, - "will be parsed as unary operator; inconsistent spacing") + # versions prior to 0.13.0 used to do this: + when false: + if p.strongSpaces: + result = true + else: + parMessage(p, warnDeprecated, + "will be parsed as unary operator; inconsistent spacing") proc checkBinary(p: TParser) {.inline.} = ## Check if the current parser token is a binary operator. diff --git a/doc/manual/syntax.txt b/doc/manual/syntax.txt index c444a3995..ca3b582ca 100644 --- a/doc/manual/syntax.txt +++ b/doc/manual/syntax.txt @@ -64,6 +64,14 @@ Precedence level Operators First charact ================ =============================================== ================== =============== +Whether an operator is used a prefix operator is also affected by preceeding whitespace (this parsing change was introduced with version 0.13.0): + +.. code-block:: nim + echo $foo + # is parsed as + echo($foo) + + Strong spaces ------------- diff --git a/todo.txt b/todo.txt index afc3776a6..eca11ee07 100644 --- a/todo.txt +++ b/todo.txt @@ -2,8 +2,10 @@ essential for 1.0 ================= - introduce newSeqOfCap(10) -- The remaining bugs of the lambda lifting pass that is responsible to enable - closures and closure iterators need to be fixed. +- annotation support for getType() +- overloading of `()` needs to be in .experimental +- do notation needs to be documented properly + - ``concept`` needs to be refined, a nice name for the feature is not enough. - Destructors need to be refined. - make '--implicitStatic:on' the default; then we can also clean up the @@ -12,7 +14,6 @@ essential for 1.0 - add "all threads are blocked" detection to 'spawn' - Deprecate ``immediate`` for templates and macros - document NimMain and check whether it works for threading -- remove echo $foo gotcha - ``not`` or ``~`` for the effects system From 0d4e52c648a5e104393b634552b8a0cd4f46c4d0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 15 Jan 2016 14:44:54 +0100 Subject: [PATCH 094/125] makes tests green --- lib/pure/events.nim | 2 +- tests/gc/gcbench.nim | 2 +- tests/generics/tgeneric3.nim | 34 +++++++++---------- .../dependencies/genpacket/genpacket_enet.nim | 2 +- .../keineschweine/lib/client_helpers.nim | 2 +- tests/manyloc/keineschweine/lib/sg_assets.nim | 4 +-- tests/manyloc/nake/nakefile.nim | 2 +- tests/stdlib/tpegs.nim | 2 +- tests/types/tillegaltyperecursion.nim | 14 ++++---- web/news.txt | 4 +++ 10 files changed, 36 insertions(+), 32 deletions(-) diff --git a/lib/pure/events.nim b/lib/pure/events.nim index 62800c5c8..23a8a2c58 100644 --- a/lib/pure/events.nim +++ b/lib/pure/events.nim @@ -57,7 +57,7 @@ proc addHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) proc removeHandler*(handler: var EventHandler, fn: proc(e: EventArgs) {.closure.}) = ## Removes the callback from the specified event handler. - for i in countup(0, len(handler.handlers) -1): + for i in countup(0, len(handler.handlers)-1): if fn == handler.handlers[i]: handler.handlers.del(i) break diff --git a/tests/gc/gcbench.nim b/tests/gc/gcbench.nim index 72337911d..782daf793 100644 --- a/tests/gc/gcbench.nim +++ b/tests/gc/gcbench.nim @@ -143,7 +143,7 @@ proc main() = # Create long-lived array, filling half of it echo(" Creating a long-lived array of " & $kArraySize & " doubles") newSeq(myarray, kArraySize) - for i in 0..kArraySize div 2 -1: + for i in 0..kArraySize div 2 - 1: myarray[i] = 1.0 / toFloat(i) PrintDiagnostics() diff --git a/tests/generics/tgeneric3.nim b/tests/generics/tgeneric3.nim index d4dac9385..0dbd5b03c 100644 --- a/tests/generics/tgeneric3.nim +++ b/tests/generics/tgeneric3.nim @@ -69,7 +69,7 @@ proc cmp[T:int8|int16|int32|int64|int] (a,b: T): T {.inline.} = template binSearchImpl *(docmp: expr) {.immediate.} = var bFound = false result = 0 - var H = haystack.len -1 + var H = haystack.len - 1 while result <= H : var I {.inject.} = (result + H) shr 1 var SW = docmp @@ -90,7 +90,7 @@ proc DeleteItem[T,D] (n: PNode[T,D], x: int): PNode[T,D] {.inline.} = return n dec(n.count) if n.count > 0 : - for i in countup(x, n.count -1) : n.slots[i] = n.slots[i + 1] + for i in countup(x, n.count - 1) : n.slots[i] = n.slots[i + 1] n.slots[n.count] = nil case n.count of cLen1 : setLen(n.slots, cLen1) @@ -121,7 +121,7 @@ proc internalDelete[T,D] (ANode: PNode[T,D], key: T, Avalue: var D): PNode[T,D] if x == 0 : n = n.left else : - x = (-x) -1 + x = (-x) - 1 if x < n.count : n = n.slots[x].node else : @@ -132,10 +132,10 @@ proc internalDelete[T,D] (ANode: PNode[T,D], key: T, Avalue: var D): PNode[T,D] Avalue = n.slots[x].value var n2 = DeleteItem(n, x) dec(h) - while (n2 != n) and (h >=0) : + while (n2 != n) and (h >= 0) : n = n2 var w = addr Path[h] - x = w.Xi -1 + x = w.Xi - 1 if x >= 0 : if (n == nil) and isClean(w.Nd, x) : n = w.Nd @@ -160,7 +160,7 @@ proc internalFind[T,D] (n: PNode[T,D], key: T): ref TItem[T,D] {.inline.} = if x == 0 : wn = wn.left else : - x = (-x) -1 + x = (-x) - 1 if x < wn.count : wn = wn.slots[x].node else : @@ -199,7 +199,7 @@ proc traceTree[T,D](root: PNode[T,D]) = if n.left != nil: traceln(space) write stdout, "left: " - doTrace(n.left, level +1) + doTrace(n.left, level+1) for i, el in n.slots : if el != nil and not isClean(el): traceln(space) @@ -208,7 +208,7 @@ proc traceTree[T,D](root: PNode[T,D]) = write stdout, "error " else: traceEl(el) - if el.node != nil: doTrace(el.node, level +1) + if el.node != nil: doTrace(el.node, level+1) else : write stdout, " empty " elif i < n.count : traceln(space) @@ -217,7 +217,7 @@ proc traceTree[T,D](root: PNode[T,D]) = when T is string : if el.key != nil: write stdout, el.key else : write stdout, el.key - if el.node != nil: doTrace(el.node, level +1) + if el.node != nil: doTrace(el.node, level+1) else : write stdout, " empty " writeLine stdout,"" @@ -245,25 +245,25 @@ proc SplitPage[T,D](n, left: PNode[T,D], xi: int, Akey:var T, Avalue:var D): PNo result.slots.newSeq(cLenCenter) result.count = cCenter if x == cCenter: - for i in 0..cCenter -1: shallowCopy(it1[i], left.slots[i]) - for i in 0..cCenter -1: shallowCopy(result.slots[i], left.slots[cCenter + i]) + for i in 0..cCenter-1: shallowCopy(it1[i], left.slots[i]) + for i in 0..cCenter-1: shallowCopy(result.slots[i], left.slots[cCenter + i]) result.left = n else : if x < cCenter : for i in 0..x-1: shallowCopy(it1[i], left.slots[i]) it1[x] = setItem(Akey, Avalue, n) - for i in x+1 .. cCenter -1: shallowCopy(it1[i], left.slots[i-1]) - var w = left.slots[cCenter -1] + for i in x+1 .. cCenter-1: shallowCopy(it1[i], left.slots[i-1]) + var w = left.slots[cCenter-1] Akey = w.key Avalue = w.value result.left = w.node - for i in 0..cCenter -1: shallowCopy(result.slots[i], left.slots[cCenter + i]) + for i in 0..cCenter-1: shallowCopy(result.slots[i], left.slots[cCenter + i]) else : - for i in 0..cCenter -1: shallowCopy(it1[i], left.slots[i]) + for i in 0..cCenter-1: shallowCopy(it1[i], left.slots[i]) x = x - (cCenter + 1) for i in 0..x-1: shallowCopy(result.slots[i], left.slots[cCenter + i + 1]) result.slots[x] = setItem(Akey, Avalue, n) - for i in x+1 .. cCenter -1: shallowCopy(result.slots[i], left.slots[cCenter + i]) + for i in x+1 .. cCenter-1: shallowCopy(result.slots[i], left.slots[cCenter + i]) var w = left.slots[cCenter] Akey = w.key Avalue = w.value @@ -290,7 +290,7 @@ proc internalPut[T,D](ANode: ref TNode[T,D], Akey: T, Avalue: D, Oldvalue: var D if x == 0 : n = n.left else : - x = (-x) -1 + x = (-x)-1 if x < n.count : n = n.slots[x].node else : diff --git a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim index 7cfd67c49..e64b818fa 100644 --- a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim +++ b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim @@ -277,7 +277,7 @@ when isMainModule: s.flush defPacket(Y, tuple[z: int8]) - proc `$`(z: Y): string = result = "Y("& $z.z &")" + proc `$`(z: Y): string = result = "Y(" & $z.z & ")" defPacket(TestPkt, tuple[x: seq[Y]]) var test = newTestPkt() test.x.add([newY(5), newY(4), newY(3), newY(2), newY(1)]) diff --git a/tests/manyloc/keineschweine/lib/client_helpers.nim b/tests/manyloc/keineschweine/lib/client_helpers.nim index f2833fe14..5f819a7d1 100644 --- a/tests/manyloc/keineschweine/lib/client_helpers.nim +++ b/tests/manyloc/keineschweine/lib/client_helpers.nim @@ -66,7 +66,7 @@ proc handlePackets*(server: PServer; buf: PBuffer) = proc updateFileProgress*() = let progress = currentFileTransfer.pos / currentFileTransfer.fullLen downloadProgress.bg.setSize(vec2f(progress * 100, 20)) - downloadProgress.setString($currentFileTransfer.pos &'/'& $currentFileTransfer.fullLen) + downloadProgress.setString($currentFileTransfer.pos & '/' & $currentFileTransfer.fullLen) ## HFileTransfer proc handleFilePartRecv*(serv: PServer; buffer: PBuffer) {.procvar.} = diff --git a/tests/manyloc/keineschweine/lib/sg_assets.nim b/tests/manyloc/keineschweine/lib/sg_assets.nim index 3b9781649..801c3456b 100644 --- a/tests/manyloc/keineschweine/lib/sg_assets.nim +++ b/tests/manyloc/keineschweine/lib/sg_assets.nim @@ -573,13 +573,13 @@ proc importItem(data: PJsonNode; errors: var seq[string]): PItemRecord = elif data[2]["bullet"].kind == JObject: result.bullet = importBullet(data[2]["bullet"], errors) else: - errors.add "UNKNOWN BULLET TYPE for item "& result.name + errors.add "UNKNOWN BULLET TYPE for item " & result.name of "ammo": result.kind = Ammo of "utility": nil else: - errors.add "Invalid item type \""& data[1].str &"\" for item "& result.name + errors.add "Invalid item type \""&data[1].str&"\" for item "&result.name proc importBullet(data: PJsonNode; errors: var seq[string]): PBulletRecord = new(result) diff --git a/tests/manyloc/nake/nakefile.nim b/tests/manyloc/nake/nakefile.nim index de6a1af40..e91b86986 100644 --- a/tests/manyloc/nake/nakefile.nim +++ b/tests/manyloc/nake/nakefile.nim @@ -146,7 +146,7 @@ task "download", "download game assets": task "zip-lib", "zip up the libs dir": var z: TZipArchive - if not z.open("libs-"& getDateStr() &".zip", fmReadWrite): + if not z.open("libs-" & getDateStr() & ".zip", fmReadWrite): quit "Could not open zip" for file in walkDirRec("libs", {pcFile, pcDir}): echo "adding file ", file diff --git a/tests/stdlib/tpegs.nim b/tests/stdlib/tpegs.nim index 3fe964d82..ec839e288 100644 --- a/tests/stdlib/tpegs.nim +++ b/tests/stdlib/tpegs.nim @@ -764,7 +764,7 @@ proc match*(s: string, pattern: TPeg, matches: var openarray[string], ## returned. var c: TCaptures c.origStart = start - result = rawMatch(s, pattern, start, c) == len(s) -start + result = rawMatch(s, pattern, start, c) == len(s)-start if result: for i in 0..c.ml-1: matches[i] = substr(s, c.matches[i][0], c.matches[i][1]) diff --git a/tests/types/tillegaltyperecursion.nim b/tests/types/tillegaltyperecursion.nim index bace2dfc8..52fbd622f 100644 --- a/tests/types/tillegaltyperecursion.nim +++ b/tests/types/tillegaltyperecursion.nim @@ -10,14 +10,14 @@ import strutils import os type - TMessageReceivedEventArgs = object of TEventArgs + TMessageReceivedEventArgs = object of EventArgs Nick*: string Message*: string TIRC = object - EventEmitter: TEventEmitter - MessageReceivedHandler*: TEventHandler - Socket: TSocket - Thread: TThread[TIRC] + EventEmitter: EventEmitter + MessageReceivedHandler*: EventHandler + Socket: Socket + Thread: Thread[TIRC] proc initIRC*(): TIRC = result.Socket = socket() @@ -49,8 +49,8 @@ proc handleData(irc: TIRC) {.thread.} = return proc Connect*(irc: var TIRC, nick: string, host: string, port: int = 6667) = - connect(irc.Socket ,host ,TPort(port),TDomain.AF_INET) - send(irc.Socket,"USER " & nick & " " & nick & " " & nick & " " & nick &"\r\L") + connect(irc.Socket, host, TPort(port), TDomain.AF_INET) + send(irc.Socket,"USER " & nick & " " & nick & " " & nick & " " & nick & "\r\L") send(irc.Socket,"NICK " & nick & "\r\L") var thread: TThread[TIRC] createThread(thread, handleData, irc) diff --git a/web/news.txt b/web/news.txt index dfeec45e6..4bb34400e 100644 --- a/web/news.txt +++ b/web/news.txt @@ -33,6 +33,10 @@ News ``echo $ foo`` continues to be parsed as ``(echo) $ (foo)``. + This also means that ``-1`` is always parsed as prefix operator so + code like ``0..kArraySize div 2 -1`` needs to be changed to + ``0..kArraySize div 2 - 1``. + Iterator changes ~~~~~~~~~~~~~~~~ From 34a783a61c0b59327891b4a026b80b75acbf022f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 15 Jan 2016 14:56:01 +0100 Subject: [PATCH 095/125] fixes #3723 --- compiler/ccgexprs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index ff1501cc2..6f513c165 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1839,7 +1839,7 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = if isConstClosure(n): inc(p.module.labels) - var tmp = "LOC" & rope(p.module.labels) + var tmp = "CNSTCLOSURE" & rope(p.module.labels) addf(p.module.s[cfsData], "static NIM_CONST $1 $2 = $3;$n", [getTypeDesc(p.module, n.typ), tmp, genConstExpr(p, n)]) putIntoDest(p, d, n.typ, tmp, OnStatic) From 480f36983a6888ae542557acdf630dc1e41ff771 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 16 Jan 2016 13:09:28 +0100 Subject: [PATCH 096/125] todo update --- todo.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/todo.txt b/todo.txt index afc3776a6..ae11e0fb2 100644 --- a/todo.txt +++ b/todo.txt @@ -4,6 +4,7 @@ essential for 1.0 - introduce newSeqOfCap(10) - The remaining bugs of the lambda lifting pass that is responsible to enable closures and closure iterators need to be fixed. +- find a solution for the x.f[T](y) gotcha - ``concept`` needs to be refined, a nice name for the feature is not enough. - Destructors need to be refined. - make '--implicitStatic:on' the default; then we can also clean up the From 2910096ccec456aa6493d06c9cc36bb70acd2988 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 16 Jan 2016 13:43:22 +0100 Subject: [PATCH 097/125] updated the documentation for the 'do' notation --- doc/manual/procs.txt | 8 ++++---- todo.txt | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/manual/procs.txt b/doc/manual/procs.txt index ee74b2ea6..654893286 100644 --- a/doc/manual/procs.txt +++ b/doc/manual/procs.txt @@ -236,8 +236,6 @@ executable code. Do notation ----------- -**Note:** The future of the ``do`` notation is uncertain. - As a special more convenient notation, proc expressions involved in procedure calls can use the ``do`` keyword: @@ -251,10 +249,12 @@ calls can use the ``do`` keyword: ``do`` is written after the parentheses enclosing the regular proc params. The proc expression represented by the do block is appended to them. -More than one ``do`` block can appear in a single call: +``do`` with parentheses is an anonymous ``proc``; however a ``do`` without +parentheses is just a block of code. The ``do`` notation can be used to +pass multiple blocks to a macro: .. code-block:: nim - proc performWithUndo(task: proc(), undo: proc()) = ... + macro performWithUndo(task, undo: untyped) = ... performWithUndo do: # multiple-line block of code diff --git a/todo.txt b/todo.txt index ac2041f66..ca8d60762 100644 --- a/todo.txt +++ b/todo.txt @@ -4,7 +4,6 @@ essential for 1.0 - introduce newSeqOfCap(10) - annotation support for getType() - overloading of `()` needs to be in .experimental -- do notation needs to be documented properly - find a solution for the x.f[T](y) gotcha - ``concept`` needs to be refined, a nice name for the feature is not enough. - Destructors need to be refined. From b6e8a1437bf007aa9914f10a3f1c78b27b2a359d Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 13:32:26 +0000 Subject: [PATCH 098/125] Added test for #3304 --- tests/closure/tclosureinference3304.nim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/closure/tclosureinference3304.nim diff --git a/tests/closure/tclosureinference3304.nim b/tests/closure/tclosureinference3304.nim new file mode 100644 index 000000000..db4aa1d04 --- /dev/null +++ b/tests/closure/tclosureinference3304.nim @@ -0,0 +1,15 @@ +discard """ + output: '''@[1, 2, 5]''' +""" + +import future, sequtils + +type + List[T] = ref object + val: T + +proc foo[T](l: List[T]): seq[int] = + @[1,2,3,5].filter(x => x != l.val) + +when isMainModule: + echo(foo(List[int](val: 3))) From 40ae1ad3639040592877adf0b235e45d3b1785fd Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 13:49:37 +0000 Subject: [PATCH 099/125] Added test for #2138. --- tests/closure/tfutclosure2138.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/closure/tfutclosure2138.nim diff --git a/tests/closure/tfutclosure2138.nim b/tests/closure/tfutclosure2138.nim new file mode 100644 index 000000000..e18834074 --- /dev/null +++ b/tests/closure/tfutclosure2138.nim @@ -0,0 +1,10 @@ +import future, sequtils + +proc any[T](list: varargs[T], pred: (T) -> bool): bool = + for item in list: + if pred(item): + result = true + break + +proc contains(s: string, words: varargs[string]): bool = + any(words, (word) => s.contains(word)) \ No newline at end of file From bc65a7c76b627beef294e87bc1b1db540e4ee610 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 13:54:28 +0000 Subject: [PATCH 100/125] Added test for #1923. --- tests/closure/tcodegenerr1923.nim | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/closure/tcodegenerr1923.nim diff --git a/tests/closure/tcodegenerr1923.nim b/tests/closure/tcodegenerr1923.nim new file mode 100644 index 000000000..ee131ae15 --- /dev/null +++ b/tests/closure/tcodegenerr1923.nim @@ -0,0 +1,9 @@ +type + Foo[M] = proc() : M + +proc bar[M](f : Foo[M]) = + discard f() + +proc baz() : int = 42 + +bar(baz) \ No newline at end of file From ac71ece6738b14019d0512b4d93394cea2ab924d Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 13:56:44 +0000 Subject: [PATCH 101/125] Added test for #1911. --- tests/closure/tissue1911.nim | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/closure/tissue1911.nim diff --git a/tests/closure/tissue1911.nim b/tests/closure/tissue1911.nim new file mode 100644 index 000000000..311d99134 --- /dev/null +++ b/tests/closure/tissue1911.nim @@ -0,0 +1,7 @@ +proc foo(x: int) : auto = + + proc helper() : int = x + proc bar() : int = helper() + proc baz() : int = helper() + + return (bar, baz) \ No newline at end of file From 19ea2672c545288c8cf4b4bd925ed3fd3bdd88b2 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 13:58:40 +0000 Subject: [PATCH 102/125] Added test for #1846. --- tests/closure/tissue1846.nim | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/closure/tissue1846.nim diff --git a/tests/closure/tissue1846.nim b/tests/closure/tissue1846.nim new file mode 100644 index 000000000..3fbef169d --- /dev/null +++ b/tests/closure/tissue1846.nim @@ -0,0 +1,16 @@ +type + TBinOp*[T] = proc (x,y: T): bool + + THeap*[T] = object + cmp*: TBinOp[T] + +proc less*[T](x,y: T): bool = + x < y + +proc initHeap*[T](cmp: TBinOp[T]): THeap[T] = + result.cmp = cmp + +when isMainModule: + var h = initHeap[int](less[int]) + + echo h.cmp(2,3) \ No newline at end of file From abc72eb9195929ad9837783d0b450ce2da52e3d1 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 14:02:25 +0000 Subject: [PATCH 103/125] Added test for #1642. --- tests/closure/tissue1642.nim | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/closure/tissue1642.nim diff --git a/tests/closure/tissue1642.nim b/tests/closure/tissue1642.nim new file mode 100644 index 000000000..5b921fc05 --- /dev/null +++ b/tests/closure/tissue1642.nim @@ -0,0 +1,3 @@ +block: + var i = 0 + proc p() = inc(i) \ No newline at end of file From 23cebff5fc18941ea72d0afb8235536b0b8dcab1 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 14:09:28 +0000 Subject: [PATCH 104/125] Added test for #1512. --- tests/closure/tmacrobust1512.nim | 137 +++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 tests/closure/tmacrobust1512.nim diff --git a/tests/closure/tmacrobust1512.nim b/tests/closure/tmacrobust1512.nim new file mode 100644 index 000000000..95681e750 --- /dev/null +++ b/tests/closure/tmacrobust1512.nim @@ -0,0 +1,137 @@ +import macros, strutils + +# https://github.com/nim-lang/Nim/issues/1512 + +proc macrobust0 (raw_input: string) = + var output = "" + proc p1 (a:string) = + output.add (a) + + proc p2 (a:string) = p1 (a) + proc p3 (a:string) = p2 (a) + proc p4 (a:string) = p3 (a) + proc p5 (a:string) = p4 (a) + proc p6 (a:string) = p5 (a) + proc p7 (a:string) = p6 (a) + proc p8 (a:string) = p7 (a) + proc p9 (a:string) = p8 (a) + proc p10 (a:string) = p9 (a) + proc p11 (a:string) = p10 (a) + proc p12 (a:string) = p11 (a) + proc p13 (a:string) = p12 (a) + proc p14 (a:string) = p13 (a) + proc p15 (a:string) = p14 (a) + proc p16 (a:string) = p15 (a) + proc p17 (a:string) = p16 (a) + proc p18 (a:string) = p17 (a) + proc p19 (a:string) = p18 (a) + proc p20 (a:string) = p19 (a) + + let input = $raw_input + + for a in input.split (): + p20 (a) + p19 (a) + + + p18 (a) + p17 (a) + p16 (a) + p15 (a) + p14 (a) + p13 (a) + p12 (a) + p11 (a) + p10 (a) + p9 (a) + p8 (a) + p7 (a) + p6 (a) + p5 (a) + p4 (a) + p3 (a) + p2 (a) + p1 (a) + + + echo output + +macro macrobust (raw_input: expr) : stmt = + + var output = "" + proc p1 (a:string) = + output.add (a) + + proc p2 (a:string) = p1 (a) + proc p3 (a:string) = p2 (a) + proc p4 (a:string) = p3 (a) + proc p5 (a:string) = p4 (a) + proc p6 (a:string) = p5 (a) + proc p7 (a:string) = p6 (a) + proc p8 (a:string) = p7 (a) + proc p9 (a:string) = p8 (a) + proc p10 (a:string) = p9 (a) + proc p11 (a:string) = p10 (a) + proc p12 (a:string) = p11 (a) + proc p13 (a:string) = p12 (a) + proc p14 (a:string) = p13 (a) + proc p15 (a:string) = p14 (a) + proc p16 (a:string) = p15 (a) + proc p17 (a:string) = p16 (a) + proc p18 (a:string) = p17 (a) + proc p19 (a:string) = p18 (a) + proc p20 (a:string) = p19 (a) + + let input = $raw_input + + for a in input.split (): + p20 (a) + p19 (a) + + p18 (a) + p17 (a) + p16 (a) + p15 (a) + p14 (a) + p13 (a) + p12 (a) + p11 (a) + p10 (a) + p9 (a) + p8 (a) + p7 (a) + p6 (a) + p5 (a) + p4 (a) + p3 (a) + p2 (a) + + echo output + discard result + +macrobust """ + fdsasadfsdfa sadfsdafsdaf + dsfsdafdsfadsfa fsdaasdfasdf + fsdafsadfsad asdfasdfasdf + fdsasdfasdfa sadfsadfsadf + sadfasdfsdaf sadfsdafsdaf dsfasdaf + sadfsdafsadf fdsasdafsadf fdsasadfsdaf + sdfasadfsdafdfsa sadfsadfsdaf + sdafsdaffsda sdfasadfsadf + fsdasdafsdfa sdfasdfafsda + sdfasdafsadf sdfasdafsdaf sdfasdafsdaf +""" + + +macrobust0 """ + fdsasadfsdfa sadfsdafsdaf + dsfsdafdsfadsfa fsdaasdfasdf + fsdafsadfsad asdfasdfasdf + fdsasdfasdfa sadfsadfsadf + sadfasdfsdaf sadfsdafsdaf dsfasdaf + sadfsdafsadf fdsasdafsadf fdsasadfsdaf + sdfasadfsdafdfsa sadfsadfsdaf + sdafsdaffsda sdfasadfsadf + fsdasdafsdfa sdfasdfafsda + sdfasdafsadf sdfasdafsdaf sdfasdafsdaf +""" \ No newline at end of file From a395182dd7577ffa099b351ff7adfd3bb6939d54 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 14:14:29 +0000 Subject: [PATCH 105/125] Added test for code in @def-'s comment in #1502. --- tests/closure/tissue1502def.nim | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/closure/tissue1502def.nim diff --git a/tests/closure/tissue1502def.nim b/tests/closure/tissue1502def.nim new file mode 100644 index 000000000..0aa6b16e3 --- /dev/null +++ b/tests/closure/tissue1502def.nim @@ -0,0 +1,6 @@ +import sequtils +let xs: seq[tuple[key: string, val: seq[string]]] = @[("foo", @["bar"])] + +let maps = xs.map( + proc(x: auto): tuple[typ: string, maps: seq[string]] = + (x.key, x.val.map(proc(x: string): string = x))) \ No newline at end of file From d0d1b396ed8a61cf6c13df26e3b8d43741033cf9 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 14:29:48 +0000 Subject: [PATCH 106/125] Added test for #600. --- tests/closure/tissue600.nim | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/closure/tissue600.nim diff --git a/tests/closure/tissue600.nim b/tests/closure/tissue600.nim new file mode 100644 index 000000000..eacc7a123 --- /dev/null +++ b/tests/closure/tissue600.nim @@ -0,0 +1,4 @@ +for i in 1..1: + var reported = false + proc report() = + reported = true \ No newline at end of file From eb1bedc3c26967b12b07a84621e3f2ce3fb65dd8 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 16 Jan 2016 14:58:12 +0000 Subject: [PATCH 107/125] Removed redis module from standard library. --- lib/pure/redis.nim | 1096 -------------------------------------------- web/news.txt | 3 + 2 files changed, 3 insertions(+), 1096 deletions(-) delete mode 100644 lib/pure/redis.nim diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim deleted file mode 100644 index 6741d3c8e..000000000 --- a/lib/pure/redis.nim +++ /dev/null @@ -1,1096 +0,0 @@ -# -# -# Nim's Runtime Library -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## This module implements a redis client. It allows you to connect to a -## redis-server instance, send commands and receive replies. -## -## **Beware**: Most (if not all) functions that return a ``RedisString`` may -## return ``redisNil``, and functions which return a ``RedisList`` -## may return ``nil``. - -import net, os, strutils, parseutils - -const - redisNil* = "\0\0" - -type - Pipeline = ref object - enabled: bool - buffer: string - expected: int ## number of replies expected if pipelined - -type - SendMode = enum - normal, pipelined, multiple - -type - Redis* = object - socket: net.Socket - connected: bool - pipeline: Pipeline - - RedisStatus* = string - RedisInteger* = BiggestInt - RedisString* = string ## Bulk reply - RedisList* = seq[RedisString] ## Multi-bulk reply - - ReplyError* = object of IOError ## Invalid reply from redis - RedisError* = object of IOError ## Error in redis - -{.deprecated: [TSendMode: SendMode, TRedis: Redis, TRedisStatus: RedisStatus, - TRedisInteger: RedisInteger, TRedisString: RedisString, - TRedisList: RedisList, EInvalidReply: ReplyError, ERedis: RedisError].} - -proc newPipeline(): Pipeline = - new(result) - result.buffer = "" - result.enabled = false - result.expected = 0 - -proc open*(host = "localhost", port = 6379.Port): Redis = - ## Opens a connection to the redis server. - result.socket = newSocket(buffered = false) - - result.socket.connect(host, port) - result.pipeline = newPipeline() - -proc raiseInvalidReply(expected, got: char) = - raise newException(ReplyError, - "Expected '$1' at the beginning of a status reply got '$2'" % - [$expected, $got]) - -proc raiseNoOK(status: string, pipelineEnabled: bool) = - if pipelineEnabled and not (status == "QUEUED" or status == "PIPELINED"): - raise newException(ReplyError, "Expected \"QUEUED\" or \"PIPELINED\" got \"$1\"" % status) - elif not pipelineEnabled and status != "OK": - raise newException(ReplyError, "Expected \"OK\" got \"$1\"" % status) - -template readSocket(r: Redis, dummyVal:expr): stmt = - var line {.inject.}: TaintedString = "" - if r.pipeline.enabled: - return dummyVal - else: - readLine(r.socket, line) - -proc parseStatus(r: Redis, line: string = ""): RedisStatus = - if r.pipeline.enabled: - return "PIPELINED" - - if line == "": - raise newException(RedisError, "Server closed connection prematurely") - - if line[0] == '-': - raise newException(RedisError, strip(line)) - if line[0] != '+': - raiseInvalidReply('+', line[0]) - - return line.substr(1) # Strip '+' - -proc readStatus(r:Redis): RedisStatus = - r.readSocket("PIPELINED") - return r.parseStatus(line) - -proc parseInteger(r: Redis, line: string = ""): RedisInteger = - if r.pipeline.enabled: return -1 - - #if line == "+QUEUED": # inside of multi - # return -1 - - if line == "": - raise newException(RedisError, "Server closed connection prematurely") - - if line[0] == '-': - raise newException(RedisError, strip(line)) - if line[0] != ':': - raiseInvalidReply(':', line[0]) - - # Strip ':' - if parseBiggestInt(line, result, 1) == 0: - raise newException(ReplyError, "Unable to parse integer.") - -proc readInteger(r: Redis): RedisInteger = - r.readSocket(-1) - return r.parseInteger(line) - -proc recv(sock: Socket, size: int): TaintedString = - result = newString(size).TaintedString - if sock.recv(cstring(result), size) != size: - raise newException(ReplyError, "recv failed") - -proc parseSingleString(r: Redis, line:string, allowMBNil = false): RedisString = - if r.pipeline.enabled: return "" - - # Error. - if line[0] == '-': - raise newException(RedisError, strip(line)) - - # Some commands return a /bulk/ value or a /multi-bulk/ nil. Odd. - if allowMBNil: - if line == "*-1": - return redisNil - - if line[0] != '$': - raiseInvalidReply('$', line[0]) - - var numBytes = parseInt(line.substr(1)) - if numBytes == -1: - return redisNil - - var s = r.socket.recv(numBytes+2) - result = strip(s.string) - -proc readSingleString(r: Redis): RedisString = - r.readSocket("") - return r.parseSingleString(line) - -proc readNext(r: Redis): RedisList - -proc parseArrayLines(r: Redis, countLine:string): RedisList = - if countLine.string[0] != '*': - raiseInvalidReply('*', countLine.string[0]) - - var numElems = parseInt(countLine.string.substr(1)) - if numElems == -1: return nil - result = @[] - - for i in 1..numElems: - var parsed = r.readNext() - if not isNil(parsed): - for item in parsed: - result.add(item) - -proc readArrayLines(r: Redis): RedisList = - r.readSocket(nil) - return r.parseArrayLines(line) - -proc parseBulkString(r: Redis, allowMBNil = false, line:string = ""): RedisString = - if r.pipeline.enabled: return "" - - return r.parseSingleString(line, allowMBNil) - -proc readBulkString(r: Redis, allowMBNil = false): RedisString = - r.readSocket("") - return r.parseBulkString(allowMBNil, line) - -proc readArray(r: Redis): RedisList = - r.readSocket(@[]) - return r.parseArrayLines(line) - -proc readNext(r: Redis): RedisList = - r.readSocket(@[]) - - var res = case line[0] - of '+', '-': @[r.parseStatus(line)] - of ':': @[$(r.parseInteger(line))] - of '$': @[r.parseBulkString(true,line)] - of '*': r.parseArrayLines(line) - else: - raise newException(ReplyError, "readNext failed on line: " & line) - nil - r.pipeline.expected -= 1 - return res - -proc flushPipeline*(r: Redis, wasMulti = false): RedisList = - ## Send buffered commands, clear buffer, return results - if r.pipeline.buffer.len > 0: - r.socket.send(r.pipeline.buffer) - r.pipeline.buffer = "" - - r.pipeline.enabled = false - result = @[] - - var tot = r.pipeline.expected - - for i in 0..tot-1: - var ret = r.readNext() - for item in ret: - if not (item.contains("OK") or item.contains("QUEUED")): - result.add(item) - - r.pipeline.expected = 0 - -proc startPipelining*(r: Redis) = - ## Enable command pipelining (reduces network roundtrips). - ## Note that when enabled, you must call flushPipeline to actually send commands, except - ## for multi/exec() which enable and flush the pipeline automatically. - ## Commands return immediately with dummy values; actual results returned from - ## flushPipeline() or exec() - r.pipeline.expected = 0 - r.pipeline.enabled = true - -proc sendCommand(r: Redis, cmd: string, args: varargs[string]) = - var request = "*" & $(1 + args.len()) & "\c\L" - request.add("$" & $cmd.len() & "\c\L") - request.add(cmd & "\c\L") - for i in items(args): - request.add("$" & $i.len() & "\c\L") - request.add(i & "\c\L") - - if r.pipeline.enabled: - r.pipeline.buffer.add(request) - r.pipeline.expected += 1 - else: - r.socket.send(request) - -proc sendCommand(r: Redis, cmd: string, arg1: string, - args: varargs[string]) = - var request = "*" & $(2 + args.len()) & "\c\L" - request.add("$" & $cmd.len() & "\c\L") - request.add(cmd & "\c\L") - request.add("$" & $arg1.len() & "\c\L") - request.add(arg1 & "\c\L") - for i in items(args): - request.add("$" & $i.len() & "\c\L") - request.add(i & "\c\L") - - if r.pipeline.enabled: - r.pipeline.expected += 1 - r.pipeline.buffer.add(request) - else: - r.socket.send(request) - -# Keys - -proc del*(r: Redis, keys: varargs[string]): RedisInteger = - ## Delete a key or multiple keys - r.sendCommand("DEL", keys) - return r.readInteger() - -proc exists*(r: Redis, key: string): bool = - ## Determine if a key exists - r.sendCommand("EXISTS", key) - return r.readInteger() == 1 - -proc expire*(r: Redis, key: string, seconds: int): bool = - ## Set a key's time to live in seconds. Returns `false` if the key could - ## not be found or the timeout could not be set. - r.sendCommand("EXPIRE", key, $seconds) - return r.readInteger() == 1 - -proc expireAt*(r: Redis, key: string, timestamp: int): bool = - ## Set the expiration for a key as a UNIX timestamp. Returns `false` - ## if the key could not be found or the timeout could not be set. - r.sendCommand("EXPIREAT", key, $timestamp) - return r.readInteger() == 1 - -proc keys*(r: Redis, pattern: string): RedisList = - ## Find all keys matching the given pattern - r.sendCommand("KEYS", pattern) - return r.readArray() - -proc scan*(r: Redis, cursor: var BiggestInt): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using default Redis values for MATCH and COUNT parameters - r.sendCommand("SCAN", $cursor) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc scan*(r: Redis, cursor: var BiggestInt, pattern: string): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using cursor as a client query identifier. Using default Redis value for COUNT argument - r.sendCommand("SCAN", $cursor, ["MATCH", pattern]) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc scan*(r: Redis, cursor: var BiggestInt, pattern: string, count: int): RedisList = - ## Find all keys matching the given pattern and yield it to client in portions - ## using cursor as a client query identifier. - r.sendCommand("SCAN", $cursor, ["MATCH", pattern, "COUNT", $count]) - let reply = r.readArray() - cursor = strutils.parseBiggestInt(reply[0]) - return reply[1..high(reply)] - -proc move*(r: Redis, key: string, db: int): bool = - ## Move a key to another database. Returns `true` on a successful move. - r.sendCommand("MOVE", key, $db) - return r.readInteger() == 1 - -proc persist*(r: Redis, key: string): bool = - ## Remove the expiration from a key. - ## Returns `true` when the timeout was removed. - r.sendCommand("PERSIST", key) - return r.readInteger() == 1 - -proc randomKey*(r: Redis): RedisString = - ## Return a random key from the keyspace - r.sendCommand("RANDOMKEY") - return r.readBulkString() - -proc rename*(r: Redis, key, newkey: string): RedisStatus = - ## Rename a key. - ## - ## **WARNING:** Overwrites `newkey` if it exists! - r.sendCommand("RENAME", key, newkey) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc renameNX*(r: Redis, key, newkey: string): bool = - ## Same as ``rename`` but doesn't continue if `newkey` exists. - ## Returns `true` if key was renamed. - r.sendCommand("RENAMENX", key, newkey) - return r.readInteger() == 1 - -proc ttl*(r: Redis, key: string): RedisInteger = - ## Get the time to live for a key - r.sendCommand("TTL", key) - return r.readInteger() - -proc keyType*(r: Redis, key: string): RedisStatus = - ## Determine the type stored at key - r.sendCommand("TYPE", key) - return r.readStatus() - - -# Strings - -proc append*(r: Redis, key, value: string): RedisInteger = - ## Append a value to a key - r.sendCommand("APPEND", key, value) - return r.readInteger() - -proc decr*(r: Redis, key: string): RedisInteger = - ## Decrement the integer value of a key by one - r.sendCommand("DECR", key) - return r.readInteger() - -proc decrBy*(r: Redis, key: string, decrement: int): RedisInteger = - ## Decrement the integer value of a key by the given number - r.sendCommand("DECRBY", key, $decrement) - return r.readInteger() - -proc get*(r: Redis, key: string): RedisString = - ## Get the value of a key. Returns `redisNil` when `key` doesn't exist. - r.sendCommand("GET", key) - return r.readBulkString() - -proc getBit*(r: Redis, key: string, offset: int): RedisInteger = - ## Returns the bit value at offset in the string value stored at key - r.sendCommand("GETBIT", key, $offset) - return r.readInteger() - -proc getRange*(r: Redis, key: string, start, stop: int): RedisString = - ## Get a substring of the string stored at a key - r.sendCommand("GETRANGE", key, $start, $stop) - return r.readBulkString() - -proc getSet*(r: Redis, key: string, value: string): RedisString = - ## Set the string value of a key and return its old value. Returns `redisNil` - ## when key doesn't exist. - r.sendCommand("GETSET", key, value) - return r.readBulkString() - -proc incr*(r: Redis, key: string): RedisInteger = - ## Increment the integer value of a key by one. - r.sendCommand("INCR", key) - return r.readInteger() - -proc incrBy*(r: Redis, key: string, increment: int): RedisInteger = - ## Increment the integer value of a key by the given number - r.sendCommand("INCRBY", key, $increment) - return r.readInteger() - -proc setk*(r: Redis, key, value: string) = - ## Set the string value of a key. - ## - ## NOTE: This function had to be renamed due to a clash with the `set` type. - r.sendCommand("SET", key, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc setNX*(r: Redis, key, value: string): bool = - ## Set the value of a key, only if the key does not exist. Returns `true` - ## if the key was set. - r.sendCommand("SETNX", key, value) - return r.readInteger() == 1 - -proc setBit*(r: Redis, key: string, offset: int, - value: string): RedisInteger = - ## Sets or clears the bit at offset in the string value stored at key - r.sendCommand("SETBIT", key, $offset, value) - return r.readInteger() - -proc setEx*(r: Redis, key: string, seconds: int, value: string): RedisStatus = - ## Set the value and expiration of a key - r.sendCommand("SETEX", key, $seconds, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc setRange*(r: Redis, key: string, offset: int, - value: string): RedisInteger = - ## Overwrite part of a string at key starting at the specified offset - r.sendCommand("SETRANGE", key, $offset, value) - return r.readInteger() - -proc strlen*(r: Redis, key: string): RedisInteger = - ## Get the length of the value stored in a key. Returns 0 when key doesn't - ## exist. - r.sendCommand("STRLEN", key) - return r.readInteger() - -# Hashes -proc hDel*(r: Redis, key, field: string): bool = - ## Delete a hash field at `key`. Returns `true` if the field was removed. - r.sendCommand("HDEL", key, field) - return r.readInteger() == 1 - -proc hExists*(r: Redis, key, field: string): bool = - ## Determine if a hash field exists. - r.sendCommand("HEXISTS", key, field) - return r.readInteger() == 1 - -proc hGet*(r: Redis, key, field: string): RedisString = - ## Get the value of a hash field - r.sendCommand("HGET", key, field) - return r.readBulkString() - -proc hGetAll*(r: Redis, key: string): RedisList = - ## Get all the fields and values in a hash - r.sendCommand("HGETALL", key) - return r.readArray() - -proc hIncrBy*(r: Redis, key, field: string, incr: int): RedisInteger = - ## Increment the integer value of a hash field by the given number - r.sendCommand("HINCRBY", key, field, $incr) - return r.readInteger() - -proc hKeys*(r: Redis, key: string): RedisList = - ## Get all the fields in a hash - r.sendCommand("HKEYS", key) - return r.readArray() - -proc hLen*(r: Redis, key: string): RedisInteger = - ## Get the number of fields in a hash - r.sendCommand("HLEN", key) - return r.readInteger() - -proc hMGet*(r: Redis, key: string, fields: varargs[string]): RedisList = - ## Get the values of all the given hash fields - r.sendCommand("HMGET", key, fields) - return r.readArray() - -proc hMSet*(r: Redis, key: string, - fieldValues: openArray[tuple[field, value: string]]) = - ## Set multiple hash fields to multiple values - var args = @[key] - for field, value in items(fieldValues): - args.add(field) - args.add(value) - r.sendCommand("HMSET", args) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc hSet*(r: Redis, key, field, value: string): RedisInteger = - ## Set the string value of a hash field - r.sendCommand("HSET", key, field, value) - return r.readInteger() - -proc hSetNX*(r: Redis, key, field, value: string): RedisInteger = - ## Set the value of a hash field, only if the field does **not** exist - r.sendCommand("HSETNX", key, field, value) - return r.readInteger() - -proc hVals*(r: Redis, key: string): RedisList = - ## Get all the values in a hash - r.sendCommand("HVALS", key) - return r.readArray() - -# Lists - -proc bLPop*(r: Redis, keys: varargs[string], timeout: int): RedisList = - ## Remove and get the *first* element in a list, or block until - ## one is available - var args: seq[string] = @[] - for i in items(keys): args.add(i) - args.add($timeout) - r.sendCommand("BLPOP", args) - return r.readArray() - -proc bRPop*(r: Redis, keys: varargs[string], timeout: int): RedisList = - ## Remove and get the *last* element in a list, or block until one - ## is available. - var args: seq[string] = @[] - for i in items(keys): args.add(i) - args.add($timeout) - r.sendCommand("BRPOP", args) - return r.readArray() - -proc bRPopLPush*(r: Redis, source, destination: string, - timeout: int): RedisString = - ## Pop a value from a list, push it to another list and return it; or - ## block until one is available. - ## - ## http://redis.io/commands/brpoplpush - r.sendCommand("BRPOPLPUSH", source, destination, $timeout) - return r.readBulkString(true) # Multi-Bulk nil allowed. - -proc lIndex*(r: Redis, key: string, index: int): RedisString = - ## Get an element from a list by its index - r.sendCommand("LINDEX", key, $index) - return r.readBulkString() - -proc lInsert*(r: Redis, key: string, before: bool, pivot, value: string): - RedisInteger = - ## Insert an element before or after another element in a list - var pos = if before: "BEFORE" else: "AFTER" - r.sendCommand("LINSERT", key, pos, pivot, value) - return r.readInteger() - -proc lLen*(r: Redis, key: string): RedisInteger = - ## Get the length of a list - r.sendCommand("LLEN", key) - return r.readInteger() - -proc lPop*(r: Redis, key: string): RedisString = - ## Remove and get the first element in a list - r.sendCommand("LPOP", key) - return r.readBulkString() - -proc lPush*(r: Redis, key, value: string, create: bool = true): RedisInteger = - ## Prepend a value to a list. Returns the length of the list after the push. - ## The ``create`` param specifies whether a list should be created if it - ## doesn't exist at ``key``. More specifically if ``create`` is true, `LPUSH` - ## will be used, otherwise `LPUSHX`. - if create: - r.sendCommand("LPUSH", key, value) - else: - r.sendCommand("LPUSHX", key, value) - return r.readInteger() - -proc lRange*(r: Redis, key: string, start, stop: int): RedisList = - ## Get a range of elements from a list. Returns `nil` when `key` - ## doesn't exist. - r.sendCommand("LRANGE", key, $start, $stop) - return r.readArray() - -proc lRem*(r: Redis, key: string, value: string, count: int = 0): RedisInteger = - ## Remove elements from a list. Returns the number of elements that have been - ## removed. - r.sendCommand("LREM", key, $count, value) - return r.readInteger() - -proc lSet*(r: Redis, key: string, index: int, value: string) = - ## Set the value of an element in a list by its index - r.sendCommand("LSET", key, $index, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc lTrim*(r: Redis, key: string, start, stop: int) = - ## Trim a list to the specified range - r.sendCommand("LTRIM", key, $start, $stop) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc rPop*(r: Redis, key: string): RedisString = - ## Remove and get the last element in a list - r.sendCommand("RPOP", key) - return r.readBulkString() - -proc rPopLPush*(r: Redis, source, destination: string): RedisString = - ## Remove the last element in a list, append it to another list and return it - r.sendCommand("RPOPLPUSH", source, destination) - return r.readBulkString() - -proc rPush*(r: Redis, key, value: string, create: bool = true): RedisInteger = - ## Append a value to a list. Returns the length of the list after the push. - ## The ``create`` param specifies whether a list should be created if it - ## doesn't exist at ``key``. More specifically if ``create`` is true, `RPUSH` - ## will be used, otherwise `RPUSHX`. - if create: - r.sendCommand("RPUSH", key, value) - else: - r.sendCommand("RPUSHX", key, value) - return r.readInteger() - -# Sets - -proc sadd*(r: Redis, key: string, member: string): RedisInteger = - ## Add a member to a set - r.sendCommand("SADD", key, member) - return r.readInteger() - -proc scard*(r: Redis, key: string): RedisInteger = - ## Get the number of members in a set - r.sendCommand("SCARD", key) - return r.readInteger() - -proc sdiff*(r: Redis, keys: varargs[string]): RedisList = - ## Subtract multiple sets - r.sendCommand("SDIFF", keys) - return r.readArray() - -proc sdiffstore*(r: Redis, destination: string, - keys: varargs[string]): RedisInteger = - ## Subtract multiple sets and store the resulting set in a key - r.sendCommand("SDIFFSTORE", destination, keys) - return r.readInteger() - -proc sinter*(r: Redis, keys: varargs[string]): RedisList = - ## Intersect multiple sets - r.sendCommand("SINTER", keys) - return r.readArray() - -proc sinterstore*(r: Redis, destination: string, - keys: varargs[string]): RedisInteger = - ## Intersect multiple sets and store the resulting set in a key - r.sendCommand("SINTERSTORE", destination, keys) - return r.readInteger() - -proc sismember*(r: Redis, key: string, member: string): RedisInteger = - ## Determine if a given value is a member of a set - r.sendCommand("SISMEMBER", key, member) - return r.readInteger() - -proc smembers*(r: Redis, key: string): RedisList = - ## Get all the members in a set - r.sendCommand("SMEMBERS", key) - return r.readArray() - -proc smove*(r: Redis, source: string, destination: string, - member: string): RedisInteger = - ## Move a member from one set to another - r.sendCommand("SMOVE", source, destination, member) - return r.readInteger() - -proc spop*(r: Redis, key: string): RedisString = - ## Remove and return a random member from a set - r.sendCommand("SPOP", key) - return r.readBulkString() - -proc srandmember*(r: Redis, key: string): RedisString = - ## Get a random member from a set - r.sendCommand("SRANDMEMBER", key) - return r.readBulkString() - -proc srem*(r: Redis, key: string, member: string): RedisInteger = - ## Remove a member from a set - r.sendCommand("SREM", key, member) - return r.readInteger() - -proc sunion*(r: Redis, keys: varargs[string]): RedisList = - ## Add multiple sets - r.sendCommand("SUNION", keys) - return r.readArray() - -proc sunionstore*(r: Redis, destination: string, - key: varargs[string]): RedisInteger = - ## Add multiple sets and store the resulting set in a key - r.sendCommand("SUNIONSTORE", destination, key) - return r.readInteger() - -# Sorted sets - -proc zadd*(r: Redis, key: string, score: int, member: string): RedisInteger = - ## Add a member to a sorted set, or update its score if it already exists - r.sendCommand("ZADD", key, $score, member) - return r.readInteger() - -proc zcard*(r: Redis, key: string): RedisInteger = - ## Get the number of members in a sorted set - r.sendCommand("ZCARD", key) - return r.readInteger() - -proc zcount*(r: Redis, key: string, min: string, max: string): RedisInteger = - ## Count the members in a sorted set with scores within the given values - r.sendCommand("ZCOUNT", key, min, max) - return r.readInteger() - -proc zincrby*(r: Redis, key: string, increment: string, - member: string): RedisString = - ## Increment the score of a member in a sorted set - r.sendCommand("ZINCRBY", key, increment, member) - return r.readBulkString() - -proc zinterstore*(r: Redis, destination: string, numkeys: string, - keys: openArray[string], weights: openArray[string] = [], - aggregate: string = ""): RedisInteger = - ## Intersect multiple sorted sets and store the resulting sorted set in - ## a new key - var args = @[destination, numkeys] - for i in items(keys): args.add(i) - - if weights.len != 0: - args.add("WITHSCORE") - for i in items(weights): args.add(i) - if aggregate.len != 0: - args.add("AGGREGATE") - args.add(aggregate) - - r.sendCommand("ZINTERSTORE", args) - - return r.readInteger() - -proc zrange*(r: Redis, key: string, start: string, stop: string, - withScores: bool): RedisList = - ## Return a range of members in a sorted set, by index - if not withScores: - r.sendCommand("ZRANGE", key, start, stop) - else: - r.sendCommand("ZRANGE", "WITHSCORES", key, start, stop) - return r.readArray() - -proc zrangebyscore*(r: Redis, key: string, min: string, max: string, - withScore: bool = false, limit: bool = false, - limitOffset: int = 0, limitCount: int = 0): RedisList = - ## Return a range of members in a sorted set, by score - var args = @[key, min, max] - - if withScore: args.add("WITHSCORE") - if limit: - args.add("LIMIT") - args.add($limitOffset) - args.add($limitCount) - - r.sendCommand("ZRANGEBYSCORE", args) - return r.readArray() - -proc zrank*(r: Redis, key: string, member: string): RedisString = - ## Determine the index of a member in a sorted set - r.sendCommand("ZRANK", key, member) - return r.readBulkString() - -proc zrem*(r: Redis, key: string, member: string): RedisInteger = - ## Remove a member from a sorted set - r.sendCommand("ZREM", key, member) - return r.readInteger() - -proc zremrangebyrank*(r: Redis, key: string, start: string, - stop: string): RedisInteger = - ## Remove all members in a sorted set within the given indexes - r.sendCommand("ZREMRANGEBYRANK", key, start, stop) - return r.readInteger() - -proc zremrangebyscore*(r: Redis, key: string, min: string, - max: string): RedisInteger = - ## Remove all members in a sorted set within the given scores - r.sendCommand("ZREMRANGEBYSCORE", key, min, max) - return r.readInteger() - -proc zrevrange*(r: Redis, key: string, start: string, stop: string, - withScore: bool): RedisList = - ## Return a range of members in a sorted set, by index, - ## with scores ordered from high to low - if withScore: - r.sendCommand("ZREVRANGE", "WITHSCORE", key, start, stop) - else: r.sendCommand("ZREVRANGE", key, start, stop) - return r.readArray() - -proc zrevrangebyscore*(r: Redis, key: string, min: string, max: string, - withScore: bool = false, limit: bool = false, - limitOffset: int = 0, limitCount: int = 0): RedisList = - ## Return a range of members in a sorted set, by score, with - ## scores ordered from high to low - var args = @[key, min, max] - - if withScore: args.add("WITHSCORE") - if limit: - args.add("LIMIT") - args.add($limitOffset) - args.add($limitCount) - - r.sendCommand("ZREVRANGEBYSCORE", args) - return r.readArray() - -proc zrevrank*(r: Redis, key: string, member: string): RedisString = - ## Determine the index of a member in a sorted set, with - ## scores ordered from high to low - r.sendCommand("ZREVRANK", key, member) - return r.readBulkString() - -proc zscore*(r: Redis, key: string, member: string): RedisString = - ## Get the score associated with the given member in a sorted set - r.sendCommand("ZSCORE", key, member) - return r.readBulkString() - -proc zunionstore*(r: Redis, destination: string, numkeys: string, - keys: openArray[string], weights: openArray[string] = [], - aggregate: string = ""): RedisInteger = - ## Add multiple sorted sets and store the resulting sorted set in a new key - var args = @[destination, numkeys] - for i in items(keys): args.add(i) - - if weights.len != 0: - args.add("WEIGHTS") - for i in items(weights): args.add(i) - if aggregate.len != 0: - args.add("AGGREGATE") - args.add(aggregate) - - r.sendCommand("ZUNIONSTORE", args) - - return r.readInteger() - -# HyperLogLog - -proc pfadd*(r: Redis, key: string, elements: varargs[string]): RedisInteger = - ## Add variable number of elements into special 'HyperLogLog' set type - r.sendCommand("PFADD", key, elements) - return r.readInteger() - -proc pfcount*(r: Redis, key: string): RedisInteger = - ## Count approximate number of elements in 'HyperLogLog' - r.sendCommand("PFCOUNT", key) - return r.readInteger() - -proc pfmerge*(r: Redis, destination: string, sources: varargs[string]) = - ## Merge several source HyperLogLog's into one specified by destKey - r.sendCommand("PFMERGE", destination, sources) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -# Pub/Sub - -# TODO: pub/sub -- I don't think this will work synchronously. -discard """ -proc psubscribe*(r: Redis, pattern: openarray[string]): ???? = - ## Listen for messages published to channels matching the given patterns - r.socket.send("PSUBSCRIBE $#\c\L" % pattern) - return ??? - -proc publish*(r: Redis, channel: string, message: string): RedisInteger = - ## Post a message to a channel - r.socket.send("PUBLISH $# $#\c\L" % [channel, message]) - return r.readInteger() - -proc punsubscribe*(r: Redis, [pattern: openarray[string], : string): ???? = - ## Stop listening for messages posted to channels matching the given patterns - r.socket.send("PUNSUBSCRIBE $# $#\c\L" % [[pattern.join(), ]) - return ??? - -proc subscribe*(r: Redis, channel: openarray[string]): ???? = - ## Listen for messages published to the given channels - r.socket.send("SUBSCRIBE $#\c\L" % channel.join) - return ??? - -proc unsubscribe*(r: Redis, [channel: openarray[string], : string): ???? = - ## Stop listening for messages posted to the given channels - r.socket.send("UNSUBSCRIBE $# $#\c\L" % [[channel.join(), ]) - return ??? - -""" - -# Transactions - -proc discardMulti*(r: Redis) = - ## Discard all commands issued after MULTI - r.sendCommand("DISCARD") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc exec*(r: Redis): RedisList = - ## Execute all commands issued after MULTI - r.sendCommand("EXEC") - r.pipeline.enabled = false - # Will reply with +OK for MULTI/EXEC and +QUEUED for every command - # between, then with the results - return r.flushPipeline(true) - - -proc multi*(r: Redis) = - ## Mark the start of a transaction block - r.startPipelining() - r.sendCommand("MULTI") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc unwatch*(r: Redis) = - ## Forget about all watched keys - r.sendCommand("UNWATCH") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc watch*(r: Redis, key: varargs[string]) = - ## Watch the given keys to determine execution of the MULTI/EXEC block - r.sendCommand("WATCH", key) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -# Connection - -proc auth*(r: Redis, password: string) = - ## Authenticate to the server - r.sendCommand("AUTH", password) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc echoServ*(r: Redis, message: string): RedisString = - ## Echo the given string - r.sendCommand("ECHO", message) - return r.readBulkString() - -proc ping*(r: Redis): RedisStatus = - ## Ping the server - r.sendCommand("PING") - return r.readStatus() - -proc quit*(r: Redis) = - ## Close the connection - r.sendCommand("QUIT") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - r.socket.close() - -proc select*(r: Redis, index: int): RedisStatus = - ## Change the selected database for the current connection - r.sendCommand("SELECT", $index) - return r.readStatus() - -# Server - -proc bgrewriteaof*(r: Redis) = - ## Asynchronously rewrite the append-only file - r.sendCommand("BGREWRITEAOF") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc bgsave*(r: Redis) = - ## Asynchronously save the dataset to disk - r.sendCommand("BGSAVE") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc configGet*(r: Redis, parameter: string): RedisList = - ## Get the value of a configuration parameter - r.sendCommand("CONFIG", "GET", parameter) - return r.readArray() - -proc configSet*(r: Redis, parameter: string, value: string) = - ## Set a configuration parameter to the given value - r.sendCommand("CONFIG", "SET", parameter, value) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc configResetStat*(r: Redis) = - ## Reset the stats returned by INFO - r.sendCommand("CONFIG", "RESETSTAT") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc dbsize*(r: Redis): RedisInteger = - ## Return the number of keys in the selected database - r.sendCommand("DBSIZE") - return r.readInteger() - -proc debugObject*(r: Redis, key: string): RedisStatus = - ## Get debugging information about a key - r.sendCommand("DEBUG", "OBJECT", key) - return r.readStatus() - -proc debugSegfault*(r: Redis) = - ## Make the server crash - r.sendCommand("DEBUG", "SEGFAULT") - -proc flushall*(r: Redis): RedisStatus = - ## Remove all keys from all databases - r.sendCommand("FLUSHALL") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc flushdb*(r: Redis): RedisStatus = - ## Remove all keys from the current database - r.sendCommand("FLUSHDB") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc info*(r: Redis): RedisString = - ## Get information and statistics about the server - r.sendCommand("INFO") - return r.readBulkString() - -proc lastsave*(r: Redis): RedisInteger = - ## Get the UNIX time stamp of the last successful save to disk - r.sendCommand("LASTSAVE") - return r.readInteger() - -discard """ -proc monitor*(r: Redis) = - ## Listen for all requests received by the server in real time - r.socket.send("MONITOR\c\L") - raiseNoOK(r.readStatus(), r.pipeline.enabled) -""" - -proc save*(r: Redis) = - ## Synchronously save the dataset to disk - r.sendCommand("SAVE") - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -proc shutdown*(r: Redis) = - ## Synchronously save the dataset to disk and then shut down the server - r.sendCommand("SHUTDOWN") - var s = "".TaintedString - r.socket.readLine(s) - if s.string.len != 0: raise newException(RedisError, s.string) - -proc slaveof*(r: Redis, host: string, port: string) = - ## Make the server a slave of another instance, or promote it as master - r.sendCommand("SLAVEOF", host, port) - raiseNoOK(r.readStatus(), r.pipeline.enabled) - -iterator hPairs*(r: Redis, key: string): tuple[key, value: string] = - ## Iterator for keys and values in a hash. - var - contents = r.hGetAll(key) - k = "" - for i in items(contents): - if k == "": - k = i - else: - yield (k, i) - k = "" - -proc someTests(r: Redis, how: SendMode):seq[string] = - var list:seq[string] = @[] - - if how == pipelined: - r.startPipelining() - elif how == multiple: - r.multi() - - r.setk("nim:test", "Testing something.") - r.setk("nim:utf8", "こんにちは") - r.setk("nim:esc", "\\ths ągt\\") - r.setk("nim:int", "1") - list.add(r.get("nim:esc")) - list.add($(r.incr("nim:int"))) - list.add(r.get("nim:int")) - list.add(r.get("nim:utf8")) - list.add($(r.hSet("test1", "name", "A Test"))) - var res = r.hGetAll("test1") - for r in res: - list.add(r) - list.add(r.get("invalid_key")) - list.add($(r.lPush("mylist","itema"))) - list.add($(r.lPush("mylist","itemb"))) - r.lTrim("mylist",0,1) - var p = r.lRange("mylist", 0, -1) - - for i in items(p): - if not isNil(i): - list.add(i) - - list.add(r.debugObject("mylist")) - - r.configSet("timeout", "299") - var g = r.configGet("timeout") - for i in items(g): - list.add(i) - - list.add(r.echoServ("BLAH")) - - case how - of normal: - return list - of pipelined: - return r.flushPipeline() - of multiple: - return r.exec() - -proc assertListsIdentical(listA, listB: seq[string]) = - assert(listA.len == listB.len) - var i = 0 - for item in listA: - assert(item == listB[i]) - i = i + 1 - -when not defined(testing) and isMainModule: - when false: - var r = open() - - # Test with no pipelining - var listNormal = r.someTests(normal) - - # Test with pipelining enabled - var listPipelined = r.someTests(pipelined) - assertListsIdentical(listNormal, listPipelined) - - # Test with multi/exec() (automatic pipelining) - var listMulti = r.someTests(multiple) - assertListsIdentical(listNormal, listMulti) diff --git a/web/news.txt b/web/news.txt index dfeec45e6..6a67c5fb8 100644 --- a/web/news.txt +++ b/web/news.txt @@ -21,6 +21,9 @@ News the language. - Top level routines cannot have the calling convention ``closure`` anymore. + - The ``redis`` module has been moved out of the standard library. It can + now be installed via Nimble and is located here: + https://github.com/nim-lang/redis Syntax changes From 26f2bb0eabb030df8276f7ec1566b56f9b410db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 16 Jan 2016 17:06:03 +0100 Subject: [PATCH 108/125] add support for uint32 and uint64 in repr --- lib/system/repr.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/system/repr.nim b/lib/system/repr.nim index 1f81a0813..986994203 100644 --- a/lib/system/repr.nim +++ b/lib/system/repr.nim @@ -259,8 +259,10 @@ when not defined(useNimRtl): of tyInt16: add result, $int(cast[ptr int16](p)[]) of tyInt32: add result, $int(cast[ptr int32](p)[]) of tyInt64: add result, $(cast[ptr int64](p)[]) - of tyUInt8: add result, $ze(cast[ptr int8](p)[]) - of tyUInt16: add result, $ze(cast[ptr int16](p)[]) + of tyUInt8: add result, $(cast[ptr uint8](p)[]) + of tyUInt16: add result, $(cast[ptr uint16](p)[]) + of tyUInt32: add result, $(cast[ptr uint32](p)[]) + of tyUInt64: add result, $(cast[ptr uint64](p)[]) of tyFloat: add result, $(cast[ptr float](p)[]) of tyFloat32: add result, $(cast[ptr float32](p)[]) From 2e71bd50b26e6165121a3076f7dc70175e9f1f55 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 16 Jan 2016 21:40:54 +0100 Subject: [PATCH 109/125] implements multi-line-comments; pounds are stripped from the doc comments --- compiler/docgen.nim | 4 +- compiler/lexer.nim | 87 +++++++++++++++++++++++++----- compiler/parser.nim | 7 +-- compiler/renderer.nim | 19 ++----- doc/manual/lexing.txt | 38 +++++++++++++ lib/packages/docutils/highlite.nim | 17 +++++- todo.txt | 2 +- 7 files changed, 137 insertions(+), 37 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 952366f93..8555ec4f0 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string], proc genComment(d: PDoc, n: PNode): string = result = "" var dummyHasToc: bool - if n.comment != nil and startsWith(n.comment, "##"): + if n.comment != nil: renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), toLinenumber(n.info), toColumn(n.info), - dummyHasToc, d.options + {roSkipPounds}), result) + dummyHasToc, d.options), result) proc genRecComment(d: PDoc, n: PNode): Rope = if n == nil: return nil diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 9a69ede3e..70a555e15 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -769,24 +769,78 @@ proc getOperator(L: var TLexer, tok: var TToken) = if buf[pos] in {CR, LF, nimlexbase.EndOfFile}: tok.strongSpaceB = -1 +proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; + isDoc: bool) = + var pos = start + var buf = L.buf + while buf[pos] == '[': + inc pos + let brackets = pos - start + var toStrip = 0 + # detect the amount of indentation: + if isDoc: + toStrip = getColNumber(L, pos) + while buf[pos] == ' ': inc pos + if buf[pos] in {CR, LF}: + pos = handleCRLF(L, pos) + buf = L.buf + toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip + while true: + case buf[pos] + of ']': + var p = pos + while buf[p] == ']': + inc p + if p-pos == brackets: + pos = p + break + else: + if isDoc: tok.literal.add ']' + inc pos + of '\t': + lexMessagePos(L, errTabulatorsAreNotAllowed, pos) + inc(pos) + if isDoc: tok.literal.add '\t' + of CR, LF: + pos = handleCRLF(L, pos) + buf = L.buf + # strip leading whitespace: + if isDoc: + tok.literal.add "\n" + inc tok.iNumber + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c + of nimlexbase.EndOfFile: + lexMessagePos(L, errGenerated, pos, "end of multiline comment expected") + break + else: + if isDoc: tok.literal.add buf[pos] + inc(pos) + L.bufpos = pos + proc scanComment(L: var TLexer, tok: var TToken) = var pos = L.bufpos var buf = L.buf - when not defined(nimfix): - assert buf[pos+1] == '#' - if buf[pos+2] == '[': - if buf[pos+3] == ']': - # ##[] is the (rather complex) "cursor token" for idetools - tok.tokType = tkComment - tok.literal = "[]" - inc(L.bufpos, 4) - return - else: - lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['") - tok.tokType = tkComment # iNumber contains the number of '\n' in the token tok.iNumber = 0 + when not defined(nimfix): + assert buf[pos+1] == '#' + if buf[pos+2] == '[': + skipMultiLineComment(L, tok, pos+2, true) + return + inc(pos, 2) + + var toStrip = 0 + while buf[pos] == ' ': + inc pos + inc toStrip + when defined(nimfix): var col = getColNumber(L, pos) while true: @@ -820,6 +874,12 @@ proc scanComment(L: var TLexer, tok: var TToken) = if doContinue(): tok.literal.add "\n" when defined(nimfix): col = indent + else: + inc(pos, 2) + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c inc tok.iNumber else: if buf[pos] > ' ': @@ -863,7 +923,8 @@ proc skip(L: var TLexer, tok: var TToken) = # do not skip documentation comment: if buf[pos+1] == '#': break if buf[pos+1] == '[': - lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['") + skipMultiLineComment(L, tok, pos+1, false) + return while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop diff --git a/compiler/parser.nim b/compiler/parser.nim index eb0ef071d..c4681a5cd 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -112,12 +112,7 @@ proc rawSkipComment(p: var TParser, node: PNode) = if p.tok.tokType == tkComment: if node != nil: if node.comment == nil: node.comment = "" - if p.tok.literal == "[]": - node.flags.incl nfIsCursor - #echo "parser: " - #debug node - else: - add(node.comment, p.tok.literal) + add(node.comment, p.tok.literal) else: parMessage(p, errInternal, "skipComment") getTok(p) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index c4d1222c4..8e4aa1831 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -167,33 +167,24 @@ proc makeNimString(s: string): string = proc putComment(g: var TSrcGen, s: string) = if s.isNil: return var i = 0 - var comIndent = 1 var isCode = (len(s) >= 2) and (s[1] != ' ') var ind = g.lineLen - var com = "" + var com = "## " while true: case s[i] of '\0': break of '\x0D': put(g, tkComment, com) - com = "" + com = "## " inc(i) if s[i] == '\x0A': inc(i) optNL(g, ind) of '\x0A': put(g, tkComment, com) - com = "" + com = "## " inc(i) optNL(g, ind) - of '#': - add(com, s[i]) - inc(i) - comIndent = 0 - while s[i] == ' ': - add(com, s[i]) - inc(i) - inc(comIndent) of ' ', '\x09': add(com, s[i]) inc(i) @@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) = if not isCode and (g.lineLen + (j - i) > MaxLineLen): put(g, tkComment, com) optNL(g, ind) - com = '#' & spaces(comIndent) + com = "## " while s[i] > ' ': add(com, s[i]) inc(i) @@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool = result = false if n.comment != nil: result = (renderNoComments notin g.flags) or - (renderDocComments in g.flags) and startsWith(n.comment, "##") + (renderDocComments in g.flags) proc gcom(g: var TSrcGen, n: PNode) = assert(n != nil) diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index 7f81ab422..a5c3dfd6d 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -69,6 +69,44 @@ Documentation comments are tokens; they are only allowed at certain places in the input file as they belong to the syntax tree! +Multiline comments +------------------ + +Starting with version 0.13.0 of the language Nim supports multiline comments. +They look like: + +.. code-block:: nim + #[Comment here. + Multiple lines + are not a problem.] + +.. code-block:: nim + #[[comment here]] + +.. code-block:: nim + #[[[comment here]]] + +.. code-block:: nim + #[[[[comment here]]]] + +The number of opening brackets must match the number of closing brackets +but they do not nest in the traditional sense: + +.. code-block:: nim + # Does not comment out 'p' properly: + #[ + proc p[T](x: T) = discard + ] + +Multiline documentation comments look like: + +.. code-block:: nim + proc foo = + ##[Long documentation comment + here. + ] + + Identifiers & Keywords ---------------------- diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 640b8cd5a..570465e14 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -173,7 +173,22 @@ proc nimNextToken(g: var GeneralTokenizer) = while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos) of '#': g.kind = gtComment - while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos) + inc(pos) + if g.buf[pos] == '#': inc(pos) + if g.buf[pos] == '[': + g.kind = gtLongComment + var brackets = 0 + while g.buf[pos] == '[': + inc(pos) + inc(brackets) + while g.buf[pos] != '\0': + if g.buf[pos] == ']': + var q = pos + while g.buf[pos] == ']': inc(pos) + if pos-q == brackets: break + inc(pos) + else: + while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': var id = "" while g.buf[pos] in SymChars + {'_'}: diff --git a/todo.txt b/todo.txt index ca8d60762..6c1c602f5 100644 --- a/todo.txt +++ b/todo.txt @@ -10,7 +10,6 @@ essential for 1.0 - make '--implicitStatic:on' the default; then we can also clean up the 'static[T]' mess in the compiler! -- add "all threads are blocked" detection to 'spawn' - Deprecate ``immediate`` for templates and macros - document NimMain and check whether it works for threading - ``not`` or ``~`` for the effects system @@ -19,6 +18,7 @@ essential for 1.0 Not critical for 1.0 ==================== +- add "all threads are blocked" detection to 'spawn' - figure out why C++ bootstrapping is so much slower - The bitwise 'not' operator cold be renamed to 'bnot' to prevent 'not 4 == 5' from compiling. -> requires 'mixin' annotation for procs! From ab79c36169e37d8d964e0e4089f0b877ae0149ca Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Sun, 17 Jan 2016 10:44:04 +0200 Subject: [PATCH 110/125] Use real JS bool literals instead of 1 and 0. Fixes #3722. --- compiler/jsgen.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 2f6496b9b..14cdf0174 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -1636,7 +1636,10 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) = of nkSym: genSym(p, n, r) of nkCharLit..nkInt64Lit: - r.res = rope(n.intVal) + if n.typ.kind == tyBool: + r.res = if n.intVal == 0: rope"false" else: rope"true" + else: + r.res = rope(n.intVal) r.kind = resExpr of nkNilLit: if isEmptyType(n.typ): From aad55ae772b652ee252b2e2c94cc61013ca05a99 Mon Sep 17 00:00:00 2001 From: xyz Date: Sun, 17 Jan 2016 13:56:41 -0500 Subject: [PATCH 111/125] Add support for ioctl. "ioctl (an abbreviation of input/output control) is a system call for device-specific input/output operations and other operations which cannot be expressed by regular system calls." --- lib/system.nim | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index bb8254364..1e090cfbd 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2694,6 +2694,11 @@ when not defined(JS): #and not defined(nimscript): proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. + proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", + header: "", varargs, tags: [WriteIOEffect].} + ## A system call for device-specific input/output operations and other + ## operations which cannot be expressed by regular system calls + proc readChar*(f: File): char {. importc: "fgetc", header: "", tags: [ReadIOEffect].} ## Reads a single character from the stream `f`. From a4aeb6fbecaabed2ae14c2405d9caa6699aec43e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 20:29:50 +0100 Subject: [PATCH 112/125] implements final version of multiline comments --- compiler/lexer.nim | 38 ++++++++++------- doc/manual/lexing.txt | 23 ++++------- lib/packages/docutils/highlite.nim | 41 ++++++++++++++----- .../dependencies/genpacket/genpacket_enet.nim | 4 +- 4 files changed, 63 insertions(+), 43 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 70a555e15..4c1dd366b 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -773,9 +773,6 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; isDoc: bool) = var pos = start var buf = L.buf - while buf[pos] == '[': - inc pos - let brackets = pos - start var toStrip = 0 # detect the amount of indentation: if isDoc: @@ -788,18 +785,31 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int; while buf[pos] == ' ': inc pos inc toStrip + var nesting = 0 while true: case buf[pos] + of '#': + if isDoc: + if buf[pos+1] == '#' and buf[pos+2] == '[': + inc nesting + tok.literal.add '#' + elif buf[pos+1] == '[': + inc nesting + inc pos of ']': - var p = pos - while buf[p] == ']': - inc p - if p-pos == brackets: - pos = p - break - else: - if isDoc: tok.literal.add ']' - inc pos + if isDoc: + if buf[pos+1] == '#' and buf[pos+2] == '#': + if nesting == 0: + inc(pos, 3) + break + dec nesting + tok.literal.add ']' + elif buf[pos+1] == '#': + if nesting == 0: + inc(pos, 2) + break + dec nesting + inc pos of '\t': lexMessagePos(L, errTabulatorsAreNotAllowed, pos) inc(pos) @@ -832,7 +842,7 @@ proc scanComment(L: var TLexer, tok: var TToken) = when not defined(nimfix): assert buf[pos+1] == '#' if buf[pos+2] == '[': - skipMultiLineComment(L, tok, pos+2, true) + skipMultiLineComment(L, tok, pos+3, true) return inc(pos, 2) @@ -923,7 +933,7 @@ proc skip(L: var TLexer, tok: var TToken) = # do not skip documentation comment: if buf[pos+1] == '#': break if buf[pos+1] == '[': - skipMultiLineComment(L, tok, pos+1, false) + skipMultiLineComment(L, tok, pos+2, false) return while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index a5c3dfd6d..7655964ee 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -78,33 +78,24 @@ They look like: .. code-block:: nim #[Comment here. Multiple lines - are not a problem.] + are not a problem.]# -.. code-block:: nim - #[[comment here]] - -.. code-block:: nim - #[[[comment here]]] - -.. code-block:: nim - #[[[[comment here]]]] - -The number of opening brackets must match the number of closing brackets -but they do not nest in the traditional sense: +Multiline comments support nesting: .. code-block:: nim # Does not comment out 'p' properly: - #[ + #[ #[ Multiline comment in already + commented out code. ]# proc p[T](x: T) = discard - ] + ]# -Multiline documentation comments look like: +Multiline documentation comments look like and support nesting too: .. code-block:: nim proc foo = ##[Long documentation comment here. - ] + ]## Identifiers & Keywords diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 570465e14..1bc0af1b6 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -174,19 +174,38 @@ proc nimNextToken(g: var GeneralTokenizer) = of '#': g.kind = gtComment inc(pos) - if g.buf[pos] == '#': inc(pos) + var isDoc = false + if g.buf[pos] == '#': + inc(pos) + isDoc = true if g.buf[pos] == '[': g.kind = gtLongComment - var brackets = 0 - while g.buf[pos] == '[': - inc(pos) - inc(brackets) - while g.buf[pos] != '\0': - if g.buf[pos] == ']': - var q = pos - while g.buf[pos] == ']': inc(pos) - if pos-q == brackets: break - inc(pos) + var nesting = 0 + while true: + case g.buf[pos] + of '\0': break + of '#': + if isDoc: + if g.buf[pos+1] == '#' and g.buf[pos+2] == '[': + inc nesting + elif g.buf[pos+1] == '[': + inc nesting + inc pos + of ']': + if isDoc: + if g.buf[pos+1] == '#' and g.buf[pos+2] == '#': + if nesting == 0: + inc(pos, 3) + break + dec nesting + elif g.buf[pos+1] == '#': + if nesting == 0: + inc(pos, 2) + break + dec nesting + inc pos + else: + inc pos else: while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos) of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF': diff --git a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim index e64b818fa..142b190ab 100644 --- a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim +++ b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim @@ -131,10 +131,10 @@ macro defPacket*(typeNameN: expr, typeFields: expr): stmt {.immediate.} = emptyNode())), emptyNode(), emptyNode(), - newNimNode(nnkStmtList).und(#[6] + newNimNode(nnkStmtList).und(# [6] newNimNode(nnkAsgn).und( ^"result", ## result = - newNimNode(nnkCall).und(#[6][0][1] + newNimNode(nnkCall).und(# [6][0][1] ^"format", ## format emptyNode())))) ## "[TypeName $1 $2]" formatStr = "["& $typeName.ident From f02701c0b05ef5d10fd0d7f040ea71fc67a8c3f7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 20:34:10 +0100 Subject: [PATCH 113/125] removed wrong documentation --- doc/manual/lexing.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index 7655964ee..5990dff07 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -83,7 +83,6 @@ They look like: Multiline comments support nesting: .. code-block:: nim - # Does not comment out 'p' properly: #[ #[ Multiline comment in already commented out code. ]# proc p[T](x: T) = discard From b4d6346bae928f37b249952a5cae262397259967 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 20:40:38 +0100 Subject: [PATCH 114/125] fixes #3720 --- lib/wrappers/linenoise/clinenoise.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/wrappers/linenoise/clinenoise.c b/lib/wrappers/linenoise/clinenoise.c index b4ae32472..dd3aa736c 100644 --- a/lib/wrappers/linenoise/clinenoise.c +++ b/lib/wrappers/linenoise/clinenoise.c @@ -417,10 +417,10 @@ void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { size_t len = strlen(str); char *copy, **cvec; - copy = malloc(len+1); + copy = (char*)malloc(len+1); if (copy == NULL) return; memcpy(copy,str,len+1); - cvec = realloc(lc->cvec,sizeof(char*)*(lc->len+1)); + cvec = (char**)realloc(lc->cvec,sizeof(char*)*(lc->len+1)); if (cvec == NULL) { free(copy); return; @@ -446,11 +446,11 @@ static void abInit(struct abuf *ab) { } static void abAppend(struct abuf *ab, const char *s, int len) { - char *new = realloc(ab->b,ab->len+len); + char *neww = (char*)realloc(ab->b,ab->len+len); - if (new == NULL) return; - memcpy(new+ab->len,s,len); - ab->b = new; + if (neww == NULL) return; + memcpy(neww+ab->len,s,len); + ab->b = neww; ab->len += len; } @@ -1016,7 +1016,7 @@ int linenoiseHistoryAdd(const char *line) { /* Initialization on first call. */ if (history == NULL) { - history = malloc(sizeof(char*)*history_max_len); + history = (char**)malloc(sizeof(char*)*history_max_len); if (history == NULL) return 0; memset(history,0,(sizeof(char*)*history_max_len)); } @@ -1043,14 +1043,14 @@ int linenoiseHistoryAdd(const char *line) { * just the latest 'len' elements if the new history length value is smaller * than the amount of items already inside the history. */ int linenoiseHistorySetMaxLen(int len) { - char **new; + char **neww; if (len < 1) return 0; if (history) { int tocopy = history_len; - new = malloc(sizeof(char*)*len); - if (new == NULL) return 0; + neww = (char**)malloc(sizeof(char*)*len); + if (neww == NULL) return 0; /* If we can't copy everything, free the elements we'll not use. */ if (len < tocopy) { @@ -1059,10 +1059,10 @@ int linenoiseHistorySetMaxLen(int len) { for (j = 0; j < tocopy-len; j++) free(history[j]); tocopy = len; } - memset(new,0,sizeof(char*)*len); - memcpy(new,history+(history_len-tocopy), sizeof(char*)*tocopy); + memset(neww,0,sizeof(char*)*len); + memcpy(neww,history+(history_len-tocopy), sizeof(char*)*tocopy); free(history); - history = new; + history = neww; } history_max_len = len; if (history_len > history_max_len) From 2befe48e4d177dfc047fb56c762d638fdc47cadb Mon Sep 17 00:00:00 2001 From: xyz Date: Sun, 17 Jan 2016 15:12:48 -0500 Subject: [PATCH 115/125] Moved ioctl to posix.nim --- lib/posix/posix.nim | 4 ++++ lib/system.nim | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 5f1dfcfcd..40b48f992 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -2165,6 +2165,10 @@ proc pwrite*(a1: cint, a2: pointer, a3: int, a4: Off): int {. importc, header: "".} proc read*(a1: cint, a2: pointer, a3: int): int {.importc, header: "".} proc readlink*(a1, a2: cstring, a3: int): int {.importc, header: "".} +proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", + header: "", varargs, tags: [WriteIOEffect].} + ## A system call for device-specific input/output operations and other + ## operations which cannot be expressed by regular system calls proc rmdir*(a1: cstring): cint {.importc, header: "".} proc setegid*(a1: Gid): cint {.importc, header: "".} diff --git a/lib/system.nim b/lib/system.nim index 1e090cfbd..bb8254364 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2694,11 +2694,6 @@ when not defined(JS): #and not defined(nimscript): proc endOfFile*(f: File): bool {.tags: [], benign.} ## Returns true iff `f` is at the end. - proc ioctl*(f: FileHandle, device: uint): int {.importc: "ioctl", - header: "", varargs, tags: [WriteIOEffect].} - ## A system call for device-specific input/output operations and other - ## operations which cannot be expressed by regular system calls - proc readChar*(f: File): char {. importc: "fgetc", header: "", tags: [ReadIOEffect].} ## Reads a single character from the stream `f`. From b79a8d60259a5837fa6bf34724b1a4e8e2e946bc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 21:28:36 +0100 Subject: [PATCH 116/125] nkEmpty is cleaner than nkCommentStmt --- compiler/transf.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index 50db2c7e8..3e074841e 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -792,7 +792,7 @@ proc transform(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkCommentStmt, n.info, 0) tryStmt.addSon(deferPart) # disable the original 'defer' statement: - n.kind = nkCommentStmt + n.kind = nkEmpty of nkContinueStmt: result = PTransNode(newNodeI(nkBreakStmt, n.info)) var labl = c.contSyms[c.contSyms.high] From 2c2ab6884c1055dedc9dbafbeeb492fecf251ea8 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 21:29:25 +0100 Subject: [PATCH 117/125] fixes strutils.unescape; refs #3634 --- lib/pure/parseutils.nim | 7 +++++-- lib/pure/strutils.nim | 10 +++++----- tests/parser/tmultiline_comments.nim | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 tests/parser/tmultiline_comments.nim diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index b3708838a..698bde42a 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -25,7 +25,7 @@ const proc toLower(c: char): char {.inline.} = result = if c in {'A'..'Z'}: chr(ord(c)-ord('A')+ord('a')) else: c -proc parseHex*(s: string, number: var int, start = 0): int {. +proc parseHex*(s: string, number: var int, start = 0; maxLen = 0): int {. rtl, extern: "npuParseHex", noSideEffect.} = ## Parses a hexadecimal number and stores its value in ``number``. ## @@ -45,11 +45,14 @@ proc parseHex*(s: string, number: var int, start = 0): int {. ## discard parseHex("0x38", value) ## assert value == -200 ## + ## If 'maxLen==0' the length of the hexadecimal number has no + ## upper bound. Not more than ```maxLen`` characters are parsed. var i = start var foundDigit = false if s[i] == '0' and (s[i+1] == 'x' or s[i+1] == 'X'): inc(i, 2) elif s[i] == '#': inc(i) - while true: + let last = if maxLen == 0: s.len else: i+maxLen + while i < last: case s[i] of '_': discard of '0'..'9': diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index aa29bb073..a446f85b4 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1210,22 +1210,21 @@ proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, ## If `s` does not begin with ``prefix`` and end with ``suffix`` a ## ValueError exception will be raised. result = newStringOfCap(s.len) - var i = 0 + var i = prefix.len if not s.startsWith(prefix): raise newException(ValueError, "String does not start with a prefix of: " & prefix) - inc(i) while true: if i == s.len-suffix.len: break case s[i] of '\\': case s[i+1]: of 'x': - inc i + inc i, 2 var c: int - i += parseutils.parseHex(s, c, i) + i += parseutils.parseHex(s, c, i, maxLen=2) result.add(chr(c)) - inc(i, 2) + dec i, 2 of '\\': result.add('\\') of '\'': @@ -1721,3 +1720,4 @@ when isMainModule: doAssert isUpper("ABC") doAssert(not isUpper("AAcc")) doAssert(not isUpper("A#$")) + doAssert(unescape(r"\x013", "", "") == "\x013") diff --git a/tests/parser/tmultiline_comments.nim b/tests/parser/tmultiline_comments.nim new file mode 100644 index 000000000..16abc7c3f --- /dev/null +++ b/tests/parser/tmultiline_comments.nim @@ -0,0 +1,18 @@ +discard """ + output: '''3''' +""" + +proc main* = + ##[Mutltie akdlsf comment with #[nesting]. + Yay, that is so cool. + ]## + echo "foo bar" + +var foo #[ Test the new inline comments ]#: int = 3 +##[ A +novel documentation comment +#[Nesting works to some extend] +##[ Nested doc comment! ]## +]# +]## +echo $foo From 71cf1f650e2812e43ed5a6edfb36666cfc145fdd Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 21:42:13 +0100 Subject: [PATCH 118/125] fixes multiline comments --- compiler/lexer.nim | 6 ++++-- tests/parser/tmultiline_comments.nim | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 4c1dd366b..e51d8db8a 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -934,8 +934,10 @@ proc skip(L: var TLexer, tok: var TToken) = if buf[pos+1] == '#': break if buf[pos+1] == '[': skipMultiLineComment(L, tok, pos+2, false) - return - while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) + pos = L.bufpos + buf = L.buf + else: + while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop L.bufpos = pos diff --git a/tests/parser/tmultiline_comments.nim b/tests/parser/tmultiline_comments.nim index 16abc7c3f..72832479d 100644 --- a/tests/parser/tmultiline_comments.nim +++ b/tests/parser/tmultiline_comments.nim @@ -7,6 +7,8 @@ proc main* = Yay, that is so cool. ]## echo "foo bar" + for s in ["one", "two", #["three",]# "four"]: + echo s var foo #[ Test the new inline comments ]#: int = 3 ##[ A @@ -16,3 +18,17 @@ novel documentation comment ]# ]## echo $foo + + #[Comment here. + Multiple lines + are not a problem.]# + + #[ #[ Multiline comment in already + commented out code. ]# + proc p[T](x: T) = discard + ]# + +proc bar = + ##[Long documentation comment + here. + ]## From f92f00b022fd367ded609fd9a8c547377a9ced51 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 22:18:06 +0100 Subject: [PATCH 119/125] fixes #3665 --- compiler/nimsets.nim | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/nimsets.nim b/compiler/nimsets.nim index 055bae909..f15ad6368 100644 --- a/compiler/nimsets.nim +++ b/compiler/nimsets.nim @@ -106,13 +106,17 @@ proc toTreeSet(s: TBitSet, settype: PType, info: TLineInfo): PNode = inc(b) if (b >= len(s) * ElemSize) or not bitSetIn(s, b): break dec(b) + let aa = newIntTypeNode(nkIntLit, a + first, elemType) + aa.info = info if a == b: - addSon(result, newIntTypeNode(nkIntLit, a + first, elemType)) + addSon(result, aa) else: n = newNodeI(nkRange, info) n.typ = elemType - addSon(n, newIntTypeNode(nkIntLit, a + first, elemType)) - addSon(n, newIntTypeNode(nkIntLit, b + first, elemType)) + addSon(n, aa) + let bb = newIntTypeNode(nkIntLit, b + first, elemType) + bb.info = info + addSon(n, bb) addSon(result, n) e = b inc(e) From 07d5adf9eeaec6adfb6dcc8c572bb4080766d7ed Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 22:55:14 +0100 Subject: [PATCH 120/125] fixes #3720 --- lib/wrappers/linenoise/clinenoise.c | 20 ++++++++++---------- lib/wrappers/linenoise/clinenoise.h | 20 ++++++-------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/lib/wrappers/linenoise/clinenoise.c b/lib/wrappers/linenoise/clinenoise.c index dd3aa736c..a03e6f379 100644 --- a/lib/wrappers/linenoise/clinenoise.c +++ b/lib/wrappers/linenoise/clinenoise.c @@ -139,7 +139,7 @@ struct linenoiseState { int ofd; /* Terminal stdout file descriptor. */ char *buf; /* Edited line buffer. */ size_t buflen; /* Edited line buffer size. */ - const char *prompt; /* Prompt to display. */ + char *prompt; /* Prompt to display. */ size_t plen; /* Prompt length. */ size_t pos; /* Current cursor position. */ size_t oldpos; /* Previous refresh cursor position. */ @@ -172,7 +172,7 @@ enum KEY_ACTION{ }; static void linenoiseAtExit(void); -int linenoiseHistoryAdd(const char *line); +int linenoiseHistoryAdd(char *line); static void refreshLine(struct linenoiseState *l); /* Debugging macro. */ @@ -413,7 +413,7 @@ void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { * in order to add completion options given the input string when the * user typed . See the example.c source code for a very easy to * understand example. */ -void linenoiseAddCompletion(linenoiseCompletions *lc, const char *str) { +void linenoiseAddCompletion(linenoiseCompletions *lc, char *str) { size_t len = strlen(str); char *copy, **cvec; @@ -445,7 +445,7 @@ static void abInit(struct abuf *ab) { ab->len = 0; } -static void abAppend(struct abuf *ab, const char *s, int len) { +static void abAppend(struct abuf *ab, char *s, int len) { char *neww = (char*)realloc(ab->b,ab->len+len); if (neww == NULL) return; @@ -723,7 +723,7 @@ void linenoiseEditDeletePrevWord(struct linenoiseState *l) { * when ctrl+d is typed. * * The function returns the length of the current buffer. */ -static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, const char *prompt) +static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, char *prompt) { struct linenoiseState l; @@ -929,7 +929,7 @@ void linenoisePrintKeyCodes(void) { /* This function calls the line editing function linenoiseEdit() using * the STDIN file descriptor set in raw mode. */ -static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { +static int linenoiseRaw(char *buf, size_t buflen, char *prompt) { int count; if (buflen == 0) { @@ -959,7 +959,7 @@ static int linenoiseRaw(char *buf, size_t buflen, const char *prompt) { * for a blacklist of stupid terminals, and later either calls the line * editing function or uses dummy fgets() so that you will be able to type * something even in the most desperate of the conditions. */ -char *linenoise(const char *prompt) { +char *linenoise(char *prompt) { char buf[LINENOISE_MAX_LINE]; int count; @@ -1009,7 +1009,7 @@ static void linenoiseAtExit(void) { * histories, but will work well for a few hundred of entries. * * Using a circular buffer is smarter, but a bit more complex to handle. */ -int linenoiseHistoryAdd(const char *line) { +int linenoiseHistoryAdd(char *line) { char *linecopy; if (history_max_len == 0) return 0; @@ -1072,7 +1072,7 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ -int linenoiseHistorySave(const char *filename) { +int linenoiseHistorySave(char *filename) { FILE *fp = fopen(filename,"w"); int j; @@ -1088,7 +1088,7 @@ int linenoiseHistorySave(const char *filename) { * * If the file exists and the operation succeeded 0 is returned, otherwise * on error -1 is returned. */ -int linenoiseHistoryLoad(const char *filename) { +int linenoiseHistoryLoad(char *filename) { FILE *fp = fopen(filename,"r"); char buf[LINENOISE_MAX_LINE]; diff --git a/lib/wrappers/linenoise/clinenoise.h b/lib/wrappers/linenoise/clinenoise.h index fbb01cfaa..a15845f86 100644 --- a/lib/wrappers/linenoise/clinenoise.h +++ b/lib/wrappers/linenoise/clinenoise.h @@ -39,30 +39,22 @@ #ifndef __LINENOISE_H #define __LINENOISE_H -#ifdef __cplusplus -extern "C" { -#endif - typedef struct linenoiseCompletions { size_t len; char **cvec; } linenoiseCompletions; -typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef void(linenoiseCompletionCallback)(char *, linenoiseCompletions *); void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); -void linenoiseAddCompletion(linenoiseCompletions *, const char *); +void linenoiseAddCompletion(linenoiseCompletions *, char *); -char *linenoise(const char *prompt); -int linenoiseHistoryAdd(const char *line); +char *linenoise(char *prompt); +int linenoiseHistoryAdd(char *line); int linenoiseHistorySetMaxLen(int len); -int linenoiseHistorySave(const char *filename); -int linenoiseHistoryLoad(const char *filename); +int linenoiseHistorySave(char *filename); +int linenoiseHistoryLoad(char *filename); void linenoiseClearScreen(void); void linenoiseSetMultiLine(int ml); void linenoisePrintKeyCodes(void); -#ifdef __cplusplus -} -#endif - #endif /* __LINENOISE_H */ From a9f6e2a843f2152676e1eb7fd043022a2fece114 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 23:00:04 +0100 Subject: [PATCH 121/125] fixes #3689 --- web/news.txt | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/web/news.txt b/web/news.txt index 9e6cb4e1b..91adef807 100644 --- a/web/news.txt +++ b/web/news.txt @@ -208,6 +208,34 @@ Changes affecting backwards compatibility is renamed to ``apply``. - The template ``mapIt`` now doesn't require the result's type parameter. Also the inplace ``mapIt`` is renamed to ``apply``. +- The compiler is now stricter with what is allowed as a case object + discriminator. The following code used to compile but was not supported + completely and so now fails: + +.. code-block:: nim + type + DataType* {.pure.} = enum + Char = 1, + Int8 = 2, + Int16 = 3, + Int32 = 4, + Int64 = 5, + Float32 = 6, + Float64 = 7 + + DataSeq* = object + case kind* : DataType + of DataType.Char: charSeq* : seq[char] + of DataType.Int8: int8Seq* : seq[int8] + of DataType.Int16: int16Seq* : seq[int16] + of DataType.Int32: int32Seq* : seq[int32] + of DataType.Int64: int64Seq* : seq[int64] + of DataType.Float32: float32Seq* : seq[float32] + of DataType.Float64: float64Seq* : seq[float64] + + length* : int + + Library Additions ----------------- From 3114523a6ff0f0d19aeffcb25a21a7f954dc74fa Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 17 Jan 2016 23:46:23 +0100 Subject: [PATCH 122/125] fixes multi-line comments --- compiler/lexer.nim | 13 +++++++++--- tests/parser/tmultiline_comments.nim | 30 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/compiler/lexer.nim b/compiler/lexer.nim index e51d8db8a..fc43f8d6a 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -913,9 +913,16 @@ proc skip(L: var TLexer, tok: var TToken) = pos = handleCRLF(L, pos) buf = L.buf var indent = 0 - while buf[pos] == ' ': - inc(pos) - inc(indent) + while true: + if buf[pos] == ' ': + inc(pos) + inc(indent) + elif buf[pos] == '#' and buf[pos+1] == '[': + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf + else: + break tok.strongSpaceA = 0 when defined(nimfix): template doBreak(): expr = buf[pos] > ' ' diff --git a/tests/parser/tmultiline_comments.nim b/tests/parser/tmultiline_comments.nim index 72832479d..7a3bb5304 100644 --- a/tests/parser/tmultiline_comments.nim +++ b/tests/parser/tmultiline_comments.nim @@ -32,3 +32,33 @@ proc bar = ##[Long documentation comment here. ]## + + +proc write(a: auto, x: varargs[string, `$`]) = + stdout.write ($a) + for o in x: + stdout.write(o) + +proc writeln(a: auto, x: varargs[string, `$`]) = + write a, x + stdout.write "\n" + +proc write() = write(stdout) +proc writeln() = + stdout.write "\n" + +#[ #[ Multiline comment in already + commented out code. ]# +proc p[T](x: T) = discard +]# + +var hello = #[(x in bar)^^ "Hello" # greetings +]#"Hello" +proc maino = + write hello, " Test Me " + writeln() + write 3 + block: + write() + write " times more" + #[ test ]# writeln " Again" From bfb629b087c29add9f6c049427ac0bab0b717e94 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 18 Jan 2016 00:02:09 +0100 Subject: [PATCH 123/125] yay, better commercial support --- web/support.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/web/support.txt b/web/support.txt index c0ffafcbc..17faef7bf 100644 --- a/web/support.txt +++ b/web/support.txt @@ -32,8 +32,9 @@ Commercial support includes: All interested parties should email ``support@nim-lang.org``. The bid for contracting work is a commercial offer provided by: -| **Andreas Rumpf** -| St.-Quentin-Ring 47 -| 67663 Kaiserslautern +| **METATEXX GmbH** +| Spicher Str. 30 +| 53859 Niederkassel | GERMANY -| EU VAT-IN: DE297783450 +| EU VAT-IN: DE287088604 +| http://metatexx.de/index.php?index=12 From bd933e6009b01768bfc62f4386aea9a09282beb6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 18 Jan 2016 00:02:22 +0100 Subject: [PATCH 124/125] documentation updates --- web/news.txt | 9 ++++++--- web/website.ini | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/web/news.txt b/web/news.txt index 91adef807..bd74ae1a3 100644 --- a/web/news.txt +++ b/web/news.txt @@ -6,9 +6,6 @@ News 2016-01-XX Version 0.13.0 released ================================== - This release implements no new features. Instead existing features have - been fleshed out as we're heading to version 1. - Changes affecting backwards compatibility ----------------------------------------- @@ -24,6 +21,8 @@ News - The ``redis`` module has been moved out of the standard library. It can now be installed via Nimble and is located here: https://github.com/nim-lang/redis + - ``math.RunningStat`` and its associated procs have been moved from + the ``math`` module to a new ``stats`` module. Syntax changes @@ -40,6 +39,10 @@ News code like ``0..kArraySize div 2 -1`` needs to be changed to ``0..kArraySize div 2 - 1``. + This release also adds multi-line comments to Nim. They look + like ``#[ comment here ]#``. For more details read the section of + the `manual `_. + Iterator changes ~~~~~~~~~~~~~~~~ diff --git a/web/website.ini b/web/website.ini index 023262aa6..6343193d5 100644 --- a/web/website.ini +++ b/web/website.ini @@ -39,6 +39,7 @@ srcdoc2: "impure/re;pure/typetraits" srcdoc2: "pure/concurrency/threadpool.nim;pure/concurrency/cpuinfo.nim" srcdoc: "system/threads.nim;system/channels.nim;js/dom" srcdoc2: "pure/os;pure/strutils;pure/math;pure/matchers;pure/algorithm" +srcdoc2: "pure/stats;impure/nre" srcdoc2: "pure/complex;pure/times;pure/osproc;pure/pegs;pure/dynlib" srcdoc2: "pure/parseopt;pure/parseopt2;pure/hashes;pure/strtabs;pure/lexbase" srcdoc2: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql" From 2309975f782bf59b240e3874d5c31b4b299eca5d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 18 Jan 2016 03:00:14 +0100 Subject: [PATCH 125/125] fixes yet another LL regression (Aporia compiles again) --- compiler/lambdalifting.nim | 2 +- tests/closure/texplicit_dummy_closure.nim | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/closure/texplicit_dummy_closure.nim diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 20f903da9..9265d09e3 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -362,6 +362,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) = # this handles the case that the inner proc was declared as # .closure but does not actually capture anything: addClosureParam(c, s) + c.somethingToDo = true let innerProc = isInnerProc(s) if innerProc: @@ -654,7 +655,6 @@ proc wrapIterBody(n: PNode; owner: PSym): PNode = proc symToClosure(n: PNode; owner: PSym; d: DetectionPass; c: var LiftingPass): PNode = let s = n.sym - if s == owner: # recursive calls go through (lambda, hiddenParam): let available = getHiddenParam(owner) diff --git a/tests/closure/texplicit_dummy_closure.nim b/tests/closure/texplicit_dummy_closure.nim new file mode 100644 index 000000000..ec608b31a --- /dev/null +++ b/tests/closure/texplicit_dummy_closure.nim @@ -0,0 +1,22 @@ + +# This is a regression of the new lambda lifting; detected by Aporia +import asyncio, sockets +import os + +type + Window = object + oneInstSock*: PAsyncSocket + IODispatcher*: PDispatcher + +var + win: Window + +proc initSocket() = + win.oneInstSock = asyncSocket() + #win.oneInstSock.handleAccept = + proc test(s: PAsyncSocket) = + var client: PAsyncSocket + proc dummy(c: PAsyncSocket) {.closure.} = + discard + client.handleRead = dummy + test(win.oneInstSock)