jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'devel' into sighashes

Browse source
This commit is contained in:
Araq 2016-11-28 20:59:30 +01:00
commit 27723af469
75 changed files with 1258 additions and 361 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/nimbase.h
View file

@ -459,3 +459,7 @@ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == siz
#elif defined(__FreeBSD__)
# include <sys/types.h>
#endif
/* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */
#define NIM_CHECK_SIZE(typ, sz) \
_Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size")

49
lib/pure/asyncdispatch.nim
View file

@ -11,7 +11,7 @@ include "system/inclrtl"
import os, oids, tables, strutils, times, heapqueue
import nativesockets, net, queues
import nativesockets, net, deques
export Port, SocketFlag
@ -164,7 +164,7 @@ include includes/asyncfutures
type
PDispatcherBase = ref object of RootRef
timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
callbacks: Queue[proc ()]
callbacks: Deque[proc ()]
proc processTimers(p: PDispatcherBase) {.inline.} =
while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt:
@ -172,7 +172,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} =
proc processPendingCallbacks(p: PDispatcherBase) =
while p.callbacks.len > 0:
var cb = p.callbacks.dequeue()
var cb = p.callbacks.popFirst()
cb()
proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
@ -230,7 +230,7 @@ when defined(windows) or defined(nimdoc):
result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1)
result.handles = initSet[AsyncFD]()
result.timers.newHeapQueue()
result.callbacks = initQueue[proc ()](64)
result.callbacks = initDeque[proc ()](64)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
@ -987,7 +987,7 @@ else:
new result
result.selector = newSelector()
result.timers.newHeapQueue()
result.callbacks = initQueue[proc ()](64)
result.callbacks = initDeque[proc ()](64)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
@ -1043,6 +1043,26 @@ else:
p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb)
update(fd, p.selector[fd.SocketHandle].events + {EvWrite})
template processCallbacks(callbacks: expr) =
# Callback may add items to ``callbacks`` which causes issues if
# we are iterating over it at the same time. We therefore
# make a copy to iterate over.
let currentCBs = callbacks
callbacks = @[]
# Using another sequence because callbacks themselves can add
# other callbacks.
var newCBs: seq[Callback] = @[]
for cb in currentCBs:
if newCBs.len > 0:
# A callback has already returned with EAGAIN, don't call any
# others until next `poll`.
newCBs.add(cb)
else:
if not cb(data.fd):
# Callback wants to be called again.
newCBs.add(cb)
callbacks = newCBs & callbacks
proc poll*(timeout = 500) =
let p = getGlobalDispatcher()
@ -1056,23 +1076,10 @@ else:
# `recv(...)` routines.
if EvRead in info.events or info.events == {EvError}:
# Callback may add items to ``data.readCBs`` which causes issues if
# we are iterating over ``data.readCBs`` at the same time. We therefore
# make a copy to iterate over.
let currentCBs = data.readCBs
data.readCBs = @[]
for cb in currentCBs:
if not cb(data.fd):
# Callback wants to be called again.
data.readCBs.add(cb)
processCallbacks(data.readCBs)
if EvWrite in info.events or info.events == {EvError}:
let currentCBs = data.writeCBs
data.writeCBs = @[]
for cb in currentCBs:
if not cb(data.fd):
# Callback wants to be called again.
data.writeCBs.add(cb)
processCallbacks(data.writeCBs)
if info.key in p.selector:
var newEvents: set[Event]
@ -1410,7 +1417,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
proc callSoon*(cbproc: proc ()) =
## Schedule `cbproc` to be called as soon as possible.
## The callback is called when control returns to the event loop.
getGlobalDispatcher().callbacks.enqueue(cbproc)
getGlobalDispatcher().callbacks.addLast(cbproc)
proc runForever*() =
## Begins a never ending global dispatcher poll loop.

17
lib/pure/asyncfile.nim
View file

@ -118,8 +118,8 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
## Read ``size`` bytes from the specified file asynchronously starting at
## the current position of the file pointer.
##
## If the file pointer is past the end of the file then an empty string is
## returned.
## If the file pointer is past the end of the file then zero is returned
## and no bytes are read into ``buf``
var retFuture = newFuture[int]("asyncfile.readBuffer")
when defined(windows) or defined(nimdoc):
@ -149,7 +149,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
let err = osLastError()
if err.int32 != ERROR_IO_PENDING:
GC_unref(ol)
retFuture.fail(newException(OSError, osErrorMsg(err)))
if err.int32 == ERROR_HANDLE_EOF:
# This happens in Windows Server 2003
retFuture.complete(0)
else:
retFuture.fail(newException(OSError, osErrorMsg(err)))
else:
# Request completed immediately.
var bytesRead: DWord
@ -233,7 +237,12 @@ proc read*(f: AsyncFile, size: int): Future[string] =
dealloc buffer
buffer = nil
GC_unref(ol)
retFuture.fail(newException(OSError, osErrorMsg(err)))
if err.int32 == ERROR_HANDLE_EOF:
# This happens in Windows Server 2003
retFuture.complete("")
else:
retFuture.fail(newException(OSError, osErrorMsg(err)))
else:
# Request completed immediately.
var bytesRead: DWord

266
lib/pure/collections/deques.nim Normal file
View file

@ -0,0 +1,266 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2012 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Implementation of a `deque`:idx: (double-ended queue).
## The underlying implementation uses a ``seq``.
##
## None of the procs that get an individual value from the deque can be used
## on an empty deque.
## If compiled with `boundChecks` option, those procs will raise an `IndexError`
## on such access. This should not be relied upon, as `-d:release` will
## disable those checks and may return garbage or crash the program.
##
## As such, a check to see if the deque is empty is needed before any
## access, unless your program logic guarantees it indirectly.
##
## .. code-block:: Nim
## proc foo(a, b: Positive) = # assume random positive values for `a` and `b`
## var deq = initDeque[int]() # initializes the object
## for i in 1 ..< a: deq.addLast i # populates the deque
##
## if b < deq.len: # checking before indexed access
## echo "The element at index position ", b, " is ", deq[b]
##
## # The following two lines don't need any checking on access due to the
## # logic of the program, but that would not be the case if `a` could be 0.
## assert deq.peekFirst == 1
## assert deq.peekLast == a
##
## while deq.len > 0: # checking if the deque is empty
## echo deq.removeLast()
##
## Note: For inter thread communication use
## a `Channel <channels.html>`_ instead.
import math
type
Deque*[T] = object
## A double-ended queue backed with a ringed seq buffer.
data: seq[T]
head, tail, count, mask: int
proc initDeque*[T](initialSize: int = 4): Deque[T] =
## Create a new deque.
## Optionally, the initial capacity can be reserved via `initialSize` as a
## performance optimization. The length of a newly created deque will still
## be 0.
##
## `initialSize` needs to be a power of two. If you need to accept runtime
## values for this you could use the ``nextPowerOfTwo`` proc from the
## `math <math.html>`_ module.
assert isPowerOfTwo(initialSize)
result.mask = initialSize-1
newSeq(result.data, initialSize)
proc len*[T](deq: Deque[T]): int {.inline.} =
## Return the number of elements of `deq`.
result = deq.count
template emptyCheck(deq) =
# Bounds check for the regular deque access.
when compileOption("boundChecks"):
if unlikely(deq.count < 1):
raise newException(IndexError, "Empty deque.")
template xBoundsCheck(deq, i) =
# Bounds check for the array like accesses.
when compileOption("boundChecks"): # d:release should disable this.
if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter
raise newException(IndexError,
"Out of bounds: " & $i & " > " & $(deq.count - 1))
proc `[]`*[T](deq: Deque[T], i: Natural) : T {.inline.} =
## Access the i-th element of `deq` by order from first to last.
## deq[0] is the first, deq[^1] is the last.
xBoundsCheck(deq, i)
return deq.data[(deq.first + i) and deq.mask]
proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} =
## Access the i-th element of `deq` and returns a mutable
## reference to it.
xBoundsCheck(deq, i)
return deq.data[(deq.head + i) and deq.mask]
proc `[]=`* [T] (deq: var Deque[T], i: Natural, val : T) {.inline.} =
## Change the i-th element of `deq`.
xBoundsCheck(deq, i)
deq.data[(deq.head + i) and deq.mask] = val
iterator items*[T](deq: Deque[T]): T =
## Yield every element of `deq`.
var i = deq.head
for c in 0 ..< deq.count:
yield deq.data[i]
i = (i + 1) and deq.mask
iterator mitems*[T](deq: var Deque[T]): var T =
## Yield every element of `deq`.
var i = deq.head
for c in 0 ..< deq.count:
yield deq.data[i]
i = (i + 1) and deq.mask
iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] =
## Yield every (position, value) of `deq`.
var i = deq.head
for c in 0 ..< deq.count:
yield (c, deq.data[i])
i = (i + 1) and deq.mask
proc contains*[T](deq: Deque[T], item: T): bool {.inline.} =
## Return true if `item` is in `deq` or false if not found. Usually used
## via the ``in`` operator. It is the equivalent of ``deq.find(item) >= 0``.
##
## .. code-block:: Nim
## if x in q:
## assert q.contains x
for e in deq:
if e == item: return true
return false
proc expandIfNeeded[T](deq: var Deque[T]) =
var cap = deq.mask + 1
if unlikely(deq.count >= cap):
var n = newSeq[T](cap * 2)
for i, x in deq: # don't use copyMem because the GC and because it's slower.
shallowCopy(n[i], x)
shallowCopy(deq.data, n)
deq.mask = cap * 2 - 1
deq.tail = deq.count
deq.head = 0
proc addFirst*[T](deq: var Deque[T], item: T) =
## Add an `item` to the beginning of the `deq`.
expandIfNeeded(deq)
inc deq.count
deq.head = (deq.head - 1) and deq.mask
deq.data[deq.head] = item
proc addLast*[T](deq: var Deque[T], item: T) =
## Add an `item` to the end of the `deq`.
expandIfNeeded(deq)
inc deq.count
deq.data[deq.tail] = item
deq.tail = (deq.tail + 1) and deq.mask
proc peekFirst*[T](deq: Deque[T]): T {.inline.}=
## Returns the first element of `deq`, but does not remove it from the deque.
emptyCheck(deq)
result = deq.data[deq.head]
proc peekLast*[T](deq: Deque[T]): T {.inline.} =
## Returns the last element of `deq`, but does not remove it from the deque.
emptyCheck(deq)
result = deq.data[(deq.tail - 1) and deq.mask]
proc default[T](t: typedesc[T]): T {.inline.} = discard
proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} =
## Remove and returns the first element of the `deq`.
emptyCheck(deq)
dec deq.count
result = deq.data[deq.head]
deq.data[deq.head] = default(type(result))
deq.head = (deq.head + 1) and deq.mask
proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} =
## Remove and returns the last element of the `deq`.
emptyCheck(deq)
dec deq.count
deq.tail = (deq.tail - 1) and deq.mask
result = deq.data[deq.tail]
deq.data[deq.tail] = default(type(result))
proc `$`*[T](deq: Deque[T]): string =
## Turn a deque into its string representation.
result = "["
for x in deq:
if result.len > 1: result.add(", ")
result.add($x)
result.add("]")
when isMainModule:
var deq = initDeque[int](1)
deq.addLast(4)
deq.addFirst(9)
deq.addFirst(123)
var first = deq.popFirst()
deq.addLast(56)
assert(deq.peekLast() == 56)
deq.addLast(6)
assert(deq.peekLast() == 6)
var second = deq.popFirst()
deq.addLast(789)
assert(deq.peekLast() == 789)
assert first == 123
assert second == 9
assert($deq == "[4, 56, 6, 789]")
assert deq[0] == deq.peekFirst and deq.peekFirst == 4
assert deq[^1] == deq.peekLast and deq.peekLast == 789
deq[0] = 42
deq[^1] = 7
assert 6 in deq and 789 notin deq
assert deq.find(6) >= 0
assert deq.find(789) < 0
for i in -2 .. 10:
if i in deq:
assert deq.contains(i) and deq.find(i) >= 0
else:
assert(not deq.contains(i) and deq.find(i) < 0)
when compileOption("boundChecks"):
try:
echo deq[99]
assert false
except IndexError:
discard
try:
assert deq.len == 4
for i in 0 ..< 5: deq.popFirst()
assert false
except IndexError:
discard
# grabs some types of resize error.
deq = initDeque[int]()
for i in 1 .. 4: deq.addLast i
deq.popFirst()
deq.popLast()
for i in 5 .. 8: deq.addFirst i
assert $deq == "[8, 7, 6, 5, 2, 3]"
# Similar to proc from the documentation example
proc foo(a, b: Positive) = # assume random positive values for `a` and `b`.
var deq = initDeque[int]()
assert deq.len == 0
for i in 1 .. a: deq.addLast i
if b < deq.len: # checking before indexed access.
assert deq[b] == b + 1
# The following two lines don't need any checking on access due to the logic
# of the program, but that would not be the case if `a` could be 0.
assert deq.peekFirst == 1
assert deq.peekLast == a
while deq.len > 0: # checking if the deque is empty
assert deq.popFirst() > 0
#foo(0,0)
foo(8,5)
foo(10,9)
foo(1,1)
foo(2,1)
foo(1,5)
foo(3,2)

4
lib/pure/collections/queues.nim
View file

@ -39,8 +39,10 @@
import math
{.warning: "`queues` module is deprecated - use `deques` instead".}
type
Queue*[T] = object ## A queue.
Queue* {.deprecated.} [T] = object ## A queue.
data: seq[T]
rd, wr, count, mask: int

14
lib/pure/collections/tableimpl.nim
View file

@ -39,16 +39,22 @@ template rawGetKnownHCImpl() {.dirty.} =
h = nextTry(h, maxHash(t))
result = -1 - h # < 0 => MISSING; insert idx = -1 - result
template rawGetImpl() {.dirty.} =
template genHashImpl(key, hc: typed) =
hc = hash(key)
if hc == 0: # This almost never taken branch should be very predictable.
hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine.
template genHash(key: typed): Hash =
var res: Hash
genHashImpl(key, res)
res
template rawGetImpl() {.dirty.} =
genHashImpl(key, hc)
rawGetKnownHCImpl()
template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add
hc = hash(key)
if hc == 0:
hc = 314159265
genHashImpl(key, hc)
var h: Hash = hc and maxHash(t)
while isFilled(t.data[h].hcode):
h = nextTry(h, maxHash(t))

53
lib/pure/collections/tables.nim
View file

@ -224,7 +224,7 @@ template withValue*[A, B](t: var Table[A, B], key: A,
iterator allValues*[A, B](t: Table[A, B]; key: A): B =
## iterates over any value in the table `t` that belongs to the given `key`.
var h: Hash = hash(key) and high(t.data)
var h: Hash = genHash(key) and high(t.data)
while isFilled(t.data[h].hcode):
if t.data[h].key == key:
yield t.data[h].val
@ -479,7 +479,7 @@ proc clear*[A, B](t: var OrderedTableRef[A, B]) =
## Resets the table so that is is empty.
clear(t[])
template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} =
template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} =
var h = t.first
while h >= 0:
var nxt = t.data[h].next
@ -674,13 +674,6 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} =
## returns the number of keys in `t`.
result = t.counter
template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} =
var h = t.first
while h >= 0:
var nxt = t.data[h].next
if isFilled(t.data[h].hcode): yieldStmt
h = nxt
iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) =
## iterates over any (key, value) pair in the table `t` in insertion
## order.
@ -785,20 +778,22 @@ proc sort*[A, B](t: OrderedTableRef[A, B],
proc del*[A, B](t: var OrderedTable[A, B], key: A) =
## deletes `key` from ordered hash table `t`. O(n) comlexity.
var prev = -1
let hc = hash(key)
forAllOrderedPairs:
if t.data[h].hcode == hc:
if t.first == h:
t.first = t.data[h].next
var n: OrderedKeyValuePairSeq[A, B]
newSeq(n, len(t.data))
var h = t.first
t.first = -1
t.last = -1
swap(t.data, n)
let hc = genHash(key)
while h >= 0:
var nxt = n[h].next
if isFilled(n[h].hcode):
if n[h].hcode == hc and n[h].key == key:
dec t.counter
else:
t.data[prev].next = t.data[h].next
var zeroValue : type(t.data[h])
t.data[h] = zeroValue
dec t.counter
break
else:
prev = h
var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode)
rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j)
h = nxt
proc del*[A, B](t: var OrderedTableRef[A, B], key: A) =
## deletes `key` from ordered hash table `t`. O(n) comlexity.
@ -1164,6 +1159,20 @@ when isMainModule:
doAssert(prev < i)
prev = i
block: # Deletion from OrderedTable should account for collision groups. See issue #5057.
# The bug is reproducible only with exact keys
const key1 = "boy_jackpot.inGamma"
const key2 = "boy_jackpot.outBlack"
var t = {
key1: 0,
key2: 0
}.toOrderedTable()
t.del(key1)
assert(t.len == 1)
assert(key2 in t)
var
t1 = initCountTable[string]()
t2 = initCountTable[string]()

29
lib/pure/includes/asyncfutures.nim
View file

@ -263,13 +263,13 @@ proc all*[T](futs: varargs[Future[T]]): auto =
for fut in futs:
fut.callback = proc(f: Future[T]) =
if f.failed:
retFuture.fail(f.error)
elif not retFuture.finished:
inc(completedFutures)
if completedFutures == totalFutures:
retFuture.complete()
inc(completedFutures)
if not retFuture.finished:
if f.failed:
retFuture.fail(f.error)
else:
if completedFutures == totalFutures:
retFuture.complete()
if totalFutures == 0:
retFuture.complete()
@ -285,14 +285,15 @@ proc all*[T](futs: varargs[Future[T]]): auto =
for i, fut in futs:
proc setCallback(i: int) =
fut.callback = proc(f: Future[T]) =
if f.failed:
retFuture.fail(f.error)
elif not retFuture.finished:
retValues[i] = f.read()
inc(completedFutures)
inc(completedFutures)
if not retFuture.finished:
if f.failed:
retFuture.fail(f.error)
else:
retValues[i] = f.read()
if completedFutures == len(retValues):
retFuture.complete(retValues)
if completedFutures == len(retValues):
retFuture.complete(retValues)
setCallback(i)

44
lib/pure/json.nim
View file

@ -954,9 +954,11 @@ proc newIndent(curr, indent: int, ml: bool): int =
proc nl(s: var string, ml: bool) =
if ml: s.add("\n")
proc escapeJson*(s: string): string =
proc escapeJson*(s: string; result: var string) =
## Converts a string `s` to its JSON representation.
result = newStringOfCap(s.len + s.len shr 3)
## Appends to ``result``.
const
HexChars = "0123456789ABCDEF"
result.add("\"")
for x in runes(s):
var r = int(x)
@ -967,10 +969,19 @@ proc escapeJson*(s: string): string =
of '\\': result.add("\\\\")
else: result.add(c)
else:
result.add("\\u")
result.add(toHex(r, 4))
# toHex inlined for more speed (saves stupid string allocations):
result.add("\\u0000")
let start = result.len - 4
for j in countdown(3, 0):
result[j+start] = HexChars[r and 0xF]
r = r shr 4
result.add("\"")
proc escapeJson*(s: string): string =
## Converts a string `s` to its JSON representation.
result = newStringOfCap(s.len + s.len shr 3)
escapeJson(s, result)
proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
lstArr = false, currIndent = 0) =
case node.kind
@ -988,7 +999,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
inc i
# Need to indent more than {
result.indent(newIndent(currIndent, indent, ml))
result.add(escapeJson(key))
escapeJson(key, result)
result.add(": ")
toPretty(result, val, indent, ml, false,
newIndent(currIndent, indent, ml))
@ -999,16 +1010,19 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true,
result.add("{}")
of JString:
if lstArr: result.indent(currIndent)
result.add(escapeJson(node.str))
escapeJson(node.str, result)
of JInt:
if lstArr: result.indent(currIndent)
result.add($node.num)
when defined(js): result.add($node.num)
else: result.add(node.num)
of JFloat:
if lstArr: result.indent(currIndent)
result.add($node.fnum)
# Fixme: implement new system.add ops for the JS target
when defined(js): result.add($node.fnum)
else: result.add(node.fnum)
of JBool:
if lstArr: result.indent(currIndent)
result.add($node.bval)
result.add(if node.bval: "true" else: "false")
of JArray:
if lstArr: result.indent(currIndent)
if len(node.elems) != 0:
@ -1057,16 +1071,18 @@ proc toUgly*(result: var string, node: JsonNode) =
for key, value in pairs(node.fields):
if comma: result.add ","
else: comma = true
result.add key.escapeJson()
key.escapeJson(result)
result.add ":"
result.toUgly value
result.add "}"
of JString:
result.add node.str.escapeJson()
node.str.escapeJson(result)
of JInt:
result.add($node.num)
when defined(js): result.add($node.num)
else: result.add(node.num)
of JFloat:
result.add($node.fnum)
when defined(js): result.add($node.fnum)
else: result.add(node.fnum)
of JBool:
result.add(if node.bval: "true" else: "false")
of JNull:
@ -1394,4 +1410,6 @@ when isMainModule:
var parsed2 = parseFile("tests/testdata/jsontest2.json")
doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}")
doAssert escapeJson("\10FoobarÄ") == "\"\\u000AFoobar\\u00C4\""
echo("Tests succeeded!")

18
lib/pure/logging.nim
View file

@ -172,18 +172,26 @@ when not defined(js):
var (path, name, _) = splitFile(getAppFilename())
result = changeFileExt(path / name, "log")
proc newFileLogger*(file: File,
levelThreshold = lvlAll,
fmtStr = defaultFmtStr): FileLogger =
## Creates a new file logger. This logger logs to ``file``.
new(result)
result.file = file
result.levelThreshold = levelThreshold
result.fmtStr = fmtStr
proc newFileLogger*(filename = defaultFilename(),
mode: FileMode = fmAppend,
levelThreshold = lvlAll,
fmtStr = defaultFmtStr,
bufSize: int = -1): FileLogger =
## Creates a new file logger. This logger logs to a file.
## Creates a new file logger. This logger logs to a file, specified
## by ``fileName``.
## Use ``bufSize`` as size of the output buffer when writing the file
## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size).
new(result)
result.levelThreshold = levelThreshold
result.file = open(filename, mode, bufSize = bufSize)
result.fmtStr = fmtStr
let file = open(filename, mode, bufSize = bufSize)
newFileLogger(file, levelThreshold, fmtStr)
# ------

7
lib/pure/marshal.nim
View file

@ -9,6 +9,7 @@
## This module contains procs for `serialization`:idx: and `deseralization`:idx:
## of arbitrary Nim data structures. The serialization format uses `JSON`:idx:.
## Warning: The serialization format could change in future!
##
## **Restriction**: For objects their type is **not** serialized. This means
## essentially that it does not work if the object has some other runtime
@ -29,6 +30,12 @@
## a = b
## echo($$a[]) # produces "{}", not "{f: 0}"
##
## # unmarshal
## let c = to[B]("""{"f": 2}""")
##
## # marshal
## let s = $$c
## **Note**: The ``to`` and ``$$`` operations are available at compile-time!
import streams, typeinfo, json, intsets, tables, unicode

2
lib/pure/net.nim
View file

@ -1250,7 +1250,7 @@ proc IPv6_loopback*(): IpAddress =
address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1])
proc `==`*(lhs, rhs: IpAddress): bool =
## Compares two IpAddresses for Equality. Returns two if the addresses are equal
## Compares two IpAddresses for Equality. Returns true if the addresses are equal
if lhs.family != rhs.family: return false
if lhs.family == IpAddressFamily.IPv4:
for i in low(lhs.address_v4) .. high(lhs.address_v4):

80
lib/pure/nimtracker.nim Normal file
View file

@ -0,0 +1,80 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2016 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Memory tracking support for Nim.
when not defined(memTracker) and not isMainModule:
{.error: "Memory tracking support is turned off!".}
{.push memtracker: off.}
# we import the low level wrapper and are careful not to use Nim's
# memory manager for anything here.
import sqlite3
var
dbHandle: PSqlite3
insertStmt: Pstmt
template sbind(x: int; value) =
when value is cstring:
let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT)
if ret != SQLITE_OK:
quit "could not bind value"
else:
let ret = insertStmt.bindInt64(x, value)
if ret != SQLITE_OK:
quit "could not bind value"
when defined(memTracker):
proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [].} =
for i in 0..log.count-1:
var success = false
let e = log.data[i]
discard sqlite3.reset(insertStmt)
discard clearBindings(insertStmt)
sbind 1, e.op
sbind(2, cast[int](e.address))
sbind 3, e.size
sbind 4, e.file
sbind 5, e.line
if step(insertStmt) == SQLITE_DONE:
success = true
if not success:
quit "could not write to database!"
proc execQuery(q: string) =
var s: Pstmt
if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK:
discard step(s)
if finalize(s) != SQLITE_OK:
quit "could not finalize " & $sqlite3.errmsg(dbHandle)
else:
quit "could not prepare statement " & $sqlite3.errmsg(dbHandle)
proc setupDb() =
execQuery """create table if not exists tracking(
id integer primary key,
op varchar not null,
address integer not null,
size integer not null,
file varchar not null,
line integer not null
)"""
execQuery "delete from tracking"
if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK:
setupDb()
const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)"
if prepare_v2(dbHandle, query,
query.len, insertStmt, nil) == SQLITE_OK:
when defined(memTracker):
setTrackLogger logEntries
else:
quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle)
{.pop.}

2
lib/pure/strutils.nim
View file

@ -1309,10 +1309,12 @@ proc join*[T: not string](a: openArray[T], sep: string = ""): string {.
type
SkipTable = array[char, int]
{.push profiler: off.}
proc preprocessSub(sub: string, a: var SkipTable) =
var m = len(sub)
for i in 0..0xff: a[chr(i)] = m+1
for i in 0..m-1: a[sub[i]] = m-i
{.pop.}
proc findAux(s, sub: string, start: int, a: SkipTable): int =
# Fast "quick search" algorithm:

111
lib/pure/times.nim
View file

@ -144,8 +144,9 @@ type
yearday*: range[0..365] ## The number of days since January 1,
## in the range 0 to 365.
## Always 0 if the target is JS.
isDST*: bool ## Determines whether DST is in effect. Always
## ``False`` if time is UTC.
isDST*: bool ## Determines whether DST is in effect.
## Semantically, this adds another negative hour
## offset to the time in addition to the timezone.
timezone*: int ## The offset of the (non-DST) timezone in seconds
## west of UTC. Note that the sign of this number
## is the opposite of the one in a formatted
@ -278,16 +279,20 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval =
carryO = `div`(ti1.months + ti2.months, 12)
result.years = carryO + ti1.years + ti2.years
proc `-`*(ti: TimeInterval): TimeInterval =
result = TimeInterval(
milliseconds: -ti.milliseconds,
seconds: -ti.seconds,
minutes: -ti.minutes,
hours: -ti.hours,
days: -ti.days,
months: -ti.months,
years: -ti.years
)
proc `-`*(ti1, ti2: TimeInterval): TimeInterval =
## Subtracts TimeInterval ``ti1`` from ``ti2``.
result = ti1
result.milliseconds -= ti2.milliseconds
result.seconds -= ti2.seconds
result.minutes -= ti2.minutes
result.hours -= ti2.hours
result.days -= ti2.days
result.months -= ti2.months
result.years -= ti2.years
result = ti1 + (-ti2)
proc isLeapYear*(year: int): bool =
## returns true if ``year`` is a leap year
@ -363,16 +368,9 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo =
##
## **Note:** This has been only briefly tested, it is inaccurate especially
## when you subtract so much that you reach the Julian calendar.
let t = toSeconds(toTime(a))
var intval: TimeInterval
intval.milliseconds = - interval.milliseconds
intval.seconds = - interval.seconds
intval.minutes = - interval.minutes
intval.hours = - interval.hours
intval.days = - interval.days
intval.months = - interval.months
intval.years = - interval.years
let secs = toSeconds(a, intval)
let
t = toSeconds(toTime(a))
secs = toSeconds(a, -interval)
if a.timezone == 0:
result = getGMTime(fromSeconds(t + secs))
else:
@ -732,6 +730,7 @@ const
secondsInMin = 60
secondsInHour = 60*60
secondsInDay = 60*60*24
minutesInHour = 60
epochStartYear = 1970
proc formatToken(info: TimeInfo, token: string, buf: var string) =
@ -823,28 +822,32 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) =
if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear
buf.add(fyear)
of "z":
let hours = abs(info.timezone) div secondsInHour
if info.timezone < 0: buf.add('-')
else: buf.add('+')
let
nonDstTz = info.timezone - int(info.isDst) * secondsInHour
hours = abs(nonDstTz) div secondsInHour
if nonDstTz <= 0: buf.add('+')
else: buf.add('-')
buf.add($hours)
of "zz":
let hours = abs(info.timezone) div secondsInHour
if info.timezone < 0: buf.add('-')
else: buf.add('+')
let
nonDstTz = info.timezone - int(info.isDst) * secondsInHour
hours = abs(nonDstTz) div secondsInHour
if nonDstTz <= 0: buf.add('+')
else: buf.add('-')
if hours < 10: buf.add('0')
buf.add($hours)
of "zzz":
let
hours = abs(info.timezone) div secondsInHour
minutes = abs(info.timezone) mod 60
if info.timezone < 0: buf.add('-')
else: buf.add('+')
nonDstTz = info.timezone - int(info.isDst) * secondsInHour
hours = abs(nonDstTz) div secondsInHour
minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour
if nonDstTz <= 0: buf.add('+')
else: buf.add('-')
if hours < 10: buf.add('0')
buf.add($hours)
buf.add(':')
if minutes < 10: buf.add('0')
buf.add($minutes)
of "":
discard
else:
@ -999,7 +1002,6 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
of "M":
var pd = parseInt(value[j..j+1], sv)
info.month = Month(sv-1)
info.monthday = sv
j += pd
of "MM":
var month = value[j..j+1].parseInt()
@ -1088,27 +1090,42 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
info.year = value[j..j+3].parseInt()
j += 4
of "z":
info.isDST = false
if value[j] == '+':
info.timezone = 0 - parseInt($value[j+1]) * secondsInHour
elif value[j] == '-':
info.timezone = parseInt($value[j+1]) * secondsInHour
elif value[j] == 'Z':
info.timezone = 0
j += 1
return
else:
raise newException(ValueError,
"Couldn't parse timezone offset (z), got: " & value[j])
j += 2
of "zz":
info.isDST = false
if value[j] == '+':
info.timezone = 0 - value[j+1..j+2].parseInt() * secondsInHour
elif value[j] == '-':
info.timezone = value[j+1..j+2].parseInt() * secondsInHour
elif value[j] == 'Z':
info.timezone = 0
j += 1
return
else:
raise newException(ValueError,
"Couldn't parse timezone offset (zz), got: " & value[j])
j += 3
of "zzz":
info.isDST = false
var factor = 0
if value[j] == '+': factor = -1
elif value[j] == '-': factor = 1
elif value[j] == 'Z':
info.timezone = 0
j += 1
return
else:
raise newException(ValueError,
"Couldn't parse timezone offset (zzz), got: " & value[j])
@ -1121,8 +1138,11 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) =
j += token.len
proc parse*(value, layout: string): TimeInfo =
## This function parses a date/time string using the standard format identifiers (below)
## The function defaults information not provided in the format string from the running program (timezone, month, year, etc)
## This function parses a date/time string using the standard format
## identifiers as listed below. The function defaults information not provided
## in the format string from the running program (timezone, month, year, etc).
## Daylight saving time is only set if no timezone is given and the given date
## lies within the DST period of the current locale.
##
## ========== ================================================================================= ================================================
## Specifier Description Example
@ -1147,7 +1167,7 @@ proc parse*(value, layout: string): TimeInfo =
## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively.
## yy Displays the year to two digits. ``2012 -> 12``
## yyyy Displays the year to four digits. ``2012 -> 2012``
## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5``
## z Displays the timezone offset from UTC. ``Z`` is parsed as ``+0`` ``GMT+7 -> +7``, ``GMT-5 -> -5``
## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05``
## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00``
## ========== ================================================================================= ================================================
@ -1165,6 +1185,8 @@ proc parse*(value, layout: string): TimeInfo =
info.hour = 0
info.minute = 0
info.second = 0
info.isDST = true # using this is flag for checking whether a timezone has \
# been read (because DST is always false when a tz is parsed)
while true:
case layout[i]
of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',':
@ -1194,16 +1216,17 @@ proc parse*(value, layout: string): TimeInfo =
parseToken(info, token, value, j)
token = ""
# We are going to process the date to find out if we are in DST, because the
# default based on the current time may be wrong. Calling getLocalTime will
# set this correctly, but the actual time may be offset from when we called
# toTime with a possibly incorrect DST setting, so we are only going to take
# the isDST from this result.
let correctDST = getLocalTime(toTime(info))
info.isDST = correctDST.isDST
if info.isDST:
# means that no timezone has been parsed. In this case, we need to check
# whether the date is within DST of the local time.
let tmp = getLocalTime(toTime(info))
# correctly set isDST so that the following step works on the correct time
info.isDST = tmp.isDST
# Now we process it again with the correct isDST to correct things like
# weekday and yearday.
# Correct weekday and yearday; transform timestamp to local time.
# There currently is no way of returning this with the original (parsed)
# timezone while also setting weekday and yearday (we are depending on stdlib
# to provide this calculation).
return getLocalTime(toTime(info))
# Leap year calculations are adapted from:

4
lib/pure/unittest.nim
View file

@ -98,7 +98,7 @@ proc startSuite(name: string) =
template rawPrint() = echo("\n[Suite] ", name)
when not defined(ECMAScript):
if colorOutput:
styledEcho styleBright, fgBlue, "\n[Suite] ", fgWhite, name
styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, name
else: rawPrint()
else: rawPrint()
@ -159,7 +159,7 @@ proc testDone(name: string, s: TestStatus, indent: bool) =
of FAILED: fgRed
of SKIPPED: fgYellow
else: fgWhite
styledEcho styleBright, color, prefix, "[", $s, "] ", fgWhite, name
styledEcho styleBright, color, prefix, "[", $s, "] ", resetStyle, name
else:
rawPrint()
else:

10
lib/system.nim
View file

@ -1272,12 +1272,14 @@ const
seqShallowFlag = low(int)
{.push profiler: off.}
when defined(nimKnowsNimvm):
let nimvm* {.magic: "Nimvm".}: bool = false
## may be used only in "when" expression.
## It is true in Nim VM context and false otherwise
else:
const nimvm*: bool = false
{.pop.}
proc compileOption*(option: string): bool {.
magic: "CompileOption", noSideEffect.}
@ -2544,6 +2546,7 @@ when hostOS == "standalone":
include "$projectpath/panicoverride"
when not declared(sysFatal):
{.push profiler: off.}
when hostOS == "standalone":
proc sysFatal(exceptn: typedesc, message: string) {.inline.} =
panic(message)
@ -2563,6 +2566,7 @@ when not declared(sysFatal):
new(e)
e.msg = message & arg
raise e
{.pop.}
proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.}
## get type information for `x`. Ordinary code should not use this, but
@ -2616,8 +2620,10 @@ when not defined(JS): #and not defined(nimscript):
when declared(setStackBottom):
setStackBottom(locals)
{.push profiler: off.}
var
strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic})
{.pop.}
# ----------------- IO Part ------------------------------------------------
@ -2950,6 +2956,8 @@ when not defined(JS): #and not defined(nimscript):
## lead to the ``raise`` statement. This only works for debug builds.
{.push stack_trace: off, profiler:off.}
when defined(memtracker):
include "system/memtracker"
when hostOS == "standalone":
include "system/embedded"
else:
@ -2992,7 +3000,9 @@ when not defined(JS): #and not defined(nimscript):
else:
result = n.sons[n.len]
{.push profiler:off.}
when hasAlloc: include "system/mmdisp"
{.pop.}
{.push stack_trace: off, profiler:off.}
when hasAlloc: include "system/sysstr"
{.pop.}

6
lib/system/alloc.nim
View file

@ -15,6 +15,10 @@
include osalloc
template track(op, address, size) =
when defined(memTracker):
memTrackerOp(op, address, size)
# We manage *chunks* of memory. Each chunk is a multiple of the page size.
# Each chunk starts at an address that is divisible by the page size. Chunks
# that are bigger than ``ChunkOsReturn`` are returned back to the operating
@ -645,6 +649,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer =
cast[ptr FreeCell](result).zeroField = 1 # mark it as used
sysAssert(not isAllocatedPtr(allocator, result), "alloc")
result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell))
track("alloc", result, size)
proc alloc0(allocator: var MemRegion, size: Natural): pointer =
result = alloc(allocator, size)
@ -658,6 +663,7 @@ proc dealloc(allocator: var MemRegion, p: pointer) =
sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2")
rawDealloc(allocator, x)
sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3")
track("dealloc", p, 0)
proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
if newsize > 0:

111
lib/system/deepcopy.nim
View file

@ -7,18 +7,66 @@
# distribution, for details about the copyright.
#
proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) {.benign.}
proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} =
type
PtrTable = ptr object
counter, max: int
data: array[0..0xff_ffff, (pointer, pointer)]
template hashPtr(key: pointer): int = cast[int](key) shr 8
template allocPtrTable: untyped =
cast[PtrTable](alloc0(sizeof(int)*2 + sizeof(pointer)*2*cap))
proc rehash(t: PtrTable): PtrTable =
let cap = (t.max+1) * 2
result = allocPtrTable()
result.counter = t.counter
result.max = cap-1
for i in 0..t.max:
let k = t.data[i][0]
if k != nil:
var h = hashPtr(k)
while result.data[h and result.max][0] != nil: inc h
result.data[h and result.max] = t.data[i]
dealloc t
proc initPtrTable(): PtrTable =
const cap = 32
result = allocPtrTable()
result.counter = 0
result.max = cap-1
template deinit(t: PtrTable) = dealloc(t)
proc get(t: PtrTable; key: pointer): pointer =
var h = hashPtr(key)
while true:
let k = t.data[h and t.max][0]
if k == nil: break
if k == key:
return t.data[h and t.max][1]
inc h
proc put(t: var PtrTable; key, val: pointer) =
if (t.max+1) * 2 < t.counter * 3: t = rehash(t)
var h = hashPtr(key)
while t.data[h and t.max][0] != nil: inc h
t.data[h and t.max] = (key, val)
inc t.counter
proc genericDeepCopyAux(dest, src: pointer, mt: PNimType;
tab: var PtrTable) {.benign.}
proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode;
tab: var PtrTable) {.benign.} =
var
d = cast[ByteAddress](dest)
s = cast[ByteAddress](src)
case n.kind
of nkSlot:
genericDeepCopyAux(cast[pointer](d +% n.offset),
cast[pointer](s +% n.offset), n.typ)
cast[pointer](s +% n.offset), n.typ, tab)
of nkList:
for i in 0..n.len-1:
genericDeepCopyAux(dest, src, n.sons[i])
genericDeepCopyAux(dest, src, n.sons[i], tab)
of nkCase:
var dd = selectBranch(dest, n)
var m = selectBranch(src, n)
@ -29,10 +77,10 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} =
copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset),
n.typ.size)
if m != nil:
genericDeepCopyAux(dest, src, m)
genericDeepCopyAux(dest, src, m, tab)
of nkNone: sysAssert(false, "genericDeepCopyAux")
proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) =
var
d = cast[ByteAddress](dest)
s = cast[ByteAddress](src)
@ -60,22 +108,22 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize),
cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +%
GenericSeqSize),
mt.base)
mt.base, tab)
of tyObject:
# we need to copy m_type field for tyObject, as it could be empty for
# sequence reallocations:
if mt.base != nil:
genericDeepCopyAux(dest, src, mt.base)
genericDeepCopyAux(dest, src, mt.base, tab)
else:
var pint = cast[ptr PNimType](dest)
pint[] = cast[ptr PNimType](src)[]
genericDeepCopyAux(dest, src, mt.node)
genericDeepCopyAux(dest, src, mt.node, tab)
of tyTuple:
genericDeepCopyAux(dest, src, mt.node)
genericDeepCopyAux(dest, src, mt.node, tab)
of tyArray, tyArrayConstr:
for i in 0..(mt.size div mt.base.size)-1:
genericDeepCopyAux(cast[pointer](d +% i*% mt.base.size),
cast[pointer](s +% i*% mt.base.size), mt.base)
cast[pointer](s +% i*% mt.base.size), mt.base, tab)
of tyRef:
let s2 = cast[PPointer](src)[]
if s2 == nil:
@ -84,30 +132,29 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
let z = mt.base.deepcopy(s2)
unsureAsgnRef(cast[PPointer](dest), z)
else:
# we modify the header of the cell temporarily; instead of the type
# field we store a forwarding pointer. XXX This is bad when the cloning
# fails due to OOM etc.
when declared(usrToCell):
# unfortunately we only have cycle detection for our native GCs.
let x = usrToCell(s2)
let forw = cast[int](x.typ)
if (forw and 1) == 1:
# we stored a forwarding pointer, so let's use that:
let z = cast[pointer](forw and not 1)
unsureAsgnRef(cast[PPointer](dest), z)
else:
let z = tab.get(s2)
if z == nil:
when declared(usrToCell):
let x = usrToCell(s2)
let realType = x.typ
let z = newObj(realType, realType.base.size)
unsureAsgnRef(cast[PPointer](dest), z)
x.typ = cast[PNimType](cast[int](z) or 1)
genericDeepCopyAux(z, s2, realType.base)
x.typ = realType
tab.put(s2, z)
genericDeepCopyAux(z, s2, realType.base, tab)
else:
when false:
# addition check disabled
let x = usrToCell(s2)
let realType = x.typ
sysAssert realType == mt, " types do differ"
# this version should work for any possible GC:
let size = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[].size else: mt.base.size
let z = newObj(mt, size)
unsureAsgnRef(cast[PPointer](dest), z)
tab.put(s2, z)
genericDeepCopyAux(z, s2, mt.base, tab)
else:
let size = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[].size
else: mt.base.size
let z = newObj(mt, size)
unsureAsgnRef(cast[PPointer](dest), z)
genericDeepCopyAux(z, s2, mt.base)
of tyPtr:
# no cycle check here, but also not really required
let s2 = cast[PPointer](src)[]
@ -120,7 +167,9 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) =
proc genericDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} =
GC_disable()
genericDeepCopyAux(dest, src, mt)
var tab = initPtrTable()
genericDeepCopyAux(dest, src, mt, tab)
deinit tab
GC_enable()
proc genericSeqDeepCopy(dest, src: pointer, mt: PNimType) {.compilerProc.} =

2
lib/system/excpt.nim
View file

@ -339,6 +339,8 @@ when not defined(noSignalHandler):
action("unknown signal\n")
# print stack trace and quit
when defined(memtracker):
logPendingOps()
when hasSomeStackTrace:
GC_disable()
var buf = newStringOfCap(2000)

32
lib/system/gc.nim
View file

@ -445,6 +445,15 @@ proc gcInvariant*() =
markForDebug(gch)
{.pop.}
template setFrameInfo(c: PCell) =
when leakDetector:
if framePtr != nil and framePtr.prev != nil:
c.filename = framePtr.prev.filename
c.line = framePtr.prev.line
else:
c.filename = nil
c.line = 0
proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
# generates a new object and sets its reference counter to 0
sysAssert(allocInv(gch.region), "rawNewObj begin")
@ -455,19 +464,14 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
gcAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
# now it is buffered in the ZCT
res.typ = typ
when leakDetector:
res.filename = nil
res.line = 0
when not hasThreadSupport:
if framePtr != nil and framePtr.prev != nil:
res.filename = framePtr.prev.filename
res.line = framePtr.prev.line
setFrameInfo(res)
# refcount is zero, color is black, but mark it to be in the ZCT
res.refcount = ZctFlag
sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
# its refcount is zero, so add it to the ZCT:
addNewObjToZCT(res, gch)
when logGC: writeCell("new cell", res)
track("rawNewObj", res, size)
gcTrace(res, csAllocated)
release(gch)
when useCellIds:
@ -509,16 +513,11 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "newObj: 2")
# now it is buffered in the ZCT
res.typ = typ
when leakDetector:
res.filename = nil
res.line = 0
when not hasThreadSupport:
if framePtr != nil and framePtr.prev != nil:
res.filename = framePtr.prev.filename
res.line = framePtr.prev.line
setFrameInfo(res)
res.refcount = rcIncrement # refcount is 1
sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3")
when logGC: writeCell("new cell", res)
track("newObjRC1", res, size)
gcTrace(res, csAllocated)
release(gch)
when useCellIds:
@ -561,6 +560,8 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer =
writeCell("growObj new cell", res)
gcTrace(ol, csZctFreed)
gcTrace(res, csAllocated)
track("growObj old", ol, 0)
track("growObj new", res, newsize)
when reallyDealloc:
sysAssert(allocInv(gch.region), "growObj before dealloc")
if ol.refcount shr rcShift <=% 1:
@ -604,6 +605,7 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} =
proc freeCyclicCell(gch: var GcHeap, c: PCell) =
prepareDealloc(c)
gcTrace(c, csCycFreed)
track("cycle collector dealloc cell", c, 0)
when logGC: writeCell("cycle collector dealloc cell", c)
when reallyDealloc:
sysAssert(allocInv(gch.region), "free cyclic cell")
@ -673,6 +675,7 @@ proc doOperation(p: pointer, op: WalkOp) =
gcAssert(c.refcount >=% rcIncrement, "doOperation 2")
#c.refcount = c.refcount -% rcIncrement
when logGC: writeCell("decref (from doOperation)", c)
track("waZctDecref", p, 0)
decRef(c)
#if c.refcount <% rcIncrement: addZCT(gch.zct, c)
of waPush:
@ -765,6 +768,7 @@ proc collectZCT(gch: var GcHeap): bool =
# In any case, it should be removed from the ZCT. But not
# freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!**
when logGC: writeCell("zct dealloc cell", c)
track("zct dealloc cell", c, 0)
gcTrace(c, csZctFreed)
# We are about to free the object, call the finalizer BEFORE its
# children are deleted as well, because otherwise the finalizer may

2
lib/system/hti.nim
View file

@ -86,6 +86,8 @@ type
finalizer: pointer # the finalizer for the type
marker: proc (p: pointer, op: int) {.nimcall, benign.} # marker proc for GC
deepcopy: proc (p: pointer): pointer {.nimcall, benign.}
when defined(nimTypeNames):
name: cstring
PNimType = ptr TNimType
# node.len may be the ``first`` element of a set

70
lib/system/memtracker.nim Normal file
View file

@ -0,0 +1,70 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2016 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Memory tracking support for Nim.
when not defined(memTracker):
{.error: "Memory tracking support is turned off! Enable memory tracking by passing `--memtracker:on` to the compiler (see the Nim Compiler User Guide for more options).".}
when defined(noSignalHandler):
{.error: "Memory tracking works better with the default signal handler.".}
# We don't want to memtrack the tracking code ...
{.push memtracker: off.}
type
LogEntry* = object
op*: cstring
address*: pointer
size*: int
file*: cstring
line*: int
TrackLog* = object
count*: int
disabled: bool
data*: array[4000, LogEntry]
TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], locks: 0.}
var
gLog*: TrackLog
gLogger*: TrackLogger = proc (log: TrackLog) = discard
proc setTrackLogger*(logger: TrackLogger) =
gLogger = logger
proc addEntry(entry: LogEntry) =
if not gLog.disabled:
if gLog.count > high(gLog.data):
gLogger(gLog)
gLog.count = 0
gLog.data[gLog.count] = entry
inc gLog.count
proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.compilerProc.} =
addEntry LogEntry(op: "write", address: address,
size: size, file: file, line: line)
proc memTrackerOp*(op: cstring; address: pointer; size: int) =
addEntry LogEntry(op: op, address: address, size: size,
file: "", line: 0)
proc memTrackerDisable*() =
gLog.disabled = true
proc memTrackerEnable*() =
gLog.disabled = false
proc logPendingOps() {.noconv.} =
# forward declared and called from Nim's signal handler.
gLogger(gLog)
gLog.count = 0
addQuitProc logPendingOps
{.pop.}

51
lib/system/sysstr.nim
View file

@ -263,27 +263,32 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
result.len = newLen
# --------------- other string routines ----------------------------------
proc nimIntToStr(x: int): string {.compilerRtl.} =
result = newString(sizeof(x)*4)
proc add*(result: var string; x: int64) =
let base = result.len
setLen(result, base + sizeof(x)*4)
var i = 0
var y = x
while true:
var d = y div 10
result[i] = chr(abs(int(y - d*10)) + ord('0'))
result[base+i] = chr(abs(int(y - d*10)) + ord('0'))
inc(i)
y = d
if y == 0: break
if x < 0:
result[i] = '-'
result[base+i] = '-'
inc(i)
setLen(result, i)
setLen(result, base+i)
# mirror the string:
for j in 0..i div 2 - 1:
swap(result[j], result[i-j-1])
swap(result[base+j], result[base+i-j-1])
proc nimFloatToStr(f: float): string {.compilerproc.} =
proc nimIntToStr(x: int): string {.compilerRtl.} =
result = newStringOfCap(sizeof(x)*4)
result.add x
proc add*(result: var string; x: float) =
var buf: array[0..64, char]
var n: int = c_sprintf(buf, "%.16g", f)
var n: int = c_sprintf(buf, "%.16g", x)
var hasDot = false
for i in 0..n-1:
if buf[i] == ',':
@ -298,14 +303,18 @@ proc nimFloatToStr(f: float): string {.compilerproc.} =
# On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' are produced.
# We want to get rid of these here:
if buf[n-1] in {'n', 'N'}:
result = "nan"
result.add "nan"
elif buf[n-1] == 'F':
if buf[0] == '-':
result = "-inf"
result.add "-inf"
else:
result = "inf"
result.add "inf"
else:
result = $buf
result.add buf
proc nimFloatToStr(f: float): string {.compilerproc.} =
result = newStringOfCap(8)
result.add f
proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {.
importc: "strtod", header: "<stdlib.h>", noSideEffect.}
@ -469,22 +478,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
number = c_strtod(t, nil)
proc nimInt64ToStr(x: int64): string {.compilerRtl.} =
result = newString(sizeof(x)*4)
var i = 0
var y = x
while true:
var d = y div 10
result[i] = chr(abs(int(y - d*10)) + ord('0'))
inc(i)
y = d
if y == 0: break
if x < 0:
result[i] = '-'
inc(i)
setLen(result, i)
# mirror the string:
for j in 0..i div 2 - 1:
swap(result[j], result[i-j-1])
result = newStringOfCap(sizeof(x)*4)
result.add x
proc nimBoolToStr(x: bool): string {.compilerRtl.} =
return if x: "true" else: "false"

106
lib/upcoming/asyncdispatch.nim
View file

@ -9,9 +9,9 @@
include "system/inclrtl"
import os, oids, tables, strutils, times, heapqueue
import os, oids, tables, strutils, times, heapqueue, lists
import nativesockets, net, queues
import nativesockets, net, deques
export Port, SocketFlag
@ -135,7 +135,7 @@ include "../includes/asyncfutures"
type
PDispatcherBase = ref object of RootRef
timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
callbacks: Queue[proc ()]
callbacks: Deque[proc ()]
proc processTimers(p: PDispatcherBase) {.inline.} =
while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt:
@ -143,7 +143,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} =
proc processPendingCallbacks(p: PDispatcherBase) =
while p.callbacks.len > 0:
var cb = p.callbacks.dequeue()
var cb = p.callbacks.popFirst()
cb()
proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
@ -729,7 +729,7 @@ when defined(windows) or defined(nimdoc):
var lpOutputBuf = newString(lpOutputLen)
var dwBytesReceived: Dword
let dwReceiveDataLength = 0.Dword # We don't want any data to be read.
let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16)
let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16)
let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16)
template completeAccept() {.dirty.} =
@ -1095,9 +1095,11 @@ else:
AsyncFD* = distinct cint
Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.}
DoublyLinkedListRef = ref DoublyLinkedList[Callback]
AsyncData = object
readCB: Callback
writeCB: Callback
readCBs: DoublyLinkedListRef
writeCBs: DoublyLinkedListRef
AsyncEvent* = distinct SelectEvent
@ -1112,7 +1114,7 @@ else:
new result
result.selector = newSelector[AsyncData]()
result.timers.newHeapQueue()
result.callbacks = initQueue[proc ()](64)
result.callbacks = initDeque[proc ()](64)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
@ -1121,7 +1123,10 @@ else:
proc register*(fd: AsyncFD) =
let p = getGlobalDispatcher()
var data = AsyncData()
var data = AsyncData(
readCBs: DoublyLinkedListRef(),
writeCBs: DoublyLinkedListRef()
)
p.selector.registerHandle(fd.SocketHandle, {}, data)
proc newAsyncNativeSocket*(domain: cint, sockType: cint,
@ -1156,8 +1161,9 @@ else:
let p = getGlobalDispatcher()
var newEvents = {Event.Read}
withData(p.selector, fd.SocketHandle, adata) do:
adata.readCB = cb
if adata.writeCB != nil:
adata.readCBs[].append(cb)
newEvents.incl(Event.Read)
if not isNil(adata.writeCBs.head):
newEvents.incl(Event.Write)
do:
raise newException(ValueError, "File descriptor not registered.")
@ -1167,8 +1173,9 @@ else:
let p = getGlobalDispatcher()
var newEvents = {Event.Write}
withData(p.selector, fd.SocketHandle, adata) do:
adata.writeCB = cb
if adata.readCB != nil:
adata.writeCBs[].append(cb)
newEvents.incl(Event.Write)
if not isNil(adata.readCBs.head):
newEvents.incl(Event.Read)
do:
raise newException(ValueError, "File descriptor not registered.")
@ -1195,31 +1202,32 @@ else:
let events = keys[i].events
if Event.Read in events or events == {Event.Error}:
let cb = keys[i].data.readCB
if cb != nil:
if cb(fd.AsyncFD):
p.selector.withData(fd, adata) do:
if adata.readCB == cb:
adata.readCB = nil
for node in keys[i].data.readCBs[].nodes():
let cb = node.value
if cb != nil:
if cb(fd.AsyncFD):
keys[i].data.readCBs[].remove(node)
else:
break
if Event.Write in events or events == {Event.Error}:
let cb = keys[i].data.writeCB
if cb != nil:
if cb(fd.AsyncFD):
p.selector.withData(fd, adata) do:
if adata.writeCB == cb:
adata.writeCB = nil
for node in keys[i].data.writeCBs[].nodes():
let cb = node.value
if cb != nil:
if cb(fd.AsyncFD):
keys[i].data.writeCBs[].remove(node)
else:
break
when supportedPlatform:
if (customSet * events) != {}:
let cb = keys[i].data.readCB
doAssert(cb != nil)
custom = true
if cb(fd.AsyncFD):
p.selector.withData(fd, adata) do:
if adata.readCB == cb:
adata.readCB = nil
p.selector.unregister(fd)
for node in keys[i].data.readCBs[].nodes():
let cb = node.value
doAssert(cb != nil)
custom = true
if cb(fd.AsyncFD):
keys[i].data.readCBs[].remove(node)
p.selector.unregister(fd)
# because state `data` can be modified in callback we need to update
# descriptor events with currently registered callbacks.
@ -1227,8 +1235,8 @@ else:
var update = false
var newEvents: set[Event] = {}
p.selector.withData(fd, adata) do:
if adata.readCB != nil: incl(newEvents, Event.Read)
if adata.writeCB != nil: incl(newEvents, Event.Write)
if not isNil(adata.readCBs.head): incl(newEvents, Event.Read)
if not isNil(adata.writeCBs.head): incl(newEvents, Event.Write)
update = true
if update:
p.selector.updateHandle(fd, newEvents)
@ -1491,21 +1499,33 @@ else:
## ``oneshot`` - if ``true`` only one event will be dispatched,
## if ``false`` continuous events every ``timeout`` milliseconds.
let p = getGlobalDispatcher()
var data = AsyncData(readCB: cb)
var data = AsyncData(
readCBs: DoublyLinkedListRef(),
writeCBs: DoublyLinkedListRef()
)
data.readCBs[].append(cb)
p.selector.registerTimer(timeout, oneshot, data)
proc addSignal*(signal: int, cb: Callback) =
## Start watching signal ``signal``, and when signal appears, call the
## callback ``cb``.
let p = getGlobalDispatcher()
var data = AsyncData(readCB: cb)
var data = AsyncData(
readCBs: DoublyLinkedListRef(),
writeCBs: DoublyLinkedListRef()
)
data.readCBs[].append(cb)
p.selector.registerSignal(signal, data)
proc addProcess*(pid: int, cb: Callback) =
## Start watching for process exit with pid ``pid``, and then call
## the callback ``cb``.
let p = getGlobalDispatcher()
var data = AsyncData(readCB: cb)
var data = AsyncData(
readCBs: DoublyLinkedListRef(),
writeCBs: DoublyLinkedListRef()
)
data.readCBs[].append(cb)
p.selector.registerProcess(pid, data)
proc newAsyncEvent*(): AsyncEvent =
@ -1524,7 +1544,11 @@ else:
## Start watching for event ``ev``, and call callback ``cb``, when
## ev will be set to signaled state.
let p = getGlobalDispatcher()
var data = AsyncData(readCB: cb)
var data = AsyncData(
readCBs: DoublyLinkedListRef(),
writeCBs: DoublyLinkedListRef()
)
data.readCBs[].append(cb)
p.selector.registerEvent(SelectEvent(ev), data)
proc sleepAsync*(ms: int): Future[void] =
@ -1591,7 +1615,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
## **Note**: This procedure is mostly used for testing. You likely want to
## use ``asyncnet.recvLine`` instead.
template addNLIfEmpty(): stmt =
template addNLIfEmpty(): typed =
if result.len == 0:
result.add("\c\L")
@ -1614,7 +1638,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
proc callSoon*(cbproc: proc ()) =
## Schedule `cbproc` to be called as soon as possible.
## The callback is called when control returns to the event loop.
getGlobalDispatcher().callbacks.enqueue(cbproc)
getGlobalDispatcher().callbacks.addLast(cbproc)
proc runForever*() =
## Begins a never ending global dispatcher poll loop.

9
lib/wrappers/openssl.nim
View file

@ -261,11 +261,14 @@ proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl,
proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.}
proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.}
proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".}
when defined(android):
template OpenSSL_add_all_algorithms*() = discard
else:
proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".}
proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.}
when not useWinVersion and not defined(macosx):
when not useWinVersion and not defined(macosx) and not defined(android):
proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl,
dynlib: DLLUtilName, importc.}
@ -279,7 +282,7 @@ when not useWinVersion and not defined(macosx):
if p != nil: dealloc(p)
proc CRYPTO_malloc_init*() =
when not useWinVersion and not defined(macosx):
when not useWinVersion and not defined(macosx) and not defined(android):
CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper)
proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cInt, larg: int, parg: pointer): int{.

13
lib/wrappers/sqlite3.nim
View file

@ -96,10 +96,6 @@ const
#define SQLITE_TRANSIENT ((void(*)(void *))-1)
SQLITE_DETERMINISTIC* = 0x800
const
SQLITE_STATIC* = nil
SQLITE_TRANSIENT* = cast[pointer](- 1)
type
Sqlite3 {.pure, final.} = object
PSqlite3* = ptr Sqlite3
@ -114,7 +110,7 @@ type
Callback* = proc (para1: pointer, para2: int32, para3,
para4: cstringArray): int32{.cdecl.}
Tbind_destructor_func* = proc (para1: pointer){.cdecl.}
Tbind_destructor_func* = proc (para1: pointer){.cdecl, locks: 0, tags: [].}
Create_function_step_func* = proc (para1: Pcontext, para2: int32,
para3: PValueArg){.cdecl.}
Create_function_func_func* = proc (para1: Pcontext, para2: int32,
@ -132,6 +128,10 @@ type
Tresult_func: Result_func, Tcreate_collation_func: Create_collation_func,
Tcollation_needed_func: Collation_needed_func].}
const
SQLITE_STATIC* = nil
SQLITE_TRANSIENT* = cast[Tbind_destructor_func](-1)
proc close*(para1: PSqlite3): int32{.cdecl, dynlib: Lib, importc: "sqlite3_close".}
proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer,
errmsg: var cstring): int32{.cdecl, dynlib: Lib,
@ -234,7 +234,8 @@ proc bind_parameter_name*(para1: Pstmt, para2: int32): cstring{.cdecl,
dynlib: Lib, importc: "sqlite3_bind_parameter_name".}
proc bind_parameter_index*(para1: Pstmt, zName: cstring): int32{.cdecl,
dynlib: Lib, importc: "sqlite3_bind_parameter_index".}
#function sqlite3_clear_bindings(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_clear_bindings';
proc clear_bindings*(para1: Pstmt): int32 {.cdecl,
dynlib: Lib, importc: "sqlite3_clear_bindings".}
proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib,
importc: "sqlite3_column_count".}
proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib,