jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'devel' into araq

Browse source
This commit is contained in:
Andreas Rumpf 2017-09-15 09:27:51 +02:00
commit 39f0195ebf
334 changed files with 4636 additions and 2763 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/core/locks.nim
View file

@ -19,6 +19,8 @@ type
{.deprecated: [TLock: Lock, TCond: Cond].}
{.push stackTrace: off.}
proc initLock*(lock: var Lock) {.inline.} =
## Initializes the given lock.
initSysLock(lock)
@ -59,9 +61,12 @@ proc signal*(cond: var Cond) {.inline.} =
template withLock*(a: Lock, body: untyped) =
## Acquires the given lock, executes the statements in body and
## releases the lock after the statements finish executing.
mixin acquire, release
a.acquire()
{.locks: [a].}:
try:
body
finally:
a.release()
{.pop.}

150
lib/core/macros.nim
View file

@ -94,8 +94,9 @@ type
ntyVarargs,
ntyUnused,
ntyError,
ntyBuiltinTypeClass, ntyConcept, ntyConceptInst, ntyComposite,
ntyAnd, ntyOr, ntyNot
ntyBuiltinTypeClass, ntyUserTypeClass, ntyUserTypeClassInst,
ntyCompositeTypeClass, ntyInferred, ntyAnd, ntyOr, ntyNot,
ntyAnything, ntyStatic, ntyFromExpr, ntyFieldAccessor, ntyVoid
TNimTypeKinds* {.deprecated.} = set[NimTypeKind]
NimSymKind* = enum
@ -149,6 +150,9 @@ proc `==`*(a, b: NimIdent): bool {.magic: "EqIdent", noSideEffect.}
proc `==`*(a, b: NimNode): bool {.magic: "EqNimrodNode", noSideEffect.}
## compares two Nim nodes
proc `==`*(a, b: NimSym): bool {.magic: "EqNimrodNode", noSideEffect.}
## compares two Nim symbols
proc sameType*(a, b: NimNode): bool {.magic: "SameNodeType", noSideEffect.} =
## compares two Nim nodes' types. Return true if the types are the same,
## eg. true when comparing alias with original type.
@ -317,10 +321,30 @@ proc toStrLit*(n: NimNode): NimNode {.compileTime.} =
## in a string literal node
return newStrLitNode(repr(n))
proc lineinfo*(n: NimNode): string {.magic: "NLineInfo", noSideEffect.}
type
LineInfo* = object
filename*: string
line*,column*: int
proc `$`*(arg: Lineinfo): string =
result = arg.filename & "(" & $arg.line & ", " & $arg.column & ")"
#proc lineinfo*(n: NimNode): LineInfo {.magic: "NLineInfo", noSideEffect.}
## returns the position the node appears in the original source file
## in the form filename(line, col)
proc getLine(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.}
proc getColumn(arg: NimNode): int {.magic: "NLineInfo", noSideEffect.}
proc getFile(arg: NimNode): string {.magic: "NLineInfo", noSideEffect.}
proc lineInfoObj*(n: NimNode): LineInfo {.compileTime.} =
result.filename = n.getFile
result.line = n.getLine
result.column = n.getColumn
proc lineInfo*(arg: NimNode): string {.compileTime.} =
$arg.lineInfoObj
proc internalParseExpr(s: string): NimNode {.
magic: "ParseExprToAst", noSideEffect.}
@ -527,10 +551,17 @@ proc newLit*[N,T](arg: array[N,T]): NimNode {.compileTime.} =
result.add newLit(x)
proc newLit*[T](arg: seq[T]): NimNode {.compileTime.} =
result = nnkBracket.newTree
var bracket = nnkBracket.newTree
for x in arg:
result.add newLit(x)
result = nnkPrefix.newTree(bindSym"@", result)
bracket.add newLit(x)
result = nnkCall.newTree(
nnkBracketExpr.newTree(
nnkAccQuoted.newTree( bindSym"@" ),
getTypeInst( bindSym"T" )
),
bracket
)
proc newLit*(arg: tuple): NimNode {.compileTime.} =
result = nnkPar.newTree
@ -557,7 +588,8 @@ proc nestList*(theProc: NimIdent,
proc treeRepr*(n: NimNode): string {.compileTime, benign.} =
## Convert the AST `n` to a human-readable tree-like string.
##
## See also `repr` and `lispRepr`.
## See also `repr`, `lispRepr`, and `astGenRepr`.
proc traverse(res: var string, level: int, n: NimNode) {.benign.} =
for i in 0..level-1: res.add " "
res.add(($n.kind).substr(3))
@ -582,7 +614,7 @@ proc treeRepr*(n: NimNode): string {.compileTime, benign.} =
proc lispRepr*(n: NimNode): string {.compileTime, benign.} =
## Convert the AST `n` to a human-readable lisp-like string,
##
## See also `repr` and `treeRepr`.
## See also `repr`, `treeRepr`, and `astGenRepr`.
result = ($n.kind).substr(3)
add(result, "(")
@ -605,9 +637,96 @@ proc lispRepr*(n: NimNode): string {.compileTime, benign.} =
add(result, ")")
proc astGenRepr*(n: NimNode): string {.compileTime, benign.} =
## Convert the AST `n` to the code required to generate that AST. So for example
##
## .. code-block:: nim
## astGenRepr:
## echo "Hello world"
##
## Would output:
##
## .. code-block:: nim
## nnkStmtList.newTree(
## nnkCommand.newTree(
## newIdentNode(!"echo"),
## newLit("Hello world")
## )
## )
##
## See also `repr`, `treeRepr`, and `lispRepr`.
const
NodeKinds = {nnkEmpty, nnkNilLit, nnkIdent, nnkSym, nnkNone}
LitKinds = {nnkCharLit..nnkInt64Lit, nnkFloatLit..nnkFloat64Lit, nnkStrLit..nnkTripleStrLit}
proc escape(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect.} =
## Functions copied from strutils
proc toHex(x: BiggestInt, len: Positive): string {.noSideEffect, rtl.} =
const
HexChars = "0123456789ABCDEF"
var
t = x
result = newString(len)
for j in countdown(len-1, 0):
result[j] = HexChars[t and 0xF]
t = t shr 4
# handle negative overflow
if t == 0 and x < 0: t = -1
result = newStringOfCap(s.len + s.len shr 2)
result.add(prefix)
for c in items(s):
case c
of '\0'..'\31', '\128'..'\255':
add(result, "\\x")
add(result, toHex(ord(c), 2))
of '\\': add(result, "\\\\")
of '\'': add(result, "\\'")
of '\"': add(result, "\\\"")
else: add(result, c)
add(result, suffix)
proc traverse(res: var string, level: int, n: NimNode) {.benign.} =
for i in 0..level-1: res.add " "
if n.kind in NodeKinds:
res.add("new" & ($n.kind).substr(3) & "Node(")
elif n.kind in LitKinds:
res.add("newLit(")
else:
res.add($n.kind)
case n.kind
of nnkEmpty: discard
of nnkNilLit: res.add("nil")
of nnkCharLit: res.add("'" & $chr(n.intVal) & "'")
of nnkIntLit..nnkInt64Lit: res.add($n.intVal)
of nnkFloatLit..nnkFloat64Lit: res.add($n.floatVal)
of nnkStrLit..nnkTripleStrLit: res.add($n.strVal.escape())
of nnkIdent: res.add("!" & ($n.ident).escape())
of nnkSym: res.add(($n.symbol).escape())
of nnkNone: assert false
else:
res.add(".newTree(")
for j in 0..<n.len:
res.add "\n"
traverse(res, level + 1, n[j])
if j != n.len-1:
res.add(",")
res.add("\n")
for i in 0..level-1: res.add " "
res.add(")")
if n.kind in NodeKinds+LitKinds:
res.add(")")
result = ""
traverse(result, 0, n)
macro dumpTree*(s: untyped): untyped = echo s.treeRepr
## Accepts a block of nim code and prints the parsed abstract syntax
## tree using the `toTree` function. Printing is done *at compile time*.
## tree using the `treeRepr` function. Printing is done *at compile time*.
##
## You can use this as a tool to explore the Nim's abstract syntax
## tree and to discover what kind of nodes must be created to represent
@ -615,7 +734,16 @@ macro dumpTree*(s: untyped): untyped = echo s.treeRepr
macro dumpLisp*(s: untyped): untyped = echo s.lispRepr
## Accepts a block of nim code and prints the parsed abstract syntax
## tree using the `toLisp` function. Printing is done *at compile time*.
## tree using the `lispRepr` function. Printing is done *at compile time*.
##
## See `dumpTree`.
macro dumpAstGen*(s: untyped): untyped = echo s.astGenRepr
## Accepts a block of nim code and prints the parsed abstract syntax
## tree using the `astGenRepr` function. Printing is done *at compile time*.
##
## You can use this as a tool to write macros quicker by writing example
## outputs and then copying the snippets into the macro for modification.
##
## See `dumpTree`.
@ -766,6 +894,8 @@ template expectRoutine(node: NimNode) =
proc name*(someProc: NimNode): NimNode {.compileTime.} =
someProc.expectRoutine
result = someProc[0]
if result.kind == nnkPostfix:
result = result[1]
proc `name=`*(someProc: NimNode; val: NimNode) {.compileTime.} =
someProc.expectRoutine
someProc[0] = val

2
lib/core/typeinfo.nim
View file

@ -97,7 +97,7 @@ proc newObj(typ: PNimType, size: int): pointer {.importCompilerProc.}
proc newSeq(typ: PNimType, len: int): pointer {.importCompilerProc.}
proc objectInit(dest: pointer, typ: PNimType) {.importCompilerProc.}
template `+!!`(a, b: expr): expr = cast[pointer](cast[ByteAddress](a) + b)
template `+!!`(a, b): untyped = cast[pointer](cast[ByteAddress](a) + b)
proc getDiscriminant(aa: pointer, n: ptr TNimNode): int =
assert(n.kind == nkCase)

37
lib/deprecated/pure/sockets.nim
View file

@ -42,6 +42,8 @@ from times import epochTime
when defined(ssl):
import openssl
else:
type SSLAcceptResult = int
when defined(Windows):
import winlean
@ -206,16 +208,16 @@ proc htons*(x: int16): int16 =
## order, this is a no-op; otherwise, it performs a 2-byte swap operation.
result = sockets.ntohs(x)
template ntohl(x: uint32): expr =
template ntohl(x: uint32): uint32 =
cast[uint32](sockets.ntohl(cast[int32](x)))
template ntohs(x: uint16): expr =
template ntohs(x: uint16): uint16 =
cast[uint16](sockets.ntohs(cast[int16](x)))
template htonl(x: uint32): expr =
template htonl(x: uint32): uint32 =
sockets.ntohl(x)
template htons(x: uint16): expr =
template htons(x: uint16): uint16 =
sockets.ntohs(x)
when defined(Posix):
@ -442,14 +444,13 @@ proc parseIp4*(s: string): BiggestInt =
if s[i] != '\0': invalidIp4(s)
result = BiggestInt(a shl 24 or b shl 16 or c shl 8 or d)
template gaiNim(a, p, h, list: expr): stmt =
block:
var gaiResult = getaddrinfo(a, $p, addr(h), list)
if gaiResult != 0'i32:
when defined(windows):
raiseOSError(osLastError())
else:
raiseOSError(osLastError(), $gai_strerror(gaiResult))
template gaiNim(a, p, h, list: untyped): untyped =
var gaiResult = getaddrinfo(a, $p, addr(h), list)
if gaiResult != 0'i32:
when defined(windows):
raiseOSError(osLastError())
else:
raiseOSError(osLastError(), $gai_strerror(gaiResult))
proc bindAddr*(socket: Socket, port = Port(0), address = "") {.
tags: [ReadIOEffect].} =
@ -493,8 +494,8 @@ proc getSockName*(socket: Socket): Port =
raiseOSError(osLastError())
result = Port(sockets.ntohs(name.sin_port))
template acceptAddrPlain(noClientRet, successRet: expr,
sslImplementation: stmt): stmt {.immediate.} =
template acceptAddrPlain(noClientRet, successRet: SSLAcceptResult or int,
sslImplementation: untyped): untyped =
assert(client != nil)
var sockAddress: Sockaddr_in
var addrLen = sizeof(sockAddress).SockLen
@ -550,7 +551,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string) {.
##
## **Warning:** When using SSL with non-blocking sockets, it is best to use
## the acceptAddrSSL procedure as this procedure will most likely block.
acceptAddrPlain(-1, -1):
acceptAddrPlain(SSLAcceptResult(-1), SSLAcceptResult(-1)):
when defined(ssl):
if server.isSSL:
# We must wrap the client sock in a ssl context.
@ -594,7 +595,7 @@ when defined(ssl):
##
## ``AcceptNoClient`` will be returned when no client is currently attempting
## to connect.
template doHandshake(): stmt =
template doHandshake(): untyped =
when defined(ssl):
if server.isSSL:
client.setBlocking(false)
@ -1278,7 +1279,7 @@ proc recvLine*(socket: Socket, line: var TaintedString, timeout = -1): bool {.
## **Deprecated since version 0.9.2**: This function has been deprecated in
## favour of readLine.
template addNLIfEmpty(): stmt =
template addNLIfEmpty(): untyped =
if line.len == 0:
line.add("\c\L")
@ -1319,7 +1320,7 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1) {.
## A timeout can be specified in milliseconds, if data is not received within
## the specified time an ETimeout exception will be raised.
template addNLIfEmpty(): stmt =
template addNLIfEmpty(): untyped =
if line.len == 0:
line.add("\c\L")

2
lib/impure/nre/private/util.nim
View file

@ -16,7 +16,7 @@ proc checkNil(arg: string): string =
else:
return arg
template formatStr*(howExpr, namegetter, idgetter: expr): expr =
template formatStr*(howExpr, namegetter, idgetter): untyped =
let how = howExpr
var val = newStringOfCap(how.len)
var i = 0

7
lib/js/jsffi.nim
View file

@ -408,20 +408,19 @@ macro `{}`*(typ: typedesc, xs: varargs[untyped]): auto =
kString = quote do:
when compiles($`k`): $`k` else: "invalid"
v = x[1]
body.add(quote do:
body.add quote do:
when compiles(`a`.`k`):
`a`.`k` = `v`
elif compiles(`a`[`k`]):
`a`[`k`] = `v`
else:
`a`[`kString`] = `v`
)
else:
error("Expression `" & $x.toStrLit & "` not allowed in `{}` macro")
body.add(quote do:
body.add quote do:
return `a`
)
result = quote do:
proc inner(): `typ` {.gensym.} =

4
lib/packages/docutils/rstgen.nim
View file

@ -765,7 +765,7 @@ proc renderTocEntries*(d: var RstGenerator, j: var int, lvl: int,
result.add(tmp)
proc renderImage(d: PDoc, n: PRstNode, result: var string) =
template valid(s): expr =
template valid(s): bool =
s.len > 0 and allCharsInSet(s, {'.','/',':','%','_','\\','\128'..'\xFF'} +
Digits + Letters + WhiteSpace)
let
@ -1194,7 +1194,7 @@ proc defaultConfig*(): StringTableRef =
## ``rstToHtml`` to generate the bare minimum HTML.
result = newStringTable(modeStyleInsensitive)
template setConfigVar(key, val: expr) =
template setConfigVar(key, val) =
result[key] = val
# If you need to modify these values, it might be worth updating the template

2
lib/posix/posix.nim
View file

@ -92,7 +92,7 @@ else:
# There used to be this name in posix.nim a long time ago, not sure why!
{.deprecated: [cSIG_HOLD: SIG_HOLD].}
when not defined(macosx):
when not defined(macosx) and not defined(android):
proc st_atime*(s: Stat): Time {.inline.} =
## Second-granularity time of last access
result = s.st_atim.tv_sec

11
lib/posix/posix_other.nim
View file

@ -15,7 +15,7 @@ const
hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays
hasAioH = defined(linux)
when defined(linux):
when defined(linux) and not defined(android):
# On Linux:
# timer_{create,delete,settime,gettime},
# clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt
@ -46,9 +46,9 @@ type
d_ino*: Ino ## File serial number.
when defined(dragonfly):
# DragonflyBSD doesn't have `d_reclen` field.
d_type*: uint8
d_type*: uint8
elif defined(linux) or defined(macosx) or defined(freebsd) or
defined(netbsd) or defined(openbsd):
defined(netbsd) or defined(openbsd) or defined(genode):
d_reclen*: cshort ## Length of this record. (not POSIX)
d_type*: int8 ## Type of file; not supported by all filesystem types.
## (not POSIX)
@ -215,7 +215,7 @@ type
## For a typed memory object, the length in bytes.
## For other file types, the use of this field is
## unspecified.
when defined(macosx):
when defined(macosx) or defined(android):
st_atime*: Time ## Time of last access.
st_mtime*: Time ## Time of last data modification.
st_ctime*: Time ## Time of last status change.
@ -572,7 +572,8 @@ else:
MAP_POPULATE*: cint = 0
when defined(linux) or defined(nimdoc):
when defined(alpha) or defined(mips) or defined(parisc) or
when defined(alpha) or defined(mips) or defined(mipsel) or
defined(mips64) or defined(mips64el) or defined(parisc) or
defined(sparc) or defined(nimdoc):
const SO_REUSEPORT* = cint(0x0200)
## Multiple binding: load balancing on incoming TCP connections

2
lib/posix/termios.nim
View file

@ -174,7 +174,7 @@ var
# Compare a character C to a value VAL from the `cc' array in a
# `struct termios'. If VAL is _POSIX_VDISABLE, no character can match it.
template cceq*(val, c: expr): expr =
template cceq*(val, c): untyped =
c == val and val != POSIX_VDISABLE
# Return the output baud rate stored in *TERMIOS_P.

36
lib/pure/asyncdispatch.nim
View file

@ -9,11 +9,12 @@
include "system/inclrtl"
import os, tables, strutils, times, heapqueue, options
import os, tables, strutils, times, heapqueue, options, asyncstreams
import asyncfutures except callSoon
import nativesockets, net, deques
export Port, SocketFlag
export asyncfutures, asyncstreams
#{.injectStmt: newGcInvariant().}
@ -159,8 +160,6 @@ export Port, SocketFlag
# TODO: Check if yielded future is nil and throw a more meaningful exception
include includes/asyncfutures
type
PDispatcherBase = ref object of RootRef
timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
@ -190,6 +189,12 @@ proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
result = int((timerTimeout - curTime) * 1000)
if result < 0: result = 0
proc callSoon(cbproc: proc ()) {.gcsafe.}
proc initCallSoonProc =
if asyncfutures.getCallSoonProc().isNil:
asyncfutures.setCallSoonProc(callSoon)
when defined(windows) or defined(nimdoc):
import winlean, sets, hashes
type
@ -237,15 +242,17 @@ when defined(windows) or defined(nimdoc):
result.callbacks = initDeque[proc ()](64)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
## Retrieves the global thread-local dispatcher.
if gDisp.isNil: gDisp = newDispatcher()
result = gDisp
proc setGlobalDispatcher*(disp: PDispatcher) =
if not gDisp.isNil:
assert gDisp.callbacks.len == 0
gDisp = disp
initCallSoonProc()
proc getGlobalDispatcher*(): PDispatcher =
if gDisp.isNil:
setGlobalDispatcher(newDispatcher())
result = gDisp
proc register*(fd: AsyncFD) =
## Registers ``fd`` with the dispatcher.
@ -932,14 +939,17 @@ else:
result.callbacks = initDeque[proc ()](64)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
if gDisp.isNil: gDisp = newDispatcher()
result = gDisp
proc setGlobalDispatcher*(disp: PDispatcher) =
if not gDisp.isNil:
assert gDisp.callbacks.len == 0
gDisp = disp
initCallSoonProc()
proc getGlobalDispatcher*(): PDispatcher =
if gDisp.isNil:
setGlobalDispatcher(newDispatcher())
result = gDisp
proc update(fd: AsyncFD, events: set[Event]) =
let p = getGlobalDispatcher()
@ -1307,7 +1317,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
##
## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead.
template addNLIfEmpty(): stmt =
template addNLIfEmpty(): untyped =
if result.len == 0:
result.add("\c\L")
@ -1327,7 +1337,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} =
return
add(result, c)
proc callSoon*(cbproc: proc ()) =
proc callSoon(cbproc: proc ()) =
## Schedule `cbproc` to be called as soon as possible.
## The callback is called when control returns to the event loop.
getGlobalDispatcher().callbacks.addLast(cbproc)

21
lib/pure/asyncfile.nim
View file

@ -81,27 +81,32 @@ proc getFileSize*(f: AsyncFile): int64 =
else:
result = lseek(f.fd.cint, 0, SEEK_END)
proc newAsyncFile*(fd: AsyncFd): AsyncFile =
## Creates `AsyncFile` with a previously opened file descriptor `fd`.
new result
result.fd = fd
register(result.fd)
proc openAsync*(filename: string, mode = fmRead): AsyncFile =
## Opens a file specified by the path in ``filename`` using
## the specified ``mode`` asynchronously.
new result
when defined(windows) or defined(nimdoc):
let flags = FILE_FLAG_OVERLAPPED or FILE_ATTRIBUTE_NORMAL
let desiredAccess = getDesiredAccess(mode)
let creationDisposition = getCreationDisposition(mode, filename)
when useWinUnicode:
result.fd = createFileW(newWideCString(filename), desiredAccess,
let fd = createFileW(newWideCString(filename), desiredAccess,
FILE_SHARE_READ,
nil, creationDisposition, flags, 0).AsyncFd
else:
result.fd = createFileA(filename, desiredAccess,
let fd = createFileA(filename, desiredAccess,
FILE_SHARE_READ,
nil, creationDisposition, flags, 0).AsyncFd
if result.fd.Handle == INVALID_HANDLE_VALUE:
if fd.Handle == INVALID_HANDLE_VALUE:
raiseOSError(osLastError())
register(result.fd)
result = newAsyncFile(fd)
if mode == fmAppend:
result.offset = getFileSize(result)
@ -110,11 +115,11 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile =
let flags = getPosixFlags(mode)
# RW (Owner), RW (Group), R (Other)
let perm = S_IRUSR or S_IWUSR or S_IRGRP or S_IWGRP or S_IROTH
result.fd = open(filename, flags, perm).AsyncFD
if result.fd.cint == -1:
let fd = open(filename, flags, perm).AsyncFD
if fd.cint == -1:
raiseOSError(osLastError())
register(result.fd)
result = newAsyncFile(fd)
proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] =
## Read ``size`` bytes from the specified file asynchronously starting at

214
lib/pure/includes/asyncfutures.nim → lib/pure/asyncfutures.nim
View file

@ -1,8 +1,16 @@
import os, tables, strutils, times, heapqueue, options, deques
# TODO: This shouldn't need to be included, but should ideally be exported.
type
CallbackFunc = proc () {.closure, gcsafe.}
CallbackList = object
function: CallbackFunc
next: ref CallbackList
FutureBase* = ref object of RootObj ## Untyped future.
cb: proc () {.closure,gcsafe.}
callbacks: CallbackList
finished: bool
error*: ref Exception ## Stored exception
errorStackTrace*: string
@ -16,12 +24,6 @@ type
FutureVar*[T] = distinct Future[T]
FutureStream*[T] = ref object of FutureBase ## Special future that acts as
## a queue. Its API is still
## experimental and so is
## subject to change.
queue: Deque[T]
FutureError* = object of Exception
cause*: FutureBase
@ -30,7 +32,27 @@ type
when not defined(release):
var currentID = 0
proc callSoon*(cbproc: proc ()) {.gcsafe.}
var callSoonProc {.threadvar.}: proc (cbproc: proc ()) {.gcsafe.}
proc getCallSoonProc*(): (proc(cbproc: proc ()) {.gcsafe.}) =
## Get current implementation of ``callSoon``.
return callSoonProc
proc setCallSoonProc*(p: (proc(cbproc: proc ()) {.gcsafe.})) =
## Change current implementation of ``callSoon``. This is normally called when dispatcher from ``asyncdispatcher`` is initialized.
callSoonProc = p
proc callSoon*(cbproc: proc ()) =
## Call ``cbproc`` "soon".
##
## If async dispatcher is running, ``cbproc`` will be executed during next dispatcher tick.
##
## If async dispatcher is not running, ``cbproc`` will be executed immediately.
if callSoonProc.isNil:
# Loop not initialized yet. Call the function directly to allow setup code to use futures.
cbproc()
else:
callSoonProc(cbproc)
template setupFutureBase(fromProc: string) =
new(result)
@ -56,22 +78,6 @@ proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] =
## that this future belongs to, is a good habit as it helps with debugging.
result = FutureVar[T](newFuture[T](fromProc))
proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] =
## Create a new ``FutureStream``. This future's callback is activated when
## two events occur:
##
## * New data is written into the future stream.
## * The future stream is completed (this means that no more data will be
## written).
##
## Specifying ``fromProc``, which is a string specifying the name of the proc
## that this future belongs to, is a good habit as it helps with debugging.
##
## **Note:** The API of FutureStream is still new and so has a higher
## likelihood of changing in the future.
setupFutureBase(fromProc)
result.queue = initDeque[T]()
proc clean*[T](future: FutureVar[T]) =
## Resets the ``finished`` status of ``future``.
Future[T](future).finished = false
@ -98,6 +104,33 @@ proc checkFinished[T](future: Future[T]) =
err.cause = future
raise err
proc call(callbacks: var CallbackList) =
var current = callbacks
while true:
if not current.function.isNil:
callSoon(current.function)
if current.next.isNil:
break
else:
current = current.next[]
# callback will be called only once, let GC collect them now
callbacks.next = nil
callbacks.function = nil
proc add(callbacks: var CallbackList, function: CallbackFunc) =
if callbacks.function.isNil:
callbacks.function = function
assert callbacks.next == nil
else:
let newNext = new(ref CallbackList)
newNext.function = callbacks.function
newNext.next = callbacks.next
callbacks.next = newNext
callbacks.function = function
proc complete*[T](future: Future[T], val: T) =
## Completes ``future`` with value ``val``.
#assert(not future.finished, "Future already finished, cannot finish twice.")
@ -105,8 +138,7 @@ proc complete*[T](future: Future[T], val: T) =
assert(future.error == nil)
future.value = val
future.finished = true
if future.cb != nil:
future.cb()
future.callbacks.call()
proc complete*(future: Future[void]) =
## Completes a void ``future``.
@ -114,8 +146,7 @@ proc complete*(future: Future[void]) =
checkFinished(future)
assert(future.error == nil)
future.finished = true
if future.cb != nil:
future.cb()
future.callbacks.call()
proc complete*[T](future: FutureVar[T]) =
## Completes a ``FutureVar``.
@ -123,8 +154,7 @@ proc complete*[T](future: FutureVar[T]) =
checkFinished(fut)
assert(fut.error == nil)
fut.finished = true
if fut.cb != nil:
fut.cb()
fut.callbacks.call()
proc complete*[T](future: FutureVar[T], val: T) =
## Completes a ``FutureVar`` with value ``val``.
@ -135,14 +165,7 @@ proc complete*[T](future: FutureVar[T], val: T) =
assert(fut.error.isNil())
fut.finished = true
fut.value = val
if not fut.cb.isNil():
fut.cb()
proc complete*[T](future: FutureStream[T]) =
## Completes a ``FutureStream`` signalling the end of data.
future.finished = true
if not future.cb.isNil():
future.cb()
fut.callbacks.call()
proc fail*[T](future: Future[T], error: ref Exception) =
## Completes ``future`` with ``error``.
@ -152,26 +175,40 @@ proc fail*[T](future: Future[T], error: ref Exception) =
future.error = error
future.errorStackTrace =
if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error)
if future.cb != nil:
future.cb()
future.callbacks.call()
proc clearCallbacks(future: FutureBase) =
future.callbacks.function = nil
future.callbacks.next = nil
proc addCallback*(future: FutureBase, cb: proc() {.closure,gcsafe.}) =
## Adds the callbacks proc to be called when the future completes.
##
## If future has already completed then ``cb`` will be called immediately.
assert cb != nil
if future.finished:
callSoon(cb)
else:
# This is to prevent exceptions from being silently ignored when a future
# is discarded.
# TODO: This may turn out to be a bad idea.
# Turns out this is a bad idea.
#raise error
discard
future.callbacks.add cb
proc addCallback*[T](future: Future[T],
cb: proc (future: Future[T]) {.closure,gcsafe.}) =
## Adds the callbacks proc to be called when the future completes.
##
## If future has already completed then ``cb`` will be called immediately.
future.addCallback(
proc() =
cb(future)
)
proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) =
## Sets the callback proc to be called when the future completes.
## Clears the list of callbacks and sets the callback proc to be called when the future completes.
##
## If future has already completed then ``cb`` will be called immediately.
##
## **Note**: You most likely want the other ``callback`` setter which
## passes ``future`` as a param to the callback.
future.cb = cb
if future.finished:
callSoon(future.cb)
## It's recommended to use ``addCallback`` or ``then`` instead.
future.clearCallbacks
future.addCallback cb
proc `callback=`*[T](future: Future[T],
cb: proc (future: Future[T]) {.closure,gcsafe.}) =
@ -180,20 +217,6 @@ proc `callback=`*[T](future: Future[T],
## If future has already completed then ``cb`` will be called immediately.
future.callback = proc () = cb(future)
proc `callback=`*[T](future: FutureStream[T],
cb: proc (future: FutureStream[T]) {.closure,gcsafe.}) =
## Sets the callback proc to be called when data was placed inside the
## future stream.
##
## The callback is also called when the future is completed. So you should
## use ``finished`` to check whether data is available.
##
## If the future stream already has data or is finished then ``cb`` will be
## called immediately.
future.cb = proc () = cb(future)
if future.queue.len > 0 or future.finished:
callSoon(future.cb)
proc injectStacktrace[T](future: Future[T]) =
# TODO: Come up with something better.
when not defined(release):
@ -240,18 +263,12 @@ proc mget*[T](future: FutureVar[T]): var T =
## Future has not been finished.
result = Future[T](future).value
proc finished*[T](future: Future[T] | FutureVar[T] | FutureStream[T]): bool =
proc finished*[T](future: Future[T] | FutureVar[T]): bool =
## Determines whether ``future`` has completed.
##
## ``True`` may indicate an error or a value. Use ``failed`` to distinguish.
##
## For a ``FutureStream`` a ``true`` value means that no more data will be
## placed inside the stream _and_ that there is no data waiting to be
## retrieved.
when future is FutureVar[T]:
result = (Future[T](future)).finished
elif future is FutureStream[T]:
result = future.finished and future.queue.len == 0
else:
result = future.finished
@ -259,57 +276,6 @@ proc failed*(future: FutureBase): bool =
## Determines whether ``future`` completed with an error.
return future.error != nil
proc write*[T](future: FutureStream[T], value: T): Future[void] =
## Writes the specified value inside the specified future stream.
##
## This will raise ``ValueError`` if ``future`` is finished.
result = newFuture[void]("FutureStream.put")
if future.finished:
let msg = "FutureStream is finished and so no longer accepts new data."
result.fail(newException(ValueError, msg))
return
# TODO: Implement limiting of the streams storage to prevent it growing
# infinitely when no reads are occuring.
future.queue.addLast(value)
if not future.cb.isNil: future.cb()
result.complete()
proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
## Returns a future that will complete when the ``FutureStream`` has data
## placed into it. The future will be completed with the oldest
## value stored inside the stream. The return value will also determine
## whether data was retrieved, ``false`` means that the future stream was
## completed and no data was retrieved.
##
## This function will remove the data that was returned from the underlying
## ``FutureStream``.
var resFut = newFuture[(bool, T)]("FutureStream.take")
let savedCb = future.cb
future.callback =
proc (fs: FutureStream[T]) =
# We don't want this callback called again.
future.cb = nil
# The return value depends on whether the FutureStream has finished.
var res: (bool, T)
if finished(fs):
# Remember, this callback is called when the FutureStream is completed.
res[0] = false
else:
res[0] = true
res[1] = fs.queue.popFirst()
if not resFut.finished:
resFut.complete(res)
# If the saved callback isn't nil then let's call it.
if not savedCb.isNil: savedCb()
return resFut
proc len*[T](future: FutureStream[T]): int =
## Returns the amount of data pieces inside the stream.
future.queue.len
proc asyncCheck*[T](future: Future[T]) =
## Sets a callback on ``future`` which raises an exception if the future
## finished with an error.

4
lib/pure/asynchttpserver.nim
View file

@ -127,7 +127,7 @@ proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] =
i.inc protocol.parseInt(result.minor, i)
proc sendStatus(client: AsyncSocket, status: string): Future[void] =
client.send("HTTP/1.1 " & status & "\c\L\c\L")
client.send("HTTP/1.1 " & status & "\c\L\c\L")
proc processClient(client: AsyncSocket, address: string,
callback: proc (request: Request):
@ -233,7 +233,7 @@ proc processClient(client: AsyncSocket, address: string,
await request.respond(Http400, "Bad Request. Content-Length does not match actual.")
continue
elif request.reqMethod == HttpPost:
await request.respond(Http400, "Bad Request. No Content-Length.")
await request.respond(Http411, "Content-Length required.")
continue
# Call the user's callback.

14
lib/pure/asyncmacro.nim
View file

@ -28,7 +28,7 @@ template createCb(retFutureSym, iteratorNameSym,
name, futureVarCompletions: untyped) =
var nameIterVar = iteratorNameSym
#{.push stackTrace: off.}
proc cb {.closure,gcsafe.} =
proc cb0 {.closure.} =
try:
if not nameIterVar.finished:
var next = nameIterVar()
@ -38,7 +38,10 @@ template createCb(retFutureSym, iteratorNameSym,
"`nil` Future?"
raise newException(AssertionError, msg % name)
else:
next.callback = cb
{.gcsafe.}:
{.push hint[ConvFromXtoItselfNotNeeded]: off.}
next.callback = (proc() {.closure, gcsafe.})(cb0)
{.pop.}
except:
futureVarCompletions
@ -49,7 +52,7 @@ template createCb(retFutureSym, iteratorNameSym,
else:
retFutureSym.fail(getCurrentException())
cb()
cb0()
#{.pop.}
proc generateExceptionCheck(futSym,
tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
@ -379,7 +382,10 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
procBody, nnkIteratorDef)
closureIterator.pragma = newNimNode(nnkPragma, lineInfoFrom=prc.body)
closureIterator.addPragma(newIdentNode("closure"))
closureIterator.addPragma(newIdentNode("gcsafe"))
# If proc has an explicit gcsafe pragma, we add it to iterator as well.
if prc.pragma.findChild(it.kind in {nnkSym, nnkIdent} and $it == "gcsafe") != nil:
closureIterator.addPragma(newIdentNode("gcsafe"))
outerProcBody.add(closureIterator)
# -> createCb(retFuture)

4
lib/pure/asyncnet.nim
View file

@ -220,7 +220,7 @@ when defineSsl:
raiseSSLError("Cannot appease SSL.")
template sslLoop(socket: AsyncSocket, flags: set[SocketFlag],
op: expr) =
op: untyped) =
var opResult {.inject.} = -1.cint
while opResult < 0:
# Call the desired operation.
@ -490,7 +490,7 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string],
# them when the result future is completed.
# Can we replace the result future with the FutureVar?
template addNLIfEmpty(): stmt =
template addNLIfEmpty(): untyped =
if resString.mget.len == 0:
resString.mget.add("\c\L")

105
lib/pure/asyncstreams.nim Normal file
View file

@ -0,0 +1,105 @@
import asyncfutures
import deques
type
FutureStream*[T] = ref object ## Special future that acts as
## a queue. Its API is still
## experimental and so is
## subject to change.
queue: Deque[T]
finished: bool
cb: proc () {.closure, gcsafe.}
proc newFutureStream*[T](fromProc = "unspecified"): FutureStream[T] =
## Create a new ``FutureStream``. This future's callback is activated when
## two events occur:
##
## * New data is written into the future stream.
## * The future stream is completed (this means that no more data will be
## written).
##
## Specifying ``fromProc``, which is a string specifying the name of the proc
## that this future belongs to, is a good habit as it helps with debugging.
##
## **Note:** The API of FutureStream is still new and so has a higher
## likelihood of changing in the future.
result = FutureStream[T](finished: false, cb: nil)
result.queue = initDeque[T]()
proc complete*[T](future: FutureStream[T]) =
## Completes a ``FutureStream`` signalling the end of data.
future.finished = true
if not future.cb.isNil:
future.cb()
proc `callback=`*[T](future: FutureStream[T],
cb: proc (future: FutureStream[T]) {.closure,gcsafe.}) =
## Sets the callback proc to be called when data was placed inside the
## future stream.
##
## The callback is also called when the future is completed. So you should
## use ``finished`` to check whether data is available.
##
## If the future stream already has data or is finished then ``cb`` will be
## called immediately.
future.cb = proc () = cb(future)
if future.queue.len > 0 or future.finished:
callSoon(future.cb)
proc finished*[T](future: FutureStream[T]): bool =
## Check if a ``FutureStream`` is finished. ``true`` value means that
## no more data will be placed inside the stream _and_ that there is
## no data waiting to be retrieved.
result = future.finished and future.queue.len == 0
proc write*[T](future: FutureStream[T], value: T): Future[void] =
## Writes the specified value inside the specified future stream.
##
## This will raise ``ValueError`` if ``future`` is finished.
result = newFuture[void]("FutureStream.put")
if future.finished:
let msg = "FutureStream is finished and so no longer accepts new data."
result.fail(newException(ValueError, msg))
return
# TODO: Implement limiting of the streams storage to prevent it growing
# infinitely when no reads are occuring.
future.queue.addLast(value)
if not future.cb.isNil: future.cb()
result.complete()
proc read*[T](future: FutureStream[T]): Future[(bool, T)] =
## Returns a future that will complete when the ``FutureStream`` has data
## placed into it. The future will be completed with the oldest
## value stored inside the stream. The return value will also determine
## whether data was retrieved, ``false`` means that the future stream was
## completed and no data was retrieved.
##
## This function will remove the data that was returned from the underlying
## ``FutureStream``.
var resFut = newFuture[(bool, T)]("FutureStream.take")
let savedCb = future.cb
future.callback =
proc (fs: FutureStream[T]) =
# We don't want this callback called again.
future.cb = nil
# The return value depends on whether the FutureStream has finished.
var res: (bool, T)
if finished(fs):
# Remember, this callback is called when the FutureStream is completed.
res[0] = false
else:
res[0] = true
res[1] = fs.queue.popFirst()
if not resFut.finished:
resFut.complete(res)
# If the saved callback isn't nil then let's call it.
if not savedCb.isNil: savedCb()
return resFut
proc len*[T](future: FutureStream[T]): int =
## Returns the amount of data pieces inside the stream.
future.queue.len

6
lib/pure/basic2d.nim
View file

@ -116,13 +116,13 @@ proc safeArccos(v:float):float=
return arccos(clamp(v,-1.0,1.0))
template makeBinOpVector(s:expr)=
template makeBinOpVector(s) =
## implements binary operators ``+``, ``-``, ``*`` and ``/`` for vectors
proc s*(a,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a.x,b.x),s(a.y,b.y))
proc s*(a:Vector2d,b:float):Vector2d {.inline,noInit.} = vector2d(s(a.x,b),s(a.y,b))
proc s*(a:float,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a,b.x),s(a,b.y))
template makeBinOpAssignVector(s:expr)=
template makeBinOpAssignVector(s)=
## implements inplace binary operators ``+=``, ``-=``, ``/=`` and ``*=`` for vectors
proc s*(a:var Vector2d,b:Vector2d) {.inline.} = s(a.x,b.x) ; s(a.y,b.y)
proc s*(a:var Vector2d,b:float) {.inline.} = s(a.x,b) ; s(a.y,b)
@ -853,5 +853,3 @@ proc degToRad*(deg:float):float {.inline.}=
proc radToDeg*(rad:float):float {.inline.}=
## converts `rad` radians to degrees
rad * RAD2DEGCONST

4
lib/pure/basic3d.nim
View file

@ -116,7 +116,7 @@ proc safeArccos(v:float):float=
## due to rounding issues
return arccos(clamp(v,-1.0,1.0))
template makeBinOpVector(s:expr)=
template makeBinOpVector(s) =
proc s*(a,b:Vector3d):Vector3d {.inline,noInit.} =
vector3d(s(a.x,b.x),s(a.y,b.y),s(a.z,b.z))
proc s*(a:Vector3d,b:float):Vector3d {.inline,noInit.} =
@ -124,7 +124,7 @@ template makeBinOpVector(s:expr)=
proc s*(a:float,b:Vector3d):Vector3d {.inline,noInit.} =
vector3d(s(a,b.x),s(a,b.y),s(a,b.z))
template makeBinOpAssignVector(s:expr)=
template makeBinOpAssignVector(s) =
proc s*(a:var Vector3d,b:Vector3d) {.inline.} =
s(a.x,b.x); s(a.y,b.y); s(a.z,b.z)
proc s*(a:var Vector3d,b:float) {.inline.} =

7
lib/pure/collections/critbits.nim
View file

@ -143,15 +143,16 @@ proc `[]=`*[T](c: var CritBitTree[T], key: string, val: T) =
var n = rawInsert(c, key)
n.val = val
template get[T](c: CritBitTree[T], key: string): T {.immediate.} =
template get[T](c: CritBitTree[T], key: string): T =
let n = rawGet(c, key)
if n != nil: result = n.val
else:
if n == nil:
when compiles($key):
raise newException(KeyError, "key not found: " & $key)
else:
raise newException(KeyError, "key not found")
n.val
proc `[]`*[T](c: CritBitTree[T], key: string): T {.inline, deprecatedGet.} =
## retrieves the value at ``c[key]``. If `key` is not in `t`, the
## ``KeyError`` exception is raised. One can check with ``hasKey`` whether

2
lib/pure/collections/deques.nim
View file

@ -33,7 +33,7 @@
## assert deq.peekLast == a
##
## while deq.len > 0: # checking if the deque is empty
## echo deq.removeLast()
## echo deq.popLast()
##
## Note: For inter thread communication use
## a `Channel <channels.html>`_ instead.

179
lib/pure/collections/intsets.nim
View file

@ -31,16 +31,18 @@ const
type
PTrunk = ref Trunk
Trunk {.final.} = object
Trunk = object
next: PTrunk # all nodes are connected with this pointer
key: int # start address at bit 0
bits: array[0..IntsPerTrunk - 1, BitScalar] # a bit vector
TrunkSeq = seq[PTrunk]
IntSet* = object ## an efficient set of 'int' implemented as a sparse bit set
elems: int # only valid for small numbers
counter, max: int
head: PTrunk
data: TrunkSeq
a: array[0..33, int] # profiling shows that 34 elements are enough
{.deprecated: [TIntSet: IntSet, TTrunk: Trunk, TTrunkSeq: TrunkSeq].}
@ -95,101 +97,154 @@ proc intSetPut(t: var IntSet, key: int): PTrunk =
proc contains*(s: IntSet, key: int): bool =
## returns true iff `key` is in `s`.
var t = intSetGet(s, `shr`(key, TrunkShift))
if t != nil:
var u = key and TrunkMask
result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
if s.elems <= s.a.len:
for i in 0..<s.elems:
if s.a[i] == key: return true
else:
result = false
var t = intSetGet(s, `shr`(key, TrunkShift))
if t != nil:
var u = key and TrunkMask
result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
else:
result = false
proc incl*(s: var IntSet, key: int) =
## includes an element `key` in `s`.
proc bitincl(s: var IntSet, key: int) {.inline.} =
var t = intSetPut(s, `shr`(key, TrunkShift))
var u = key and TrunkMask
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
`shl`(1, u and IntMask)
proc incl*(s: var IntSet, key: int) =
## includes an element `key` in `s`.
if s.elems <= s.a.len:
for i in 0..<s.elems:
if s.a[i] == key: return
if s.elems < s.a.len:
s.a[s.elems] = key
inc s.elems
return
newSeq(s.data, InitIntSetSize)
s.max = InitIntSetSize-1
for i in 0..<s.elems:
bitincl(s, s.a[i])
s.elems = s.a.len + 1
# fall through:
bitincl(s, key)
proc excl*(s: var IntSet, key: int) =
## excludes `key` from the set `s`.
var t = intSetGet(s, `shr`(key, TrunkShift))
if t != nil:
var u = key and TrunkMask
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and
not `shl`(1, u and IntMask)
if s.elems <= s.a.len:
for i in 0..<s.elems:
if s.a[i] == key:
s.a[i] = s.a[s.elems-1]
dec s.elems
return
else:
var t = intSetGet(s, `shr`(key, TrunkShift))
if t != nil:
var u = key and TrunkMask
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and
not `shl`(1, u and IntMask)
proc containsOrIncl*(s: var IntSet, key: int): bool =
## returns true if `s` contains `key`, otherwise `key` is included in `s`
## and false is returned.
var t = intSetGet(s, `shr`(key, TrunkShift))
if t != nil:
var u = key and TrunkMask
result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
if not result:
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
`shl`(1, u and IntMask)
else:
if s.elems <= s.a.len:
for i in 0..<s.elems:
if s.a[i] == key:
return true
incl(s, key)
result = false
else:
var t = intSetGet(s, `shr`(key, TrunkShift))
if t != nil:
var u = key and TrunkMask
result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
if not result:
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
`shl`(1, u and IntMask)
else:
incl(s, key)
result = false
proc initIntSet*: IntSet =
## creates a new int set that is empty.
newSeq(result.data, InitIntSetSize)
result.max = InitIntSetSize-1
#newSeq(result.data, InitIntSetSize)
#result.max = InitIntSetSize-1
result.data = nil
result.max = 0
result.counter = 0
result.head = nil
result.elems = 0
proc clear*(result: var IntSet) =
setLen(result.data, InitIntSetSize)
for i in 0..InitIntSetSize-1: result.data[i] = nil
result.max = InitIntSetSize-1
#setLen(result.data, InitIntSetSize)
#for i in 0..InitIntSetSize-1: result.data[i] = nil
#result.max = InitIntSetSize-1
result.data = nil
result.max = 0
result.counter = 0
result.head = nil
result.elems = 0
proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil
proc isNil*(x: IntSet): bool {.inline.} = x.head.isNil and x.elems == 0
proc assign*(dest: var IntSet, src: IntSet) =
## copies `src` to `dest`. `dest` does not need to be initialized by
## `initIntSet`.
dest.counter = src.counter
dest.max = src.max
newSeq(dest.data, src.data.len)
if src.elems <= src.a.len:
dest.data = nil
dest.max = 0
dest.counter = src.counter
dest.head = nil
dest.elems = src.elems
dest.a = src.a
else:
dest.counter = src.counter
dest.max = src.max
newSeq(dest.data, src.data.len)
var it = src.head
while it != nil:
var it = src.head
while it != nil:
var h = it.key and dest.max
while dest.data[h] != nil: h = nextTry(h, dest.max)
assert(dest.data[h] == nil)
var h = it.key and dest.max
while dest.data[h] != nil: h = nextTry(h, dest.max)
assert(dest.data[h] == nil)
var n: PTrunk
new(n)
n.next = dest.head
n.key = it.key
n.bits = it.bits
dest.head = n
dest.data[h] = n
var n: PTrunk
new(n)
n.next = dest.head
n.key = it.key
n.bits = it.bits
dest.head = n
dest.data[h] = n
it = it.next
it = it.next
iterator items*(s: IntSet): int {.inline.} =
## iterates over any included element of `s`.
var r = s.head
while r != nil:
var i = 0
while i <= high(r.bits):
var w = r.bits[i]
# taking a copy of r.bits[i] here is correct, because
# modifying operations are not allowed during traversation
var j = 0
while w != 0: # test all remaining bits for zero
if (w and 1) != 0: # the bit is set!
yield (r.key shl TrunkShift) or (i shl IntShift +% j)
inc(j)
w = w shr 1
inc(i)
r = r.next
if s.elems <= s.a.len:
for i in 0..<s.elems:
yield s.a[i]
else:
var r = s.head
while r != nil:
var i = 0
while i <= high(r.bits):
var w = r.bits[i]
# taking a copy of r.bits[i] here is correct, because
# modifying operations are not allowed during traversation
var j = 0
while w != 0: # test all remaining bits for zero
if (w and 1) != 0: # the bit is set!
yield (r.key shl TrunkShift) or (i shl IntShift +% j)
inc(j)
w = w shr 1
inc(i)
r = r.next
template dollarImpl(): stmt =
template dollarImpl(): untyped =
result = "{"
for key in items(s):
if result.len > 1: result.add(", ")
@ -225,3 +280,9 @@ when isMainModule:
ys.sort(cmp[int])
assert ys == @[1, 2, 7, 1056]
var z: IntSet
for i in 0..1000:
incl z, i
for i in 0..1000:
assert i in z

2
lib/pure/collections/rtarrays.nim
View file

@ -19,7 +19,7 @@ type
L: Natural
spart: seq[T]
apart: array[ArrayPartSize, T]
UncheckedArray* {.unchecked.}[T] = array[0..100_000_000, T]
UncheckedArray* {.unchecked.}[T] = array[0, T]
template usesSeqPart(x): untyped = x.L > ArrayPartSize

2
lib/pure/collections/sequtils.nim
View file

@ -455,7 +455,7 @@ template anyIt*(seq1, pred: untyped): bool =
break
result
template toSeq*(iter: untyped): untyped {.oldimmediate.} =
template toSeq*(iter: untyped): untyped =
## Transforms any iterator into a sequence.
##
## Example:

4
lib/pure/collections/sharedstrings.nim
View file

@ -9,10 +9,8 @@
## Shared string support for Nim.
const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000
type
UncheckedCharArray {.unchecked.} = array[0..ArrayDummySize, char]
UncheckedCharArray = UncheckedArray[char]
type
Buffer = ptr object

2
lib/pure/collections/sharedtables.nim
View file

@ -25,7 +25,7 @@ type
counter, dataLen: int
lock: Lock
template maxHash(t): expr = t.dataLen-1
template maxHash(t): untyped = t.dataLen-1
include tableimpl

4
lib/pure/collections/tableimpl.nim
View file

@ -85,7 +85,7 @@ template addImpl(enlarge) {.dirty.} =
rawInsert(t, t.data, key, val, hc, j)
inc(t.counter)
template maybeRehashPutImpl(enlarge) {.oldimmediate, dirty.} =
template maybeRehashPutImpl(enlarge) {.dirty.} =
if mustRehash(t.dataLen, t.counter):
enlarge(t)
index = rawGetKnownHC(t, key, hc)
@ -93,7 +93,7 @@ template maybeRehashPutImpl(enlarge) {.oldimmediate, dirty.} =
rawInsert(t, t.data, key, val, hc, index)
inc(t.counter)
template putImpl(enlarge) {.oldimmediate, dirty.} =
template putImpl(enlarge) {.dirty.} =
var hc: Hash
var index = rawGet(t, key, hc)
if index >= 0: t.data[index].val = val

18
lib/pure/collections/tables.nim
View file

@ -648,7 +648,7 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool =
var nxtt = t.data[ht].next
var nxts = s.data[hs].next
if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode):
if (s.data[hs].key != t.data[ht].key) and (s.data[hs].val != t.data[ht].val):
if (s.data[hs].key != t.data[ht].key) or (s.data[hs].val != t.data[ht].val):
return false
ht = nxtt
hs = nxts
@ -939,7 +939,7 @@ proc enlarge[A](t: var CountTable[A]) =
proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
## puts a (key, value)-pair into `t`.
assert val > 0
assert val >= 0
var h = rawGet(t, key)
if h >= 0:
t.data[h].val = val
@ -1311,3 +1311,17 @@ when isMainModule:
assert a == c
block: #6250
let
a = {3: 1}.toOrderedTable
b = {3: 2}.toOrderedTable
assert((a == b) == false)
assert((b == a) == false)
block: #6250
let
a = {3: 2}.toOrderedTable
b = {3: 2}.toOrderedTable
assert((a == b) == true)
assert((b == a) == true)

12
lib/pure/colors.nim
View file

@ -19,18 +19,18 @@ type
proc `==` *(a, b: Color): bool {.borrow.}
## compares two colors.
template extract(a: Color, r, g, b: expr) {.immediate.}=
template extract(a: Color, r, g, b: untyped) =
var r = a.int shr 16 and 0xff
var g = a.int shr 8 and 0xff
var b = a.int and 0xff
template rawRGB(r, g, b: int): expr =
template rawRGB(r, g, b: int): Color =
Color(r shl 16 or g shl 8 or b)
template colorOp(op: expr) {.immediate.} =
template colorOp(op): Color =
extract(a, ar, ag, ab)
extract(b, br, bg, bb)
result = rawRGB(op(ar, br), op(ag, bg), op(ab, bb))
rawRGB(op(ar, br), op(ag, bg), op(ab, bb))
proc satPlus(a, b: int): int {.inline.} =
result = a +% b
@ -67,12 +67,12 @@ proc intensity*(a: Color, f: float): Color =
if b >% 255: b = 255
result = rawRGB(r, g, b)
template mix*(a, b: Color, fn: expr): expr =
template mix*(a, b: Color, fn: untyped): untyped =
## uses `fn` to mix the colors `a` and `b`. `fn` is invoked for each component
## R, G, and B. This is a template because `fn` should be inlined and the
## compiler cannot inline proc pointers yet. If `fn`'s result is not in the
## range[0..255], it will be saturated to be so.
template `><` (x: expr): expr =
template `><` (x: untyped): untyped =
# keep it in the range 0..255
block:
var y = x # eval only once

13
lib/pure/concurrency/threadpool.nim
View file

@ -409,20 +409,20 @@ proc preferSpawn*(): bool =
## it is not necessary to call this directly; use 'spawnX' instead.
result = gSomeReady.counter > 0
proc spawn*(call: expr): expr {.magic: "Spawn".}
proc spawn*(call: typed): void {.magic: "Spawn".}
## always spawns a new task, so that the 'call' is never executed on
## the calling thread. 'call' has to be proc call 'p(...)' where 'p'
## is gcsafe and has a return type that is either 'void' or compatible
## with ``FlowVar[T]``.
proc pinnedSpawn*(id: ThreadId; call: expr): expr {.magic: "Spawn".}
proc pinnedSpawn*(id: ThreadId; call: typed): void {.magic: "Spawn".}
## always spawns a new task on the worker thread with ``id``, so that
## the 'call' is **always** executed on
## the thread. 'call' has to be proc call 'p(...)' where 'p'
## is gcsafe and has a return type that is either 'void' or compatible
## with ``FlowVar[T]``.
template spawnX*(call: expr): expr =
template spawnX*(call): void =
## spawns a new task if a CPU core is ready, otherwise executes the
## call in the calling thread. Usually it is advised to
## use 'spawn' in order to not block the producer for an unknown
@ -431,7 +431,7 @@ template spawnX*(call: expr): expr =
## with ``FlowVar[T]``.
(if preferSpawn(): spawn call else: call)
proc parallel*(body: stmt) {.magic: "Parallel".}
proc parallel*(body: untyped) {.magic: "Parallel".}
## a parallel section can be used to execute a block in parallel. ``body``
## has to be in a DSL that is a particular subset of the language. Please
## refer to the manual for further information.
@ -530,6 +530,7 @@ proc nimSpawn4(fn: WorkerProc; data: pointer; id: ThreadId) {.compilerProc.} =
proc sync*() =
## a simple barrier to wait for all spawn'ed tasks. If you need more elaborate
## waiting, you have to use an explicit barrier.
var toRelease = 0
while true:
var allReady = true
for i in 0 .. <currentPoolSize:
@ -537,5 +538,9 @@ proc sync*() =
allReady = allReady and workersData[i].ready
if allReady: break
await(gSomeReady)
inc toRelease
for i in 0 ..< toRelease:
signal(gSomeReady)
setup()

2
lib/pure/httpclient.nim
View file

@ -1245,6 +1245,8 @@ proc downloadFile*(client: HttpClient | AsyncHttpClient,
url: string, filename: string): Future[void] {.multisync.} =
## Downloads ``url`` and saves it to ``filename``.
client.getBody = false
defer:
client.getBody = true
let resp = await client.get(url)
when client is HttpClient:

159
lib/pure/includes/osenv.nim Normal file
View file

@ -0,0 +1,159 @@
## Include file that implements 'getEnv' and friends. Do not import it!
when not declared(ospaths):
{.error: "This is an include file for ospaths.nim!".}
proc c_getenv(env: cstring): cstring {.
importc: "getenv", header: "<stdlib.h>".}
proc c_putenv(env: cstring): cint {.
importc: "putenv", header: "<stdlib.h>".}
# Environment handling cannot be put into RTL, because the ``envPairs``
# iterator depends on ``environment``.
var
envComputed {.threadvar.}: bool
environment {.threadvar.}: seq[string]
when defined(windows) and not defined(nimscript):
# because we support Windows GUI applications, things get really
# messy here...
when useWinUnicode:
when defined(cpp):
proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".}
else:
proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
importc: "wcschr", header: "<string.h>".}
else:
proc strEnd(cstr: cstring, c = 0'i32): cstring {.
importc: "strchr", header: "<string.h>".}
proc getEnvVarsC() =
if not envComputed:
environment = @[]
when useWinUnicode:
var
env = getEnvironmentStringsW()
e = env
if e == nil: return # an error occurred
while true:
var eend = strEnd(e)
add(environment, $e)
e = cast[WideCString](cast[ByteAddress](eend)+2)
if eend[1].int == 0: break
discard freeEnvironmentStringsW(env)
else:
var
env = getEnvironmentStringsA()
e = env
if e == nil: return # an error occurred
while true:
var eend = strEnd(e)
add(environment, $e)
e = cast[cstring](cast[ByteAddress](eend)+1)
if eend[1] == '\0': break
discard freeEnvironmentStringsA(env)
envComputed = true
else:
const
useNSGetEnviron = (defined(macosx) and not defined(ios)) or defined(nimscript)
when useNSGetEnviron:
# From the manual:
# Shared libraries and bundles don't have direct access to environ,
# which is only available to the loader ld(1) when a complete program
# is being linked.
# The environment routines can still be used, but if direct access to
# environ is needed, the _NSGetEnviron() routine, defined in
# <crt_externs.h>, can be used to retrieve the address of environ
# at runtime.
proc NSGetEnviron(): ptr cstringArray {.
importc: "_NSGetEnviron", header: "<crt_externs.h>".}
else:
var gEnv {.importc: "environ".}: cstringArray
proc getEnvVarsC() =
# retrieves the variables of char** env of C's main proc
if not envComputed:
environment = @[]
when useNSGetEnviron:
var gEnv = NSGetEnviron()[]
var i = 0
while true:
if gEnv[i] == nil: break
add environment, $gEnv[i]
inc(i)
envComputed = true
proc findEnvVar(key: string): int =
getEnvVarsC()
var temp = key & '='
for i in 0..high(environment):
if startsWith(environment[i], temp): return i
return -1
proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} =
## Returns the value of the `environment variable`:idx: named `key`.
##
## If the variable does not exist, "" is returned. To distinguish
## whether a variable exists or it's value is just "", call
## `existsEnv(key)`.
when nimvm:
discard "built into the compiler"
else:
var i = findEnvVar(key)
if i >= 0:
return TaintedString(substr(environment[i], find(environment[i], '=')+1))
else:
var env = c_getenv(key)
if env == nil: return TaintedString("")
result = TaintedString($env)
proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
## Checks whether the environment variable named `key` exists.
## Returns true if it exists, false otherwise.
when nimvm:
discard "built into the compiler"
else:
if c_getenv(key) != nil: return true
else: return findEnvVar(key) >= 0
proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
## Sets the value of the `environment variable`:idx: named `key` to `val`.
## If an error occurs, `EInvalidEnvVar` is raised.
# Note: by storing the string in the environment sequence,
# we guarantee that we don't free the memory before the program
# ends (this is needed for POSIX compliance). It is also needed so that
# the process itself may access its modified environment variables!
when nimvm:
discard "built into the compiler"
else:
var indx = findEnvVar(key)
if indx >= 0:
environment[indx] = key & '=' & val
else:
add environment, (key & '=' & val)
indx = high(environment)
when defined(windows) and not defined(nimscript):
when useWinUnicode:
var k = newWideCString(key)
var v = newWideCString(val)
if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError())
else:
if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError())
else:
if c_putenv(environment[indx]) != 0'i32:
raiseOSError(osLastError())
iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} =
## Iterate over all `environments variables`:idx:. In the first component
## of the tuple is the name of the current variable stored, in the second
## its value.
getEnvVarsC()
for i in 0..high(environment):
var p = find(environment[i], '=')
yield (TaintedString(substr(environment[i], 0, p-1)),
TaintedString(substr(environment[i], p+1)))

135
lib/pure/includes/oserr.nim Normal file
View file

@ -0,0 +1,135 @@
## Include file that implements 'osErrorMsg' and friends. Do not import it!
when not declared(ospaths):
{.error: "This is an include file for ospaths.nim!".}
when not defined(nimscript):
var errno {.importc, header: "<errno.h>".}: cint
proc c_strerror(errnum: cint): cstring {.
importc: "strerror", header: "<string.h>".}
when defined(windows):
import winlean
proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} =
## Retrieves the operating system's error flag, ``errno``.
## On Windows ``GetLastError`` is checked before ``errno``.
## Returns "" if no error occurred.
##
## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc.
result = ""
when defined(Windows) and not defined(nimscript):
var err = getLastError()
if err != 0'i32:
when useWinUnicode:
var msgbuf: WideCString
if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(cast[pointer](msgbuf))
else:
var msgbuf: cstring
if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(msgbuf)
when not defined(nimscript):
if errno != 0'i32:
result = $c_strerror(errno)
{.push warning[deprecated]: off.}
proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1",
deprecated.} =
## raises an OSError exception with the given message ``msg``.
## If ``msg == ""``, the operating system's error flag
## (``errno``) is converted to a readable error message. On Windows
## ``GetLastError`` is checked before ``errno``.
## If no error flag is set, the message ``unknown OS error`` is used.
##
## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc.
if len(msg) == 0:
var m = osErrorMsg()
raise newException(OSError, if m.len > 0: m else: "unknown OS error")
else:
raise newException(OSError, msg)
{.pop.}
when not defined(nimfix):
{.deprecated: [osError: raiseOSError].}
proc `==`*(err1, err2: OSErrorCode): bool {.borrow.}
proc `$`*(err: OSErrorCode): string {.borrow.}
proc osErrorMsg*(errorCode: OSErrorCode): string =
## Converts an OS error code into a human readable string.
##
## The error code can be retrieved using the ``osLastError`` proc.
##
## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be
## returned.
##
## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to
## make this procedure use the non-unicode Win API calls to retrieve the
## message.
result = ""
when defined(nimscript):
discard
elif defined(Windows):
if errorCode != OSErrorCode(0'i32):
when useWinUnicode:
var msgbuf: WideCString
if formatMessageW(0x00000100 or 0x00001000 or 0x00000200,
nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(cast[pointer](msgbuf))
else:
var msgbuf: cstring
if formatMessageA(0x00000100 or 0x00001000 or 0x00000200,
nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(msgbuf)
else:
if errorCode != OSErrorCode(0'i32):
result = $c_strerror(errorCode.int32)
proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} =
## Raises an ``OSError`` exception. The ``errorCode`` will determine the
## message, ``osErrorMsg`` will be used to get this message.
##
## The error code can be retrieved using the ``osLastError`` proc.
##
## If the error code is ``0`` or an error message could not be retrieved,
## the message ``unknown OS error`` will be used.
var e: ref OSError; new(e)
e.errorCode = errorCode.int32
if additionalInfo.len == 0:
e.msg = osErrorMsg(errorCode)
else:
e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo
if e.msg == "":
e.msg = "unknown OS error"
raise e
{.push stackTrace:off.}
proc osLastError*(): OSErrorCode =
## Retrieves the last operating system error code.
##
## This procedure is useful in the event when an OS call fails. In that case
## this procedure will return the error code describing the reason why the
## OS call failed. The ``OSErrorMsg`` procedure can then be used to convert
## this code into a string.
##
## **Warning**:
## The behaviour of this procedure varies between Windows and POSIX systems.
## On Windows some OS calls can reset the error code to ``0`` causing this
## procedure to return ``0``. It is therefore advised to call this procedure
## immediately after an OS call fails. On POSIX systems this is not a problem.
when defined(nimscript):
discard
elif defined(windows):
result = OSErrorCode(getLastError())
else:
result = OSErrorCode(errno)
{.pop.}

2
lib/pure/ioselectors.nim
View file

@ -208,7 +208,7 @@ else:
import locks
type
SharedArray {.unchecked.}[T] = array[0..100, T]
SharedArray[T] = UncheckedArray[T]
proc allocSharedArray[T](nsize: int): ptr SharedArray[T] =
result = cast[ptr SharedArray[T]](allocShared0(sizeof(T) * nsize))

6
lib/pure/memfiles.nim
View file

@ -123,7 +123,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
result.size = 0
when defined(windows):
template fail(errCode: OSErrorCode, msg: expr) =
template fail(errCode: OSErrorCode, msg: untyped) =
rollback()
if result.fHandle != 0: discard closeHandle(result.fHandle)
if result.mapHandle != 0: discard closeHandle(result.mapHandle)
@ -131,7 +131,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
# return false
#raise newException(EIO, msg)
template callCreateFile(winApiProc, filename: expr): expr =
template callCreateFile(winApiProc, filename): untyped =
winApiProc(
filename,
# GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
@ -198,7 +198,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
result.fHandle = INVALID_HANDLE_VALUE
else:
template fail(errCode: OSErrorCode, msg: expr) =
template fail(errCode: OSErrorCode, msg: string) =
rollback()
if result.handle != -1: discard close(result.handle)
raiseOSError(errCode)

2
lib/pure/nativesockets.nim
View file

@ -233,7 +233,7 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET,
# OpenBSD doesn't support AI_V4MAPPED and doesn't define the macro AI_V4MAPPED.
# FreeBSD doesn't support AI_V4MAPPED but defines the macro.
# https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=198092
when not defined(freebsd) and not defined(openbsd) and not defined(netbsd):
when not defined(freebsd) and not defined(openbsd) and not defined(netbsd) and not defined(android):
if domain == AF_INET6:
hints.ai_flags = AI_V4MAPPED
var gaiResult = getaddrinfo(address, $port, addr(hints), result)

2
lib/pure/net.nim
View file

@ -797,7 +797,7 @@ when false: #defineSsl:
##
## ``AcceptNoClient`` will be returned when no client is currently attempting
## to connect.
template doHandshake(): stmt =
template doHandshake(): untyped =
when defineSsl:
if server.isSSL:
client.setBlocking(false)

336
lib/pure/os.nim
View file

@ -29,150 +29,17 @@ else:
import ospaths
export ospaths
when defined(posix):
when NoFakeVars:
const pathMax = 5000 # doesn't matter really. The concept of PATH_MAX
# doesn't work anymore on modern OSes.
else:
var
pathMax {.importc: "PATH_MAX", header: "<stdlib.h>".}: cint
proc c_remove(filename: cstring): cint {.
importc: "remove", header: "<stdio.h>".}
proc c_rename(oldname, newname: cstring): cint {.
importc: "rename", header: "<stdio.h>".}
proc c_system(cmd: cstring): cint {.
importc: "system", header: "<stdlib.h>".}
proc c_strerror(errnum: cint): cstring {.
importc: "strerror", header: "<string.h>".}
proc c_strlen(a: cstring): cint {.
importc: "strlen", header: "<string.h>", noSideEffect.}
proc c_getenv(env: cstring): cstring {.
importc: "getenv", header: "<stdlib.h>".}
proc c_putenv(env: cstring): cint {.
importc: "putenv", header: "<stdlib.h>".}
proc c_free(p: pointer) {.
importc: "free", header: "<stdlib.h>".}
var errno {.importc, header: "<errno.h>".}: cint
proc osErrorMsg*(): string {.rtl, extern: "nos$1", deprecated.} =
## Retrieves the operating system's error flag, ``errno``.
## On Windows ``GetLastError`` is checked before ``errno``.
## Returns "" if no error occurred.
##
## **Deprecated since version 0.9.4**: use the other ``osErrorMsg`` proc.
result = ""
when defined(Windows):
var err = getLastError()
if err != 0'i32:
when useWinUnicode:
var msgbuf: WideCString
if formatMessageW(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(cast[pointer](msgbuf))
else:
var msgbuf: cstring
if formatMessageA(0x00000100 or 0x00001000 or 0x00000200 or 0x000000FF,
nil, err, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(msgbuf)
if errno != 0'i32:
result = $os.c_strerror(errno)
{.push warning[deprecated]: off.}
proc raiseOSError*(msg: string = "") {.noinline, rtl, extern: "nos$1",
deprecated.} =
## raises an OSError exception with the given message ``msg``.
## If ``msg == ""``, the operating system's error flag
## (``errno``) is converted to a readable error message. On Windows
## ``GetLastError`` is checked before ``errno``.
## If no error flag is set, the message ``unknown OS error`` is used.
##
## **Deprecated since version 0.9.4**: use the other ``raiseOSError`` proc.
if len(msg) == 0:
var m = osErrorMsg()
raise newException(OSError, if m.len > 0: m else: "unknown OS error")
else:
raise newException(OSError, msg)
{.pop.}
when not defined(nimfix):
{.deprecated: [osError: raiseOSError].}
proc `==`*(err1, err2: OSErrorCode): bool {.borrow.}
proc `$`*(err: OSErrorCode): string {.borrow.}
proc osErrorMsg*(errorCode: OSErrorCode): string =
## Converts an OS error code into a human readable string.
##
## The error code can be retrieved using the ``osLastError`` proc.
##
## If conversion fails, or ``errorCode`` is ``0`` then ``""`` will be
## returned.
##
## On Windows, the ``-d:useWinAnsi`` compilation flag can be used to
## make this procedure use the non-unicode Win API calls to retrieve the
## message.
result = ""
when defined(Windows):
if errorCode != OSErrorCode(0'i32):
when useWinUnicode:
var msgbuf: WideCString
if formatMessageW(0x00000100 or 0x00001000 or 0x00000200,
nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(cast[pointer](msgbuf))
else:
var msgbuf: cstring
if formatMessageA(0x00000100 or 0x00001000 or 0x00000200,
nil, errorCode.int32, 0, addr(msgbuf), 0, nil) != 0'i32:
result = $msgbuf
if msgbuf != nil: localFree(msgbuf)
else:
if errorCode != OSErrorCode(0'i32):
result = $os.c_strerror(errorCode.int32)
proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} =
## Raises an ``OSError`` exception. The ``errorCode`` will determine the
## message, ``osErrorMsg`` will be used to get this message.
##
## The error code can be retrieved using the ``osLastError`` proc.
##
## If the error code is ``0`` or an error message could not be retrieved,
## the message ``unknown OS error`` will be used.
var e: ref OSError; new(e)
e.errorCode = errorCode.int32
if additionalInfo.len == 0:
e.msg = osErrorMsg(errorCode)
else:
e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo
if e.msg == "":
e.msg = "unknown OS error"
raise e
{.push stackTrace:off.}
proc osLastError*(): OSErrorCode =
## Retrieves the last operating system error code.
##
## This procedure is useful in the event when an OS call fails. In that case
## this procedure will return the error code describing the reason why the
## OS call failed. The ``OSErrorMsg`` procedure can then be used to convert
## this code into a string.
##
## **Warning**:
## The behaviour of this procedure varies between Windows and POSIX systems.
## On Windows some OS calls can reset the error code to ``0`` causing this
## procedure to return ``0``. It is therefore advised to call this procedure
## immediately after an OS call fails. On POSIX systems this is not a problem.
when defined(windows):
result = OSErrorCode(getLastError())
else:
result = OSErrorCode(errno)
{.pop.}
when defined(windows):
when useWinUnicode:
@ -252,6 +119,60 @@ proc dirExists*(dir: string): bool {.inline.} =
## Synonym for existsDir
existsDir(dir)
when not defined(windows):
proc checkSymlink(path: string): bool =
var rawInfo: Stat
if lstat(path, rawInfo) < 0'i32: result = false
else: result = S_ISLNK(rawInfo.st_mode)
const
ExeExts* = when defined(windows): ["exe", "cmd", "bat"] else: [""] ## \
## platform specific file extension for executables. On Windows
## ``["exe", "cmd", "bat"]``, on Posix ``[""]``.
proc findExe*(exe: string, followSymlinks: bool = true;
extensions: openarray[string]=ExeExts): string {.
tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect].} =
## Searches for `exe` in the current working directory and then
## in directories listed in the ``PATH`` environment variable.
## Returns "" if the `exe` cannot be found. `exe`
## is added the `ExeExts <#ExeExts>`_ file extensions if it has none.
## If the system supports symlinks it also resolves them until it
## meets the actual file. This behavior can be disabled if desired.
for ext in extensions:
result = addFileExt(exe, ext)
if existsFile(result): return
var path = string(getEnv("PATH"))
for candidate in split(path, PathSep):
when defined(windows):
var x = (if candidate[0] == '"' and candidate[^1] == '"':
substr(candidate, 1, candidate.len-2) else: candidate) /
exe
else:
var x = expandTilde(candidate) / exe
for ext in extensions:
var x = addFileExt(x, ext)
if existsFile(x):
when not defined(windows):
while followSymlinks: # doubles as if here
if x.checkSymlink:
var r = newString(256)
var len = readlink(x, r, 256)
if len < 0:
raiseOSError(osLastError())
if len > 256:
r = newString(len+1)
len = readlink(x, r, len)
setLen(r, len)
if isAbsolute(r):
x = r
else:
x = parentDir(x) / r
else:
break
return x
result = ""
proc getLastModificationTime*(file: string): Time {.rtl, extern: "nos$1".} =
## Returns the `file`'s last modification time.
when defined(posix):
@ -714,147 +635,6 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
else:
result = c_system(command)
# Environment handling cannot be put into RTL, because the ``envPairs``
# iterator depends on ``environment``.
var
envComputed {.threadvar.}: bool
environment {.threadvar.}: seq[string]
when defined(windows):
# because we support Windows GUI applications, things get really
# messy here...
when useWinUnicode:
when defined(cpp):
proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
importcpp: "(NI16*)wcschr((const wchar_t *)#, #)", header: "<string.h>".}
else:
proc strEnd(cstr: WideCString, c = 0'i32): WideCString {.
importc: "wcschr", header: "<string.h>".}
else:
proc strEnd(cstr: cstring, c = 0'i32): cstring {.
importc: "strchr", header: "<string.h>".}
proc getEnvVarsC() =
if not envComputed:
environment = @[]
when useWinUnicode:
var
env = getEnvironmentStringsW()
e = env
if e == nil: return # an error occurred
while true:
var eend = strEnd(e)
add(environment, $e)
e = cast[WideCString](cast[ByteAddress](eend)+2)
if eend[1].int == 0: break
discard freeEnvironmentStringsW(env)
else:
var
env = getEnvironmentStringsA()
e = env
if e == nil: return # an error occurred
while true:
var eend = strEnd(e)
add(environment, $e)
e = cast[cstring](cast[ByteAddress](eend)+1)
if eend[1] == '\0': break
discard freeEnvironmentStringsA(env)
envComputed = true
else:
const
useNSGetEnviron = defined(macosx) and not defined(ios)
when useNSGetEnviron:
# From the manual:
# Shared libraries and bundles don't have direct access to environ,
# which is only available to the loader ld(1) when a complete program
# is being linked.
# The environment routines can still be used, but if direct access to
# environ is needed, the _NSGetEnviron() routine, defined in
# <crt_externs.h>, can be used to retrieve the address of environ
# at runtime.
proc NSGetEnviron(): ptr cstringArray {.
importc: "_NSGetEnviron", header: "<crt_externs.h>".}
else:
var gEnv {.importc: "environ".}: cstringArray
proc getEnvVarsC() =
# retrieves the variables of char** env of C's main proc
if not envComputed:
environment = @[]
when useNSGetEnviron:
var gEnv = NSGetEnviron()[]
var i = 0
while true:
if gEnv[i] == nil: break
add environment, $gEnv[i]
inc(i)
envComputed = true
proc findEnvVar(key: string): int =
getEnvVarsC()
var temp = key & '='
for i in 0..high(environment):
if startsWith(environment[i], temp): return i
return -1
proc getEnv*(key: string): TaintedString {.tags: [ReadEnvEffect].} =
## Returns the value of the `environment variable`:idx: named `key`.
##
## If the variable does not exist, "" is returned. To distinguish
## whether a variable exists or it's value is just "", call
## `existsEnv(key)`.
var i = findEnvVar(key)
if i >= 0:
return TaintedString(substr(environment[i], find(environment[i], '=')+1))
else:
var env = c_getenv(key)
if env == nil: return TaintedString("")
result = TaintedString($env)
proc existsEnv*(key: string): bool {.tags: [ReadEnvEffect].} =
## Checks whether the environment variable named `key` exists.
## Returns true if it exists, false otherwise.
if c_getenv(key) != nil: return true
else: return findEnvVar(key) >= 0
proc putEnv*(key, val: string) {.tags: [WriteEnvEffect].} =
## Sets the value of the `environment variable`:idx: named `key` to `val`.
## If an error occurs, `EInvalidEnvVar` is raised.
# Note: by storing the string in the environment sequence,
# we guarantee that we don't free the memory before the program
# ends (this is needed for POSIX compliance). It is also needed so that
# the process itself may access its modified environment variables!
var indx = findEnvVar(key)
if indx >= 0:
environment[indx] = key & '=' & val
else:
add environment, (key & '=' & val)
indx = high(environment)
when defined(windows):
when useWinUnicode:
var k = newWideCString(key)
var v = newWideCString(val)
if setEnvironmentVariableW(k, v) == 0'i32: raiseOSError(osLastError())
else:
if setEnvironmentVariableA(key, val) == 0'i32: raiseOSError(osLastError())
else:
if c_putenv(environment[indx]) != 0'i32:
raiseOSError(osLastError())
iterator envPairs*(): tuple[key, value: TaintedString] {.tags: [ReadEnvEffect].} =
## Iterate over all `environments variables`:idx:. In the first component
## of the tuple is the name of the current variable stored, in the second
## its value.
getEnvVarsC()
for i in 0..high(environment):
var p = find(environment[i], '=')
yield (TaintedString(substr(environment[i], 0, p-1)),
TaintedString(substr(environment[i], p+1)))
# Templates for filtering directories and files
when defined(windows):
template isDir(f: WIN32_FIND_DATA): bool =
@ -1017,7 +797,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
y = dir / y
var k = pcFile
when defined(linux) or defined(macosx) or defined(bsd):
when defined(linux) or defined(macosx) or defined(bsd) or defined(genode):
if x.d_type != DT_UNKNOWN:
if x.d_type == DT_DIR: k = pcDir
if x.d_type == DT_LNK:
@ -1184,7 +964,9 @@ proc createSymlink*(src, dest: string) =
## Some OS's (such as Microsoft Windows) restrict the creation
## of symlinks to root users (administrators).
when defined(Windows):
let flag = dirExists(src).int32
# 2 is the SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE. This allows
# anyone with developer mode on to create a link
let flag = dirExists(src).int32 or 2
when useWinUnicode:
var wSrc = newWideCString(src)
var wDst = newWideCString(dest)
@ -1386,7 +1168,7 @@ proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect].} =
copyDir(source, dest)
removeDir(source)
include ospaths
#include ospaths
proc expandSymlink*(symlinkPath: string): string =
## Returns a string representing the path to which the symbolic link points.

1080
lib/pure/ospaths.nim
View file

File diff suppressed because it is too large Load diff

34
lib/pure/osproc.nim
View file

@ -119,7 +119,8 @@ proc execProcess*(command: string,
poUsePath,
poEvalCommand}): TaintedString {.
rtl, extern: "nosp$1",
tags: [ExecIOEffect, ReadIOEffect].}
tags: [ExecIOEffect, ReadIOEffect,
RootEffect].}
## A convenience procedure that executes ``command`` with ``startProcess``
## and returns its output as a string.
## WARNING: this function uses poEvalCommand by default for backward compatibility.
@ -131,7 +132,8 @@ proc execProcess*(command: string,
## # Note: outp may have an interleave of text from the nim compile
## # and any output from mytestfile when it runs
proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect].}
proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [ExecIOEffect,
ReadIOEffect, RootEffect].}
## Executes ``command`` and returns its error code. Standard input, output,
## error streams are inherited from the calling process. This operation
## is also often called `system`:idx:.
@ -145,7 +147,8 @@ proc startProcess*(command: string,
args: openArray[string] = [],
env: StringTableRef = nil,
options: set[ProcessOption] = {poStdErrToStdOut}):
Process {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadEnvEffect].}
Process {.rtl, extern: "nosp$1", tags: [ExecIOEffect, ReadEnvEffect,
RootEffect].}
## Starts a process. `Command` is the executable file, `workingDir` is the
## process's working directory. If ``workingDir == ""`` the current directory
## is used. `args` are the command line arguments that are passed to the
@ -170,7 +173,7 @@ proc startProcess*(command: string,
proc startCmd*(command: string, options: set[ProcessOption] = {
poStdErrToStdOut, poUsePath}): Process {.
tags: [ExecIOEffect, ReadEnvEffect], deprecated.} =
tags: [ExecIOEffect, ReadEnvEffect, RootEffect], deprecated.} =
## Deprecated - use `startProcess` directly.
result = startProcess(command=command, options=options + {poEvalCommand})
@ -721,7 +724,7 @@ elif not defined(useNimRtl):
inc(i)
type StartProcessData = object
sysCommand: cstring
sysCommand: string
sysArgs: cstringArray
sysEnv: cstringArray
workingDir: cstring
@ -735,13 +738,13 @@ elif not defined(useNimRtl):
not defined(useClone) and not defined(linux)
when useProcessAuxSpawn:
proc startProcessAuxSpawn(data: StartProcessData): Pid {.
tags: [ExecIOEffect, ReadEnvEffect], gcsafe.}
tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.}
else:
proc startProcessAuxFork(data: StartProcessData): Pid {.
tags: [ExecIOEffect, ReadEnvEffect], gcsafe.}
tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], gcsafe.}
{.push stacktrace: off, profiler: off.}
proc startProcessAfterFork(data: ptr StartProcessData) {.
tags: [ExecIOEffect, ReadEnvEffect], cdecl, gcsafe.}
tags: [ExecIOEffect, ReadEnvEffect, ReadDirEffect, RootEffect], cdecl, gcsafe.}
{.pop.}
proc startProcess(command: string,
@ -762,7 +765,8 @@ elif not defined(useNimRtl):
var sysCommand: string
var sysArgsRaw: seq[string]
if poEvalCommand in options:
sysCommand = "/bin/sh"
const useShPath {.strdefine.} = "/bin/sh"
sysCommand = useShPath
sysArgsRaw = @[sysCommand, "-c", command]
assert args.len == 0, "`args` has to be empty when using poEvalCommand."
else:
@ -784,7 +788,7 @@ elif not defined(useNimRtl):
defer: deallocCStringArray(sysEnv)
var data: StartProcessData
data.sysCommand = sysCommand
shallowCopy(data.sysCommand, sysCommand)
data.sysArgs = sysArgs
data.sysEnv = sysEnv
data.pStdin = pStdin
@ -949,11 +953,10 @@ elif not defined(useNimRtl):
discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
if data.optionPoUsePath:
when defined(uClibc):
when defined(uClibc) or defined(linux):
# uClibc environment (OpenWrt included) doesn't have the full execvpe
discard execve(data.sysCommand, data.sysArgs, data.sysEnv)
elif defined(linux) and not defined(android):
discard execvpe(data.sysCommand, data.sysArgs, data.sysEnv)
let exe = findExe(data.sysCommand)
discard execve(exe, data.sysArgs, data.sysEnv)
else:
# MacOSX doesn't have execvpe, so we need workaround.
# On MacOSX we can arrive here only from fork, so this is safe:
@ -1264,7 +1267,8 @@ elif not defined(useNimRtl):
proc execCmdEx*(command: string, options: set[ProcessOption] = {
poStdErrToStdOut, poUsePath}): tuple[
output: TaintedString,
exitCode: int] {.tags: [ExecIOEffect, ReadIOEffect], gcsafe.} =
exitCode: int] {.tags:
[ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
## a convenience proc that runs the `command`, grabs all its output and
## exit code and returns both.
##

10
lib/pure/parsecfg.nim
View file

@ -320,9 +320,13 @@ proc rawGetTok(c: var CfgParser, tok: var Token) =
tok.literal = "="
of '-':
inc(c.bufpos)
if c.buf[c.bufpos] == '-': inc(c.bufpos)
tok.kind = tkDashDash
tok.literal = "--"
if c.buf[c.bufpos] == '-':
inc(c.bufpos)
tok.kind = tkDashDash
tok.literal = "--"
else:
dec(c.bufpos)
getSymbol(c, tok)
of ':':
tok.kind = tkColon
inc(c.bufpos)

88
lib/pure/parsesql.nim
View file

@ -496,6 +496,7 @@ type
nkPrimaryKey,
nkForeignKey,
nkNotNull,
nkNull,
nkStmtList,
nkDot,
@ -565,8 +566,13 @@ proc newNode(k: SqlNodeKind, s: string): SqlNode =
result.strVal = s
proc len*(n: SqlNode): int =
if isNil(n.sons): result = 0
else: result = n.sons.len
if n.kind in {nkIdent, nkStringLit, nkBitStringLit, nkHexStringLit,
nkIntegerLit, nkNumericLit}:
result = 0
else:
result = n.sons.len
proc `[]`*(n: SqlNode; i: int): SqlNode = n.sons[i]
proc add*(father, n: SqlNode) =
if isNil(father.sons): father.sons = @[]
@ -613,6 +619,9 @@ proc eat(p: var SqlParser, keyw: string) =
else:
sqlError(p, keyw.toUpper() & " expected")
proc opt(p: var SqlParser, kind: TokKind) =
if p.tok.kind == kind: getTok(p)
proc parseDataType(p: var SqlParser): SqlNode =
if isKeyw(p, "enum"):
result = newNode(nkEnumDef)
@ -705,7 +714,7 @@ proc primary(p: var SqlParser): SqlNode =
result = newNode(nkCall)
result.add(a)
getTok(p)
while true:
while p.tok.kind != tkParRi:
result.add(parseExpr(p))
if p.tok.kind == tkComma: getTok(p)
else: break
@ -776,9 +785,19 @@ proc parseConstraint(p: var SqlParser): SqlNode =
expectIdent(p)
result.add(newNode(nkIdent, p.tok.literal))
getTok(p)
eat(p, "check")
optKeyw(p, "check")
result.add(parseExpr(p))
proc parseParIdentList(p: var SqlParser, father: SqlNode) =
eat(p, tkParLe)
while true:
expectIdent(p)
father.add(newNode(nkIdent, p.tok.literal))
getTok(p)
if p.tok.kind != tkComma: break
getTok(p)
eat(p, tkParRi)
proc parseColumnConstraints(p: var SqlParser, result: SqlNode) =
while true:
if isKeyw(p, "default"):
@ -795,6 +814,9 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) =
getTok(p)
eat(p, "null")
result.add(newNode(nkNotNull))
elif isKeyw(p, "null"):
getTok(p)
result.add(newNode(nkNull))
elif isKeyw(p, "identity"):
getTok(p)
result.add(newNode(nkIdentity))
@ -807,6 +829,7 @@ proc parseColumnConstraints(p: var SqlParser, result: SqlNode) =
elif isKeyw(p, "constraint"):
result.add(parseConstraint(p))
elif isKeyw(p, "unique"):
getTok(p)
result.add(newNode(nkUnique))
else:
break
@ -829,16 +852,6 @@ proc parseIfNotExists(p: var SqlParser, k: SqlNodeKind): SqlNode =
else:
result = newNode(k)
proc parseParIdentList(p: var SqlParser, father: SqlNode) =
eat(p, tkParLe)
while true:
expectIdent(p)
father.add(newNode(nkIdent, p.tok.literal))
getTok(p)
if p.tok.kind != tkComma: break
getTok(p)
eat(p, tkParRi)
proc parseTableConstraint(p: var SqlParser): SqlNode =
if isKeyw(p, "primary"):
getTok(p)
@ -866,20 +879,34 @@ proc parseTableConstraint(p: var SqlParser): SqlNode =
else:
sqlError(p, "column definition expected")
proc parseUnique(p: var SqlParser): SqlNode =
result = parseExpr(p)
if result.kind == nkCall: result.kind = nkUnique
proc parseTableDef(p: var SqlParser): SqlNode =
result = parseIfNotExists(p, nkCreateTable)
expectIdent(p)
result.add(newNode(nkIdent, p.tok.literal))
getTok(p)
if p.tok.kind == tkParLe:
while true:
getTok(p)
if p.tok.kind == tkIdentifier or p.tok.kind == tkQuotedIdentifier:
getTok(p)
while p.tok.kind != tkParRi:
if isKeyw(p, "constraint"):
result.add parseConstraint(p)
elif isKeyw(p, "primary") or isKeyw(p, "foreign"):
result.add parseTableConstraint(p)
elif isKeyw(p, "unique"):
result.add parseUnique(p)
elif p.tok.kind == tkIdentifier or p.tok.kind == tkQuotedIdentifier:
result.add(parseColumnDef(p))
else:
result.add(parseTableConstraint(p))
if p.tok.kind != tkComma: break
getTok(p)
eat(p, tkParRi)
# skip additional crap after 'create table (...) crap;'
while p.tok.kind notin {tkSemicolon, tkEof}:
getTok(p)
proc parseTypeDef(p: var SqlParser): SqlNode =
result = parseIfNotExists(p, nkCreateType)
@ -1046,7 +1073,7 @@ proc parseSelect(p: var SqlParser): SqlNode =
getTok(p)
result.add(n)
proc parseStmt(p: var SqlParser): SqlNode =
proc parseStmt(p: var SqlParser; parent: SqlNode) =
if isKeyw(p, "create"):
getTok(p)
optKeyw(p, "cached")
@ -1058,21 +1085,23 @@ proc parseStmt(p: var SqlParser): SqlNode =
optKeyw(p, "unique")
optKeyw(p, "hash")
if isKeyw(p, "table"):
result = parseTableDef(p)
parent.add parseTableDef(p)
elif isKeyw(p, "type"):
result = parseTypeDef(p)
parent.add parseTypeDef(p)
elif isKeyw(p, "index"):
result = parseIndexDef(p)
parent.add parseIndexDef(p)
else:
sqlError(p, "TABLE expected")
elif isKeyw(p, "insert"):
result = parseInsert(p)
parent.add parseInsert(p)
elif isKeyw(p, "update"):
result = parseUpdate(p)
parent.add parseUpdate(p)
elif isKeyw(p, "delete"):
result = parseDelete(p)
parent.add parseDelete(p)
elif isKeyw(p, "select"):
result = parseSelect(p)
parent.add parseSelect(p)
elif isKeyw(p, "begin"):
getTok(p)
else:
sqlError(p, "CREATE expected")
@ -1089,9 +1118,8 @@ proc parse(p: var SqlParser): SqlNode =
## Syntax errors raise an `EInvalidSql` exception.
result = newNode(nkStmtList)
while p.tok.kind != tkEof:
var s = parseStmt(p)
parseStmt(p, result)
eat(p, tkSemicolon)
result.add(s)
if result.len == 1:
result = result.sons[0]
@ -1147,6 +1175,8 @@ proc ra(n: SqlNode, s: var string, indent: int) =
rs(n, s, indent)
of nkNotNull:
s.add(" not null")
of nkNull:
s.add(" null")
of nkDot:
ra(n.sons[0], s, indent)
s.add(".")
@ -1330,6 +1360,10 @@ proc renderSQL*(n: SqlNode): string =
result = ""
ra(n, result, 0)
proc `$`*(n: SqlNode): string =
## an alias for `renderSQL`.
renderSQL(n)
when not defined(testing) and isMainModule:
echo(renderSQL(parseSQL(newStringStream("""
CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic');

22
lib/pure/pegs.nim
View file

@ -139,7 +139,7 @@ proc addChoice(dest: var Peg, elem: Peg) =
else: add(dest, elem)
else: add(dest, elem)
template multipleOp(k: PegKind, localOpt: expr) =
template multipleOp(k: PegKind, localOpt: untyped) =
result.kind = k
result.sons = @[]
for x in items(a):
@ -328,32 +328,32 @@ proc newNonTerminal*(name: string, line, column: int): NonTerminal {.
result.line = line
result.col = column
template letters*: expr =
template letters*: Peg =
## expands to ``charset({'A'..'Z', 'a'..'z'})``
charSet({'A'..'Z', 'a'..'z'})
template digits*: expr =
template digits*: Peg =
## expands to ``charset({'0'..'9'})``
charSet({'0'..'9'})
template whitespace*: expr =
template whitespace*: Peg =
## expands to ``charset({' ', '\9'..'\13'})``
charSet({' ', '\9'..'\13'})
template identChars*: expr =
template identChars*: Peg =
## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})``
charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})
template identStartChars*: expr =
template identStartChars*: Peg =
## expands to ``charset({'A'..'Z', 'a'..'z', '_'})``
charSet({'a'..'z', 'A'..'Z', '_'})
template ident*: expr =
template ident*: Peg =
## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier
sequence(charSet({'a'..'z', 'A'..'Z', '_'}),
*charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}))
template natural*: expr =
template natural*: Peg =
## same as ``\d+``
+digits
@ -514,10 +514,10 @@ proc bounds*(c: Captures,
when not useUnicode:
type
Rune = char
template fastRuneAt(s, i, ch: expr) =
template fastRuneAt(s, i, ch) =
ch = s[i]
inc(i)
template runeLenAt(s, i: expr): expr = 1
template runeLenAt(s, i): untyped = 1
proc isAlpha(a: char): bool {.inline.} = return a in {'a'..'z','A'..'Z'}
proc isUpper(a: char): bool {.inline.} = return a in {'A'..'Z'}
@ -735,7 +735,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
else: result = -1
of pkRule, pkList: assert false
template fillMatches(s, caps, c: expr) =
template fillMatches(s, caps, c) =
for k in 0..c.ml-1:
let startIdx = c.matches[k][0]
let endIdx = c.matches[k][1]

4
lib/pure/random.nim
View file

@ -101,7 +101,7 @@ proc random*[T](a: openArray[T]): T =
## returns a random element from the openarray `a`.
result = a[random(a.low..a.len)]
proc randomize*(seed: int) {.benign.} =
proc randomize*(seed: int64) {.benign.} =
## Initializes the random number generator with a specific seed.
state.a0 = ui(seed shr 16)
state.a1 = ui(seed and 0xffff)
@ -123,7 +123,7 @@ when not defined(nimscript):
proc getMil(t: Time): int {.importcpp: "getTime", nodecl.}
randomize(getMil times.getTime())
else:
let time = int(times.epochTime() * 1_000_000_000)
let time = int64(times.epochTime() * 1_000_000_000)
randomize(time)
{.pop.}

4
lib/pure/streams.nim
View file

@ -411,12 +411,12 @@ when not defined(js):
result.writeDataImpl = fsWriteData
result.flushImpl = fsFlush
proc newFileStream*(filename: string, mode: FileMode = fmRead): FileStream =
proc newFileStream*(filename: string, mode: FileMode = fmRead, bufSize: int = -1): FileStream =
## creates a new stream from the file named `filename` with the mode `mode`.
## If the file cannot be opened, nil is returned. See the `system
## <system.html>`_ module for a list of available FileMode enums.
var f: File
if open(f, filename, mode): result = newFileStream(f)
if open(f, filename, mode, bufSize): result = newFileStream(f)
when true:

25
lib/pure/strscans.nim
View file

@ -187,7 +187,6 @@ overloaded to handle both single characters and sets of character.
if scanp(content, idx, +( ~{'\L', '\0'} -> entry.add(peekChar($input))), '\L'):
result.add entry
Calling ordinary Nim procs inside the macro is possible:
.. code-block:: nim
@ -253,6 +252,30 @@ is performed.
for r in collectLinks(body):
echo r
In this example both macros are combined seamlessly in order to maximise
efficiency and perform different checks.
.. code-block:: nim
iterator parseIps*(soup: string): string =
## ipv4 only!
const digits = {'0'..'9'}
var a, b, c, d: int
var buf = ""
var idx = 0
while idx < soup.len:
if scanp(soup, idx, (`digits`{1,3}, '.', `digits`{1,3}, '.',
`digits`{1,3}, '.', `digits`{1,3}) -> buf.add($_)):
discard buf.scanf("$i.$i.$i.$i", a, b, c, d)
if (a >= 0 and a <= 254) and
(b >= 0 and b <= 254) and
(c >= 0 and c <= 254) and
(d >= 0 and d <= 254):
yield buf
buf.setLen(0) # need to clear `buf` each time, cause it might contain garbage
idx.inc
]##

35
lib/pure/strutils.nim
View file

@ -1551,6 +1551,37 @@ proc replaceWord*(s, sub: string, by = ""): string {.noSideEffect,
# copy the rest:
add result, substr(s, i)
proc multiReplace*(s: string, replacements: varargs[(string, string)]): string {.noSideEffect.} =
## Same as replace, but specialized for doing multiple replacements in a single
## pass through the input string.
##
## Calling replace multiple times after each other is inefficient and result in too many allocations
## follwed by immediate deallocations as portions of the string gets replaced.
## multiReplace performs all replacements in a single pass.
##
## If the resulting string is not longer than the original input string, only a single
## memory allocation is required.
##
## The order of the replacements does matter. Earlier replacements are preferred over later
## replacements in the argument list.
result = newStringOfCap(s.len)
var i = 0
var fastChk: set[char] = {}
for tup in replacements: fastChk.incl(tup[0][0]) # Include first character of all replacements
while i < s.len:
block sIteration:
# Assume most chars in s are not candidates for any replacement operation
if s[i] in fastChk:
for tup in replacements:
if s.continuesWith(tup[0], i):
add result, tup[1]
inc(i, tup[0].len)
break sIteration
# No matching replacement found
# copy current character from s
add result, s[i]
inc(i)
proc delete*(s: var string, first, last: int) {.noSideEffect,
rtl, extern: "nsuDelete".} =
## Deletes in `s` the characters at position `first` .. `last`.
@ -2324,6 +2355,10 @@ when isMainModule:
doAssert " foo\n bar".indent(4, "Q") == "QQQQ foo\nQQQQ bar"
doAssert "abba".multiReplace(("a", "b"), ("b", "a")) == "baab"
doAssert "Hello World.".multiReplace(("ello", "ELLO"), ("World.", "PEOPLE!")) == "HELLO PEOPLE!"
doAssert "aaaa".multiReplace(("a", "aa"), ("aa", "bb")) == "aaaaaaaa"
doAssert isAlphaAscii('r')
doAssert isAlphaAscii('A')
doAssert(not isAlphaAscii('$'))

4
lib/pure/subexes.nim
View file

@ -46,12 +46,12 @@ type
num, i, lineLen: int
{.deprecated: [TFormatParser: FormatParser].}
template call(x: stmt) {.immediate.} =
template call(x: untyped): untyped =
p.i = i
x
i = p.i
template callNoLineLenTracking(x: stmt) {.immediate.} =
template callNoLineLenTracking(x: untyped): untyped =
let oldLineLen = p.lineLen
p.i = i
x

11
lib/pure/terminal.nim
View file

@ -578,7 +578,7 @@ template styledEchoProcessArg(f: File, cmd: TerminalCmd) =
when cmd == resetStyle:
resetAttributes(f)
macro styledWriteLine*(f: File, m: varargs[expr]): stmt =
macro styledWriteLine*(f: File, m: varargs[typed]): untyped =
## Similar to ``writeLine``, but treating terminal style arguments specially.
## When some argument is ``Style``, ``set[Style]``, ``ForegroundColor``,
## ``BackgroundColor`` or ``TerminalCmd`` then it is not sent directly to
@ -614,16 +614,13 @@ macro styledWriteLine*(f: File, m: varargs[expr]): stmt =
result.add(newCall(bindSym"write", f, newStrLitNode("\n")))
if reset: result.add(newCall(bindSym"resetAttributes", f))
macro callStyledEcho(args: varargs[expr]): stmt =
macro styledEcho*(args: varargs[untyped]): untyped =
## Echoes styles arguments to stdout using ``styledWriteLine``.
result = newCall(bindSym"styledWriteLine")
result.add(bindSym"stdout")
for arg in children(args[0][1]):
for arg in children(args):
result.add(arg)
template styledEcho*(args: varargs[expr]): expr =
## Echoes styles arguments to stdout using ``styledWriteLine``.
callStyledEcho(args)
proc getch*(): char =
## Read a single character from the terminal, blocking until it is entered.
## The character is not printed to the terminal.

17
lib/pure/times.nim
View file

@ -222,6 +222,12 @@ proc toSeconds*(time: Time): float {.tags: [], raises: [], benign.}
proc `-`*(a, b: Time): int64 {.
rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign.}
## computes the difference of two calendar times. Result is in seconds.
## .. code-block:: nim
##
## let a = fromSeconds(1_000_000_000)
## let b = fromSeconds(1_500_000_000)
## echo initInterval(seconds=int(b - a))
## # (milliseconds: 0, seconds: 20, minutes: 53, hours: 0, days: 5787, months: 0, years: 0)
proc `<`*(a, b: Time): bool {.
rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect.} =
@ -301,6 +307,11 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval =
result.years = carryO + ti1.years + ti2.years
proc `-`*(ti: TimeInterval): TimeInterval =
## Reverses a time interval
## .. code-block:: nim
##
## let day = -initInterval(hours=24)
## echo day # -> (milliseconds: 0, seconds: 0, minutes: 0, hours: 0, days: -1, months: 0, years: 0)
result = TimeInterval(
milliseconds: -ti.milliseconds,
seconds: -ti.seconds,
@ -313,6 +324,12 @@ proc `-`*(ti: TimeInterval): TimeInterval =
proc `-`*(ti1, ti2: TimeInterval): TimeInterval =
## Subtracts TimeInterval ``ti1`` from ``ti2``.
## .. code-block:: nim
##
## let a = fromSeconds(1_000_000_000)
## let b = fromSeconds(1_500_000_000)
## echo b.toTimeInterval - a.toTimeInterval
## # (milliseconds: 0, seconds: -40, minutes: -6, hours: 1, days: -2, months: -2, years: 16)
result = ti1 + (-ti2)
proc isLeapYear*(year: int): bool =

55
lib/pure/unittest.nim
View file

@ -13,7 +13,24 @@
##
## The test status and name is printed after any output or traceback.
##
## Example:
## Tests can be nested, however failure of a nested test will not mark the
## parent test as failed. Setup and teardown are inherited. Setup can be
## overridden locally.
##
## Compiled test files return the number of failed test as exit code, while
## ``nim c -r <testfile.nim>`` exits with 0 or 1
##
## Running a single test
## ---------------------
##
## Simply specify the test name as a command line argument.
##
## .. code::
##
## nim c -r test "my super awesome test name"
##
## Example
## -------
##
## .. code:: nim
##
@ -42,16 +59,9 @@
## discard v[4]
##
## echo "suite teardown: run once after the tests"
##
##
## Tests can be nested, however failure of a nested test will not mark the
## parent test as failed. Setup and teardown are inherited. Setup can be
## overridden locally.
## Compiled test files return the number of failed test as exit code, while
## nim c -r <testfile.nim> exits with 0 or 1
import
macros, strutils, streams, times
macros, strutils, streams, times, sets
when declared(stdout):
import os
@ -111,6 +121,7 @@ var
checkpoints {.threadvar.}: seq[string]
formatters {.threadvar.}: seq[OutputFormatter]
testsToRun {.threadvar.}: HashSet[string]
when declared(stdout):
abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR")
@ -290,12 +301,22 @@ method suiteEnded*(formatter: JUnitOutputFormatter) =
formatter.stream.writeLine("\t</testsuite>")
proc shouldRun(testName: string): bool =
result = true
if testsToRun.len == 0:
return true
proc ensureFormattersInitialized() =
result = testName in testsToRun
proc ensureInitialized() =
if formatters == nil:
formatters = @[OutputFormatter(defaultConsoleFormatter())]
if not testsToRun.isValid:
testsToRun.init()
when declared(os):
# Read tests to run from the command line.
for i in 1 .. paramCount():
testsToRun.incl(paramStr(i))
# These two procs are added as workarounds for
# https://github.com/nim-lang/Nim/issues/5549
proc suiteEnded() =
@ -335,7 +356,7 @@ template suite*(name, body) {.dirty.} =
## [Suite] test suite for addition
## [OK] 2 + 2 = 4
## [OK] (2 + -2) != 4
bind formatters, ensureFormattersInitialized, suiteEnded
bind formatters, ensureInitialized, suiteEnded
block:
template setup(setupBody: untyped) {.dirty, used.} =
@ -348,7 +369,7 @@ template suite*(name, body) {.dirty.} =
let testSuiteName {.used.} = name
ensureFormattersInitialized()
ensureInitialized()
try:
for formatter in formatters:
formatter.suiteStarted(name)
@ -370,9 +391,9 @@ template test*(name, body) {.dirty.} =
## .. code-block::
##
## [OK] roses are red
bind shouldRun, checkpoints, formatters, ensureFormattersInitialized, testEnded
bind shouldRun, checkpoints, formatters, ensureInitialized, testEnded
ensureFormattersInitialized()
ensureInitialized()
if shouldRun(name):
checkpoints = @[]
@ -433,14 +454,14 @@ template fail* =
## fail()
##
## outputs "Checkpoint A" before quitting.
bind ensureFormattersInitialized
bind ensureInitialized
when declared(testStatusIMPL):
testStatusIMPL = FAILED
else:
programResult += 1
ensureFormattersInitialized()
ensureInitialized()
# var stackTrace: string = nil
for formatter in formatters:

4
lib/pure/uri.nim
View file

@ -81,7 +81,7 @@ proc parsePath(uri: string, i: var int, result: var Uri) =
i.inc parseUntil(uri, result.path, {'?', '#'}, i)
# The 'mailto' scheme's PATH actually contains the hostname/username
if result.scheme.toLower == "mailto":
if cmpIgnoreCase(result.scheme, "mailto") == 0:
parseAuthority(result.path, result)
result.path.setLen(0)
@ -215,7 +215,7 @@ proc combine*(base: Uri, reference: Uri): Uri =
## let bar = combine(parseUri("http://example.com/foo/bar/"), parseUri("baz"))
## assert bar.path == "/foo/bar/baz"
template setAuthority(dest, src: expr): stmt =
template setAuthority(dest, src): untyped =
dest.hostname = src.hostname
dest.username = src.username
dest.port = src.port

2
lib/pure/xmltree.nim
View file

@ -338,7 +338,7 @@ proc xmlConstructor(e: NimNode): NimNode {.compileTime.} =
else:
result = newCall("newXmlTree", toStrLit(a))
macro `<>`*(x: expr): expr {.immediate.} =
macro `<>`*(x: untyped): untyped =
## Constructor macro for XML. Example usage:
##
## .. code-block:: nim

134
lib/system.nim
View file

@ -80,9 +80,10 @@ type
`nil` {.magic: "Nil".}
expr* {.magic: Expr, deprecated.} ## meta type to denote an expression (for templates)
## **Deprecated** since version 0.15. Use ``untyped`` instead.
## **Deprecated** since version 0.15. Use ``untyped`` instead.
stmt* {.magic: Stmt, deprecated.} ## meta type to denote a statement (for templates)
## **Deprecated** since version 0.15. Use ``typed`` instead.
## **Deprecated** since version 0.15. Use ``typed`` instead.
void* {.magic: "VoidType".} ## meta type to denote the absence of any type
auto* {.magic: Expr.} ## meta type for automatic type determination
any* = distinct auto ## meta type for any supported type
@ -230,31 +231,6 @@ proc reset*[T](obj: var T) {.magic: "Reset", noSideEffect.}
## resets an object `obj` to its initial (binary zero) value. This needs to
## be called before any possible `object branch transition`:idx:.
# for low and high the return type T may not be correct, but
# we handle that with compiler magic in semLowHigh()
proc high*[T](x: T): T {.magic: "High", noSideEffect.}
## returns the highest possible index of an array, a sequence, a string or
## the highest possible value of an ordinal value `x`. As a special
## semantic rule, `x` may also be a type identifier.
## ``high(int)`` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:.
##
## .. code-block:: nim
## var arr = [1,2,3,4,5,6,7]
## high(arr) #=> 6
## high(2) #=> 9223372036854775807
## high(int) #=> 9223372036854775807
proc low*[T](x: T): T {.magic: "Low", noSideEffect.}
## returns the lowest possible index of an array, a sequence, a string or
## the lowest possible value of an ordinal value `x`. As a special
## semantic rule, `x` may also be a type identifier.
##
## .. code-block:: nim
## var arr = [1,2,3,4,5,6,7]
## low(arr) #=> 0
## low(2) #=> -9223372036854775808
## low(int) #=> -9223372036854775808
type
range*{.magic: "Range".}[T] ## Generic type to construct range types.
array*{.magic: "Array".}[I, T] ## Generic type to construct
@ -267,6 +243,45 @@ type
seq*{.magic: "Seq".}[T] ## Generic type to construct sequences.
set*{.magic: "Set".}[T] ## Generic type to construct bit sets.
UncheckedArray* {.unchecked.}[T] = array[0, T]
## Array with no bounds checking
proc high*[T: Ordinal](x: T): T {.magic: "High", noSideEffect.}
## returns the highest possible index of an array, a sequence, a string or
## the highest possible value of an ordinal value `x`. As a special
## semantic rule, `x` may also be a type identifier.
## ``high(int)`` is Nim's way of writing `INT_MAX`:idx: or `MAX_INT`:idx:.
##
## .. code-block:: nim
## var arr = [1,2,3,4,5,6,7]
## high(arr) #=> 6
## high(2) #=> 9223372036854775807
## high(int) #=> 9223372036854775807
proc high*[T: Ordinal](x: typeDesc[T]): T {.magic: "High", noSideEffect.}
proc high*[T](x: openArray[T]): int {.magic: "High", noSideEffect.}
proc high*[I, T](x: array[I, T]): I {.magic: "High", noSideEffect.}
proc high*[I, T](x: typeDesc[array[I, T]]): I {.magic: "High", noSideEffect.}
proc high*(x: cstring): int {.magic: "High", noSideEffect.}
proc high*(x: string): int {.magic: "High", noSideEffect.}
proc low*[T: Ordinal](x: typeDesc[T]): T {.magic: "Low", noSideEffect.}
proc low*[T](x: openArray[T]): int {.magic: "Low", noSideEffect.}
proc low*[I, T](x: array[I, T]): I {.magic: "Low", noSideEffect.}
proc low*[T](x: T): T {.magic: "Low", noSideEffect.}
proc low*[I, T](x: typeDesc[array[I, T]]): I {.magic: "Low", noSideEffect.}
proc low*(x: cstring): int {.magic: "Low", noSideEffect.}
proc low*(x: string): int {.magic: "Low", noSideEffect.}
## returns the lowest possible index of an array, a sequence, a string or
## the lowest possible value of an ordinal value `x`. As a special
## semantic rule, `x` may also be a type identifier.
##
## .. code-block:: nim
## var arr = [1,2,3,4,5,6,7]
## low(arr) #=> 0
## low(2) #=> -9223372036854775808
## low(int) #=> -9223372036854775808
when defined(nimArrIdx):
# :array|openarray|string|seq|cstring|tuple
proc `[]`*[I: Ordinal;T](a: T; i: I): T {.
@ -380,8 +395,6 @@ include "system/inclrtl"
const NoFakeVars* = defined(nimscript) ## true if the backend doesn't support \
## "fake variables" like 'var EBADF {.importc.}: cint'.
const ArrayDummySize = when defined(cpu16): 10_000 else: 100_000_000
when not defined(JS):
type
TGenericSeq {.compilerproc, pure, inheritable.} = object
@ -389,10 +402,9 @@ when not defined(JS):
when defined(gogc):
elemSize: int
PGenericSeq {.exportc.} = ptr TGenericSeq
UncheckedCharArray {.unchecked.} = array[0..ArrayDummySize, char]
# len and space without counting the terminating zero:
NimStringDesc {.compilerproc, final.} = object of TGenericSeq
data: UncheckedCharArray
data: UncheckedArray[char]
NimString = ptr NimStringDesc
when not defined(JS) and not defined(nimscript):
@ -413,7 +425,7 @@ type
## is an int type ranging from one to the maximum value
## of an int. This type is often useful for documentation and debugging.
RootObj* {.exportc: "TNimObject", inheritable.} =
RootObj* {.compilerProc, inheritable.} =
object ## the root of Nim's object hierarchy. Objects should
## inherit from RootObj or one of its descendants. However,
## objects that have no ancestor are allowed.
@ -421,7 +433,7 @@ type
RootEffect* {.compilerproc.} = object of RootObj ## \
## base effect class; each effect should
## inherit from `TEffect` unless you know what
## inherit from `RootEffect` unless you know what
## you doing.
TimeEffect* = object of RootEffect ## Time effect.
IOEffect* = object of RootEffect ## IO effect.
@ -1174,6 +1186,8 @@ proc `is` *[T, S](x: T, y: S): bool {.magic: "Is", noSideEffect.}
template `isnot` *(x, y: untyped): untyped = not (x is y)
## Negated version of `is`. Equivalent to ``not(x is y)``.
proc `of` *[T, S](x: typeDesc[T], y: typeDesc[S]): bool {.magic: "Of", noSideEffect.}
proc `of` *[T, S](x: T, y: typeDesc[S]): bool {.magic: "Of", noSideEffect.}
proc `of` *[T, S](x: T, y: S): bool {.magic: "Of", noSideEffect.}
## Checks if `x` has a type of `y`
##
@ -1312,7 +1326,7 @@ const
hostCPU* {.magic: "HostCPU".}: string = ""
## a string that describes the host CPU. Possible values:
## "i386", "alpha", "powerpc", "powerpc64", "powerpc64el", "sparc",
## "amd64", "mips", "mipsel", "arm", "arm64".
## "amd64", "mips", "mipsel", "arm", "arm64", "mips64", "mips64el".
seqShallowFlag = low(int)
@ -1599,8 +1613,7 @@ type # these work for most platforms:
culonglong* {.importc: "unsigned long long", nodecl.} = uint64
## This is the same as the type ``unsigned long long`` in *C*.
cstringArray* {.importc: "char**", nodecl.} = ptr
array[0..ArrayDummySize, cstring]
cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring]
## This is binary compatible to the type ``char**`` in *C*. The array's
## high value is large enough to disable bounds checking in practice.
## Use `cstringArrayToSeq` to convert it into a ``seq[string]``.
@ -1951,30 +1964,34 @@ iterator countdown*[T](a, b: T, step = 1): T {.inline.} =
yield res
dec(res, step)
template countupImpl(incr: untyped) {.oldimmediate, dirty.} =
when T is IntLikeForCount:
var res = int(a)
while res <= int(b):
yield T(res)
incr
else:
var res: T = T(a)
while res <= b:
yield res
incr
iterator countup*[S, T](a: S, b: T, step = 1): T {.inline.} =
## Counts from ordinal value `a` up to `b` (inclusive) with the given
## step count. `S`, `T` may be any ordinal type, `step` may only
## be positive. **Note**: This fails to count to ``high(int)`` if T = int for
## efficiency reasons.
countupImpl:
inc(res, step)
when T is IntLikeForCount:
var res = int(a)
while res <= int(b):
yield T(res)
inc(res, step)
else:
var res: T = T(a)
while res <= b:
yield res
inc(res, step)
iterator `..`*[S, T](a: S, b: T): T {.inline.} =
## An alias for `countup`.
countupImpl:
inc(res)
when T is IntLikeForCount:
var res = int(a)
while res <= int(b):
yield T(res)
inc(res)
else:
var res: T = T(a)
while res <= b:
yield res
inc(res)
iterator `||`*[S, T](a: S, b: T, annotation=""): T {.
inline, magic: "OmpParFor", sideEffect.} =
@ -2551,7 +2568,7 @@ const NimStackTrace = compileOption("stacktrace")
template coroutinesSupportedPlatform(): bool =
when defined(sparc) or defined(ELATE) or compileOption("gc", "v2") or
defined(boehmgc) or defined(gogc) or defined(nogc) or defined(gcStack) or
defined(boehmgc) or defined(gogc) or defined(nogc) or defined(gcRegions) or
defined(gcMarkAndSweep):
false
else:
@ -2752,10 +2769,10 @@ when not defined(JS): #and not defined(nimscript):
{.push stack_trace: off, profiler:off.}
when hasAlloc:
when not defined(gcStack):
when not defined(gcRegions):
proc initGC() {.gcsafe.}
when not defined(boehmgc) and not defined(useMalloc) and
not defined(gogc) and not defined(gcStack):
not defined(gogc) and not defined(gcRegions):
proc initAllocator() {.inline.}
proc initStackBottom() {.inline, compilerproc.} =
@ -3040,7 +3057,8 @@ when not defined(JS): #and not defined(nimscript):
## creates a NULL terminated cstringArray from `a`. The result has to
## be freed with `deallocCStringArray` after it's not needed anymore.
result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring)))
let x = cast[ptr array[0..ArrayDummySize, string]](a)
let x = cast[ptr UncheckedArray[string]](a)
for i in 0 .. a.high:
result[i] = cast[cstring](alloc0(x[i].len+1))
copyMem(result[i], addr(x[i][0]), x[i].len)
@ -3269,7 +3287,7 @@ when not defined(JS): #and not defined(nimscript):
proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} =
## can be used to determine if a first class iterator has finished.
{.emit: """
`result` = *((NI*) `x`.ClE_0) < 0;
`result` = ((NI*) `x`.ClE_0)[1] < 0;
""".}
elif defined(JS):
@ -3752,7 +3770,7 @@ when hasAlloc:
proc locals*(): RootObj {.magic: "Plugin", noSideEffect.} =
## generates a tuple constructor expression listing all the local variables
## in the current scope. This is quite fast as it does not rely
## on any debug or runtime information. Note that in constrast to what
## on any debug or runtime information. Note that in contrast to what
## the official signature says, the return type is not ``RootObj`` but a
## tuple of a structure that depends on the current scope. Example:
##

2
lib/system/atomics.nim
View file

@ -172,7 +172,7 @@ elif defined(vcc) and hasThreadSupport:
header: "<intrin.h>".}
else:
proc addAndFetch*(p: ptr int, val: int): int {.
importcpp: "_InterlockedExchangeAdd(static_cast<NI volatile *>(#), #)",
importcpp: "_InterlockedExchangeAdd(reinterpret_cast<LONG volatile *>(#), static_cast<LONG>(#))",
header: "<intrin.h>".}
else:
when sizeof(int) == 8:

5
lib/system/cellsets.nim
View file

@ -30,13 +30,12 @@ type
key: ByteAddress # start address at bit 0
bits: array[BitIndex, int] # a bit vector
PPageDescArray = ptr array[0..1000_000, PPageDesc]
PPageDescArray = ptr UncheckedArray[PPageDesc]
CellSet {.final, pure.} = object
counter, max: int
head: PPageDesc
data: PPageDescArray
PCellArray = ptr array[0..100_000_000, PCell]
PCellArray = ptr UncheckedArray[PCell]
CellSeq {.final, pure.} = object
len, cap: int
d: PCellArray

45
lib/system/channels.nim
View file

@ -22,7 +22,7 @@ when not declared(NimString):
type
pbytes = ptr array[0.. 0xffff, byte]
RawChannel {.pure, final.} = object ## msg queue for a thread
rd, wr, count, mask: int
rd, wr, count, mask, maxItems: int
data: pbytes
lock: SysLock
cond: SysCond
@ -37,11 +37,12 @@ type
const ChannelDeadMask = -2
proc initRawChannel(p: pointer) =
proc initRawChannel(p: pointer, maxItems: int) =
var c = cast[PRawChannel](p)
initSysLock(c.lock)
initSysCond(c.cond)
c.mask = -1
c.maxItems = maxItems
proc deinitRawChannel(p: pointer) =
var c = cast[PRawChannel](p)
@ -203,28 +204,41 @@ proc rawRecv(q: PRawChannel, data: pointer, typ: PNimType) =
storeAux(data, addr(q.data[q.rd * typ.size]), typ, q, mLoad)
q.rd = (q.rd + 1) and q.mask
template lockChannel(q: expr, action: stmt) {.immediate.} =
template lockChannel(q, action): untyped =
acquireSys(q.lock)
action
releaseSys(q.lock)
template sendImpl(q: expr) {.immediate.} =
proc sendImpl(q: PRawChannel, typ: PNimType, msg: pointer, noBlock: bool): bool =
if q.mask == ChannelDeadMask:
sysFatal(DeadThreadError, "cannot send message; thread died")
acquireSys(q.lock)
var typ = cast[PNimType](getTypeInfo(msg))
rawSend(q, unsafeAddr(msg), typ)
if q.maxItems > 0:
# Wait until count is less than maxItems
if noBlock and q.count >= q.maxItems:
releaseSys(q.lock)
return
while q.count >= q.maxItems:
waitSysCond(q.cond, q.lock)
rawSend(q, msg, typ)
q.elemType = typ
releaseSys(q.lock)
signalSysCond(q.cond)
result = true
proc send*[TMsg](c: var Channel[TMsg], msg: TMsg) =
proc send*[TMsg](c: var Channel[TMsg], msg: TMsg) {.inline.} =
## sends a message to a thread. `msg` is deeply copied.
var q = cast[PRawChannel](addr(c))
sendImpl(q)
discard sendImpl(cast[PRawChannel](addr c), cast[PNimType](getTypeInfo(msg)), unsafeAddr(msg), false)
proc trySend*[TMsg](c: var Channel[TMsg], msg: TMsg): bool {.inline.} =
## Tries to send a message to a thread. `msg` is deeply copied. Doesn't block.
## Returns `false` if the message was not sent because number of pending items
## in the cannel exceeded `maxItems`.
sendImpl(cast[PRawChannel](addr c), cast[PNimType](getTypeInfo(msg)), unsafeAddr(msg), true)
proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) =
# to save space, the generic is as small as possible
q.ready = true
while q.count <= 0:
waitSysCond(q.cond, q.lock)
@ -233,6 +247,9 @@ proc llRecv(q: PRawChannel, res: pointer, typ: PNimType) =
releaseSys(q.lock)
sysFatal(ValueError, "cannot receive message of wrong type")
rawRecv(q, res, typ)
if q.maxItems > 0 and q.count == q.maxItems - 1:
# Parent thread is awaiting in send. Wake it up.
signalSysCond(q.cond)
proc recv*[TMsg](c: var Channel[TMsg]): TMsg =
## receives a message from the channel `c`. This blocks until
@ -267,9 +284,11 @@ proc peek*[TMsg](c: var Channel[TMsg]): int =
else:
result = -1
proc open*[TMsg](c: var Channel[TMsg]) =
## opens a channel `c` for inter thread communication.
initRawChannel(addr(c))
proc open*[TMsg](c: var Channel[TMsg], maxItems: int = 0) =
## opens a channel `c` for inter thread communication. The `send` operation
## will block until number of unprocessed items is less than `maxItems`.
## For unlimited queue set `maxItems` to 0.
initRawChannel(addr(c), maxItems)
proc close*[TMsg](c: var Channel[TMsg]) =
## closes a channel `c` and frees its associated resources.

6
lib/system/deepcopy.nim
View file

@ -148,11 +148,11 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType; tab: var PtrTable) =
let realType = x.typ
sysAssert realType == mt, " types do differ"
# this version should work for any possible GC:
let size = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[].size else: mt.base.size
let z = newObj(mt, size)
let typ = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[] else: mt.base
let z = newObj(mt, typ.size)
unsureAsgnRef(cast[PPointer](dest), z)
tab.put(s2, z)
genericDeepCopyAux(z, s2, mt.base, tab)
genericDeepCopyAux(z, s2, typ, tab)
else:
unsureAsgnRef(cast[PPointer](dest), z)
of tyPtr:

7
lib/system/excpt.nim
View file

@ -293,11 +293,11 @@ proc raiseExceptionAux(e: ref Exception) =
quitOrDebug()
else:
# ugly, but avoids heap allocations :-)
template xadd(buf, s, slen: expr) =
template xadd(buf, s, slen) =
if L + slen < high(buf):
copyMem(addr(buf[L]), cstring(s), slen)
inc L, slen
template add(buf, s: expr) =
template add(buf, s) =
xadd(buf, s, s.len)
var buf: array[0..2000, char]
var L = 0
@ -404,7 +404,8 @@ when not defined(noSignalHandler):
GC_enable()
else:
var msg: cstring
template asgn(y: expr) = msg = y
template asgn(y) =
msg = y
processSignal(sign, asgn)
showErrorMessage(msg)
when defined(endb): dbgAborting = true

36
lib/system/gc.nim
View file

@ -238,21 +238,6 @@ proc nimGCunref(p: pointer) {.compilerProc.} =
include gc_common
proc prepareDealloc(cell: PCell) =
when useMarkForDebug:
gcAssert(cell notin gch.marked, "Cell still alive!")
let t = cell.typ
if t.finalizer != nil:
# the finalizer could invoke something that
# allocates memory; this could trigger a garbage
# collection. Since we are already collecting we
# prevend recursive entering here by a lock.
# XXX: we should set the cell's children to nil!
inc(gch.recGcLock)
(cast[Finalizer](t.finalizer))(cellToUsr(cell))
dec(gch.recGcLock)
decTypeSize(cell, t)
template beforeDealloc(gch: var GcHeap; c: PCell; msg: typed) =
when false:
for i in 0..gch.decStack.len-1:
@ -274,6 +259,9 @@ proc nimGCunrefNoCycle(p: pointer) {.compilerProc, inline.} =
sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 2")
sysAssert(allocInv(gch.region), "end nimGCunrefNoCycle 5")
proc nimGCunrefRC1(p: pointer) {.compilerProc, inline.} =
decRef(usrToCell(p))
proc asgnRef(dest: PPointer, src: pointer) {.compilerProc, inline.} =
# the code generator calls this proc!
gcAssert(not isOnStack(dest), "asgnRef")
@ -754,8 +742,8 @@ proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
#[
This method is conditionally marked with an attribute so that it gets ignored by the LLVM ASAN
(Address SANitizer) intrumentation as it will raise false errors due to the implementation of
garbage collection that is used by Nim. For more information, please see the documentation of
(Address SANitizer) intrumentation as it will raise false errors due to the implementation of
garbage collection that is used by Nim. For more information, please see the documentation of
`CLANG_NO_SANITIZE_ADDRESS` in `lib/nimbase.h`.
]#
proc markStackAndRegisters(gch: var GcHeap) {.noinline, cdecl, codegenDecl: "CLANG_NO_SANITIZE_ADDRESS $# $#$#".} =
@ -920,11 +908,13 @@ when not defined(useNimRtl):
else:
inc(gch.recGcLock)
proc GC_enable() =
if gch.recGcLock > 0:
when hasThreadSupport and hasSharedHeap:
discard atomicDec(gch.recGcLock, 1)
else:
dec(gch.recGcLock)
if gch.recGcLock <= 0:
raise newException(AssertionError,
"API usage error: GC_enable called but GC is already enabled")
when hasThreadSupport and hasSharedHeap:
discard atomicDec(gch.recGcLock, 1)
else:
dec(gch.recGcLock)
proc GC_setStrategy(strategy: GC_Strategy) =
discard
@ -945,7 +935,6 @@ when not defined(useNimRtl):
release(gch)
proc GC_getStatistics(): string =
GC_disable()
result = "[GC] total memory: " & $(getTotalMem()) & "\n" &
"[GC] occupied memory: " & $(getOccupiedMem()) & "\n" &
"[GC] stack scans: " & $gch.stat.stackScans & "\n" &
@ -961,6 +950,5 @@ when not defined(useNimRtl):
result = result & "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & cast[pointer](stack.maxStackSize).repr & "\n"
else:
result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n"
GC_enable()
{.pop.} # profiler: off, stackTrace: off

32
lib/system/gc_common.nim
View file

@ -351,3 +351,35 @@ else:
# ----------------------------------------------------------------------------
# end of non-portable code
# ----------------------------------------------------------------------------
proc prepareDealloc(cell: PCell) =
when declared(useMarkForDebug):
when useMarkForDebug:
gcAssert(cell notin gch.marked, "Cell still alive!")
let t = cell.typ
if t.finalizer != nil:
# the finalizer could invoke something that
# allocates memory; this could trigger a garbage
# collection. Since we are already collecting we
# prevend recursive entering here by a lock.
# XXX: we should set the cell's children to nil!
inc(gch.recGcLock)
(cast[Finalizer](t.finalizer))(cellToUsr(cell))
dec(gch.recGcLock)
decTypeSize(cell, t)
proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) =
## Frees the thread local heap. Runs every finalizer if ``runFinalizers```
## is true. If ``allowGcAfterwards`` is true, a minimal amount of allocation
## happens to ensure the GC can continue to work after the call
## to ``deallocHeap``.
if runFinalizers:
for x in allObjects(gch.region):
if isCell(x):
# cast to PCell is correct here:
var c = cast[PCell](x)
prepareDealloc(c)
deallocOsPages(gch.region)
zeroMem(addr gch.region, sizeof(gch.region))
if allowGcAfterwards:
initGC()

26
lib/system/gc_ms.nim
View file

@ -221,18 +221,6 @@ when defined(nimGcRefLeak):
include gc_common
proc prepareDealloc(cell: PCell) =
if cell.typ.finalizer != nil:
# the finalizer could invoke something that
# allocates memory; this could trigger a garbage
# collection. Since we are already collecting we
# prevend recursive entering here by a lock.
# XXX: we should set the cell's children to nil!
inc(gch.recGcLock)
(cast[Finalizer](cell.typ.finalizer))(cellToUsr(cell))
dec(gch.recGcLock)
decTypeSize cell, cell.typ
proc initGC() =
when not defined(useNimRtl):
gch.cycleThreshold = InitialThreshold
@ -506,11 +494,13 @@ when not defined(useNimRtl):
else:
inc(gch.recGcLock)
proc GC_enable() =
if gch.recGcLock > 0:
when hasThreadSupport and hasSharedHeap:
atomicDec(gch.recGcLock, 1)
else:
dec(gch.recGcLock)
if gch.recGcLock <= 0:
raise newException(AssertionError,
"API usage error: GC_enable called but GC is already enabled")
when hasThreadSupport and hasSharedHeap:
atomicDec(gch.recGcLock, 1)
else:
dec(gch.recGcLock)
proc GC_setStrategy(strategy: GC_Strategy) = discard
@ -530,7 +520,6 @@ when not defined(useNimRtl):
release(gch)
proc GC_getStatistics(): string =
GC_disable()
result = "[GC] total memory: " & $getTotalMem() & "\n" &
"[GC] occupied memory: " & $getOccupiedMem() & "\n" &
"[GC] collections: " & $gch.stat.collections & "\n" &
@ -542,6 +531,5 @@ when not defined(useNimRtl):
result = result & "[GC] stack " & stack.bottom.repr & "[GC] max stack size " & $stack.maxStackSize & "\n"
else:
result = result & "[GC] max stack size: " & $gch.stat.maxStackSize & "\n"
GC_enable()
{.pop.}

298
lib/system/gc_stack.nim → lib/system/gc_regions.nim
View file

@ -44,10 +44,6 @@ type
typ: PNimType
nextFinal: ptr ObjHeader # next object with finalizer
Hole = object # stacks can have holes. Otherwise 'growObj' would be insane.
zeroTyp: pointer # overlaid with 'typ' field. Always 'nil'.
size: int # size of the free slot
Chunk = ptr BaseChunk
BaseChunk = object
next: Chunk
@ -55,7 +51,15 @@ type
head, tail: ptr ObjHeader # first and last object in chunk that
# has a finalizer attached to it
const
MaxSmallObject = 128
type
FreeEntry = ptr object
next: FreeEntry
SizedFreeEntry = ptr object
next: SizedFreeEntry
size: int
StackPtr = object
bump: pointer
remaining: int
@ -66,12 +70,21 @@ type
bump: pointer
head, tail: Chunk
nextChunkSize, totalSize: int
hole: ptr Hole # we support individual freeing
freeLists: array[MaxSmallObject div MemAlign, FreeEntry]
holes: SizedFreeEntry
when hasThreadSupport:
lock: SysLock
SeqHeader = object # minor hack ahead: Since we know that seqs
# and strings cannot have finalizers, we use the field
# instead for a 'region' field so that they can grow
# and shrink safely.
typ: PNimType
region: ptr MemRegion
var
tlRegion {.threadVar.}: MemRegion
# tempStrRegion {.threadVar.}: MemRegion # not yet used
template withRegion*(r: MemRegion; body: untyped) =
let oldRegion = tlRegion
@ -85,6 +98,9 @@ template withRegion*(r: MemRegion; body: untyped) =
template inc(p: pointer, s: int) =
p = cast[pointer](cast[int](p) +% s)
template dec(p: pointer, s: int) =
p = cast[pointer](cast[int](p) -% s)
template `+!`(p: pointer, s: int): pointer =
cast[pointer](cast[int](p) +% s)
@ -128,7 +144,22 @@ proc allocSlowPath(r: var MemRegion; size: int) =
r.tail = fresh
r.remaining = s - sizeof(BaseChunk)
proc alloc(r: var MemRegion; size: int): pointer {.inline.} =
proc alloc(r: var MemRegion; size: int): pointer =
if size <= MaxSmallObject:
var it = r.freeLists[size div MemAlign]
if it != nil:
r.freeLists[size div MemAlign] = it.next
return pointer(it)
else:
var it = r.holes
var prev: SizedFreeEntry = nil
while it != nil:
if it.size >= size:
if prev != nil: prev.next = it.next
else: r.holes = it.next
return pointer(it)
prev = it
it = it.next
if size > r.remaining:
allocSlowPath(r, size)
sysAssert(size <= r.remaining, "size <= r.remaining")
@ -145,12 +176,23 @@ proc runFinalizers(c: Chunk) =
(cast[Finalizer](it.typ.finalizer))(it+!sizeof(ObjHeader))
it = it.nextFinal
when false:
proc dealloc(r: var MemRegion; p: pointer) =
let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader))
if it.typ != nil and it.typ.finalizer != nil:
(cast[Finalizer](it.typ.finalizer))(p)
it.typ = nil
proc dealloc(r: var MemRegion; p: pointer; size: int) =
let it = cast[ptr ObjHeader](p-!sizeof(ObjHeader))
if it.typ != nil and it.typ.finalizer != nil:
(cast[Finalizer](it.typ.finalizer))(p)
it.typ = nil
# it is benefitial to not use the free lists here:
if r.bump -! size == p:
dec r.bump, size
elif size <= MaxSmallObject:
let it = cast[FreeEntry](p)
it.next = r.freeLists[size div MemAlign]
r.freeLists[size div MemAlign] = it
else:
let it = cast[SizedFreeEntry](p)
it.size = size
it.next = r.holes
r.holes = it
proc deallocAll(r: var MemRegion; head: Chunk) =
var it = head
@ -175,12 +217,15 @@ template computeRemaining(r): untyped =
proc setObstackPtr*(r: var MemRegion; sp: StackPtr) =
# free everything after 'sp':
if sp.current != nil:
if sp.current.next != nil:
deallocAll(r, sp.current.next)
sp.current.next = nil
else:
deallocAll(r, r.head)
r.head = nil
# better leak this memory than be sorry:
for i in 0..high(r.freeLists): r.freeLists[i] = nil
r.holes = nil
#else:
# deallocAll(r, r.head)
# r.head = nil
r.bump = sp.bump
r.tail = sp.current
r.remaining = sp.remaining
@ -191,17 +236,28 @@ proc deallocAll*() = tlRegion.deallocAll()
proc deallocOsPages(r: var MemRegion) = r.deallocAll()
proc joinRegion*(dest: var MemRegion; src: MemRegion) =
# merging is not hard.
if dest.head.isNil:
dest.head = src.head
else:
dest.tail.next = src.head
dest.tail = src.tail
dest.bump = src.bump
dest.remaining = src.remaining
dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize)
inc dest.totalSize, src.totalSize
template withScratchRegion*(body: untyped) =
var scratch: MemRegion
let oldRegion = tlRegion
tlRegion = scratch
try:
body
finally:
tlRegion = oldRegion
deallocAll(scratch)
when false:
proc joinRegion*(dest: var MemRegion; src: MemRegion) =
# merging is not hard.
if dest.head.isNil:
dest.head = src.head
else:
dest.tail.next = src.head
dest.tail = src.tail
dest.bump = src.bump
dest.remaining = src.remaining
dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize)
inc dest.totalSize, src.totalSize
proc isOnHeap*(r: MemRegion; p: pointer): bool =
# the tail chunk is the largest, so check it first. It's also special
@ -213,159 +269,6 @@ proc isOnHeap*(r: MemRegion; p: pointer): bool =
if it >= p and p <= it+!it.size: return true
it = it.next
when false:
# essential feature for later: copy data over from one region to another
proc isInteriorPointer(r: MemRegion; p: pointer): pointer =
discard " we cannot patch stack pointers anyway!"
type
PointerStackChunk = object
next, prev: ptr PointerStackChunk
len: int
data: array[128, pointer]
template head(s: PointerStackChunk): untyped = s.prev
template tail(s: PointerStackChunk): untyped = s.next
include chains
proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) =
if s.len < high(s.data):
s.data[s.len] = x
inc s.len
else:
let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk)))
fresh.len = 1
fresh.data[0] = x
fresh.next = nil
fresh.prev = nil
append(s, fresh)
proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk;
dest, src: pointer, mt: PNimType) {.benign.}
proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk;
dest, src: pointer, n: ptr TNimNode) {.benign.} =
var
d = cast[ByteAddress](dest)
s = cast[ByteAddress](src)
case n.kind
of nkSlot:
genericDeepCopyAux(cast[pointer](d +% n.offset),
cast[pointer](s +% n.offset), n.typ)
of nkList:
for i in 0..n.len-1:
genericDeepCopyAux(dest, src, n.sons[i])
of nkCase:
var dd = selectBranch(dest, n)
var m = selectBranch(src, n)
# reset if different branches are in use; note different branches also
# imply that's not self-assignment (``x = x``)!
if m != dd and dd != nil:
genericResetAux(dest, dd)
copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset),
n.typ.size)
if m != nil:
genericDeepCopyAux(dest, src, m)
of nkNone: sysAssert(false, "genericDeepCopyAux")
proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} =
result = rawNewStringNoInit(dr, src.len)
result.len = src.len
copyMem(result.data, src.data, src.len + 1)
proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk;
dest, src: pointer, mt: PNimType) =
var
d = cast[ByteAddress](dest)
s = cast[ByteAddress](src)
sysAssert(mt != nil, "genericDeepCopyAux 2")
case mt.kind
of tyString:
var x = cast[PPointer](dest)
var s2 = cast[PPointer](s)[]
if s2 == nil:
x[] = nil
else:
x[] = copyDeepString(cast[NimString](s2))
of tySequence:
var s2 = cast[PPointer](src)[]
var seq = cast[PGenericSeq](s2)
var x = cast[PPointer](dest)
if s2 == nil:
x[] = nil
return
sysAssert(dest != nil, "genericDeepCopyAux 3")
x[] = newSeq(mt, seq.len)
var dst = cast[ByteAddress](cast[PPointer](dest)[])
for i in 0..seq.len-1:
genericDeepCopyAux(dr, stack,
cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize),
cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +%
GenericSeqSize),
mt.base)
of tyObject:
# we need to copy m_type field for tyObject, as it could be empty for
# sequence reallocations:
var pint = cast[ptr PNimType](dest)
pint[] = cast[ptr PNimType](src)[]
if mt.base != nil:
genericDeepCopyAux(dr, stack, dest, src, mt.base)
genericDeepCopyAux(dr, stack, dest, src, mt.node)
of tyTuple:
genericDeepCopyAux(dr, stack, dest, src, mt.node)
of tyArray, tyArrayConstr:
for i in 0..(mt.size div mt.base.size)-1:
genericDeepCopyAux(dr, stack,
cast[pointer](d +% i*% mt.base.size),
cast[pointer](s +% i*% mt.base.size), mt.base)
of tyRef:
let s2 = cast[PPointer](src)[]
if s2 == nil:
cast[PPointer](dest)[] = nil
else:
# we modify the header of the cell temporarily; instead of the type
# field we store a forwarding pointer. XXX This is bad when the cloning
# fails due to OOM etc.
let x = usrToCell(s2)
let forw = cast[int](x.typ)
if (forw and 1) == 1:
# we stored a forwarding pointer, so let's use that:
let z = cast[pointer](forw and not 1)
unsureAsgnRef(cast[PPointer](dest), z)
else:
let realType = x.typ
let z = newObj(realType, realType.base.size)
unsureAsgnRef(cast[PPointer](dest), z)
x.typ = cast[PNimType](cast[int](z) or 1)
genericDeepCopyAux(dr, stack, z, s2, realType.base)
x.typ = realType
else:
copyMem(dest, src, mt.size)
proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion;
root: pointer): pointer =
# we mark the alive data and copy only alive data over to 'dest'.
# This is O(liveset) but it nicely compacts memory, so it's fine.
# We use the 'typ' field as a forwarding pointer. The forwarding
# pointers have bit 0 set, so we can disambiguate them.
# We allocate a temporary stack in 'src' that we later free:
var s: PointerStackChunk
s.len = 1
s.data[0] = root
while s.len > 0:
var p: pointer
if s.tail == nil:
p = s.data[s.len-1]
dec s.len
else:
p = s.tail.data[s.tail.len-1]
dec s.tail.len
if s.tail.len == 0:
unlink(s, s.tail)
proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer =
var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader)))
res.typ = typ
@ -374,6 +277,12 @@ proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer =
r.head.head = res
result = res +! sizeof(ObjHeader)
proc rawNewSeq(r: var MemRegion, typ: PNimType, size: int): pointer =
var res = cast[ptr SeqHeader](alloc(r, size + sizeof(SeqHeader)))
res.typ = typ
res.region = addr(r)
result = res +! sizeof(SeqHeader)
proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} =
result = rawNewObj(tlRegion, typ, size)
zeroMem(result, size)
@ -384,28 +293,37 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} =
when defined(memProfiler): nimProfile(size)
proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} =
let size = addInt(mulInt(len, typ.base.size), GenericSeqSize)
result = newObj(typ, size)
let size = roundup(addInt(mulInt(len, typ.base.size), GenericSeqSize),
MemAlign)
result = rawNewSeq(tlRegion, typ, size)
zeroMem(result, size)
cast[PGenericSeq](result).len = len
cast[PGenericSeq](result).reserved = len
proc newStr(typ: PNimType, len: int; init: bool): pointer {.compilerRtl.} =
let size = roundup(addInt(len, GenericSeqSize), MemAlign)
result = rawNewSeq(tlRegion, typ, size)
if init: zeroMem(result, size)
cast[PGenericSeq](result).len = 0
cast[PGenericSeq](result).reserved = len
proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
result = rawNewObj(tlRegion, typ, size)
zeroMem(result, size)
proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} =
let size = addInt(mulInt(len, typ.base.size), GenericSeqSize)
result = newObj(typ, size)
cast[PGenericSeq](result).len = len
cast[PGenericSeq](result).reserved = len
result = newSeq(typ, len)
proc growObj(region: var MemRegion; old: pointer, newsize: int): pointer =
let typ = cast[ptr ObjHeader](old -! sizeof(ObjHeader)).typ
result = rawNewObj(region, typ, newsize)
proc growObj(regionUnused: var MemRegion; old: pointer, newsize: int): pointer =
let sh = cast[ptr SeqHeader](old -! sizeof(SeqHeader))
let typ = sh.typ
result = rawNewSeq(sh.region[], typ,
roundup(newsize, MemAlign))
let elemSize = if typ.kind == tyString: 1 else: typ.base.size
let oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize
copyMem(result, old, oldsize)
zeroMem(result +! oldsize, newsize-oldsize)
copyMem(result, old, oldsize)
dealloc(sh.region[], old, roundup(oldsize, MemAlign))
proc growObj(old: pointer, newsize: int): pointer {.rtl.} =
result = growObj(tlRegion, old, newsize)

6
lib/system/jssys.nim
View file

@ -53,7 +53,7 @@ proc isNimException(): bool {.asmNoStackFrame.} =
else:
asm "return `lastJSError`.m_type;"
proc getCurrentException*(): ref Exception =
proc getCurrentException*(): ref Exception {.compilerRtl, benign.} =
if isNimException(): result = cast[ref Exception](lastJSError)
proc getCurrentExceptionMsg*(): string =
@ -157,10 +157,10 @@ proc reraiseException() {.compilerproc, asmNoStackFrame.} =
asm "throw lastJSError;"
proc raiseOverflow {.exportc: "raiseOverflow", noreturn.} =
proc raiseOverflow {.exportc: "raiseOverflow", noreturn, compilerProc.} =
raise newException(OverflowError, "over- or underflow")
proc raiseDivByZero {.exportc: "raiseDivByZero", noreturn.} =
proc raiseDivByZero {.exportc: "raiseDivByZero", noreturn, compilerProc.} =
raise newException(DivByZeroError, "division by zero")
proc raiseRangeError() {.compilerproc, noreturn.} =

10
lib/system/mmdisp.nim
View file

@ -16,7 +16,7 @@
const
debugGC = false # we wish to debug the GC...
logGC = false
traceGC = defined(smokeCycles) # extensive debugging
traceGC = false # extensive debugging
alwaysCycleGC = defined(smokeCycles)
alwaysGC = defined(fulldebug) # collect after every memory
# allocation (for debugging)
@ -34,7 +34,7 @@ const
type
PPointer = ptr pointer
ByteArray = array[0..ArrayDummySize, byte]
ByteArray = UncheckedArray[byte]
PByte = ptr ByteArray
PString = ptr string
{.deprecated: [TByteArray: ByteArray].}
@ -543,7 +543,7 @@ elif defined(nogc):
include "system/cellsets"
else:
when not defined(gcStack):
when not defined(gcRegions):
include "system/alloc"
include "system/cellsets"
@ -551,9 +551,9 @@ else:
sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell")
when compileOption("gc", "v2"):
include "system/gc2"
elif defined(gcStack):
elif defined(gcRegions):
# XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here
include "system/gc_stack"
include "system/gc_regions"
elif defined(gcMarkAndSweep):
# XXX use 'compileOption' here
include "system/gc_ms"

8
lib/system/platforms.nim
View file

@ -24,6 +24,8 @@ type
amd64, ## x86_64 (AMD64); 64 bit x86 compatible CPU
mips, ## Mips based processor
mipsel, ## Little Endian Mips based processor
mips64, ## 64-bit MIPS processor
mips64el, ## Little Endian 64-bit MIPS processor
arm, ## ARM based processor
arm64, ## ARM64 based processor
vm, ## Some Virtual machine: Nim's VM or JavaScript
@ -33,7 +35,8 @@ type
OsPlatform* {.pure.} = enum ## the OS this program will run on.
none, dos, windows, os2, linux, morphos, skyos, solaris,
irix, netbsd, freebsd, openbsd, aix, palmos, qnx, amiga,
atari, netware, macos, macosx, haiku, js, nimVM, standalone
atari, netware, macos, macosx, haiku, android, js, nimVM,
standalone
const
targetOS* = when defined(windows): OsPlatform.windows
@ -56,6 +59,7 @@ const
elif defined(macosx): OsPlatform.macosx
elif defined(macos): OsPlatform.macos
elif defined(haiku): OsPlatform.haiku
elif defined(android): OsPlatform.android
elif defined(js): OsPlatform.js
elif defined(nimrodVM): OsPlatform.nimVM
elif defined(standalone): OsPlatform.standalone
@ -73,6 +77,8 @@ const
elif defined(amd64): CpuPlatform.amd64
elif defined(mips): CpuPlatform.mips
elif defined(mipsel): CpuPlatform.mipsel
elif defined(mips64): CpuPlatform.mips64
elif defined(mips64el): CpuPlatform.mips64el
elif defined(arm): CpuPlatform.arm
elif defined(arm64): CpuPlatform.arm64
elif defined(vm): CpuPlatform.vm

8
lib/system/sysio.nim
View file

@ -86,11 +86,11 @@ proc writeBuffer(f: File, buffer: pointer, len: Natural): int =
checkErr(f)
proc writeBytes(f: File, a: openArray[int8|uint8], start, len: Natural): int =
var x = cast[ptr array[0..1000_000_000, int8]](a)
result = writeBuffer(f, addr(x[start]), len)
var x = cast[ptr UncheckedArray[int8]](a)
result = writeBuffer(f, addr(x[int(start)]), len)
proc writeChars(f: File, a: openArray[char], start, len: Natural): int =
var x = cast[ptr array[0..1000_000_000, int8]](a)
result = writeBuffer(f, addr(x[start]), len)
var x = cast[ptr UncheckedArray[int8]](a)
result = writeBuffer(f, addr(x[int(start)]), len)
proc write(f: File, s: string) =
if writeBuffer(f, cstring(s), s.len) != s.len:

11
lib/system/sysstr.nim
View file

@ -38,6 +38,13 @@ when declared(allocAtomic):
template allocStrNoInit(size: untyped): untyped =
cast[NimString](boehmAllocAtomic(size))
elif defined(gcRegions):
template allocStr(size: untyped): untyped =
cast[NimString](newStr(addr(strDesc), size, true))
template allocStrNoInit(size: untyped): untyped =
cast[NimString](newStr(addr(strDesc), size, false))
else:
template allocStr(size: untyped): untyped =
cast[NimString](newObj(addr(strDesc), size))
@ -99,7 +106,7 @@ proc copyString(src: NimString): NimString {.compilerRtl.} =
proc copyStringRC1(src: NimString): NimString {.compilerRtl.} =
if src != nil:
when declared(newObjRC1):
when declared(newObjRC1) and not defined(gcRegions):
var s = src.len
if s < 7: s = 7
result = cast[NimString](newObjRC1(addr(strDesc), sizeof(TGenericSeq) +
@ -235,7 +242,7 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
# we need to decref here, otherwise the GC leaks!
when not defined(boehmGC) and not defined(nogc) and
not defined(gcMarkAndSweep) and not defined(gogc) and
not defined(gcStack):
not defined(gcRegions):
when false: # compileOption("gc", "v2"):
for i in newLen..result.len-1:
let len0 = gch.tempStack.len

140
lib/system/threads.nim
View file

@ -115,10 +115,6 @@ when defined(windows):
proc setThreadAffinityMask(hThread: SysThread, dwThreadAffinityMask: uint) {.
importc: "SetThreadAffinityMask", stdcall, header: "<windows.h>".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
result = int(getCurrentThreadId())
elif defined(genode):
const
GenodeHeader = "genode_cpp/threads.h"
@ -249,48 +245,6 @@ else:
proc setAffinity(thread: SysThread; setsize: csize; s: var CpuSet) {.
importc: "pthread_setaffinity_np", header: pthreadh.}
when defined(linux):
proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".}
var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: int
#type Pid {.importc: "pid_t", header: "<sys/types.h>".} = distinct int
#proc gettid(): Pid {.importc, header: "<sys/types.h>".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
result = int(syscall(NR_gettid))
elif defined(dragonfly):
proc lwp_gettid(): int32 {.importc, header: "unistd.h".}
proc getThreadId*(): int =
result = int(lwp_gettid())
elif defined(openbsd):
proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".}
proc getThreadId*(): int =
result = int(getthrid())
elif defined(netbsd):
proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".}
proc getThreadId*(): int =
result = int(lwp_self())
elif defined(macosx) or defined(freebsd):
proc pthread_threadid_np(y: pointer; x: var uint64): cint {.importc, header: "pthread.h".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
var x: uint64
result = pthread_threadid_np(nil, x)
result = int(x)
elif defined(solaris):
# just a guess really:
type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int
proc thr_self(): thread_t {.importc, header: "<thread.h>".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
result = int(thr_self())
const
emulatedThreadVars = compileOption("tlsEmulation")
@ -302,8 +256,9 @@ when emulatedThreadVars:
# we preallocate a fixed size for thread local storage, so that no heap
# allocations are needed. Currently less than 7K are used on a 64bit machine.
# We use ``float`` for proper alignment:
const nimTlsSize {.intdefine.} = 8000
type
ThreadLocalStorage = array[0..1_000, float]
ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float]
PGcThread = ptr GcThread
GcThread {.pure, inheritable.} = object
@ -369,7 +324,11 @@ when not defined(useNimRtl):
when emulatedThreadVars:
if nimThreadVarsSize() > sizeof(ThreadLocalStorage):
echo "too large thread local storage size requested"
echo "too large thread local storage size requested ",
"(", nimThreadVarsSize(), "/", sizeof(ThreadLocalStorage), "). ",
"Use -d:\"nimTlsSize=", nimThreadVarsSize(),
"\" to preallocate sufficient storage."
quit 1
when hasSharedHeap and not defined(boehmgc) and not defined(gogc) and not defined(nogc):
@ -437,7 +396,7 @@ template afterThreadRuns() =
for i in countdown(threadDestructionHandlers.len-1, 0):
threadDestructionHandlers[i]()
when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcstack):
when not defined(boehmgc) and not hasSharedHeap and not defined(gogc) and not defined(gcRegions):
proc deallocOsPages()
when defined(boehmgc):
@ -475,7 +434,7 @@ else:
proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) =
when defined(boehmgc):
boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd)
elif not defined(nogc) and not defined(gogc) and not defined(gcstack):
elif not defined(nogc) and not defined(gogc) and not defined(gcRegions):
var p {.volatile.}: proc(a: ptr Thread[TArg]) {.nimcall.} =
threadProcWrapDispatch[TArg]
when not hasSharedHeap:
@ -493,7 +452,7 @@ proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) =
else:
threadProcWrapDispatch(thrd)
template threadProcWrapperBody(closure: expr) {.immediate.} =
template threadProcWrapperBody(closure: untyped): untyped =
var thrd = cast[ptr Thread[TArg]](closure)
var core = thrd.core
when declared(globalsSlot): threadVarSetValue(globalsSlot, thrd.core)
@ -665,3 +624,82 @@ when useStackMaskHack:
var mainThread: Thread[pointer]
createThread(mainThread, tp)
joinThread(mainThread)
## we need to cache current threadId to not perform syscall all the time
var threadId {.threadvar.}: int
when defined(windows):
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(getCurrentThreadId())
result = threadId
elif defined(linux):
proc syscall(arg: clong): clong {.varargs, importc: "syscall", header: "<unistd.h>".}
var NR_gettid {.importc: "__NR_gettid", header: "<sys/syscall.h>".}: int
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(syscall(NR_gettid))
result = threadId
elif defined(dragonfly):
proc lwp_gettid(): int32 {.importc, header: "unistd.h".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(lwp_gettid())
result = threadId
elif defined(openbsd):
proc getthrid(): int32 {.importc: "getthrid", header: "<unistd.h>".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(getthrid())
result = threadId
elif defined(netbsd):
proc lwp_self(): int32 {.importc: "_lwp_self", header: "<lwp.h>".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(lwp_self())
result = threadId
elif defined(freebsd):
proc syscall(arg: cint, arg0: ptr cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".}
var SYS_thr_self {.importc:"SYS_thr_self", header:"<sys/syscall.h>"}: cint
proc getThreadId*(): int =
## get the ID of the currently running thread.
var tid = 0.cint
if threadId == 0:
discard syscall(SYS_thr_self, addr tid)
threadId = tid
result = threadId
elif defined(macosx):
proc syscall(arg: cint): cint {.varargs, importc: "syscall", header: "<unistd.h>".}
var SYS_thread_selfid {.importc:"SYS_thread_selfid", header:"<sys/syscall.h>".}: cint
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(syscall(SYS_thread_selfid))
result = threadId
elif defined(solaris):
type thread_t {.importc: "thread_t", header: "<thread.h>".} = distinct int
proc thr_self(): thread_t {.importc, header: "<thread.h>".}
proc getThreadId*(): int =
## get the ID of the currently running thread.
if threadId == 0:
threadId = int(thr_self())
result = threadId

2
lib/system/widestrs.nim
View file

@ -15,7 +15,7 @@ when not declared(NimString):
type
Utf16Char* = distinct int16
WideCString* = ref array[0.. 1_000_000, Utf16Char]
WideCString* = ref UncheckedArray[Utf16Char]
{.deprecated: [TUtf16Char: Utf16Char].}
proc len*(w: WideCString): int =

33
lib/upcoming/asyncdispatch.nim
View file

@ -9,11 +9,13 @@
include "system/inclrtl"
import os, tables, strutils, times, heapqueue, lists, options
import os, tables, strutils, times, heapqueue, lists, options, asyncstreams
import asyncfutures except callSoon
import nativesockets, net, deques
export Port, SocketFlag
export asyncfutures, asyncstreams
#{.injectStmt: newGcInvariant().}
@ -130,8 +132,6 @@ export Port, SocketFlag
# TODO: Check if yielded future is nil and throw a more meaningful exception
include "../includes/asyncfutures"
type
PDispatcherBase = ref object of RootRef
timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
@ -161,6 +161,12 @@ proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
result = int((timerTimeout - curTime) * 1000)
if result < 0: result = 0
proc callSoon(cbproc: proc ()) {.gcsafe.}
proc initCallSoonProc =
if asyncfutures.getCallSoonProc().isNil:
asyncfutures.setCallSoonProc(callSoon)
when defined(windows) or defined(nimdoc):
import winlean, sets, hashes
type
@ -214,15 +220,17 @@ when defined(windows) or defined(nimdoc):
result.callbacks = initDeque[proc ()](64)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
## Retrieves the global thread-local dispatcher.
if gDisp.isNil: gDisp = newDispatcher()
result = gDisp
proc setGlobalDispatcher*(disp: PDispatcher) =
if not gDisp.isNil:
assert gDisp.callbacks.len == 0
gDisp = disp
initCallSoonProc()
proc getGlobalDispatcher*(): PDispatcher =
if gDisp.isNil:
setGlobalDispatcher(newDispatcher())
result = gDisp
proc register*(fd: AsyncFD) =
## Registers ``fd`` with the dispatcher.
@ -1081,14 +1089,17 @@ else:
result.callbacks = initDeque[proc ()](InitDelayedCallbackListSize)
var gDisp{.threadvar.}: PDispatcher ## Global dispatcher
proc getGlobalDispatcher*(): PDispatcher =
if gDisp.isNil: gDisp = newDispatcher()
result = gDisp
proc setGlobalDispatcher*(disp: PDispatcher) =
if not gDisp.isNil:
assert gDisp.callbacks.len == 0
gDisp = disp
initCallSoonProc()
proc getGlobalDispatcher*(): PDispatcher =
if gDisp.isNil:
setGlobalDispatcher(newDispatcher())
result = gDisp
proc register*(fd: AsyncFD) =
let p = getGlobalDispatcher()
@ -1601,7 +1612,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} =
return
add(result, c)
proc callSoon*(cbproc: proc ()) =
proc callSoon(cbproc: proc ()) =
## Schedule `cbproc` to be called as soon as possible.
## The callback is called when control returns to the event loop.
getGlobalDispatcher().callbacks.addLast(cbproc)

4
lib/wrappers/openssl.nim
View file

@ -588,13 +588,13 @@ proc md5*(d: ptr cuchar; n: csize; md: ptr cuchar): ptr cuchar{.importc: "MD5".}
proc md5_Transform*(c: var MD5_CTX; b: ptr cuchar){.importc: "MD5_Transform".}
{.pop.}
from strutils import toHex,toLower
from strutils import toHex, toLowerAscii
proc hexStr (buf:cstring): string =
# turn md5s output into a nice hex str
result = newStringOfCap(32)
for i in 0 .. <16:
result.add toHex(buf[i].ord, 2).toLower
result.add toHex(buf[i].ord, 2).toLowerAscii
proc md5_File* (file: string): string {.raises: [IOError,Exception].} =
## Generate MD5 hash for a file. Result is a 32 character