jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'devel' into async-improvements

Browse source
This commit is contained in:
Dominik Picheta 2018-01-17 16:28:00 +00:00
commit 47d05b3f2e
201 changed files with 6604 additions and 2628 deletions
Download patch file Download diff file Expand all files Collapse all files

35
lib/core/allocators.nim Normal file
View file

@ -0,0 +1,35 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
type
Allocator* = ptr object {.inheritable.}
alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall.}
dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.}
realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.}
var
currentAllocator {.threadvar.}: Allocator
proc getCurrentAllocator*(): Allocator =
result = currentAllocator
proc setCurrentAllocator*(a: Allocator) =
currentAllocator = a
proc alloc*(size: int; alignment: int = 8): pointer =
let a = getCurrentAllocator()
result = a.alloc(a, size, alignment)
proc dealloc*(p: pointer; size: int) =
let a = getCurrentAllocator()
a.dealloc(a, p, size)
proc realloc*(p: pointer; oldSize, newSize: int): pointer =
let a = getCurrentAllocator()
result = a.realloc(a, p, oldSize, newSize)

56
lib/core/macros.nim
View file

@ -21,7 +21,7 @@ type
nnkInt16Lit, nnkInt32Lit, nnkInt64Lit, nnkUIntLit, nnkUInt8Lit,
nnkUInt16Lit, nnkUInt32Lit, nnkUInt64Lit, nnkFloatLit,
nnkFloat32Lit, nnkFloat64Lit, nnkFloat128Lit, nnkStrLit, nnkRStrLit,
nnkTripleStrLit, nnkNilLit, nnkMetaNode, nnkDotCall,
nnkTripleStrLit, nnkNilLit, nnkComesFrom, nnkDotCall,
nnkCommand, nnkCall, nnkCallStrLit, nnkInfix,
nnkPrefix, nnkPostfix, nnkHiddenCallConv,
nnkExprEqExpr,
@ -130,6 +130,7 @@ const
nnkLiterals* = {nnkCharLit..nnkNilLit}
nnkCallKinds* = {nnkCall, nnkInfix, nnkPrefix, nnkPostfix, nnkCommand,
nnkCallStrLit}
nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit}
proc `!`*(s: string): NimIdent {.magic: "StrToIdent", noSideEffect, deprecated.}
## constructs an identifier from the string `s`
@ -1213,6 +1214,59 @@ macro expandMacros*(body: typed): untyped =
result = getAst(inner(body))
echo result.toStrLit
proc customPragmaNode(n: NimNode): NimNode =
expectKind(n, {nnkSym, nnkDotExpr})
if n.kind == nnkSym:
let sym = n.symbol.getImpl()
sym.expectRoutine()
result = sym.pragma
elif n.kind == nnkDotExpr:
let typDef = getImpl(getTypeInst(n[0]).symbol)
typDef.expectKind(nnkTypeDef)
typDef[2].expectKind(nnkObjectTy)
let recList = typDef[2][2]
for identDefs in recList:
for i in 0 .. identDefs.len - 3:
if identDefs[i].kind == nnkPragmaExpr and
identDefs[i][0].kind == nnkIdent and $identDefs[i][0] == $n[1]:
return identDefs[i][1]
macro hasCustomPragma*(n: typed, cp: typed{nkSym}): untyped =
## Expands to `true` if expression `n` which is expected to be `nnkDotExpr`
## has custom pragma `cp`.
##
## .. code-block:: nim
## template myAttr() {.pragma.}
## type
## MyObj = object
## myField {.myAttr.}: int
## var o: MyObj
## assert(o.myField.hasCustomPragma(myAttr) == 0)
let pragmaNode = customPragmaNode(n)
for p in pragmaNode:
if (p.kind == nnkSym and p == cp) or
(p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp):
return newLit(true)
return newLit(false)
macro getCustomPragmaVal*(n: typed, cp: typed{nkSym}): untyped =
## Expands to value of custom pragma `cp` of expression `n` which is expected
## to be `nnkDotExpr`.
##
## .. code-block:: nim
## template serializationKey(key: string) {.pragma.}
## type
## MyObj = object
## myField {.serializationKey: "mf".}: int
## var o: MyObj
## assert(o.myField.getCustomPragmaVal(serializationKey) == "mf")
let pragmaNode = customPragmaNode(n)
for p in pragmaNode:
if p.kind in nnkPragmaCallKinds and p.len > 0 and p[0].kind == nnkSym and p[0] == cp:
return p[1]
return newEmptyNode()
when not defined(booting):
template emit*(e: static[string]): untyped {.deprecated.} =
## accepts a single string argument and treats it as nim code

97
lib/core/refs.nim Normal file
View file

@ -0,0 +1,97 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Default ref implementation used by Nim's core.
# We cannot use the allocator interface here as we require a heap walker to
# exist. Thus we import 'alloc' directly here to get our own heap that is
# all under the GC's control and can use the ``allObjects`` iterator which
# is crucial for the "sweep" phase.
import typelayouts, alloc
type
TracingGc = ptr object of Allocator
visit*: proc (fieldAddr: ptr pointer; a: Allocator) {.nimcall.}
GcColor = enum
white = 0, black = 1, grey = 2 ## to flip the meaning of white/black
## perform (1 - col)
GcHeader = object
t: ptr TypeLayout
color: GcColor
Cell = ptr GcHeader
GcFrame {.core.} = object
prev: ptr GcFrame
marker: proc (self: GcFrame; a: Allocator)
Phase = enum
None, Marking, Sweeping
GcHeap = object
r: MemRegion
phase: Phase
currBlack, currWhite: GcColor
greyStack: seq[Cell]
var
gch {.threadvar.}: GcHeap
proc `=trace`[T](a: ref T) =
if not marked(a):
mark(a)
`=trace`(a[])
template usrToCell(p: pointer): Cell =
template cellToUsr(cell: Cell): pointer =
cast[pointer](cast[ByteAddress](cell)+%ByteAddress(sizeof(GcHeader)))
template usrToCell(usr: pointer): Cell =
cast[Cell](cast[ByteAddress](usr)-%ByteAddress(sizeof(GcHeader)))
template markGrey(x: Cell) =
if x.color == gch.currWhite and phase == Marking:
x.color = grey
add(gch.greyStack, x)
proc `=`[T](dest: var ref T; src: ref T) =
## full write barrier implementation.
if src != nil:
let s = usrToCell(src)
markGrey(s)
system.`=`(dest, src)
proc linkGcFrame(f: ptr GcFrame) {.core.}
proc unlinkGcFrame() {.core.}
proc setGcFrame(f: ptr GcFrame) {.core.}
proc registerGlobal(p: pointer; t: ptr TypeLayout) {.core.}
proc unregisterGlobal(p: pointer; t: ptr TypeLayout) {.core.}
proc registerThreadvar(p: pointer; t: ptr TypeLayout) {.core.}
proc unregisterThreadvar(p: pointer; t: ptr TypeLayout) {.core.}
proc newImpl(t: ptr TypeLayout): pointer =
let r = cast[Cell](rawAlloc(t.size + sizeof(GcHeader)))
r.typ = t
result = r +! sizeof(GcHeader)
template new*[T](x: var ref T) =
x = newImpl(getTypeLayout(x))
when false:
# implement these if your GC requires them:
proc writeBarrierLocal() {.core.}
proc writeBarrierGlobal() {.core.}
proc writeBarrierGeneric() {.core.}

139
lib/core/seqs.nim Normal file
View file

@ -0,0 +1,139 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
import allocators, typetraits
## Default seq implementation used by Nim's core.
type
seq*[T] = object
len, cap: int
data: ptr UncheckedArray[T]
template frees(s) = dealloc(s.data, s.cap * sizeof(T))
# XXX make code memory safe for overflows in '*'
proc nimSeqLiteral[T](x: openArray[T]): seq[T] {.core.} =
seq[T](len: x.len, cap: x.len, data: x)
when defined(nimHasTrace):
proc `=trace`[T](s: seq[T]; a: Allocator) =
for i in 0 ..< s.len: `=trace`(s.data[i], a)
proc `=destroy`[T](x: var seq[T]) =
if x.data != nil:
when not supportsCopyMem(T):
for i in 0..<x.len: `=destroy`(x[i])
frees(x)
x.data = nil
x.len = 0
x.cap = 0
proc `=`[T](a: var seq[T]; b: seq[T]) =
if a.data == b.data: return
if a.data != nil:
frees(a)
a.data = nil
a.len = b.len
a.cap = b.cap
if b.data != nil:
a.data = cast[type(a.data)](alloc(a.cap * sizeof(T)))
when supportsCopyMem(T):
copyMem(a.data, b.data, a.cap * sizeof(T))
else:
for i in 0..<a.len:
a.data[i] = b.data[i]
proc `=sink`[T](a: var seq[T]; b: seq[T]) =
if a.data != nil and a.data != b.data:
frees(a)
a.len = b.len
a.cap = b.cap
a.data = b.data
proc resize[T](s: var seq[T]) =
let old = s.cap
if old == 0: s.cap = 8
else: s.cap = (s.cap * 3) shr 1
s.data = cast[type(s.data)](realloc(s.data, old * sizeof(T), s.cap * sizeof(T)))
proc reserveSlot[T](x: var seq[T]): ptr T =
if x.len >= x.cap: resize(x)
result = addr(x.data[x.len])
inc x.len
template add*[T](x: var seq[T]; y: T) =
reserveSlot(x)[] = y
proc shrink*[T](x: var seq[T]; newLen: int) =
assert newLen <= x.len
assert newLen >= 0
when not supportsCopyMem(T):
for i in countdown(x.len - 1, newLen - 1):
`=destroy`(x.data[i])
x.len = newLen
proc grow*[T](x: var seq[T]; newLen: int; value: T) =
if newLen <= x.len: return
assert newLen >= 0
if x.cap == 0: x.cap = newLen
else: x.cap = max(newLen, (x.cap * 3) shr 1)
x.data = cast[type(x.data)](realloc(x.data, x.cap * sizeof(T)))
for i in x.len..<newLen:
x.data[i] = value
x.len = newLen
template default[T](t: typedesc[T]): T =
var v: T
v
proc setLen*[T](x: var seq[T]; newLen: int) {.deprecated.} =
if newlen < x.len: shrink(x, newLen)
else: grow(x, newLen, default(T))
template `[]`*[T](x: seq[T]; i: Natural): T =
assert i < x.len
x.data[i]
template `[]=`*[T](x: seq[T]; i: Natural; y: T) =
assert i < x.len
x.data[i] = y
proc `@`*[T](elems: openArray[T]): seq[T] =
result.cap = elems.len
result.len = elems.len
result.data = cast[type(result.data)](alloc(result.cap * sizeof(T)))
when supportsCopyMem(T):
copyMem(result.data, unsafeAddr(elems[0]), result.cap * sizeof(T))
else:
for i in 0..<result.len:
result.data[i] = elems[i]
proc len*[T](x: seq[T]): int {.inline.} = x.len
proc `$`*[T](x: seq[T]): string =
result = "@["
var firstElement = true
for i in 0..<x.len:
let
value = x.data[i]
if firstElement:
firstElement = false
else:
result.add(", ")
when compiles(value.isNil):
# this branch should not be necessary
if value.isNil:
result.add "nil"
else:
result.addQuoted(value)
else:
result.addQuoted(value)
result.add("]")

111
lib/core/strs.nim Normal file
View file

@ -0,0 +1,111 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Default string implementation used by Nim's core.
import allocators
type
string {.core.} = object
len, cap: int
data: ptr UncheckedArray[char]
proc nimStringLiteral(x: cstring; len: int): string {.core.} =
string(len: len, cap: len, data: x)
template frees(s) = dealloc(s.data, s.cap + 1)
proc `=destroy`(s: var string) =
if s.data != nil:
frees(s)
s.data = nil
s.len = 0
s.cap = 0
proc `=sink`(a: var string, b: string) =
# we hope this is optimized away for not yet alive objects:
if a.data != nil and a.data != b.data:
frees(a)
a.len = b.len
a.cap = b.cap
a.data = b.data
proc `=`(a: var string; b: string) =
if a.data != nil and a.data != b.data:
frees(a)
a.data = nil
a.len = b.len
a.cap = b.cap
if b.data != nil:
a.data = cast[type(a.data)](alloc(a.cap + 1))
copyMem(a.data, b.data, a.cap+1)
proc resize(s: var string) =
let old = s.cap
if old == 0: s.cap = 8
else: s.cap = (s.cap * 3) shr 1
s.data = cast[type(s.data)](realloc(s.data, old + 1, s.cap + 1))
proc add*(s: var string; c: char) =
if s.len >= s.cap: resize(s)
s.data[s.len] = c
s.data[s.len+1] = '\0'
inc s.len
proc ensure(s: var string; newLen: int) =
let old = s.cap
if newLen >= old:
s.cap = max((old * 3) shr 1, newLen)
if s.cap > 0:
s.data = cast[type(s.data)](realloc(s.data, old + 1, s.cap + 1))
proc add*(s: var string; y: string) =
if y.len != 0:
let newLen = s.len + y.len
ensure(s, newLen)
copyMem(addr s.data[len], y.data, y.data.len + 1)
s.len = newLen
proc len*(s: string): int {.inline.} = s.len
proc newString*(len: int): string =
result.len = len
result.cap = len
if len > 0:
result.data = alloc0(len+1)
converter toCString(x: string): cstring {.core.} =
if x.len == 0: cstring"" else: cast[cstring](x.data)
proc newStringOfCap*(cap: int): string =
result.len = 0
result.cap = cap
if cap > 0:
result.data = alloc(cap+1)
proc `&`*(a, b: string): string =
let sum = a.len + b.len
result = newStringOfCap(sum)
result.len = sum
copyMem(addr result.data[0], a.data, a.len)
copyMem(addr result.data[a.len], b.data, b.len)
if sum > 0:
result.data[sum] = '\0'
proc concat(x: openArray[string]): string {.core.} =
## used be the code generator to optimize 'x & y & z ...'
var sum = 0
for i in 0 ..< x.len: inc(sum, x[i].len)
result = newStringOfCap(sum)
sum = 0
for i in 0 ..< x.len:
let L = x[i].len
copyMem(addr result.data[sum], x[i].data, L)
inc(sum, L)

19
lib/core/typelayouts.nim Normal file
View file

@ -0,0 +1,19 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
type
TypeLayout* = object
size*, alignment*: int
destructor*: proc (self: pointer; a: Allocator) {.nimcall.}
trace*: proc (self: pointer; a: Allocator) {.nimcall.}
when false:
construct*: proc (self: pointer; a: Allocator) {.nimcall.}
copy*, deepcopy*, sink*: proc (self, other: pointer; a: Allocator) {.nimcall.}
proc getTypeLayout(t: typedesc): ptr TypeLayout {.magic: "getTypeLayout".}

141
lib/js/asyncjs.nim Normal file
View file

@ -0,0 +1,141 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim Authors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
## This module implements types and macros for writing asynchronous code
## for the JS backend. It provides tools for interaction with JavaScript async API-s
## and libraries, writing async procedures in Nim and converting callback-based code
## to promises.
##
## A Nim procedure is asynchronous when it includes the ``{.async.}`` pragma. It
## should always have a ``Future[T]`` return type or not have a return type at all.
## A ``Future[void]`` return type is assumed by default.
##
## This is roughly equivalent to the ``async`` keyword in JavaScript code.
##
## .. code-block:: nim
## proc loadGame(name: string): Future[Game] {.async.} =
## # code
##
## should be equivalent to
##
## .. code-block:: javascript
## async function loadGame(name) {
## // code
## }
##
## A call to an asynchronous procedure usually needs ``await`` to wait for
## the completion of the ``Future``.
##
## .. code-block:: nim
## var game = await loadGame(name)
##
## Often, you might work with callback-based API-s. You can wrap them with
## asynchronous procedures using promises and ``newPromise``:
##
## .. code-block:: nim
## proc loadGame(name: string): Future[Game] =
## var promise = newPromise() do (resolve: proc(response: Game)):
## cbBasedLoadGame(name) do (game: Game):
## resolve(game)
## return promise
##
## Forward definitions work properly, you just need to always add the ``{.async.}`` pragma:
##
## .. code-block:: nim
## proc loadGame(name: string): Future[Game] {.async.}
##
## JavaScript compatibility
## ~~~~~~~~~~~~~~~~~~~~~~~~~
##
## Nim currently generates `async/await` JavaScript code which is supported in modern
## EcmaScript and most modern versions of browsers, Node.js and Electron.
## If you need to use this module with older versions of JavaScript, you can
## use a tool that backports the resulting JavaScript code, as babel.
import jsffi
import macros
when not defined(js) and not defined(nimdoc) and not defined(nimsuggest):
{.fatal: "Module asyncjs is designed to be used with the JavaScript backend.".}
type
Future*[T] = ref object
future*: T
## Wraps the return type of an asynchronous procedure.
PromiseJs* {.importcpp: "Promise".} = ref object
## A JavaScript Promise
proc replaceReturn(node: var NimNode) =
var z = 0
for s in node:
var son = node[z]
if son.kind == nnkReturnStmt:
node[z] = nnkReturnStmt.newTree(nnkCall.newTree(ident("jsResolve"), son[0]))
elif son.kind == nnkAsgn and son[0].kind == nnkIdent and $son[0] == "result":
node[z] = nnkAsgn.newTree(son[0], nnkCall.newTree(ident("jsResolve"), son[1]))
else:
replaceReturn(son)
inc z
proc isFutureVoid(node: NimNode): bool =
result = node.kind == nnkBracketExpr and
node[0].kind == nnkIdent and $node[0] == "Future" and
node[1].kind == nnkIdent and $node[1] == "void"
proc generateJsasync(arg: NimNode): NimNode =
assert arg.kind == nnkProcDef
result = arg
var isVoid = false
var jsResolveNode = ident("jsResolve")
if arg.params[0].kind == nnkEmpty:
result.params[0] = nnkBracketExpr.newTree(ident("Future"), ident("void"))
isVoid = true
elif isFutureVoid(arg.params[0]):
isVoid = true
var code = result.body
replaceReturn(code)
result.body = nnkStmtList.newTree()
if len(code) > 0:
var awaitFunction = quote:
proc await[T](f: Future[T]): T {.importcpp: "(await #)".}
result.body.add(awaitFunction)
var resolve: NimNode
if isVoid:
resolve = quote:
var `jsResolveNode` {.importcpp: "undefined".}: Future[void]
else:
resolve = quote:
proc jsResolve[T](a: T): Future[T] {.importcpp: "#".}
result.body.add(resolve)
else:
result.body = newEmptyNode()
for child in code:
result.body.add(child)
if len(code) > 0 and isVoid:
var voidFix = quote:
return `jsResolveNode`
result.body.add(voidFix)
result.pragma = quote:
{.codegenDecl: "async function $2($3)".}
macro async*(arg: untyped): untyped =
## Macro which converts normal procedures into
## javascript-compatible async procedures
generateJsasync(arg)
proc newPromise*[T](handler: proc(resolve: proc(response: T))): Future[T] {.importcpp: "(new Promise(#))".}
## A helper for wrapping callback-based functions
## into promises and async procedures

7
lib/js/dom.nim
View file

@ -134,9 +134,9 @@ type
# https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement
HtmlElement* = ref object of Element
contentEditable*: string
contentEditable*: cstring
isContentEditable*: bool
dir*: string
dir*: cstring
offsetHeight*: int
offsetWidth*: int
offsetLeft*: int
@ -405,7 +405,7 @@ type
# EventTarget "methods"
proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false)
proc addEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), options: AddEventListenerOptions)
proc removeEventListener*(et: EventTarget, ev: cstring, cb: proc(ev: Event), useCapture: bool = false)
# Window "methods"
proc alert*(w: Window, msg: cstring)
@ -507,6 +507,7 @@ proc replace*(loc: Location, s: cstring)
proc back*(h: History)
proc forward*(h: History)
proc go*(h: History, pagesToJump: int)
proc pushState*[T](h: History, stateObject: T, title, url: cstring)
# Navigator "methods"
proc javaEnabled*(h: Navigator): bool

17
lib/nimbase.h
View file

@ -70,7 +70,7 @@ __clang__
#if defined(_MSC_VER)
# pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309)
# pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706)
# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297)
# pragma warning(disable: 4710 4711 4774 4800 4809 4820 4996 4090 4297)
#endif
/* ------------------------------------------------------------------------- */
@ -159,6 +159,7 @@ __clang__
/* ------------------------------------------------------------------- */
#if defined(WIN32) || defined(_WIN32) /* only Windows has this mess... */
# define N_LIB_PRIVATE
# define N_CDECL(rettype, name) rettype __cdecl name
# define N_STDCALL(rettype, name) rettype __stdcall name
# define N_SYSCALL(rettype, name) rettype __syscall name
@ -178,6 +179,7 @@ __clang__
# endif
# define N_LIB_IMPORT extern __declspec(dllimport)
#else
# define N_LIB_PRIVATE __attribute__((visibility("hidden")))
# if defined(__GNUC__)
# define N_CDECL(rettype, name) rettype name
# define N_STDCALL(rettype, name) rettype name
@ -398,11 +400,11 @@ typedef struct TStringDesc* string;
// NAN definition copied from math.h included in the Windows SDK version 10.0.14393.0
#ifndef NAN
#ifndef _HUGE_ENUF
#define _HUGE_ENUF 1e+300 // _HUGE_ENUF*_HUGE_ENUF must overflow
#endif
#define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF))
#define NAN ((float)(NAN_INFINITY * 0.0F))
# ifndef _HUGE_ENUF
# define _HUGE_ENUF 1e+300 // _HUGE_ENUF*_HUGE_ENUF must overflow
# endif
# define NAN_INFINITY ((float)(_HUGE_ENUF * _HUGE_ENUF))
# define NAN ((float)(NAN_INFINITY * 0.0F))
#endif
#ifndef INF
@ -480,7 +482,6 @@ static inline void GCGuard (void *ptr) { asm volatile ("" :: "X" (ptr)); }
On disagreement, your C compiler will say something like:
"error: 'Nim_and_C_compiler_disagree_on_target_architecture' declared as an array with a negative size" */
typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == sizeof(void*) && NIM_INTBITS == sizeof(NI)*8 ? 1 : -1];
#endif
#ifdef __cplusplus
# define NIM_EXTERNC extern "C"
@ -507,3 +508,5 @@ extern Libc::Env *genodeEnv;
/* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */
#define NIM_CHECK_SIZE(typ, sz) \
_Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size")
#endif /* NIMBASE_H */

3
lib/posix/posix.nim
View file

@ -609,11 +609,12 @@ proc clock_nanosleep*(a1: ClockId, a2: cint, a3: var Timespec,
proc clock_settime*(a1: ClockId, a2: var Timespec): cint {.
importc, header: "<time.h>".}
proc `==`*(a, b: Time): bool {.borrow.}
proc `-`*(a, b: Time): Time {.borrow.}
proc ctime*(a1: var Time): cstring {.importc, header: "<time.h>".}
proc ctime_r*(a1: var Time, a2: cstring): cstring {.importc, header: "<time.h>".}
proc difftime*(a1, a2: Time): cdouble {.importc, header: "<time.h>".}
proc getdate*(a1: cstring): ptr Tm {.importc, header: "<time.h>".}
proc gmtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".}
proc gmtime_r*(a1: var Time, a2: var Tm): ptr Tm {.importc, header: "<time.h>".}
proc localtime*(a1: var Time): ptr Tm {.importc, header: "<time.h>".}

6
lib/posix/posix_linux_amd64.nim
View file

@ -12,8 +12,6 @@
# To be included from posix.nim!
from times import Time
const
hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays
hasAioH = defined(linux)
@ -40,13 +38,15 @@ type
const SIG_HOLD* = cast[SigHandler](2)
type
Time* {.importc: "time_t", header: "<time.h>".} = distinct clong
Timespec* {.importc: "struct timespec",
header: "<time.h>", final, pure.} = object ## struct timespec
tv_sec*: Time ## Seconds.
tv_nsec*: clong ## Nanoseconds.
Dirent* {.importc: "struct dirent",
header: "<dirent.h>", final, pure.} = object ## dirent_t struct
header: "<dirent.h>", final, pure.} = object ## dirent_t struct
d_ino*: Ino
d_off*: Off
d_reclen*: cushort

36
lib/posix/posix_other.nim
View file

@ -9,8 +9,6 @@
{.deadCodeElim:on.}
from times import Time
const
hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays
hasAioH = defined(linux)
@ -36,6 +34,8 @@ type
{.deprecated: [TSocketHandle: SocketHandle].}
type
Time* {.importc: "time_t", header: "<time.h>".} = distinct clong
Timespec* {.importc: "struct timespec",
header: "<time.h>", final, pure.} = object ## struct timespec
tv_sec*: Time ## Seconds.
@ -209,24 +209,24 @@ type
st_gid*: Gid ## Group ID of file.
st_rdev*: Dev ## Device ID (if file is character or block special).
st_size*: Off ## For regular files, the file size in bytes.
## For symbolic links, the length in bytes of the
## pathname contained in the symbolic link.
## For a shared memory object, the length in bytes.
## For a typed memory object, the length in bytes.
## For other file types, the use of this field is
## unspecified.
## For symbolic links, the length in bytes of the
## pathname contained in the symbolic link.
## For a shared memory object, the length in bytes.
## For a typed memory object, the length in bytes.
## For other file types, the use of this field is
## unspecified.
when defined(macosx) or defined(android):
st_atime*: Time ## Time of last access.
st_mtime*: Time ## Time of last data modification.
st_ctime*: Time ## Time of last status change.
st_atime*: Time ## Time of last access.
st_mtime*: Time ## Time of last data modification.
st_ctime*: Time ## Time of last status change.
else:
st_atim*: Timespec ## Time of last access.
st_mtim*: Timespec ## Time of last data modification.
st_ctim*: Timespec ## Time of last status change.
st_blksize*: Blksize ## A file system-specific preferred I/O block size
## for this object. In some file system types, this
## may vary from file to file.
st_blocks*: Blkcnt ## Number of blocks allocated for this object.
st_atim*: Timespec ## Time of last access.
st_mtim*: Timespec ## Time of last data modification.
st_ctim*: Timespec ## Time of last status change.
st_blksize*: Blksize ## A file system-specific preferred I/O block size
## for this object. In some file system types, this
## may vary from file to file.
st_blocks*: Blkcnt ## Number of blocks allocated for this object.
Statvfs* {.importc: "struct statvfs", header: "<sys/statvfs.h>",

45
lib/pure/asyncdispatch.nim
View file

@ -168,18 +168,20 @@ type
timers*: HeapQueue[tuple[finishAt: float, fut: Future[void]]]
callbacks*: Deque[proc ()]
proc processTimers(p: PDispatcherBase) {.inline.} =
proc processTimers(p: PDispatcherBase; didSomeWork: var bool) {.inline.} =
#Process just part if timers at a step
var count = p.timers.len
let t = epochTime()
while count > 0 and t >= p.timers[0].finishAt:
p.timers.pop().fut.complete()
dec count
didSomeWork = true
proc processPendingCallbacks(p: PDispatcherBase) =
proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) =
while p.callbacks.len > 0:
var cb = p.callbacks.popFirst()
cb()
didSomeWork = true
proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} =
# If dispatcher has active timers this proc returns the timeout
@ -298,14 +300,13 @@ when defined(windows) or defined(nimdoc):
let p = getGlobalDispatcher()
p.handles.len != 0 or p.timers.len != 0 or p.callbacks.len != 0
proc poll*(timeout = 500) =
## Waits for completion events and processes them. Raises ``ValueError``
## if there are no pending operations.
proc runOnce(timeout = 500): bool =
let p = getGlobalDispatcher()
if p.handles.len == 0 and p.timers.len == 0 and p.callbacks.len == 0:
raise newException(ValueError,
"No handles or timers registered in dispatcher.")
result = false
if p.handles.len != 0:
let at = p.adjustedTimeout(timeout)
var llTimeout =
@ -318,6 +319,7 @@ when defined(windows) or defined(nimdoc):
let res = getQueuedCompletionStatus(p.ioPort,
addr lpNumberOfBytesTransferred, addr lpCompletionKey,
cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool
result = true
# http://stackoverflow.com/a/12277264/492186
# TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html
@ -347,13 +349,14 @@ when defined(windows) or defined(nimdoc):
else:
if errCode.int32 == WAIT_TIMEOUT:
# Timed out
discard
result = false
else: raiseOSError(errCode)
# Timer processing.
processTimers(p)
processTimers(p, result)
# Callback queue processing
processPendingCallbacks(p)
processPendingCallbacks(p, result)
var acceptEx: WSAPROC_ACCEPTEX
var connectEx: WSAPROC_CONNECTEX
@ -1229,7 +1232,7 @@ else:
# descriptor was unregistered in callback via `unregister()`.
discard
proc poll*(timeout = 500) =
proc runOnce(timeout = 500): bool =
let p = getGlobalDispatcher()
when ioselSupportedPlatform:
let customSet = {Event.Timer, Event.Signal, Event.Process,
@ -1239,6 +1242,7 @@ else:
raise newException(ValueError,
"No handles or timers registered in dispatcher.")
result = false
if not p.selector.isEmpty():
var keys: array[64, ReadyKey]
var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys)
@ -1251,20 +1255,24 @@ else:
if Event.Read in events or events == {Event.Error}:
processBasicCallbacks(fd, readList)
result = true
if Event.Write in events or events == {Event.Error}:
processBasicCallbacks(fd, writeList)
result = true
if Event.User in events or events == {Event.Error}:
if Event.User in events:
processBasicCallbacks(fd, readList)
custom = true
if rLength == 0:
p.selector.unregister(fd)
result = true
when ioselSupportedPlatform:
if (customSet * events) != {}:
custom = true
processCustomCallbacks(fd)
result = true
# because state `data` can be modified in callback we need to update
# descriptor events with currently registered callbacks.
@ -1276,9 +1284,9 @@ else:
p.selector.updateHandle(SocketHandle(fd), newEvents)
# Timer processing.
processTimers(p)
processTimers(p, result)
# Callback queue processing
processPendingCallbacks(p)
processPendingCallbacks(p, result)
proc recv*(socket: AsyncFD, size: int,
flags = {SocketFlag.SafeDisconn}): Future[string] =
@ -1501,6 +1509,19 @@ else:
data.readList.add(cb)
p.selector.registerEvent(SelectEvent(ev), data)
proc drain*(timeout = 500) =
## Waits for completion events and processes them. Raises ``ValueError``
## if there are no pending operations. In contrast to ``poll`` this
## processes as many events as are available.
if runOnce(timeout):
while hasPendingOperations() and runOnce(0): discard
proc poll*(timeout = 500) =
## Waits for completion events and processes them. Raises ``ValueError``
## if there are no pending operations. This runs the underlying OS
## `epoll`:idx: or `kqueue`:idx: primitive only once.
discard runOnce(timeout)
# Common procedures between current and upcoming asyncdispatch
include includes.asynccommon

89
lib/pure/asyncfutures.nim
View file

@ -1,4 +1,4 @@
import os, tables, strutils, times, heapqueue, options, deques
import os, tables, strutils, times, heapqueue, options, deques, cstrutils
# TODO: This shouldn't need to be included, but should ideally be exported.
type
@ -217,17 +217,78 @@ proc `callback=`*[T](future: Future[T],
## If future has already completed then ``cb`` will be called immediately.
future.callback = proc () = cb(future)
proc injectStacktrace[T](future: Future[T]) =
# TODO: Come up with something better.
when not defined(release):
var msg = ""
msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:")
proc getHint(entry: StackTraceEntry): string =
## We try to provide some hints about stack trace entries that the user
## may not be familiar with, in particular calls inside the stdlib.
result = ""
if entry.procname == "processPendingCallbacks":
if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
return "Executes pending callbacks"
elif entry.procname == "poll":
if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
return "Processes asynchronous completion events"
if not future.errorStackTrace.isNil and future.errorStackTrace != "":
msg.add("\n" & indent(future.errorStackTrace.strip(), 4))
else:
msg.add("\n Empty or nil stack trace.")
future.error.msg.add(msg)
if entry.procname.endsWith("_continue"):
if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0:
return "Resumes an async procedure"
proc `$`*(entries: seq[StackTraceEntry]): string =
result = ""
# Find longest filename & line number combo for alignment purposes.
var longestLeft = 0
for entry in entries:
if entry.procName.isNil: continue
let left = $entry.filename & $entry.line
if left.len > longestLeft:
longestLeft = left.len
var indent = 2
# Format the entries.
for entry in entries:
if entry.procName.isNil:
if entry.line == -10:
result.add(spaces(indent) & "#[\n")
indent.inc(2)
else:
indent.dec(2)
result.add(spaces(indent)& "]#\n")
continue
let left = "$#($#)" % [$entry.filename, $entry.line]
result.add((spaces(indent) & "$#$# $#\n") % [
left,
spaces(longestLeft - left.len + 2),
$entry.procName
])
let hint = getHint(entry)
if hint.len > 0:
result.add(spaces(indent+2) & "## " & hint & "\n")
proc injectStacktrace[T](future: Future[T]) =
when not defined(release):
const header = "\nAsync traceback:\n"
var exceptionMsg = future.error.msg
if header in exceptionMsg:
# This is messy: extract the original exception message from the msg
# containing the async traceback.
let start = exceptionMsg.find(header)
exceptionMsg = exceptionMsg[0..<start]
var newMsg = exceptionMsg & header
let entries = getStackTraceEntries(future.error)
newMsg.add($entries)
newMsg.add("Exception message: " & exceptionMsg & "\n")
newMsg.add("Exception type:")
# # For debugging purposes
# for entry in getStackTraceEntries(future.error):
# newMsg.add "\n" & $entry
future.error.msg = newMsg
proc read*[T](future: Future[T] | FutureVar[T]): T =
## Retrieves the value of ``future``. Future must be finished otherwise
@ -263,12 +324,12 @@ proc mget*[T](future: FutureVar[T]): var T =
## Future has not been finished.
result = Future[T](future).value
proc finished*[T](future: Future[T] | FutureVar[T]): bool =
proc finished*(future: FutureBase | FutureVar): bool =
## Determines whether ``future`` has completed.
##
## ``True`` may indicate an error or a value. Use ``failed`` to distinguish.
when future is FutureVar[T]:
result = (Future[T](future)).finished
when future is FutureVar:
result = (FutureBase(future)).finished
else:
result = future.finished

21
lib/pure/asyncmacro.nim
View file

@ -25,22 +25,28 @@ proc skipStmtList(node: NimNode): NimNode {.compileTime.} =
result = node[0]
template createCb(retFutureSym, iteratorNameSym,
name, futureVarCompletions: untyped) =
strName, identName, futureVarCompletions: untyped) =
var nameIterVar = iteratorNameSym
#{.push stackTrace: off.}
proc cb0 {.closure.} =
proc identName {.closure.} =
try:
if not nameIterVar.finished:
var next = nameIterVar()
# Continue while the yielded future is already finished.
while (not next.isNil) and next.finished:
next = nameIterVar()
if nameIterVar.finished:
break
if next == nil:
if not retFutureSym.finished:
let msg = "Async procedure ($1) yielded `nil`, are you await'ing a " &
"`nil` Future?"
raise newException(AssertionError, msg % name)
raise newException(AssertionError, msg % strName)
else:
{.gcsafe.}:
{.push hint[ConvFromXtoItselfNotNeeded]: off.}
next.callback = (proc() {.closure, gcsafe.})(cb0)
next.callback = (proc() {.closure, gcsafe.})(identName)
{.pop.}
except:
futureVarCompletions
@ -52,7 +58,7 @@ template createCb(retFutureSym, iteratorNameSym,
else:
retFutureSym.fail(getCurrentException())
cb0()
identName()
#{.pop.}
proc generateExceptionCheck(futSym,
tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
@ -389,9 +395,12 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
outerProcBody.add(closureIterator)
# -> createCb(retFuture)
#var cbName = newIdentNode("cb")
# NOTE: The "_continue" suffix is checked for in asyncfutures.nim to produce
# friendlier stack traces:
var cbName = genSym(nskProc, prcName & "_continue")
var procCb = getAst createCb(retFutureSym, iteratorNameSym,
newStrLitNode(prcName),
cbName,
createFutureVarCompletions(futureVarIdents, nil))
outerProcBody.add procCb

2
lib/pure/asyncnet.nim
View file

@ -286,6 +286,7 @@ template readInto(buf: pointer, size: int, socket: AsyncSocket,
flags: set[SocketFlag]): int =
## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that
## this is a template and not a proc.
assert(not socket.closed, "Cannot `recv` on a closed socket")
var res = 0
if socket.isSsl:
when defineSsl:
@ -412,6 +413,7 @@ proc send*(socket: AsyncSocket, buf: pointer, size: int,
## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all
## data has been sent.
assert socket != nil
assert(not socket.closed, "Cannot `send` on a closed socket")
if socket.isSsl:
when defineSsl:
sslLoop(socket, flags,

6
lib/pure/collections/critbits.nim
View file

@ -141,8 +141,8 @@ proc excl*[T](c: var CritBitTree[T], key: string) =
proc missingOrExcl*[T](c: var CritBitTree[T], key: string): bool =
## Returns true iff `c` does not contain the given `key`. If the key
## does exist, c.excl(key) is performed.
let oldCount = c.count
## does exist, c.excl(key) is performed.
let oldCount = c.count
var n = exclImpl(c, key)
result = c.count == oldCount
@ -326,7 +326,7 @@ proc `$`*[T](c: CritBitTree[T]): string =
result.add($key)
when T isnot void:
result.add(": ")
result.add($val)
result.addQuoted(val)
result.add("}")
when isMainModule:

2
lib/pure/collections/deques.nim
View file

@ -185,7 +185,7 @@ proc `$`*[T](deq: Deque[T]): string =
result = "["
for x in deq:
if result.len > 1: result.add(", ")
result.add($x)
result.addQuoted(x)
result.add("]")
when isMainModule:

2
lib/pure/collections/lists.nim
View file

@ -135,7 +135,7 @@ proc `$`*[T](L: SomeLinkedCollection[T]): string =
result = "["
for x in nodes(L):
if result.len > 1: result.add(", ")
result.add($x.value)
result.addQuoted(x.value)
result.add("]")
proc find*[T](L: SomeLinkedCollection[T], value: T): SomeLinkedNode[T] =

2
lib/pure/collections/sets.nim
View file

@ -406,7 +406,7 @@ template dollarImpl() {.dirty.} =
result = "{"
for key in items(s):
if result.len > 1: result.add(", ")
result.add($key)
result.addQuoted(key)
result.add("}")
proc `$`*[A](s: HashSet[A]): string =

15
lib/pure/collections/sharedlist.nim
View file

@ -73,10 +73,10 @@ proc add*[A](x: var SharedList[A]; y: A) =
node.d[node.dataLen] = y
inc(node.dataLen)
proc initSharedList*[A](): SharedList[A] =
initLock result.lock
result.head = nil
result.tail = nil
proc init*[A](t: var SharedList[A]) =
initLock t.lock
t.head = nil
t.tail = nil
proc clear*[A](t: var SharedList[A]) =
withLock(t):
@ -92,4 +92,11 @@ proc deinitSharedList*[A](t: var SharedList[A]) =
clear(t)
deinitLock t.lock
proc initSharedList*[A](): SharedList[A] {.deprecated.} =
## Deprecated. Use `init` instead.
## This is not posix compliant, may introduce undefined behavior.
initLock result.lock
result.head = nil
result.tail = nil
{.pop.}

21
lib/pure/collections/sharedtables.nim
View file

@ -183,6 +183,7 @@ proc `[]=`*[A, B](t: var SharedTable[A, B], key: A, val: B) =
proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) =
## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
## This can introduce duplicate keys into the table!
withLock t:
addImpl(enlarge)
@ -191,19 +192,29 @@ proc del*[A, B](t: var SharedTable[A, B], key: A) =
withLock t:
delImpl()
proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] =
proc init*[A, B](t: var SharedTable[A, B], initialSize=64) =
## creates a new hash table that is empty.
##
## `initialSize` needs to be a power of two. If you need to accept runtime
## values for this you could use the ``nextPowerOfTwo`` proc from the
## `math <math.html>`_ module or the ``rightSize`` proc from this module.
assert isPowerOfTwo(initialSize)
result.counter = 0
result.dataLen = initialSize
result.data = cast[KeyValuePairSeq[A, B]](allocShared0(
t.counter = 0
t.dataLen = initialSize
t.data = cast[KeyValuePairSeq[A, B]](allocShared0(
sizeof(KeyValuePair[A, B]) * initialSize))
initLock result.lock
initLock t.lock
proc deinitSharedTable*[A, B](t: var SharedTable[A, B]) =
deallocShared(t.data)
deinitLock t.lock
proc initSharedTable*[A, B](initialSize=64): SharedTable[A, B] {.deprecated.} =
## Deprecated. Use `init` instead.
## This is not posix compliant, may introduce undefined behavior.
assert isPowerOfTwo(initialSize)
result.counter = 0
result.dataLen = initialSize
result.data = cast[KeyValuePairSeq[A, B]](allocShared0(
sizeof(KeyValuePair[A, B]) * initialSize))
initLock result.lock

24
lib/pure/collections/tables.nim
View file

@ -308,6 +308,7 @@ proc `[]=`*[A, B](t: var Table[A, B], key: A, val: B) =
proc add*[A, B](t: var Table[A, B], key: A, val: B) =
## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
## This can introduce duplicate keys into the table!
addImpl(enlarge)
proc len*[A, B](t: TableRef[A, B]): int =
@ -337,9 +338,9 @@ template dollarImpl(): untyped {.dirty.} =
result = "{"
for key, val in pairs(t):
if result.len > 1: result.add(", ")
result.add($key)
result.addQuoted(key)
result.add(": ")
result.add($val)
result.addQuoted(val)
result.add("}")
proc `$`*[A, B](t: Table[A, B]): string =
@ -430,6 +431,7 @@ proc `[]=`*[A, B](t: TableRef[A, B], key: A, val: B) =
proc add*[A, B](t: TableRef[A, B], key: A, val: B) =
## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
## This can introduce duplicate keys into the table!
t[].add(key, val)
proc del*[A, B](t: TableRef[A, B], key: A) =
@ -604,6 +606,7 @@ proc `[]=`*[A, B](t: var OrderedTable[A, B], key: A, val: B) =
proc add*[A, B](t: var OrderedTable[A, B], key: A, val: B) =
## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
## This can introduce duplicate keys into the table!
addImpl(enlarge)
proc mgetOrPut*[A, B](t: var OrderedTable[A, B], key: A, val: B): var B =
@ -770,6 +773,7 @@ proc `[]=`*[A, B](t: OrderedTableRef[A, B], key: A, val: B) =
proc add*[A, B](t: OrderedTableRef[A, B], key: A, val: B) =
## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists.
## This can introduce duplicate keys into the table!
t[].add(key, val)
proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] =
@ -962,9 +966,10 @@ proc initCountTable*[A](initialSize=64): CountTable[A] =
newSeq(result.data, initialSize)
proc toCountTable*[A](keys: openArray[A]): CountTable[A] =
## creates a new count table with every key in `keys` having a count of 1.
## creates a new count table with every key in `keys` having a count
## of how many times it occurs in `keys`.
result = initCountTable[A](rightSize(keys.len))
for key in items(keys): result[key] = 1
for key in items(keys): result.inc key
proc `$`*[A](t: CountTable[A]): string =
## The `$` operator for count tables.
@ -989,9 +994,10 @@ proc inc*[A](t: var CountTable[A], key: A, val = 1) =
proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] =
## returns the (key,val)-pair with the smallest `val`. Efficiency: O(n)
assert t.len > 0
var minIdx = 0
for h in 1..high(t.data):
if t.data[h].val > 0 and t.data[minIdx].val > t.data[h].val: minIdx = h
var minIdx = -1
for h in 0..high(t.data):
if t.data[h].val > 0 and (minIdx == -1 or t.data[minIdx].val > t.data[h].val):
minIdx = h
result.key = t.data[minIdx].key
result.val = t.data[minIdx].val
@ -1325,3 +1331,7 @@ when isMainModule:
assert((a == b) == true)
assert((b == a) == true)
block: # CountTable.smallest
var t = initCountTable[int]()
for v in items([0, 0, 5, 5, 5]): t.inc(v)
doAssert t.smallest == (0, 2)

6
lib/pure/cookies.nim
View file

@ -51,7 +51,7 @@ proc setCookie*(key, value: string, domain = "", path = "",
if secure: result.add("; Secure")
if httpOnly: result.add("; HttpOnly")
proc setCookie*(key, value: string, expires: TimeInfo,
proc setCookie*(key, value: string, expires: DateTime,
domain = "", path = "", noName = false,
secure = false, httpOnly = false): string =
## Creates a command in the format of
@ -63,9 +63,9 @@ proc setCookie*(key, value: string, expires: TimeInfo,
noname, secure, httpOnly)
when isMainModule:
var tim = Time(int(getTime()) + 76 * (60 * 60 * 24))
var tim = fromUnix(getTime().toUnix + 76 * (60 * 60 * 24))
let cookie = setCookie("test", "value", tim.getGMTime())
let cookie = setCookie("test", "value", tim.utc)
when not defined(testing):
echo cookie
let start = "Set-Cookie: test=value; Expires="

79
lib/pure/cstrutils.nim Normal file
View file

@ -0,0 +1,79 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## This module supports helper routines for working with ``cstring``
## without having to convert ``cstring`` to ``string`` in order to
## save allocations.
include "system/inclrtl"
proc toLowerAscii(c: char): char {.inline.} =
if c in {'A'..'Z'}:
result = chr(ord(c) + (ord('a') - ord('A')))
else:
result = c
proc startsWith*(s, prefix: cstring): bool {.noSideEffect,
rtl, extern: "csuStartsWith".} =
## Returns true iff ``s`` starts with ``prefix``.
##
## If ``prefix == ""`` true is returned.
var i = 0
while true:
if prefix[i] == '\0': return true
if s[i] != prefix[i]: return false
inc(i)
proc endsWith*(s, suffix: cstring): bool {.noSideEffect,
rtl, extern: "csuEndsWith".} =
## Returns true iff ``s`` ends with ``suffix``.
##
## If ``suffix == ""`` true is returned.
let slen = s.len
var i = 0
var j = slen - len(suffix)
while i+j <% slen:
if s[i+j] != suffix[i]: return false
inc(i)
if suffix[i] == '\0': return true
proc cmpIgnoreStyle*(a, b: cstring): int {.noSideEffect,
rtl, extern: "csuCmpIgnoreStyle".} =
## Compares two strings normalized (i.e. case and
## underscores do not matter). Returns:
##
## | 0 iff a == b
## | < 0 iff a < b
## | > 0 iff a > b
var i = 0
var j = 0
while true:
while a[i] == '_': inc(i)
while b[j] == '_': inc(j) # BUGFIX: typo
var aa = toLowerAscii(a[i])
var bb = toLowerAscii(b[j])
result = ord(aa) - ord(bb)
if result != 0 or aa == '\0': break
inc(i)
inc(j)
proc cmpIgnoreCase*(a, b: cstring): int {.noSideEffect,
rtl, extern: "csuCmpIgnoreCase".} =
## Compares two strings in a case insensitive manner. Returns:
##
## | 0 iff a == b
## | < 0 iff a < b
## | > 0 iff a > b
var i = 0
while true:
var aa = toLowerAscii(a[i])
var bb = toLowerAscii(b[i])
result = ord(aa) - ord(bb)
if result != 0 or aa == '\0': break
inc(i)

12
lib/pure/httpclient.nim
View file

@ -923,8 +923,14 @@ proc parseChunks(client: HttpClient | AsyncHttpClient): Future[void]
if chunkSize <= 0:
discard await recvFull(client, 2, client.timeout, false) # Skip \c\L
break
discard await recvFull(client, chunkSize, client.timeout, true)
discard await recvFull(client, 2, client.timeout, false) # Skip \c\L
var bytesRead = await recvFull(client, chunkSize, client.timeout, true)
if bytesRead != chunkSize:
httpError("Server terminated connection prematurely")
bytesRead = await recvFull(client, 2, client.timeout, false) # Skip \c\L
if bytesRead != 2:
httpError("Server terminated connection prematurely")
# Trailer headers will only be sent if the request specifies that we want
# them: http://tools.ietf.org/html/rfc2616#section-3.6.1
@ -965,7 +971,7 @@ proc parseBody(client: HttpClient | AsyncHttpClient,
if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0":
while true:
let recvLen = await client.recvFull(4000, client.timeout, true)
if recvLen == 0:
if recvLen != 4000:
client.close()
break

7
lib/pure/ioselects/ioselectors_epoll.nim
View file

@ -277,15 +277,16 @@ proc registerTimer*[T](s: Selector[T], timeout: int, oneshot: bool,
var events = {Event.Timer}
var epv = EpollEvent(events: EPOLLIN or EPOLLRDHUP)
epv.data.u64 = fdi.uint
if oneshot:
new_ts.it_interval.tv_sec = 0.Time
new_ts.it_interval.tv_sec = posix.Time(0)
new_ts.it_interval.tv_nsec = 0
new_ts.it_value.tv_sec = (timeout div 1_000).Time
new_ts.it_value.tv_sec = posix.Time(timeout div 1_000)
new_ts.it_value.tv_nsec = (timeout %% 1_000) * 1_000_000
incl(events, Event.Oneshot)
epv.events = epv.events or EPOLLONESHOT
else:
new_ts.it_interval.tv_sec = (timeout div 1000).Time
new_ts.it_interval.tv_sec = posix.Time(timeout div 1000)
new_ts.it_interval.tv_nsec = (timeout %% 1_000) * 1_000_000
new_ts.it_value.tv_sec = new_ts.it_interval.tv_sec
new_ts.it_value.tv_nsec = new_ts.it_interval.tv_nsec

4
lib/pure/ioselects/ioselectors_kqueue.nim
View file

@ -452,10 +452,10 @@ proc selectInto*[T](s: Selector[T], timeout: int,
if timeout != -1:
if timeout >= 1000:
tv.tv_sec = (timeout div 1_000).Time
tv.tv_sec = posix.Time(timeout div 1_000)
tv.tv_nsec = (timeout %% 1_000) * 1_000_000
else:
tv.tv_sec = 0.Time
tv.tv_sec = posix.Time(0)
tv.tv_nsec = timeout * 1_000_000
else:
ptv = nil

15
lib/pure/logging.nim
View file

@ -107,9 +107,14 @@ var
proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): string =
## Format a log message using the ``frmt`` format string, ``level`` and varargs.
## See the module documentation for the format string syntax.
const nilString = "nil"
var msgLen = 0
for arg in args:
msgLen += arg.len
if arg.isNil:
msgLen += nilString.len
else:
msgLen += arg.len
result = newStringOfCap(frmt.len + msgLen + 20)
var i = 0
while i < frmt.len:
@ -136,7 +141,10 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str
of "levelname": result.add(LevelNames[level])
else: discard
for arg in args:
result.add(arg)
if arg.isNil:
result.add(nilString)
else:
result.add(arg)
method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {.
raises: [Exception], gcsafe,
@ -361,3 +369,6 @@ when not defined(testing) and isMainModule:
addHandler(L)
for i in 0 .. 25:
info("hello", i)
var nilString: string
info "hello ", nilString

19
lib/pure/math.nim
View file

@ -291,6 +291,8 @@ when not defined(JS):
## echo fmod(-2.5, 0.3) ## -0.1
else:
proc trunc*(x: float32): float32 {.importc: "Math.trunc", nodecl.}
proc trunc*(x: float64): float64 {.importc: "Math.trunc", nodecl.}
proc floor*(x: float32): float32 {.importc: "Math.floor", nodecl.}
proc floor*(x: float64): float64 {.importc: "Math.floor", nodecl.}
proc ceil*(x: float32): float32 {.importc: "Math.ceil", nodecl.}
@ -349,15 +351,19 @@ proc round*[T: float32|float64](x: T, places: int = 0): T =
result = round0(x*mult)/mult
when not defined(JS):
proc frexp*(x: float32, exponent: var int): float32 {.
proc c_frexp*(x: float32, exponent: var int32): float32 {.
importc: "frexp", header: "<math.h>".}
proc frexp*(x: float64, exponent: var int): float64 {.
proc c_frexp*(x: float64, exponent: var int32): float64 {.
importc: "frexp", header: "<math.h>".}
proc frexp*[T, U](x: T, exponent: var U): T =
## Split a number into mantissa and exponent.
## `frexp` calculates the mantissa m (a float greater than or equal to 0.5
## and less than 1) and the integer value n such that `x` (the original
## float value) equals m * 2**n. frexp stores n in `exponent` and returns
## m.
var exp: int32
result = c_frexp(x, exp)
exponent = exp
else:
proc frexp*[T: float32|float64](x: T, exponent: var int): T =
if x == 0.0:
@ -366,9 +372,14 @@ else:
elif x < 0.0:
result = -frexp(-x, exponent)
else:
var ex = floor(log2(x))
exponent = round(ex)
var ex = trunc(log2(x))
exponent = int(ex)
result = x / pow(2.0, ex)
if abs(result) >= 1:
inc(exponent)
result = result / 2
if exponent == 1024 and result == 0.0:
result = 0.99999999999999988898
proc splitDecimal*[T: float32|float64](x: T): tuple[intpart: T, floatpart: T] =
## Breaks `x` into an integral and a fractional part.

11
lib/pure/memfiles.nim
View file

@ -257,10 +257,13 @@ proc close*(f: var MemFile) =
when defined(windows):
if f.wasOpened:
error = unmapViewOfFile(f.mem) == 0
lastErr = osLastError()
error = (closeHandle(f.mapHandle) == 0) or error
if f.fHandle != INVALID_HANDLE_VALUE:
error = (closeHandle(f.fHandle) == 0) or error
if not error:
error = closeHandle(f.mapHandle) == 0
if not error and f.fHandle != INVALID_HANDLE_VALUE:
discard closeHandle(f.fHandle)
f.fHandle = INVALID_HANDLE_VALUE
if error:
lastErr = osLastError()
else:
error = munmap(f.mem, f.size) != 0
lastErr = osLastError()

47
lib/pure/net.nim
View file

@ -413,7 +413,7 @@ proc isIpAddress*(address_str: string): bool {.tags: [].} =
when defineSsl:
CRYPTO_malloc_init()
SslLibraryInit()
doAssert SslLibraryInit() == 1
SslLoadErrorStrings()
ErrLoadBioStrings()
OpenSSL_add_all_algorithms()
@ -864,6 +864,7 @@ proc close*(socket: Socket) =
socket.sslHandle = nil
socket.fd.close()
socket.fd = osInvalidSocket
when defined(posix):
from posix import TCP_NODELAY
@ -1001,15 +1002,25 @@ proc select(readfd: Socket, timeout = 500): int =
var fds = @[readfd.fd]
result = select(fds, timeout)
proc isClosed(socket: Socket): bool =
socket.fd == osInvalidSocket
proc uniRecv(socket: Socket, buffer: pointer, size, flags: cint): int =
## Handles SSL and non-ssl recv in a nice package.
##
## In particular handles the case where socket has been closed properly
## for both SSL and non-ssl.
result = 0
assert(not socket.isClosed, "Cannot `recv` on a closed socket")
when defineSsl:
if socket.isSsl:
return SSLRead(socket.sslHandle, buffer, size)
return recv(socket.fd, buffer, size, flags)
proc readIntoBuf(socket: Socket, flags: int32): int =
result = 0
when defineSsl:
if socket.isSSL:
result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high))
else:
result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags)
else:
result = recv(socket.fd, addr(socket.buffer), cint(socket.buffer.high), flags)
result = uniRecv(socket, addr(socket.buffer), socket.buffer.high, flags)
if result < 0:
# Save it in case it gets reset (the Nim codegen occasionally may call
# Win API functions which reset it).
@ -1055,16 +1066,16 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect]
else:
when defineSsl:
if socket.isSSL:
if socket.sslHasPeekChar:
if socket.sslHasPeekChar: # TODO: Merge this peek char mess into uniRecv
copyMem(data, addr(socket.sslPeekChar), 1)
socket.sslHasPeekChar = false
if size-1 > 0:
var d = cast[cstring](data)
result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1
result = uniRecv(socket, addr(d[1]), cint(size-1), 0'i32) + 1
else:
result = 1
else:
result = SSLRead(socket.sslHandle, data, size)
result = uniRecv(socket, data, size.cint, 0'i32)
else:
result = recv(socket.fd, data, size.cint, 0'i32)
else:
@ -1131,17 +1142,21 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1,
##
## When 0 is returned the socket's connection has been closed.
##
## This function will throw an EOS exception when an error occurs. A value
## This function will throw an OSError exception when an error occurs. A value
## lower than 0 is never returned.
##
## A timeout may be specified in milliseconds, if enough data is not received
## within the time specified an ETimeout exception will be raised.
## within the time specified an TimeoutError exception will be raised.
##
## **Note**: ``data`` must be initialised.
##
## **Warning**: Only the ``SafeDisconn`` flag is currently supported.
data.setLen(size)
result = recv(socket, cstring(data), size, timeout)
result =
if timeout == -1:
recv(socket, cstring(data), size)
else:
recv(socket, cstring(data), size, timeout)
if result < 0:
data.setLen(0)
let lastError = getSocketError(socket)
@ -1178,7 +1193,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} =
when defineSsl:
if socket.isSSL:
if not socket.sslHasPeekChar:
result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1)
result = uniRecv(socket, addr(socket.sslPeekChar), 1, 0'i32)
socket.sslHasPeekChar = true
c = socket.sslPeekChar
@ -1312,6 +1327,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {.
##
## **Note**: This is a low-level version of ``send``. You likely should use
## the version below.
assert(not socket.isClosed, "Cannot `send` on a closed socket")
when defineSsl:
if socket.isSSL:
return SSLWrite(socket.sslHandle, cast[cstring](data), size)
@ -1356,6 +1372,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, data: pointer,
## which is defined below.
##
## **Note:** This proc is not available for SSL sockets.
assert(not socket.isClosed, "Cannot `sendTo` on a closed socket")
var aiList = getAddrInfo(address, port, af)
# try all possibilities:

2
lib/pure/oids.nim
View file

@ -88,7 +88,7 @@ proc generatedTime*(oid: Oid): Time =
var tmp: int32
var dummy = oid.time
bigEndian32(addr(tmp), addr(dummy))
result = Time(tmp)
result = fromUnix(tmp)
when not defined(testing) and isMainModule:
let xo = genOid()

75
lib/pure/os.nim
View file

@ -173,33 +173,33 @@ proc findExe*(exe: string, followSymlinks: bool = true;
return x
result = ""
proc getLastModificationTime*(file: string): Time {.rtl, extern: "nos$1".} =
proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1".} =
## Returns the `file`'s last modification time.
when defined(posix):
var res: Stat
if stat(file, res) < 0'i32: raiseOSError(osLastError())
return res.st_mtime
return fromUnix(res.st_mtime.int64)
else:
var f: WIN32_FIND_DATA
var h = findFirstFile(file, f)
if h == -1'i32: raiseOSError(osLastError())
result = winTimeToUnixTime(rdFileTime(f.ftLastWriteTime))
result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastWriteTime)).int64)
findClose(h)
proc getLastAccessTime*(file: string): Time {.rtl, extern: "nos$1".} =
proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} =
## Returns the `file`'s last read or write access time.
when defined(posix):
var res: Stat
if stat(file, res) < 0'i32: raiseOSError(osLastError())
return res.st_atime
return fromUnix(res.st_atime.int64)
else:
var f: WIN32_FIND_DATA
var h = findFirstFile(file, f)
if h == -1'i32: raiseOSError(osLastError())
result = winTimeToUnixTime(rdFileTime(f.ftLastAccessTime))
result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftLastAccessTime)).int64)
findClose(h)
proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} =
proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} =
## Returns the `file`'s creation time.
##
## **Note:** Under POSIX OS's, the returned time may actually be the time at
@ -208,12 +208,12 @@ proc getCreationTime*(file: string): Time {.rtl, extern: "nos$1".} =
when defined(posix):
var res: Stat
if stat(file, res) < 0'i32: raiseOSError(osLastError())
return res.st_ctime
return fromUnix(res.st_ctime.int64)
else:
var f: WIN32_FIND_DATA
var h = findFirstFile(file, f)
if h == -1'i32: raiseOSError(osLastError())
result = winTimeToUnixTime(rdFileTime(f.ftCreationTime))
result = fromUnix(winTimeToUnixTime(rdFileTime(f.ftCreationTime)).int64)
findClose(h)
proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} =
@ -672,7 +672,10 @@ template walkCommon(pattern: string, filter) =
if dotPos < 0 or idx >= ff.len or ff[idx] == '.' or
pattern[dotPos+1] == '*':
yield splitFile(pattern).dir / extractFilename(ff)
if findNextFile(res, f) == 0'i32: break
if findNextFile(res, f) == 0'i32:
let errCode = getLastError()
if errCode == ERROR_NO_MORE_FILES: break
else: raiseOSError(errCode.OSErrorCode)
else: # here we use glob
var
f: Glob
@ -782,7 +785,10 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
let xx = if relative: extractFilename(getFilename(f))
else: dir / extractFilename(getFilename(f))
yield (k, xx)
if findNextFile(h, f) == 0'i32: break
if findNextFile(h, f) == 0'i32:
let errCode = getLastError()
if errCode == ERROR_NO_MORE_FILES: break
else: raiseOSError(errCode.OSErrorCode)
else:
var d = opendir(dir)
if d != nil:
@ -816,32 +822,40 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
k = getSymlinkFileKind(y)
yield (k, y)
iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {.
tags: [ReadDirEffect].} =
## Recursively walks over the directory `dir` and yields for each file in `dir`.
## The full path for each file is returned. Directories are not returned.
iterator walkDirRec*(dir: string, yieldFilter = {pcFile},
followFilter = {pcDir}): string {.tags: [ReadDirEffect].} =
## Recursively walks over the directory `dir` and yields for each file
## or directory in `dir`.
## The full path for each file or directory is returned.
## **Warning**:
## Modifying the directory structure while the iterator
## is traversing may result in undefined behavior!
##
## Walking is recursive. `filter` controls the behaviour of the iterator:
## Walking is recursive. `filters` controls the behaviour of the iterator:
##
## --------------------- ---------------------------------------------
## filter meaning
## yieldFilter meaning
## --------------------- ---------------------------------------------
## ``pcFile`` yield real files
## ``pcLinkToFile`` yield symbolic links to files
## ``pcDir`` yield real directories
## ``pcLinkToDir`` yield symbolic links to directories
## --------------------- ---------------------------------------------
##
## --------------------- ---------------------------------------------
## followFilter meaning
## --------------------- ---------------------------------------------
## ``pcDir`` follow real directories
## ``pcLinkToDir`` follow symbolic links to directories
## --------------------- ---------------------------------------------
##
var stack = @[dir]
while stack.len > 0:
for k,p in walkDir(stack.pop()):
if k in filter:
case k
of pcFile, pcLinkToFile: yield p
of pcDir, pcLinkToDir: stack.add(p)
for k, p in walkDir(stack.pop()):
if k in {pcDir, pcLinkToDir} and k in followFilter:
stack.add(p)
if k in yieldFilter:
yield p
proc rawRemoveDir(dir: string) =
when defined(windows):
@ -1443,7 +1457,7 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [TimeEffect].} =
winlean.sleep(int32(milsecs))
else:
var a, b: Timespec
a.tv_sec = Time(milsecs div 1000)
a.tv_sec = posix.Time(milsecs div 1000)
a.tv_nsec = (milsecs mod 1000) * 1000 * 1000
discard posix.nanosleep(a, b)
@ -1481,16 +1495,17 @@ type
size*: BiggestInt # Size of file.
permissions*: set[FilePermission] # File permissions
linkCount*: BiggestInt # Number of hard links the file object has.
lastAccessTime*: Time # Time file was last accessed.
lastWriteTime*: Time # Time file was last modified/written to.
creationTime*: Time # Time file was created. Not supported on all systems!
lastAccessTime*: times.Time # Time file was last accessed.
lastWriteTime*: times.Time # Time file was last modified/written to.
creationTime*: times.Time # Time file was created. Not supported on all systems!
template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
## Transforms the native file info structure into the one nim uses.
## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows,
## or a 'Stat' structure on posix
when defined(Windows):
template toTime(e: FILETIME): untyped {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics
template toTime(e: FILETIME): untyped {.gensym.} =
fromUnix(winTimeToUnixTime(rdFileTime(e)).int64) # local templates default to bind semantics
template merge(a, b): untyped = a or (b shl 32)
formalInfo.id.device = rawInfo.dwVolumeSerialNumber
formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh)
@ -1522,9 +1537,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino)
formalInfo.size = rawInfo.st_size
formalInfo.linkCount = rawInfo.st_Nlink.BiggestInt
formalInfo.lastAccessTime = rawInfo.st_atime
formalInfo.lastWriteTime = rawInfo.st_mtime
formalInfo.creationTime = rawInfo.st_ctime
formalInfo.lastAccessTime = fromUnix(rawInfo.st_atime.int64)
formalInfo.lastWriteTime = fromUnix(rawInfo.st_mtime.int64)
formalInfo.creationTime = fromUnix(rawInfo.st_ctime.int64)
result.permissions = {}
checkAndIncludeMode(S_IRUSR, fpUserRead)

183
lib/pure/osproc.nim
View file

@ -47,6 +47,7 @@ type
ProcessObj = object of RootObj
when defined(windows):
fProcessHandle: Handle
fThreadHandle: Handle
inHandle, outHandle, errHandle: FileHandle
id: Handle
else:
@ -54,6 +55,7 @@ type
inStream, outStream, errStream: Stream
id: Pid
exitStatus: cint
exitFlag: bool
options: set[ProcessOption]
Process* = ref ProcessObj ## represents an operating system process
@ -237,11 +239,13 @@ proc execProcesses*(cmds: openArray[string],
if n > 1:
var i = 0
var q = newSeq[Process](n)
var m = min(n, cmds.len)
when defined(windows):
var w: WOHandleArray
var m = min(min(n, MAXIMUM_WAIT_OBJECTS), cmds.len)
var wcount = m
else:
var m = min(n, cmds.len)
while i < m:
if beforeRunEvent != nil:
@ -253,6 +257,7 @@ proc execProcesses*(cmds: openArray[string],
var ecount = len(cmds)
while ecount > 0:
var rexit = -1
when defined(windows):
# waiting for all children, get result if any child exits
var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32,
@ -262,22 +267,37 @@ proc execProcesses*(cmds: openArray[string],
discard
elif ret == WAIT_FAILED:
raiseOSError(osLastError())
else:
var status: int32
for r in 0..m-1:
if not isNil(q[r]) and q[r].fProcessHandle == w[ret]:
discard getExitCodeProcess(q[r].fProcessHandle, status)
q[r].exitFlag = true
q[r].exitStatus = status
rexit = r
break
else:
var status : cint = 1
var status: cint = 1
# waiting for all children, get result if any child exits
let res = waitpid(-1, status, 0)
if res > 0:
for r in 0..m-1:
if not isNil(q[r]) and q[r].id == res:
# we updating `exitStatus` manually, so `running()` can work.
if WIFEXITED(status) or WIFSIGNALED(status):
q[r].exitFlag = true
q[r].exitStatus = status
rexit = r
break
else:
let err = osLastError()
if err == OSErrorCode(ECHILD):
# some child exits, we need to check our childs exit codes
discard
for r in 0..m-1:
if (not isNil(q[r])) and (not running(q[r])):
q[r].exitFlag = true
q[r].exitStatus = status
rexit = r
break
elif err == OSErrorCode(EINTR):
# signal interrupted our syscall, lets repeat it
continue
@ -285,26 +305,27 @@ proc execProcesses*(cmds: openArray[string],
# all other errors are exceptions
raiseOSError(err)
for r in 0..m-1:
if not isNil(q[r]):
if not running(q[r]):
result = max(result, q[r].peekExitCode())
if afterRunEvent != nil: afterRunEvent(r, q[r])
close(q[r])
if i < len(cmds):
if beforeRunEvent != nil: beforeRunEvent(i)
q[r] = startProcess(cmds[i],
if rexit >= 0:
result = max(result, q[rexit].peekExitCode())
if afterRunEvent != nil: afterRunEvent(rexit, q[rexit])
close(q[rexit])
if i < len(cmds):
if beforeRunEvent != nil: beforeRunEvent(i)
q[rexit] = startProcess(cmds[i],
options = options + {poEvalCommand})
when defined(windows):
w[r] = q[r].fProcessHandle
inc(i)
else:
q[r] = nil
when defined(windows):
for c in r..MAXIMUM_WAIT_OBJECTS - 2:
w[c] = w[c + 1]
when defined(windows):
w[rexit] = q[rexit].fProcessHandle
inc(i)
else:
when defined(windows):
for k in 0..wcount - 1:
if w[k] == q[rexit].fProcessHandle:
w[k] = w[wcount - 1]
w[wcount - 1] = 0
dec(wcount)
dec(ecount)
break
q[rexit] = nil
dec(ecount)
else:
for i in 0..high(cmds):
if beforeRunEvent != nil:
@ -491,6 +512,7 @@ when defined(Windows) and not defined(useNimRtl):
hi, ho, he: Handle
new(result)
result.options = options
result.exitFlag = true
si.cb = sizeof(si).cint
if poParentStreams notin options:
si.dwFlags = STARTF_USESTDHANDLES # STARTF_USESHOWWINDOW or
@ -559,28 +581,31 @@ when defined(Windows) and not defined(useNimRtl):
"Requested command not found: '$1'. OS error:" % command)
else:
raiseOSError(lastError, command)
# Close the handle now so anyone waiting is woken:
discard closeHandle(procInfo.hThread)
result.fProcessHandle = procInfo.hProcess
result.fThreadHandle = procInfo.hThread
result.id = procInfo.dwProcessId
result.exitFlag = false
proc close(p: Process) =
if poInteractive in p.options:
# somehow this is not always required on Windows:
if poParentStreams notin p.options:
discard closeHandle(p.inHandle)
discard closeHandle(p.outHandle)
discard closeHandle(p.errHandle)
#discard closeHandle(p.FProcessHandle)
discard closeHandle(p.fThreadHandle)
discard closeHandle(p.fProcessHandle)
proc suspend(p: Process) =
discard suspendThread(p.fProcessHandle)
discard suspendThread(p.fThreadHandle)
proc resume(p: Process) =
discard resumeThread(p.fProcessHandle)
discard resumeThread(p.fThreadHandle)
proc running(p: Process): bool =
var x = waitForSingleObject(p.fProcessHandle, 50)
return x == WAIT_TIMEOUT
if p.exitFlag:
return false
else:
var x = waitForSingleObject(p.fProcessHandle, 0)
return x == WAIT_TIMEOUT
proc terminate(p: Process) =
if running(p):
@ -590,22 +615,37 @@ when defined(Windows) and not defined(useNimRtl):
terminate(p)
proc waitForExit(p: Process, timeout: int = -1): int =
discard waitForSingleObject(p.fProcessHandle, timeout.int32)
if p.exitFlag:
return p.exitStatus
var res: int32
discard getExitCodeProcess(p.fProcessHandle, res)
result = res
p.exitStatus = res
discard closeHandle(p.fProcessHandle)
let res = waitForSingleObject(p.fProcessHandle, timeout.int32)
if res == WAIT_TIMEOUT:
terminate(p)
var status: int32
discard getExitCodeProcess(p.fProcessHandle, status)
if status != STILL_ACTIVE:
p.exitFlag = true
p.exitStatus = status
discard closeHandle(p.fThreadHandle)
discard closeHandle(p.fProcessHandle)
result = status
else:
result = -1
proc peekExitCode(p: Process): int =
var b = waitForSingleObject(p.fProcessHandle, 50) == WAIT_TIMEOUT
if b: result = -1
else:
var res: int32
discard getExitCodeProcess(p.fProcessHandle, res)
if res == 0: return p.exitStatus
return res
if p.exitFlag:
return p.exitStatus
result = -1
var b = waitForSingleObject(p.fProcessHandle, 0) == WAIT_TIMEOUT
if not b:
var status: int32
discard getExitCodeProcess(p.fProcessHandle, status)
p.exitFlag = true
p.exitStatus = status
discard closeHandle(p.fThreadHandle)
discard closeHandle(p.fProcessHandle)
result = status
proc inputStream(p: Process): Stream =
streamAccess(p)
@ -737,7 +777,8 @@ elif not defined(useNimRtl):
pStdin, pStdout, pStderr: array[0..1, cint]
new(result)
result.options = options
result.exitStatus = -3 # for ``waitForExit``
result.exitFlag = true
if poParentStreams notin options:
if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or
pipe(pStderr) != 0'i32:
@ -792,6 +833,7 @@ elif not defined(useNimRtl):
if poEchoCmd in options:
echo(command, " ", join(args, " "))
result.id = pid
result.exitFlag = false
if poParentStreams in options:
# does not make much sense, but better than nothing:
@ -968,14 +1010,14 @@ elif not defined(useNimRtl):
if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError())
proc running(p: Process): bool =
if p.exitStatus != -3:
if p.exitFlag:
return false
else:
var ret : int
var status : cint = 1
ret = waitpid(p.id, status, WNOHANG)
var status: cint = 1
let ret = waitpid(p.id, status, WNOHANG)
if ret == int(p.id):
if isExitStatus(status):
p.exitFlag = true
p.exitStatus = status
return false
else:
@ -998,13 +1040,14 @@ elif not defined(useNimRtl):
import kqueue, times
proc waitForExit(p: Process, timeout: int = -1): int =
if p.exitStatus != -3:
if p.exitFlag:
return exitStatus(p.exitStatus)
if timeout == -1:
var status : cint = 1
var status: cint = 1
if waitpid(p.id, status, 0) < 0:
raiseOSError(osLastError())
p.exitFlag = true
p.exitStatus = status
else:
var kqFD = kqueue()
@ -1017,15 +1060,15 @@ elif not defined(useNimRtl):
var tmspec: Timespec
if timeout >= 1000:
tmspec.tv_sec = (timeout div 1_000).Time
tmspec.tv_sec = posix.Time(timeout div 1_000)
tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000
else:
tmspec.tv_sec = 0.Time
tmspec.tv_sec = posix.Time(0)
tmspec.tv_nsec = (timeout * 1_000_000)
try:
while true:
var status : cint = 1
var status: cint = 1
var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1,
addr(tmspec))
if count < 0:
@ -1038,12 +1081,14 @@ elif not defined(useNimRtl):
raiseOSError(osLastError())
if waitpid(p.id, status, 0) < 0:
raiseOSError(osLastError())
p.exitFlag = true
p.exitStatus = status
break
else:
if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC:
if waitpid(p.id, status, 0) < 0:
raiseOSError(osLastError())
p.exitFlag = true
p.exitStatus = status
break
else:
@ -1064,36 +1109,33 @@ elif not defined(useNimRtl):
var b: Timespec
b.tv_sec = e.tv_sec
b.tv_nsec = e.tv_nsec
e.tv_sec = (e.tv_sec - s.tv_sec).Time
e.tv_sec = e.tv_sec - s.tv_sec
if e.tv_nsec >= s.tv_nsec:
e.tv_nsec -= s.tv_nsec
else:
if e.tv_sec == 0.Time:
if e.tv_sec == posix.Time(0):
raise newException(ValueError, "System time was modified")
else:
diff = s.tv_nsec - e.tv_nsec
e.tv_nsec = 1_000_000_000 - diff
t.tv_sec = (t.tv_sec - e.tv_sec).Time
t.tv_sec = t.tv_sec - e.tv_sec
if t.tv_nsec >= e.tv_nsec:
t.tv_nsec -= e.tv_nsec
else:
t.tv_sec = (int(t.tv_sec) - 1).Time
t.tv_sec = t.tv_sec - posix.Time(1)
diff = e.tv_nsec - t.tv_nsec
t.tv_nsec = 1_000_000_000 - diff
s.tv_sec = b.tv_sec
s.tv_nsec = b.tv_nsec
#if waitPid(p.id, p.exitStatus, 0) == int(p.id):
# ``waitPid`` fails if the process is not running anymore. But then
# ``running`` probably set ``p.exitStatus`` for us. Since ``p.exitStatus`` is
# initialized with -3, wrong success exit codes are prevented.
if p.exitStatus != -3:
if p.exitFlag:
return exitStatus(p.exitStatus)
if timeout == -1:
var status : cint = 1
var status: cint = 1
if waitpid(p.id, status, 0) < 0:
raiseOSError(osLastError())
p.exitFlag = true
p.exitStatus = status
else:
var nmask, omask: Sigset
@ -1112,10 +1154,10 @@ elif not defined(useNimRtl):
raiseOSError(osLastError())
if timeout >= 1000:
tmspec.tv_sec = (timeout div 1_000).Time
tmspec.tv_sec = posix.Time(timeout div 1_000)
tmspec.tv_nsec = (timeout %% 1_000) * 1_000_000
else:
tmspec.tv_sec = 0.Time
tmspec.tv_sec = posix.Time(0)
tmspec.tv_nsec = (timeout * 1_000_000)
try:
@ -1125,9 +1167,10 @@ elif not defined(useNimRtl):
let res = sigtimedwait(nmask, sinfo, tmspec)
if res == SIGCHLD:
if sinfo.si_pid == p.id:
var status : cint = 1
var status: cint = 1
if waitpid(p.id, status, 0) < 0:
raiseOSError(osLastError())
p.exitFlag = true
p.exitStatus = status
break
else:
@ -1148,9 +1191,10 @@ elif not defined(useNimRtl):
# timeout expired, so we trying to kill process
if posix.kill(p.id, SIGKILL) == -1:
raiseOSError(osLastError())
var status : cint = 1
var status: cint = 1
if waitpid(p.id, status, 0) < 0:
raiseOSError(osLastError())
p.exitFlag = true
p.exitStatus = status
break
else:
@ -1168,12 +1212,13 @@ elif not defined(useNimRtl):
proc peekExitCode(p: Process): int =
var status = cint(0)
result = -1
if p.exitStatus != -3:
if p.exitFlag:
return exitStatus(p.exitStatus)
var ret = waitpid(p.id, status, WNOHANG)
if ret > 0:
if isExitStatus(status):
p.exitFlag = true
p.exitStatus = status
result = exitStatus(status)

2
lib/pure/parsecsv.nim
View file

@ -32,7 +32,7 @@
## import parsecsv
## import os
## # Prepare a file
## var csv_content = """One,Two,Three,Four
## var content = """One,Two,Three,Four
## 1,2,3,4
## 10,20,30,40
## 100,200,300,400

463
lib/pure/parsesql.nim
View file

@ -55,6 +55,13 @@ const
";", ":", ",", "(", ")", "[", "]", "."
]
reservedKeywords = @[
# statements
"select", "from", "where", "group", "limit", "having",
# functions
"count",
]
proc open(L: var SqlLexer, input: Stream, filename: string) =
lexbase.open(L, input)
L.filename = filename
@ -274,16 +281,16 @@ proc getSymbol(c: var SqlLexer, tok: var Token) =
c.bufpos = pos
tok.kind = tkIdentifier
proc getQuotedIdentifier(c: var SqlLexer, tok: var Token) =
proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') =
var pos = c.bufpos + 1
var buf = c.buf
tok.kind = tkQuotedIdentifier
while true:
var ch = buf[pos]
if ch == '\"':
if buf[pos+1] == '\"':
if ch == quote:
if buf[pos+1] == quote:
inc(pos, 2)
add(tok.literal, '\"')
add(tok.literal, quote)
else:
inc(pos)
break
@ -442,7 +449,8 @@ proc getTok(c: var SqlLexer, tok: var Token) =
add(tok.literal, '.')
of '0'..'9': getNumeric(c, tok)
of '\'': getString(c, tok, tkStringConstant)
of '"': getQuotedIdentifier(c, tok)
of '"': getQuotedIdentifier(c, tok, '"')
of '`': getQuotedIdentifier(c, tok, '`')
of lexbase.EndOfFile:
tok.kind = tkEof
tok.literal = "[EOF]"
@ -450,7 +458,7 @@ proc getTok(c: var SqlLexer, tok: var Token) =
'\128'..'\255':
getSymbol(c, tok)
of '+', '-', '*', '/', '<', '>', '=', '~', '!', '@', '#', '%',
'^', '&', '|', '`', '?':
'^', '&', '|', '?':
getOperator(c, tok)
else:
add(tok.literal, c.buf[c.bufpos])
@ -462,27 +470,27 @@ proc errorStr(L: SqlLexer, msg: string): string =
# ----------------------------- parser ----------------------------------------
# Operator/Element Associativity Description
# . left table/column name separator
# :: left PostgreSQL-style typecast
# [ ] left array element selection
# - right unary minus
# ^ left exponentiation
# * / % left multiplication, division, modulo
# + - left addition, subtraction
# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL
# ISNULL test for null
# NOTNULL test for not null
# (any other) left all other native and user-defined oprs
# IN set membership
# BETWEEN range containment
# OVERLAPS time interval overlap
# LIKE ILIKE SIMILAR string pattern matching
# < > less than, greater than
# = right equality, assignment
# NOT right logical negation
# AND left logical conjunction
# OR left logical disjunction
# Operator/Element Associativity Description
# . left table/column name separator
# :: left PostgreSQL-style typecast
# [ ] left array element selection
# - right unary minus
# ^ left exponentiation
# * / % left multiplication, division, modulo
# + - left addition, subtraction
# IS IS TRUE, IS FALSE, IS UNKNOWN, IS NULL
# ISNULL test for null
# NOTNULL test for not null
# (any other) left all other native and user-defined oprs
# IN set membership
# BETWEEN range containment
# OVERLAPS time interval overlap
# LIKE ILIKE SIMILAR string pattern matching
# < > less than, greater than
# = right equality, assignment
# NOT right logical negation
# AND left logical conjunction
# OR left logical disjunction
type
SqlNodeKind* = enum ## kind of SQL abstract syntax tree
@ -504,6 +512,7 @@ type
nkPrefix,
nkInfix,
nkCall,
nkPrGroup,
nkColumnReference,
nkReferences,
nkDefault,
@ -518,11 +527,15 @@ type
nkSelect,
nkSelectDistinct,
nkSelectColumns,
nkSelectPair,
nkAsgn,
nkFrom,
nkFromItemPair,
nkGroup,
nkLimit,
nkHaving,
nkOrder,
nkJoin,
nkDesc,
nkUnion,
nkIntersect,
@ -658,10 +671,12 @@ proc getPrecedence(p: SqlParser): int =
elif isOpr(p, "=") or isOpr(p, "<") or isOpr(p, ">") or isOpr(p, ">=") or
isOpr(p, "<=") or isOpr(p, "<>") or isOpr(p, "!=") or isKeyw(p, "is") or
isKeyw(p, "like"):
result = 3
result = 4
elif isKeyw(p, "and"):
result = 2
result = 3
elif isKeyw(p, "or"):
result = 2
elif isKeyw(p, "between"):
result = 1
elif p.tok.kind == tkOperator:
# user-defined operator:
@ -670,6 +685,7 @@ proc getPrecedence(p: SqlParser): int =
result = - 1
proc parseExpr(p: var SqlParser): SqlNode
proc parseSelect(p: var SqlParser): SqlNode
proc identOrLiteral(p: var SqlParser): SqlNode =
case p.tok.kind
@ -693,7 +709,8 @@ proc identOrLiteral(p: var SqlParser): SqlNode =
getTok(p)
of tkParLe:
getTok(p)
result = parseExpr(p)
result = newNode(nkPrGroup)
result.add(parseExpr(p))
eat(p, tkParRi)
else:
sqlError(p, "expression expected")
@ -745,7 +762,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int =
result = opPred
while opPred > limit:
node = newNode(nkInfix)
opNode = newNode(nkIdent, p.tok.literal)
opNode = newNode(nkIdent, p.tok.literal.toLower())
getTok(p)
result = lowestExprAux(p, v2, opPred)
node.add(opNode)
@ -921,6 +938,19 @@ proc parseWhere(p: var SqlParser): SqlNode =
result = newNode(nkWhere)
result.add(parseExpr(p))
proc parseFromItem(p: var SqlParser): SqlNode =
result = newNode(nkFromItemPair)
if p.tok.kind == tkParLe:
getTok(p)
var select = parseSelect(p)
result.add(select)
eat(p, tkParRi)
else:
result.add(parseExpr(p))
if isKeyw(p, "as"):
getTok(p)
result.add(parseExpr(p))
proc parseIndexDef(p: var SqlParser): SqlNode =
result = parseIfNotExists(p, nkCreateIndex)
if isKeyw(p, "primary"):
@ -997,6 +1027,8 @@ proc parseUpdate(p: var SqlParser): SqlNode =
proc parseDelete(p: var SqlParser): SqlNode =
getTok(p)
if isOpr(p, "*"):
getTok(p)
result = newNode(nkDelete)
eat(p, "from")
result.add(primary(p))
@ -1019,7 +1051,12 @@ proc parseSelect(p: var SqlParser): SqlNode =
a.add(newNode(nkIdent, "*"))
getTok(p)
else:
a.add(parseExpr(p))
var pair = newNode(nkSelectPair)
pair.add(parseExpr(p))
a.add(pair)
if isKeyw(p, "as"):
getTok(p)
pair.add(parseExpr(p))
if p.tok.kind != tkComma: break
getTok(p)
result.add(a)
@ -1027,7 +1064,7 @@ proc parseSelect(p: var SqlParser): SqlNode =
var f = newNode(nkFrom)
while true:
getTok(p)
f.add(parseExpr(p))
f.add(parseFromItem(p))
if p.tok.kind != tkComma: break
result.add(f)
if isKeyw(p, "where"):
@ -1041,6 +1078,11 @@ proc parseSelect(p: var SqlParser): SqlNode =
if p.tok.kind != tkComma: break
getTok(p)
result.add(g)
if isKeyw(p, "limit"):
getTok(p)
var l = newNode(nkLimit)
l.add(parseExpr(p))
result.add(l)
if isKeyw(p, "having"):
var h = newNode(nkHaving)
while true:
@ -1073,6 +1115,19 @@ proc parseSelect(p: var SqlParser): SqlNode =
if p.tok.kind != tkComma: break
getTok(p)
result.add(n)
if isKeyw(p, "join") or isKeyw(p, "inner") or isKeyw(p, "outer") or isKeyw(p, "cross"):
var join = newNode(nkJoin)
result.add(join)
if isKeyw(p, "join"):
join.add(newNode(nkIdent, ""))
getTok(p)
else:
join.add(newNode(nkIdent, p.tok.literal.toLower()))
getTok(p)
eat(p, "join")
join.add(parseFromItem(p))
eat(p, "on")
join.add(parseExpr(p))
proc parseStmt(p: var SqlParser; parent: SqlNode) =
if isKeyw(p, "create"):
@ -1104,7 +1159,7 @@ proc parseStmt(p: var SqlParser; parent: SqlNode) =
elif isKeyw(p, "begin"):
getTok(p)
else:
sqlError(p, "CREATE expected")
sqlError(p, "SELECT, CREATE, UPDATE or DELETE expected")
proc open(p: var SqlParser, input: Stream, filename: string) =
## opens the parser `p` and assigns the input stream `input` to it.
@ -1116,13 +1171,13 @@ proc open(p: var SqlParser, input: Stream, filename: string) =
proc parse(p: var SqlParser): SqlNode =
## parses the content of `p`'s input stream and returns the SQL AST.
## Syntax errors raise an `EInvalidSql` exception.
## Syntax errors raise an `SqlParseError` exception.
result = newNode(nkStmtList)
while p.tok.kind != tkEof:
parseStmt(p, result)
if p.tok.kind == tkEof:
break
eat(p, tkSemicolon)
if result.len == 1:
result = result.sons[0]
proc close(p: var SqlParser) =
## closes the parser `p`. The associated input stream is closed too.
@ -1131,7 +1186,7 @@ proc close(p: var SqlParser) =
proc parseSQL*(input: Stream, filename: string): SqlNode =
## parses the SQL from `input` into an AST and returns the AST.
## `filename` is only used for error messages.
## Syntax errors raise an `EInvalidSql` exception.
## Syntax errors raise an `SqlParseError` exception.
var p: SqlParser
open(p, input, filename)
try:
@ -1139,24 +1194,69 @@ proc parseSQL*(input: Stream, filename: string): SqlNode =
finally:
close(p)
proc ra(n: SqlNode, s: var string, indent: int)
proc parseSQL*(input: string, filename=""): SqlNode =
## parses the SQL from `input` into an AST and returns the AST.
## `filename` is only used for error messages.
## Syntax errors raise an `SqlParseError` exception.
parseSQL(newStringStream(input), "")
proc rs(n: SqlNode, s: var string, indent: int,
prefix = "(", suffix = ")",
sep = ", ") =
type
SqlWriter = object
indent: int
upperCase: bool
buffer: string
proc add(s: var SqlWriter, thing: char) =
s.buffer.add(thing)
proc add(s: var SqlWriter, thing: string) =
if s.buffer.len > 0 and s.buffer[^1] notin {' ', '\L', '(', '.'}:
s.buffer.add(" ")
s.buffer.add(thing)
proc addKeyw(s: var SqlWriter, thing: string) =
var keyw = thing
if s.upperCase:
keyw = keyw.toUpper()
s.add(keyw)
proc addIden(s: var SqlWriter, thing: string) =
var iden = thing
if iden.toLower() in reservedKeywords:
iden = '"' & iden & '"'
s.add(iden)
proc ra(n: SqlNode, s: var SqlWriter)
proc rs(n: SqlNode, s: var SqlWriter, prefix = "(", suffix = ")", sep = ", ") =
if n.len > 0:
s.add(prefix)
for i in 0 .. n.len-1:
if i > 0: s.add(sep)
ra(n.sons[i], s, indent)
ra(n.sons[i], s)
s.add(suffix)
proc ra(n: SqlNode, s: var string, indent: int) =
proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',') =
if n.len > 0:
for i in 0 .. n.len-1:
if i > 0: s.add(sep)
ra(n.sons[i], s)
proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) =
if n.len > 0:
s.add(prefix)
for i in 0 .. n.len-1:
if i > 0: s.add(sep)
ra(n.sons[i], s)
s.add(suffix)
proc ra(n: SqlNode, s: var SqlWriter) =
if n == nil: return
case n.kind
of nkNone: discard
of nkIdent:
if allCharsInSet(n.strVal, {'\33'..'\127'}):
if allCharsInSet(n.strVal, {'\33'..'\127'}) and n.strVal.toLower() notin reservedKeywords:
s.add(n.strVal)
else:
s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"")
@ -1169,217 +1269,206 @@ proc ra(n: SqlNode, s: var string, indent: int) =
of nkIntegerLit, nkNumericLit:
s.add(n.strVal)
of nkPrimaryKey:
s.add(" primary key")
rs(n, s, indent)
s.addKeyw("primary key")
rs(n, s)
of nkForeignKey:
s.add(" foreign key")
rs(n, s, indent)
s.addKeyw("foreign key")
rs(n, s)
of nkNotNull:
s.add(" not null")
s.addKeyw("not null")
of nkNull:
s.add(" null")
s.addKeyw("null")
of nkDot:
ra(n.sons[0], s, indent)
s.add(".")
ra(n.sons[1], s, indent)
ra(n.sons[0], s)
s.add('.')
ra(n.sons[1], s)
of nkDotDot:
ra(n.sons[0], s, indent)
ra(n.sons[0], s)
s.add(". .")
ra(n.sons[1], s, indent)
ra(n.sons[1], s)
of nkPrefix:
s.add('(')
ra(n.sons[0], s, indent)
ra(n.sons[0], s)
s.add(' ')
ra(n.sons[1], s, indent)
s.add(')')
ra(n.sons[1], s)
of nkInfix:
s.add('(')
ra(n.sons[1], s, indent)
ra(n.sons[1], s)
s.add(' ')
ra(n.sons[0], s, indent)
ra(n.sons[0], s)
s.add(' ')
ra(n.sons[2], s, indent)
s.add(')')
ra(n.sons[2], s)
of nkCall, nkColumnReference:
ra(n.sons[0], s, indent)
ra(n.sons[0], s)
s.add('(')
for i in 1..n.len-1:
if i > 1: s.add(", ")
ra(n.sons[i], s, indent)
if i > 1: s.add(',')
ra(n.sons[i], s)
s.add(')')
of nkPrGroup:
s.add('(')
s.addMulti(n)
s.add(')')
of nkReferences:
s.add(" references ")
ra(n.sons[0], s, indent)
s.addKeyw("references")
ra(n.sons[0], s)
of nkDefault:
s.add(" default ")
ra(n.sons[0], s, indent)
s.addKeyw("default")
ra(n.sons[0], s)
of nkCheck:
s.add(" check ")
ra(n.sons[0], s, indent)
s.addKeyw("check")
ra(n.sons[0], s)
of nkConstraint:
s.add(" constraint ")
ra(n.sons[0], s, indent)
s.add(" check ")
ra(n.sons[1], s, indent)
s.addKeyw("constraint")
ra(n.sons[0], s)
s.addKeyw("check")
ra(n.sons[1], s)
of nkUnique:
s.add(" unique")
rs(n, s, indent)
s.addKeyw("unique")
rs(n, s)
of nkIdentity:
s.add(" identity")
s.addKeyw("identity")
of nkColumnDef:
s.add("\n ")
rs(n, s, indent, "", "", " ")
rs(n, s, "", "", " ")
of nkStmtList:
for i in 0..n.len-1:
ra(n.sons[i], s, indent)
s.add("\n")
ra(n.sons[i], s)
s.add(';')
of nkInsert:
assert n.len == 3
s.add("insert into ")
ra(n.sons[0], s, indent)
ra(n.sons[1], s, indent)
s.addKeyw("insert into")
ra(n.sons[0], s)
s.add(' ')
ra(n.sons[1], s)
if n.sons[2].kind == nkDefault:
s.add("default values")
s.addKeyw("default values")
else:
s.add("\n")
ra(n.sons[2], s, indent)
s.add(';')
ra(n.sons[2], s)
of nkUpdate:
s.add("update ")
ra(n.sons[0], s, indent)
s.add(" set ")
s.addKeyw("update")
ra(n.sons[0], s)
s.addKeyw("set")
var L = n.len
for i in 1 .. L-2:
if i > 1: s.add(", ")
var it = n.sons[i]
assert it.kind == nkAsgn
ra(it, s, indent)
ra(n.sons[L-1], s, indent)
s.add(';')
ra(it, s)
ra(n.sons[L-1], s)
of nkDelete:
s.add("delete from ")
ra(n.sons[0], s, indent)
ra(n.sons[1], s, indent)
s.add(';')
s.addKeyw("delete from")
ra(n.sons[0], s)
ra(n.sons[1], s)
of nkSelect, nkSelectDistinct:
s.add("select ")
s.addKeyw("select")
if n.kind == nkSelectDistinct:
s.add("distinct ")
rs(n.sons[0], s, indent, "", "", ", ")
for i in 1 .. n.len-1: ra(n.sons[i], s, indent)
s.add(';')
s.addKeyw("distinct")
s.addMulti(n.sons[0])
for i in 1 .. n.len-1:
ra(n.sons[i], s)
of nkSelectColumns:
assert(false)
of nkSelectPair:
ra(n.sons[0], s)
if n.sons.len == 2:
s.addKeyw("as")
ra(n.sons[1], s)
of nkFromItemPair:
if n.sons[0].kind == nkIdent:
ra(n.sons[0], s)
else:
assert n.sons[0].kind == nkSelect
s.add('(')
ra(n.sons[0], s)
s.add(')')
if n.sons.len == 2:
s.addKeyw("as")
ra(n.sons[1], s)
of nkAsgn:
ra(n.sons[0], s, indent)
ra(n.sons[0], s)
s.add(" = ")
ra(n.sons[1], s, indent)
ra(n.sons[1], s)
of nkFrom:
s.add("\nfrom ")
rs(n, s, indent, "", "", ", ")
s.addKeyw("from")
s.addMulti(n)
of nkGroup:
s.add("\ngroup by")
rs(n, s, indent, "", "", ", ")
s.addKeyw("group by")
s.addMulti(n)
of nkLimit:
s.addKeyw("limit")
s.addMulti(n)
of nkHaving:
s.add("\nhaving")
rs(n, s, indent, "", "", ", ")
s.addKeyw("having")
s.addMulti(n)
of nkOrder:
s.add("\norder by ")
rs(n, s, indent, "", "", ", ")
s.addKeyw("order by")
s.addMulti(n)
of nkJoin:
var joinType = n.sons[0].strVal
if joinType == "":
joinType = "join"
else:
joinType &= " " & "join"
s.addKeyw(joinType)
ra(n.sons[1], s)
s.addKeyw("on")
ra(n.sons[2], s)
of nkDesc:
ra(n.sons[0], s, indent)
s.add(" desc")
ra(n.sons[0], s)
s.addKeyw("desc")
of nkUnion:
s.add(" union")
s.addKeyw("union")
of nkIntersect:
s.add(" intersect")
s.addKeyw("intersect")
of nkExcept:
s.add(" except")
s.addKeyw("except")
of nkColumnList:
rs(n, s, indent)
rs(n, s)
of nkValueList:
s.add("values ")
rs(n, s, indent)
s.addKeyw("values")
rs(n, s)
of nkWhere:
s.add("\nwhere ")
ra(n.sons[0], s, indent)
s.addKeyw("where")
ra(n.sons[0], s)
of nkCreateTable, nkCreateTableIfNotExists:
s.add("create table ")
s.addKeyw("create table")
if n.kind == nkCreateTableIfNotExists:
s.add("if not exists ")
ra(n.sons[0], s, indent)
s.addKeyw("if not exists")
ra(n.sons[0], s)
s.add('(')
for i in 1..n.len-1:
if i > 1: s.add(", ")
ra(n.sons[i], s, indent)
if i > 1: s.add(',')
ra(n.sons[i], s)
s.add(");")
of nkCreateType, nkCreateTypeIfNotExists:
s.add("create type ")
s.addKeyw("create type")
if n.kind == nkCreateTypeIfNotExists:
s.add("if not exists ")
ra(n.sons[0], s, indent)
s.add(" as ")
ra(n.sons[1], s, indent)
s.add(';')
s.addKeyw("if not exists")
ra(n.sons[0], s)
s.addKeyw("as")
ra(n.sons[1], s)
of nkCreateIndex, nkCreateIndexIfNotExists:
s.add("create index ")
s.addKeyw("create index")
if n.kind == nkCreateIndexIfNotExists:
s.add("if not exists ")
ra(n.sons[0], s, indent)
s.add(" on ")
ra(n.sons[1], s, indent)
s.addKeyw("if not exists")
ra(n.sons[0], s)
s.addKeyw("on")
ra(n.sons[1], s)
s.add('(')
for i in 2..n.len-1:
if i > 2: s.add(", ")
ra(n.sons[i], s, indent)
ra(n.sons[i], s)
s.add(");")
of nkEnumDef:
s.add("enum ")
rs(n, s, indent)
s.addKeyw("enum")
rs(n, s)
# What I want:
#
#select(columns = [T1.all, T2.name],
# fromm = [T1, T2],
# where = T1.name ==. T2.name,
# orderby = [name]):
#
#for row in dbQuery(db, """select x, y, z
# from a, b
# where a.name = b.name"""):
#
#select x, y, z:
# fromm: Table1, Table2
# where: x.name == y.name
#db.select(fromm = [t1, t2], where = t1.name == t2.name):
#for x, y, z in db.select(fromm = a, b where = a.name == b.name):
# writeLine x, y, z
proc renderSQL*(n: SqlNode): string =
proc renderSQL*(n: SqlNode, upperCase=false): string =
## Converts an SQL abstract syntax tree to its string representation.
result = ""
ra(n, result, 0)
var s: SqlWriter
s.buffer = ""
s.upperCase = upperCase
ra(n, s)
return s.buffer
proc `$`*(n: SqlNode): string =
## an alias for `renderSQL`.
renderSQL(n)
when not defined(testing) and isMainModule:
echo(renderSQL(parseSQL(newStringStream("""
CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic');
CREATE TABLE holidays (
num_weeks int,
happiness happiness
);
CREATE INDEX table1_attr1 ON table1(attr1);
SELECT * FROM myTab WHERE col1 = 'happy';
"""), "stdin")))
# CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic');
# CREATE TABLE holidays (
# num_weeks int,
# happiness happiness
# );
# CREATE INDEX table1_attr1 ON table1(attr1)

17
lib/pure/parseutils.nim
View file

@ -87,6 +87,23 @@ proc parseOct*(s: string, number: var int, start = 0): int {.
inc(i)
if foundDigit: result = i-start
proc parseBin*(s: string, number: var int, start = 0): int {.
rtl, extern: "npuParseBin", noSideEffect.} =
## parses an binary number and stores its value in ``number``. Returns
## the number of the parsed characters or 0 in case of an error.
var i = start
var foundDigit = false
if s[i] == '0' and (s[i+1] == 'b' or s[i+1] == 'B'): inc(i, 2)
while true:
case s[i]
of '_': discard
of '0'..'1':
number = number shl 1 or (ord(s[i]) - ord('0'))
foundDigit = true
else: break
inc(i)
if foundDigit: result = i-start
proc parseIdent*(s: string, ident: var string, start = 0): int =
## parses an identifier and stores it in ``ident``. Returns
## the number of the parsed characters or 0 in case of an error.

135
lib/pure/random.nim
View file

@ -7,16 +7,16 @@
# distribution, for details about the copyright.
#
## Nim's standard random number generator. Based on the ``xoroshiro128+`` (xor/rotate/shift/rotate) library.
## Nim's standard random number generator. Based on
## the ``xoroshiro128+`` (xor/rotate/shift/rotate) library.
## * More information: http://xoroshiro.di.unimi.it/
## * C implementation: http://xoroshiro.di.unimi.it/xoroshiro128plus.c
##
## Do not use this module for cryptographic use!
## **Do not use this module for cryptographic purposes!**
include "system/inclrtl"
{.push debugger:off.}
# XXX Expose RandomGenState
when defined(JS):
type ui = uint32
@ -27,31 +27,34 @@ else:
const randMax = 18_446_744_073_709_551_615u64
type
RandomGenState = object
Rand* = object ## State of the random number generator.
## The procs that use the default state
## are **not** thread-safe!
a0, a1: ui
when defined(JS):
var state = RandomGenState(
var state = Rand(
a0: 0x69B4C98Cu32,
a1: 0xFED1DD30u32) # global for backwards compatibility
else:
# racy for multi-threading but good enough for now:
var state = RandomGenState(
var state = Rand(
a0: 0x69B4C98CB8530805u64,
a1: 0xFED1DD3004688D67CAu64) # global for backwards compatibility
proc rotl(x, k: ui): ui =
result = (x shl k) or (x shr (ui(64) - k))
proc next(s: var RandomGenState): uint64 =
let s0 = s.a0
var s1 = s.a1
proc next*(r: var Rand): uint64 =
## Uses the state to compute a new ``uint64`` random number.
let s0 = r.a0
var s1 = r.a1
result = s0 + s1
s1 = s1 xor s0
s.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b
s.a1 = rotl(s1, 36) # c
r.a0 = rotl(s0, 55) xor s1 xor (s1 shl 14) # a, b
r.a1 = rotl(s1, 36) # c
proc skipRandomNumbers(s: var RandomGenState) =
proc skipRandomNumbers*(s: var Rand) =
## This is the jump function for the generator. It is equivalent
## to 2^64 calls to next(); it can be used to generate 2^64
## non-overlapping subsequences for parallel computations.
@ -71,21 +74,23 @@ proc skipRandomNumbers(s: var RandomGenState) =
s.a0 = s0
s.a1 = s1
proc random*(max: int): int {.benign.} =
proc random*(max: int): int {.benign, deprecated.} =
## Returns a random number in the range 0..max-1. The sequence of
## random number is always the same, unless `randomize` is called
## which initializes the random number generator with a "random"
## number, i.e. a tickcount.
## number, i.e. a tickcount. **Deprecated since version 0.18.0**.
## Use ``rand`` instead.
while true:
let x = next(state)
if x < randMax - (randMax mod ui(max)):
return int(x mod uint64(max))
proc random*(max: float): float {.benign.} =
proc random*(max: float): float {.benign, deprecated.} =
## Returns a random number in the range 0..<max. The sequence of
## random number is always the same, unless `randomize` is called
## which initializes the random number generator with a "random"
## number, i.e. a tickcount.
## number, i.e. a tickcount. **Deprecated since version 0.18.0**.
## Use ``rand`` instead.
let x = next(state)
when defined(JS):
result = (float(x) / float(high(uint32))) * max
@ -93,25 +98,91 @@ proc random*(max: float): float {.benign.} =
let u = (0x3FFu64 shl 52u64) or (x shr 12u64)
result = (cast[float](u) - 1.0) * max
proc random*[T](x: HSlice[T, T]): T =
proc random*[T](x: HSlice[T, T]): T {.deprecated.} =
## For a slice `a .. b` returns a value in the range `a .. b-1`.
## **Deprecated since version 0.18.0**.
## Use ``rand`` instead.
result = T(random(x.b - x.a)) + x.a
proc random*[T](a: openArray[T]): T =
proc random*[T](a: openArray[T]): T {.deprecated.} =
## returns a random element from the openarray `a`.
## **Deprecated since version 0.18.0**.
## Use ``rand`` instead.
result = a[random(a.low..a.len)]
proc rand*(r: var Rand; max: int): int {.benign.} =
## Returns a random number in the range 0..max. The sequence of
## random number is always the same, unless `randomize` is called
## which initializes the random number generator with a "random"
## number, i.e. a tickcount.
while true:
let x = next(r)
if x <= randMax - (randMax mod ui(max)):
return int(x mod (uint64(max)+1u64))
proc rand*(max: int): int {.benign.} =
## Returns a random number in the range 0..max. The sequence of
## random number is always the same, unless `randomize` is called
## which initializes the random number generator with a "random"
## number, i.e. a tickcount.
rand(state, max)
proc rand*(r: var Rand; max: float): float {.benign.} =
## Returns a random number in the range 0..max. The sequence of
## random number is always the same, unless `randomize` is called
## which initializes the random number generator with a "random"
## number, i.e. a tickcount.
let x = next(r)
when defined(JS):
result = (float(x) / float(high(uint32))) * max
else:
let u = (0x3FFu64 shl 52u64) or (x shr 12u64)
result = (cast[float](u) - 1.0) * max
proc rand*(max: float): float {.benign.} =
## Returns a random number in the range 0..max. The sequence of
## random number is always the same, unless `randomize` is called
## which initializes the random number generator with a "random"
## number, i.e. a tickcount.
rand(state, max)
proc rand*[T](r: var Rand; x: HSlice[T, T]): T =
## For a slice `a .. b` returns a value in the range `a .. b`.
result = T(rand(r, x.b - x.a)) + x.a
proc rand*[T](x: HSlice[T, T]): T =
## For a slice `a .. b` returns a value in the range `a .. b`.
result = rand(state, x)
proc rand*[T](r: var Rand; a: openArray[T]): T =
## returns a random element from the openarray `a`.
result = a[rand(r, a.low..a.high)]
proc rand*[T](a: openArray[T]): T =
## returns a random element from the openarray `a`.
result = a[rand(a.low..a.high)]
proc initRand*(seed: int64): Rand =
## Creates a new ``Rand`` state from ``seed``.
result.a0 = ui(seed shr 16)
result.a1 = ui(seed and 0xffff)
discard next(result)
proc randomize*(seed: int64) {.benign.} =
## Initializes the random number generator with a specific seed.
state.a0 = ui(seed shr 16)
state.a1 = ui(seed and 0xffff)
discard next(state)
## Initializes the default random number generator
## with a specific seed.
state = initRand(seed)
proc shuffle*[T](r: var Rand; x: var openArray[T]) =
## Swaps the positions of elements in a sequence randomly.
for i in countdown(x.high, 1):
let j = r.rand(i)
swap(x[i], x[j])
proc shuffle*[T](x: var openArray[T]) =
## Will randomly swap the positions of elements in a sequence.
for i in countdown(x.high, 1):
let j = random(i + 1)
swap(x[i], x[j])
## Swaps the positions of elements in a sequence randomly.
shuffle(state, x)
when not defined(nimscript):
import times
@ -119,12 +190,8 @@ when not defined(nimscript):
proc randomize*() {.benign.} =
## Initializes the random number generator with a "random"
## number, i.e. a tickcount. Note: Does not work for NimScript.
when defined(JS):
proc getMil(t: Time): int {.importcpp: "getTime", nodecl.}
randomize(getMil times.getTime())
else:
let time = int64(times.epochTime() * 1_000_000_000)
randomize(time)
let time = int64(times.epochTime() * 1_000_000_000)
randomize(time)
{.pop.}
@ -134,12 +201,12 @@ when isMainModule:
var x = 8234
for i in 0..100_000:
x = random(len(occur)) # myrand(x)
x = rand(high(occur))
inc occur[x]
for i, oc in occur:
if oc < 69:
doAssert false, "too few occurrences of " & $i
elif oc > 130:
elif oc > 150:
doAssert false, "too many occurrences of " & $i
var a = [0, 1]

11
lib/pure/rationals.nim
View file

@ -39,7 +39,7 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] =
result.num = x
result.den = 1
proc toRational*(x: float, n: int = high(int32)): Rational[int] =
proc toRational*(x: float, n: int = high(int) shr (sizeof(int) div 2 * 8)): Rational[int] =
## Calculates the best rational numerator and denominator
## that approximates to `x`, where the denominator is
## smaller than `n` (default is the largest possible
@ -323,8 +323,13 @@ when isMainModule:
assert abs(toFloat(y) - 0.4814814814814815) < 1.0e-7
assert toInt(z) == 0
assert toRational(0.98765432) == 2111111029 // 2137499919
assert toRational(PI) == 817696623 // 260280919
when sizeof(int) == 8:
assert toRational(0.98765432) == 2111111029 // 2137499919
assert toRational(PI) == 817696623 // 260280919
when sizeof(int) == 4:
assert toRational(0.98765432) == 80 // 81
assert toRational(PI) == 355 // 113
assert toRational(0.1) == 1 // 10
assert toRational(0.9) == 9 // 10

638
lib/pure/strformat.nim Normal file
View file

@ -0,0 +1,638 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Nim contributors
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
##[
String `interpolation`:idx: / `format`:idx: inspired by
Python's ``f``-strings.
Examples:
.. code-block:: nim
doAssert &"""{"abc":>4}""" == " abc"
doAssert &"""{"abc":<4}""" == "abc "
doAssert &"{-12345:08}" == "-0012345"
doAssert &"{-1:3}" == " -1"
doAssert &"{-1:03}" == "-01"
doAssert &"{16:#X}" == "0x10"
doAssert &"{123.456}" == "123.456"
doAssert &"{123.456:>9.3f}" == " 123.456"
doAssert &"{123.456:9.3f}" == " 123.456"
doAssert &"{123.456:9.4f}" == " 123.4560"
doAssert &"{123.456:>9.0f}" == " 123."
doAssert &"{123.456:<9.4f}" == "123.4560 "
doAssert &"{123.456:e}" == "1.234560e+02"
doAssert &"{123.456:>13e}" == " 1.234560e+02"
doAssert &"{123.456:13e}" == " 1.234560e+02"
An expression like ``&"{key} is {value:arg} {{z}}"`` is transformed into:
.. code-block:: nim
var temp = newStringOfCap(educatedCapGuess)
format(key, temp)
format(" is ", temp)
format(value, arg, temp)
format(" {z}", temp)
temp
Parts of the string that are enclosed in the curly braces are interpreted
as Nim code, to escape an ``{`` or ``}`` double it.
``&`` delegates most of the work to an open overloaded set
of ``format`` procs. The required signature for a type ``T`` that supports
formatting is usually ``proc format(x: T; result: var string)`` for efficiency
but can also be ``proc format(x: T): string``. ``add`` and ``$`` procs are
used as the fallback implementation.
This is the concrete lookup algorithm that ``&`` uses:
.. code-block:: nim
when compiles(format(arg, res)):
format(arg, res)
elif compiles(format(arg)):
res.add format(arg)
elif compiles(add(res, arg)):
res.add(arg)
else:
res.add($arg)
The subexpression after the colon
(``arg`` in ``&"{key} is {value:arg} {{z}}"``) is an optional argument
passed to ``format``.
If an optional argument is present the following lookup algorithm is used:
.. code-block:: nim
when compiles(format(arg, option, res)):
format(arg, option, res)
else:
res.add format(arg, option)
For strings and numeric types the optional argument is a so-called
"standard format specifier".
Standard format specifier
=========================
The general form of a standard format specifier is::
[[fill]align][sign][#][0][minimumwidth][.precision][type]
The square brackets ``[]`` indicate an optional element.
The optional align flag can be one of the following:
'<'
Forces the field to be left-aligned within the available
space. (This is the default for strings.)
'>'
Forces the field to be right-aligned within the available space.
(This is the default for numbers.)
'^'
Forces the field to be centered within the available space.
Note that unless a minimum field width is defined, the field width
will always be the same size as the data to fill it, so that the alignment
option has no meaning in this case.
The optional 'fill' character defines the character to be used to pad
the field to the minimum width. The fill character, if present, must be
followed by an alignment flag.
The 'sign' option is only valid for numeric types, and can be one of the following:
================= ====================================================
Sign Meaning
================= ====================================================
``+`` Indicates that a sign should be used for both
positive as well as negative numbers.
``-`` Indicates that a sign should be used only for
negative numbers (this is the default behavior).
(space) Indicates that a leading space should be used on
positive numbers.
================= ====================================================
If the '#' character is present, integers use the 'alternate form' for formatting.
This means that binary, octal, and hexadecimal output will be prefixed
with '0b', '0o', and '0x', respectively.
'width' is a decimal integer defining the minimum field width. If not specified,
then the field width will be determined by the content.
If the width field is preceded by a zero ('0') character, this enables
zero-padding.
The 'precision' is a decimal number indicating how many digits should be displayed
after the decimal point in a floating point conversion. For non-numeric types the
field indicates the maximum field size - in other words, how many characters will
be used from the field content. The precision is ignored for integer conversions.
Finally, the 'type' determines how the data should be presented.
The available integer presentation types are:
================= ====================================================
Type Result
================= ====================================================
``b`` Binary. Outputs the number in base 2.
``d`` Decimal Integer. Outputs the number in base 10.
``o`` Octal format. Outputs the number in base 8.
``x`` Hex format. Outputs the number in base 16, using
lower-case letters for the digits above 9.
``X`` Hex format. Outputs the number in base 16, using
uppercase letters for the digits above 9.
(None) the same as 'd'
================= ====================================================
The available floating point presentation types are:
================= ====================================================
Type Result
================= ====================================================
``e`` Exponent notation. Prints the number in scientific
notation using the letter 'e' to indicate the
exponent.
``E`` Exponent notation. Same as 'e' except it converts
the number to uppercase.
``f`` Fixed point. Displays the number as a fixed-point
number.
``F`` Fixed point. Same as 'f' except it converts the
number to uppercase.
``g`` General format. This prints the number as a
fixed-point number, unless the number is too
large, in which case it switches to 'e'
exponent notation.
``G`` General format. Same as 'g' except switches to 'E'
if the number gets to large.
(None) similar to 'g', except that it prints at least one
digit after the decimal point.
================= ====================================================
Future directions
=================
A curly expression with commas in it like ``{x, argA, argB}`` could be
transformed to ``format(x, argA, argB, res)`` in order to support
formatters that do not need to parse a custom language within a custom
language but instead prefer to use Nim's existing syntax. This also
helps in readability since there is only so much you can cram into
single letter DSLs.
]##
import macros, parseutils, unicode
import strutils
template callFormat(res, arg) {.dirty.} =
when arg is string:
# workaround in order to circumvent 'strutils.format' which matches
# too but doesn't adhere to our protocol.
res.add arg
elif compiles(format(arg, res)):
format(arg, res)
elif compiles(format(arg)):
res.add format(arg)
elif compiles(add(res, arg)):
res.add(arg)
else:
res.add($arg)
template callFormatOption(res, arg, option) {.dirty.} =
when compiles(format(arg, option, res)):
format(arg, option, res)
elif compiles(format(arg, option)):
res.add format(arg, option)
else:
format($arg, option, res)
macro `&`*(pattern: string): untyped =
## For a specification of the ``&`` macro, see the module level documentation.
runnableExamples:
template check(actual, expected: string) =
doAssert actual == expected
from strutils import toUpperAscii, repeat
# Basic tests
let s = "string"
check &"{0} {s}", "0 string"
check &"{s[0..2].toUpperAscii}", "STR"
check &"{-10:04}", "-010"
check &"{-10:<04}", "-010"
check &"{-10:>04}", "-010"
check &"0x{10:02X}", "0x0A"
check &"{10:#04X}", "0x0A"
check &"""{"test":#>5}""", "#test"
check &"""{"test":>5}""", " test"
check &"""{"test":#^7}""", "#test##"
check &"""{"test": <5}""", "test "
check &"""{"test":<5}""", "test "
check &"{1f:.3f}", "1.000"
check &"Hello, {s}!", "Hello, string!"
# Tests for identifers without parenthesis
check &"{s} works{s}", "string worksstring"
check &"{s:>7}", " string"
doAssert(not compiles(&"{s_works}")) # parsed as identifier `s_works`
# Misc general tests
check &"{{}}", "{}"
check &"{0}%", "0%"
check &"{0}%asdf", "0%asdf"
check &("\n{\"\\n\"}\n"), "\n\n\n"
check &"""{"abc"}s""", "abcs"
# String tests
check &"""{"abc"}""", "abc"
check &"""{"abc":>4}""", " abc"
check &"""{"abc":<4}""", "abc "
check &"""{"":>4}""", " "
check &"""{"":<4}""", " "
# Int tests
check &"{12345}", "12345"
check &"{ - 12345}", "-12345"
check &"{12345:6}", " 12345"
check &"{12345:>6}", " 12345"
check &"{12345:4}", "12345"
check &"{12345:08}", "00012345"
check &"{-12345:08}", "-0012345"
check &"{0:0}", "0"
check &"{0:02}", "00"
check &"{-1:3}", " -1"
check &"{-1:03}", "-01"
check &"{10}", "10"
check &"{16:#X}", "0x10"
check &"{16:^#7X}", " 0x10 "
check &"{16:^+#7X}", " +0x10 "
# Hex tests
check &"{0:x}", "0"
check &"{-0:x}", "0"
check &"{255:x}", "ff"
check &"{255:X}", "FF"
check &"{-255:x}", "-ff"
check &"{-255:X}", "-FF"
check &"{255:x} uNaffeCteD CaSe", "ff uNaffeCteD CaSe"
check &"{255:X} uNaffeCteD CaSe", "FF uNaffeCteD CaSe"
check &"{255:4x}", " ff"
check &"{255:04x}", "00ff"
check &"{-255:4x}", " -ff"
check &"{-255:04x}", "-0ff"
# Float tests
check &"{123.456}", "123.456"
check &"{-123.456}", "-123.456"
check &"{123.456:.3f}", "123.456"
check &"{123.456:+.3f}", "+123.456"
check &"{-123.456:+.3f}", "-123.456"
check &"{-123.456:.3f}", "-123.456"
check &"{123.456:1g}", "123.456"
check &"{123.456:.1f}", "123.5"
check &"{123.456:.0f}", "123."
#check &"{123.456:.0f}", "123."
check &"{123.456:>9.3f}", " 123.456"
check &"{123.456:9.3f}", " 123.456"
check &"{123.456:>9.4f}", " 123.4560"
check &"{123.456:>9.0f}", " 123."
check &"{123.456:<9.4f}", "123.4560 "
# Float (scientific) tests
check &"{123.456:e}", "1.234560e+02"
check &"{123.456:>13e}", " 1.234560e+02"
check &"{123.456:<13e}", "1.234560e+02 "
check &"{123.456:.1e}", "1.2e+02"
check &"{123.456:.2e}", "1.23e+02"
check &"{123.456:.3e}", "1.235e+02"
# Note: times.format adheres to the format protocol. Test that this
# works:
import times
var nullTime: DateTime
check &"{nullTime:yyyy-mm-dd}", "0000-00-00"
# Unicode string tests
check &"""{"αβγ"}""", "αβγ"
check &"""{"αβγ":>5}""", " αβγ"
check &"""{"αβγ":<5}""", "αβγ "
check &"""a{"a"}α{"α"}€{"€"}𐍈{"𐍈"}""", "aaαα€€𐍈𐍈"
check &"""a{"a":2}α{"α":2}€{"€":2}𐍈{"𐍈":2}""", "aa αα €€ 𐍈𐍈 "
# Invalid unicode sequences should be handled as plain strings.
# Invalid examples taken from: https://stackoverflow.com/a/3886015/1804173
let invalidUtf8 = [
"\xc3\x28", "\xa0\xa1",
"\xe2\x28\xa1", "\xe2\x82\x28",
"\xf0\x28\x8c\xbc", "\xf0\x90\x28\xbc", "\xf0\x28\x8c\x28"
]
for s in invalidUtf8:
check &"{s:>5}", repeat(" ", 5-s.len) & s
if pattern.kind notin {nnkStrLit..nnkTripleStrLit}:
error "& only works with string literals", pattern
let f = pattern.strVal
var i = 0
let res = genSym(nskVar, "fmtRes")
result = newNimNode(nnkStmtListExpr, lineInfoFrom=pattern)
result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + count(f, '{')*10)))
var strlit = ""
while i < f.len:
if f[i] == '{':
inc i
if f[i] == '{':
inc i
strlit.add '{'
else:
if strlit.len > 0:
result.add newCall(bindSym"add", res, newLit(strlit))
strlit = ""
var subexpr = ""
while i < f.len and f[i] != '}' and f[i] != ':':
subexpr.add f[i]
inc i
let x = parseExpr(subexpr)
if f[i] == ':':
inc i
var options = ""
while i < f.len and f[i] != '}':
options.add f[i]
inc i
result.add getAst(callFormatOption(res, x, newLit(options)))
else:
result.add getAst(callFormat(res, x))
if f[i] == '}':
inc i
else:
doAssert false, "invalid format string: missing '}'"
elif f[i] == '}':
if f[i+1] == '}':
strlit.add '}'
inc i, 2
else:
doAssert false, "invalid format string: '}' instead of '}}'"
inc i
else:
strlit.add f[i]
inc i
if strlit.len > 0:
result.add newCall(bindSym"add", res, newLit(strlit))
result.add res
when defined(debugFmtDsl):
echo repr result
template fmt*(pattern: string): untyped =
## An alias for ``&``.
## **Examples:**
##
## .. code-block:: nim
## import json
## import strformat except `&`
##
## let example = "oh, look no conflicts anymore"
## echo fmt"{example}"
bind `&`
&pattern
proc mkDigit(v: int, typ: char): string {.inline.} =
assert(v < 26)
if v < 10:
result = $chr(ord('0') + v)
else:
result = $chr(ord(if typ == 'x': 'a' else: 'A') + v - 10)
proc alignString*(s: string, minimumWidth: int; align = '\0'; fill = ' '): string =
## Aligns ``s`` using ``fill`` char.
## This is only of interest if you want to write a custom ``format`` proc that
## should support the standard format specifiers.
if minimumWidth == 0:
result = s
else:
let sRuneLen = if s.validateUtf8 == -1: s.runeLen else: s.len
let toFill = minimumWidth - sRuneLen
if toFill <= 0:
result = s
elif align == '<' or align == '\0':
result = s & repeat(fill, toFill)
elif align == '^':
let half = toFill div 2
result = repeat(fill, half) & s & repeat(fill, toFill - half)
else:
result = repeat(fill, toFill) & s
type
StandardFormatSpecifier* = object ## Type that describes "standard format specifiers".
fill*, align*: char ## Desired fill and alignment.
sign*: char ## Desired sign.
alternateForm*: bool ## Whether to prefix binary, octal and hex numbers
## with ``0b``, ``0o``, ``0x``.
padWithZero*: bool ## Whether to pad with zeros rather than spaces.
minimumWidth*, precision*: int ## Desired minium width and precision.
typ*: char ## Type like 'f', 'g' or 'd'.
endPosition*: int ## End position in the format specifier after
## ``parseStandardFormatSpecifier`` returned.
proc formatInt(n: SomeNumber; radix: int; spec: StandardFormatSpecifier): string =
## Converts ``n`` to string. If ``n`` is `SomeReal`, it casts to `int64`.
## Conversion is done using ``radix``. If result's length is lesser than
## ``minimumWidth``, it aligns result to the right or left (depending on ``a``)
## with ``fill`` char.
when n is SomeUnsignedInt:
var v = n.uint64
let negative = false
else:
var v = n.int64
let negative = v.int64 < 0
if negative:
# FIXME: overflow error for low(int64)
v = v * -1
var xx = ""
if spec.alternateForm:
case spec.typ
of 'X': xx = "0x"
of 'x': xx = "0x"
of 'b': xx = "0b"
of 'o': xx = "0o"
else: discard
if v == 0:
result = "0"
else:
result = ""
while v > type(v)(0):
let d = v mod type(v)(radix)
v = v div type(v)(radix)
result.add(mkDigit(d.int, spec.typ))
for idx in 0..<(result.len div 2):
swap result[idx], result[result.len - idx - 1]
if spec.padWithZero:
let sign = negative or spec.sign != '-'
let toFill = spec.minimumWidth - result.len - xx.len - ord(sign)
if toFill > 0:
result = repeat('0', toFill) & result
if negative:
result = "-" & xx & result
elif spec.sign != '-':
result = spec.sign & xx & result
else:
result = xx & result
if spec.align == '<':
for i in result.len..<spec.minimumWidth:
result.add(spec.fill)
else:
let toFill = spec.minimumWidth - result.len
if spec.align == '^':
let half = toFill div 2
result = repeat(spec.fill, half) & result & repeat(spec.fill, toFill - half)
else:
if toFill > 0:
result = repeat(spec.fill, toFill) & result
proc parseStandardFormatSpecifier*(s: string; start = 0;
ignoreUnknownSuffix = false): StandardFormatSpecifier =
## An exported helper proc that parses the "standard format specifiers",
## as specified by the grammar::
##
## [[fill]align][sign][#][0][minimumwidth][.precision][type]
##
## This is only of interest if you want to write a custom ``format`` proc that
## should support the standard format specifiers. If ``ignoreUnknownSuffix`` is true,
## an unknown suffix after the ``type`` field is not an error.
const alignChars = {'<', '>', '^'}
result.fill = ' '
result.align = '\0'
result.sign = '-'
var i = start
if i + 1 < s.len and s[i+1] in alignChars:
result.fill = s[i]
result.align = s[i+1]
inc i, 2
elif i < s.len and s[i] in alignChars:
result.align = s[i]
inc i
if i < s.len and s[i] in {'-', '+', ' '}:
result.sign = s[i]
inc i
if i < s.len and s[i] == '#':
result.alternateForm = true
inc i
if i+1 < s.len and s[i] == '0' and s[i+1] in {'0'..'9'}:
result.padWithZero = true
inc i
let parsedLength = parseSaturatedNatural(s, result.minimumWidth, i)
inc i, parsedLength
if i < s.len and s[i] == '.':
inc i
let parsedLengthB = parseSaturatedNatural(s, result.precision, i)
inc i, parsedLengthB
else:
result.precision = -1
if i < s.len and s[i] in {'A'..'Z', 'a'..'z'}:
result.typ = s[i]
inc i
result.endPosition = i
if i != s.len and not ignoreUnknownSuffix:
raise newException(ValueError,
"invalid format string, cannot parse: " & s[i..^1])
proc format*(value: SomeInteger; specifier: string; res: var string) =
## Standard format implementation for ``SomeInteger``. It makes little
## sense to call this directly, but it is required to exist
## by the ``&`` macro.
let spec = parseStandardFormatSpecifier(specifier)
var radix = 10
case spec.typ
of 'x', 'X': radix = 16
of 'd', '\0': discard
of 'b': radix = 2
of 'o': radix = 8
else:
raise newException(ValueError,
"invalid type in format string for number, expected one " &
" of 'x', 'X', 'b', 'd', 'o' but got: " & spec.typ)
res.add formatInt(value, radix, spec)
proc format*(value: SomeReal; specifier: string; res: var string) =
## Standard format implementation for ``SomeReal``. It makes little
## sense to call this directly, but it is required to exist
## by the ``&`` macro.
let spec = parseStandardFormatSpecifier(specifier)
var fmode = ffDefault
case spec.typ
of 'e', 'E':
fmode = ffScientific
of 'f', 'F':
fmode = ffDecimal
of 'g', 'G':
fmode = ffDefault
of '\0': discard
else:
raise newException(ValueError,
"invalid type in format string for number, expected one " &
" of 'e', 'E', 'f', 'F', 'g', 'G' but got: " & spec.typ)
var f = formatBiggestFloat(value, fmode, spec.precision)
if value >= 0.0 and spec.sign != '-':
f = spec.sign & f
# the default for numbers is right-alignment:
let align = if spec.align == '\0': '>' else: spec.align
let result = alignString(f, spec.minimumWidth,
align, spec.fill)
if spec.typ in {'A'..'Z'}:
res.add toUpperAscii(result)
else:
res.add result
proc format*(value: string; specifier: string; res: var string) =
## Standard format implementation for ``string``. It makes little
## sense to call this directly, but it is required to exist
## by the ``&`` macro.
let spec = parseStandardFormatSpecifier(specifier)
case spec.typ
of 's', '\0': discard
else:
raise newException(ValueError,
"invalid type in format string for string, expected 's', but got " &
spec.typ)
res.add alignString(value, spec.minimumWidth, spec.align, spec.fill)
when isMainModule:
import json
doAssert fmt"{'a'} {'b'}" == "a b"

43
lib/pure/strscans.nim
View file

@ -31,7 +31,10 @@ As can be seen from the examples, strings are matched verbatim except for
substrings starting with ``$``. These constructions are available:
================= ========================================================
``$i`` Matches an integer. This uses ``parseutils.parseInt``.
``$b`` Matches a binary integer. This uses ``parseutils.parseBin``.
``$o`` Matches an octal integer. This uses ``parseutils.parseOct``.
``$i`` Matches a decimal integer. This uses ``parseutils.parseInt``.
``$h`` Matches a hex integer. This uses ``parseutils.parseHex``.
``$f`` Matches a floating pointer number. Uses ``parseFloat``.
``$w`` Matches an ASCII identifier: ``[A-Z-a-z_][A-Za-z_0-9]*``.
``$s`` Skips optional whitespace.
@ -330,19 +333,37 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b
conds.add resLen.notZero
conds.add resLen
of 'w':
if i < results.len or getType(results[i]).typeKind != ntyString:
if i < results.len and getType(results[i]).typeKind == ntyString:
matchBind "parseIdent"
else:
error("no string var given for $w")
inc i
of 'b':
if i < results.len and getType(results[i]).typeKind == ntyInt:
matchBind "parseBin"
else:
error("no int var given for $b")
inc i
of 'o':
if i < results.len and getType(results[i]).typeKind == ntyInt:
matchBind "parseOct"
else:
error("no int var given for $o")
inc i
of 'i':
if i < results.len or getType(results[i]).typeKind != ntyInt:
if i < results.len and getType(results[i]).typeKind == ntyInt:
matchBind "parseInt"
else:
error("no int var given for $d")
error("no int var given for $i")
inc i
of 'h':
if i < results.len and getType(results[i]).typeKind == ntyInt:
matchBind "parseHex"
else:
error("no int var given for $h")
inc i
of 'f':
if i < results.len or getType(results[i]).typeKind != ntyFloat:
if i < results.len and getType(results[i]).typeKind == ntyFloat:
matchBind "parseFloat"
else:
error("no float var given for $f")
@ -357,7 +378,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b
else:
error("invalid format string")
of '*', '+':
if i < results.len or getType(results[i]).typeKind != ntyString:
if i < results.len and getType(results[i]).typeKind == ntyString:
var min = ord(pattern[p] == '+')
var q=p+1
var token = ""
@ -441,7 +462,7 @@ template success*(x: int): bool = x != 0
template nxt*(input: string; idx, step: int = 1) = inc(idx, step)
macro scanp*(input, idx: typed; pattern: varargs[untyped]): bool =
## See top level documentation of his module of how ``scanp`` works.
## ``scanp`` is currently undocumented.
type StmtTriple = tuple[init, cond, action: NimNode]
template interf(x): untyped = bindSym(x, brForceOpen)
@ -645,6 +666,14 @@ when isMainModule:
doAssert intval == 89
doAssert floatVal == 33.25
var binval: int
var octval: int
var hexval: int
doAssert scanf("0b0101 0o1234 0xabcd", "$b$s$o$s$h", binval, octval, hexval)
doAssert binval == 0b0101
doAssert octval == 0o1234
doAssert hexval == 0xabcd
let xx = scanf("$abc", "$$$i", intval)
doAssert xx == false

24
lib/pure/strutils.nim
View file

@ -1761,29 +1761,15 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect,
proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,
rtl, extern: "nsuEscape".} =
## Escapes a string `s`.
## Escapes a string `s`. See `system.addEscapedChar <system.html#addEscapedChar>`_
## for the escaping scheme.
##
## This does these operations (at the same time):
## * replaces any ``\`` by ``\\``
## * replaces any ``'`` by ``\'``
## * replaces any ``"`` by ``\"``
## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}``
## by ``\xHH`` where ``HH`` is its hexadecimal value.
## The procedure has been designed so that its output is usable for many
## different common syntaxes. The resulting string is prefixed with
## `prefix` and suffixed with `suffix`. Both may be empty strings.
## **Note**: This is not correct for producing Ansi C code!
## The resulting string is prefixed with `prefix` and suffixed with `suffix`.
## Both may be empty strings.
result = newStringOfCap(s.len + s.len shr 2)
result.add(prefix)
for c in items(s):
case c
of '\0'..'\31', '\127'..'\255':
add(result, "\\x")
add(result, toHex(ord(c), 2))
of '\\': add(result, "\\\\")
of '\'': add(result, "\\'")
of '\"': add(result, "\\\"")
else: add(result, c)
result.addEscapedChar(c)
add(result, suffix)
proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect,

20
lib/pure/terminal.nim
View file

@ -391,12 +391,11 @@ proc eraseLine*(f: File) =
origin.X = 0'i16
if setConsoleCursorPosition(h, origin) == 0:
raiseOSError(osLastError())
var ht = scrbuf.dwSize.Y - origin.Y
var wt = scrbuf.dwSize.X - origin.X
if fillConsoleOutputCharacter(h, ' ', ht*wt,
var wt: DWORD = scrbuf.dwSize.X - origin.X
if fillConsoleOutputCharacter(h, ' ', wt,
origin, addr(numwrote)) == 0:
raiseOSError(osLastError())
if fillConsoleOutputAttribute(h, scrbuf.wAttributes, ht * wt,
if fillConsoleOutputAttribute(h, scrbuf.wAttributes, wt,
scrbuf.dwCursorPosition, addr(numwrote)) == 0:
raiseOSError(osLastError())
else:
@ -634,7 +633,10 @@ proc getch*(): char =
doAssert(readConsoleInput(fd, addr(keyEvent), 1, addr(numRead)) != 0)
if numRead == 0 or keyEvent.eventType != 1 or keyEvent.bKeyDown == 0:
continue
return char(keyEvent.uChar)
if keyEvent.uChar == 0:
return char(keyEvent.wVirtualKeyCode)
else:
return char(keyEvent.uChar)
else:
let fd = getFileHandle(stdin)
var oldMode: Termios
@ -650,10 +652,10 @@ template setCursorPos*(x, y: int) = setCursorPos(stdout, x, y)
template setCursorXPos*(x: int) = setCursorXPos(stdout, x)
when defined(windows):
template setCursorYPos(x: int) = setCursorYPos(stdout, x)
template cursorUp*(count=1) = cursorUp(stdout, f)
template cursorDown*(count=1) = cursorDown(stdout, f)
template cursorForward*(count=1) = cursorForward(stdout, f)
template cursorBackward*(count=1) = cursorBackward(stdout, f)
template cursorUp*(count=1) = cursorUp(stdout, count)
template cursorDown*(count=1) = cursorDown(stdout, count)
template cursorForward*(count=1) = cursorForward(stdout, count)
template cursorBackward*(count=1) = cursorBackward(stdout, count)
template eraseLine*() = eraseLine(stdout)
template eraseScreen*() = eraseScreen(stdout)
template setStyle*(style: set[Style]) =

1647
lib/pure/times.nim
View file

File diff suppressed because it is too large Load diff

91
lib/pure/unittest.nim
View file

@ -21,13 +21,41 @@
## ``nim c -r <testfile.nim>`` exits with 0 or 1
##
## Running a single test
## ---------------------
## =====================
##
## Simply specify the test name as a command line argument.
## Specify the test name as a command line argument.
##
## .. code::
##
## nim c -r test "my super awesome test name"
## nim c -r test "my test name" "another test"
##
## Multiple arguments can be used.
##
## Running a single test suite
## ===========================
##
## Specify the suite name delimited by ``"::"``.
##
## .. code::
##
## nim c -r test "my test name::"
##
## Selecting tests by pattern
## ==========================
##
## A single ``"*"`` can be used for globbing.
##
## Delimit the end of a suite name with ``"::"``.
##
## Tests matching **any** of the arguments are executed.
##
## .. code::
##
## nim c -r test fast_suite::mytest1 fast_suite::mytest2
## nim c -r test "fast_suite::mytest*"
## nim c -r test "auth*::" "crypto::hashing*"
## # Run suites starting with 'bug #' and standalone tests starting with '#'
## nim c -r test 'bug #*::' '::#*'
##
## Example
## -------
@ -121,7 +149,7 @@ var
checkpoints {.threadvar.}: seq[string]
formatters {.threadvar.}: seq[OutputFormatter]
testsToRun {.threadvar.}: HashSet[string]
testsFilters {.threadvar.}: HashSet[string]
when declared(stdout):
abortOnError = existsEnv("NIMTEST_ABORT_ON_ERROR")
@ -300,22 +328,63 @@ method testEnded*(formatter: JUnitOutputFormatter, testResult: TestResult) =
method suiteEnded*(formatter: JUnitOutputFormatter) =
formatter.stream.writeLine("\t</testsuite>")
proc shouldRun(testName: string): bool =
if testsToRun.len == 0:
proc glob(matcher, filter: string): bool =
## Globbing using a single `*`. Empty `filter` matches everything.
if filter.len == 0:
return true
result = testName in testsToRun
if not filter.contains('*'):
return matcher == filter
let beforeAndAfter = filter.split('*', maxsplit=1)
if beforeAndAfter.len == 1:
# "foo*"
return matcher.startswith(beforeAndAfter[0])
if matcher.len < filter.len - 1:
return false # "12345" should not match "123*345"
return matcher.startsWith(beforeAndAfter[0]) and matcher.endsWith(beforeAndAfter[1])
proc matchFilter(suiteName, testName, filter: string): bool =
if filter == "":
return true
if testName == filter:
# corner case for tests containing "::" in their name
return true
let suiteAndTestFilters = filter.split("::", maxsplit=1)
if suiteAndTestFilters.len == 1:
# no suite specified
let test_f = suiteAndTestFilters[0]
return glob(testName, test_f)
return glob(suiteName, suiteAndTestFilters[0]) and glob(testName, suiteAndTestFilters[1])
when defined(testing): export matchFilter
proc shouldRun(currentSuiteName, testName: string): bool =
## Check if a test should be run by matching suiteName and testName against
## test filters.
if testsFilters.len == 0:
return true
for f in testsFilters:
if matchFilter(currentSuiteName, testName, f):
return true
return false
proc ensureInitialized() =
if formatters == nil:
formatters = @[OutputFormatter(defaultConsoleFormatter())]
if not testsToRun.isValid:
testsToRun.init()
if not testsFilters.isValid:
testsFilters.init()
when declared(paramCount):
# Read tests to run from the command line.
for i in 1 .. paramCount():
testsToRun.incl(paramStr(i))
testsFilters.incl(paramStr(i))
# These two procs are added as workarounds for
# https://github.com/nim-lang/Nim/issues/5549
@ -395,7 +464,7 @@ template test*(name, body) {.dirty.} =
ensureInitialized()
if shouldRun(name):
if shouldRun(when declared(testSuiteName): testSuiteName else: "", name):
checkpoints = @[]
var testStatusIMPL {.inject.} = OK

96
lib/system.nim
View file

@ -249,6 +249,10 @@ type
when defined(nimHasOpt):
type opt*{.magic: "Opt".}[T]
when defined(nimNewRuntime):
type sink*{.magic: "BuiltinType".}[T]
type lent*{.magic: "BuiltinType".}[T]
proc high*[T: Ordinal](x: T): T {.magic: "High", noSideEffect.}
## returns the highest possible index of an array, a sequence, a string or
## the highest possible value of an ordinal value `x`. As a special
@ -463,7 +467,7 @@ type
line*: int ## line number of the proc that is currently executing
filename*: cstring ## filename of the proc that is currently executing
Exception* {.compilerproc.} = object of RootObj ## \
Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \
## Base exception class.
##
## Each exception has to inherit from `Exception`. See the full `exception
@ -743,6 +747,18 @@ proc newSeqOfCap*[T](cap: Natural): seq[T] {.
## ``cap``.
discard
when not defined(JS):
proc newSeqUninitialized*[T: SomeNumber](len: Natural): seq[T] =
## creates a new sequence of type ``seq[T]`` with length ``len``.
##
## Only available for numbers types. Note that the sequence will be
## uninitialized. After the creation of the sequence you should assign
## entries to the sequence instead of adding them.
result = newSeqOfCap[T](len)
var s = cast[PGenericSeq](result)
s.len = len
proc len*[TOpenArray: openArray|varargs](x: TOpenArray): int {.
magic: "LengthOpenArray", noSideEffect.}
proc len*(x: string): int {.magic: "LengthStr", noSideEffect.}
@ -1325,6 +1341,7 @@ proc add*(x: var string, y: string) {.magic: "AppendStrStr", noSideEffect.}
## tmp.add("cd")
## assert(tmp == "abcd")
type
Endianness* = enum ## is a type describing the endianness of a processor.
littleEndian, bigEndian
@ -2503,9 +2520,9 @@ proc `$`*[T: tuple|object](x: T): string =
when compiles($value):
when compiles(value.isNil):
if value.isNil: result.add "nil"
else: result.add($value)
else: result.addQuoted(value)
else:
result.add($value)
result.addQuoted(value)
firstElement = false
else:
result.add("...")
@ -2525,12 +2542,9 @@ proc collectionToString[T](x: T, prefix, separator, suffix: string): string =
if value.isNil:
result.add "nil"
else:
result.add($value)
# prevent temporary string allocation
elif compiles(result.add(value)):
result.add(value)
result.addQuoted(value)
else:
result.add($value)
result.addQuoted(value)
result.add(suffix)
@ -2918,7 +2932,10 @@ when not defined(JS): #and not defined(nimscript):
elif x > y: result = 1
else: result = 0
else:
result = int(c_strcmp(x, y))
let minlen = min(x.len, y.len)
result = int(c_memcmp(x.cstring, y.cstring, minlen.csize))
if result == 0:
result = x.len - y.len
when defined(nimscript):
proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.}
@ -3893,6 +3910,65 @@ proc compiles*(x: untyped): bool {.magic: "Compiles", noSideEffect, compileTime.
when declared(initDebugger):
initDebugger()
proc addEscapedChar*(s: var string, c: char) {.noSideEffect, inline.} =
## Adds a char to string `s` and applies the following escaping:
##
## * replaces any ``\`` by ``\\``
## * replaces any ``'`` by ``\'``
## * replaces any ``"`` by ``\"``
## * replaces any other character in the set ``{'\0'..'\31', '\127'..'\255'}``
## by ``\xHH`` where ``HH`` is its hexadecimal value.
##
## The procedure has been designed so that its output is usable for many
## different common syntaxes.
## **Note**: This is not correct for producing Ansi C code!
case c
of '\0'..'\31', '\127'..'\255':
add(s, "\\x")
const HexChars = "0123456789ABCDEF"
let n = ord(c)
s.add(HexChars[int((n and 0xF0) shr 4)])
s.add(HexChars[int(n and 0xF)])
of '\\': add(s, "\\\\")
of '\'': add(s, "\\'")
of '\"': add(s, "\\\"")
else: add(s, c)
proc addQuoted*[T](s: var string, x: T) =
## Appends `x` to string `s` in place, applying quoting and escaping
## if `x` is a string or char. See
## `addEscapedChar <system.html#addEscapedChar>`_
## for the escaping scheme.
##
## The Nim standard library uses this function on the elements of
## collections when producing a string representation of a collection.
## It is recommended to use this function as well for user-side collections.
## Users may overload `addQuoted` for custom (string-like) types if
## they want to implement a customized element representation.
##
## .. code-block:: Nim
## var tmp = ""
## tmp.addQuoted(1)
## tmp.add(", ")
## tmp.addQuoted("string")
## tmp.add(", ")
## tmp.addQuoted('c')
## assert(tmp == """1, "string", 'c'""")
when T is string:
s.add("\"")
for c in x:
s.addEscapedChar(c)
s.add("\"")
elif T is char:
s.add("'")
s.addEscapedChar(x)
s.add("'")
# prevent temporary string allocation
elif compiles(s.add(x)):
s.add(x)
else:
s.add($x)
when hasAlloc:
# XXX: make these the default (or implement the NilObject optimization)
proc safeAdd*[T](x: var seq[T], y: T) {.noSideEffect.} =
@ -4034,4 +4110,4 @@ template doAssertRaises*(exception, code: untyped): typed =
except Exception as exc:
raiseAssert(astToStr(exception) &
" wasn't raised, another error was raised instead by:\n"&
astToStr(code))
astToStr(code))

8
lib/system/alloc.nim
View file

@ -813,13 +813,13 @@ proc alloc0(allocator: var MemRegion, size: Natural): pointer =
zeroMem(result, size)
proc dealloc(allocator: var MemRegion, p: pointer) =
sysAssert(p != nil, "dealloc 0")
sysAssert(p != nil, "dealloc: p is nil")
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
sysAssert(x != nil, "dealloc 1")
sysAssert(x != nil, "dealloc: x is nil")
sysAssert(isAccessible(allocator, x), "is not accessible")
sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2")
sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted")
rawDealloc(allocator, x)
sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3")
sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible")
track("dealloc", p, 0)
proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =

1
lib/system/chcks.nim
View file

@ -63,7 +63,6 @@ proc chckObj(obj, subclass: PNimType) {.compilerproc.} =
while x != subclass:
if x == nil:
sysFatal(ObjectConversionError, "invalid object conversion")
break
x = x.base
proc chckObjAsgn(a, b: PNimType) {.compilerproc, inline.} =

12
lib/system/excpt.nim
View file

@ -70,6 +70,18 @@ proc getFrame*(): PFrame {.compilerRtl, inl.} = framePtr
proc popFrame {.compilerRtl, inl.} =
framePtr = framePtr.prev
when false:
proc popFrameOfAddr(s: PFrame) {.compilerRtl.} =
var it = framePtr
if it == s:
framePtr = framePtr.prev
else:
while it != nil:
if it == s:
framePtr = it.prev
break
it = it.prev
proc setFrame*(s: PFrame) {.compilerRtl, inl.} =
framePtr = s

133
lib/system/gc.nim
View file

@ -21,10 +21,6 @@ const
# reaches this threshold
# this seems to be a good value
withRealTime = defined(useRealtimeGC)
useMarkForDebug = defined(gcGenerational)
useBackupGc = true # use a simple M&S GC to collect
# cycles instead of the complex
# algorithm
when withRealTime and not declared(getTicks):
include "system/timers"
@ -92,14 +88,12 @@ type
maxPause: Nanos # max allowed pause in nanoseconds; active if > 0
region: MemRegion # garbage collected region
stat: GcStat
when useMarkForDebug or useBackupGc:
marked: CellSet
additionalRoots: CellSeq # dummy roots for GC_ref/unref
marked: CellSet
additionalRoots: CellSeq # dummy roots for GC_ref/unref
when hasThreadSupport:
toDispose: SharedList[pointer]
gcThreadId: int
{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap,
TGcStat: GcStat].}
var
gch {.rtlThreadVar.}: GcHeap
@ -165,12 +159,12 @@ when defined(logGC):
if not c.typ.name.isNil:
typName = c.typ.name
when leakDetector:
c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld from %s(%ld)\n",
msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line)
else:
c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; color=%ld\n",
msg, c, kind, typName, c.refcount shr rcShift, c.color)
when leakDetector:
c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld from %s(%ld)\n",
msg, c, kind, typName, c.refcount shr rcShift, c.filename, c.line)
else:
c_fprintf(stdout, "[GC] %s: %p %d %s rc=%ld; thread=%ld\n",
msg, c, kind, typName, c.refcount shr rcShift, gch.gcThreadId)
template gcTrace(cell, state: untyped) =
when traceGC: traceCell(cell, state)
@ -314,27 +308,12 @@ proc initGC() =
init(gch.zct)
init(gch.tempStack)
init(gch.decStack)
when useMarkForDebug or useBackupGc:
init(gch.marked)
init(gch.additionalRoots)
init(gch.marked)
init(gch.additionalRoots)
when hasThreadSupport:
gch.toDispose = initSharedList[pointer]()
when useMarkForDebug or useBackupGc:
type
GlobalMarkerProc = proc () {.nimcall, benign.}
{.deprecated: [TGlobalMarkerProc: GlobalMarkerProc].}
var
globalMarkersLen: int
globalMarkers: array[0.. 7_000, GlobalMarkerProc]
proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} =
if globalMarkersLen <= high(globalMarkers):
globalMarkers[globalMarkersLen] = markerProc
inc globalMarkersLen
else:
echo "[GC] cannot register global variable; too many global variables"
quit 1
init(gch.toDispose)
gch.gcThreadId = atomicInc(gHeapidGenerator) - 1
gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID")
proc cellsetReset(s: var CellSet) =
deinit(s)
@ -377,10 +356,10 @@ proc forAllChildrenAux(dest: pointer, mt: PNimType, op: WalkOp) =
else: discard
proc forAllChildren(cell: PCell, op: WalkOp) =
gcAssert(cell != nil, "forAllChildren: 1")
gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: 2")
gcAssert(cell.typ != nil, "forAllChildren: 3")
gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: 4"
gcAssert(cell != nil, "forAllChildren: cell is nil")
gcAssert(isAllocatedPtr(gch.region, cell), "forAllChildren: pointer not part of the heap")
gcAssert(cell.typ != nil, "forAllChildren: cell.typ is nil")
gcAssert cell.typ.kind in {tyRef, tyOptAsRef, tySequence, tyString}, "forAllChildren: unknown GC'ed type"
let marker = cell.typ.marker
if marker != nil:
marker(cellToUsr(cell), op.int)
@ -481,7 +460,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
release(gch)
when useCellIds:
inc gch.idGenerator
res.id = gch.idGenerator
res.id = gch.idGenerator * 1000_000 + gch.gcThreadId
result = cellToUsr(res)
sysAssert(allocInv(gch.region), "rawNewObj end")
@ -528,7 +507,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
release(gch)
when useCellIds:
inc gch.idGenerator
res.id = gch.idGenerator
res.id = gch.idGenerator * 1000_000 + gch.gcThreadId
result = cellToUsr(res)
zeroMem(result, size)
sysAssert(allocInv(gch.region), "newObjRC1 end")
@ -598,7 +577,7 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer =
release(gch)
when useCellIds:
inc gch.idGenerator
res.id = gch.idGenerator
res.id = gch.idGenerator * 1000_000 + gch.gcThreadId
result = cellToUsr(res)
sysAssert(allocInv(gch.region), "growObj end")
when defined(memProfiler): nimProfile(newsize-oldsize)
@ -623,29 +602,31 @@ proc freeCyclicCell(gch: var GcHeap, c: PCell) =
gcAssert(c.typ != nil, "freeCyclicCell")
zeroMem(c, sizeof(Cell))
when useBackupGc:
proc sweep(gch: var GcHeap) =
for x in allObjects(gch.region):
if isCell(x):
# cast to PCell is correct here:
var c = cast[PCell](x)
if c notin gch.marked: freeCyclicCell(gch, c)
proc sweep(gch: var GcHeap) =
for x in allObjects(gch.region):
if isCell(x):
# cast to PCell is correct here:
var c = cast[PCell](x)
if c notin gch.marked: freeCyclicCell(gch, c)
when useMarkForDebug or useBackupGc:
proc markS(gch: var GcHeap, c: PCell) =
incl(gch.marked, c)
gcAssert gch.tempStack.len == 0, "stack not empty!"
forAllChildren(c, waMarkPrecise)
while gch.tempStack.len > 0:
dec gch.tempStack.len
var d = gch.tempStack.d[gch.tempStack.len]
if not containsOrIncl(gch.marked, d):
forAllChildren(d, waMarkPrecise)
proc markS(gch: var GcHeap, c: PCell) =
gcAssert isAllocatedPtr(gch.region, c), "markS: foreign heap root detected A!"
incl(gch.marked, c)
gcAssert gch.tempStack.len == 0, "stack not empty!"
forAllChildren(c, waMarkPrecise)
while gch.tempStack.len > 0:
dec gch.tempStack.len
var d = gch.tempStack.d[gch.tempStack.len]
gcAssert isAllocatedPtr(gch.region, d), "markS: foreign heap root detected B!"
if not containsOrIncl(gch.marked, d):
forAllChildren(d, waMarkPrecise)
proc markGlobals(gch: var GcHeap) =
proc markGlobals(gch: var GcHeap) =
if gch.gcThreadId == 0:
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
let d = gch.additionalRoots.d
for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i])
for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]()
let d = gch.additionalRoots.d
for i in 0 .. gch.additionalRoots.len-1: markS(gch, d[i])
when logGC:
var
@ -689,16 +670,9 @@ proc doOperation(p: pointer, op: WalkOp) =
of waPush:
add(gch.tempStack, c)
of waMarkGlobal:
when useMarkForDebug or useBackupGc:
when hasThreadSupport:
# could point to a cell which we don't own and don't want to touch/trace
if isAllocatedPtr(gch.region, c):
markS(gch, c)
else:
markS(gch, c)
markS(gch, c)
of waMarkPrecise:
when useMarkForDebug or useBackupGc:
add(gch.tempStack, c)
add(gch.tempStack, c)
#of waDebug: debugGraph(c)
proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} =
@ -712,14 +686,13 @@ proc collectCycles(gch: var GcHeap) =
nimGCunref(c)
# ensure the ZCT 'color' is not used:
while gch.zct.len > 0: discard collectZCT(gch)
when useBackupGc:
cellsetReset(gch.marked)
var d = gch.decStack.d
for i in 0..gch.decStack.len-1:
sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles"
markS(gch, d[i])
markGlobals(gch)
sweep(gch)
cellsetReset(gch.marked)
var d = gch.decStack.d
for i in 0..gch.decStack.len-1:
sysAssert isAllocatedPtr(gch.region, d[i]), "collectCycles"
markS(gch, d[i])
markGlobals(gch)
sweep(gch)
proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
# the addresses are not as cells on the stack, so turn them to cells:
@ -860,7 +833,7 @@ proc collectCT(gch: var GcHeap) =
if (gch.zct.len >= stackMarkCosts or (cycleGC and
getOccupiedMem(gch.region)>=gch.cycleThreshold) or alwaysGC) and
gch.recGcLock == 0:
when useMarkForDebug:
when false:
prepareForInteriorPointerChecking(gch.region)
cellsetReset(gch.marked)
markForDebug(gch)

77
lib/system/gc2.nim
View file

@ -104,6 +104,7 @@ type
pDumpHeapFile: pointer # File that is used for GC_dumpHeap
when hasThreadSupport:
toDispose: SharedList[pointer]
gcThreadId: int
var
gch {.rtlThreadVar.}: GcHeap
@ -119,22 +120,6 @@ template release(gch: GcHeap) =
when hasThreadSupport and hasSharedHeap:
releaseSys(HeapLock)
proc initGC() =
when not defined(useNimRtl):
gch.red = (1-gch.black)
gch.cycleThreshold = InitialCycleThreshold
gch.stat.stackScans = 0
gch.stat.completedCollections = 0
gch.stat.maxThreshold = 0
gch.stat.maxStackSize = 0
gch.stat.maxStackCells = 0
gch.stat.cycleTableSize = 0
# init the rt
init(gch.additionalRoots)
init(gch.greyStack)
when hasThreadSupport:
gch.toDispose = initSharedList[pointer]()
# Which color to use for new objects is tricky: When we're marking,
# they have to be *white* so that everything is marked that is only
# reachable from them. However, when we are sweeping, they have to
@ -284,20 +269,6 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.compilerProc.} =
if not isOnStack(dest): markGrey(s)
dest[] = src
type
GlobalMarkerProc = proc () {.nimcall, benign.}
var
globalMarkersLen: int
globalMarkers: array[0.. 7_000, GlobalMarkerProc]
proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} =
if globalMarkersLen <= high(globalMarkers):
globalMarkers[globalMarkersLen] = markerProc
inc globalMarkersLen
else:
echo "[GC] cannot register global variable; too many global variables"
quit 1
proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} =
var d = cast[ByteAddress](dest)
case n.kind
@ -354,6 +325,24 @@ proc gcInvariant*() =
include gc_common
proc initGC() =
when not defined(useNimRtl):
gch.red = (1-gch.black)
gch.cycleThreshold = InitialCycleThreshold
gch.stat.stackScans = 0
gch.stat.completedCollections = 0
gch.stat.maxThreshold = 0
gch.stat.maxStackSize = 0
gch.stat.maxStackCells = 0
gch.stat.cycleTableSize = 0
# init the rt
init(gch.additionalRoots)
init(gch.greyStack)
when hasThreadSupport:
init(gch.toDispose)
gch.gcThreadId = atomicInc(gHeapidGenerator) - 1
gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID")
proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer =
# generates a new object and sets its reference counter to 0
sysAssert(allocInv(gch.region), "rawNewObj begin")
@ -492,7 +481,9 @@ proc GC_dumpHeap*(file: File) =
c_fprintf(file, "onstack %p\n", d[i])
else:
c_fprintf(file, "onstack_invalid %p\n", d[i])
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
if gch.gcThreadId == 0:
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]()
while true:
let x = allObjectsAsProc(gch.region, addr spaceIter)
if spaceIter.state < 0: break
@ -579,7 +570,9 @@ proc markIncremental(gch: var GcHeap): bool =
result = true
proc markGlobals(gch: var GcHeap) =
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
if gch.gcThreadId == 0:
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]()
proc doOperation(p: pointer, op: WalkOp) =
if p == nil: return
@ -599,22 +592,14 @@ proc doOperation(p: pointer, op: WalkOp) =
markRoot(gch, c)
else:
dumpRoot(gch, c)
when hasThreadSupport:
# could point to a cell which we don't own and don't want to touch/trace
if isAllocatedPtr(gch.region, c): handleRoot()
else:
#gcAssert(isAllocatedPtr(gch.region, c), "doOperation: waMarkGlobal")
if not isAllocatedPtr(gch.region, c):
c_fprintf(stdout, "[GC] not allocated anymore: MarkGlobal %p\n", c)
#GC_dumpHeap()
sysAssert(false, "wtf")
handleRoot()
handleRoot()
discard allocInv(gch.region)
of waMarkGrey:
if not isAllocatedPtr(gch.region, c):
c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c)
#GC_dumpHeap()
sysAssert(false, "wtf")
when false:
if not isAllocatedPtr(gch.region, c):
c_fprintf(stdout, "[GC] not allocated anymore: MarkGrey %p\n", c)
#GC_dumpHeap()
sysAssert(false, "wtf")
if c.color == 1-gch.black:
c.setColor(rcGrey)
add(gch.greyStack, c)

25
lib/system/gc_common.nim
View file

@ -393,3 +393,28 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) =
zeroMem(addr gch.region, sizeof(gch.region))
if allowGcAfterwards:
initGC()
type
GlobalMarkerProc = proc () {.nimcall, benign.}
var
globalMarkersLen: int
globalMarkers: array[0.. 3499, GlobalMarkerProc]
threadLocalMarkersLen: int
threadLocalMarkers: array[0.. 3499, GlobalMarkerProc]
gHeapidGenerator: int
proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} =
if globalMarkersLen <= high(globalMarkers):
globalMarkers[globalMarkersLen] = markerProc
inc globalMarkersLen
else:
echo "[GC] cannot register global variable; too many global variables"
quit 1
proc nimRegisterThreadLocalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} =
if threadLocalMarkersLen <= high(threadLocalMarkers):
threadLocalMarkers[threadLocalMarkersLen] = markerProc
inc threadLocalMarkersLen
else:
echo "[GC] cannot register thread local variable; too many thread local variables"
quit 1

34
lib/system/gc_ms.nim
View file

@ -40,8 +40,6 @@ type
# A ref type can have a finalizer that is called before the object's
# storage is freed.
GlobalMarkerProc = proc () {.nimcall, benign.}
GcStat = object
collections: int # number of performed full collections
maxThreshold: int # max threshold that has been set
@ -75,9 +73,9 @@ type
stat: GcStat
when hasThreadSupport:
toDispose: SharedList[pointer]
gcThreadId: int
additionalRoots: CellSeq # dummy roots for GC_ref/unref
{.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat,
TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].}
var
gch {.rtlThreadVar.}: GcHeap
@ -119,18 +117,6 @@ proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} =
proc internRefcount(p: pointer): int {.exportc: "getRefcount".} =
result = 0
var
globalMarkersLen: int
globalMarkers: array[0.. 7_000, GlobalMarkerProc]
proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} =
if globalMarkersLen <= high(globalMarkers):
globalMarkers[globalMarkersLen] = markerProc
inc globalMarkersLen
else:
echo "[GC] cannot register global variable; too many global variables"
quit 1
# this that has to equals zero, otherwise we have to round up UnitsPerPage:
when BitsPerPage mod (sizeof(int)*8) != 0:
{.error: "(BitsPerPage mod BitsPerUnit) should be zero!".}
@ -233,7 +219,9 @@ proc initGC() =
init(gch.allocated)
init(gch.marked)
when hasThreadSupport:
gch.toDispose = initSharedList[pointer]()
init(gch.toDispose)
gch.gcThreadId = atomicInc(gHeapidGenerator) - 1
gcAssert(gch.gcThreadId >= 0, "invalid computed thread ID")
proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} =
var d = cast[ByteAddress](dest)
@ -407,13 +395,7 @@ proc doOperation(p: pointer, op: WalkOp) =
var c: PCell = usrToCell(p)
gcAssert(c != nil, "doOperation: 1")
case op
of waMarkGlobal:
when hasThreadSupport:
# could point to a cell which we don't own and don't want to touch/trace
if isAllocatedPtr(gch.region, c):
mark(gch, c)
else:
mark(gch, c)
of waMarkGlobal: mark(gch, c)
of waMarkPrecise: add(gch.tempStack, c)
proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} =
@ -450,7 +432,9 @@ when false:
quit 1
proc markGlobals(gch: var GcHeap) =
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
if gch.gcThreadId == 0:
for i in 0 .. globalMarkersLen-1: globalMarkers[i]()
for i in 0 .. threadLocalMarkersLen-1: threadLocalMarkers[i]()
let d = gch.additionalRoots.d
for i in 0 .. gch.additionalRoots.len-1: mark(gch, d[i])

114
lib/system/genodealloc.nim Normal file
View file

@ -0,0 +1,114 @@
#
#
# Nim's Runtime Library
# (c) Copyright 2017 Emery Hemingway
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
# Low level dataspace allocator for Genode.
when not defined(genode):
{.error: "Genode only module".}
type DataspaceCapability {.
importcpp: "Genode::Dataspace_capability", pure.} = object
type
Map = object
attachment: pointer
size: int
ds: DataspaceCapability
SlabMeta = object
next: ptr MapSlab
ds: DataspaceCapability
MapSlab = object
meta: SlabMeta
maps: array[1,Map]
const SlabBackendSize = 4096
proc ramAvail(): int {.
importcpp: "genodeEnv->pd().avail_ram().value".}
## Return number of bytes available for allocation.
proc capsAvail(): int {.
importcpp: "genodeEnv->pd().avail_caps().value".}
## Return the number of available capabilities.
## Each dataspace allocation consumes a capability.
proc allocDataspace(size: int): DataspaceCapability {.
importcpp: "genodeEnv->pd().alloc(@)".}
## Allocate a dataspace and its capability.
proc attachDataspace(ds: DataspaceCapability): pointer {.
importcpp: "genodeEnv->rm().attach(@)".}
## Attach a dataspace into the component address-space.
proc detachAddress(p: pointer) {.
importcpp: "genodeEnv->rm().detach(@)".}
## Detach a dataspace from the component address-space.
proc freeDataspace(ds: DataspaceCapability) {.
importcpp: "genodeEnv->pd().free(@)".}
## Free a dataspace.
proc newMapSlab(): ptr MapSlab =
let
ds = allocDataspace SlabBackendSize
p = attachDataspace ds
result = cast[ptr MapSlab](p)
result.meta.ds = ds
iterator items(s: ptr MapSlab): ptr Map =
let mapCount = (SlabBackendSize - sizeof(SlabMeta)) div sizeof(Map)
for i in 0 .. <mapCount:
yield s.maps[i].addr
var slabs: ptr MapSlab
proc osAllocPages(size: int): pointer =
if slabs.isNil:
slabs = newMapSlab()
var
slab = slabs
map: ptr Map
let mapCount = (SlabBackendSize - sizeof(SlabMeta)) div sizeof(Map)
block findFreeMap:
while true:
# lookup first free spot in slabs
for m in slab.items:
if m.attachment.isNil:
map = m
break findFreeMap
if slab.meta.next.isNil:
slab.meta.next = newMapSlab()
# tack a new slab on the tail
slab = slab.meta.next
# move to next slab in linked list
map.ds = allocDataspace size
map.size = size
map.attachment = attachDataspace map.ds
result = map.attachment
proc osTryAllocPages(size: int): pointer =
if ramAvail() >= size and capsAvail() > 1:
result = osAllocPages size
proc osDeallocPages(p: pointer; size: int) =
var slab = slabs
while not slab.isNil:
# lookup first free spot in slabs
for m in slab.items:
if m.attachment == p:
if m.size != size:
echo "cannot partially detach dataspace"
quit -1
detachAddress m.attachment
freeDataspace m.ds
m[] = Map()
return
slab = slab.meta.next

4
lib/system/mmdisp.nim
View file

@ -109,7 +109,6 @@ when defined(boehmgc):
if result == nil: raiseOutOfMem()
proc alloc0(size: Natural): pointer =
result = alloc(size)
zeroMem(result, size)
proc realloc(p: pointer, newsize: Natural): pointer =
result = boehmRealloc(p, newsize)
if result == nil: raiseOutOfMem()
@ -119,8 +118,7 @@ when defined(boehmgc):
result = boehmAlloc(size)
if result == nil: raiseOutOfMem()
proc allocShared0(size: Natural): pointer =
result = alloc(size)
zeroMem(result, size)
result = allocShared(size)
proc reallocShared(p: pointer, newsize: Natural): pointer =
result = boehmRealloc(p, newsize)
if result == nil: raiseOutOfMem()

12
lib/system/osalloc.nim
View file

@ -78,17 +78,7 @@ when defined(emscripten):
munmap(mmapDescr.realPointer, mmapDescr.realSize)
elif defined(genode):
proc osAllocPages(size: int): pointer {.
importcpp: "genodeEnv->rm().attach(genodeEnv->ram().alloc(@))".}
proc osTryAllocPages(size: int): pointer =
{.emit: """try {""".}
result = osAllocPages size
{.emit: """} catch (...) { }""".}
proc osDeallocPages(p: pointer, size: int) {.
importcpp: "genodeEnv->rm().detach(#)".}
include genodealloc # osAllocPages, osTryAllocPages, osDeallocPages
elif defined(posix):
const

5
lib/system/sysio.nim
View file

@ -406,7 +406,8 @@ proc setStdIoUnbuffered() =
when declared(stdout):
proc echoBinSafe(args: openArray[string]) {.compilerProc.} =
when not defined(windows):
# flockfile deadlocks some versions of Android 5.x.x
when not defined(windows) and not defined(android):
proc flockfile(f: File) {.importc, noDecl.}
proc funlockfile(f: File) {.importc, noDecl.}
flockfile(stdout)
@ -415,7 +416,7 @@ when declared(stdout):
const linefeed = "\n" # can be 1 or more chars
discard c_fwrite(linefeed.cstring, linefeed.len, 1, stdout)
discard c_fflush(stdout)
when not defined(windows):
when not defined(windows) and not defined(android):
funlockfile(stdout)
{.pop.}

15
lib/system/sysstr.nim
View file

@ -24,10 +24,10 @@ proc cmpStrings(a, b: NimString): int {.inline, compilerProc.} =
if a == b: return 0
if a == nil: return -1
if b == nil: return 1
when defined(nimNoArrayToCstringConversion):
return c_strcmp(addr a.data, addr b.data)
else:
return c_strcmp(a.data, b.data)
let minlen = min(a.len, b.len)
result = c_memcmp(addr a.data, addr b.data, minlen.csize)
if result == 0:
result = a.len - b.len
proc eqStrings(a, b: NimString): bool {.inline, compilerProc.} =
if a == b: return true
@ -386,8 +386,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
kdigits, fdigits = 0
exponent: int
integer: uint64
fraction: uint64
frac_exponent= 0
frac_exponent = 0
exp_sign = 1
first_digit = -1
has_sign = false
@ -480,7 +479,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
# if integer is representable in 53 bits: fast path
# max fast path integer is 1<<53 - 1 or 8999999999999999 (16 digits)
if kdigits + fdigits <= 16 and first_digit <= 8:
let digits = kdigits + fdigits
if digits <= 15 or (digits <= 16 and first_digit <= 8):
# max float power of ten with set bits above the 53th bit is 10^22
if abs_exponent <= 22:
if exp_negative:
@ -504,6 +504,7 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
result = i - start
i = start
# re-parse without error checking, any error should be handled by the code above.
if s[i] == '.': i.inc
while s[i] in {'0'..'9','+','-'}:
if ti < maxlen:
t[ti] = s[i]; inc(ti)

2
lib/windows/winlean.nim
View file

@ -111,6 +111,7 @@ const
WAIT_TIMEOUT* = 0x00000102'i32
WAIT_FAILED* = 0xFFFFFFFF'i32
INFINITE* = -1'i32
STILL_ACTIVE* = 0x00000103'i32
STD_INPUT_HANDLE* = -10'i32
STD_OUTPUT_HANDLE* = -11'i32
@ -685,6 +686,7 @@ const
ERROR_FILE_NOT_FOUND* = 2
ERROR_PATH_NOT_FOUND* = 3
ERROR_ACCESS_DENIED* = 5
ERROR_NO_MORE_FILES* = 18
ERROR_HANDLE_EOF* = 38
ERROR_BAD_ARGUMENTS* = 165

105
lib/wrappers/openssl.nim
View file

@ -8,6 +8,17 @@
#
## OpenSSL support
##
## When OpenSSL is dynamically linked, the wrapper provides partial forward and backward
## compatibility for OpenSSL versions above and below 1.1.0
##
## OpenSSL can be also statically linked using dynlibOverride:ssl for OpenSSL >= 1.1.0
##
## Build and test examples:
##
## .. code-block::
## ./bin/nim c -d:ssl -p:. -r tests/untestable/tssl.nim
## ./bin/nim c -d:ssl -p:. --dynlibOverride:ssl --passL:-lcrypto --passL:-lssl -r tests/untestable/tssl.nim
{.deadCodeElim: on.}
@ -25,8 +36,8 @@ when useWinVersion:
from winlean import SocketHandle
else:
const
versions = "(|.38|.39|.41|.43|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8)"
const versions = "(.1.1|.38|.39|.41|.43|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)"
when defined(macosx):
const
DLLSSLName = "libssl" & versions & ".dylib"
@ -64,6 +75,8 @@ type
des_key_schedule* = array[1..16, des_ks_struct]
pem_password_cb* = proc(buf: cstring, size, rwflag: cint, userdata: pointer): cint {.cdecl.}
{.deprecated: [PSSL: SslPtr, PSSL_CTX: SslCtx, PBIO: BIO].}
const
@ -138,6 +151,7 @@ const
SSL_OP_NO_SSLv2* = 0x01000000
SSL_OP_NO_SSLv3* = 0x02000000
SSL_OP_NO_TLSv1* = 0x04000000
SSL_OP_NO_TLSv1_1* = 0x08000000
SSL_OP_ALL* = 0x000FFFFF
SSL_VERIFY_NONE* = 0x00000000
SSL_VERIFY_PEER* = 0x00000001
@ -189,16 +203,39 @@ const
proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
when compileOption("dynlibOverride", "ssl"):
proc SSL_library_init*(): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.}
proc SSL_load_error_strings*() {.cdecl, dynlib: DLLSSLName, importc.}
proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
# TLS_method(), TLS_server_method(), TLS_client_method() are introduced in 1.1.0
# and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2
# SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0
proc SSLv23_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
when compileOption("dynlibOverride", "ssl"):
# Static linking
proc OPENSSL_init_ssl*(opts: uint64, settings: uint8): cint {.cdecl, dynlib: DLLSSLName, importc, discardable.}
proc SSL_library_init*(): cint {.discardable.} =
## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0
return OPENSSL_init_ssl(0.uint64, 0.uint8)
proc TLS_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
proc SSLv23_method*(): PSSL_METHOD =
TLS_method()
proc SSLv23_client_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
proc SSLv2_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
proc SSLv3_method*(): PSSL_METHOD {.cdecl, dynlib: DLLSSLName, importc.}
template OpenSSL_add_all_algorithms*() = discard
proc OpenSSL_version_num(): culong {.cdecl, dynlib: DLLSSLName, importc.}
proc getOpenSSLVersion*(): culong =
## Return OpenSSL version as unsigned long
OpenSSL_version_num()
proc SSL_load_error_strings*() =
## Removed from OpenSSL 1.1.0
# This proc prevents breaking existing code calling SslLoadErrorStrings
# Static linking against OpenSSL < 1.1.0 is not supported
discard
else:
# Here we're trying to stay compatible with openssl 1.0.* and 1.1.*. Some
# symbols are loaded dynamically and we don't use them if not found.
@ -221,38 +258,58 @@ else:
if not dl.isNil:
result = symAddr(dl, name)
proc loadPSSLMethod(method1, method2: string): PSSL_METHOD =
## Load <method1> from OpenSSL if available, otherwise <method2>
let m1 = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method1))
if not m1.isNil:
return m1()
cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method2))()
proc SSL_library_init*(): cint {.discardable.} =
let theProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init"))
if not theProc.isNil: result = theProc()
## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise
## SSL_library_init
let theProc = cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](sslSym("OPENSSL_init_ssl"))
if not theProc.isNil:
return theProc(0, 0)
let olderProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init"))
if not olderProc.isNil: result = olderProc()
proc SSL_load_error_strings*() =
let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings"))
if not theProc.isNil: theProc()
proc SSLv23_client_method*(): PSSL_METHOD =
let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_client_method"))
if not theProc.isNil: result = theProc()
else: result = TLSv1_method()
loadPSSLMethod("SSLv23_client_method", "TLS_client_method")
proc SSLv23_method*(): PSSL_METHOD =
let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv23_method"))
if not theProc.isNil: result = theProc()
else: result = TLSv1_method()
loadPSSLMethod("SSLv23_method", "TLS_method")
proc SSLv2_method*(): PSSL_METHOD =
let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv2_method"))
if not theProc.isNil: result = theProc()
else: result = TLSv1_method()
loadPSSLMethod("SSLv2_method", "TLS_method")
proc SSLv3_method*(): PSSL_METHOD =
let theProc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym("SSLv3_method"))
if not theProc.isNil: result = theProc()
else: result = TLSv1_method()
loadPSSLMethod("SSLv3_method", "TLS_method")
proc TLS_method*(): PSSL_METHOD =
loadPSSLMethod("TLS_method", "SSLv23_method")
proc TLS_client_method*(): PSSL_METHOD =
loadPSSLMethod("TLS_client_method", "SSLv23_client_method")
proc TLS_server_method*(): PSSL_METHOD =
loadPSSLMethod("TLS_server_method", "SSLv23_server_method")
proc OpenSSL_add_all_algorithms*() =
let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf"))
if not theProc.isNil: theProc()
proc getOpenSSLVersion*(): culong =
## Return OpenSSL version as unsigned long or 0 if not available
let theProc = cast[proc(): culong {.cdecl.}](sslSym("OpenSSL_version_num"))
result =
if theProc.isNil: 0.culong
else: theProc()
proc ERR_load_BIO_strings*(){.cdecl, dynlib: DLLUtilName, importc.}
proc SSL_new*(context: SslCtx): SslPtr{.cdecl, dynlib: DLLSSLName, importc.}
@ -432,6 +489,12 @@ proc ErrClearError*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_clear_error".}
proc ErrFreeStrings*(){.cdecl, dynlib: DLLUtilName, importc: "ERR_free_strings".}
proc ErrRemoveState*(pid: cInt){.cdecl, dynlib: DLLUtilName, importc: "ERR_remove_state".}
proc PEM_read_bio_RSA_PUBKEY*(bp: BIO, x: ptr PRSA, pw: pem_password_cb, u: pointer): PRSA {.cdecl,
dynlib: DLLSSLName, importc.}
proc RSA_verify*(kind: cint, origMsg: pointer, origMsgLen: cuint, signature: pointer,
signatureLen: cuint, rsa: PRSA): cint {.cdecl, dynlib: DLLSSLName, importc.}
when true:
discard
else: