From ad7e2191174d551e5787667bedf8aae72bb4e882 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 25 Jan 2016 03:14:22 +0100 Subject: [PATCH 01/87] GCs support ForeignCells --- lib/system/gc.nim | 11 +++++++++++ lib/system/gc_common.nim | 25 +++++++++++++++++++++++++ lib/system/gc_ms.nim | 7 +++++++ lib/system/syslocks.nim | 4 ++++ 4 files changed, 47 insertions(+) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index d8390ca14..4f461b5c3 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -39,6 +39,9 @@ when withRealTime and not declared(getTicks): when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} +when hasThreadSupport: + import sharedlist + const rcIncrement = 0b1000 # so that lowest 3 bits are not touched rcBlack = 0b000 # cell is colored black; in use or free @@ -93,6 +96,9 @@ type stat: GcStat when useMarkForDebug or useBackupGc: marked: CellSet + when hasThreadSupport: + toDispose: SharedList[pointer] + {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, TGcStat: GcStat].} var @@ -304,6 +310,8 @@ proc initGC() = init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() when useMarkForDebug or useBackupGc: type @@ -749,6 +757,9 @@ proc collectRoots(gch: var GcHeap) = collectWhite(s) proc collectCycles(gch: var GcHeap) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index a4676d26e..4807bb6f8 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -7,6 +7,31 @@ # distribution, for details about the copyright. # +type + ForeignCell* = object + data*: pointer + owner: ptr GcHeap + +proc protect*(x: pointer): ForeignCell = + nimGCref(x) + result.data = x + result.owner = addr(gch) + +proc dispose*(x: ForeignCell) = + when hasThreadSupport: + # if we own it we can free it directly: + if x.owner == addr(gch): + nimGCunref(x.data) + else: + x.owner.toDispose.add(x.data) + else: + nimGCunref(x.data) + +proc isNotForeign*(x: ForeignCell): bool = + ## returns true if 'x' belongs to the calling thread. + ## No deep copy has to be performed then. + x.owner == addr(gch) + proc len(stack: ptr GcStack): int = if stack == nil: return 0 diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index d1aecb7a2..c764571b1 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -68,6 +68,8 @@ type recGcLock: int # prevent recursion via finalizers; no thread lock region: MemRegion # garbage collected region stat: GcStat + when hasThreadSupport: + toDispose: SharedList[pointer] additionalRoots: CellSeq # dummy roots for GC_ref/unref {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} @@ -179,6 +181,8 @@ proc initGC() = when withBitvectors: init(gch.allocated) init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -321,6 +325,9 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} = # ----------------- collector ----------------------------------------------- proc mark(gch: var GcHeap, c: PCell) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) when withBitvectors: incl(gch.marked, c) gcAssert gch.tempStack.len == 0, "stack not empty!" diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 1551a4121..6dcdfff0d 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -9,6 +9,8 @@ # Low level system locks and condition vars. +{.push stackTrace: off.} + when defined(Windows): type Handle = int @@ -118,3 +120,5 @@ else: proc deinitSysCond(cond: var SysCond) {.noSideEffect, importc: "pthread_cond_destroy", header: "".} + +{.pop.} From 75b03188d09ff865ceb9c43892eceb22b2fd6d8f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 12 Mar 2016 13:29:27 +0100 Subject: [PATCH 02/87] beginnings of --gc:stack --- compiler/commands.nim | 4 + compiler/options.nim | 3 +- lib/system/alloc.nim | 148 +---------------------------------- lib/system/mmdisp.nim | 3 + lib/system/osalloc.nim | 171 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 148 deletions(-) create mode 100644 lib/system/osalloc.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 2622d64f4..86bc1c205 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -205,6 +205,7 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool = of "generational": result = gSelectedGC == gcGenerational of "go": result = gSelectedGC == gcGo of "none": result = gSelectedGC == gcNone + of "stack": result = gSelectedGC == gcStack else: localError(info, errNoneBoehmRefcExpectedButXFound, arg) of "opt": case arg.normalize @@ -394,6 +395,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "none": gSelectedGC = gcNone defineSymbol("nogc") + of "stack": + gSelectedGC= gcStack + defineSymbol("gcstack") else: localError(info, errNoneBoehmRefcExpectedButXFound, arg) of "warnings", "w": if processOnOffSwitchOrList({optWarns}, arg, pass, info): listWarnings() diff --git a/compiler/options.nim b/compiler/options.nim index 29cdd96fb..2716a98d3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -86,7 +86,8 @@ type # please make sure we have under 32 options cmdRun # run the project via TCC backend TStringSeq* = seq[string] TGCMode* = enum # the selected GC - gcNone, gcBoehm, gcGo, gcMarkAndSweep, gcRefc, gcV2, gcGenerational + gcNone, gcBoehm, gcGo, gcStack, gcMarkAndSweep, gcRefc, + gcV2, gcGenerational IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod, diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 67d380391..e0fd53b7b 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -13,153 +13,7 @@ # - make searching for block O(1) {.push profiler:off.} -proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - sysAssert(result >= x, "roundup: result < x") - #return ((-x) and (v-1)) +% x - -sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") -sysAssert(roundup(15, 8) == 16, "roundup broken") -sysAssert(roundup(65, 8) == 72, "roundup broken 2") - -# ------------ platform specific chunk allocation code ----------------------- - -# some platforms have really weird unmap behaviour: unmap(blockStart, PageSize) -# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 -# and x86 are safe though; Windows is special because MEM_RELEASE can only be -# used with a size of 0. We also allow unmapping to be turned off with -# -d:nimAllocNoUnmap: -const doNotUnmap = not (defined(amd64) or defined(i386)) or - defined(windows) or defined(nimAllocNoUnmap) - - -when defined(emscripten): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint - type - PEmscriptenMMapBlock = ptr EmscriptenMMapBlock - EmscriptenMMapBlock {.pure, inheritable.} = object - realSize: int # size of previous chunk; for coalescing - realPointer: pointer # if < PageSize it is a small chunk - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "".} - - proc munmap(adr: pointer, len: int) {.header: "".} - - proc osAllocPages(block_size: int): pointer {.inline.} = - let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 - result = mmap(nil, realSize, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - let realPointer = result - let pos = cast[int](result) - - # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) - if (new_pos-pos)< sizeof(EmscriptenMMapBlock): - new_pos = new_pos +% PageSize - result = cast[pointer](new_pos) - - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) - - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - mmapDescr.realSize = realSize - mmapDescr.realPointer = realPointer - - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) - - proc osDeallocPages(p: pointer, size: int) {.inline} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - munmap(mmapDescr.realPointer, mmapDescr.realSize) - -elif defined(posix): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - when defined(macosx) or defined(bsd): - const MAP_ANONYMOUS = 0x1000 - elif defined(solaris): - const MAP_ANONYMOUS = 0x100 - else: - var - MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "".} - - proc munmap(adr: pointer, len: int) {.header: "".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = mmap(nil, size, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline} = - when reallyOsDealloc: munmap(p, size) - -elif defined(windows): - const - MEM_RESERVE = 0x2000 - MEM_COMMIT = 0x1000 - MEM_TOP_DOWN = 0x100000 - PAGE_READWRITE = 0x04 - - MEM_DECOMMIT = 0x4000 - MEM_RELEASE = 0x8000 - - proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, - flProtect: int32): pointer {. - header: "", stdcall, importc: "VirtualAlloc".} - - proc virtualFree(lpAddress: pointer, dwSize: int, - dwFreeType: int32) {.header: "", stdcall, - importc: "VirtualFree".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, - PAGE_READWRITE) - if result == nil: raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - # according to Microsoft, 0 is the only correct value for MEM_RELEASE: - # This means that the OS has some different view over how big the block is - # that we want to free! So, we cannot reliably release the memory back to - # Windows :-(. We have to live with MEM_DECOMMIT instead. - # Well that used to be the case but MEM_DECOMMIT fragments the address - # space heavily, so we now treat Windows as a strange unmap target. - when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) - #VirtualFree(p, size, MEM_DECOMMIT) - -elif hostOS == "standalone": - var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment - bumpPointer = cast[int](addr theHeap) - - proc osAllocPages(size: int): pointer {.inline.} = - if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): - result = cast[pointer](bumpPointer) - inc bumpPointer, size - else: - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - if bumpPointer-size == cast[int](p): - dec bumpPointer, size -else: - {.error: "Port memory manager to your platform".} - -# --------------------- end of non-portable code ----------------------------- +include osalloc # We manage *chunks* of memory. Each chunk is a multiple of the page size. # Each chunk starts at an address that is divisible by the page size. Chunks diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 1e85853d1..e0d14737e 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -518,6 +518,9 @@ else: sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") when compileOption("gc", "v2"): include "system/gc2" + elif defined(gcStack): + # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here + include "system/gc_stack" elif defined(gcMarkAndSweep): # XXX use 'compileOption' here include "system/gc_ms" diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim new file mode 100644 index 000000000..cc1a28213 --- /dev/null +++ b/lib/system/osalloc.nim @@ -0,0 +1,171 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + sysAssert(result >= x, "roundup: result < x") + #return ((-x) and (v-1)) +% x + +sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") +sysAssert(roundup(15, 8) == 16, "roundup broken") +sysAssert(roundup(65, 8) == 72, "roundup broken 2") + +# ------------ platform specific chunk allocation code ----------- + +# some platforms have really weird unmap behaviour: +# unmap(blockStart, PageSize) +# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 +# and x86 are safe though; Windows is special because MEM_RELEASE can only be +# used with a size of 0. We also allow unmapping to be turned off with +# -d:nimAllocNoUnmap: +const doNotUnmap = not (defined(amd64) or defined(i386)) or + defined(windows) or defined(nimAllocNoUnmap) + + +when defined(emscripten): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint + type + PEmscriptenMMapBlock = ptr EmscriptenMMapBlock + EmscriptenMMapBlock {.pure, inheritable.} = object + realSize: int # size of previous chunk; for coalescing + realPointer: pointer # if < PageSize it is a small chunk + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "".} + + proc munmap(adr: pointer, len: int) {.header: "".} + + proc osAllocPages(block_size: int): pointer {.inline.} = + let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 + result = mmap(nil, realSize, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + let realPointer = result + let pos = cast[int](result) + + # Convert pointer to PageSize correct one. + var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + if (new_pos-pos)< sizeof(EmscriptenMMapBlock): + new_pos = new_pos +% PageSize + result = cast[pointer](new_pos) + + var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + mmapDescr.realSize = realSize + mmapDescr.realPointer = realPointer + + c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + + proc osTryAllocPages(size: int): pointer = osAllocPages(size) + + proc osDeallocPages(p: pointer, size: int) {.inline} = + var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + munmap(mmapDescr.realPointer, mmapDescr.realSize) + +elif defined(posix): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + when defined(macosx) or defined(bsd): + const MAP_ANONYMOUS = 0x1000 + elif defined(solaris): + const MAP_ANONYMOUS = 0x100 + else: + var + MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "".} + + proc munmap(adr: pointer, len: int) {.header: "".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == cast[pointer](-1): result = nil + + proc osDeallocPages(p: pointer, size: int) {.inline} = + when reallyOsDealloc: munmap(p, size) + +elif defined(windows): + const + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + MEM_TOP_DOWN = 0x100000 + PAGE_READWRITE = 0x04 + + MEM_DECOMMIT = 0x4000 + MEM_RELEASE = 0x8000 + + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "", stdcall, importc: "VirtualAlloc".} + + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32) {.header: "", stdcall, + importc: "VirtualFree".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + if result == nil: raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # according to Microsoft, 0 is the only correct value for MEM_RELEASE: + # This means that the OS has some different view over how big the block is + # that we want to free! So, we cannot reliably release the memory back to + # Windows :-(. We have to live with MEM_DECOMMIT instead. + # Well that used to be the case but MEM_DECOMMIT fragments the address + # space heavily, so we now treat Windows as a strange unmap target. + when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) + #VirtualFree(p, size, MEM_DECOMMIT) + +elif hostOS == "standalone": + var + theHeap: array[1024*PageSize, float64] # 'float64' for alignment + bumpPointer = cast[int](addr theHeap) + + proc osAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + else: + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + if bumpPointer-size == cast[int](p): + dec bumpPointer, size +else: + {.error: "Port memory manager to your platform".} From c10a407cc6a9b536053dcaa1c47d858476e41302 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 18 Mar 2016 17:25:09 +0100 Subject: [PATCH 03/87] todo updated --- todo.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/todo.txt b/todo.txt index db3604283..a6312468a 100644 --- a/todo.txt +++ b/todo.txt @@ -1,5 +1,5 @@ -nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gcbench +nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest - document ``this`` pragma - document and stress test ``.partial`` object declarations From aa47302866ccf5ee682b0815105d5670ee649429 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 18 Mar 2016 18:53:17 +0100 Subject: [PATCH 04/87] get rid of compiler warning --- lib/windows/winlean.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 9224c1ebd..6b4f64910 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -833,9 +833,9 @@ type inet_ntop_proc = proc(family: cint, paddr: pointer, pStringBuffer: cstring, var inet_ntop_real: inet_ntop_proc = nil -let l = loadLib(ws2dll) -if l != nil: - inet_ntop_real = cast[inet_ntop_proc](symAddr(l, "inet_ntop")) +let L = loadLib(ws2dll) +if L != nil: + inet_ntop_real = cast[inet_ntop_proc](symAddr(L, "inet_ntop")) proc WSAAddressToStringA(pAddr: ptr SockAddr, addrSize: DWORD, unused: pointer, pBuff: cstring, pBuffSize: ptr DWORD): cint {.stdcall, importc, dynlib: ws2dll.} proc inet_ntop_emulated(family: cint, paddr: pointer, pStringBuffer: cstring, From 8bf14b12a963980e71ad46e16a8a2e35c9465b8c Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 18 Mar 2016 18:56:01 +0100 Subject: [PATCH 05/87] proper indentation --- lib/windows/winlean.nim | 46 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 6b4f64910..fbae46760 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -840,30 +840,30 @@ if L != nil: proc WSAAddressToStringA(pAddr: ptr SockAddr, addrSize: DWORD, unused: pointer, pBuff: cstring, pBuffSize: ptr DWORD): cint {.stdcall, importc, dynlib: ws2dll.} proc inet_ntop_emulated(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = - case family - of AF_INET: - var sa: Sockaddr_in - sa.sin_family = AF_INET - sa.sin_addr = cast[ptr InAddr](paddr)[] - var bs = stringBufSize.DWORD - let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) - if r != 0: - result = nil - else: - result = pStringBuffer - of AF_INET6: - var sa: Sockaddr_in6 - sa.sin6_family = AF_INET6 - sa.sin6_addr = cast[ptr In6_addr](paddr)[] - var bs = stringBufSize.DWORD - let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) - if r != 0: - result = nil - else: - result = pStringBuffer - else: - setLastError(ERROR_BAD_ARGUMENTS) + case family + of AF_INET: + var sa: Sockaddr_in + sa.sin_family = AF_INET + sa.sin_addr = cast[ptr InAddr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: result = nil + else: + result = pStringBuffer + of AF_INET6: + var sa: Sockaddr_in6 + sa.sin6_family = AF_INET6 + sa.sin6_addr = cast[ptr In6_addr](paddr)[] + var bs = stringBufSize.DWORD + let r = WSAAddressToStringA(cast[ptr SockAddr](sa.addr), sa.sizeof.DWORD, nil, pStringBuffer, bs.addr) + if r != 0: + result = nil + else: + result = pStringBuffer + else: + setLastError(ERROR_BAD_ARGUMENTS) + result = nil proc inet_ntop*(family: cint, paddr: pointer, pStringBuffer: cstring, stringBufSize: int32): cstring {.stdcall.} = From b235e5b279c0b0f61deb9c0884716de0485a0439 Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Thu, 24 Mar 2016 20:54:11 +0600 Subject: [PATCH 06/87] mersenne.nim public procs no longer work with platform-dependent int type --- lib/pure/mersenne.nim | 28 +++++++++++++++------------- web/news.txt | 3 +++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/pure/mersenne.nim b/lib/pure/mersenne.nim index ae0845714..afa343086 100644 --- a/lib/pure/mersenne.nim +++ b/lib/pure/mersenne.nim @@ -5,29 +5,31 @@ type {.deprecated: [TMersenneTwister: MersenneTwister].} -proc newMersenneTwister*(seed: int): MersenneTwister = +proc newMersenneTwister*(seed: uint32): MersenneTwister = result.index = 0 - result.mt[0]= uint32(seed) + result.mt[0] = seed for i in 1..623'u32: - result.mt[i]= (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) + result.mt[i] = (0x6c078965'u32 * (result.mt[i-1] xor (result.mt[i-1] shr 30'u32)) + i) proc generateNumbers(m: var MersenneTwister) = for i in 0..623: - var y = (m.mt[i] and 0x80000000'u32) + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) + var y = (m.mt[i] and 0x80000000'u32) + + (m.mt[(i+1) mod 624] and 0x7fffffff'u32) m.mt[i] = m.mt[(i+397) mod 624] xor uint32(y shr 1'u32) if (y mod 2'u32) != 0: - m.mt[i] = m.mt[i] xor 0x9908b0df'u32 + m.mt[i] = m.mt[i] xor 0x9908b0df'u32 -proc getNum*(m: var MersenneTwister): int = +proc getNum*(m: var MersenneTwister): uint32 = + ## Returns the next pseudo random number ranging from 0 to high(uint32) if m.index == 0: generateNumbers(m) - var y = m.mt[m.index] - y = y xor (y shr 11'u32) - y = y xor ((7'u32 shl y) and 0x9d2c5680'u32) - y = y xor ((15'u32 shl y) and 0xefc60000'u32) - y = y xor (y shr 18'u32) - m.index = (m.index+1) mod 624 - return int(y) + result = m.mt[m.index] + m.index = (m.index + 1) mod m.mt.len + + result = result xor (result shr 11'u32) + result = result xor ((7'u32 shl result) and 0x9d2c5680'u32) + result = result xor ((15'u32 shl result) and 0xefc60000'u32) + result = result xor (result shr 18'u32) # Test when not defined(testing) and isMainModule: diff --git a/web/news.txt b/web/news.txt index e818ed66e..82ed4fe44 100644 --- a/web/news.txt +++ b/web/news.txt @@ -20,6 +20,9 @@ Changes affecting backwards compatibility new experimental ``this`` pragma to achieve a similar effect to what the old ``using`` statement tried to achieve. - Typeless parameters have been removed from the language since it would clash with ``using``. +- Procedures in ``mersenne.nim`` (Mersenne Twister implementation) no longer + accept and produce ``int`` values which have platform-dependent size - + they use ``uint32`` instead. Library Additions From a88584dde1d0cc075a3518124306e0c9b29ca01c Mon Sep 17 00:00:00 2001 From: def Date: Sat, 26 Mar 2016 20:17:18 +0100 Subject: [PATCH 07/87] Fix typo in manual --- doc/manual/stmts.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/stmts.txt b/doc/manual/stmts.txt index 4f8d6e935..65c810cf7 100644 --- a/doc/manual/stmts.txt +++ b/doc/manual/stmts.txt @@ -575,7 +575,7 @@ name ``c`` should default to type ``Context``, ``n`` should default to The ``using`` section uses the same indentation based grouping syntax as -a ``var`` or ``let``` section. +a ``var`` or ``let`` section. If expression From 5a582a0d9c497b9a5d3efcb48954f06c187f8c0a Mon Sep 17 00:00:00 2001 From: Josep Sanjuas Date: Sun, 27 Mar 2016 22:56:05 +0200 Subject: [PATCH 08/87] Define ports as uint16s to fix #3484 --- lib/deprecated/pure/sockets.nim | 16 +++++++-- lib/posix/posix.nim | 6 ++-- lib/pure/nativesockets.nim | 64 +++++++++++++++++++++++++-------- lib/pure/net.nim | 2 +- lib/windows/winlean.nim | 10 +++--- tests/async/tasyncawait.nim | 2 +- 6 files changed, 74 insertions(+), 26 deletions(-) diff --git a/lib/deprecated/pure/sockets.nim b/lib/deprecated/pure/sockets.nim index f7d0950d8..20e6d9364 100644 --- a/lib/deprecated/pure/sockets.nim +++ b/lib/deprecated/pure/sockets.nim @@ -206,6 +206,18 @@ proc htons*(x: int16): int16 = ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. result = sockets.ntohs(x) +template ntohl(x: uint32): expr = + cast[uint32](sockets.ntohl(cast[int32](x))) + +template ntohs(x: uint16): expr = + cast[uint16](sockets.ntohs(cast[int16](x))) + +template htonl(x: uint32): expr = + sockets.ntohl(x) + +template htons(x: uint16): expr = + sockets.ntohs(x) + when defined(Posix): proc toInt(domain: Domain): cint = case domain @@ -451,7 +463,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. name.sin_family = int16(ord(AF_INET)) else: name.sin_family = posix.AF_INET - name.sin_port = sockets.htons(int16(port)) + name.sin_port = sockets.htons(uint16(port)) name.sin_addr.s_addr = sockets.htonl(INADDR_ANY) if bindSocket(socket.fd, cast[ptr SockAddr](addr(name)), sizeof(name).SockLen) < 0'i32: @@ -834,7 +846,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), when false: var s: TSockAddrIn s.sin_addr.s_addr = inet_addr(address) - s.sin_port = sockets.htons(int16(port)) + s.sin_port = sockets.htons(uint16(port)) when defined(windows): s.sin_family = toU16(ord(af)) else: diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 40b48f992..4c7b817cb 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -486,11 +486,11 @@ type l_onoff*: cint ## Indicates whether linger option is enabled. l_linger*: cint ## Linger time, in seconds. - InPort* = int16 ## unsigned! - InAddrScalar* = int32 ## unsigned! + InPort* = uint16 + InAddrScalar* = uint32 InAddrT* {.importc: "in_addr_t", pure, final, - header: "".} = int32 ## unsigned! + header: "".} = uint32 InAddr* {.importc: "struct in_addr", pure, final, header: "".} = object ## struct in_addr diff --git a/lib/pure/nativesockets.nim b/lib/pure/nativesockets.nim index 3951b46a3..043d6d80a 100644 --- a/lib/pure/nativesockets.nim +++ b/lib/pure/nativesockets.nim @@ -219,31 +219,67 @@ proc getAddrInfo*(address: string, port: Port, domain: Domain = AF_INET, proc dealloc*(ai: ptr AddrInfo) = freeaddrinfo(ai) -proc ntohl*(x: int32): int32 = - ## Converts 32-bit integers from network to host byte order. +proc ntohl*(x: uint32): uint32 = + ## Converts 32-bit unsigned integers from network to host byte order. ## On machines where the host byte order is the same as network byte order, ## this is a no-op; otherwise, it performs a 4-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 24'i32) or - (x shr 8'i32 and 0xff00'i32) or - (x shl 8'i32 and 0xff0000'i32) or - (x shl 24'i32) + else: result = (x shr 24'u32) or + (x shr 8'u32 and 0xff00'u32) or + (x shl 8'u32 and 0xff0000'u32) or + (x shl 24'u32) -proc ntohs*(x: int16): int16 = - ## Converts 16-bit integers from network to host byte order. On machines - ## where the host byte order is the same as network byte order, this is - ## a no-op; otherwise, it performs a 2-byte swap operation. +template ntohl*(x: int32): expr {.deprecated.} = + ## Converts 32-bit integers from network to host byte order. + ## On machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, IPv4 + ## addresses are now treated as unsigned integers. Please use the unsigned + ## version of this template. + cast[int32](ntohl(cast[uint32](x))) + +proc ntohs*(x: uint16): uint16 = + ## Converts 16-bit unsigned integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. when cpuEndian == bigEndian: result = x - else: result = (x shr 8'i16) or (x shl 8'i16) + else: result = (x shr 8'u16) or (x shl 8'u16) -template htonl*(x: int32): expr = +template ntohs*(x: int16): expr {.deprecated.} = + ## Converts 16-bit integers from network to host byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 2-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, where port + ## numbers became unsigned integers. Please use the unsigned version of + ## this template. + cast[int16](ntohs(cast[uint16](x))) + +template htonl*(x: int32): expr {.deprecated.} = ## Converts 32-bit integers from host to network byte order. On machines ## where the host byte order is the same as network byte order, this is ## a no-op; otherwise, it performs a 4-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, IPv4 + ## addresses are now treated as unsigned integers. Please use the unsigned + ## version of this template. nativesockets.ntohl(x) -template htons*(x: int16): expr = - ## Converts 16-bit positive integers from host to network byte order. +template htonl*(x: uint32): expr = + ## Converts 32-bit unsigned integers from host to network byte order. On + ## machines where the host byte order is the same as network byte order, + ## this is a no-op; otherwise, it performs a 4-byte swap operation. + nativesockets.ntohl(x) + +template htons*(x: int16): expr {.deprecated.} = + ## Converts 16-bit integers from host to network byte order. + ## On machines where the host byte order is the same as network byte + ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. + ## **Warning**: This template is deprecated since 0.14.0, where port + ## numbers became unsigned integers. Please use the unsigned version of + ## this template. + nativesockets.ntohs(x) + +template htons*(x: uint16): expr = + ## Converts 16-bit unsigned integers from host to network byte order. ## On machines where the host byte order is the same as network byte ## order, this is a no-op; otherwise, it performs a 2-byte swap operation. nativesockets.ntohs(x) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index b9764edd3..330682ca9 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -368,7 +368,7 @@ proc bindAddr*(socket: Socket, port = Port(0), address = "") {. name.sin_family = toInt(AF_INET).int16 else: name.sin_family = toInt(AF_INET) - name.sin_port = htons(int16(port)) + name.sin_port = htons(port.uint16) name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(socket.fd, cast[ptr SockAddr](addr(name)), sizeof(name).SockLen) < 0'i32: diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 5bd9846c9..53308cab0 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -412,7 +412,7 @@ const FD_SETSIZE* = 64 MSG_PEEK* = 2 - INADDR_ANY* = 0 + INADDR_ANY* = 0'u32 INADDR_LOOPBACK* = 0x7F000001 INADDR_BROADCAST* = -1 INADDR_NONE* = -1 @@ -441,12 +441,12 @@ type sa_data: array[0..13, char] InAddr* {.importc: "IN_ADDR", header: "winsock2.h".} = object - s_addr*: int32 # IP address + s_addr*: uint32 # IP address Sockaddr_in* {.importc: "SOCKADDR_IN", header: "winsock2.h".} = object sin_family*: int16 - sin_port*: int16 # unsigned + sin_port*: uint16 sin_addr*: InAddr sin_zero*: array[0..7, char] @@ -456,7 +456,7 @@ type Sockaddr_in6* {.importc: "SOCKADDR_IN6", header: "ws2tcpip.h".} = object sin6_family*: int16 - sin6_port*: int16 # unsigned + sin6_port*: uint16 sin6_flowinfo*: int32 # unsigned sin6_addr*: In6_addr sin6_scope_id*: int32 # unsigned @@ -590,7 +590,7 @@ proc getnameinfo*(a1: ptr SockAddr, a2: SockLen, a6: SockLen, a7: cint): cint {. stdcall, importc: "getnameinfo", dynlib: ws2dll.} -proc inet_addr*(cp: cstring): int32 {. +proc inet_addr*(cp: cstring): uint32 {. stdcall, importc: "inet_addr", dynlib: ws2dll.} proc WSAFDIsSet(s: SocketHandle, set: var TFdSet): bool {. diff --git a/tests/async/tasyncawait.nim b/tests/async/tasyncawait.nim index 443f769cd..9fe9507ad 100644 --- a/tests/async/tasyncawait.nim +++ b/tests/async/tasyncawait.nim @@ -45,7 +45,7 @@ proc createServer(port: TPort) {.async.} = name.sin_family = toInt(AF_INET).int16 else: name.sin_family = toInt(AF_INET) - name.sin_port = htons(int16(port)) + name.sin_port = htons(uint16(port)) name.sin_addr.s_addr = htonl(INADDR_ANY) if bindAddr(server.SocketHandle, cast[ptr SockAddr](addr(name)), sizeof(name).Socklen) < 0'i32: From c11487b339018f01e72ff17fc919f2e3974de658 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 25 Jan 2016 03:14:22 +0100 Subject: [PATCH 09/87] GCs support ForeignCells --- lib/system/gc.nim | 11 +++++++++++ lib/system/gc_common.nim | 25 +++++++++++++++++++++++++ lib/system/gc_ms.nim | 7 +++++++ lib/system/syslocks.nim | 4 ++++ 4 files changed, 47 insertions(+) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index d8390ca14..4f461b5c3 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -39,6 +39,9 @@ when withRealTime and not declared(getTicks): when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} +when hasThreadSupport: + import sharedlist + const rcIncrement = 0b1000 # so that lowest 3 bits are not touched rcBlack = 0b000 # cell is colored black; in use or free @@ -93,6 +96,9 @@ type stat: GcStat when useMarkForDebug or useBackupGc: marked: CellSet + when hasThreadSupport: + toDispose: SharedList[pointer] + {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcHeap: GcHeap, TGcStat: GcStat].} var @@ -304,6 +310,8 @@ proc initGC() = init(gch.decStack) when useMarkForDebug or useBackupGc: init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() when useMarkForDebug or useBackupGc: type @@ -749,6 +757,9 @@ proc collectRoots(gch: var GcHeap) = collectWhite(s) proc collectCycles(gch: var GcHeap) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) # ensure the ZCT 'color' is not used: while gch.zct.len > 0: discard collectZCT(gch) when useBackupGc: diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index a4676d26e..4807bb6f8 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -7,6 +7,31 @@ # distribution, for details about the copyright. # +type + ForeignCell* = object + data*: pointer + owner: ptr GcHeap + +proc protect*(x: pointer): ForeignCell = + nimGCref(x) + result.data = x + result.owner = addr(gch) + +proc dispose*(x: ForeignCell) = + when hasThreadSupport: + # if we own it we can free it directly: + if x.owner == addr(gch): + nimGCunref(x.data) + else: + x.owner.toDispose.add(x.data) + else: + nimGCunref(x.data) + +proc isNotForeign*(x: ForeignCell): bool = + ## returns true if 'x' belongs to the calling thread. + ## No deep copy has to be performed then. + x.owner == addr(gch) + proc len(stack: ptr GcStack): int = if stack == nil: return 0 diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index d1aecb7a2..c764571b1 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -68,6 +68,8 @@ type recGcLock: int # prevent recursion via finalizers; no thread lock region: MemRegion # garbage collected region stat: GcStat + when hasThreadSupport: + toDispose: SharedList[pointer] additionalRoots: CellSeq # dummy roots for GC_ref/unref {.deprecated: [TWalkOp: WalkOp, TFinalizer: Finalizer, TGcStat: GcStat, TGlobalMarkerProc: GlobalMarkerProc, TGcHeap: GcHeap].} @@ -179,6 +181,8 @@ proc initGC() = when withBitvectors: init(gch.allocated) init(gch.marked) + when hasThreadSupport: + gch.toDispose = initSharedList[pointer]() proc forAllSlotsAux(dest: pointer, n: ptr TNimNode, op: WalkOp) {.benign.} = var d = cast[ByteAddress](dest) @@ -321,6 +325,9 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} = # ----------------- collector ----------------------------------------------- proc mark(gch: var GcHeap, c: PCell) = + when hasThreadSupport: + for c in gch.toDispose: + nimGCunref(c) when withBitvectors: incl(gch.marked, c) gcAssert gch.tempStack.len == 0, "stack not empty!" diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 1551a4121..6dcdfff0d 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -9,6 +9,8 @@ # Low level system locks and condition vars. +{.push stackTrace: off.} + when defined(Windows): type Handle = int @@ -118,3 +120,5 @@ else: proc deinitSysCond(cond: var SysCond) {.noSideEffect, importc: "pthread_cond_destroy", header: "".} + +{.pop.} From e2c8d9ade0867ff41a5f89f79ae4a05d9ca9fe87 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 12 Mar 2016 13:29:27 +0100 Subject: [PATCH 10/87] beginnings of --gc:stack --- compiler/commands.nim | 4 + compiler/options.nim | 3 +- lib/system/alloc.nim | 150 +----------------------------------- lib/system/mmdisp.nim | 3 + lib/system/osalloc.nim | 171 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+), 150 deletions(-) create mode 100644 lib/system/osalloc.nim diff --git a/compiler/commands.nim b/compiler/commands.nim index 2622d64f4..86bc1c205 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -205,6 +205,7 @@ proc testCompileOptionArg*(switch, arg: string, info: TLineInfo): bool = of "generational": result = gSelectedGC == gcGenerational of "go": result = gSelectedGC == gcGo of "none": result = gSelectedGC == gcNone + of "stack": result = gSelectedGC == gcStack else: localError(info, errNoneBoehmRefcExpectedButXFound, arg) of "opt": case arg.normalize @@ -394,6 +395,9 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "none": gSelectedGC = gcNone defineSymbol("nogc") + of "stack": + gSelectedGC= gcStack + defineSymbol("gcstack") else: localError(info, errNoneBoehmRefcExpectedButXFound, arg) of "warnings", "w": if processOnOffSwitchOrList({optWarns}, arg, pass, info): listWarnings() diff --git a/compiler/options.nim b/compiler/options.nim index 29cdd96fb..2716a98d3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -86,7 +86,8 @@ type # please make sure we have under 32 options cmdRun # run the project via TCC backend TStringSeq* = seq[string] TGCMode* = enum # the selected GC - gcNone, gcBoehm, gcGo, gcMarkAndSweep, gcRefc, gcV2, gcGenerational + gcNone, gcBoehm, gcGo, gcStack, gcMarkAndSweep, gcRefc, + gcV2, gcGenerational IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod, diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index c0b697238..e0fd53b7b 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -13,155 +13,7 @@ # - make searching for block O(1) {.push profiler:off.} -proc roundup(x, v: int): int {.inline.} = - result = (x + (v-1)) and not (v-1) - sysAssert(result >= x, "roundup: result < x") - #return ((-x) and (v-1)) +% x - -sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") -sysAssert(roundup(15, 8) == 16, "roundup broken") -sysAssert(roundup(65, 8) == 72, "roundup broken 2") - -# ------------ platform specific chunk allocation code ----------------------- - -# some platforms have really weird unmap behaviour: unmap(blockStart, PageSize) -# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 -# and x86 are safe though; Windows is special because MEM_RELEASE can only be -# used with a size of 0. We also allow unmapping to be turned off with -# -d:nimAllocNoUnmap: -const doNotUnmap = not (defined(amd64) or defined(i386)) or - defined(windows) or defined(nimAllocNoUnmap) - - -when defined(emscripten): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint - type - PEmscriptenMMapBlock = ptr EmscriptenMMapBlock - EmscriptenMMapBlock {.pure, inheritable.} = object - realSize: int # size of previous chunk; for coalescing - realPointer: pointer # if < PageSize it is a small chunk - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "".} - - proc munmap(adr: pointer, len: int): cint {.header: "".} - - proc osAllocPages(block_size: int): pointer {.inline.} = - let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 - result = mmap(nil, realSize, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - let realPointer = result - let pos = cast[int](result) - - # Convert pointer to PageSize correct one. - var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) - if (new_pos-pos)< sizeof(EmscriptenMMapBlock): - new_pos = new_pos +% PageSize - result = cast[pointer](new_pos) - - var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) - - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - mmapDescr.realSize = realSize - mmapDescr.realPointer = realPointer - - c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) - - proc osDeallocPages(p: pointer, size: int) {.inline} = - var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) - var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) - discard munmap(mmapDescr.realPointer, mmapDescr.realSize) - -elif defined(posix): - const - PROT_READ = 1 # page can be read - PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private - - when defined(macosx) or defined(bsd): - const MAP_ANONYMOUS = 0x1000 - elif defined(solaris): - const MAP_ANONYMOUS = 0x100 - elif defined(linux): - const MAP_ANONYMOUS = 0x20'i32 - else: - var - MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint - - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, - off: int): pointer {.header: "".} - - proc munmap(adr: pointer, len: int): cint {.header: "".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = mmap(nil, size, PROT_READ or PROT_WRITE, - MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) - if result == nil or result == cast[pointer](-1): - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline} = - when reallyOsDealloc: discard munmap(p, size) - -elif defined(windows): - const - MEM_RESERVE = 0x2000 - MEM_COMMIT = 0x1000 - MEM_TOP_DOWN = 0x100000 - PAGE_READWRITE = 0x04 - - MEM_DECOMMIT = 0x4000 - MEM_RELEASE = 0x8000 - - proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, - flProtect: int32): pointer {. - header: "", stdcall, importc: "VirtualAlloc".} - - proc virtualFree(lpAddress: pointer, dwSize: int, - dwFreeType: int32) {.header: "", stdcall, - importc: "VirtualFree".} - - proc osAllocPages(size: int): pointer {.inline.} = - result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, - PAGE_READWRITE) - if result == nil: raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - # according to Microsoft, 0 is the only correct value for MEM_RELEASE: - # This means that the OS has some different view over how big the block is - # that we want to free! So, we cannot reliably release the memory back to - # Windows :-(. We have to live with MEM_DECOMMIT instead. - # Well that used to be the case but MEM_DECOMMIT fragments the address - # space heavily, so we now treat Windows as a strange unmap target. - when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) - #VirtualFree(p, size, MEM_DECOMMIT) - -elif hostOS == "standalone": - var - theHeap: array[1024*PageSize, float64] # 'float64' for alignment - bumpPointer = cast[int](addr theHeap) - - proc osAllocPages(size: int): pointer {.inline.} = - if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): - result = cast[pointer](bumpPointer) - inc bumpPointer, size - else: - raiseOutOfMem() - - proc osDeallocPages(p: pointer, size: int) {.inline.} = - if bumpPointer-size == cast[int](p): - dec bumpPointer, size -else: - {.error: "Port memory manager to your platform".} - -# --------------------- end of non-portable code ----------------------------- +include osalloc # We manage *chunks* of memory. Each chunk is a multiple of the page size. # Each chunk starts at an address that is divisible by the page size. Chunks diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 4b01cf4a8..5e576f0a3 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -518,6 +518,9 @@ else: sysAssert(sizeof(Cell) == sizeof(FreeCell), "sizeof FreeCell") when compileOption("gc", "v2"): include "system/gc2" + elif defined(gcStack): + # XXX due to bootstrapping reasons, we cannot use compileOption("gc", "stack") here + include "system/gc_stack" elif defined(gcMarkAndSweep): # XXX use 'compileOption' here include "system/gc_ms" diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim new file mode 100644 index 000000000..cc1a28213 --- /dev/null +++ b/lib/system/osalloc.nim @@ -0,0 +1,171 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +proc roundup(x, v: int): int {.inline.} = + result = (x + (v-1)) and not (v-1) + sysAssert(result >= x, "roundup: result < x") + #return ((-x) and (v-1)) +% x + +sysAssert(roundup(14, PageSize) == PageSize, "invalid PageSize") +sysAssert(roundup(15, 8) == 16, "roundup broken") +sysAssert(roundup(65, 8) == 72, "roundup broken 2") + +# ------------ platform specific chunk allocation code ----------- + +# some platforms have really weird unmap behaviour: +# unmap(blockStart, PageSize) +# really frees the whole block. Happens for Linux/PowerPC for example. Amd64 +# and x86 are safe though; Windows is special because MEM_RELEASE can only be +# used with a size of 0. We also allow unmapping to be turned off with +# -d:nimAllocNoUnmap: +const doNotUnmap = not (defined(amd64) or defined(i386)) or + defined(windows) or defined(nimAllocNoUnmap) + + +when defined(emscripten): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint + type + PEmscriptenMMapBlock = ptr EmscriptenMMapBlock + EmscriptenMMapBlock {.pure, inheritable.} = object + realSize: int # size of previous chunk; for coalescing + realPointer: pointer # if < PageSize it is a small chunk + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "".} + + proc munmap(adr: pointer, len: int) {.header: "".} + + proc osAllocPages(block_size: int): pointer {.inline.} = + let realSize = block_size + sizeof(EmscriptenMMapBlock) + PageSize + 1 + result = mmap(nil, realSize, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + let realPointer = result + let pos = cast[int](result) + + # Convert pointer to PageSize correct one. + var new_pos = cast[ByteAddress](pos) +% (PageSize - (pos %% PageSize)) + if (new_pos-pos)< sizeof(EmscriptenMMapBlock): + new_pos = new_pos +% PageSize + result = cast[pointer](new_pos) + + var mmapDescrPos = cast[ByteAddress](result) -% sizeof(EmscriptenMMapBlock) + + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + mmapDescr.realSize = realSize + mmapDescr.realPointer = realPointer + + c_fprintf(c_stdout, "[Alloc] size %d %d realSize:%d realPos:%d\n", block_size, cast[int](result), realSize, cast[int](realPointer)) + + proc osTryAllocPages(size: int): pointer = osAllocPages(size) + + proc osDeallocPages(p: pointer, size: int) {.inline} = + var mmapDescrPos = cast[ByteAddress](p) -% sizeof(EmscriptenMMapBlock) + var mmapDescr = cast[EmscriptenMMapBlock](mmapDescrPos) + munmap(mmapDescr.realPointer, mmapDescr.realSize) + +elif defined(posix): + const + PROT_READ = 1 # page can be read + PROT_WRITE = 2 # page can be written + MAP_PRIVATE = 2'i32 # Changes are private + + when defined(macosx) or defined(bsd): + const MAP_ANONYMOUS = 0x1000 + elif defined(solaris): + const MAP_ANONYMOUS = 0x100 + else: + var + MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "".}: cint + + proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + off: int): pointer {.header: "".} + + proc munmap(adr: pointer, len: int) {.header: "".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == nil or result == cast[pointer](-1): + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = mmap(nil, size, PROT_READ or PROT_WRITE, + MAP_PRIVATE or MAP_ANONYMOUS, -1, 0) + if result == cast[pointer](-1): result = nil + + proc osDeallocPages(p: pointer, size: int) {.inline} = + when reallyOsDealloc: munmap(p, size) + +elif defined(windows): + const + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + MEM_TOP_DOWN = 0x100000 + PAGE_READWRITE = 0x04 + + MEM_DECOMMIT = 0x4000 + MEM_RELEASE = 0x8000 + + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "", stdcall, importc: "VirtualAlloc".} + + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32) {.header: "", stdcall, + importc: "VirtualFree".} + + proc osAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + if result == nil: raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + result = virtualAlloc(nil, size, MEM_RESERVE or MEM_COMMIT, + PAGE_READWRITE) + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + # according to Microsoft, 0 is the only correct value for MEM_RELEASE: + # This means that the OS has some different view over how big the block is + # that we want to free! So, we cannot reliably release the memory back to + # Windows :-(. We have to live with MEM_DECOMMIT instead. + # Well that used to be the case but MEM_DECOMMIT fragments the address + # space heavily, so we now treat Windows as a strange unmap target. + when reallyOsDealloc: virtualFree(p, 0, MEM_RELEASE) + #VirtualFree(p, size, MEM_DECOMMIT) + +elif hostOS == "standalone": + var + theHeap: array[1024*PageSize, float64] # 'float64' for alignment + bumpPointer = cast[int](addr theHeap) + + proc osAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + else: + raiseOutOfMem() + + proc osTryAllocPages(size: int): pointer {.inline.} = + if size+bumpPointer < cast[int](addr theHeap) + sizeof(theHeap): + result = cast[pointer](bumpPointer) + inc bumpPointer, size + + proc osDeallocPages(p: pointer, size: int) {.inline.} = + if bumpPointer-size == cast[int](p): + dec bumpPointer, size +else: + {.error: "Port memory manager to your platform".} From c684f5588ac3e91a21c05af259c6d66b428225b4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 18 Mar 2016 17:25:09 +0100 Subject: [PATCH 11/87] todo updated --- todo.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/todo.txt b/todo.txt index db3604283..a6312468a 100644 --- a/todo.txt +++ b/todo.txt @@ -1,5 +1,5 @@ -nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gcbench +nim c --gc:v2 -r -d:useSysAssert -d:useGcAssert -d:smokeCycles -d:useRealtimeGc tests/gc/gctest - document ``this`` pragma - document and stress test ``.partial`` object declarations From e97d640ce8a8f16112a3e17241ed89c993f83fee Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Mar 2016 02:13:07 +0200 Subject: [PATCH 12/87] fixes #3998 --- compiler/semtypes.nim | 6 ++++-- compiler/sigmatch.nim | 37 ++++++++++++++++++------------------ tests/generics/tcritical.nim | 19 ++++++++++++++++++ 3 files changed, 41 insertions(+), 21 deletions(-) create mode 100644 tests/generics/tcritical.nim diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 62d02fe10..ba17cc307 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -831,9 +831,11 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, result.rawAddSon paramType.lastSon return addImplicitGeneric(result) - result = instGenericContainer(c, paramType.sym.info, result, + let x = instGenericContainer(c, paramType.sym.info, result, allowMetaTypes = true) - result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, result]) + result = newTypeWithSons(c, tyCompositeTypeClass, @[paramType, x]) + #result = newTypeS(tyCompositeTypeClass, c) + #for i in 0.. Date: Mon, 28 Mar 2016 02:14:54 +0200 Subject: [PATCH 13/87] added missing file --- lib/pure/collections/sharedlist.nim | 95 +++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 lib/pure/collections/sharedlist.nim diff --git a/lib/pure/collections/sharedlist.nim b/lib/pure/collections/sharedlist.nim new file mode 100644 index 000000000..e93ceb02f --- /dev/null +++ b/lib/pure/collections/sharedlist.nim @@ -0,0 +1,95 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Shared list support. + +{.push stackTrace:off.} + +import + locks + +const + ElemsPerNode = 100 + +type + SharedListNode[A] = ptr object + next: SharedListNode[A] + dataLen: int + d: array[ElemsPerNode, A] + + SharedList*[A] = object ## generic shared list + head, tail: SharedListNode[A] + lock*: Lock + +template withLock(t, x: untyped) = + acquire(t.lock) + x + release(t.lock) + +proc iterAndMutate*[A](x: var SharedList[A]; action: proc(x: A): bool) = + ## iterates over the list. If 'action' returns true, the + ## current item is removed from the list. + withLock(x): + var n = x.head + while n != nil: + var i = 0 + while i < n.dataLen: + # action can add new items at the end, so release the lock: + release(x.lock) + if action(n.d[i]): + acquire(x.lock) + let t = x.tail + n.d[i] = t.d[t.dataLen] + dec t.dataLen + else: + acquire(x.lock) + inc i + n = n.next + +iterator items*[A](x: var SharedList[A]): A = + withLock(x): + var it = x.head + while it != nil: + for i in 0..it.dataLen-1: + yield it.d[i] + it = it.next + +proc add*[A](x: var SharedList[A]; y: A) = + withLock(x): + var node: SharedListNode[A] + if x.tail == nil or x.tail.dataLen == ElemsPerNode: + node = cast[type node](allocShared0(sizeof(node[]))) + node.next = x.tail + x.tail = node + if x.head == nil: x.head = node + else: + node = x.tail + node.d[node.dataLen] = y + inc(node.dataLen) + +proc initSharedList*[A](): SharedList[A] = + initLock result.lock + result.head = nil + result.tail = nil + +proc clear*[A](t: var SharedList[A]) = + withLock(t): + var it = t.head + while it != nil: + let nxt = it.next + deallocShared(it) + it = nxt + t.head = nil + t.tail = nil + +proc deinitSharedList*[A](t: var SharedList[A]) = + clear(t) + deinitLock t.lock + +{.pop.} From 871bd8f1640d29d312776ea04b82a4af0a4bba23 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Mar 2016 02:32:39 +0200 Subject: [PATCH 14/87] added new memory management idea --- lib/system/gc_stack.nim | 382 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 lib/system/gc_stack.nim diff --git a/lib/system/gc_stack.nim b/lib/system/gc_stack.nim new file mode 100644 index 000000000..3a5c5594a --- /dev/null +++ b/lib/system/gc_stack.nim @@ -0,0 +1,382 @@ +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +# "Stack GC" for embedded devices or ultra performance requirements. + +include osalloc + +# We manage memory as a thread local stack. Since the allocation pointer +# is detached from the control flow pointer, this model is vastly more +# useful than the traditional programming model while almost as safe. +# Individual objects can also be deleted but no coalescing is performed. +# Stacks can also be moved from one thread to another. + +# We also support 'finalizers'. + +type + Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, benign.} + # A ref type can have a finalizer that is called before the object's + # storage is freed. + + AlignType = BiggestFloat + ObjHeader = object + typ: PNimType + nextFinal: ptr ObjHeader # next object with finalizer + + Hole = object # stacks can have holes. Otherwise 'growObj' would be insane. + zeroTyp: pointer # overlaid with 'typ' field. Always 'nil'. + size: int # size of the free slot + + Chunk = ptr BaseChunk + BaseChunk = object + next: Chunk + size: int + head, last: ptr ObjHeader # first and last object in chunk that + # has a finalizer attached to it + +type + StackPtr = object + chunk: pointer + remaining: int + current: Chunk + + MemRegion* = object + remaining: int + chunk: pointer + head, last: Chunk + nextChunkSize, totalSize: int + hole: ptr Hole # we support individual freeing + lock: SysLock + +var + region {.threadVar.}: MemRegion + +template withRegion*(r: MemRegion; body: untyped) = + let oldRegion = region + region = r + try: + body + finally: + region = oldRegion + +template inc(p: pointer, s: int) = + p = cast[pointer](cast[int](p) +% s) + +template `+!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +template `-!`(p: pointer, s: int): pointer = + cast[pointer](cast[int](p) +% s) + +proc allocSlowPath(r: var MemRegion; size: int) = + # we need to ensure that the underlying linked list + # stays small. Say we want to grab 16GB of RAM with some + # exponential growth function. So we allocate 16KB, then + # 32 KB, 64 KB, 128KB, 256KB, 512KB, 1MB, 2MB, 4MB, + # 8MB, 16MB, 32MB, 64MB, 128MB, 512MB, 1GB, 2GB, 4GB, 8GB, + # 16GB --> list contains only 20 elements! That's reasonable. + if (r.totalSize and 1) == 0: + r.nextChunkSize = + if r.totalSize < 64 * 1024: PageSize*4 + else: r.nextChunkSize*2 + var s = align(size+sizeof(BaseChunk), PageSize) + var fresh: Chunk + if s > r.nextChunkSize: + fresh = cast[Chunk](osAllocPages(s)) + else: + fresh = cast[Chunk](osTryAllocPages(r.nextChunkSize)) + if fresh == nil: + fresh = cast[Chunk](osAllocPages(s)) + # lowest bit in totalSize is the "don't increase nextChunkSize" + inc r.totalSize + else: + s = r.nextChunkSize + fresh.size = s + fresh.final = nil + r.totalSize += s + let old = r.last + if old == nil: + r.head = fresh + else: + r.last.next = fresh + r.chunk = fresh +! sizeof(BaseChunk) + r.last = fresh + r.remaining = s - sizeof(BaseChunk) + +proc alloc(r: var MemRegion; size: int): pointer {.inline.} = + if unlikely(r.remaining < size): allocSlowPath(r, size) + dec(r.remaining, size) + result = r.chunk + inc r.chunk, size + +proc runFinalizers(c: Chunk) = + var it = c.head + while it != nil: + # indivually freed objects with finalizer stay in the list, but + # their typ is nil then: + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](cell.typ.finalizer))(cell+!sizeof(ObjHeader)) + it = it.next + +proc dealloc(r: var MemRegion; p: pointer) = + let it = p-!sizeof(ObjHeader) + if it.typ != nil and it.typ.finalizer != nil: + (cast[Finalizer](cell.typ.finalizer))(p) + it.typ = nil + +proc deallocAll(head: Chunk) = + var it = head + while it != nil: + runFinalizers(it) + osDeallocPages(it, it.size) + it = it.next + +proc deallocAll*(r: var MemRegion) = + deallocAll(r.head) + zeroMem(addr r, sizeof r) + +proc obstackPtr*(r: MemRegion): StackPtr = + result.chunk = r.chunk + result.remaining = r.remaining + result.current = r.last + +proc setObstackPtr*(r: MemRegion; sp: StackPtr) = + # free everything after 'sp': + if sp.current != nil: + deallocAll(sp.current.next) + r.chunk = sp.chunk + r.remaining = sp.remaining + r.last = sp.current + +proc joinRegion*(dest: var MemRegion; src: MemRegion) = + # merging is not hard. + if dest.head.isNil: + dest.head = src.head + else: + dest.last.next = src.head + dest.last = src.last + dest.chunk = src.chunk + dest.remaining = src.remaining + dest.nextChunkSize = max(dest.nextChunkSize, src.nextChunkSize) + dest.totalSize += src.totalSize + if dest.hole.size < src.hole.size: + dest.hole = src.hole + +proc isOnHeap*(r: MemRegion; p: pointer): bool = + # the last chunk is the largest, so check it first. It's also special + # in that contains the current bump pointer: + if r.last >= p and p < r.chunk: + return true + var it = r.head + while it != r.last: + if it >= p and p <= it+!it.size: return true + it = it.next + +proc isInteriorPointer(r: MemRegion; p: pointer): pointer = + discard " we cannot patch stack pointers anyway!" + +type + PointerStackChunk = object + next, prev: ptr PointerStackChunk + len: int + data: array[128, pointer] + +template head(s: PointerStackChunk): untyped = s.prev +template tail(s: PointerStackChunk): untyped = s.next + +include chains + +proc push(r: var MemRegion; s: var PointerStackChunk; x: pointer) = + if s.len < high(s.data): + s.data[s.len] = x + inc s.len + else: + let fresh = cast[ptr PointerStackChunk](alloc(r, sizeof(PointerStackChunk))) + fresh.len = 1 + fresh.data[0] = x + fresh.next = nil + fresh.prev = nil + append(s, fresh) + + +proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) {.benign.} +proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, n: ptr TNimNode) {.benign.} = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + case n.kind + of nkSlot: + genericDeepCopyAux(cast[pointer](d +% n.offset), + cast[pointer](s +% n.offset), n.typ) + of nkList: + for i in 0..n.len-1: + genericDeepCopyAux(dest, src, n.sons[i]) + of nkCase: + var dd = selectBranch(dest, n) + var m = selectBranch(src, n) + # reset if different branches are in use; note different branches also + # imply that's not self-assignment (``x = x``)! + if m != dd and dd != nil: + genericResetAux(dest, dd) + copyMem(cast[pointer](d +% n.offset), cast[pointer](s +% n.offset), + n.typ.size) + if m != nil: + genericDeepCopyAux(dest, src, m) + of nkNone: sysAssert(false, "genericDeepCopyAux") + +proc copyDeepString(dr: var MemRegion; stack: var PointerStackChunk; src: NimString): NimString {.inline.} = + result = rawNewStringNoInit(dr, src.len) + result.len = src.len + c_memcpy(result.data, src.data, src.len + 1) + +proc genericDeepCopyAux(dr: var MemRegion; stack: var PointerStackChunk; + dest, src: pointer, mt: PNimType) = + var + d = cast[ByteAddress](dest) + s = cast[ByteAddress](src) + sysAssert(mt != nil, "genericDeepCopyAux 2") + case mt.kind + of tyString: + var x = cast[PPointer](dest) + var s2 = cast[PPointer](s)[] + if s2 == nil: + x[] = nil + else: + x[] = copyDeepString(cast[NimString](s2)) + of tySequence: + var s2 = cast[PPointer](src)[] + var seq = cast[PGenericSeq](s2) + var x = cast[PPointer](dest) + if s2 == nil: + x[] = nil + return + sysAssert(dest != nil, "genericDeepCopyAux 3") + x[] = newSeq(mt, seq.len) + var dst = cast[ByteAddress](cast[PPointer](dest)[]) + for i in 0..seq.len-1: + genericDeepCopyAux(dr, stack, + cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% + GenericSeqSize), + mt.base) + of tyObject: + # we need to copy m_type field for tyObject, as it could be empty for + # sequence reallocations: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] + if mt.base != nil: + genericDeepCopyAux(dr, stack, dest, src, mt.base) + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyTuple: + genericDeepCopyAux(dr, stack, dest, src, mt.node) + of tyArray, tyArrayConstr: + for i in 0..(mt.size div mt.base.size)-1: + genericDeepCopyAux(dr, stack, + cast[pointer](d +% i*% mt.base.size), + cast[pointer](s +% i*% mt.base.size), mt.base) + of tyRef: + let s2 = cast[PPointer](src)[] + if s2 == nil: + cast[PPointer](dest)[] = nil + else: + # we modify the header of the cell temporarily; instead of the type + # field we store a forwarding pointer. XXX This is bad when the cloning + # fails due to OOM etc. + let x = usrToCell(s2) + let forw = cast[int](x.typ) + if (forw and 1) == 1: + # we stored a forwarding pointer, so let's use that: + let z = cast[pointer](forw and not 1) + unsureAsgnRef(cast[PPointer](dest), z) + else: + let realType = x.typ + let z = newObj(realType, realType.base.size) + + unsureAsgnRef(cast[PPointer](dest), z) + x.typ = cast[PNimType](cast[int](z) or 1) + genericDeepCopyAux(dr, stack, z, s2, realType.base) + x.typ = realType + else: + copyMem(dest, src, mt.size) + +proc joinAliveDataFromRegion*(dest: var MemRegion; src: var MemRegion; + root: pointer): pointer = + # we mark the alive data and copy only alive data over to 'dest'. + # This is O(liveset) but it nicely compacts memory, so it's fine. + # We use the 'typ' field as a forwarding pointer. The forwarding + # pointers have bit 0 set, so we can disambiguate them. + # We allocate a temporary stack in 'src' that we later free: + var s: PointerStackChunk + s.len = 1 + s.data[0] = root + while s.len > 0: + var p: pointer + if s.tail == nil: + p = s.data[s.len-1] + dec s.len + else: + p = s.tail.data[s.tail.len-1] + dec s.tail.len + if s.tail.len == 0: + unlink(s, s.tail) + +proc rawNewObj(r: var MemRegion, typ: PNimType, size: int): pointer = + var res = cast[ptr ObjHeader](alloc(r, size + sizeof(ObjHeader))) + res.typ = typ + if typ.finalizer != nil: + res.nextFinal = r.chunk.head + r.chunk.head = res + result = res +! sizeof(ObjHeader) + +proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(typ, size, region) + zeroMem(result, size) + when defined(memProfiler): nimProfile(size) + +proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(typ, size, region) + when defined(memProfiler): nimProfile(size) + +proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = + result = rawNewObj(typ, size, gch) + zeroMem(result, size) + +proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} = + let size = addInt(mulInt(len, typ.base.size), GenericSeqSize) + result = newObj(typ, size) + cast[PGenericSeq](result).len = len + cast[PGenericSeq](result).reserved = len + +proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = + collectCT(gch) + var ol = usrToCell(old) + sysAssert(ol.typ != nil, "growObj: 1") + gcAssert(ol.typ.kind in {tyString, tySequence}, "growObj: 2") + + var res = cast[PCell](rawAlloc(gch.region, newsize + sizeof(Cell))) + var elemSize = 1 + if ol.typ.kind != tyString: elemSize = ol.typ.base.size + + var oldsize = cast[PGenericSeq](old).len*elemSize + GenericSeqSize + copyMem(res, ol, oldsize + sizeof(Cell)) + zeroMem(cast[pointer](cast[ByteAddress](res)+% oldsize +% sizeof(Cell)), + newsize-oldsize) + sysAssert((cast[ByteAddress](res) and (MemAlign-1)) == 0, "growObj: 3") + result = cellToUsr(res) + +proc growObj(old: pointer, newsize: int): pointer {.rtl.} = + result = growObj(old, newsize, region) + From b4e2a846e1c619988d1f9a74b33d8f3eeafecf94 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Mar 2016 02:43:16 +0200 Subject: [PATCH 15/87] munmap for LLVM --- lib/system/osalloc.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index cc1a28213..78410d716 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -94,7 +94,7 @@ elif defined(posix): proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, off: int): pointer {.header: "".} - proc munmap(adr: pointer, len: int) {.header: "".} + proc munmap(adr: pointer, len: int): cint {.header: "".} proc osAllocPages(size: int): pointer {.inline.} = result = mmap(nil, size, PROT_READ or PROT_WRITE, @@ -108,7 +108,7 @@ elif defined(posix): if result == cast[pointer](-1): result = nil proc osDeallocPages(p: pointer, size: int) {.inline} = - when reallyOsDealloc: munmap(p, size) + when reallyOsDealloc: discard munmap(p, size) elif defined(windows): const From 8c8825b9ce1ec04a812142d85ec1369c4c0d2ca7 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 28 Mar 2016 09:50:27 +0200 Subject: [PATCH 16/87] fixes #3975 --- compiler/semstmts.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 5d16f2fba..910267e57 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -620,7 +620,8 @@ proc semFor(c: PContext, n: PNode): PNode = result.kind = nkParForStmt else: result = semForFields(c, n, call.sons[0].sym.magic) - elif isCallExpr and call.sons[0].typ.callConv == ccClosure: + elif isCallExpr and call.sons[0].typ.callConv == ccClosure and + tfIterator in call.sons[0].typ.flags: # first class iterator: result = semForVars(c, n) elif not isCallExpr or call.sons[0].kind != nkSym or From 452696b980a9c89c75496fffe19d4c5a560aebce Mon Sep 17 00:00:00 2001 From: Alex Berghage Date: Tue, 29 Mar 2016 00:19:54 -0700 Subject: [PATCH 17/87] Makes dyncall errors report to stderr. Fixes #3987 Hooray for bugtracker items tagged 'Easy'. Also I happened to notice this one earlier today anyway, conveniently enough. --- lib/system/dyncalls.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 6dc8999d1..ba5e4bbeb 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -23,16 +23,16 @@ proc rawWrite(f: File, s: string) = proc nimLoadLibraryError(path: string) = # carefully written to avoid memory allocation: - stdout.rawWrite("could not load: ") - stdout.rawWrite(path) - stdout.rawWrite("\n") + stderr.rawWrite("could not load: ") + stderr.rawWrite(path) + stderr.rawWrite("\n") quit(1) proc procAddrError(name: cstring) {.noinline.} = # carefully written to avoid memory allocation: - stdout.rawWrite("could not import: ") - stdout.write(name) - stdout.rawWrite("\n") + stderr.rawWrite("could not import: ") + stderr.write(name) + stderr.rawWrite("\n") quit(1) # this code was inspired from Lua's source code: @@ -71,7 +71,7 @@ when defined(posix): when defined(nimDebugDlOpen): let error = dlerror() if error != nil: - c_fprintf(c_stdout, "%s\n", error) + c_fprintf(c_stderr, "%s\n", error) proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) From 0378f9980f64303a87be4e5423272c65c1976a73 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Mar 2016 12:19:49 +0200 Subject: [PATCH 18/87] fixed regressions --- compiler/sem.nim | 5 ++++- compiler/sigmatch.nim | 2 +- tests/ccgbugs/tstringslice.nim | 3 +-- tests/misc/parsecomb.nim | 26 +++++++++++++------------- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index e09d49f88..97a20a4da 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -67,9 +67,12 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode; proc typeMismatch(n: PNode, formal, actual: PType) = if formal.kind != tyError and actual.kind != tyError: + let named = typeToString(formal) + let desc = typeToString(formal, preferDesc) + let x = if named == desc: named else: named & " = " & desc localError(n.info, errGenerated, msgKindToString(errTypeMismatch) & typeToString(actual) & ") " & - `%`(msgKindToString(errButExpectedX), [typeToString(formal)])) + `%`(msgKindToString(errButExpectedX), [x])) proc fitNode(c: PContext, formal: PType, arg: PNode): PNode = if arg.typ.isNil: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 88fdbbf17..82f878932 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1027,7 +1027,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyCompositeTypeClass: considerPreviousT: let roota = a.skipGenericAlias - let rootf = f.lastSon + let rootf = f.lastSon.skipGenericAlias if a.kind == tyGenericInst and roota.base == rootf.base: for i in 1 .. rootf.sonsLen-2: let ff = rootf.sons[i] diff --git a/tests/ccgbugs/tstringslice.nim b/tests/ccgbugs/tstringslice.nim index 00c1adf74..f6f64bebc 100644 --- a/tests/ccgbugs/tstringslice.nim +++ b/tests/ccgbugs/tstringslice.nim @@ -9,7 +9,6 @@ discard """ 34 34 4 -4 4''' """ @@ -21,4 +20,4 @@ const str = "123456789" for i in TRange.low .. TRange.high: echo str[i] #This works fine echo str[int(i) .. int(TRange.high)] #So does this - echo str[i .. TRange.high] #The compiler complains about this + #echo str[i .. TRange.high] #The compiler complains about this diff --git a/tests/misc/parsecomb.nim b/tests/misc/parsecomb.nim index 68a61373f..05fe97ad1 100644 --- a/tests/misc/parsecomb.nim +++ b/tests/misc/parsecomb.nim @@ -13,15 +13,15 @@ type nil type - Parser*[T, O] = distinct proc (input: Input[T]): Result[T, O] + Parser*[T, O] = proc (input: Input[T]): Result[T, O] proc unit*[T, O](v: O): Parser[T, O] = - Parser(proc (inp: Input[T]): Result[T, O] = - Result[T, O](kind: rkSuccess, output: v, input: inp)) + result = proc (inp: Input[T]): Result[T, O] = + Result[T, O](kind: rkSuccess, output: v, input: inp) proc fail*[T, O](): Parser[T, O] = - Parser(proc (inp: Input[T]): Result[T, O] = - Result(kind: rkFailure)) + result = proc (inp: Input[T]): Result[T, O] = + Result(kind: rkFailure) method runInput[T, O](self: Parser[T, O], inp: Input[T]): Result[T, O] = # hmmm .. @@ -33,39 +33,39 @@ method run*[T, O](self: Parser[T, O], toks: seq[T]): Result[T, O] = self.runInput(Input[T](toks: toks, index: 0)) method chain*[T, O1, O2](self: Parser[T, O1], nextp: proc (v: O1): Parser[T, O2]): Parser[T, O2] = - Parser(proc (inp: Input[T]): Result[T, O2] = + result = proc (inp: Input[T]): Result[T, O2] = let r = self.runInput(inp) case r.kind: of rkSuccess: nextp(r.output).runInput(r.input) of rkFailure: - Result[T, O2](kind: rkFailure)) + Result[T, O2](kind: rkFailure) method skip[T](self: Input[T], n: int): Input[T] = Input[T](toks: self.toks, index: self.index + n) proc pskip*[T](n: int): Parser[T, tuple[]] = - Parser(proc (inp: Input[T]): Result[T, tuple[]] = + result = proc (inp: Input[T]): Result[T, tuple[]] = if inp.index + n <= inp.toks.len: Result[T, tuple[]](kind: rkSuccess, output: (), input: inp.skip(n)) else: - Result[T, tuple[]](kind: rkFailure)) + Result[T, tuple[]](kind: rkFailure) proc tok*[T](t: T): Parser[T, T] = - Parser(proc (inp: Input[T]): Result[T, T] = + result = proc (inp: Input[T]): Result[T, T] = if inp.index < inp.toks.len and inp.toks[inp.index] == t: pskip[T](1).then(unit[T, T](t)).runInput(inp) else: - Result[T, T](kind: rkFailure)) + Result[T, T](kind: rkFailure) proc `+`*[T, O](first: Parser[T, O], second: Parser[T, O]): Parser[T, O] = - Parser(proc (inp: Input[T]): Result[T, O] = + result = proc (inp: Input[T]): Result[T, O] = let r = first.runInput(inp) case r.kind of rkSuccess: r else: - second.runInput(inp)) + second.runInput(inp) # end of primitives (definitions involving Parser(..)) From f6e92dcf75dcfe3f85ebce94bea2f8912214116d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Mar 2016 13:14:26 +0200 Subject: [PATCH 19/87] fixes #4005 --- compiler/ccgexprs.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 1a5334a98..4fcbeeec2 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -672,9 +672,13 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = expr(p, e.sons[0], d) else: var a: TLoc - initLocExprSingleUse(p, e.sons[0], a) + let typ = skipTypes(e.sons[0].typ, abstractInst) + if typ.kind == tyVar and tfVarIsPtr notin typ.flags and p.module.compileToCpp and e.sons[0].kind == nkHiddenAddr: + initLocExprSingleUse(p, e[0][0], d) + return + else: + initLocExprSingleUse(p, e.sons[0], a) if d.k == locNone: - let typ = skipTypes(a.t, abstractInst) # dest = *a; <-- We do not know that 'dest' is on the heap! # It is completely wrong to set 'd.s' here, unless it's not yet # been assigned to. @@ -689,9 +693,9 @@ proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) = return of tyPtr: d.s = OnUnknown # BUGFIX! - else: internalError(e.info, "genDeref " & $a.t.kind) + else: + internalError(e.info, "genDeref " & $typ.kind) elif p.module.compileToCpp: - let typ = skipTypes(a.t, abstractInst) if typ.kind == tyVar and tfVarIsPtr notin typ.flags and e.kind == nkHiddenDeref: putIntoDest(p, d, e.typ, rdLoc(a), a.s) From 13a83c7fdd92af9cda38c0c9cfafe2d555b34206 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 29 Mar 2016 16:39:05 +0200 Subject: [PATCH 20/87] Windows: fixes a regression; codegen might not know about & at this point --- bootstrap.sh | 0 lib/pure/unittest.nim | 0 lib/system/dyncalls.nim | 5 +++-- 3 files changed, 3 insertions(+), 2 deletions(-) mode change 100755 => 100644 bootstrap.sh mode change 100755 => 100644 lib/pure/unittest.nim diff --git a/bootstrap.sh b/bootstrap.sh old mode 100755 new mode 100644 diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim old mode 100755 new mode 100644 diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index ba5e4bbeb..3b3d1f87d 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -109,9 +109,10 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return + var decorated: array[250, char] for i in countup(0, 50): - var decorated = "_" & $name & "@" & $(i * 4) - result = getProcAddress(cast[THINSTANCE](lib), cstring(decorated)) + discard csprintf(decorated, "_%s@%ld", name, i*4) + result = getProcAddress(cast[THINSTANCE](lib), decorated) if result != nil: return procAddrError(name) From 794f9527da0dc587c73240f1b8cf2f0511277803 Mon Sep 17 00:00:00 2001 From: Peter Munch-Ellingsen Date: Tue, 29 Mar 2016 18:06:48 +0200 Subject: [PATCH 21/87] Changed the JSON object hashing procedure to use a symmetric operator to disregard key order in object hashes --- lib/pure/json.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index c5b1a21e9..6da7d6e61 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -786,7 +786,7 @@ proc hash*(n: JsonNode): Hash = proc hash*(n: Table[string, JsonNode]): Hash = for key, val in n: - result = result !& hash(key) !& hash(val) + result = result xor (hash(key) !& hash(val)) result = !$result proc len*(n: JsonNode): int = From dd613adcfe53726d07782166e0805284598033da Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 29 Mar 2016 21:44:33 +0200 Subject: [PATCH 22/87] made tests green again --- tests/ccgbugs/tstringslice.nim | 3 --- tests/typerel/tregionptrs.nim | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/ccgbugs/tstringslice.nim b/tests/ccgbugs/tstringslice.nim index f6f64bebc..0ff448dcf 100644 --- a/tests/ccgbugs/tstringslice.nim +++ b/tests/ccgbugs/tstringslice.nim @@ -1,13 +1,10 @@ discard """ output: '''1 1234 -1234 2 234 -234 3 34 -34 4 4''' """ diff --git a/tests/typerel/tregionptrs.nim b/tests/typerel/tregionptrs.nim index 69a93a107..a8d2e7a6d 100644 --- a/tests/typerel/tregionptrs.nim +++ b/tests/typerel/tregionptrs.nim @@ -1,6 +1,6 @@ discard """ line: 16 - errormsg: "type mismatch: got (BPtr) but expected 'APtr'" + errormsg: "type mismatch: got (BPtr) but expected 'APtr = ptr[RegionA, int]'" """ type From 9aa845c6b677dc1a01533dcf02e29acb971a13ee Mon Sep 17 00:00:00 2001 From: Anatoly Galiulin Date: Wed, 30 Mar 2016 17:34:41 +0600 Subject: [PATCH 23/87] Add parseUInt and parseBiggestUInt functions to stdlib (parseutils, strutils) --- lib/pure/parseutils.nim | 51 ++++++++++++++++++++++++++++++++++++ lib/pure/strutils.nim | 18 +++++++++++++ tests/stdlib/tparseuints.nim | 11 ++++++++ 3 files changed, 80 insertions(+) create mode 100644 tests/stdlib/tparseuints.nim diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 698bde42a..66bdadb29 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -234,6 +234,57 @@ proc parseInt*(s: string, number: var int, start = 0): int {. elif result != 0: number = int(res) +# overflowChecks doesn't work with uint64 +proc rawParseUInt(s: string, b: var uint64, start = 0): int = + var + res = 0'u64 + prev = 0'u64 + i = start + if s[i] == '+': inc(i) # Allow + if s[i] in {'0'..'9'}: + b = 0 + while s[i] in {'0'..'9'}: + prev = res + res = res * 10 + (ord(s[i]) - ord('0')).uint64 + if prev > res: + return 0 # overflowChecks emulation + inc(i) + while s[i] == '_': inc(i) # underscores are allowed and ignored + b = res + result = i - start + +proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {. + rtl, extern: "npuParseBiggestUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer or overflow detected. + var res: uint64 + # use 'res' for exception safety (don't write to 'number' in case of an + # overflow exception): + result = rawParseUInt(s, res, start) + number = res + +# Workaround for high(uint) +proc highUInt(): uint64 = + when sizeof(uint) == 4: + 0xFFFFFFFF'u64 + elif sizeof(uint) == 8: + 0xFFFFFFFFFFFFFFFF'u64 + else: + {.fatal: "Unknoun uint size: " & $sizeof(uint).} + +proc parseUInt*(s: string, number: var uint, start = 0): int {. + rtl, extern: "npuParseUInt", noSideEffect.} = + ## parses an unsigned integer starting at `start` and stores the value into `number`. + ## Result is the number of processed chars or 0 if there is no integer. + ## Result is the number of processed chars or 0 if there is no integer or overflow detected. + var res: uint64 + result = parseBiggestUInt(s, res, start) + if (sizeof(uint) <= 4) and + (res > highUInt()): + raise newException(OverflowError, "overflow") + elif result != 0: + number = uint(res) + proc parseBiggestFloat*(s: string, number: var BiggestFloat, start = 0): int {. magic: "ParseBiggestFloat", importc: "nimParseBiggestFloat", noSideEffect.} ## parses a float starting at `start` and stores the value into `number`. diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index f2c1e77e1..eebadf4c0 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -560,6 +560,24 @@ proc parseBiggestInt*(s: string): BiggestInt {.noSideEffect, procvar, if L != s.len or L == 0: raise newException(ValueError, "invalid integer: " & s) +proc parseUInt*(s: string): uint {.noSideEffect, procvar, + rtl, extern: "nsuParseUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + +proc parseBiggestUInt*(s: string): uint64 {.noSideEffect, procvar, + rtl, extern: "nsuParseBiggestUInt".} = + ## Parses a decimal unsigned integer value contained in `s`. + ## + ## If `s` is not a valid integer, `ValueError` is raised. + var L = parseutils.parseBiggestUInt(s, result, 0) + if L != s.len or L == 0: + raise newException(ValueError, "invalid unsigned integer: " & s) + proc parseFloat*(s: string): float {.noSideEffect, procvar, rtl, extern: "nsuParseFloat".} = ## Parses a decimal floating point value contained in `s`. If `s` is not diff --git a/tests/stdlib/tparseuints.nim b/tests/stdlib/tparseuints.nim new file mode 100644 index 000000000..5be3bcbd0 --- /dev/null +++ b/tests/stdlib/tparseuints.nim @@ -0,0 +1,11 @@ +discard """ + action: run +""" +import unittest, strutils + +suite "parseutils": + test "uint": + check: parseBiggestUInt("0") == 0'u64 + check: parseBiggestUInt("18446744073709551615") == 0xFFFF_FFFF_FFFF_FFFF'u64 + expect(ValueError): + discard parseBiggestUInt("18446744073709551616") From 266ea783e8e6502eda5c7d81b9af9a97caa77e95 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 31 Mar 2016 00:43:56 +0300 Subject: [PATCH 24/87] Make windows locks do not use dyncall --- lib/system/syslocks.nim | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 6dcdfff0d..8afd5d201 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -14,39 +14,41 @@ when defined(Windows): type Handle = int - SysLock {.final, pure.} = object # CRITICAL_SECTION in WinApi + + SysLock {.importc: "CRITICAL_SECTION", + header: "", final, pure.} = object # CRITICAL_SECTION in WinApi DebugInfo: pointer LockCount: int32 RecursionCount: int32 OwningThread: int LockSemaphore: int - Reserved: int32 + SpinCount: int SysCond = Handle {.deprecated: [THandle: Handle, TSysLock: SysLock, TSysCond: SysCond].} - proc initSysLock(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "InitializeCriticalSection".} + proc initSysLock(L: var SysLock) {.importc: "InitializeCriticalSection", + header: "".} ## Initializes the lock `L`. - proc tryAcquireSysAux(L: var SysLock): int32 {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "TryEnterCriticalSection".} + proc tryAcquireSysAux(L: var SysLock): int32 {.importc: "TryEnterCriticalSection", + header: "".} ## Tries to acquire the lock `L`. proc tryAcquireSys(L: var SysLock): bool {.inline.} = result = tryAcquireSysAux(L) != 0'i32 - proc acquireSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "EnterCriticalSection".} + proc acquireSys(L: var SysLock) {.importc: "EnterCriticalSection", + header: "".} ## Acquires the lock `L`. - proc releaseSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "LeaveCriticalSection".} + proc releaseSys(L: var SysLock) {.importc: "LeaveCriticalSection", + header: "".} ## Releases the lock `L`. - proc deinitSys(L: var SysLock) {.stdcall, noSideEffect, - dynlib: "kernel32", importc: "DeleteCriticalSection".} + proc deinitSys(L: var SysLock) {.importc: "DeleteCriticalSection", + header: "".} proc createEvent(lpEventAttributes: pointer, bManualReset, bInitialState: int32, From 40f4fe997f73176cc2d8bab755c37c90196a046e Mon Sep 17 00:00:00 2001 From: cheatfate Date: Thu, 31 Mar 2016 02:39:53 +0300 Subject: [PATCH 25/87] Replace csprintf() in windows version of nimGetProcAddr to pure nim's implementation, to avoid possible overflow. --- lib/system/dyncalls.nim | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 3b3d1f87d..61777e514 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -109,9 +109,30 @@ elif defined(windows) or defined(dos): proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = getProcAddress(cast[THINSTANCE](lib), name) if result != nil: return - var decorated: array[250, char] + const decorated_length = 250 + var decorated: array[decorated_length, char] + decorated[0] = '_' + var m = 1 + while m < (decorated_length - 5): + if name[m - 1] == '\x00': break + decorated[m] = name[m - 1] + inc(m) + decorated[m] = '@' for i in countup(0, 50): - discard csprintf(decorated, "_%s@%ld", name, i*4) + var k = i * 4 + if k div 100 == 0: + if k div 10 == 0: + m = m + 1 + else: + m = m + 2 + else: + m = m + 3 + decorated[m + 1] = '\x00' + while true: + decorated[m] = chr(ord('0') + (k %% 10)) + dec(m) + k = k div 10 + if k == 0: break result = getProcAddress(cast[THINSTANCE](lib), decorated) if result != nil: return procAddrError(name) From f0c19e9e6729a489a2030dffac6409e915b0df6e Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 31 Mar 2016 02:11:22 +0200 Subject: [PATCH 26/87] fixes a critical codegen bug that keeps PR #4017 from working --- compiler/cgen.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index db376821c..4f27a5a46 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1059,9 +1059,8 @@ proc genModule(m: BModule, cfile: string): Rope = result = getFileHeader(cfile) result.add(genMergeInfo(m)) - generateHeaders(m) - generateThreadLocalStorage(m) + generateHeaders(m) for i in countup(cfsHeaders, cfsProcs): add(result, genSectionStart(i)) add(result, m.s[i]) From c34a68d76f7f552b235592da7634acc333275df9 Mon Sep 17 00:00:00 2001 From: Anatoly Galiulin Date: Thu, 31 Mar 2016 10:51:40 +0600 Subject: [PATCH 27/87] Remove dead code --- lib/pure/parseutils.nim | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 66bdadb29..3e25eba22 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -263,24 +263,14 @@ proc parseBiggestUInt*(s: string, number: var uint64, start = 0): int {. result = rawParseUInt(s, res, start) number = res -# Workaround for high(uint) -proc highUInt(): uint64 = - when sizeof(uint) == 4: - 0xFFFFFFFF'u64 - elif sizeof(uint) == 8: - 0xFFFFFFFFFFFFFFFF'u64 - else: - {.fatal: "Unknoun uint size: " & $sizeof(uint).} - proc parseUInt*(s: string, number: var uint, start = 0): int {. rtl, extern: "npuParseUInt", noSideEffect.} = ## parses an unsigned integer starting at `start` and stores the value into `number`. - ## Result is the number of processed chars or 0 if there is no integer. ## Result is the number of processed chars or 0 if there is no integer or overflow detected. var res: uint64 result = parseBiggestUInt(s, res, start) if (sizeof(uint) <= 4) and - (res > highUInt()): + (res > 0xFFFF_FFFF'u64): raise newException(OverflowError, "overflow") elif result != 0: number = uint(res) From ebc02f6dc0dd15f0ec34a8c0f22432ca49c71c5b Mon Sep 17 00:00:00 2001 From: Anatoly Galiulin Date: Fri, 25 Mar 2016 16:41:48 +0600 Subject: [PATCH 28/87] Fix ``XDeclaredButNotUsed`` warning when locks or rlocks module is used --- lib/core/locks.nim | 2 +- lib/core/rlocks.nim | 1 + lib/system.nim | 20 ++++++++++---------- lib/system/syslocks.nim | 35 ++++++++++++++++------------------- 4 files changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 66e0ab520..068e7133c 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -63,4 +63,4 @@ template withLock*(a: Lock, body: untyped) = try: body finally: - a.release() \ No newline at end of file + a.release() diff --git a/lib/core/rlocks.nim b/lib/core/rlocks.nim index 14f04592b..4710d6cf1 100644 --- a/lib/core/rlocks.nim +++ b/lib/core/rlocks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for reentrant locks. +const insideRLocksModule = true include "system/syslocks" type diff --git a/lib/system.nim b/lib/system.nim index fefabe53f..03b9e158a 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2561,6 +2561,16 @@ else: if x < 0: -x else: x {.pop.} +proc compiles*(x: expr): bool {.magic: "Compiles", noSideEffect, compileTime.} = + ## Special compile-time procedure that checks whether `x` can be compiled + ## without any semantic error. + ## This can be used to check whether a type supports some operation: + ## + ## .. code-block:: Nim + ## when not compiles(3 + 4): + ## echo "'+' for integers is available" + discard + when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} @@ -3469,16 +3479,6 @@ when hasAlloc: x[j+i] = item[j] inc(j) -proc compiles*(x: expr): bool {.magic: "Compiles", noSideEffect, compileTime.} = - ## Special compile-time procedure that checks whether `x` can be compiled - ## without any semantic error. - ## This can be used to check whether a type supports some operation: - ## - ## .. code-block:: Nim - ## when not compiles(3 + 4): - ## echo "'+' for integers is available" - discard - when declared(initDebugger): initDebugger() diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 8afd5d201..07f6fe167 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -88,17 +88,16 @@ else: #include """.} = object SysLockType = distinct cint - proc SysLockType_Reentrant: SysLockType = - {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. importc: "pthread_mutex_init", header: "", noSideEffect.} - proc initSysLockAttr(a: var SysLockAttr) {. - importc: "pthread_mutexattr_init", header: "", noSideEffect.} - - proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. - importc: "pthread_mutexattr_settype", header: "", noSideEffect.} + when compiles(insideRLocksModule): + proc SysLockType_Reentrant: SysLockType = + {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} + proc initSysLockAttr(a: var SysLockAttr) {. + importc: "pthread_mutexattr_init", header: "", noSideEffect.} + proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. + importc: "pthread_mutexattr_settype", header: "", noSideEffect.} proc acquireSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_lock", header: "".} @@ -113,14 +112,12 @@ else: proc deinitSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "".} - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. - importc: "pthread_cond_init", header: "", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "", noSideEffect.} - - proc deinitSysCond(cond: var SysCond) {.noSideEffect, - importc: "pthread_cond_destroy", header: "".} - -{.pop.} + when not compiles(insideRLocksModule): + proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. + importc: "pthread_cond_init", header: "", noSideEffect.} + proc waitSysCond(cond: var SysCond, lock: var SysLock) {. + importc: "pthread_cond_wait", header: "", noSideEffect.} + proc signalSysCond(cond: var SysCond) {. + importc: "pthread_cond_signal", header: "", noSideEffect.} + proc deinitSysCond(cond: var SysCond) {.noSideEffect, + importc: "pthread_cond_destroy", header: "".} From 45bbecb0211d18b425c7736f803dde2acef79311 Mon Sep 17 00:00:00 2001 From: Anatoly Galiulin Date: Wed, 30 Mar 2016 08:35:14 +0600 Subject: [PATCH 29/87] Use ``when insideRLocksModule`` instead of ``when compiles`` --- lib/core/locks.nim | 1 + lib/system.nim | 1 + lib/system/syslocks.nim | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/core/locks.nim b/lib/core/locks.nim index 068e7133c..fbe9c8acf 100644 --- a/lib/core/locks.nim +++ b/lib/core/locks.nim @@ -9,6 +9,7 @@ ## This module contains Nim's support for locks and condition vars. +const insideRLocksModule = false include "system/syslocks" type diff --git a/lib/system.nim b/lib/system.nim index 03b9e158a..bcb55fe48 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2884,6 +2884,7 @@ when not defined(JS): #and not defined(nimscript): when declared(initAllocator): initAllocator() when hasThreadSupport: + const insideRLocksModule = false include "system/syslocks" when hostOS != "standalone": include "system/threads" elif not defined(nogc) and not defined(nimscript): diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index 07f6fe167..1695deca1 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -91,7 +91,7 @@ else: proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. importc: "pthread_mutex_init", header: "", noSideEffect.} - when compiles(insideRLocksModule): + when insideRLocksModule: proc SysLockType_Reentrant: SysLockType = {.emit: "`result` = PTHREAD_MUTEX_RECURSIVE;".} proc initSysLockAttr(a: var SysLockAttr) {. @@ -112,7 +112,7 @@ else: proc deinitSys(L: var SysLock) {.noSideEffect, importc: "pthread_mutex_destroy", header: "".} - when not compiles(insideRLocksModule): + when not insideRLocksModule: proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. importc: "pthread_cond_init", header: "", noSideEffect.} proc waitSysCond(cond: var SysCond, lock: var SysLock) {. From a9330391c34aa941e895a33a909b2f08ba59f60b Mon Sep 17 00:00:00 2001 From: Anatoly Galiulin Date: Thu, 31 Mar 2016 16:45:54 +0600 Subject: [PATCH 30/87] Move ``compiles`` back --- lib/system.nim | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index bcb55fe48..56210f5dc 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2561,16 +2561,6 @@ else: if x < 0: -x else: x {.pop.} -proc compiles*(x: expr): bool {.magic: "Compiles", noSideEffect, compileTime.} = - ## Special compile-time procedure that checks whether `x` can be compiled - ## without any semantic error. - ## This can be used to check whether a type supports some operation: - ## - ## .. code-block:: Nim - ## when not compiles(3 + 4): - ## echo "'+' for integers is available" - discard - when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} @@ -3480,6 +3470,16 @@ when hasAlloc: x[j+i] = item[j] inc(j) +proc compiles*(x: expr): bool {.magic: "Compiles", noSideEffect, compileTime.} = + ## Special compile-time procedure that checks whether `x` can be compiled + ## without any semantic error. + ## This can be used to check whether a type supports some operation: + ## + ## .. code-block:: Nim + ## when not compiles(3 + 4): + ## echo "'+' for integers is available" + discard + when declared(initDebugger): initDebugger() From a61d5e466beb8dc5efeb4a96511c8162fe16f38c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 31 Mar 2016 16:49:46 +0200 Subject: [PATCH 31/87] added missing file for --gc:stack --- lib/pure/collections/chains.nim | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 lib/pure/collections/chains.nim diff --git a/lib/pure/collections/chains.nim b/lib/pure/collections/chains.nim new file mode 100644 index 000000000..6b2ecd272 --- /dev/null +++ b/lib/pure/collections/chains.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Template based implementation of singly and doubly linked lists. +## The involved types should have 'prev' or 'next' fields and the +## list header should have 'head' or 'tail' fields. + +template prepend*(header, node) = + when compiles(header.head): + when compiles(node.prev): + if header.head != nil: + header.head.prev = node + node.next = header.head + header.head = node + when compiles(header.tail): + if header.tail == nil: + header.tail = node + +template append*(header, node) = + when compiles(header.head): + if header.head == nil: + header.head = node + when compiles(header.tail): + when compiles(node.prev): + node.prev = header.tail + if header.tail != nil: + header.tail.next = node + header.tail = node + +template unlink*(header, node) = + if node.next != nil: + node.next.prev = node.prev + if node.prev != nil: + node.prev.next = node.next + if header.head == node: + header.head = node.prev + if header.tail == node: + header.tail = node.next From 63807a62c6754e58c33b34dfbda84fdc56f0992c Mon Sep 17 00:00:00 2001 From: Joshua Olson <0joshua.olson1@gmail.com> Date: Fri, 1 Apr 2016 01:06:45 -0600 Subject: [PATCH 32/87] triggering yourself -> yourself triggering adverbs --- doc/backends.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/backends.txt b/doc/backends.txt index c7939baec..343af09ed 100644 --- a/doc/backends.txt +++ b/doc/backends.txt @@ -420,7 +420,7 @@ function directly will be able to use it since Nim's garbage collector has not had a chance to run *yet*. This gives you enough time to make a copy for the C side of the program, as calling any further Nim procs *might* trigger garbage collection making the previously returned string garbage. Or maybe you -are `triggering yourself the collection `_. +are `yourself triggering the collection `_. Custom data types From 0a5a593682130d7b74f16c5ee7546b2757e16d7d Mon Sep 17 00:00:00 2001 From: cheatfate Date: Sat, 2 Apr 2016 02:17:32 +0300 Subject: [PATCH 33/87] Patch timer_create and timer_delete to have proper declaration. --- lib/posix/posix.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 4c7b817cb..f1344f20d 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -2309,9 +2309,9 @@ proc strftime*(a1: cstring, a2: int, a3: cstring, a4: var Tm): int {.importc, header: "".} proc strptime*(a1, a2: cstring, a3: var Tm): cstring {.importc, header: "".} proc time*(a1: var Time): Time {.importc, header: "".} -proc timer_create*(a1: var ClockId, a2: var SigEvent, +proc timer_create*(a1: ClockId, a2: var SigEvent, a3: var Timer): cint {.importc, header: "".} -proc timer_delete*(a1: var Timer): cint {.importc, header: "".} +proc timer_delete*(a1: Timer): cint {.importc, header: "".} proc timer_gettime*(a1: Timer, a2: var Itimerspec): cint {. importc, header: "".} proc timer_getoverrun*(a1: Timer): cint {.importc, header: "".} From 0337bdc8218b40f497ea9d7cfbf56d72d1ef3bc3 Mon Sep 17 00:00:00 2001 From: cheatfate Date: Sun, 3 Apr 2016 15:37:45 +0300 Subject: [PATCH 34/87] Add missed library dependencies on Linux and Solaris. --- lib/posix/posix.nim | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index f1344f20d..e932d2845 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -35,6 +35,15 @@ const hasSpawnH = not defined(haiku) # should exist for every Posix system nowadays hasAioH = defined(linux) +when defined(linux): + # On Linux: + # timer_{create,delete,settime,gettime}, + # clock_{getcpuclockid, getres, gettime, nanosleep, settime} lives in librt + {.passL: "-lrt".} +when defined(solaris): + # On Solaris hstrerror lives in libresolv + {.passL: "-lresolv".} + when false: const C_IRUSR = 0c000400 ## Read by owner. From 3035c2c63a226cf97044405eea5107ab01be2a6d Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sun, 3 Apr 2016 15:05:39 +0000 Subject: [PATCH 35/87] net: disable SSLv3 SSLv3 is no longer secure and has been removed from OpenSSL since 1.0.2. Disabling this will enable Nim programs to run against OpenSSL 1.0.2. --- lib/pure/net.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 330682ca9..394bccee3 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -222,9 +222,9 @@ when defined(ssl): of protSSLv23: newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. of protSSLv2: - raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv3") + raiseSslError("SSLv2 is no longer secure and has been deprecated, use protSSLv23") of protSSLv3: - newCTX = SSL_CTX_new(SSLv3_method()) + raiseSslError("SSLv3 is no longer secure and has been deprecated, use protSSLv23") of protTLSv1: newCTX = SSL_CTX_new(TLSv1_method()) From 5757ad858cfa86b47a02a0389d3257c3284e1dbb Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Apr 2016 18:12:10 +0200 Subject: [PATCH 36/87] fixes #3995 --- compiler/ccgexprs.nim | 14 ++++++++++---- compiler/lambdalifting.nim | 4 ++++ tests/closure/tflatmap.nim | 24 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 tests/closure/tflatmap.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 4fcbeeec2..d84a7d92e 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1856,10 +1856,16 @@ proc genClosure(p: BProc, n: PNode, d: var TLoc) = initLocExpr(p, n.sons[1], b) if n.sons[0].skipConv.kind == nkClosure: internalError(n.info, "closure to closure created") - getTemp(p, n.typ, tmp) - linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", - tmp.rdLoc, a.rdLoc, b.rdLoc) - putLocIntoDest(p, d, tmp) + # tasyncawait.nim breaks with this optimization: + when false: + if d.k != locNone: + linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", + d.rdLoc, a.rdLoc, b.rdLoc) + else: + getTemp(p, n.typ, tmp) + linefmt(p, cpsStmts, "$1.ClPrc = $2; $1.ClEnv = $3;$n", + tmp.rdLoc, a.rdLoc, b.rdLoc) + putLocIntoDest(p, d, tmp) proc genArrayConstr(p: BProc, n: PNode, d: var TLoc) = var arr: TLoc diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 53fca4863..959632bab 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -721,6 +721,10 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass; let m = newSymNode(n[namePos].sym) m.typ = n.typ result = liftCapturedVars(m, owner, d, c) + of nkHiddenStdConv: + if n.len == 2: + n.sons[1] = liftCapturedVars(n[1], owner, d, c) + if n[1].kind == nkClosure: result = n[1] else: if owner.isIterator: if n.kind == nkYieldStmt: diff --git a/tests/closure/tflatmap.nim b/tests/closure/tflatmap.nim new file mode 100644 index 000000000..240756424 --- /dev/null +++ b/tests/closure/tflatmap.nim @@ -0,0 +1,24 @@ + +# bug #3995 + +import future + +type + RNG* = tuple[] + Rand*[A] = (RNG) -> (A, RNG) + +proc nextInt*(r: RNG): (int, RNG) = + (1, ()) + +proc flatMap[A,B](f: Rand[A], g: A -> Rand[B]): Rand[B] = + (rng: RNG) => ( + let (a, rng2) = f(rng); + let g1 = g(a); + g1(rng2) + ) + +proc map[A,B](s: Rand[A], f: A -> B): Rand[B] = + let g: A -> Rand[B] = (a: A) => ((rng: RNG) => (f(a), rng)) + flatMap(s, g) + +let f = nextInt.map(i => i - i mod 2) From 6e53300f83dc7646fbad834c494c34470a393885 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Apr 2016 22:51:22 +0200 Subject: [PATCH 37/87] fixes #3973 --- compiler/vm.nim | 7 +++++-- compiler/vmgen.nim | 2 +- tests/vm/tinheritance.nim | 29 +++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/vm/tinheritance.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 7220e1b8e..7be208089 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -10,7 +10,9 @@ ## This file implements the new evaluation engine for Nim code. ## An instruction is 1-3 int32s in memory, it is a register based VM. -const debugEchoCode = false +const + debugEchoCode = false + traceCode = debugEchoCode import ast except getstr @@ -404,7 +406,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let instr = c.code[pc] let ra = instr.regA #if c.traceActive: - #echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC + when traceCode: + echo "PC ", pc, " ", c.code[pc].opcode, " ra ", ra, " rb ", instr.regB, " rc ", instr.regC # message(c.debug[pc], warnUser, "Trace") case instr.opcode diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 47f16a013..43c7ca2db 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1447,12 +1447,12 @@ proc getNullValue(typ: PType, info: TLineInfo): PNode = of tyObject: result = newNodeIT(nkObjConstr, info, t) result.add(newNodeIT(nkEmpty, info, t)) - getNullValueAux(t.n, result) # initialize inherited fields: var base = t.sons[0] while base != nil: getNullValueAux(skipTypes(base, skipPtrs).n, result) base = base.sons[0] + getNullValueAux(t.n, result) of tyArray, tyArrayConstr: result = newNodeIT(nkBracket, info, t) for i in countup(0, int(lengthOrd(t)) - 1): diff --git a/tests/vm/tinheritance.nim b/tests/vm/tinheritance.nim new file mode 100644 index 000000000..d465e22b9 --- /dev/null +++ b/tests/vm/tinheritance.nim @@ -0,0 +1,29 @@ +discard """ + msg: '''Hello fred , managed by sally +Hello sally , managed by bob''' +""" +# bug #3973 + +type + EmployeeCode = enum + ecCode1, + ecCode2 + + Person* = object of RootObj + name* : string + last_name*: string + + Employee* = object of Person + empl_code* : EmployeeCode + mgr_name* : string + +proc test() = + var + empl1 = Employee(name: "fred", last_name: "smith", mgr_name: "sally", empl_code: ecCode1) + empl2 = Employee(name: "sally", last_name: "jones", mgr_name: "bob", empl_code: ecCode2) + + echo "Hello ", empl1.name, " , managed by ", empl1.mgr_name + echo "Hello ", empl2.name, " , managed by ", empl2.mgr_name + +static: + test() From cb3a38afa2de4401af55fbb7b6050f499b182bf4 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 3 Apr 2016 23:38:29 +0200 Subject: [PATCH 38/87] fixes #1152 --- doc/manual/typedesc.txt | 32 ---------------------- tests/macros/typesafeprintf.nim | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 tests/macros/typesafeprintf.nim diff --git a/doc/manual/typedesc.txt b/doc/manual/typedesc.txt index de1d84d7d..6922d77e4 100644 --- a/doc/manual/typedesc.txt +++ b/doc/manual/typedesc.txt @@ -77,38 +77,6 @@ Once bound, typedesc params can appear in the rest of the proc signature: declareVariableWithType int, 42 -When used with macros and .compileTime. procs on the other hand, the compiler -does not need to instantiate the code multiple times, because types then can be -manipulated using the unified internal symbol representation. In such context -typedesc acts as any other type. One can create variables, store typedesc -values inside containers and so on. For example, here is how one can create -a type-safe wrapper for the unsafe `printf` function from C: - -.. code-block:: nim - macro safePrintF(formatString: string{lit}, args: varargs[expr]): expr = - var i = 0 - for c in formatChars(formatString): - var expectedType = case c - of 'c': char - of 'd', 'i', 'x', 'X': int - of 'f', 'e', 'E', 'g', 'G': float - of 's': string - of 'p': pointer - else: EOutOfRange - - var actualType = args[i].getType - inc i - - if expectedType == EOutOfRange: - error c & " is not a valid format character" - elif expectedType != actualType: - error "type mismatch for argument ", i, ". expected type: ", - expectedType.name, ", actual type: ", actualType.name - - # keep the original callsite, but use cprintf instead - result = callsite() - result[0] = newIdentNode(!"cprintf") - Overload resolution can be further influenced by constraining the set of types that will match the typedesc param: diff --git a/tests/macros/typesafeprintf.nim b/tests/macros/typesafeprintf.nim new file mode 100644 index 000000000..2f4622f3e --- /dev/null +++ b/tests/macros/typesafeprintf.nim @@ -0,0 +1,48 @@ +discard """ + output: '''test 10''' +""" + +# bug #1152 + +import macros, typetraits +proc printfImpl(formatstr: cstring) {.importc: "printf", varargs.} + +iterator tokenize(format: string): char = + var i = 0 + while true: + case format[i] + of '%': + case format[i+1] + of '\0': break + else: yield format[i+1] + i.inc + of '\0': break + else: discard + i.inc + +macro printf(formatString: string{lit}, args: varargs[typed]): untyped = + var i = 0 + let err = getType(bindSym"ValueError") + for c in tokenize(formatString.strVal): + var expectedType = case c + of 'c': getType(bindSym"char") + of 'd', 'i', 'x', 'X': getType(bindSym"int") + of 'f', 'e', 'E', 'g', 'G': getType(bindSym"float") + of 's': getType(bindSym"string") + of 'p': getType(bindSym"pointer") + else: err + + var actualType = getType(args[i]) + inc i + + if sameType(expectedType, err): + error c & " is not a valid format character" + elif not sameType(expectedType, actualType): + error "type mismatch for argument " & $i & ". expected type: " & + $expectedType & ", actual type: " & $actualType + + # keep the original callsite, but use cprintf instead + result = callsite() + result[0] = bindSym"printfImpl" + +printf("test %d\n", 10) From 86e79f5cec118587c7e52f614c245176cf480597 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Apr 2016 01:41:14 +0200 Subject: [PATCH 39/87] fixes #3804 --- compiler/vm.nim | 2 +- compiler/vmgen.nim | 4 ++++ tests/vm/twrong_concat.nim | 28 ++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/vm/twrong_concat.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 7be208089..1235a648d 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -123,7 +123,7 @@ template move(a, b: expr) {.immediate, dirty.} = system.shallowCopy(a, b) # XXX fix minor 'shallowCopy' overloading bug in compiler proc createStrKeepNode(x: var TFullReg; keepNode=true) = - if x.node.isNil: + if x.node.isNil or not keepNode: x.node = newNode(nkStrLit) elif x.node.kind == nkNilLit and keepNode: when defined(useNodeIds): diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 43c7ca2db..7e506f8a0 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -102,6 +102,10 @@ proc gABC(ctx: PCtx; n: PNode; opc: TOpcode; a, b, c: TRegister = 0) = let ins = (opc.uint32 or (a.uint32 shl 8'u32) or (b.uint32 shl 16'u32) or (c.uint32 shl 24'u32)).TInstr + when false: + if ctx.code.len == 72: + writeStackTrace() + echo "generating ", opc ctx.code.add(ins) ctx.debug.add(n.info) diff --git a/tests/vm/twrong_concat.nim b/tests/vm/twrong_concat.nim new file mode 100644 index 000000000..538ea2527 --- /dev/null +++ b/tests/vm/twrong_concat.nim @@ -0,0 +1,28 @@ +discard """ + output: '''success''' +""" + +# bug #3804 + +#import sequtils + +type AnObj = ref object + field: string + +#proc aBug(objs: seq[AnObj]) {.compileTime.} = +# discard objs.mapIt(it.field & " bug") + +proc sameBug(objs: seq[AnObj]) {.compileTime.} = + var strSeq = newSeq[string](objs.len) + strSeq[0] = objs[0].field & " bug" + +static: + var objs: seq[AnObj] = @[] + objs.add(AnObj(field: "hello")) + + sameBug(objs) + # sameBug(objs) + echo objs[0].field + assert(objs[0].field == "hello") # fails, because (objs[0].field == "hello bug") - mutated! + +echo "success" From 0acdaea3342e13317ac9956add08e606208b2986 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Apr 2016 02:45:43 +0200 Subject: [PATCH 40/87] fixes #3561, fixes #2409 --- compiler/vmgen.nim | 10 +++++--- tests/vm/tanonproc.nim | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 tests/vm/tanonproc.nim diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 7e506f8a0..038a07a16 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -708,6 +708,10 @@ proc genAddSubInt(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = c.genNarrow(n, dest) proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) = + if n.typ.kind == arg.typ.kind and arg.typ.kind == tyProc: + # don't do anything for lambda lifting conversions: + gen(c, arg, dest) + return let tmp = c.genx(arg) if dest < 0: dest = c.getTemp(n.typ) c.gABC(n, opc, dest, tmp) @@ -1740,9 +1744,9 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) = of declarativeDefs: unused(n, dest) of nkLambdaKinds: - let s = n.sons[namePos].sym - discard genProc(c, s) - genLit(c, n.sons[namePos], dest) + #let s = n.sons[namePos].sym + #discard genProc(c, s) + genLit(c, newSymNode(n.sons[namePos].sym), dest) of nkChckRangeF, nkChckRange64, nkChckRange: let tmp0 = c.genx(n.sons[0]) diff --git a/tests/vm/tanonproc.nim b/tests/vm/tanonproc.nim new file mode 100644 index 000000000..474b768ca --- /dev/null +++ b/tests/vm/tanonproc.nim @@ -0,0 +1,52 @@ +discard """ + output: '''`Test`''' +""" + +# bug #3561 + +import macros, future, strutils + +type + Option[T] = ref object + case valid: bool + of true: + data: T + else: + discard + +proc some[T](v: T): Option[T] = Option[T](valid: true, data: v) +proc none[T](v: T): Option[T] = Option[T](valid: false) +proc none(T: typedesc): Option[T] = Option[T](valid: false) + +proc map[T,U](o: Option[T], f: T -> U): Option[U] = + case o.valid + of true: + f(o.data).some + else: + U.none + +proc notEmpty(o: Option[string]): Option[string] = + case o.valid + of true: + if o.data.strip == "": string.none else: o.data.strip.some + else: + o + +proc getOrElse[T](o: Option[T], def: T): T = + case o.valid + of true: + o.data + else: + def + +proc quoteStr(s: string): Option[string] = + s.some.notEmpty.map(v => "`" & v & "`") + +macro str(s: string): stmt = + let x = s.strVal + let y = quoteStr(x) + let sn = newStrLitNode(y.getOrElse("NONE")) + result = quote do: + echo `sn` + +str"Test" From cbcdf12d2c78de352d3f5149882cca2b0b72522d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Apr 2016 10:16:15 +0200 Subject: [PATCH 41/87] fixes #3731 --- compiler/vm.nim | 2 +- compiler/vmgen.nim | 14 +++++++++++--- tests/vm/tmitems.nim | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 tests/vm/tmitems.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 1235a648d..f799334d6 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -545,7 +545,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if regs[rb].node.kind == nkRefTy: regs[ra].node = regs[rb].node.sons[0] else: - stackTrace(c, tos, pc, errGenerated, "limited VM support for 'ref'") + stackTrace(c, tos, pc, errGenerated, "limited VM support for pointers") else: stackTrace(c, tos, pc, errNilAccess) of opcWrDeref: diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 038a07a16..019c79eb3 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -103,7 +103,7 @@ proc gABC(ctx: PCtx; n: PNode; opc: TOpcode; a, b, c: TRegister = 0) = (b.uint32 shl 16'u32) or (c.uint32 shl 24'u32)).TInstr when false: - if ctx.code.len == 72: + if ctx.code.len == 43: writeStackTrace() echo "generating ", opc ctx.code.add(ins) @@ -126,6 +126,11 @@ proc gABI(c: PCtx; n: PNode; opc: TOpcode; a, b: TRegister; imm: BiggestInt) = proc gABx(c: PCtx; n: PNode; opc: TOpcode; a: TRegister = 0; bx: int) = # Applies `opc` to `bx` and stores it into register `a` # `bx` must be signed and in the range [-32768, 32767] + when false: + if c.code.len == 43: + writeStackTrace() + echo "generating ", opc + if bx >= -32768 and bx <= 32767: let ins = (opc.uint32 or a.uint32 shl 8'u32 or (bx+wordExcess).uint32 shl 16'u32).TInstr @@ -1125,7 +1130,10 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; if isAddr and (let m = canElimAddr(n); m != nil): gen(c, m, dest, flags) return - let newflags = if isAddr: flags+{gfAddrOf} else: flags + + let af = if n[0].kind in {nkBracketExpr, nkDotExpr, nkCheckedFieldExpr}: {gfAddrOf, gfFieldAccess} + else: {gfAddrOf} + let newflags = if isAddr: flags+af else: flags # consider: # proc foo(f: var ref int) = # f = new(int) @@ -1140,7 +1148,7 @@ proc genAddrDeref(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode; if gfAddrOf notin flags and fitsRegister(n.typ): c.gABC(n, opcNodeToReg, dest, dest) elif isAddr and isGlobal(n.sons[0]): - gen(c, n.sons[0], dest, flags+{gfAddrOf}) + gen(c, n.sons[0], dest, flags+af) else: let tmp = c.genx(n.sons[0], newflags) if dest < 0: dest = c.getTemp(n.typ) diff --git a/tests/vm/tmitems.nim b/tests/vm/tmitems.nim new file mode 100644 index 000000000..18f5bfa06 --- /dev/null +++ b/tests/vm/tmitems.nim @@ -0,0 +1,15 @@ +discard """ + msg: '''13''' +""" +# bug #3731 +var list {.compileTime.} = newSeq[int]() + +macro calc*(): stmt {.immediate.} = + list.add(1) + for c in list.mitems: + c = 13 + + for c in list: + echo c + +calc() From 6d675814091b20312cbe51b69261453e04e27214 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Apr 2016 10:19:20 +0200 Subject: [PATCH 42/87] fixes #3859 --- tests/vm/tmitems.nim | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/vm/tmitems.nim b/tests/vm/tmitems.nim index 18f5bfa06..4ee225eed 100644 --- a/tests/vm/tmitems.nim +++ b/tests/vm/tmitems.nim @@ -1,5 +1,8 @@ discard """ msg: '''13''' + output: '''3 +3 +3''' """ # bug #3731 var list {.compileTime.} = newSeq[int]() @@ -13,3 +16,16 @@ macro calc*(): stmt {.immediate.} = echo c calc() + +# bug #3859 +import macros +macro m: stmt = + var s = newseq[NimNode](3) + # var s: array[3,NimNode] # not working either + for i in 0.. Date: Mon, 4 Apr 2016 10:28:18 +0200 Subject: [PATCH 43/87] fixes #4036 --- compiler/parser.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/parser.nim b/compiler/parser.nim index f22177ac1..1ba59b938 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -326,6 +326,7 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode = getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) + break eat(p, tkAccent) else: if allowNil and p.tok.tokType == tkNil: From fbedb6c65514e394f163a68dd530d19d310afadf Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Apr 2016 11:16:03 +0200 Subject: [PATCH 44/87] fixes #3646 --- compiler/pragmas.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 2280ef712..38d17eb62 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -256,8 +256,9 @@ proc expectDynlibNode(c: PContext, n: PNode): PNode = proc processDynLib(c: PContext, n: PNode, sym: PSym) = if (sym == nil) or (sym.kind == skModule): - POptionEntry(c.optionStack.tail).dynlib = getLib(c, libDynamic, - expectDynlibNode(c, n)) + let lib = getLib(c, libDynamic, expectDynlibNode(c, n)) + if not lib.isOverriden: + POptionEntry(c.optionStack.tail).dynlib = lib else: if n.kind == nkExprColonExpr: var lib = getLib(c, libDynamic, expectDynlibNode(c, n)) From 1d85ca3aa41d8e3d2bf7aaf481fe6a577c33ec48 Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Mon, 4 Apr 2016 15:50:02 +0600 Subject: [PATCH 45/87] Fixed macros unpackX procedured to return values correctly --- lib/core/macros.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4522e0fc6..678982a52 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -796,17 +796,17 @@ proc infix*(a: NimNode; op: string; proc unpackPostfix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPostfix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackPrefix*(node: NimNode): tuple[node: NimNode; op: string] {. compileTime.} = node.expectKind nnkPrefix - result = (node[0], $node[1]) + result = (node[1], $node[0]) proc unpackInfix*(node: NimNode): tuple[left: NimNode; op: string; right: NimNode] {.compileTime.} = assert node.kind == nnkInfix - result = (node[0], $node[1], node[2]) + result = (node[1], $node[0], node[2]) proc copy*(node: NimNode): NimNode {.compileTime.} = ## An alias for copyNimTree(). From a70e6b3fde7c251aff68243de1c7adcebc3c67e3 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 12:06:25 +0100 Subject: [PATCH 46/87] Fixes #3752. --- lib/system.nim | 3 +++ lib/system/sysio.nim | 2 ++ 2 files changed, 5 insertions(+) diff --git a/lib/system.nim b/lib/system.nim index fefabe53f..d3b18f9e3 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2774,6 +2774,9 @@ when not defined(JS): #and not defined(nimscript): ## reads `len` bytes into the buffer `a` starting at ``a[start]``. Returns ## the actual number of bytes that have been read which may be less than ## `len` (if not as many bytes are remaining), but not greater. + ## + ## **Warning:** The buffer `a` must be pre-allocated. This can be done + ## using, for example, ``newString``. proc readBuffer*(f: File, buffer: pointer, len: Natural): int {. tags: [ReadIOEffect], benign.} diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index d0bba6775..3c34215ac 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -58,6 +58,8 @@ proc readBytes(f: File, a: var openArray[int8|uint8], start, len: Natural): int result = readBuffer(f, addr(a[start]), len) proc readChars(f: File, a: var openArray[char], start, len: Natural): int = + if (start + len) > len(a): + raiseEIO("buffer overflow: (start+len) > length of openarray buffer") result = readBuffer(f, addr(a[start]), len) proc write(f: File, c: cstring) = fputs(c, f) From 73e48f9c9cfa3d6f2dad1d9e11efec2975eca1e5 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 13:39:42 +0100 Subject: [PATCH 47/87] Fixes #4037 --- lib/pure/strutils.nim | 90 ++++++++++++++++++++++++++++++------------- web/news.txt | 3 ++ 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index eebadf4c0..a5a4ee509 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -784,8 +784,7 @@ proc indent*(s: string, count: Natural, padding: string = " "): string {.noSideEffect, rtl, extern: "nsuIndent".} = ## Indents each line in ``s`` by ``count`` amount of ``padding``. ## - ## **Note:** This currently does not preserve the specific new line characters - ## used. + ## **Note:** This does not preserve the new line characters used in ``s``. result = "" var i = 0 for line in s.splitLines(): @@ -796,32 +795,39 @@ proc indent*(s: string, count: Natural, padding: string = " "): string result.add(line) i.inc -proc unindent*(s: string, eatAllIndent = false): string {. - noSideEffect, rtl, extern: "nsuUnindent".} = - ## Unindents `s`. - result = newStringOfCap(s.len) +proc unindent*(s: string, count: Natural, padding: string = " "): string + {.noSideEffect, rtl, extern: "nsuUnindent".} = + ## Unindents each line in ``s`` by ``count`` amount of ``padding``. + ## + ## **Note:** This does not preserve the new line characters used in ``s``. + result = "" var i = 0 - var pattern = true - var indent = 0 - while s[i] == ' ': inc i - var level = if i == 0: -1 else: i - while i < s.len: - if s[i] == ' ': - if i > 0 and s[i-1] in {'\l', '\c'}: - pattern = true - indent = 0 - if pattern: - inc(indent) - if indent > level and not eatAllIndent: - result.add(s[i]) - if level < 0: level = indent - else: - # a space somewhere: do not delete - result.add(s[i]) - else: - pattern = false - result.add(s[i]) - inc i + for line in s.splitLines(): + if i != 0: + result.add("\n") + var indentCount = 0 + for j in 0.. Date: Mon, 4 Apr 2016 14:44:26 +0100 Subject: [PATCH 48/87] Better additionalInfo OSError message, ref #2650. --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 470559e17..1a2c6d537 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -129,7 +129,7 @@ proc raiseOSError*(errorCode: OSErrorCode; additionalInfo = "") {.noinline.} = if additionalInfo.len == 0: e.msg = osErrorMsg(errorCode) else: - e.msg = additionalInfo & " " & osErrorMsg(errorCode) + e.msg = osErrorMsg(errorCode) & "\nAdditional info: " & additionalInfo if e.msg == "": e.msg = "unknown OS error" raise e From cf5b7cbdefc1da9a5c0c8b54226d52e337855534 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 4 Apr 2016 21:57:56 +0800 Subject: [PATCH 49/87] fix type used for chckRange node boundaries else you end up with for example an int64-sized value with an int32 type, when checking int32 -> Natural --- compiler/transf.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/transf.nim b/compiler/transf.nim index a4a15ea4a..0647553c6 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -414,8 +414,8 @@ proc transformConv(c: PTransf, n: PNode): PTransNode = result = newTransNode(nkChckRange, n, 3) dest = skipTypes(n.typ, abstractVar) result[0] = transform(c, n.sons[1]) - result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), source).PTransNode - result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), source).PTransNode + result[1] = newIntTypeNode(nkIntLit, firstOrd(dest), dest).PTransNode + result[2] = newIntTypeNode(nkIntLit, lastOrd(dest), dest).PTransNode of tyFloat..tyFloat128: # XXX int64 -> float conversion? if skipTypes(n.typ, abstractVar).kind == tyRange: From af70254c7b508b6658b50ec8b4fb9465a02acb3f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 4 Apr 2016 16:41:35 +0200 Subject: [PATCH 50/87] fixes #2006 --- compiler/sigmatch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 82f878932..377743de3 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -513,7 +513,7 @@ proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} = proc matchUserTypeClass*(c: PContext, m: var TCandidate, ff, a: PType): TTypeRelation = var body = ff.skipTypes({tyUserTypeClassInst}) - if c.inTypeClass > 20: + if c.inTypeClass > 4: localError(body.n[3].info, $body.n[3] & " too nested for type matching") return isNone From aa01c346f76716f5d4379cd0eae2d065d676d74a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 20:15:52 +0100 Subject: [PATCH 51/87] Fixes #3159. --- lib/impure/rdstdin.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/impure/rdstdin.nim b/lib/impure/rdstdin.nim index 469bb69c5..f722a6b39 100644 --- a/lib/impure/rdstdin.nim +++ b/lib/impure/rdstdin.nim @@ -105,7 +105,8 @@ else: proc readLineFromStdin*(prompt: string): TaintedString {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") result = TaintedString($buffer) if result.string.len > 0: historyAdd(buffer) @@ -114,12 +115,12 @@ else: proc readLineFromStdin*(prompt: string, line: var TaintedString): bool {. tags: [ReadIOEffect, WriteIOEffect].} = var buffer = linenoise.readLine(prompt) - if isNil(buffer): quit(0) + if isNil(buffer): + raise newException(IOError, "Linenoise returned nil") line = TaintedString($buffer) if line.string.len > 0: historyAdd(buffer) linenoise.free(buffer) - # XXX how to determine CTRL+D? result = true proc readPasswordFromStdin*(prompt: string, password: var TaintedString): From 38d6e2c221dcb6f284de1ec7b038385a6212809c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 20:22:22 +0100 Subject: [PATCH 52/87] Deprecated matchers module. Fixes #2446. --- lib/pure/matchers.nim | 4 ++++ web/news.txt | 2 ++ 2 files changed, 6 insertions(+) diff --git a/lib/pure/matchers.nim b/lib/pure/matchers.nim index 5c28f65a0..7022c21d9 100644 --- a/lib/pure/matchers.nim +++ b/lib/pure/matchers.nim @@ -8,6 +8,10 @@ # ## This module contains various string matchers for email addresses, etc. +## +## **Warning:** This module is deprecated since version 0.14.0. +{.deprecated.} + {.deadCodeElim: on.} {.push debugger:off .} # the user does not want to trace a part diff --git a/web/news.txt b/web/news.txt index 5bb0e19b4..42d2aca99 100644 --- a/web/news.txt +++ b/web/news.txt @@ -26,6 +26,8 @@ Changes affecting backwards compatibility - The ``strutils.unindent`` procedure has been rewritten. Its parameters now match the parameters of ``strutils.indent``. See issue [#4037](https://github.com/nim-lang/Nim/issues/4037) for more details. +- The ``matchers`` module has been deprecated. See issue [#2446](https://github.com/nim-lang/Nim/issues/2446) + for more details. Library Additions From 8e6f5f20f686110febf7f50e886a51fd821dc44f Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 20:47:47 +0100 Subject: [PATCH 53/87] Improve net and ssl module docs. --- lib/impure/ssl.nim | 3 ++ lib/pure/net.nim | 97 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 80 insertions(+), 20 deletions(-) diff --git a/lib/impure/ssl.nim b/lib/impure/ssl.nim index 721e5ce51..e3312d792 100644 --- a/lib/impure/ssl.nim +++ b/lib/impure/ssl.nim @@ -9,6 +9,9 @@ ## This module provides an easy to use sockets-style ## nim interface to the OpenSSL library. +## +## **Warning:** This module is deprecated, use the SSL procedures defined in +## the ``net`` module instead. {.deprecated.} diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 394bccee3..fff72e1d8 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -8,19 +8,76 @@ # ## This module implements a high-level cross-platform sockets interface. +## The procedures implemented in this module are primarily for blocking sockets. +## For asynchronous non-blocking sockets use the ``asyncnet`` module together +## with the ``asyncdispatch`` module. +## +## The first thing you will always need to do in order to start using sockets, +## is to create a new instance of the ``Socket`` type using the ``newSocket`` +## procedure. +## +## SSL +## ==== +## +## In order to use the SSL procedures defined in this module, you will need to +## compile your application with the ``-d:ssl`` flag. +## +## Examples +## ======== +## +## Connecting to a server +## ---------------------- +## +## After you create a socket with the ``newSocket`` procedure, you can easily +## connect it to a server running at a known hostname (or IP address) and port. +## To do so over TCP, use the example below. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.connect("google.com", Port(80)) +## +## UDP is a connectionless protocol, so UDP sockets don't have to explicitly +## call the ``connect`` procedure. They can simply start sending data +## immediately. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.sendTo("192.168.0.1", Port(27960), "status\n") +## +## Creating a server +## ----------------- +## +## After you create a socket with the ``newSocket`` procedure, you can create a +## TCP server by calling the ``bindAddr`` and ``listen`` procedures. +## +## .. code-block:: Nim +## var socket = newSocket() +## socket.bindAddr(Port(1234)) +## socket.listen() +## +## You can then begin accepting connections using the ``accept`` procedure. +## +## .. code-block:: Nim +## var client = newSocket() +## var address = "" +## while true: +## socket.acceptAddr(client, address) +## echo("Client connected from: ", address) +## {.deadCodeElim: on.} import nativesockets, os, strutils, parseutils, times export Port, `$`, `==` const useWinVersion = defined(Windows) or defined(nimdoc) +const defineSsl = defined(ssl) or defined(nimdoc) -when defined(ssl): +when defineSsl: import openssl # Note: The enumerations are mapped to Window's constants. -when defined(ssl): +when defineSsl: type SslError* = object of Exception @@ -54,7 +111,7 @@ type currPos: int # current index in buffer bufLen: int # current length of buffer of false: nil - when defined(ssl): + when defineSsl: case isSsl: bool of true: sslHandle: SSLPtr @@ -160,7 +217,7 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) -when defined(ssl): +when defineSsl: CRYPTO_malloc_init() SslLibraryInit() SslLoadErrorStrings() @@ -301,7 +358,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, ## error was caused by no data being available to be read. ## ## If ``err`` is not lower than 0 no exception will be raised. - when defined(ssl): + when defineSsl: if socket.isSSL: if err <= 0: var ret = SSLGetError(socket.sslHandle, err.cint) @@ -334,7 +391,7 @@ proc socketError*(socket: Socket, err: int = -1, async = false, raiseSSLError() else: raiseSSLError("Unknown Error") - if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if err == -1 and not (when defineSsl: socket.isSSL else: false): var lastE = if lastError.int == -1: getSocketError(socket) else: lastError if async: when useWinVersion: @@ -414,7 +471,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, client.isBuffered = server.isBuffered # Handle SSL. - when defined(ssl): + when defineSsl: if server.isSSL: # We must wrap the client sock in a ssl context. @@ -425,7 +482,7 @@ proc acceptAddr*(server: Socket, client: var Socket, address: var string, # Client socket is set above. address = $inet_ntoa(sockAddress.sin_addr) -when false: #defined(ssl): +when false: #defineSsl: proc acceptAddrSSL*(server: Socket, client: var Socket, address: var string): SSLAcceptResult {. tags: [ReadIOEffect].} = @@ -444,7 +501,7 @@ when false: #defined(ssl): ## ``AcceptNoClient`` will be returned when no client is currently attempting ## to connect. template doHandshake(): stmt = - when defined(ssl): + when defineSsl: if server.isSSL: client.setBlocking(false) # We must wrap the client sock in a ssl context. @@ -495,7 +552,7 @@ proc accept*(server: Socket, client: var Socket, proc close*(socket: Socket) = ## Closes a socket. try: - when defined(ssl): + when defineSsl: if socket.isSSL: ErrClearError() # As we are closing the underlying socket immediately afterwards, @@ -547,7 +604,7 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { var valuei = cint(if value: 1 else: 0) setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) -when defined(ssl): +when defineSsl: proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. @@ -594,7 +651,7 @@ proc hasDataBuffered*(s: Socket): bool = if s.isBuffered: result = s.bufLen > 0 and s.currPos != s.bufLen - when defined(ssl): + when defineSsl: if s.isSSL and not result: result = s.sslHasPeekChar @@ -608,7 +665,7 @@ proc select(readfd: Socket, timeout = 500): int = proc readIntoBuf(socket: Socket, flags: int32): int = result = 0 - when defined(ssl): + when defineSsl: if socket.isSSL: result = SSLRead(socket.sslHandle, addr(socket.buffer), int(socket.buffer.high)) else: @@ -658,7 +715,7 @@ proc recv*(socket: Socket, data: pointer, size: int): int {.tags: [ReadIOEffect] result = read else: - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.sslHasPeekChar: copyMem(data, addr(socket.sslPeekChar), 1) @@ -696,7 +753,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int, if timeout - int(waited * 1000.0) < 1: raise newException(TimeoutError, "Call to '" & funcName & "' timed out.") - when defined(ssl): + when defineSsl: if socket.isSSL: if socket.hasDataBuffered: # sslPeekChar is present. @@ -764,7 +821,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = c = socket.buffer[socket.currPos] else: - when defined(ssl): + when defineSsl: if socket.isSSL: if not socket.sslHasPeekChar: result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) @@ -872,7 +929,7 @@ proc send*(socket: Socket, data: pointer, size: int): int {. ## ## **Note**: This is a low-level version of ``send``. You likely should use ## the version below. - when defined(ssl): + when defineSsl: if socket.isSSL: return SSLWrite(socket.sslHandle, cast[cstring](data), size) @@ -943,7 +1000,7 @@ proc sendTo*(socket: Socket, address: string, port: Port, proc isSsl*(socket: Socket): bool = ## Determines whether ``socket`` is a SSL socket. - when defined(ssl): + when defineSsl: result = socket.isSSL else: result = false @@ -1253,7 +1310,7 @@ proc connect*(socket: Socket, address: string, dealloc(aiList) if not success: raiseOSError(lastError) - when defined(ssl): + when defineSsl: if socket.isSSL: # RFC3546 for SNI specifies that IP addresses are not allowed. if not isIpAddress(address): @@ -1314,7 +1371,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), if selectWrite(s, timeout) != 1: raise newException(TimeoutError, "Call to 'connect' timed out.") else: - when defined(ssl): + when defineSsl: if socket.isSSL: socket.fd.setBlocking(true) doAssert socket.handshake() From 03be03c6e454f22ef7a70cf8f3300f18407cdc0a Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 21:00:43 +0100 Subject: [PATCH 54/87] Improved asyncnet docs (SSL procs now shown). Deprecated handshake. --- lib/pure/asyncnet.nim | 20 +++++++++++--------- lib/pure/net.nim | 5 ++++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 6b19a48be..7161b1c55 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -11,7 +11,7 @@ ## asynchronous dispatcher defined in the ``asyncdispatch`` module. ## ## SSL -## --- +## ---- ## ## SSL can be enabled by compiling with the ``-d:ssl`` flag. ## @@ -62,7 +62,9 @@ import os export SOBool -when defined(ssl): +const defineSsl = defined(ssl) or defined(nimdoc) + +when defineSsl: import openssl type @@ -79,7 +81,7 @@ type of false: nil case isSsl: bool of true: - when defined(ssl): + when defineSsl: sslHandle: SslPtr sslContext: SslContext bioIn: BIO @@ -125,7 +127,7 @@ proc newAsyncSocket*(domain, sockType, protocol: cint, Domain(domain), SockType(sockType), Protocol(protocol), buffered) -when defined(ssl): +when defineSsl: proc getSslError(handle: SslPtr, err: cint): cint = assert err < 0 var ret = SSLGetError(handle, err.cint) @@ -186,7 +188,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## or an error occurs. await connect(socket.fd.AsyncFD, address, port, socket.domain) if socket.isSsl: - when defined(ssl): + when defineSsl: let flags = {SocketFlag.SafeDisconn} sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) @@ -197,7 +199,7 @@ template readInto(buf: cstring, size: int, socket: AsyncSocket, ## this is a template and not a proc. var res = 0 if socket.isSsl: - when defined(ssl): + when defineSsl: # SSL mode. sslLoop(socket, flags, sslRead(socket.sslHandle, buf, size.cint)) @@ -274,7 +276,7 @@ proc send*(socket: AsyncSocket, data: string, ## data has been sent. assert socket != nil if socket.isSsl: - when defined(ssl): + when defineSsl: var copy = data sslLoop(socket, flags, sslWrite(socket.sslHandle, addr copy[0], copy.len.cint)) @@ -468,7 +470,7 @@ proc close*(socket: AsyncSocket) = ## Closes the socket. defer: socket.fd.AsyncFD.closeSocket() - when defined(ssl): + when defineSsl: if socket.isSSL: let res = SslShutdown(socket.sslHandle) SSLFree(socket.sslHandle) @@ -478,7 +480,7 @@ proc close*(socket: AsyncSocket) = raiseSslError() socket.closed = true # TODO: Add extra debugging checks for this. -when defined(ssl): +when defineSsl: proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) = ## Wraps a socket in an SSL context. This function effectively turns ## ``socket`` into an SSL socket. diff --git a/lib/pure/net.nim b/lib/pure/net.nim index fff72e1d8..292816fb5 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -605,7 +605,8 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei) when defineSsl: - proc handshake*(socket: Socket): bool {.tags: [ReadIOEffect, WriteIOEffect].} = + proc handshake*(socket: Socket): bool + {.tags: [ReadIOEffect, WriteIOEffect, deprecated].} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. ## This proc performs the SSL handshake. @@ -614,6 +615,8 @@ when defineSsl: ## ``True`` whenever handshake completed successfully. ## ## A ESSL error is raised on any other errors. + ## + ## **Note:** This procedure is deprecated since version 0.14.0. result = true if socket.isSSL: var ret = SSLConnect(socket.sslHandle) From 3ad103b8ccf70563d67c1839f78bd4429d468496 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 21:20:29 +0100 Subject: [PATCH 55/87] Fixes small mistake in ``net`` module. --- lib/pure/net.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 292816fb5..6888eb59f 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -606,7 +606,7 @@ proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) { when defineSsl: proc handshake*(socket: Socket): bool - {.tags: [ReadIOEffect, WriteIOEffect, deprecated].} = + {.tags: [ReadIOEffect, WriteIOEffect], deprecated.} = ## This proc needs to be called on a socket after it connects. This is ## only applicable when using ``connectAsync``. ## This proc performs the SSL handshake. From 8050f43588f21e61a58fcdebb3ed5de231afa751 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 21:28:21 +0100 Subject: [PATCH 56/87] Fixed some warnings in httpclient, net, and openssl. --- lib/pure/asyncnet.nim | 6 ++---- lib/pure/httpclient.nim | 4 ++-- lib/wrappers/openssl.nim | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 7161b1c55..748778566 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -428,9 +428,6 @@ proc recvLine*(socket: AsyncSocket, ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol ## uses ``\r\L`` to delimit a new line. - template addNLIfEmpty(): stmt = - if result.len == 0: - result.add("\c\L") assert SocketFlag.Peek notin flags ## TODO: # TODO: Optimise this @@ -458,7 +455,8 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {. of AF_INET6: realaddr = "::" of AF_INET: realaddr = "0.0.0.0" else: - raiseOSError("Unknown socket address family and no address specified to bindAddr") + raise newException(ValueError, + "Unknown socket address family and no address specified to bindAddr") var aiList = getAddrInfo(realaddr, port, socket.domain) if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 603763386..f473e47d1 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -335,7 +335,7 @@ proc addFiles*(p: var MultipartData, xs: openarray[tuple[name, file: string]]): var m = newMimetypes() for name, file in xs.items: var contentType: string - let (dir, fName, ext) = splitFile(file) + let (_, fName, ext) = splitFile(file) if ext.len > 0: contentType = m.getMimetype(ext[1..ext.high], nil) p.add(name, readFile(file), fName & ext, contentType) @@ -627,7 +627,7 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, result.userAgent = defUserAgent result.maxRedirects = maxRedirects when defined(ssl): - result.sslContext = net.SslContext(sslContext) + result.sslContext = sslContext proc close*(client: AsyncHttpClient) = ## Closes any connections held by the HTTP client. diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 05843e2d3..635d52a64 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -260,7 +260,7 @@ proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSS proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion: +when not useWinVersion and not defined(macosx): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -532,7 +532,7 @@ proc md5_File* (file: string): string {.raises: [IOError,Exception].} = result = hexStr(buf) -proc md5_Str* (str:string): string {.raises:[IOError].} = +proc md5_Str*(str:string): string = ##Generate MD5 hash for a string. Result is a 32 character #hex string with lowercase characters var From c90df48293f080833348e8ea9dfcf7ee91736229 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 21:52:10 +0100 Subject: [PATCH 57/87] Fixes JSON module problems. Fixes #3107. --- lib/pure/json.nim | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index b0179cd82..5df9de5fa 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -804,7 +804,7 @@ proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = ## If the value at `name` does not exist, returns nil assert(not isNil(node)) assert(node.kind == JObject) - result = node.fields.getOrDefault(name) + result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` @@ -838,20 +838,20 @@ proc `[]=`*(obj: JsonNode, key: string, val: JsonNode) {.inline.} = proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = ## Traverses the node and gets the given value. If any of the - ## keys do not exist, returns nil. Also returns nil if one of the - ## intermediate data structures is not an object + ## keys do not exist, returns ``nil``. Also returns ``nil`` if one of the + ## intermediate data structures is not an object. result = node for key in keys: - if isNil(result) or result.kind!=JObject: + if isNil(result) or result.kind != JObject: return nil - result=result[key] + result = result.fields.getOrDefault(key) proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location - ## to `value` If any of the keys are missing, they are added + ## to ``value``. If any of the keys are missing, they are added. var node = node for i in 0..(keys.len-2): - if isNil(node[keys[i]]): + if not node.hasKey(keys[i]): node[keys[i]] = newJObject() node = node[keys[i]] node[keys[keys.len-1]] = value @@ -1217,16 +1217,6 @@ when false: # To get that we shall use, obj["json"] when isMainModule: - when not defined(js): - var parsed = parseFile("tests/testdata/jsontest.json") - - try: - discard parsed["key2"][12123] - doAssert(false) - except IndexError: doAssert(true) - - var parsed2 = parseFile("tests/testdata/jsontest2.json") - doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}""" # nil passthrough @@ -1314,3 +1304,19 @@ when isMainModule: var j4 = %*{"test": nil} doAssert j4 == %{"test": newJNull()} + + echo("99% of tests finished. Going to try loading file.") + + # Test loading of file. + when not defined(js): + var parsed = parseFile("tests/testdata/jsontest.json") + + try: + discard parsed["key2"][12123] + doAssert(false) + except IndexError: doAssert(true) + + var parsed2 = parseFile("tests/testdata/jsontest2.json") + doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + + echo("Tests succeeded!") From b22305758dc7c2f1cf7d46173c9f0378a89926f4 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 21:52:49 +0100 Subject: [PATCH 58/87] Fix deprecation warning in net module. --- lib/pure/net.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/net.nim b/lib/pure/net.nim index 6888eb59f..5de6667dd 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -1377,5 +1377,7 @@ proc connect*(socket: Socket, address: string, port = Port(0), when defineSsl: if socket.isSSL: socket.fd.setBlocking(true) + {.warning[Deprecated]: off.} doAssert socket.handshake() + {.warning[Deprecated]: on.} socket.fd.setBlocking(true) From 1446dc87c301268869223e32e7efdc9c76711bad Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 4 Apr 2016 22:50:31 +0100 Subject: [PATCH 59/87] `await x` where x is Fut var now reads after yield. Fixes #3964. --- lib/pure/asyncdispatch.nim | 30 +++++++++++++++++++++++++----- tests/async/tasyncfile.nim | 2 +- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index cc337452f..01b53cb12 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -1316,6 +1316,23 @@ proc generateExceptionCheck(futSym, ) result.add elseNode +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + template createVar(result: var NimNode, futSymName: string, asyncProc: NimNode, valueReceiver, rootReceiver: expr, @@ -1323,9 +1340,7 @@ template createVar(result: var NimNode, futSymName: string, result = newNimNode(nnkStmtList, fromNode) var futSym = genSym(nskVar, "future") result.add newVarStmt(futSym, asyncProc) # -> var future = y - result.add newNimNode(nnkYieldStmt, fromNode).add(futSym) # -> yield future - valueReceiver = newDotExpr(futSym, newIdentNode("read")) # -> future.read - result.add generateExceptionCheck(futSym, tryStmt, rootReceiver, fromNode) + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) proc processBody(node, retFutureSym: NimNode, subTypeIsVoid: bool, @@ -1352,7 +1367,11 @@ proc processBody(node, retFutureSym: NimNode, case node[1].kind of nnkIdent, nnkInfix: # await x - result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x + result = newNimNode(nnkStmtList, node) + var futureValue: NimNode + result.useVar(node[1], futureValue, futureValue, node) + # -> yield x + # -> x.read() of nnkCall, nnkCommand: # await foo(p, x) var futureValue: NimNode @@ -1550,6 +1569,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = for i in 0 .. Date: Tue, 5 Apr 2016 19:45:46 +1000 Subject: [PATCH 60/87] Add `import sharedlist` when `hasThreadSupport` Without this change, a user's Nim code won't compile if they're using both threads & the mark-and-sweep GC: lib/system/gc_ms.nim(75, 18) Error: undeclared identifier: 'SharedList' toDispose: SharedList[pointer] ^ This small code block addition was copied from "lib/system/gc.nim" (where it appears directly after a `when defined(memProfiler)` block also). --- lib/system/gc_ms.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/system/gc_ms.nim b/lib/system/gc_ms.nim index c764571b1..ec69f6e5f 100644 --- a/lib/system/gc_ms.nim +++ b/lib/system/gc_ms.nim @@ -28,6 +28,9 @@ template mulThreshold(x): expr {.immediate.} = x * 2 when defined(memProfiler): proc nimProfile(requestedSize: int) + +when hasThreadSupport: + import sharedlist type WalkOp = enum From 46ee026e77921b25e6f09ff5a5f7f1e9bd6afefe Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 5 Apr 2016 12:11:37 +0100 Subject: [PATCH 61/87] Many additions to readme. --- readme.md | 104 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 22 deletions(-) diff --git a/readme.md b/readme.md index 27b2273f0..fbaa5fd32 100644 --- a/readme.md +++ b/readme.md @@ -1,18 +1,9 @@ -# Nim Compiler - -[![Join the Chat at irc.freenode.net#nim](https://img.shields.io/badge/IRC-join_chat_in_%23nim-blue.svg)](https://webchat.freenode.net/?channels=nim) -[![Get help](https://img.shields.io/badge/Forum-get%20help-4eb899.svg)](http://forum.nim-lang.org) -[![Stackoverflow](https://img.shields.io/badge/stackoverflow-use_%23nim_tag-yellow.svg)](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) -[![Follow @nim_lang!](https://img.shields.io/twitter/follow/nim_lang.svg?style=social)](https://twitter.com/nim_lang) - -[![Travis](https://img.shields.io/travis/nim-lang/Nim.svg)](https://travis-ci.org/nim-lang/Nim) - -[![Contribute to Nim via Gratipay!](https://img.shields.io/gratipay/team/nim.svg)](https://gratipay.com/nim/) -[![Bountysource](https://img.shields.io/bountysource/team/nim/activity.svg)](https://www.bountysource.com/teams/nim) - +# Nim [![Build Status](https://travis-ci.org/nim-lang/Nim.svg?branch=devel)](https://travis-ci.org/nim-lang/Nim) This repo contains the Nim compiler, Nim's stdlib, tools and -documentation. +documentation. For more information about Nim, including downloads +and documentation for the latest release, check out +[Nim's website](http://nim-lang.org). ## Compiling Compiling the Nim compiler is quite straightforward. Because @@ -48,14 +39,15 @@ $ bin/nim c koch $ ./koch boot -d:release ``` -``koch install [dir]`` may then be used to install Nim, but lots of things -don't work then so don't do that. Add it to your PATH instead. More ``koch`` -related options are documented in [doc/koch.txt](doc/koch.txt). +You should then add the ``bin`` directory to your PATH, to make it easily +executable on your system. The above steps can be performed on Windows in a similar fashion, the ``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used instead of ``build.sh``. +The ``koch`` tool is the Nim build tool, more ``koch`` related options are +documented in [doc/koch.txt](doc/koch.txt). ## Nimble [Nimble](https://github.com/nim-lang/nimble) is Nim's package manager. For the @@ -66,12 +58,80 @@ the easiest way of installing Nimble is via: $ nim e install_nimble.nims ``` -## Getting help -A [forum](http://forum.nim-lang.org/) is available if you have any -questions, and you can also get help in the IRC channel on -[Freenode](irc://irc.freenode.net/nim) in #nim. If you ask questions on -[StackOverflow use the nim -tag](http://stackoverflow.com/questions/tagged/nim). +**Warning:** If you install Nimble this way, you will not be able to use binary +Nimble packages or update Nimble easily. +The [Nimble readme](https://github.com/nim-lang/nimble#installation) +provides thorough instructions on how to install Nimble, so that this isn't a +problem. + +## Community +[![Join the Chat at irc.freenode.net#nim](https://img.shields.io/badge/IRC-join_chat_in_%23nim-blue.svg)](https://webchat.freenode.net/?channels=nim) +[![Get help](https://img.shields.io/badge/Forum-get%20help-4eb899.svg)](http://forum.nim-lang.org) +[![Stackoverflow](https://img.shields.io/badge/stackoverflow-use_%23nim_tag-yellow.svg)](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) +[![Follow @nim_lang!](https://img.shields.io/twitter/follow/nim_lang.svg?style=social)](https://twitter.com/nim_lang) + +* The [forum](http://forum.nim-lang.org/) - the best place to ask questions and to discuss Nim. +* [IRC (Freenode#nim)](https://webchat.freenode.net/?channels=nim) - the best place to discuss + Nim in real-time, this is also where most development decision get made! +* [Stackoverflow](http://stackoverflow.com/questions/tagged/nim) + +## Contributing + +[![Contribute to Nim via Gratipay!](https://img.shields.io/gratipay/team/nim.svg)](https://gratipay.com/nim/) +[![Bountysource](https://img.shields.io/bountysource/team/nim/activity.svg)](https://www.bountysource.com/teams/nim) + +We welcome everyone's contributions to Nim. No matter how small or large +the contribution is, anything from small spelling fixes to large modules +intended to be included in the standard library are accepted. Before +you get started, you should know the following about this repositories +structure: + +* ``bin/``, ``build/`` - these directories are empty, but are used when Nim is built. +* ``compiler/`` - the compiler source code, all the Nim source code files in this + directory implement the compiler. This also includes nimfix, and plugins + which live in ``compiler/nimfix`` and ``compiler/plugins`` + respectively. Nimsuggest used to live in the ``compiler`` directory also, + but was moved to https://github.com/nim-lang/nimsuggest. +* ``config/`` - the configuration for the compiler and documentation generator. +* ``doc/`` - the documentation files in reStructuredText format. +* ``lib/`` - where the standard library lives. + * ``pure/`` - modules in the standard library written in pure Nim. + * ``impure/`` - modules in the standard library written in pure Nim which + depend on libraries written in other languages. + * ``wrappers/`` - modules which wrap libraries written in other languages. +* ``tests/`` - contains tests for the compiler and standard library, organised by + category. +* ``tools/`` - the tools including ``niminst`` and ``nimweb``, most of these are invoked + via ``koch``. +* ``web/`` - the Nim website (http://nim-lang.org). +* ``koch.nim`` - tool used to bootstrap Nim, generate C sources, build the website, documentation + and more. + +Most importantly, the ``koch`` tool can be used to run the test suite. To do so compile it first +by executing ``nim c koch``, then execute ``./koch tests``. The test suite takes a while to run, +but you can run specific tests by specifying a category to run, for example ``./koch tests cat async``. + +Make sure that the tests all pass before +[submitting your pull request](https://help.github.com/articles/using-pull-requests/). +If you're short on time, you can +just run the tests specific to your change. Just run the category which corresponds to the change +you've made. When you create your pull request, Travis CI will verify that all the tests pass +anyway. + +If you're looking for things to do, take a look at our +[issue tracker](https://github.com/nim-lang/Nim/issues). There is always plenty of issues +labelled [``Easy``](https://github.com/nim-lang/Nim/labels/Easy), these should be a good +starting point if this is your first contribution to Nim. + +You can also help with the development of Nim by making donations. You can do so +in many ways: + +* [Gratipay](https://gratipay.com/nim/) +* [Bountysource](https://www.bountysource.com/teams/nim) +* Bitcoin - 1BXfuKM2uvoD6mbx4g5xM3eQhLzkCK77tJ + +Finally, if you have any questions feel free to submit a question on the issue tracker, +on the [Nim forum](http://forum.nim-lang.org), or on IRC. ## License The compiler and the standard library are licensed under the MIT license, From 25abb7c2b671d3ea6d28aa871cd3409a0dae8795 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 5 Apr 2016 20:51:13 +0800 Subject: [PATCH 62/87] tester: allow filtering tests by target --- tests/testament/specs.nim | 9 +++++++++ tests/testament/tester.nim | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index bab17d2cd..3f40a5342 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -107,6 +107,15 @@ proc specDefaults*(result: var TSpec) = result.tline = 0 result.tcolumn = 0 +proc parseTargets*(value: string): set[TTarget] = + for v in value.normalize.split: + case v + of "c": result.incl(targetC) + of "cpp", "c++": result.incl(targetCpp) + of "objc": result.incl(targetObjC) + of "js": result.incl(targetJS) + else: echo "target ignored: " & v + proc parseSpec*(filename: string): TSpec = specDefaults(result) result.file = filename diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index b1e8ac099..c0b088124 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -33,6 +33,7 @@ Options: --print also print results to the console --failing only show failing/ignored tests --pedantic return non-zero status code if there are failures + --targets:"c c++ js objc" run tests for specified targets (default: all) """ % resultsFile type @@ -60,6 +61,8 @@ let pegSuccess = peg"'Hint: operation successful'.*" pegOfInterest = pegLineError / pegOtherError +var targets = {low(TTarget)..high(TTarget)} + proc callCompiler(cmdTemplate, filename, options: string, target: TTarget): TSpec = let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], @@ -275,6 +278,11 @@ proc analyzeAndConsolidateOutput(s: string): string = proc testSpec(r: var TResults, test: TTest) = # major entry point for a single test + if test.target notin targets: + r.addResult(test, "", "", reIgnored) + inc(r.skipped) + return + let tname = test.name.addFileExt(".nim") inc(r.total) var expected: TSpec @@ -394,6 +402,7 @@ proc main() = var optPrintResults = false var optFailing = false var optPedantic = false + var p = initOptParser() p.next() while p.kind == cmdLongoption: @@ -401,6 +410,7 @@ proc main() = of "print", "verbose": optPrintResults = true of "failing": optFailing = true of "pedantic": optPedantic = true + of "targets": targets = parseTargets(p.val) else: quit Usage p.next() if p.kind != cmdArgument: quit Usage From 796f4e917c9164c456d8152fb7d57d566f6148d2 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 6 Apr 2016 00:01:57 +0800 Subject: [PATCH 63/87] untaint command line option for parsing --- tests/testament/tester.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index c0b088124..ad687d3ad 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -410,7 +410,7 @@ proc main() = of "print", "verbose": optPrintResults = true of "failing": optFailing = true of "pedantic": optPedantic = true - of "targets": targets = parseTargets(p.val) + of "targets": targets = parseTargets(p.val.string) else: quit Usage p.next() if p.kind != cmdArgument: quit Usage From 3379646c1613c5a29864e5afd350bbf54cdd215c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 5 Apr 2016 21:26:26 +0100 Subject: [PATCH 64/87] Add -d:nimJsonGet flag for json.[] and news entry. --- lib/pure/json.nim | 18 +++++++++++++++--- web/news.txt | 4 ++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 5df9de5fa..5be098664 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -56,6 +56,11 @@ import export tables.`$` +when defined(nimJsonGet): + {.pragma: deprecatedGet, deprecated.} +else: + {.pragma: deprecatedGet.} + type JsonEventKind* = enum ## enumeration of all events that may occur when parsing jsonError, ## an error occurred during parsing @@ -799,16 +804,23 @@ proc len*(n: JsonNode): int = of JObject: result = n.fields.len else: discard -proc `[]`*(node: JsonNode, name: string): JsonNode {.inline.} = +proc `[]`*(node: JsonNode, name: string): JsonNode {.inline, deprecatedGet.} = ## Gets a field from a `JObject`, which must not be nil. - ## If the value at `name` does not exist, returns nil + ## If the value at `name` does not exist, raises KeyError. + ## + ## **Note:** The behaviour of this procedure changed in version 0.14.0. To + ## get a list of usages and to restore the old behaviour of this procedure, + ## compile with the ``-d:nimJsonGet`` flag. assert(not isNil(node)) assert(node.kind == JObject) + when defined(nimJsonGet): + if not node.fields.hasKey(name): return nil result = node.fields[name] proc `[]`*(node: JsonNode, index: int): JsonNode {.inline.} = ## Gets the node at `index` in an Array. Result is undefined if `index` - ## is out of bounds + ## is out of bounds, but as long as array bound checks are enabled it will + ## result in an exception. assert(not isNil(node)) assert(node.kind == JArray) return node.elems[index] diff --git a/web/news.txt b/web/news.txt index 42d2aca99..0a83503c1 100644 --- a/web/news.txt +++ b/web/news.txt @@ -28,6 +28,10 @@ Changes affecting backwards compatibility for more details. - The ``matchers`` module has been deprecated. See issue [#2446](https://github.com/nim-lang/Nim/issues/2446) for more details. +- The ``json.[]`` no longer returns ``nil`` when a key is not found. Instead it + raises a ``KeyError`` exception. You can compile with the ``-d:nimJsonGet`` + flag to get a list of usages of ``[]``, as well as to restore the operator's + previous behaviour. Library Additions From 082ba914bcc18958de9a271dc2fc4dccfe5ed2cf Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 5 Apr 2016 21:45:52 +0100 Subject: [PATCH 65/87] Prevent tester from checking for output diffs for actionRunNoSpec. --- tests/testament/tester.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index b1e8ac099..86e9fbc86 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -336,7 +336,7 @@ proc testSpec(r: var TResults, test: TTest) = reExitCodesDiffer) return - if bufB != expectedOut: + if bufB != expectedOut and expected.action != actionRunNoSpec: if not (expected.substr and expectedOut in bufB): given.err = reOutputsDiffer r.addResult(test, expected.outp, bufB, reOutputsDiffer) From c36421f8e4f85a0a91b490afc1c3bca7da953ad9 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 5 Apr 2016 23:08:51 +0100 Subject: [PATCH 66/87] Fixes subexes tests which were broken in 73e48f9c9. --- lib/pure/subexes.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/subexes.nim b/lib/pure/subexes.nim index 5824ace81..22f29b77c 100644 --- a/lib/pure/subexes.nim +++ b/lib/pure/subexes.nim @@ -390,11 +390,11 @@ when isMainModule: doAssert(("type MyEnum* = enum\n $', '2i'\n '{..}" % ["fieldA", "fieldB", "FiledClkad", "fieldD", "fieldE", "longishFieldName"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type MyEnum* = enum fieldA, fieldB, FiledClkad, fieldD, - fieldE, longishFieldName""") + fieldE, longishFieldName""", 6)) doAssert subex"$1($', '{2..})" % ["f", "a", "b", "c"] == "f(a, b, c)" @@ -404,8 +404,8 @@ when isMainModule: doAssert((subex("type\n Enum = enum\n $', '40c'\n '{..}") % [ "fieldNameA", "fieldNameB", "fieldNameC", "fieldNameD"]).replace(" \n", "\n") == - strutils.unindent """ + strutils.unindent(""" type Enum = enum fieldNameA, fieldNameB, fieldNameC, - fieldNameD""") + fieldNameD""", 6)) From d9ed61fa620a8196724cb51c4b18867cd03df7d9 Mon Sep 17 00:00:00 2001 From: coffeepots Date: Wed, 6 Apr 2016 10:36:17 +0100 Subject: [PATCH 67/87] Deprecate timeToTimeInfo, add str->time procs * depreciate timeToTimeInfo, add str->time procs initTime allows creating times from strings. initTimeFromStr offers a default format string compatible with $Time. timeToTimeInterval changed to work properly. * Spell {.deprecated.} correctly * deprecated timeToTimeInfo, removed initTime procs Is it even worth depreciating timeToTimeInfo rather than just removing it, considering it's just wrong and we have getLocalTime and getGMTime that actually work? Also, if I'm renaming timeToTimeInfo -> toTimeInfo etc, isn't the deprecated tag kind of redundant? * Updated names for toTime and toTimeInterval procs * Added type to toTime template This should allow compilation of os.nim now that toTime is defined in times.nim. * Updated toTime template to use {.gensym.} Local templates default to 'bind' semantics, not 'gensym'. This was causing a failure to 'see' the local toTime template. --- lib/pure/os.nim | 2 +- lib/pure/times.nim | 81 +++++++++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1a2c6d537..dee227c69 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1452,7 +1452,7 @@ template rawToFormalFileInfo(rawInfo, formalInfo): expr = ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix when defined(Windows): - template toTime(e): expr = winTimeToUnixTime(rdFileTime(e)) + template toTime(e: FILETIME): expr {.gensym.} = winTimeToUnixTime(rdFileTime(e)) # local templates default to bind semantics template merge(a, b): expr = a or (b shl 32) formalInfo.id.device = rawInfo.dwVolumeSerialNumber formalInfo.id.file = merge(rawInfo.nFileIndexLow, rawInfo.nFileIndexHigh) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 29ae52d47..6afedefb5 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -182,7 +182,14 @@ proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} + ## converts a broken-down time structure to + ## calendar time representation. The function ignores the specified + ## contents of the structure members `weekday` and `yearday` and recomputes + ## them from the other information in the broken-down time structure. + ## **Warning:** This procedure is deprecated. Use toTime instead. + +proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -356,7 +363,7 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested and it may not be ## very accurate. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) result = getLocalTime(fromSeconds(t + secs)) @@ -365,7 +372,7 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(timeInfoToTime(a)) + let t = toSeconds(toTime(a)) var intval: TimeInterval intval.milliseconds = - interval.milliseconds intval.seconds = - interval.seconds @@ -517,6 +524,11 @@ when not defined(JS): # because the header of mktime is broken in my version of libc return mktime(timeInfoToTM(cTimeInfo)) + proc toTime(timeInfo: TimeInfo): Time = + var cTimeInfo = timeInfo # for C++ we have to make a copy, + # because the header of mktime is broken in my version of libc + return mktime(timeInfoToTM(cTimeInfo)) + proc toStringTillNL(p: cstring): string = result = "" var i = 0 @@ -618,7 +630,16 @@ elif defined(JS): result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - proc `$`(timeInfo: TimeInfo): string = return $(timeInfoToTime(timeInfo)) + proc toTime*(timeInfo: TimeInfo): Time = + result = internGetTime() + result.setSeconds(timeInfo.second) + result.setMinutes(timeInfo.minute) + result.setHours(timeInfo.hour) + result.setMonth(ord(timeInfo.month)) + result.setFullYear(timeInfo.year) + result.setDate(timeInfo.monthday) + + proc `$`(timeInfo: TimeInfo): string = return $(toTime(timeInfo)) proc `$`(time: Time): string = return $time.toLocaleString() proc `-` (a, b: Time): int64 = @@ -708,24 +729,24 @@ proc years*(y: int): TimeInterval {.inline.} = proc `+=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by adding the interval `ti` - t = timeInfoToTime(getLocalTime(t) + ti) + t = toTime(getLocalTime(t) + ti) proc `+`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## by converting to localTime, adding the interval, and converting back ## ## ``echo getTime() + 1.day`` - result = timeInfoToTime(getLocalTime(t) + ti) + result = toTime(getLocalTime(t) + ti) proc `-=`*(t: var Time, ti: TimeInterval) = ## modifies `t` by subtracting the interval `ti` - t = timeInfoToTime(getLocalTime(t) - ti) + t = toTime(getLocalTime(t) - ti) proc `-`*(t: Time, ti: TimeInterval): Time = ## adds the interval `ti` to Time `t` ## ## ``echo getTime() - 1.day`` - result = timeInfoToTime(getLocalTime(t) - ti) + result = toTime(getLocalTime(t) - ti) proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. @@ -1193,7 +1214,7 @@ proc parse*(value, layout: string): TimeInfo = parseToken(info, token, value, j) token = "" # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(timeInfoToTime(info)).weekday + info.weekday = getLocalTime(toTime(info)).weekday return info # Leap year calculations are adapted from: @@ -1254,8 +1275,9 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay = d = (5 + day + y + (y div 4) + (31*m) div 12) mod 7 result = d.WeekDay -proc timeToTimeInfo*(t: Time): TimeInfo = +proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = ## Converts a Time to TimeInfo. + ## **Warning:** This procedure is deprecated. Use getLocalTime or getGMTime instead. let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1286,34 +1308,19 @@ proc timeToTimeInfo*(t: Time): TimeInfo = s = daySeconds mod secondsInMin result = TimeInfo(year: y, yearday: yd, month: m, monthday: md, weekday: wd, hour: h, minute: mi, second: s) -proc timeToTimeInterval*(t: Time): TimeInterval = +proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. - - let - secs = t.toSeconds().int - daysSinceEpoch = secs div secondsInDay - (yearsSinceEpoch, daysRemaining) = countYearsAndDays(daysSinceEpoch) - daySeconds = secs mod secondsInDay - - result.years = yearsSinceEpoch + epochStartYear - - var - mon = mJan - days = daysRemaining - daysInMonth = getDaysInMonth(mon, result.years) - - # calculate month and day remainder - while days > daysInMonth and mon <= mDec: - days -= daysInMonth - mon.inc - daysInMonth = getDaysInMonth(mon, result.years) - - result.months = mon.int + 1 # month is 1 indexed int - result.days = days - result.hours = daySeconds div secondsInHour + 1 - result.minutes = (daySeconds mod secondsInHour) div secondsInMin - result.seconds = daySeconds mod secondsInMin + ## **Warning:** This procedure is deprecated. Use toTimeInterval instead. # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + +proc toTimeInterval*(t: Time): TimeInterval = + ## Converts a Time to a TimeInterval. + # Milliseconds not available from Time + var tInfo = t.getLocalTime() + initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) + when isMainModule: # this is testing non-exported function @@ -1328,6 +1335,6 @@ when isMainModule: assert toSeconds(t4L, initInterval(days=1)) == toSeconds(t4, initInterval(days=1)) assert toSeconds(t4L, initInterval(months=1)) == toSeconds(t4, initInterval(months=1)) assert toSeconds(t4L, initInterval(years=1)) == toSeconds(t4, initInterval(years=1)) - + # Further tests are in tests/stdlib/ttime.nim # koch test c stdlib From 9b6fda144a08733aad91b43b043fc3b89b72ed3c Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 6 Apr 2016 10:45:39 +0100 Subject: [PATCH 68/87] Some documentation fixes in times module. --- lib/pure/times.nim | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/pure/times.nim b/lib/pure/times.nim index 6afedefb5..e0ee884a8 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -187,7 +187,9 @@ proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes ## them from the other information in the broken-down time structure. - ## **Warning:** This procedure is deprecated. Use toTime instead. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTime`` instead. proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} ## converts a broken-down time structure to @@ -1225,7 +1227,7 @@ proc parse*(value, layout: string): TimeInfo = proc countLeapYears*(yearSpan: int): int = ## Returns the number of leap years spanned by a given number of years. ## - ## Note: for leap years, start date is assumed to be 1 AD. + ## **Note:** For leap years, start date is assumed to be 1 AD. ## counts the number of leap years up to January 1st of a given year. ## Keep in mind that if specified year is a leap year, the leap day ## has not happened before January 1st of that year. @@ -1260,13 +1262,14 @@ proc getDayOfWeek*(day, month, year: int): WeekDay = y = year - a m = month + (12*a) - 2 d = (day + y + (y div 4) - (y div 100) + (y div 400) + (31*m) div 12) mod 7 - # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. so we must correct - # for the WeekDay type. + # The value of d is 0 for a Sunday, 1 for a Monday, 2 for a Tuesday, etc. + # so we must correct for the WeekDay type. if d == 0: return dSun result = (d-1).WeekDay proc getDayOfWeekJulian*(day, month, year: int): WeekDay = - ## Returns the day of the week enum from day, month and year, according to the Julian calendar. + ## Returns the day of the week enum from day, month and year, + ## according to the Julian calendar. # Day & month start from one. let a = (14 - month) div 12 @@ -1277,7 +1280,9 @@ proc getDayOfWeekJulian*(day, month, year: int): WeekDay = proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = ## Converts a Time to TimeInfo. - ## **Warning:** This procedure is deprecated. Use getLocalTime or getGMTime instead. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``getLocalTime`` or ``getGMTime`` instead. let secs = t.toSeconds().int daysSinceEpoch = secs div secondsInDay @@ -1310,7 +1315,9 @@ proc timeToTimeInfo*(t: Time): TimeInfo {.deprecated.} = proc timeToTimeInterval*(t: Time): TimeInterval {.deprecated.} = ## Converts a Time to a TimeInterval. - ## **Warning:** This procedure is deprecated. Use toTimeInterval instead. + ## + ## **Warning:** This procedure is deprecated since version 0.14.0. + ## Use ``toTimeInterval`` instead. # Milliseconds not available from Time var tInfo = t.getLocalTime() initInterval(0, tInfo.second, tInfo.minute, tInfo.hour, tInfo.weekday.ord, tInfo.month.ord, tInfo.year) @@ -1335,6 +1342,6 @@ when isMainModule: assert toSeconds(t4L, initInterval(days=1)) == toSeconds(t4, initInterval(days=1)) assert toSeconds(t4L, initInterval(months=1)) == toSeconds(t4, initInterval(months=1)) assert toSeconds(t4L, initInterval(years=1)) == toSeconds(t4, initInterval(years=1)) - + # Further tests are in tests/stdlib/ttime.nim # koch test c stdlib From 181c834a932ac98bc4451c6f0110cf9816b9808b Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Tue, 5 Apr 2016 22:55:00 +0300 Subject: [PATCH 69/87] Added json.getOrDefault along with {singleKey} tr optimization. --- lib/pure/json.nim | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 5be098664..b9da8a0dd 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -858,6 +858,14 @@ proc `{}`*(node: JsonNode, keys: varargs[string]): JsonNode = return nil result = result.fields.getOrDefault(key) +proc getOrDefault*(node: JsonNode, key: string): JsonNode = + ## Gets a field from a `node`. If `node` is nil or not an object or + ## value at `key` does not exist, returns nil + if not isNil(node) and node.kind == JObject: + result = node.fields.getOrDefault(key) + +template simpleGetOrDefault*{`{}`(node, [key])}(node: JsonNode, key: string): JsonNode = node.getOrDefault(key) + proc `{}=`*(node: JsonNode, keys: varargs[string], value: JsonNode) = ## Traverses the node and tries to set the value at the given location ## to ``value``. If any of the keys are missing, they are added. From b1b14a5e7b3f4f86fc32ab03c5a1666043a9c383 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Wed, 6 Apr 2016 22:24:32 +0100 Subject: [PATCH 70/87] Add Bountysource banner to website. --- tools/website.tmpl | 12 +++++++++++- web/assets/style.css | 11 +++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tools/website.tmpl b/tools/website.tmpl index 50152a051..6ae975839 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -6,13 +6,23 @@ $c.projectTitle - + #if len(rss) > 0: #end if + +