From e0059287bb971a52a11d9ba5129dbbf01a6762df Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Thu, 4 May 2017 16:27:08 +0700 Subject: [PATCH 01/22] Add 'hostname' param to wrapConnectedSocket --- lib/pure/asyncnet.nim | 9 +- lib/pure/httpclient.nim | 8 +- lib/pure/net.nim | 358 ++++++++++++++++++++-------------------- 3 files changed, 195 insertions(+), 180 deletions(-) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 11b7998b2..9f73bc3cf 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -647,9 +647,12 @@ when defineSsl: sslSetBio(socket.sslHandle, socket.bioIn, socket.bioOut) proc wrapConnectedSocket*(ctx: SslContext, socket: AsyncSocket, - handshake: SslHandshakeType) = + handshake: SslHandshakeType, + hostname: string = nil) = ## Wraps a connected socket in an SSL context. This function effectively ## turns ``socket`` into an SSL socket. + ## ``hostname`` should be specified so that the client knows which hostname + ## the server certificate should be validated against. ## ## This should be called on a connected socket, and will perform ## an SSL handshake immediately. @@ -660,6 +663,10 @@ when defineSsl: case handshake of handshakeAsClient: + if not hostname.isNil and not isIpAddress(hostname): + # Set the SNI address for this connection. This call can fail if + # we're not using TLSv1+. + discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) sslSetConnectState(socket.sslHandle) of handshakeAsServer: sslSetAcceptState(socket.sslHandle) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 1b8a20b65..4f43177a8 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -512,7 +512,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", raise newException(HttpRequestError, "The proxy server rejected a CONNECT request, " & "so a secure connection could not be established.") - sslContext.wrapConnectedSocket(s, handshakeAsClient) + sslContext.wrapConnectedSocket(s, handshakeAsClient, hostUrl.hostname) else: raise newException(HttpRequestError, "SSL support not available. Cannot connect via proxy over SSL") else: @@ -1060,7 +1060,8 @@ proc newConnection(client: HttpClient | AsyncHttpClient, when defined(ssl): if isSsl: try: - client.sslContext.wrapConnectedSocket(client.socket, handshakeAsClient) + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, url.hostname) except: client.socket.close() raise getCurrentException() @@ -1102,7 +1103,8 @@ proc requestAux(client: HttpClient | AsyncHttpClient, url: string, raise newException(HttpRequestError, "The proxy server rejected a CONNECT request, " & "so a secure connection could not be established.") - client.sslContext.wrapConnectedSocket(client.socket, handshakeAsClient) + client.sslContext.wrapConnectedSocket( + client.socket, handshakeAsClient, requestUrl.hostname) client.proxy = nil else: raise newException(HttpRequestError, diff --git a/lib/pure/net.nim b/lib/pure/net.nim index d175bd537..629e916fa 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -237,6 +237,180 @@ proc newSocket*(domain: Domain = AF_INET, sockType: SockType = SOCK_STREAM, raiseOSError(osLastError()) result = newSocket(fd, domain, sockType, protocol, buffered) +proc parseIPv4Address(address_str: string): IpAddress = + ## Parses IPv4 adresses + ## Raises EInvalidValue on errors + var + byteCount = 0 + currentByte:uint16 = 0 + seperatorValid = false + + result.family = IpAddressFamily.IPv4 + + for i in 0 .. high(address_str): + if address_str[i] in strutils.Digits: # Character is a number + currentByte = currentByte * 10 + + cast[uint16](ord(address_str[i]) - ord('0')) + if currentByte > 255'u16: + raise newException(ValueError, + "Invalid IP Address. Value is out of range") + seperatorValid = true + elif address_str[i] == '.': # IPv4 address separator + if not seperatorValid or byteCount >= 3: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + result.address_v4[byteCount] = cast[uint8](currentByte) + currentByte = 0 + byteCount.inc + seperatorValid = false + else: + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not seperatorValid: + raise newException(ValueError, "Invalid IP Address") + result.address_v4[byteCount] = cast[uint8](currentByte) + +proc parseIPv6Address(address_str: string): IpAddress = + ## Parses IPv6 adresses + ## Raises EInvalidValue on errors + result.family = IpAddressFamily.IPv6 + if address_str.len < 2: + raise newException(ValueError, "Invalid IP Address") + + var + groupCount = 0 + currentGroupStart = 0 + currentShort:uint32 = 0 + seperatorValid = true + dualColonGroup = -1 + lastWasColon = false + v4StartPos = -1 + byteCount = 0 + + for i,c in address_str: + if c == ':': + if not seperatorValid: + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid seperator") + if lastWasColon: + if dualColonGroup != -1: + raise newException(ValueError, + "Invalid IP Address. Address contains more than one \"::\" seperator") + dualColonGroup = groupCount + seperatorValid = false + elif i != 0 and i != high(address_str): + if groupCount >= 8: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + currentShort = 0 + groupCount.inc() + if dualColonGroup != -1: seperatorValid = false + elif i == 0: # only valid if address starts with :: + if address_str[1] != ':': + raise newException(ValueError, + "Invalid IP Address. Address may not start with \":\"") + else: # i == high(address_str) - only valid if address ends with :: + if address_str[high(address_str)-1] != ':': + raise newException(ValueError, + "Invalid IP Address. Address may not end with \":\"") + lastWasColon = true + currentGroupStart = i + 1 + elif c == '.': # Switch to parse IPv4 mode + if i < 3 or not seperatorValid or groupCount >= 7: + raise newException(ValueError, "Invalid IP Address") + v4StartPos = currentGroupStart + currentShort = 0 + seperatorValid = false + break + elif c in strutils.HexDigits: + if c in strutils.Digits: # Normal digit + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) + elif c >= 'a' and c <= 'f': # Lower case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 + else: # Upper case hex + currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 + if currentShort > 65535'u32: + raise newException(ValueError, + "Invalid IP Address. Value is out of range") + lastWasColon = false + seperatorValid = true + else: + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid character") + + + if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff + if seperatorValid: # Copy remaining data + if groupCount >= 8: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) + result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) + groupCount.inc() + else: # Must parse IPv4 address + for i,c in address_str[v4StartPos..high(address_str)]: + if c in strutils.Digits: # Character is a number + currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) + if currentShort > 255'u32: + raise newException(ValueError, + "Invalid IP Address. Value is out of range") + seperatorValid = true + elif c == '.': # IPv4 address separator + if not seperatorValid or byteCount >= 3: + raise newException(ValueError, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + currentShort = 0 + byteCount.inc() + seperatorValid = false + else: # Invalid character + raise newException(ValueError, + "Invalid IP Address. Address contains an invalid character") + + if byteCount != 3 or not seperatorValid: + raise newException(ValueError, "Invalid IP Address") + result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) + groupCount += 2 + + # Shift and fill zeros in case of :: + if groupCount > 8: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + elif groupCount < 8: # must fill + if dualColonGroup == -1: + raise newException(ValueError, + "Invalid IP Address. The address consists of too few groups") + var toFill = 8 - groupCount # The number of groups to fill + var toShift = groupCount - dualColonGroup # Nr of known groups after :: + for i in 0..2*toShift-1: # shift + result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] + for i in 0..2*toFill-1: # fill with 0s + result.address_v6[dualColonGroup*2+i] = 0 + elif dualColonGroup != -1: + raise newException(ValueError, + "Invalid IP Address. The address consists of too many groups") + +proc parseIpAddress*(address_str: string): IpAddress = + ## Parses an IP address + ## Raises EInvalidValue on error + if address_str == nil: + raise newException(ValueError, "IP Address string is nil") + if address_str.contains(':'): + return parseIPv6Address(address_str) + else: + return parseIPv4Address(address_str) + +proc isIpAddress*(address_str: string): bool {.tags: [].} = + ## Checks if a string is an IP address + ## Returns true if it is, false otherwise + try: + discard parseIpAddress(address_str) + except ValueError: + return false + return true + when defineSsl: CRYPTO_malloc_init() SslLibraryInit() @@ -438,9 +612,12 @@ when defineSsl: raiseSSLError() proc wrapConnectedSocket*(ctx: SSLContext, socket: Socket, - handshake: SslHandshakeType) = + handshake: SslHandshakeType, + hostname: string = nil) = ## Wraps a connected socket in an SSL context. This function effectively ## turns ``socket`` into an SSL socket. + ## ``hostname`` should be specified so that the client knows which hostname + ## the server certificate should be validated against. ## ## This should be called on a connected socket, and will perform ## an SSL handshake immediately. @@ -450,6 +627,10 @@ when defineSsl: wrapSocket(ctx, socket) case handshake of handshakeAsClient: + if not hostname.isNil and not isIpAddress(hostname): + # Discard result in case OpenSSL version doesn't support SNI, or we're + # not using TLSv1+ + discard SSL_set_tlsext_host_name(socket.sslHandle, hostname) let ret = SSLConnect(socket.sslHandle) socketError(socket, ret) of handshakeAsServer: @@ -1302,181 +1483,6 @@ proc `$`*(address: IpAddress): string = mask = mask shr 4 printedLastGroup = true -proc parseIPv4Address(address_str: string): IpAddress = - ## Parses IPv4 adresses - ## Raises EInvalidValue on errors - var - byteCount = 0 - currentByte:uint16 = 0 - seperatorValid = false - - result.family = IpAddressFamily.IPv4 - - for i in 0 .. high(address_str): - if address_str[i] in strutils.Digits: # Character is a number - currentByte = currentByte * 10 + - cast[uint16](ord(address_str[i]) - ord('0')) - if currentByte > 255'u16: - raise newException(ValueError, - "Invalid IP Address. Value is out of range") - seperatorValid = true - elif address_str[i] == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - result.address_v4[byteCount] = cast[uint8](currentByte) - currentByte = 0 - byteCount.inc - seperatorValid = false - else: - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid character") - - if byteCount != 3 or not seperatorValid: - raise newException(ValueError, "Invalid IP Address") - result.address_v4[byteCount] = cast[uint8](currentByte) - -proc parseIPv6Address(address_str: string): IpAddress = - ## Parses IPv6 adresses - ## Raises EInvalidValue on errors - result.family = IpAddressFamily.IPv6 - if address_str.len < 2: - raise newException(ValueError, "Invalid IP Address") - - var - groupCount = 0 - currentGroupStart = 0 - currentShort:uint32 = 0 - seperatorValid = true - dualColonGroup = -1 - lastWasColon = false - v4StartPos = -1 - byteCount = 0 - - for i,c in address_str: - if c == ':': - if not seperatorValid: - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid seperator") - if lastWasColon: - if dualColonGroup != -1: - raise newException(ValueError, - "Invalid IP Address. Address contains more than one \"::\" seperator") - dualColonGroup = groupCount - seperatorValid = false - elif i != 0 and i != high(address_str): - if groupCount >= 8: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) - result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) - currentShort = 0 - groupCount.inc() - if dualColonGroup != -1: seperatorValid = false - elif i == 0: # only valid if address starts with :: - if address_str[1] != ':': - raise newException(ValueError, - "Invalid IP Address. Address may not start with \":\"") - else: # i == high(address_str) - only valid if address ends with :: - if address_str[high(address_str)-1] != ':': - raise newException(ValueError, - "Invalid IP Address. Address may not end with \":\"") - lastWasColon = true - currentGroupStart = i + 1 - elif c == '.': # Switch to parse IPv4 mode - if i < 3 or not seperatorValid or groupCount >= 7: - raise newException(ValueError, "Invalid IP Address") - v4StartPos = currentGroupStart - currentShort = 0 - seperatorValid = false - break - elif c in strutils.HexDigits: - if c in strutils.Digits: # Normal digit - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('0')) - elif c >= 'a' and c <= 'f': # Lower case hex - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('a')) + 10 - else: # Upper case hex - currentShort = (currentShort shl 4) + cast[uint32](ord(c) - ord('A')) + 10 - if currentShort > 65535'u32: - raise newException(ValueError, - "Invalid IP Address. Value is out of range") - lastWasColon = false - seperatorValid = true - else: - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid character") - - - if v4StartPos == -1: # Don't parse v4. Copy the remaining v6 stuff - if seperatorValid: # Copy remaining data - if groupCount >= 8: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - result.address_v6[groupCount*2] = cast[uint8](currentShort shr 8) - result.address_v6[groupCount*2+1] = cast[uint8](currentShort and 0xFF) - groupCount.inc() - else: # Must parse IPv4 address - for i,c in address_str[v4StartPos..high(address_str)]: - if c in strutils.Digits: # Character is a number - currentShort = currentShort * 10 + cast[uint32](ord(c) - ord('0')) - if currentShort > 255'u32: - raise newException(ValueError, - "Invalid IP Address. Value is out of range") - seperatorValid = true - elif c == '.': # IPv4 address separator - if not seperatorValid or byteCount >= 3: - raise newException(ValueError, "Invalid IP Address") - result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) - currentShort = 0 - byteCount.inc() - seperatorValid = false - else: # Invalid character - raise newException(ValueError, - "Invalid IP Address. Address contains an invalid character") - - if byteCount != 3 or not seperatorValid: - raise newException(ValueError, "Invalid IP Address") - result.address_v6[groupCount*2 + byteCount] = cast[uint8](currentShort) - groupCount += 2 - - # Shift and fill zeros in case of :: - if groupCount > 8: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - elif groupCount < 8: # must fill - if dualColonGroup == -1: - raise newException(ValueError, - "Invalid IP Address. The address consists of too few groups") - var toFill = 8 - groupCount # The number of groups to fill - var toShift = groupCount - dualColonGroup # Nr of known groups after :: - for i in 0..2*toShift-1: # shift - result.address_v6[15-i] = result.address_v6[groupCount*2-i-1] - for i in 0..2*toFill-1: # fill with 0s - result.address_v6[dualColonGroup*2+i] = 0 - elif dualColonGroup != -1: - raise newException(ValueError, - "Invalid IP Address. The address consists of too many groups") - - -proc parseIpAddress*(address_str: string): IpAddress = - ## Parses an IP address - ## Raises EInvalidValue on error - if address_str == nil: - raise newException(ValueError, "IP Address string is nil") - if address_str.contains(':'): - return parseIPv6Address(address_str) - else: - return parseIPv4Address(address_str) - -proc isIpAddress*(address_str: string): bool {.tags: [].} = - ## Checks if a string is an IP address - ## Returns true if it is, false otherwise - try: - discard parseIpAddress(address_str) - except ValueError: - return false - return true - proc dial*(address: string, port: Port, protocol = IPPROTO_TCP, buffered = true): Socket {.tags: [ReadIOEffect, WriteIOEffect].} = From c24dc7944aae2aa108b2638a9c70a5edf5e47915 Mon Sep 17 00:00:00 2001 From: Matt Riggott Date: Thu, 4 May 2017 18:09:24 +0000 Subject: [PATCH 02/22] Add space to fix RST in captureBetween docs (#5769) --- lib/pure/parseutils.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim index 8d53a0360..b78e8d000 100644 --- a/lib/pure/parseutils.nim +++ b/lib/pure/parseutils.nim @@ -201,7 +201,7 @@ proc parseWhile*(s: string, token: var string, validChars: set[char], proc captureBetween*(s: string, first: char, second = '\0', start = 0): string = ## Finds the first occurrence of ``first``, then returns everything from there - ## up to ``second``(if ``second`` is '\0', then ``first`` is used). + ## up to ``second`` (if ``second`` is '\0', then ``first`` is used). var i = skipUntil(s, first, start)+1+start result = "" discard s.parseUntil(result, if second == '\0': first else: second, i) From c0669326357cab1af0cc93bf562eeb5b7be484a0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 4 May 2017 22:25:29 +0200 Subject: [PATCH 03/22] added IP v6 parsing tests --- tests/stdlib/tparseipv6.nim | 217 ++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 tests/stdlib/tparseipv6.nim diff --git a/tests/stdlib/tparseipv6.nim b/tests/stdlib/tparseipv6.nim new file mode 100644 index 000000000..3e1c23e58 --- /dev/null +++ b/tests/stdlib/tparseipv6.nim @@ -0,0 +1,217 @@ +discard """ + output: "all ok" +""" + +import net + +const + positives = [ + "::f:8:a8f:218.17.235.229", + "::b:228.19.241.2", + "::8:c:a:f:8.35.8.096", + "::3:e:a:bc:04.19.2.9", + "::2:212.242.248.19", + "::df:a5f:3.250.208.9", + "::8:c:5:e63:250.208.249.0", + "::b:f:181.12.9.98", + "::a:f8:77.08.243.232", + "::a:b:85:e4d9:252.9.229.056", + "941:c:8a:c:e::917", + "e8:7a:e:ad:88a:8:203.235.225.46", + "139c:9e::f8:254.08.21.249", + "b38:f0:e::f9:89.6.12.18", + "ef::8", + "5::ab", + "a::8:255.247.96.253", + "b:c0::c:254.248.95.254", + "::8c:2:99.251.024.3", + "98::c:247.240.249.57", + "9::9", + "628::f1ed:f", + "c::cca8", + "2::3:c", + "fde::8fcc:92:e", + "f::3", + "e85::7", + "8::b:f6", + "0::6:8ca", + "c8::6e:be8", + "87::e", + "6:9::a7:9", + "c::5", + "49::1:62", + "df:c0::f:9", + "a09a:8::21:887a", + "2:f::c", + "8bf5:5::2a6e:f8f", + "a:9e::bc:a", + "f:60::c:fd", + "59::52f:0:fa7", + "8268:6cf::f:9", + "c:abb::f", + "a:ff8d::9:7", + "05:c87::9c:9a", + "e:f::c:9a:1", + "ff6:8::962:e", + "9::bd", + "68:ec::6", + "3b8:f::94:3e9:9952", + "49b4:ae::899:b4", + "cb9:8e8:af::f4", + "8::10:9ae6:f9", + "b9::2:57", + "ff:fba9::d", + "4::a:8", + "caa:c:85a::2:3", + "5::a5:9", + "c:ad::a", + "9a:f:f65::b", + "f:df::9:0", + "c:b9::8de", + "d:f::a", + "ab88:d4:0::fc:8d", + "8f:ee2::3", + "f:f8::bf2:8c8", + "8::efc", + "e:5a::b", + "c:48::94", + "a:b:5::8", + "f:88f::f0a6", + "9:f:e::3", + "b::fedd", + "7b:f::c", + "edf4:7d::88", + "89::d", + "c0:a:62::ac", + "7:f::b", + "8::a2", + "0f::1", + "::", + "b:8::", + "44:a::", + "ef8f::", + "b:4:d::", + "a::", + "5a:8::", + "ddaf:ecbf::", + "f:bb:a1::", + "f8:f::", + "::e:38:ab:f8", + "::cd:c", + "::aa3:eb", + "::bf:9f9", + "::7ef:bf8a", + "::9", + "::a:9af", + "::315", + "::a:a", + "::aed3:a", + "f0eb:0:e8:b:c:a:254.098.233.17", + "bfa:7fc:c66d:15:e9a:ded:254.119.9.9", + "d:ffa8:9:a:879:3:202.39.08.245", + "8e:2:8:fa8a:f1d1:1aa8:252.254.245.81", + "5:d4:a:e9:8:8:06.38.98.253", + "9c5:4:a5c:f:a6:8c9d:05.250.8.2", + "d19a:2:f808:be:f:c:98.86.197.249", + "8:26ac:8:8:cb:f:242.00.254.85", + "38:e:1:0b88:f:0:8.89.248.92", + "e7:ff96:a:f:f:b:253.91.052.195", + "d:8:2:5:894:5:254.0.240.199", + "2:98:9:8aa:9c8f:fa:252.98.248.17", + "e9:d4f:890:ccbe:5:8:088.200.228.216", + "3:3:9:5:6a:df5:255.251.8.12", + "0280:3:8:8:4:9:255.000.251.249", + "8:af7:db:aa:0:9:238.248.250.255", + "ff:ee:9a:9252:a:289:059.083.18.255", + "9f6:5:fc9:b:a89:a:142.1.250.254", + "e:981a:da:bf94:9:f8:254.242.18.95", + "3c:1:4:f2:89:f:8.91.255.14", + "e::9a2:c:9.050.80.8", + "9::4a:07:fb:211.241.254.228", + "9be::2:e:215.189.48.188", + "f::f:d:069.148.99.168", + "f::a:97.18.240.47", + "c::a98e:1:251.253.252.254", + "668::82:214.87.208.9", + "9c0::cf0:ecb:253.208.238.255", + "a::0:f1:210.240.238.049", + "8::a:1:251.238.34.09", + "81:dfe::b8:8.255.249.248", + "d3::7:b:9:83.189.08.244", + "8::9:8:8:00.7.11.252", + "2:8::c:a8:250.221.9.249", + "2::f:99.8.249.247", + "c:22f5::5:2c:243.15.79.89", + "e:8e::da:251.243.255.2", + "f15f:9::a:255.70.247.218", + "f:b::9f38:31.220.94.022", + "9::9a48:03.98.249.119", + "d:d:9b87::2d:a:249.253.38.8", + "d86d:99b::a9b:5:242.236.8.244", + "eb:3::f:9cf:1.253.1.228", + "b::ba2:255.247.114.64", + "2f:ec:bcb::9:219.254.250.094", + "da8a:f6::a:e0:19.251.241.251", + "5e:c1::a:021.250.8.254", + "c:9::8c9b:248.219.212.252", + "2:a::8d4a:216.255.198.223", + "1f::66:255.30.08.150", + "bc2b:8f::2ff9:6.245.99.230", + "a:8::a8:9.251.246.255", + "f:7:7::98:06.14.1.208", + "e:2::9:218.249.255.254", + "79:f::6:250.255.98.246", + "47:9:fb9f::9:038.136.17.251", + "ed::a:247.9.23.239", + "6f::f1:88.254.119.9", + "a::d:218.199.236.0", + "fc88::9:203.196.04.95", + "::8.048.255.85", + "::253.07.255.36", + "9:d::253.7.178.229", + "::250.84.158.253", + "::8.55.204.248", + "2d:c::253.18.18.252", + "df9:88ca::248.255.108.17", + "8e9b::250.206.0.82", + "::209.8.254.209", + "::247.088.8.8", + "::cb:f:ba41:250.208.19.249", + "::fe:0e8:243.240.229.5", + "::c:223.251.5.226", + "::8:08.03.8.250", + "::f:8.88.11.255", + "::fda:48:aa:05.189.07.2", + "::8:c3f:f:240.06.212.255", + "::f:0aa:244.123.99.16", + "::c9b5:c:034.8.090.196", + "::98:c9:254.14.241.81" + ] + negatives = ["foo.bar", + "::::::::::::", + "yet another failure", + "de:6:c:ab5:6a::9:252.06.06.249", + "f9:5f7:fa38:9:b::b6:09.255.248.252", + "97:c:5b:81:8a::f5dd:144.252.250.9", + "9:8:cd:8:a9::f:247.255.09.255", + "18:1:8c:2:3::9:8.254.252.139", + "e:c298:3:e:a::bb12:254.246.05.250", + "e:e:c:8e:fd::8:253.8.49.231", + "9:97f:f:e929:8a::c9:0.8.252.10", + "0df:b24:7:89:c::2b:16.249.240.092", + "b:8f5f:485:c:9a::84c:178.7.249.34", + ] + +proc ok(pos: openarray[string]) = + for p in pos: + if not isIpAddress(p): + echo "failure ", p + +proc notok(neg: openarray[string]) = + for n in neg: + if isIpAddress(n): + echo "failure ", n + +ok(positives) +notok(negatives) +echo "all ok" From 78315792d33a26e519278aec650f76022f53082b Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Fri, 5 May 2017 03:33:52 +0700 Subject: [PATCH 04/22] Implement 'take' for Table and TableRef (#5773) --- lib/pure/collections/tables.nim | 41 ++++++++++++++++++++++++--------- tests/collections/ttables.nim | 15 ++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 969802cfc..323af5a38 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -269,6 +269,18 @@ proc del*[A, B](t: var Table[A, B], key: A) = ## deletes `key` from hash table `t`. delImpl() +proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool = + ## Deletes the ``key`` from the table. + ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the + ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## unchanged. + var hc: Hash + var index = rawGet(t, key, hc) + result = index >= 0 + if result: + shallowCopy(val, t.data[index].val) + delImplIdx(t, index) + proc enlarge[A, B](t: var Table[A, B]) = var n: KeyValuePairSeq[A, B] newSeq(n, len(t.data) * growthFactor) @@ -424,6 +436,13 @@ proc del*[A, B](t: TableRef[A, B], key: A) = ## deletes `key` from hash table `t`. t[].del(key) +proc take*[A, B](t: TableRef[A, B], key: A, val: var B): bool = + ## Deletes the ``key`` from the table. + ## Returns ``true``, if the ``key`` existed, and sets ``val`` to the + ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is + ## unchanged. + result = t[].take(key, val) + proc newTable*[A, B](initialSize=64): TableRef[A, B] = new(result) result[] = initTable[A, B](initialSize) @@ -625,7 +644,7 @@ proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = while ht >= 0 and hs >= 0: var nxtt = t.data[ht].next var nxts = s.data[hs].next - if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode): + if isFilled(t.data[ht].hcode) and isFilled(s.data[hs].hcode): if (s.data[hs].key != t.data[ht].key) and (s.data[hs].val != t.data[ht].val): return false ht = nxtt @@ -829,7 +848,7 @@ proc clear*[A](t: CountTableRef[A]) = proc clear*[A](t: var CountTable[A]) = ## Resets the table so that it is empty. clearImpl() - + iterator pairs*[A](t: CountTable[A]): (A, int) = ## iterates over any (key, value) pair in the table `t`. for h in 0..high(t.data): @@ -1256,17 +1275,17 @@ when isMainModule: var b = newOrderedTable[string, string](initialSize=2) b.add("wrong?", "foo") b.add("wrong?", "foo2") - assert a == b + assert a == b block: #5482 - var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() + var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() var b = newOrderedTable[string, string](initialSize=2) b.add("wrong?", "foo") b.add("wrong?", "foo2") - assert a == b + assert a == b block: #5487 - var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() + var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() var b = newOrderedTable[string, string]() # notice, default size! b.add("wrong?", "foo") b.add("wrong?", "foo2") @@ -1279,13 +1298,13 @@ when isMainModule: b.add("wrong?", "foo2") assert a == b - block: - var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() - var b = [("wrong?","foo"), ("wrong?", "foo2")].newOrderedTable() + block: + var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable() + var b = [("wrong?","foo"), ("wrong?", "foo2")].newOrderedTable() var c = newOrderedTable[string, string]() # notice, default size! c.add("wrong?", "foo") - c.add("wrong?", "foo2") + c.add("wrong?", "foo2") assert a == b assert a == c - + diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index a2988e841..01dab44fc 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -235,6 +235,21 @@ block withKeyTest: except KeyError: discard +block takeTest: + var t = initTable[string, int]() + t["key"] = 123 + + var val = 0 + assert(t.take("key", val)) + assert(val == 123) + + val = -1 + assert(not t.take("key", val)) + assert(val == -1) + + assert(not t.take("otherkey", val)) + assert(val == -1) + proc orderedTableSortTest() = var t = initOrderedTable[string, int](2) for key, val in items(data): t[key] = val From 6c7d2ce8fb104af8f880e6a2973c012174604a5c Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 4 May 2017 22:37:07 +0200 Subject: [PATCH 05/22] disable fragile parts of thttpclient.nim --- tests/stdlib/thttpclient.nim | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim index 40324d92a..c6a857edb 100644 --- a/tests/stdlib/thttpclient.nim +++ b/tests/stdlib/thttpclient.nim @@ -104,15 +104,17 @@ proc syncTest() = client.close() - # Timeout test. - client = newHttpClient(timeout = 1) - try: - resp = client.request("http://example.com/") - doAssert false, "TimeoutError should have been raised." - except TimeoutError: - discard - except: - doAssert false, "TimeoutError should have been raised." + when false: + # Disabled for now because it causes troubles with AppVeyor + # Timeout test. + client = newHttpClient(timeout = 1) + try: + resp = client.request("http://example.com/") + doAssert false, "TimeoutError should have been raised." + except TimeoutError: + discard + except: + doAssert false, "TimeoutError should have been raised." proc makeIPv6HttpServer(hostname: string, port: Port): AsyncFD = let fd = newNativeSocket(AF_INET6) From c22e0d414ddff0404a0adb1a8fc18d867e49ecd9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 6 May 2017 11:53:15 +0200 Subject: [PATCH 06/22] fixes an internal compiler error when json.to is used wrongly --- compiler/vm.nim | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/vm.nim b/compiler/vm.nim index 6a9545193..3c475cf57 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1651,8 +1651,16 @@ proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode, for i in 0 .. Date: Sun, 7 May 2017 21:54:22 -0500 Subject: [PATCH 07/22] setGlobalDispatcher() --- lib/pure/asyncdispatch.nim | 10 ++++++++++ lib/upcoming/asyncdispatch.nim | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 6a877be30..e48fe8d94 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -242,6 +242,11 @@ when defined(windows) or defined(nimdoc): if gDisp.isNil: gDisp = newDispatcher() result = gDisp + proc setGlobalDispatcher*(disp: PDispatcher) = + if not gDisp.isNil: + assert gDisp.callbacks.len == 0 + gDisp = disp + proc register*(fd: AsyncFD) = ## Registers ``fd`` with the dispatcher. let p = getGlobalDispatcher() @@ -931,6 +936,11 @@ else: if gDisp.isNil: gDisp = newDispatcher() result = gDisp + proc setGlobalDispatcher*(disp: PDispatcher) = + if not gDisp.isNil: + assert gDisp.callbacks.len == 0 + gDisp = disp + proc update(fd: AsyncFD, events: set[Event]) = let p = getGlobalDispatcher() assert fd.SocketHandle in p.selector diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 7c0497cc6..8541bf98c 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -219,6 +219,11 @@ when defined(windows) or defined(nimdoc): if gDisp.isNil: gDisp = newDispatcher() result = gDisp + proc setGlobalDispatcher*(disp: PDispatcher) = + if not gDisp.isNil: + assert gDisp.callbacks.len == 0 + gDisp = disp + proc register*(fd: AsyncFD) = ## Registers ``fd`` with the dispatcher. let p = getGlobalDispatcher() @@ -1080,6 +1085,11 @@ else: if gDisp.isNil: gDisp = newDispatcher() result = gDisp + proc setGlobalDispatcher*(disp: PDispatcher) = + if not gDisp.isNil: + assert gDisp.callbacks.len == 0 + gDisp = disp + proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() var data = newAsyncData() From fa3436fb657141127038d88431b4aad113c27cf6 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 9 May 2017 15:14:20 +0100 Subject: [PATCH 08/22] Deprecate osproc.select(). Refs #90. --- lib/pure/osproc.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 82a0c0c65..c94a65a63 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -335,7 +335,8 @@ proc execProcesses*(cmds: openArray[string], if afterRunEvent != nil: afterRunEvent(i, p) close(p) -proc select*(readfds: var seq[Process], timeout = 500): int {.benign.} +proc select*(readfds: var seq[Process], timeout = 500): int + {.benign, deprecated.} ## `select` with a sensible Nim interface. `timeout` is in milliseconds. ## Specify -1 for no timeout. Returns the number of processes that are ## ready to read from. The processes that are ready to be read from are @@ -343,6 +344,9 @@ proc select*(readfds: var seq[Process], timeout = 500): int {.benign.} ## ## **Warning**: This function may give unexpected or completely wrong ## results on Windows. + ## + ## **Deprecated since version 0.17.0**: This procedure isn't cross-platform + ## and so should not be used in newly written code. when not defined(useNimRtl): proc execProcess(command: string, From 6e0e3b75cf419f9f2e61aeccaa59f91160e29628 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Wed, 10 May 2017 10:29:30 +0300 Subject: [PATCH 09/22] Fixed const sets in JS (#5790) --- compiler/jsgen.nim | 1 - compiler/jstypes.nim | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index eb3fb9f47..ee35356c9 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -2272,7 +2272,6 @@ proc myProcess(b: PPassContext, n: PNode): PNode = genModule(p, n) add(p.g.code, p.locals) add(p.g.code, p.body) - globals.unique = p.unique proc wholeCode(graph: ModuleGraph; m: BModule): Rope = for prc in globals.forwarded: diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index f49bd7668..ae30861e7 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -59,7 +59,7 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope = u = rope(lengthOrd(field.typ)) else: internalError(n.info, "genObjectFields(nkRecCase)") if result != nil: add(result, ", " & tnl) - addf(result, "[SetConstr($1), $2]", + addf(result, "[setConstr($1), $2]", [u, genObjectFields(p, typ, lastSon(b))]) result = ("{kind: 3, offset: \"$1\", len: $3, " & "typ: $2, name: $4, sons: [$5]}") % [ From 2e7551f17eeff16715dd788e076d40fe3eb07879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mice=20P=C3=A1pai?= Date: Wed, 10 May 2017 14:04:38 +0200 Subject: [PATCH 10/22] Fix a typo in the Result variable block --- doc/tut1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index 06ee84c0d..de89a0b2d 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -599,7 +599,7 @@ Result variable A procedure that returns a value has an implicit ``result`` variable declared that represents the return value. A ``return`` statement with no expression is a shorthand for ``return result``. The ``result`` value is always returned -automatically at the end a procedure if there is no ``return`` statement at +automatically at the end of a procedure if there is no ``return`` statement at the exit. .. code-block:: nim From 1a29069ececd4514195a58e1fe68385fa1240d46 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 11 May 2017 17:00:50 +0200 Subject: [PATCH 11/22] attempt to make nimsuggest work with choosenim --- nimsuggest/nimsuggest.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index c1adb87a1..93a418dd7 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -595,6 +595,8 @@ proc handleCmdLine(cache: IdentCache; config: ConfigRef) = raise newException(IOError, "Cannot find Nim standard library: Nim compiler not in PATH") gPrefixDir = binaryPath.splitPath().head.parentDir() + if not dirExists(gPrefixDir / "lib"): gPrefixDir = "" + #msgs.writelnHook = proc (line: string) = log(line) myLog("START " & gProjectFull) From 29507d2491939d2979fbdfc4704142bc6ef2565c Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Fri, 12 May 2017 03:30:48 +0700 Subject: [PATCH 12/22] Consider optNoCppExceptions in blockLeaveActions (#5801) --- compiler/ccgstmts.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index cc925b150..a094da783 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -332,7 +332,7 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) = var alreadyPoppedCnt = p.inExceptBlock for i in countup(1, howManyTrys): - if not p.module.compileToCpp: + if not p.module.compileToCpp or optNoCppExceptions in gGlobalOptions: # Pop safe points generated by try if alreadyPoppedCnt > 0: dec alreadyPoppedCnt @@ -354,7 +354,7 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) = for i in countdown(howManyTrys-1, 0): p.nestedTryStmts.add(stack[i]) - if not p.module.compileToCpp: + if not p.module.compileToCpp or optNoCppExceptions in gGlobalOptions: # Pop exceptions that was handled by the # except-blocks we are in for i in countdown(howManyExcepts-1, 0): From 2e420dfa382dc5af3a225a7b073da2d0eb3d899f Mon Sep 17 00:00:00 2001 From: Parashurama Date: Fri, 12 May 2017 11:21:43 +0200 Subject: [PATCH 13/22] fixes multiple nil-checks for methods (#5806) --- compiler/cgmeth.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index e14306e56..1165ec932 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -233,6 +233,12 @@ proc genDispatcher(methods: TSymSeq, relevantCols: IntSet): PSym = var disp = newNodeI(nkIfStmt, base.info) var ands = getSysSym("and") var iss = getSysSym("of") + for col in countup(1, paramLen - 1): + if contains(relevantCols, col): + let param = base.typ.n.sons[col].sym + if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}: + addSon(nilchecks, newTree(nkCall, + newSymNode(getCompilerProc"chckNilDisp"), newSymNode(param))) for meth in countup(0, high(methods)): var curr = methods[meth] # generate condition: var cond: PNode = nil @@ -242,9 +248,6 @@ proc genDispatcher(methods: TSymSeq, relevantCols: IntSet): PSym = addSon(isn, newSymNode(iss)) let param = base.typ.n.sons[col].sym addSon(isn, newSymNode(param)) - if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}: - addSon(nilchecks, newTree(nkCall, - newSymNode(getCompilerProc"chckNilDisp"), newSymNode(param))) addSon(isn, newNodeIT(nkType, base.info, curr.typ.sons[col])) if cond != nil: var a = newNodeIT(nkCall, base.info, getSysType(tyBool)) From 77cadd07f9079ef9c5f3907cbe273b075aa1f76d Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Fri, 12 May 2017 16:24:45 +0700 Subject: [PATCH 14/22] Fix atomicInc under vcc, C++, amd64 (#5809) Also, fixed inconsistent behaviour of atomicInc on vcc. Previously it would return the old value, while it must return the new value. Fixes: #5808 --- lib/system/atomics.nim | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index 3a43729dc..885b01621 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -165,8 +165,22 @@ when someGcc and hasThreadSupport: template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) elif defined(vcc) and hasThreadSupport: - proc addAndFetch*(p: ptr int, val: int): int {. - importc: "_InterlockedExchangeAdd", header: "".} + when defined(cpp): + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd64(static_cast(#), #)", + header: "".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importcpp: "_InterlockedExchangeAdd(static_cast(#), #)", + header: "".} + else: + when sizeof(int) == 8: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd64", header: "".} + else: + proc addAndFetch*(p: ptr int, val: int): int {. + importc: "_InterlockedExchangeAdd", header: "".} proc fence*() {.importc: "_ReadWriteBarrier", header: "".} @@ -180,6 +194,7 @@ proc atomicInc*(memLoc: var int, x: int = 1): int = result = atomic_add_fetch(memLoc.addr, x, ATOMIC_RELAXED) elif defined(vcc) and hasThreadSupport: result = addAndFetch(memLoc.addr, x) + inc(result, x) else: inc(memLoc, x) result = memLoc @@ -192,6 +207,7 @@ proc atomicDec*(memLoc: var int, x: int = 1): int = result = atomic_add_fetch(memLoc.addr, -x, ATOMIC_RELAXED) elif defined(vcc) and hasThreadSupport: result = addAndFetch(memLoc.addr, -x) + dec(result, x) else: dec(memLoc, x) result = memLoc From 534d8a6b59bce5eadd543929d01ccb2f40747fcb Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Fri, 12 May 2017 13:11:58 +0300 Subject: [PATCH 15/22] Fixed syslocks for ios (#5804) --- lib/system/syslocks.nim | 112 +++++++++++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 23 deletions(-) diff --git a/lib/system/syslocks.nim b/lib/system/syslocks.nim index fb354880f..f61b887ad 100644 --- a/lib/system/syslocks.nim +++ b/lib/system/syslocks.nim @@ -99,7 +99,7 @@ elif defined(genode): else: type - SysLock {.importc: "pthread_mutex_t", pure, final, + SysLockObj {.importc: "pthread_mutex_t", pure, final, header: """#include #include """.} = object when defined(linux) and defined(amd64): @@ -111,7 +111,7 @@ else: when defined(linux) and defined(amd64): abi: array[4 div sizeof(cint), cint] # actually a cint - SysCond {.importc: "pthread_cond_t", pure, final, + SysCondObj {.importc: "pthread_cond_t", pure, final, header: """#include #include """.} = object when defined(linux) and defined(amd64): @@ -119,8 +119,62 @@ else: SysLockType = distinct cint - proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) {. + proc initSysLockAux(L: var SysLockObj, attr: ptr SysLockAttr) {. importc: "pthread_mutex_init", header: "", noSideEffect.} + proc deinitSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_destroy", header: "".} + + proc acquireSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_lock", header: "".} + proc tryAcquireSysAux(L: var SysLockObj): cint {.noSideEffect, + importc: "pthread_mutex_trylock", header: "".} + + proc releaseSysAux(L: var SysLockObj) {.noSideEffect, + importc: "pthread_mutex_unlock", header: "".} + + when defined(ios): + # iOS will behave badly if sync primitives are moved in memory. In order + # to prevent this once and for all, we're doing an extra malloc when + # initializing the primitive. + type + SysLock = ptr SysLockObj + SysCond = ptr SysCondObj + + when not declared(c_malloc): + proc c_malloc(size: csize): pointer {. + importc: "malloc", header: "".} + proc c_free(p: pointer) {. + importc: "free", header: "".} + + proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) = + L = cast[SysLock](c_malloc(sizeof(SysLockObj))) + initSysLockAux(L[], attr) + + proc deinitSys(L: var SysLock) = + deinitSysAux(L[]) + c_free(L) + + template acquireSys(L: var SysLock) = + acquireSysAux(L[]) + template tryAcquireSys(L: var SysLock): bool = + tryAcquireSysAux(L[]) == 0'i32 + template releaseSys(L: var SysLock) = + releaseSysAux(L[]) + else: + type + SysLock = SysLockObj + SysCond = SysCondObj + + template initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) = + initSysLockAux(L, attr) + template deinitSys(L: var SysLock) = + deinitSysAux(L) + template acquireSys(L: var SysLock) = + acquireSysAux(L) + template tryAcquireSys(L: var SysLock): bool = + tryAcquireSysAux(L) == 0'i32 + template releaseSys(L: var SysLock) = + releaseSysAux(L) when insideRLocksModule: proc SysLockType_Reentrant: SysLockType = @@ -130,27 +184,39 @@ else: proc setSysLockType(a: var SysLockAttr, t: SysLockType) {. importc: "pthread_mutexattr_settype", header: "", noSideEffect.} - proc acquireSys(L: var SysLock) {.noSideEffect, - importc: "pthread_mutex_lock", header: "".} - proc tryAcquireSysAux(L: var SysLock): cint {.noSideEffect, - importc: "pthread_mutex_trylock", header: "".} - - proc tryAcquireSys(L: var SysLock): bool {.inline.} = - result = tryAcquireSysAux(L) == 0'i32 - - proc releaseSys(L: var SysLock) {.noSideEffect, - importc: "pthread_mutex_unlock", header: "".} - proc deinitSys(L: var SysLock) {.noSideEffect, - importc: "pthread_mutex_destroy", header: "".} - - when not insideRLocksModule: - proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) {. + else: + proc initSysCondAux(cond: var SysCondObj, cond_attr: pointer) {. importc: "pthread_cond_init", header: "", noSideEffect.} - proc waitSysCond(cond: var SysCond, lock: var SysLock) {. - importc: "pthread_cond_wait", header: "", noSideEffect.} - proc signalSysCond(cond: var SysCond) {. - importc: "pthread_cond_signal", header: "", noSideEffect.} - proc deinitSysCond(cond: var SysCond) {.noSideEffect, + proc deinitSysCondAux(cond: var SysCondObj) {.noSideEffect, importc: "pthread_cond_destroy", header: "".} + proc waitSysCondAux(cond: var SysCondObj, lock: var SysLockObj) {. + importc: "pthread_cond_wait", header: "", noSideEffect.} + proc signalSysCondAux(cond: var SysCondObj) {. + importc: "pthread_cond_signal", header: "", noSideEffect.} + + when defined(ios): + proc initSysCond(cond: var SysCond, cond_attr: pointer = nil) = + cond = cast[SysCond](c_malloc(sizeof(SysCondObj))) + initSysCondAux(cond[], cond_attr) + + proc deinitSysCond(cond: var SysCond) = + deinitSysCondAux(cond[]) + c_free(cond) + + template waitSysCond(cond: var SysCond, lock: var SysLock) = + waitSysCondAux(cond[], lock[]) + template signalSysCond(cond: var SysCond) = + signalSysCondAux(cond[]) + else: + template initSysCond(cond: var SysCond, cond_attr: pointer = nil) = + initSysCondAux(cond, cond_attr) + template deinitSysCond(cond: var SysCond) = + deinitSysCondAux(cond) + + template waitSysCond(cond: var SysCond, lock: var SysLock) = + waitSysCondAux(cond, lock) + template signalSysCond(cond: var SysCond) = + signalSysCondAux(cond) + {.pop.} From 40e9e751f0e77211442707820b4737a253334a4f Mon Sep 17 00:00:00 2001 From: Simon Krauter Date: Sat, 13 May 2017 21:50:44 +0200 Subject: [PATCH 16/22] Extend documenation of formatFloat() (#5799) Added line: "If ``precision == 0``, it tries to format it nicely." (copied from formatBiggestFloat()) --- lib/pure/strutils.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 9383675f4..458c22f3a 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -1881,6 +1881,8 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault, ## of significant digits to be printed. ## `precision`'s default value is the maximum number of meaningful digits ## after the decimal point for Nim's ``float`` type. + ## + ## If ``precision == 0``, it tries to format it nicely. result = formatBiggestFloat(f, format, precision, decimalSep) proc trimZeros*(x: var string) {.noSideEffect.} = From f5717490d9498f6399ca15a90e9ecf60488ae80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mice=20P=C3=A1pai?= Date: Sat, 13 May 2017 21:52:19 +0200 Subject: [PATCH 17/22] tut1.rst: Fix alignment, Change Subrange to MySubrange for clarity (#5798) * tut1.rst: Fix alignment * tut1.rst: Change Subrange to MySubrange for clarity --- doc/tut1.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/tut1.rst b/doc/tut1.rst index de89a0b2d..436b3880d 100644 --- a/doc/tut1.rst +++ b/doc/tut1.rst @@ -1074,8 +1074,8 @@ at runtime by 0, the second by 1 and so on. For example: Direction = enum north, east, south, west - var x = south # `x` is of type `Direction`; its value is `south` - echo x # writes "south" to `stdout` + var x = south # `x` is of type `Direction`; its value is `south` + echo x # writes "south" to `stdout` All the comparison operators can be used with enumeration types. @@ -1132,11 +1132,11 @@ A subrange type is a range of values from an integer or enumeration type .. code-block:: nim type - Subrange = range[0..5] + MySubrange = range[0..5] -``Subrange`` is a subrange of ``int`` which can only hold the values 0 -to 5. Assigning any other value to a variable of type ``Subrange`` is a +``MySubrange`` is a subrange of ``int`` which can only hold the values 0 +to 5. Assigning any other value to a variable of type ``MySubrange`` is a compile-time or runtime error. Assignments from the base type to one of its subrange types (and vice versa) are allowed. From b826f85731fb20cbdc00a2ab5a64ec852251c56d Mon Sep 17 00:00:00 2001 From: Ruslan Mustakov Date: Sun, 14 May 2017 20:07:33 +0700 Subject: [PATCH 18/22] Remove unused oids import from asyncdispatch (#5811) --- lib/pure/asyncdispatch.nim | 2 +- lib/upcoming/asyncdispatch.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index e48fe8d94..1697384e0 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -9,7 +9,7 @@ include "system/inclrtl" -import os, oids, tables, strutils, times, heapqueue, options +import os, tables, strutils, times, heapqueue, options import nativesockets, net, deques diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 8541bf98c..1623d8375 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -9,7 +9,7 @@ include "system/inclrtl" -import os, oids, tables, strutils, times, heapqueue, lists, options +import os, tables, strutils, times, heapqueue, lists, options import nativesockets, net, deques From ec50dab57d91fb33380a39479bafbeae48ed1a83 Mon Sep 17 00:00:00 2001 From: jcosborn Date: Sun, 14 May 2017 08:08:41 -0500 Subject: [PATCH 19/22] fixed macros.getTypeImpl for a symbol of type tyGenericBody (#5788) (#5812) * fixed macros.getTypeImpl for a symbol of type tyGenericBody * added test case and commented test code --- compiler/vmdeps.nim | 7 +++- tests/macros/tgettypeinst.nim | 69 +++++++++++++++++++++++--------- web/news/e031_version_0_16_2.rst | 2 + 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 8c7388643..b2b1ec92b 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -175,7 +175,12 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add mapTypeToAst(t.sons[i], info) else: result = mapTypeToAstX(t.lastSon, info, inst, allowRecursion) - of tyGenericBody, tyOrdinal: + of tyGenericBody: + if inst: + result = mapTypeToAstX(t.lastSon, info, inst, true) + else: + result = mapTypeToAst(t.lastSon, info) + of tyOrdinal: result = mapTypeToAst(t.lastSon, info) of tyDistinct: if inst: diff --git a/tests/macros/tgettypeinst.nim b/tests/macros/tgettypeinst.nim index 255eff949..f89aa5e0b 100644 --- a/tests/macros/tgettypeinst.nim +++ b/tests/macros/tgettypeinst.nim @@ -21,49 +21,67 @@ proc symToIdent(x: NimNode): NimNode = for c in x: result.add symToIdent(c) +# check getTypeInst and getTypeImpl for given symbol x macro testX(x,inst0: typed; recurse: static[bool]; implX: stmt): typed = + # check that getTypeInst(x) equals inst0 let inst = x.getTypeInst - let impl = x.getTypeImpl - let inst0r = inst0.symToIdent.treeRepr let instr = inst.symToIdent.treeRepr - #echo inst0r + let inst0r = inst0.symToIdent.treeRepr #echo instr + #echo inst0r doAssert(instr == inst0r) + + # check that getTypeImpl(x) is correct + # if implX is nil then compare to inst0 + # else we expect implX to be a type definition + # and we extract the implementation from that + let impl = x.getTypeImpl var impl0 = if implX.kind == nnkNilLit: inst0 else: implX[0][2] - let impl0r = impl0.symToIdent.treerepr let implr = impl.symToIdent.treerepr - #echo impl0r + let impl0r = impl0.symToIdent.treerepr #echo implr + #echo impl0r doAssert(implr == impl0r) - template echoString(s:string) = echo s.replace("\n","\n ") + result = newStmtList() + #template echoString(s: string) = echo s.replace("\n","\n ") #result.add getAst(echoString(" " & inst0.repr)) #result.add getAst(echoString(" " & inst.repr)) #result.add getAst(echoString(" " & impl0.repr)) #result.add getAst(echoString(" " & impl.repr)) + if recurse: - template testDecl(n, m :typed) = + # now test using a newly formed variable of type getTypeInst(x) + template testDecl(n,m: typed) = testV(n, false): type _ = m result.add getAst(testDecl(inst.symToIdent, impl.symToIdent)) +# test with a variable (instance) of type template testV(inst, recurse, impl) = block: #echo "testV(" & astToStr(inst) & ", " & $recurse & "):" & astToStr(impl) var x: inst testX(x, inst, recurse, impl) -template testT(inst, recurse) = - block: - type myType = inst - testV(myType, recurse): - type _ = inst +# test with a newly created typedesc (myType) +# using the passed type as the implementation +template testT(impl, recurse) = + block: + type myType = impl + testV(myType, recurse): + type _ = impl + +# test a built-in type whose instance is equal to the implementation template test(inst) = testT(inst, false) testV(inst, true, nil) -template test(inst, impl) = testV(inst, true, impl) + +# test a custom type with provided implementation +template test(inst, impl) = + testV(inst, true, impl) type Model = object of RootObj @@ -87,9 +105,12 @@ type value:T Foo[N:static[int],T] = object Bar[N:static[int],T] = object - #baz:Foo[N+1,GenericObject[T]] + #baz:Foo[N+1,GenericObject[T]] # currently fails baz:Foo[N,GenericObject[T]] + Generic[T] = seq[int] + Concrete = Generic[int] + test(bool) test(char) test(int) @@ -97,13 +118,17 @@ test(float) test(ptr int) test(ref int) test(array[1..10,Bar[2,Foo[3,float]]]) +test(array[MyEnum,Bar[2,Foo[3,float]]]) test(distinct Bar[2,Foo[3,float]]) test(tuple[a:int,b:Foo[-1,float]]) -#test(MyEnum): -# type _ = enum -# valueA, valueB, valueC -test(set[MyEnum]) test(seq[int]) +test(set[MyEnum]) +test(proc (a: int, b: Foo[2,float])) +test(proc (a: int, b: Foo[2,float]): Bar[3,int]) + +test(MyEnum): + type _ = enum + valueA, valueB, valueC test(Bar[2,Foo[3,float]]): type _ = object baz: Foo[2, GenericObject[Foo[3, float]]] @@ -118,8 +143,12 @@ test(Tree): value: int left: ref Tree right: ref Tree -test(proc (a: int, b: Foo[2,float])) -test(proc (a: int, b: Foo[2,float]): Bar[3,int]) +test(Concrete): + type _ = Generic[int] +test(Generic[int]): + type _ = seq[int] +test(Generic[float]): + type _ = seq[int] # bug #4862 static: diff --git a/web/news/e031_version_0_16_2.rst b/web/news/e031_version_0_16_2.rst index 63884700f..4f49bd8d9 100644 --- a/web/news/e031_version_0_16_2.rst +++ b/web/news/e031_version_0_16_2.rst @@ -45,6 +45,8 @@ Changes affecting backwards compatibility AST that is the same as what is used to define an enum. Previously the AST returned had a repeated ``EnumTy`` node and was missing the initial pragma node (which is currently empty for an enum). +- ``macros.getTypeImpl`` now correctly returns the implementation for a symbol + of type ``tyGenericBody``. - If the dispatcher parameter's value used in multi method is ``nil``, a ``NilError`` exception is raised. The old behavior was that the method would be a ``nop`` then. From 3afd852f54df2a0bc06be34d06509dfb2b9c80c5 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 14 May 2017 17:10:39 +0200 Subject: [PATCH 20/22] Fixes #5532 win async write (#5791) * nimgrab tool bugfix: don't divide by zero * fixes #5532 (asyncfile write on Windows) * add a comment about what has been tried instead --- lib/pure/asyncfile.nim | 10 ++++++---- tests/async/tasyncfilewrite.nim | 17 +++++++++++++++++ tools/nimgrab.nim | 6 +++++- 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 tests/async/tasyncfilewrite.nim diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 9bd060e30..8fb30075c 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -339,13 +339,17 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = if not retFuture.finished: if errcode == OSErrorCode(-1): assert bytesCount == size.int32 - f.offset.inc(size) retFuture.complete() else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) ) + # passing -1 here should work according to MSDN, but doesn't. For more + # information see + # http://stackoverflow.com/questions/33650899/does-asynchronous-file- + # appending-in-windows-preserve-order ol.offset = DWord(f.offset and 0xffffffff) ol.offsetHigh = DWord(f.offset shr 32) + f.offset.inc(size) # According to MSDN we're supposed to pass nil to lpNumberOfBytesWritten. let ret = writeFile(f.fd.Handle, buf, size.int32, nil, @@ -364,7 +368,6 @@ proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) else: assert bytesWritten == size.int32 - f.offset.inc(size) retFuture.complete() else: var written = 0 @@ -410,7 +413,6 @@ proc write*(f: AsyncFile, data: string): Future[void] = if not retFuture.finished: if errcode == OSErrorCode(-1): assert bytesCount == data.len.int32 - f.offset.inc(data.len) retFuture.complete() else: retFuture.fail(newException(OSError, osErrorMsg(errcode))) @@ -420,6 +422,7 @@ proc write*(f: AsyncFile, data: string): Future[void] = ) ol.offset = DWord(f.offset and 0xffffffff) ol.offsetHigh = DWord(f.offset shr 32) + f.offset.inc(data.len) # According to MSDN we're supposed to pass nil to lpNumberOfBytesWritten. let ret = writeFile(f.fd.Handle, buffer, data.len.int32, nil, @@ -441,7 +444,6 @@ proc write*(f: AsyncFile, data: string): Future[void] = retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) else: assert bytesWritten == data.len.int32 - f.offset.inc(data.len) retFuture.complete() else: var written = 0 diff --git a/tests/async/tasyncfilewrite.nim b/tests/async/tasyncfilewrite.nim new file mode 100644 index 000000000..cda612bae --- /dev/null +++ b/tests/async/tasyncfilewrite.nim @@ -0,0 +1,17 @@ +discard """ + output: '''string 1 +string 2 +string 3''' +""" +# bug #5532 +import os, asyncfile, asyncdispatch + +removeFile("test.txt") +let f = openAsync("test.txt", fmWrite) +var futs = newSeq[Future[void]]() +for i in 1..3: + futs.add(f.write("string " & $i & "\n")) +waitFor(all(futs)) +f.close() +echo readFile("test.txt") + diff --git a/tools/nimgrab.nim b/tools/nimgrab.nim index 69824369e..937a5dcd4 100644 --- a/tools/nimgrab.nim +++ b/tools/nimgrab.nim @@ -7,7 +7,11 @@ when defined(windows): proc progress(status: DownloadStatus, progress: uint, total: uint, message: string) {.procvar, gcsafe.} = echo "Downloading " & url - echo clamp(int(progress.BiggestInt*100 div total.BiggestInt), 0, 100), "%" + let t = total.BiggestInt + if t != 0: + echo clamp(int(progress.BiggestInt*100 div t), 0, 100), "%" + else: + echo "0%" downloadToFile(url, file, {optUseCache}, progress) echo "100%" From 6bbf0fb64d4037b1ae7f00af0c153b1064e05355 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 15 May 2017 22:34:30 +0800 Subject: [PATCH 21/22] compiler option for testament (#5713) * compiler option for testament * different spacing --- tests/testament/specs.nim | 11 +++++++---- tests/testament/tester.nim | 6 ++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/testament/specs.nim b/tests/testament/specs.nim index 91e8b2ec7..ab24acc70 100644 --- a/tests/testament/specs.nim +++ b/tests/testament/specs.nim @@ -9,8 +9,11 @@ import parseutils, strutils, os, osproc, streams, parsecfg -const - cmdTemplate* = r"compiler" / "nim $target --lib:lib --hints:on -d:testing $options $file" + +var compilerPrefix* = "compiler" / "nim " + +proc cmdTemplate*(): string = + compilerPrefix & "$target --lib:lib --hints:on -d:testing $options $file" type TTestAction* = enum @@ -100,7 +103,7 @@ proc specDefaults*(result: var TSpec) = result.outp = "" result.nimout = "" result.ccodeCheck = "" - result.cmd = cmdTemplate + result.cmd = cmdTemplate() result.line = 0 result.column = 0 result.tfile = "" @@ -173,7 +176,7 @@ proc parseSpec*(filename: string): TSpec = raise newException(ValueError, "cannot interpret as a bool: " & e.value) of "cmd": if e.value.startsWith("nim "): - result.cmd = "compiler" / e.value + result.cmd = compilerPrefix & e.value[4..^1] else: result.cmd = e.value of "ccodecheck": result.ccodeCheck = e.value diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 2d758ef0d..0f74de013 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -34,6 +34,7 @@ Options: --failing only show failing/ignored tests --pedantic return non-zero status code if there are failures --targets:"c c++ js objc" run tests for specified targets (default: all) + --nim:path use a particular nim executable (default: compiler/nim) """ % resultsFile type @@ -367,7 +368,7 @@ proc testNoSpec(r: var TResults, test: TTest) = # does not extract the spec because the file is not supposed to have any #let tname = test.name.addFileExt(".nim") inc(r.total) - let given = callCompiler(cmdTemplate, test.name, test.options, test.target) + let given = callCompiler(cmdTemplate(), test.name, test.options, test.target) r.addResult(test, "", given.msg, given.err) if given.err == reSuccess: inc(r.passed) @@ -376,7 +377,7 @@ proc testC(r: var TResults, test: TTest) = let tname = test.name.addFileExt(".c") inc(r.total) styledEcho "Processing ", fgCyan, extractFilename(tname) - var given = callCCompiler(cmdTemplate, test.name & ".c", test.options, test.target) + var given = callCCompiler(cmdTemplate(), test.name & ".c", test.options, test.target) if given.err != reSuccess: r.addResult(test, "", given.msg, given.err) elif test.action == actionRun: @@ -424,6 +425,7 @@ proc main() = of "failing": optFailing = true of "pedantic": optPedantic = true of "targets": targets = parseTargets(p.val.string) + of "nim": compilerPrefix = p.val.string else: quit Usage p.next() if p.kind != cmdArgument: quit Usage From 113857c7f1a6f96665fc9a6948438dfd3166f008 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 16 May 2017 14:03:33 +0200 Subject: [PATCH 22/22] better stack trace handling --- lib/system/excpt.nim | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 8ed1fbb38..096d01845 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -282,8 +282,13 @@ proc raiseExceptionAux(e: ref Exception) = proc raiseException(e: ref Exception, ename: cstring) {.compilerRtl.} = if e.name.isNil: e.name = ename when hasSomeStackTrace: - e.trace = "" - rawWriteStackTrace(e.trace) + if e.trace.isNil: + e.trace = "" + rawWriteStackTrace(e.trace) + elif framePtr != nil: + e.trace.add "[[reraised from:\n" + auxWriteStackTrace(framePtr, e.trace) + e.trace.add "]]\n" raiseExceptionAux(e) proc reraiseException() {.compilerRtl.} =