From 3f3014f626b5be347f973399988a225db79fe762 Mon Sep 17 00:00:00 2001 From: flaviut Date: Thu, 10 Apr 2014 21:59:43 -0400 Subject: [PATCH 1/4] Koch deletes the .gitignore while cleaning --- koch.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koch.nim b/koch.nim index ce01d36a5..68dea16dd 100644 --- a/koch.nim +++ b/koch.nim @@ -170,7 +170,7 @@ const ".idx" ] ignore = [ - ".bzrignore", "nimrod", "nimrod.exe", "koch", "koch.exe" + ".bzrignore", "nimrod", "nimrod.exe", "koch", "koch.exe", ".gitignore" ] proc cleanAux(dir: string) = From fac9947ebef81a88c16600db868b5b9e27541ea3 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Fri, 11 Apr 2014 22:24:49 +0200 Subject: [PATCH 2/4] Adds generic split proc to sequtils. --- lib/pure/collections/sequtils.nim | 90 +++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index b2f72ee14..8666054f4 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -88,6 +88,71 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = newSeq(result, m) for i in 0 .. m-1: result[i] = (seq1[i], seq2[i]) +proc split*[T](s: seq[T], num: int, spread = true): seq[seq[T]] = + ## Splits a sequence `s` into `num` sub sequences. + ## + ## Returns a sequence of `num` sequences. For some input values this is the + ## inverse of the `concat <#concat>`_ proc. The proc will assert in debug + ## builds if `s` is nil or `num` is less than one, and will likely crash on + ## release builds. The sequence `s` can be empty, which will produce `num` + ## empty sequences. + ## + ## If `spread` is false and the length of `s` is not a multiple of `num`, the + ## proc will max out the first sub sequences with ``1 + len(s) div num`` + ## entries, leaving the remainder of elements to the last sequence. + ## + ## On the other hand, if `spread` is true, the proc will distribute evenly + ## the remainder of the division across all sequences, which makes the result + ## more suited to multithreading where you are passing equal sized work units + ## to a thread pool and want to maximize core usage. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## let numbers = @[1, 2, 3, 4, 5, 6, 7] + ## assert numbers.split(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + ## assert numbers.split(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + ## assert numbers.split(6)[0] == @[1, 2] + ## assert numbers.split(6)[5] == @[7] + assert(not s.isNil, "`s` can't be nil") + assert(num > 0, "`num` has to be greater than zero") + if num < 2: + result = @[s] + return + + # Create the result and calculate the stride size and the remainder if any. + result = newSeq[seq[T]](num) + var + stride = s.len div num + first = 0 + last = 0 + extra = s.len mod num + + if extra == 0 or spread == false: + # Use an algorithm which overcounts the stride and minimizes reading limits. + if extra > 0: inc(stride) + + for i in 0 .. 0: + extra -= 1 + inc(last) + + result[i] = newSeq[T]() + for g in first .. Date: Sun, 13 Apr 2014 10:50:27 +0200 Subject: [PATCH 3/4] Renames split to distribute. Refs #1084. --- lib/pure/collections/sequtils.nim | 57 ++++++++++++++++--------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 8666054f4..f5db9d3fa 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -88,14 +88,14 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = newSeq(result, m) for i in 0 .. m-1: result[i] = (seq1[i], seq2[i]) -proc split*[T](s: seq[T], num: int, spread = true): seq[seq[T]] = - ## Splits a sequence `s` into `num` sub sequences. +proc distribute*[T](s: seq[T], num: int, spread = true): seq[seq[T]] = + ## Splits and distributes a sequence `s` into `num` sub sequences. ## ## Returns a sequence of `num` sequences. For some input values this is the ## inverse of the `concat <#concat>`_ proc. The proc will assert in debug ## builds if `s` is nil or `num` is less than one, and will likely crash on - ## release builds. The sequence `s` can be empty, which will produce `num` - ## empty sequences. + ## release builds. The input sequence `s` can be empty, which will produce + ## `num` empty sequences. ## ## If `spread` is false and the length of `s` is not a multiple of `num`, the ## proc will max out the first sub sequences with ``1 + len(s) div num`` @@ -110,10 +110,10 @@ proc split*[T](s: seq[T], num: int, spread = true): seq[seq[T]] = ## ## .. code-block:: nimrod ## let numbers = @[1, 2, 3, 4, 5, 6, 7] - ## assert numbers.split(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - ## assert numbers.split(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - ## assert numbers.split(6)[0] == @[1, 2] - ## assert numbers.split(6)[5] == @[7] + ## assert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + ## assert numbers.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + ## assert numbers.distribute(6)[0] == @[1, 2] + ## assert numbers.distribute(6)[5] == @[7] assert(not s.isNil, "`s` can't be nil") assert(num > 0, "`num` has to be greater than zero") if num < 2: @@ -485,29 +485,30 @@ when isMainModule: nums.mapIt(it * 3) assert nums[0] + nums[3] == 15 - block: # split tests + block: # distribute tests let numbers = @[1, 2, 3, 4, 5, 6, 7] - doAssert numbers.split(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert numbers.split(6)[0] == @[1, 2] - doAssert numbers.split(6)[5] == @[7] + doAssert numbers.distribute(3) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + doAssert numbers.distribute(6)[0] == @[1, 2] + doAssert numbers.distribute(6)[5] == @[7] let a = @[1, 2, 3, 4, 5, 6, 7] - doAssert a.split(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.split(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] - doAssert a.split(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.split(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] - doAssert a.split(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] - doAssert a.split(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] - doAssert a.split(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.split(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] - doAssert a.split(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] - doAssert a.split(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] - doAssert a.split(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] - doAssert a.split(6, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] - doAssert a.split(8, false) == a.split(8, true) - doAssert a.split(90, false) == a.split(90, true) + doAssert a.distribute(1, true) == @[@[1, 2, 3, 4, 5, 6, 7]] + doAssert a.distribute(1, false) == @[@[1, 2, 3, 4, 5, 6, 7]] + doAssert a.distribute(2, true) == @[@[1, 2, 3, 4], @[5, 6, 7]] + doAssert a.distribute(2, false) == @[@[1, 2, 3, 4], @[5, 6, 7]] + doAssert a.distribute(3, true) == @[@[1, 2, 3], @[4, 5], @[6, 7]] + doAssert a.distribute(3, false) == @[@[1, 2, 3], @[4, 5, 6], @[7]] + doAssert a.distribute(4, true) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] + doAssert a.distribute(4, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7]] + doAssert a.distribute(5, true) == @[@[1, 2], @[3, 4], @[5], @[6], @[7]] + doAssert a.distribute(5, false) == @[@[1, 2], @[3, 4], @[5, 6], @[7], @[]] + doAssert a.distribute(6, true) == @[@[1, 2], @[3], @[4], @[5], @[6], @[7]] + doAssert a.distribute(6, false) == @[ + @[1, 2], @[3, 4], @[5, 6], @[7], @[], @[]] + doAssert a.distribute(8, false) == a.distribute(8, true) + doAssert a.distribute(90, false) == a.distribute(90, true) var b = @[0] for f in 1 .. 25: b.add(f) - doAssert b.split(5, true)[4].len == 5 - doAssert b.split(5, false)[4].len == 2 + doAssert b.distribute(5, true)[4].len == 5 + doAssert b.distribute(5, false)[4].len == 2 echo "Finished doc tests" From 3612aca5fe0dfdc703115dfc27af4fef03acd2c9 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 13 Apr 2014 14:18:47 +0100 Subject: [PATCH 4/4] Implemented buffering for asynchronous sockets. --- lib/pure/asyncdispatch.nim | 2 +- lib/pure/asynchttpserver.nim | 2 +- lib/pure/asyncnet.nim | 38 ++++++++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index a8802dec3..5e638dc74 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -381,8 +381,8 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: dealloc dataBuf.buf - retFuture.fail(newException(EOS, osErrorMsg(err))) dealloc(ol) + retFuture.fail(newException(EOS, osErrorMsg(err))) elif ret == 0 and bytesReceived == 0 and dataBuf.buf[0] == '\0': # We have to ensure that the buffer is empty because WSARecv will tell # us immediatelly when it was disconnected, even when there is still diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index f1aa1a09d..2ebd7036d 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -134,7 +134,7 @@ proc processClient(client: PAsyncSocket, address: string, return case reqMethod.normalize - of "get": + of "get", "post", "head", "put", "delete", "trace", "options", "connect", "patch": await callback(request) else: request.respond(Http400, "Invalid request method. Got: " & reqMethod) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index 7cf2ad134..b1abf627b 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -80,14 +80,48 @@ proc connect*(socket: PAsyncSocket, address: string, port: TPort, ## or an error occurs. result = connect(socket.fd.TAsyncFD, address, port, af) +proc readIntoBuf(socket: PAsyncSocket, flags: int): PFuture[int] {.async.} = + var data = await recv(socket.fd.TAsyncFD, BufferSize, flags) + if data.len != 0: + copyMem(addr socket.buffer[0], addr data[0], data.len) + socket.bufLen = data.len + socket.currPos = 0 + result = data.len + proc recv*(socket: PAsyncSocket, size: int, - flags: int = 0): PFuture[string] = + flags: int = 0): PFuture[string] {.async.} = ## Reads ``size`` bytes from ``socket``. Returned future will complete once ## all of the requested data is read. If socket is disconnected during the ## recv operation then the future may complete with only a part of the ## requested data read. If socket is disconnected and no data is available ## to be read then the future will complete with a value of ``""``. - result = recv(socket.fd.TAsyncFD, size, flags) + if socket.isBuffered: + result = newString(size) + + template returnNow(readBytes: int) = + result.setLen(readBytes) + # Only increase buffer position when not peeking. + if (flags and MSG_PEEK) != MSG_PEEK: + socket.currPos.inc(readBytes) + return + + if socket.bufLen == 0: + let res = await socket.readIntoBuf(flags and (not MSG_PEEK)) + if res == 0: returnNow(0) + + var read = 0 + while read < size: + if socket.currPos >= socket.bufLen: + let res = await socket.readIntoBuf(flags and (not MSG_PEEK)) + if res == 0: returnNow(read) + + let chunk = min(socket.bufLen-socket.currPos, size-read) + copyMem(addr(result[read]), addr(socket.buffer[socket.currPos+read]), chunk) + read.inc(chunk) + + returnNow(read) + else: + result = await recv(socket.fd.TAsyncFD, size, flags) proc send*(socket: PAsyncSocket, data: string): PFuture[void] = ## Sends ``data`` to ``socket``. The returned future will complete once all