From 7dc3475e35e38e52cf7efe10b1c4a26094b1dbae Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 6 May 2017 21:10:13 +0200 Subject: [PATCH 01/12] first version of todo app with toy database --- examples/todoapp/todoapp.nim | 27 +++++----- src/kajax.nim | 18 +++++++ src/karaxdb/client.nim | 99 ++++++++++++++++++++++++++++++++++++ src/karaxdb/common.nim | 17 +++++++ src/karaxdb/server.nim | 56 ++++++++++++++++++++ 5 files changed, 204 insertions(+), 13 deletions(-) create mode 100644 src/karaxdb/client.nim create mode 100644 src/karaxdb/common.nim create mode 100644 src/karaxdb/server.nim diff --git a/examples/todoapp/todoapp.nim b/examples/todoapp/todoapp.nim index 802bfc9..d1ceb21 100644 --- a/examples/todoapp/todoapp.nim +++ b/examples/todoapp/todoapp.nim @@ -1,5 +1,5 @@ -import vdom, karax, karaxdsl, jstrutils, components, localstorage +import vdom, karax, karaxdsl, jstrutils, components, karaxdb/client type Filter = enum @@ -9,6 +9,12 @@ var selectedEntry = -1 filter: Filter entriesLen: int + data: Db + +registerOnUpdate proc(newDb: Db) = + merge(newDb, data) + data = newDb + redraw() const contentSuffix = cstring"content" @@ -16,25 +22,23 @@ const lenSuffix = cstring"entriesLen" proc getEntryContent(pos: int): cstring = - result = getItem(&pos & contentSuffix) - if result == cstring"null": - result = nil + extract(data, &pos, contentSuffix) proc isCompleted(pos: int): bool = - var value = getItem(&pos & completedSuffix) + var value = extract(data, &pos, completedSuffix) result = value == cstring"true" proc setEntryContent(pos: int, content: cstring) = - setItem(&pos & contentSuffix, content) + insert(data, &pos, contentSuffix, content) proc markAsCompleted(pos: int, completed: bool) = - setItem(&pos & completedSuffix, &completed) + insert(data, &pos, completedSuffix, &completed) proc addEntry(content: cstring, completed: bool) = setEntryContent(entriesLen, content) markAsCompleted(entriesLen, completed) inc entriesLen - setItem(lenSuffix, &entriesLen) + insert(data, lenSuffix, "equals", &entriesLen) proc updateEntry(pos: int, content: cstring, completed: bool) = setEntryContent(pos, content) @@ -61,7 +65,7 @@ proc toggleEntry(ev: Event; n: VNode) = markAsCompleted(id, not isCompleted(id)) proc onAllDone(ev: Event; n: VNode) = - clear() + insert(data, lenSuffix, "equals", "0") selectedEntry = -1 proc clearCompleted(ev: Event, n: VNode) = @@ -150,8 +154,5 @@ setOnHashChange(proc(hash: cstring) = elif hash == "#/active": filter = active ) -if hasItem(lenSuffix): - entriesLen = parseInt getItem(lenSuffix) -else: - entriesLen = 0 +entriesLen = 0 setRenderer createDom diff --git a/src/kajax.nim b/src/kajax.nim index be230f0..2d88cd1 100644 --- a/src/kajax.nim +++ b/src/kajax.nim @@ -48,3 +48,21 @@ proc ajaxGet*(url: cstring; headers: openarray[(cstring, cstring)]; proc toJson*[T](data: T): cstring {.importc: "JSON.stringify".} proc fromJson*[T](blob: cstring): T {.importc: "JSON.parse".} + +type + MessageEvent* {.importc.} = ref object + data*: cstring + ErrorEvent* {.importc.} = ref object + CloseEvent* {.importc.} = ref object + code*: int + reason*: cstring + + WebSocket* {.importc.} = ref object + onopen*: proc() + onmessage*: proc(ev: MessageEvent) + onclose*: proc(ev: CloseEvent) + onerror*: proc(ev: ErrorEvent) + +proc newWebsocket*(url, protocol: cstring): WebSocket {.importc: "new WebSocket".} + +proc send*(socket: WebSocket, data: cstring) {.importcpp.} diff --git a/src/karaxdb/client.nim b/src/karaxdb/client.nim new file mode 100644 index 0000000..7ea648b --- /dev/null +++ b/src/karaxdb/client.nim @@ -0,0 +1,99 @@ + +import "../kajax", "../jdict", common +export common +from "../karax" import kout + +type + Db* = ref object + data: seq[Triple] + version*: int + next: Db + + Query* = object + constraints*: set[TripleKind] + t*: Triple + + Message {.importc.} = ref object + kind: MessageKind + data: seq[Triple] + version: int + id: MessageId +# RequestMessage {.importc.} = ref object + +let conn = newWebSocket("ws://localhost:8080", "karaxdb") +var gid: MessageId +var pendingIds = newJDict[MessageId, Message]() +var pending: int + +#proc loadDb*(url: cstring): Db = +# result = nil + +proc newTransaction*(): Db = + result = Db(data: @[]) + +proc insert*(head, newdb: Db) = + newdb.next = head + #result = newdb + let expectedVersion = if not head.isNil: head.version + 1 else: 1 + inc gid.int + let m = Message(kind: NewData, data: newdb.data, version: expectedVersion, id: gid) + pendingIds[gid] = m + inc pending + conn.send(toJson(m)) + +proc merge*(newer, older: Db) = + newer.next = older + +proc registerOnUpdate*(update: proc(db: Db)) = + conn.onmessage = + proc (e: MessageEvent) = + let msg = fromJson[Message](e.data) + case msg.kind + of Conflict: + # conflict, so throw away the sent data, don't apply the changes: + if pending > 0: + pendingIds.del msg.id + dec pending + # server sent data that caused the conflict: + if msg.data.len > 0: + let db = Db(data: msg.data, version: msg.version) + update(db) + of Accepted: + kout cstring"accepted", pending + if pending > 0: + let d = pendingIds[msg.id] + # data was not submitted again, so we use the in-memory version + # of the data: + let db = Db(data: d.data, version: d.version) + pendingIds.del msg.id + dec pending + update(db) + of Newdata: + let db = Db(data: msg.data, version: msg.version) + update(db) + +iterator list*(db: Db, q: Query): Triple = + # XXX here the datamodel comes in! We must not + # yield outdated data! + var it = db + while it != nil: + for d in it.data: + var match = true + for k in q.constraints: + if d[k] != q.t[k]: match = false + if match: yield d + it = it.next + +proc extract*(db: Db, q: Query): Triple = + for x in list(db, q): + result = x + break + +proc extract*(db: Db, subj, pred: kstring): kstring = + let q = Query(constraints: {Subj, Pred}, t: [subj, pred, ""]) + result = extract(db, q)[Obj] + +proc insert*(db: Db, subj, pred, obj: kstring) = + let result = Db() + result.data.add([subj, pred, obj]) + insert(db, result) diff --git a/src/karaxdb/common.nim b/src/karaxdb/common.nim new file mode 100644 index 0000000..67cdf4c --- /dev/null +++ b/src/karaxdb/common.nim @@ -0,0 +1,17 @@ + +when defined(js): + type kstring* = cstring +else: + type kstring* = string + +type + TripleKind* = enum + Subj, Pred, Obj + + DbValue* = kstring + Triple* = array[TripleKind, DbValue] + + MessageId* = distinct int + + MessageKind* = enum + Newdata, Conflict, Accepted diff --git a/src/karaxdb/server.nim b/src/karaxdb/server.nim new file mode 100644 index 0000000..c0291d4 --- /dev/null +++ b/src/karaxdb/server.nim @@ -0,0 +1,56 @@ + +import asynchttpserver, asyncdispatch, asyncnet, "../../../websocket/websocket", common, json + +type + Message = object + kind: MessageKind + id: MessageId + data: seq[Triple] + version: int + +proc `%`(id: MessageId): JsonNode = %BiggestInt(id) +proc `%`(k: MessageKind): JsonNode = %BiggestInt(k) + +proc triplesFromJson(j: JsonNode): seq[Triple] = + result = newSeq[Triple](j.len) + var i = 0 + for t in j: + doAssert t.kind == JArray + result[i] = [t[0].str, t[1].str, t[2].str] + inc i + +proc messageFromJson(j: JsonNode): Message = + Message(kind: MessageKind(j["kind"].num), id: MessageId(j["id"].num), + data: triplesFromJson(j["data"]), version: j["version"].num.int) + +var server = newAsyncHttpServer() + +proc cb(req: Request) {.async.} = + let (success, error) = await(verifyWebsocketRequest(req, "karaxdb")) + if not success: + echo "WS negotiation failed: " & error + await req.respond(Http400, "Websocket negotiation failed: " & error) + req.client.close + else: + echo "New websocket customer arrived!" + while true: + try: + var f = await req.client.readData(false) + echo "(opcode: " & $f.opcode & ", data: " & $f.data.len & ")" + let m = messageFromJson(f.data.parseJson) + let om = Message(kind: Accepted, id: m.id, data: @[], version: m.version) + let oms = $(%*om) + echo "OUTPUT ", oms + if f.opcode == Opcode.Text: + waitFor req.client.sendText(oms, false) + else: + echo "protocol error" + #waitFor req.client.sendBinary(f.data, false) + except: + echo getCurrentExceptionMsg() + break + + req.client.close() + echo ".. socket went away." + +waitFor server.serve(Port(8080), cb) From df92d3a55d00bb3ae7392f99564cd53412e24082 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 10 May 2017 09:56:54 +0200 Subject: [PATCH 02/12] first version of transactor (todo-app working) --- src/karaxdb/client.nim | 35 +++------ src/karaxdb/common.nim | 2 +- src/karaxdb/server.nim | 56 -------------- src/karaxdb/transactor.nim | 151 +++++++++++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 83 deletions(-) delete mode 100644 src/karaxdb/server.nim create mode 100644 src/karaxdb/transactor.nim diff --git a/src/karaxdb/client.nim b/src/karaxdb/client.nim index 7ea648b..72c31bc 100644 --- a/src/karaxdb/client.nim +++ b/src/karaxdb/client.nim @@ -21,9 +21,7 @@ type # RequestMessage {.importc.} = ref object let conn = newWebSocket("ws://localhost:8080", "karaxdb") -var gid: MessageId -var pendingIds = newJDict[MessageId, Message]() -var pending: int +var version: int #proc loadDb*(url: cstring): Db = # result = nil @@ -34,11 +32,9 @@ proc newTransaction*(): Db = proc insert*(head, newdb: Db) = newdb.next = head #result = newdb - let expectedVersion = if not head.isNil: head.version + 1 else: 1 - inc gid.int - let m = Message(kind: NewData, data: newdb.data, version: expectedVersion, id: gid) - pendingIds[gid] = m - inc pending + let expectedVersion = version + inc version + let m = Message(kind: NewData, data: newdb.data, version: expectedVersion, id: MessageId(0)) conn.send(toJson(m)) proc merge*(newer, older: Db) = @@ -49,28 +45,15 @@ proc registerOnUpdate*(update: proc(db: Db)) = proc (e: MessageEvent) = let msg = fromJson[Message](e.data) case msg.kind - of Conflict: + of Rejected: # conflict, so throw away the sent data, don't apply the changes: - if pending > 0: - pendingIds.del msg.id - dec pending - # server sent data that caused the conflict: - if msg.data.len > 0: - let db = Db(data: msg.data, version: msg.version) - update(db) - of Accepted: - kout cstring"accepted", pending - if pending > 0: - let d = pendingIds[msg.id] - # data was not submitted again, so we use the in-memory version - # of the data: - let db = Db(data: d.data, version: d.version) - pendingIds.del msg.id - dec pending - update(db) + kout cstring"rejected" of Newdata: let db = Db(data: msg.data, version: msg.version) update(db) + of Disconnect: + kout cstring"disconnected" + else: kout cstring"something else" iterator list*(db: Db, q: Query): Triple = # XXX here the datamodel comes in! We must not diff --git a/src/karaxdb/common.nim b/src/karaxdb/common.nim index 67cdf4c..c098702 100644 --- a/src/karaxdb/common.nim +++ b/src/karaxdb/common.nim @@ -14,4 +14,4 @@ type MessageId* = distinct int MessageKind* = enum - Newdata, Conflict, Accepted + Newdata, Rejected, Disconnect diff --git a/src/karaxdb/server.nim b/src/karaxdb/server.nim deleted file mode 100644 index c0291d4..0000000 --- a/src/karaxdb/server.nim +++ /dev/null @@ -1,56 +0,0 @@ - -import asynchttpserver, asyncdispatch, asyncnet, "../../../websocket/websocket", common, json - -type - Message = object - kind: MessageKind - id: MessageId - data: seq[Triple] - version: int - -proc `%`(id: MessageId): JsonNode = %BiggestInt(id) -proc `%`(k: MessageKind): JsonNode = %BiggestInt(k) - -proc triplesFromJson(j: JsonNode): seq[Triple] = - result = newSeq[Triple](j.len) - var i = 0 - for t in j: - doAssert t.kind == JArray - result[i] = [t[0].str, t[1].str, t[2].str] - inc i - -proc messageFromJson(j: JsonNode): Message = - Message(kind: MessageKind(j["kind"].num), id: MessageId(j["id"].num), - data: triplesFromJson(j["data"]), version: j["version"].num.int) - -var server = newAsyncHttpServer() - -proc cb(req: Request) {.async.} = - let (success, error) = await(verifyWebsocketRequest(req, "karaxdb")) - if not success: - echo "WS negotiation failed: " & error - await req.respond(Http400, "Websocket negotiation failed: " & error) - req.client.close - else: - echo "New websocket customer arrived!" - while true: - try: - var f = await req.client.readData(false) - echo "(opcode: " & $f.opcode & ", data: " & $f.data.len & ")" - let m = messageFromJson(f.data.parseJson) - let om = Message(kind: Accepted, id: m.id, data: @[], version: m.version) - let oms = $(%*om) - echo "OUTPUT ", oms - if f.opcode == Opcode.Text: - waitFor req.client.sendText(oms, false) - else: - echo "protocol error" - #waitFor req.client.sendBinary(f.data, false) - except: - echo getCurrentExceptionMsg() - break - - req.client.close() - echo ".. socket went away." - -waitFor server.serve(Port(8080), cb) diff --git a/src/karaxdb/transactor.nim b/src/karaxdb/transactor.nim new file mode 100644 index 0000000..057884d --- /dev/null +++ b/src/karaxdb/transactor.nim @@ -0,0 +1,151 @@ + +import asynchttpserver, asyncdispatch, asyncnet, "../../../websocket/websocket", common, json, + strutils, times + +type + Message = object + kind: MessageKind + id: MessageId + data: seq[Triple] + version: int + +proc `%`(id: MessageId): JsonNode = %BiggestInt(id) +proc `%`(k: MessageKind): JsonNode = %BiggestInt(k) + +proc triplesFromJson(j: JsonNode): seq[Triple] = + result = newSeq[Triple](j.len) + var i = 0 + for t in j: + doAssert t.kind == JArray + let val = if t[2].kind == JNull: string(nil) else: t[2].str + result[i] = [t[0].str, t[1].str, val] + inc i + +proc messageFromJson(j: JsonNode): Message = + Message(kind: MessageKind(j["kind"].num), id: MessageId(j["id"].num), + data: triplesFromJson(j["data"]), version: j["version"].num.int) + +proc error(msg: string) = echo msg +proc warn(msg: string) = echo msg + +type + Tx = object + data: string + version: int + Client = ref object + socket: AsyncSocket + connected: bool + hostname: string + lastMessage: float + rapidMessageCount: int + + Server = ref object + clients: seq[Client] + needsUpdate: bool + txs: seq[Tx] + version: int + +proc newClient(socket: AsyncSocket, hostname: string): Client = + Client(socket: socket, connected: true, hostname: hostname) + +proc `$`(client: Client): string = + "Client(ip: $1)" % [client.hostname] + +proc updateClients(server: Server) {.async.} = + while true: + var needsUpdate = false + for client in server.clients: + if not client.connected: + needsUpdate = true + break + + server.needsUpdate = server.needsUpdate or needsUpdate + if server.needsUpdate and server.txs.len != 0: + var someDead = false + # perform a copy to prevent the race condition: + var txs = server.txs + setLen(server.txs, 0) + for tx in txs: + for c in server.clients: + if c.connected: + await c.socket.sendText(tx.data, false) + else: + someDead = true + if someDead: + var i = 0 + while i < server.clients.len: + if not server.clients[i].connected: del(server.clients, i) + else: inc i + server.needsUpdate = false + # let other stuff in the main loop run: + await sleepAsync(10) + +proc processMessage(server: Server, client: Client, data: string) {.async.} = + # Check if last message was relatively recent. If so, kick the user. + echo "processMessage ", data + if epochTime() - client.lastMessage < 0.1: # 100ms + client.rapidMessageCount.inc + else: + client.rapidMessageCount = 0 + + client.lastMessage = epochTime() + if client.rapidMessageCount > 10: + warn("Client ($1) is firing messages too rapidly. Killing." % $client) + client.connected = false + let msgj = parseJson(data) + let msg = messageFromJson(msgj) + case msg.kind + of Newdata: + if msg.version == server.version: + server.txs.add Tx(data: data, version: msg.version) + server.needsUpdate = true + inc server.version + else: + let om = Message(kind: Rejected, id: msg.id, data: @[], version: server.version) + await client.socket.sendText($(%*om), false) + else: + # either Disconnect or an invalid message type: + client.connected = false + server.needsUpdate = true + +proc processClient(server: Server, client: Client) {.async.} = + while client.connected: + var frameFut = client.socket.readData(false) + yield frameFut + if frameFut.failed: + error("Error occurred handling client messages.\n" & + frameFut.error.msg) + client.connected = false + break + + let frame = frameFut.read() + if frame.opcode == Opcode.Text: + let processFut = processMessage(server, client, frame.data) + if processFut.failed: + error("Client ($1) attempted to send bad JSON? " % $client & "\n" & + processFut.error.msg) + client.connected = false + + client.socket.close() + +proc onRequest(server: Server, req: Request) {.async.} = + let (success, error) = await verifyWebsocketRequest(req, "karaxdb") + if success: + echo("Client connected from ", req.hostname) + server.clients.add(newClient(req.client, req.hostname)) + asyncCheck processClient(server, server.clients[^1]) + else: + echo("WS negotiation failed: ", error) + await req.respond(Http400, "WebSocket negotiation failed: " & error) + req.client.close() + +proc main = + let httpServer = newAsyncHttpServer() + let server = Server(clients: @[], txs: @[]) + + proc cb(req: Request): Future[void] {.async.} = await onRequest(server, req) + + asyncCheck updateClients(server) + waitFor httpServer.serve(Port(8080), cb) + +main() From 93ce8b06e1312c6616e749e1daacafaac65624e6 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 May 2017 15:04:21 +0200 Subject: [PATCH 03/12] toy btree implementation --- src/karaxdb/btree.nim | 156 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 src/karaxdb/btree.nim diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim new file mode 100644 index 0000000..97f5b2d --- /dev/null +++ b/src/karaxdb/btree.nim @@ -0,0 +1,156 @@ + +# max children per B-tree node = M-1 +# (must be even and greater than 2) +const + M = 4 + Mhalf = M div 2 + +type + Key = string + Val = string + Entry = ref object + key: Key + val: Val # external nodes only + next: Node # internal nodes only; helper field to iterate over array entries + Node = ref object + m: int + children: array[M, Entry] + BTree = ref object + root: Node + height: int ## height + n: int ## number of key-value pairs + +proc newBTree(): BTree = BTree(root: Node(m: 0)) + +proc less(a, b: Key): bool = cmp(a, b) < 0 + +proc eq(a, b: Key): bool = cmp(a, b) == 0 + +proc search(x: Node, key: Key, ht: int): Val = + if ht == 0: + # external node + for j in 0 ..< x.m: + if eq(key, x.children[j].key): return x.children[j].val + else: + # internal node + for j in 0 ..< x.m: + if j+1 == x.m or less(key, x.children[j+1].key): + return search(x.children[j].next, key, ht-1) + return nil + +proc candidates(x: Node, key: Key, ht: int): Node = + if ht == 0: + return x + else: + # internal node + for j in 0 ..< x.m: + if j+1 == x.m or less(key, x.children[j+1].key): + return candidates(x.children[j].next, key, ht-1) + return nil + +iterator allValues(t: BTree; key: Key): Val = + let x = candidates(t.root, key, t.height) + if x != nil: + for j in 0 ..< x.m: + if eq(key, x.children[j].key): yield x.children[j].val + +proc get(t: BTree; key: Key): Val = search(t.root, key, t.height) + +proc split(h: Node): Node = + ## split node in half + result = Node(m: Mhalf) + h.m = Mhalf + for j in 0 ..< Mhalf: + result.children[j] = h.children[Mhalf + j] + +proc insert(h: Node, key: Key, val: Val, ht: int): Node = + var t = Entry(key: key, val: val, next: nil) + var j = 0 + if ht == 0: + # external node: + while j < h.m: + if less(key, h.children[j].key): break + inc j + else: + # internal node + while j < h.m: + if j+1 == h.m or less(key, h.children[j+1].key): + let u = insert(h.children[j].next, key, val, ht-1) + inc j + if u == nil: return nil + t.key = u.children[0].key + t.next = u + break + inc j + for i in countdown(h.m, j+1): + h.children[i] = h.children[i-1] + h.children[j] = t + inc h.m + return if h.m < M: nil else: split(h) + +proc put(b: BTree; key: Key; val: Val) = + let u = insert(b.root, key, val, b.height) + inc b.n + if u == nil: return + + # need to split root + let t = Node(m: 2) + t.children[0] = Entry(key: b.root.children[0].key, val: nil, next: b.root) + t.children[1] = Entry(key: u.children[0].key, val: nil, next: u) + b.root = t + inc b.height + +proc toString(h: Node, ht: int, indent: string; result: var string) = + if ht == 0: + for j in 0.. 0: result.add(indent & "(" & $h.children[j].key & ")\n") + toString(h.children[j].next, ht-1, indent & " ", result) + +proc `$`(b: BTree): string = + result = "" + toString(b.root, b.height, "", result) + +proc main = + var st = newBTree() + st.put("www.cs.princeton.edu", "128.112.136.12") + st.put("www.cs.princeton.edu", "128.112.136.11") + st.put("www.princeton.edu", "128.112.128.15") + st.put("www.yale.edu", "130.132.143.21") + st.put("www.simpsons.com", "209.052.165.60") + st.put("www.apple.com", "17.112.152.32") + st.put("www.amazon.com", "207.171.182.16") + st.put("www.ebay.com", "66.135.192.87") + st.put("www.cnn.com", "64.236.16.20") + st.put("www.google.com", "216.239.41.99") + st.put("www.nytimes.com", "199.239.136.200") + st.put("www.microsoft.com", "207.126.99.140") + st.put("www.dell.com", "143.166.224.230") + st.put("www.slashdot.org", "66.35.250.151") + st.put("www.espn.com", "199.181.135.201") + st.put("www.weather.com", "63.111.66.11") + st.put("www.yahoo.com", "216.109.118.65") + + echo("cs.princeton.edu: ", st.get("www.cs.princeton.edu")) + echo("hardvardsucks.com: ", st.get("www.harvardsucks.com")) + echo("simpsons.com: ", st.get("www.simpsons.com")) + echo("apple.com: ", st.get("www.apple.com")) + echo("ebay.com: ", st.get("www.ebay.com")) + echo("dell.com: ", st.get("www.dell.com")) + echo() + echo("size: ", st.n) + echo("height: ", st.height) + echo(st) + + var dups = newBTree() + for i in 0..20: + dups.put("testme", $i) + echo dups + for v in allValues(dups, "testme"): + echo v + echo dups.get("testme") + +main() From 927e4099f5ccf4b8e024da5db9de14788bbcdbbc Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 May 2017 17:04:12 +0200 Subject: [PATCH 04/12] BTree: restructured for memory layout efficiency --- src/karaxdb/btree.nim | 140 ++++++++++++++++++++++-------------------- 1 file changed, 73 insertions(+), 67 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index 97f5b2d..c25f48d 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -2,25 +2,26 @@ # max children per B-tree node = M-1 # (must be even and greater than 2) const - M = 4 - Mhalf = M div 2 + M = 4 + Mhalf = M div 2 type Key = string Val = string - Entry = ref object - key: Key - val: Val # external nodes only - next: Node # internal nodes only; helper field to iterate over array entries Node = ref object m: int - children: array[M, Entry] + keys: array[M, Key] + case isInternal: bool + of false: + vals: array[M, Val] + of true: + links: array[M, Node] BTree = ref object root: Node height: int ## height n: int ## number of key-value pairs -proc newBTree(): BTree = BTree(root: Node(m: 0)) +proc newBTree(): BTree = BTree(root: Node(m: 0, isInternal: false)) proc less(a, b: Key): bool = cmp(a, b) < 0 @@ -28,63 +29,61 @@ proc eq(a, b: Key): bool = cmp(a, b) == 0 proc search(x: Node, key: Key, ht: int): Val = if ht == 0: - # external node + assert(not x.isInternal) for j in 0 ..< x.m: - if eq(key, x.children[j].key): return x.children[j].val + if eq(key, x.keys[j]): return x.vals[j] else: - # internal node + assert(x.isInternal) for j in 0 ..< x.m: - if j+1 == x.m or less(key, x.children[j+1].key): - return search(x.children[j].next, key, ht-1) - return nil - -proc candidates(x: Node, key: Key, ht: int): Node = - if ht == 0: - return x - else: - # internal node - for j in 0 ..< x.m: - if j+1 == x.m or less(key, x.children[j+1].key): - return candidates(x.children[j].next, key, ht-1) - return nil - -iterator allValues(t: BTree; key: Key): Val = - let x = candidates(t.root, key, t.height) - if x != nil: - for j in 0 ..< x.m: - if eq(key, x.children[j].key): yield x.children[j].val + if j+1 == x.m or less(key, x.keys[j+1]): + return search(x.links[j], key, ht-1) proc get(t: BTree; key: Key): Val = search(t.root, key, t.height) proc split(h: Node): Node = ## split node in half - result = Node(m: Mhalf) + result = Node(m: Mhalf, isInternal: h.isInternal) h.m = Mhalf for j in 0 ..< Mhalf: - result.children[j] = h.children[Mhalf + j] + result.keys[j] = h.keys[Mhalf + j] + if h.isInternal: + for j in 0 ..< Mhalf: + result.links[j] = h.links[Mhalf + j] + else: + for j in 0 ..< Mhalf: + shallowCopy(result.vals[j], h.vals[Mhalf + j]) proc insert(h: Node, key: Key, val: Val, ht: int): Node = - var t = Entry(key: key, val: val, next: nil) + #var t = Entry(key: key, val: val, next: nil) + var newKey = key var j = 0 if ht == 0: - # external node: + assert(not h.isInternal) while j < h.m: - if less(key, h.children[j].key): break + if less(key, h.keys[j]): break inc j + for i in countdown(h.m, j+1): + shallowCopy(h.vals[i], h.vals[i-1]) + h.vals[j] = val else: - # internal node + assert h.isInternal + var newLink: Node = nil while j < h.m: - if j+1 == h.m or less(key, h.children[j+1].key): - let u = insert(h.children[j].next, key, val, ht-1) + if j+1 == h.m or less(key, h.keys[j+1]): + let u = insert(h.links[j], key, val, ht-1) inc j if u == nil: return nil - t.key = u.children[0].key - t.next = u + newKey = u.keys[0] + newLink = u break inc j + for i in countdown(h.m, j+1): + h.links[i] = h.links[i-1] + h.links[j] = newLink + for i in countdown(h.m, j+1): - h.children[i] = h.children[i-1] - h.children[j] = t + h.keys[i] = h.keys[i-1] + h.keys[j] = newKey inc h.m return if h.m < M: nil else: split(h) @@ -94,21 +93,25 @@ proc put(b: BTree; key: Key; val: Val) = if u == nil: return # need to split root - let t = Node(m: 2) - t.children[0] = Entry(key: b.root.children[0].key, val: nil, next: b.root) - t.children[1] = Entry(key: u.children[0].key, val: nil, next: u) + let t = Node(m: 2, isInternal: true) + t.keys[0] = b.root.keys[0] + t.links[0] = b.root + t.keys[1] = u.keys[0] + t.links[1] = u b.root = t inc b.height proc toString(h: Node, ht: int, indent: string; result: var string) = if ht == 0: + assert(not h.isInternal) for j in 0.. 0: result.add(indent & "(" & $h.children[j].key & ")\n") - toString(h.children[j].next, ht-1, indent & " ", result) + if j > 0: result.add(indent & "(" & $h.keys[j] & ")\n") + toString(h.links[j], ht-1, indent & " ", result) proc `$`(b: BTree): string = result = "" @@ -116,8 +119,8 @@ proc `$`(b: BTree): string = proc main = var st = newBTree() - st.put("www.cs.princeton.edu", "128.112.136.12") - st.put("www.cs.princeton.edu", "128.112.136.11") + st.put("www.cs.princeton.edu", "abc") + st.put("www.cs.princeton.edu", "xyz") st.put("www.princeton.edu", "128.112.128.15") st.put("www.yale.edu", "130.132.143.21") st.put("www.simpsons.com", "209.052.165.60") @@ -134,23 +137,26 @@ proc main = st.put("www.weather.com", "63.111.66.11") st.put("www.yahoo.com", "216.109.118.65") - echo("cs.princeton.edu: ", st.get("www.cs.princeton.edu")) - echo("hardvardsucks.com: ", st.get("www.harvardsucks.com")) - echo("simpsons.com: ", st.get("www.simpsons.com")) - echo("apple.com: ", st.get("www.apple.com")) - echo("ebay.com: ", st.get("www.ebay.com")) - echo("dell.com: ", st.get("www.dell.com")) - echo() - echo("size: ", st.n) - echo("height: ", st.height) - echo(st) + assert st.get("www.cs.princeton.edu") == "abc" + assert st.get("www.harvardsucks.com") == nil - var dups = newBTree() - for i in 0..20: - dups.put("testme", $i) - echo dups - for v in allValues(dups, "testme"): - echo v - echo dups.get("testme") + assert st.get("www.simpsons.com") == "209.052.165.60" + assert st.get("www.apple.com") == "17.112.152.32" + assert st.get("www.ebay.com") == "66.135.192.87" + assert st.get("www.dell.com") == "143.166.224.230" + assert(st.n == 17) + assert(st.height == 2) + + when true: + var b2 = newBTree() + const iters = 10_000 + for i in 1..iters: + b2.put($i, $(iters - i)) + for i in 1..iters: + let x = b2.get($i) + if x != $(iters - i): + echo "got ", x, ", but expected ", iters - i + echo b2.n + echo b2.height main() From bd67d7226a4e52d070a0c8ad69e7dfee7c7a214e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 May 2017 17:05:34 +0200 Subject: [PATCH 05/12] made M realistic --- src/karaxdb/btree.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index c25f48d..fd453af 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -2,7 +2,7 @@ # max children per B-tree node = M-1 # (must be even and greater than 2) const - M = 4 + M = 128 Mhalf = M div 2 type @@ -145,7 +145,6 @@ proc main = assert st.get("www.ebay.com") == "66.135.192.87" assert st.get("www.dell.com") == "143.166.224.230" assert(st.n == 17) - assert(st.height == 2) when true: var b2 = newBTree() From e300692c753a2bfcaf89c3e2ca65ab6eced7b3c9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 May 2017 17:11:37 +0200 Subject: [PATCH 06/12] Btree doesn't have to be a 'ref object' --- src/karaxdb/btree.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index fd453af..f7c1655 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -16,7 +16,7 @@ type vals: array[M, Val] of true: links: array[M, Node] - BTree = ref object + BTree = object root: Node height: int ## height n: int ## number of key-value pairs @@ -87,7 +87,7 @@ proc insert(h: Node, key: Key, val: Val, ht: int): Node = inc h.m return if h.m < M: nil else: split(h) -proc put(b: BTree; key: Key; val: Val) = +proc put(b: var BTree; key: Key; val: Val) = let u = insert(b.root, key, val, b.height) inc b.n if u == nil: return From 2908b1e44a4b9e479592a713d60f9da8afaacf80 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 May 2017 17:50:25 +0200 Subject: [PATCH 07/12] first version of broken immutable BTree --- src/karaxdb/btree.nim | 108 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 12 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index f7c1655..f6042bf 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -1,8 +1,12 @@ -# max children per B-tree node = M-1 -# (must be even and greater than 2) +## General purpose BTree implementation. Can also be used as a persistent +## data structure. The persistent operations use a 'Ps' suffix. +## Can also use a page manager for allocations. The page manager be used +## to off load pages to a file system or to send it over the wire. + const - M = 128 + M = 128 # max children per B-tree node = M-1 + # (must be even and greater than 2) Mhalf = M div 2 type @@ -40,18 +44,32 @@ proc search(x: Node, key: Key, ht: int): Val = proc get(t: BTree; key: Key): Val = search(t.root, key, t.height) +proc copyHalf(h, result: Node; offset: int) = + for j in 0 ..< Mhalf: + result.keys[j] = h.keys[offset + j] + if h.isInternal: + for j in 0 ..< Mhalf: + result.links[j] = h.links[offset + j] + else: + for j in 0 ..< Mhalf: + shallowCopy(result.vals[j], h.vals[offset + j]) + proc split(h: Node): Node = ## split node in half result = Node(m: Mhalf, isInternal: h.isInternal) h.m = Mhalf - for j in 0 ..< Mhalf: - result.keys[j] = h.keys[Mhalf + j] - if h.isInternal: - for j in 0 ..< Mhalf: - result.links[j] = h.links[Mhalf + j] - else: - for j in 0 ..< Mhalf: - shallowCopy(result.vals[j], h.vals[Mhalf + j]) + copyHalf(h, result, Mhalf) + +when false: + # unnecessary because when we call 'split' we know it's not a + # shared node! + proc splitPs(h: Node): (Node, Node) = + ## persistent variant of 'split'. + var a = Node(m: Mhalf, isInternal: h.isInternal) + var b = Node(m: Mhalf, isInternal: h.isInternal) + copyHalf(h, a, Mhalf) + copyHalf(h, b, 0) + result = (a, b) proc insert(h: Node, key: Key, val: Val, ht: int): Node = #var t = Entry(key: key, val: val, next: nil) @@ -87,6 +105,42 @@ proc insert(h: Node, key: Key, val: Val, ht: int): Node = inc h.m return if h.m < M: nil else: split(h) +proc insertPs(h: Node, key: Key, val: Val, ht: int): (Node, Node) = + #var t = Entry(key: key, val: val, next: nil) + var newKey = key + var j = 0 + var hh = Node(m: h.m, isInternal: h.isInternal) + if ht == 0: + assert(not h.isInternal) + while j < h.m: + if less(key, h.keys[j]): break + inc j + for i in countdown(h.m, j+1): + shallowCopy(hh.vals[i], h.vals[i-1]) + hh.vals[j] = val + else: + assert h.isInternal + var newLink: Node = nil + while j < h.m: + if j+1 == h.m or less(key, h.keys[j+1]): + let (u, root) = insertPs(h.links[j], key, val, ht-1) + if u == nil: return (u, root) + hh.links[j] = root + newKey = u.keys[0] + newLink = u + inc j + break + inc j + for i in countdown(h.m, j+1): + hh.links[i] = h.links[i-1] + hh.links[j] = newLink + + for i in countdown(h.m, j+1): + hh.keys[i] = h.keys[i-1] + hh.keys[j] = newKey + inc hh.m + return if hh.m < M: (nil, nil) else: (split(hh), hh) + proc put(b: var BTree; key: Key; val: Val) = let u = insert(b.root, key, val, b.height) inc b.n @@ -101,6 +155,24 @@ proc put(b: var BTree; key: Key; val: Val) = b.root = t inc b.height +proc putPs(b: BTree; key: Key; val: Val): BTree = + let v = insertPs(b.root, key, val, b.height) + result.n = b.n + 1 + result.height = b.height + let u = v[0] + let root = v[1] + if u == nil: + result.root = Node(m: 0, isInternal: false) + return + # need to split root + let t = Node(m: 2, isInternal: true) + t.keys[0] = root.keys[0] + t.links[0] = root + t.keys[1] = u.keys[0] + t.links[1] = u + result.root = t + inc result.height + proc toString(h: Node, ht: int, indent: string; result: var string) = if ht == 0: assert(not h.isInternal) @@ -146,7 +218,7 @@ proc main = assert st.get("www.dell.com") == "143.166.224.230" assert(st.n == 17) - when true: + when false: var b2 = newBTree() const iters = 10_000 for i in 1..iters: @@ -158,4 +230,16 @@ proc main = echo b2.n echo b2.height + when true: + var b2 = newBTree() + const iters = 10_000 + for i in 1..iters: + b2 = b2.putPs($i, $(iters - i)) + for i in 1..iters: + let x = b2.get($i) + if x != $(iters - i): + echo "got ", x, ", but expected ", iters - i + echo b2.n + echo b2.height + main() From 1f1edeb23fa753d1ab2ef46bf53e2b1441fb39de Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 12 May 2017 22:23:18 +0200 Subject: [PATCH 08/12] immutable BTree works --- src/karaxdb/btree.nim | 56 +++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index f6042bf..d1b7844 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -5,7 +5,7 @@ ## to off load pages to a file system or to send it over the wire. const - M = 128 # max children per B-tree node = M-1 + M = 4 # max children per B-tree node = M-1 # (must be even and greater than 2) Mhalf = M div 2 @@ -106,40 +106,40 @@ proc insert(h: Node, key: Key, val: Val, ht: int): Node = return if h.m < M: nil else: split(h) proc insertPs(h: Node, key: Key, val: Val, ht: int): (Node, Node) = - #var t = Entry(key: key, val: val, next: nil) - var newKey = key var j = 0 var hh = Node(m: h.m, isInternal: h.isInternal) + for i in 0 ..< h.m: hh.keys[i] = h.keys[i] if ht == 0: assert(not h.isInternal) while j < h.m: if less(key, h.keys[j]): break inc j - for i in countdown(h.m, j+1): - shallowCopy(hh.vals[i], h.vals[i-1]) + for i in countdown(h.m, j+1): shallowCopy(hh.vals[i], h.vals[i-1]) + for i in 0 ..< j: shallowCopy(hh.vals[i], h.vals[i]) + for i in countdown(h.m, j+1): hh.keys[i] = hh.keys[i-1] hh.vals[j] = val + hh.keys[j] = key else: assert h.isInternal - var newLink: Node = nil + # could optimize this copying here: + for i in 0 ..< h.m: hh.links[i] = h.links[i] while j < h.m: if j+1 == h.m or less(key, h.keys[j+1]): - let (u, root) = insertPs(h.links[j], key, val, ht-1) - if u == nil: return (u, root) + let (root, r) = insertPs(h.links[j], key, val, ht-1) hh.links[j] = root - newKey = u.keys[0] - newLink = u - inc j - break + if r == nil: + return (hh, nil) + else: + inc j + for i in countdown(h.m, j+1): + hh.links[i] = hh.links[i-1] + hh.keys[i] = hh.keys[i-1] + hh.links[j] = r + hh.keys[j] = r.keys[0] + break inc j - for i in countdown(h.m, j+1): - hh.links[i] = h.links[i-1] - hh.links[j] = newLink - - for i in countdown(h.m, j+1): - hh.keys[i] = h.keys[i-1] - hh.keys[j] = newKey inc hh.m - return if hh.m < M: (nil, nil) else: (split(hh), hh) + return if hh.m < M: (hh, nil) else: (hh, split(hh)) proc put(b: var BTree; key: Key; val: Val) = let u = insert(b.root, key, val, b.height) @@ -156,13 +156,11 @@ proc put(b: var BTree; key: Key; val: Val) = inc b.height proc putPs(b: BTree; key: Key; val: Val): BTree = - let v = insertPs(b.root, key, val, b.height) + let (root, u) = insertPs(b.root, key, val, b.height) result.n = b.n + 1 result.height = b.height - let u = v[0] - let root = v[1] if u == nil: - result.root = Node(m: 0, isInternal: false) + result.root = root return # need to split root let t = Node(m: 2, isInternal: true) @@ -231,15 +229,17 @@ proc main = echo b2.height when true: + var b1 = newBTree() var b2 = newBTree() - const iters = 10_000 + const iters = 60_000 for i in 1..iters: b2 = b2.putPs($i, $(iters - i)) + b1.put($i, $(iters - i)) for i in 1..iters: let x = b2.get($i) if x != $(iters - i): - echo "got ", x, ", but expected ", iters - i - echo b2.n - echo b2.height + echo i, "th iteration; got ", x, ", but expected ", iters - i + echo b2.n, " = ", b1.n + echo b2.height, " = ", b1.height main() From eb78bc7a514da70d35fd416e520abef240c6afd5 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 16 May 2017 09:56:55 +0200 Subject: [PATCH 09/12] more stuff that doesn't work --- src/karaxdb/btree.nim | 205 +++++++++++++++++++++++++++++++----------- 1 file changed, 154 insertions(+), 51 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index d1b7844..14e79ab 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -4,11 +4,26 @@ ## Can also use a page manager for allocations. The page manager be used ## to off load pages to a file system or to send it over the wire. +## Todo: +## - Add logic to deal with the fact that keys do not have to be unique. +## - Ranged queries +## - Make it generic and low level +## - Support for external nodes and a page cache + const M = 4 # max children per B-tree node = M-1 # (must be even and greater than 2) Mhalf = M div 2 + SupportFullTableScan = true + SupportDuplicateKeys = true + +## Due to the fact that leaves are shared among multiple BTrees the following +## fields in a Node are downright impossible: +## - parent +## - next +## - prev + type Key = string Val = string @@ -24,6 +39,12 @@ type root: Node height: int ## height n: int ## number of key-value pairs + CmpKind {.pure.} = enum + eq, le, lt, ge, gt, neq + Cursor = object + n: Node + i, dir: int + up: seq[(Node, int)] proc newBTree(): BTree = BTree(root: Node(m: 0, isInternal: false)) @@ -42,6 +63,86 @@ proc search(x: Node, key: Key, ht: int): Val = if j+1 == x.m or less(key, x.keys[j+1]): return search(x.links[j], key, ht-1) +proc `=~`(i: int; k: CmpKind): bool = + ## check if the result of 'cmp' matches what was requested by 'k': + case k + of CmpKind.eq: i == 0 + of CmpKind.le: i <= 0 + of CmpKind.lt: i < 0 + of CmpKind.ge: i >= 0 + of CmpKind.gt: i > 0 + of CmpKind.neq: i != 0 + +proc canPrune(i: int; k: CmpKind): bool = + case k + of CmpKind.eq: + # we demand equality so if bigger, we can prune: + i > 0 + of CmpKind.le: + i > 0 + of CmpKind.lt: i >= 0 + of CmpKind.ge: i < 0 + of CmpKind.gt: i <= 0 + of CmpKind.neq: i != 0 + +proc startingPoint(x: Node; key: Key; kind: CmpKind): Cursor = + if x.isInternal: + for j in 0 ..< x.m: + if j+1 == x.m or cmp(key, x.keys[j+1]) =~ kind: + return startingPoint(x.links[j], key, kind) + else: + for j in 0 ..< x.m: + let cmpRes = cmp(key, x.keys[j]) + if cmpRes =~ kind: return Cursor(n: x, i: j) + +proc dos(x: Node; key: Key; kind: CmpKind; withKey: proc(k: Key; v: Val)) = + if not x.isInternal: + echo "came here" + for j in 0 ..< x.m: + if cmp(key, x.keys[j]) =~ kind: + withKey(x.keys[j+1], x.vals[j]) + else: + for j in 0 ..< x.m-1: + let cmpRes = cmp(key, x.keys[j+1]) + echo "came here B ", canPrune(cmpRes, kind) + if not canPrune(cmpRes, kind): #cmpRes =~ kind: + dos(x.links[j], key, kind, withKey) + #if canPrune(cmpRes, kind): return + # XXX j+1==x.m case? + + + +proc init(x: Node): Cursor = + result.up = @[] + result.i = 0 + var it = x + while it.isInternal: + result.up.add((it, 0)) + it = it.links[0] + result.n = it + +proc next(c: var Cursor) = + assert c.n != nil + if c.i >= c.n.m: + # current leaf exhausted, pick the next one: + if c.up.len > 0: + (c.n, c.i) = c.up.pop() + + inc c.i + var u = 1 + while c.i > c.n.m or c.n.isInternal: + if c.up.len > 0: + (c.n, c.i) = c.up[c.up.len - u] + else: + c.n = nil + return + assert(c.n == nil or not c.n.isInternal) + +proc atEnd(c: Cursor): bool = c.n == nil + +proc getKey(c: Cursor): Key = discard +proc getVal(c: Cursor): Val = discard + proc get(t: BTree; key: Key): Val = search(t.root, key, t.height) proc copyHalf(h, result: Node; offset: int) = @@ -187,59 +288,61 @@ proc `$`(b: BTree): string = result = "" toString(b.root, b.height, "", result) -proc main = - var st = newBTree() - st.put("www.cs.princeton.edu", "abc") - st.put("www.cs.princeton.edu", "xyz") - st.put("www.princeton.edu", "128.112.128.15") - st.put("www.yale.edu", "130.132.143.21") - st.put("www.simpsons.com", "209.052.165.60") - st.put("www.apple.com", "17.112.152.32") - st.put("www.amazon.com", "207.171.182.16") - st.put("www.ebay.com", "66.135.192.87") - st.put("www.cnn.com", "64.236.16.20") - st.put("www.google.com", "216.239.41.99") - st.put("www.nytimes.com", "199.239.136.200") - st.put("www.microsoft.com", "207.126.99.140") - st.put("www.dell.com", "143.166.224.230") - st.put("www.slashdot.org", "66.35.250.151") - st.put("www.espn.com", "199.181.135.201") - st.put("www.weather.com", "63.111.66.11") - st.put("www.yahoo.com", "216.109.118.65") +when isMainModule: + proc main = + var st = newBTree() + st.put("www.cs.princeton.edu", "abc") + st.put("www.cs.princeton.edu", "xyz") + st.put("www.princeton.edu", "128.112.128.15") + st.put("www.yale.edu", "130.132.143.21") + st.put("www.simpsons.com", "209.052.165.60") + st.put("www.apple.com", "17.112.152.32") + st.put("www.amazon.com", "207.171.182.16") + st.put("www.ebay.com", "66.135.192.87") + st.put("www.cnn.com", "64.236.16.20") + st.put("www.google.com", "216.239.41.99") + st.put("www.nytimes.com", "199.239.136.200") + st.put("www.microsoft.com", "207.126.99.140") + st.put("www.dell.com", "143.166.224.230") + st.put("www.slashdot.org", "66.35.250.151") + st.put("www.espn.com", "199.181.135.201") + st.put("www.weather.com", "63.111.66.11") + st.put("www.yahoo.com", "216.109.118.65") - assert st.get("www.cs.princeton.edu") == "abc" - assert st.get("www.harvardsucks.com") == nil + assert st.get("www.cs.princeton.edu") == "abc" + assert st.get("www.harvardsucks.com") == nil - assert st.get("www.simpsons.com") == "209.052.165.60" - assert st.get("www.apple.com") == "17.112.152.32" - assert st.get("www.ebay.com") == "66.135.192.87" - assert st.get("www.dell.com") == "143.166.224.230" - assert(st.n == 17) + assert st.get("www.simpsons.com") == "209.052.165.60" + assert st.get("www.apple.com") == "17.112.152.32" + assert st.get("www.ebay.com") == "66.135.192.87" + assert st.get("www.dell.com") == "143.166.224.230" + assert(st.n == 17) - when false: - var b2 = newBTree() - const iters = 10_000 - for i in 1..iters: - b2.put($i, $(iters - i)) - for i in 1..iters: - let x = b2.get($i) - if x != $(iters - i): - echo "got ", x, ", but expected ", iters - i - echo b2.n - echo b2.height + when false: + var b2 = newBTree() + const iters = 10_000 + for i in 1..iters: + b2.put($i, $(iters - i)) + for i in 1..iters: + let x = b2.get($i) + if x != $(iters - i): + echo "got ", x, ", but expected ", iters - i + echo b2.n + echo b2.height - when true: - var b1 = newBTree() - var b2 = newBTree() - const iters = 60_000 - for i in 1..iters: - b2 = b2.putPs($i, $(iters - i)) - b1.put($i, $(iters - i)) - for i in 1..iters: - let x = b2.get($i) - if x != $(iters - i): - echo i, "th iteration; got ", x, ", but expected ", iters - i - echo b2.n, " = ", b1.n - echo b2.height, " = ", b1.height + when true: + var b1 = newBTree() + var b2 = newBTree() + const iters = 10 #60_000 + for i in 1..iters: + b2 = b2.putPs($i, $(iters - i)) + b1.put($i, $(iters - i)) + for i in 1..iters: + let x = b2.get($i) + if x != $(iters - i): + echo i, "th iteration; got ", x, ", but expected ", iters - i + echo b2.n, " = ", b1.n + echo b2.height, " = ", b1.height + dos(b1.root, "5", CmpKind.eq, proc(k: Key; v: Val) = echo("k ", k, " = ", v)) -main() + main() From 5599a59e870b4a56b02b086934754fd578590210 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 19 May 2017 13:20:15 +0200 Subject: [PATCH 10/12] simple tree iterator now works --- src/karaxdb/btree.nim | 109 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index 14e79ab..e64396f 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -73,17 +73,17 @@ proc `=~`(i: int; k: CmpKind): bool = of CmpKind.gt: i > 0 of CmpKind.neq: i != 0 -proc canPrune(i: int; k: CmpKind): bool = +proc follow(i: int; k: CmpKind): bool = case k of CmpKind.eq: - # we demand equality so if bigger, we can prune: - i > 0 + # we demand equality so if less than, follow it + i < 0 of CmpKind.le: - i > 0 - of CmpKind.lt: i >= 0 - of CmpKind.ge: i < 0 - of CmpKind.gt: i <= 0 - of CmpKind.neq: i != 0 + i <= 0 + of CmpKind.lt: i <= 0 + of CmpKind.ge: i >= 0 + of CmpKind.gt: i >= 0 + of CmpKind.neq: true proc startingPoint(x: Node; key: Key; kind: CmpKind): Cursor = if x.isInternal: @@ -95,22 +95,79 @@ proc startingPoint(x: Node; key: Key; kind: CmpKind): Cursor = let cmpRes = cmp(key, x.keys[j]) if cmpRes =~ kind: return Cursor(n: x, i: j) -proc dos(x: Node; key: Key; kind: CmpKind; withKey: proc(k: Key; v: Val)) = +proc dos(x: Node; kind: CmpKind; key: Key; withKey: proc(k: Key; v: Val)) = if not x.isInternal: - echo "came here" for j in 0 ..< x.m: - if cmp(key, x.keys[j]) =~ kind: - withKey(x.keys[j+1], x.vals[j]) + if cmp(x.keys[j], key) =~ kind: + withKey(x.keys[j], x.vals[j]) else: - for j in 0 ..< x.m-1: - let cmpRes = cmp(key, x.keys[j+1]) - echo "came here B ", canPrune(cmpRes, kind) - if not canPrune(cmpRes, kind): #cmpRes =~ kind: - dos(x.links[j], key, kind, withKey) - #if canPrune(cmpRes, kind): return - # XXX j+1==x.m case? + # we compute the range of links to follow first, before + # recursing: + var followA = 0 + var followB = -1 + case kind + of CmpKind.eq: + # want: key == 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + for j in 1..x.m: + if j == x.m or cmp(key, x.keys[j]) < 0: + followA = j-1 + followB = j-1 + break + of CmpKind.le, CmpKind.lt: + # want: key <= 10 or key < 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + # Case A: all keys are bigger: + if cmp(key, x.keys[1]) < 0: + # --> use the very first branch + followA = 0 + followB = 0 + else: + # Case B: all keys are smaller --> use all branches is covered too + # by this loop. + for j in 1..= 0: + if followB < 0: followA = j-1 + # if the keys are identical and we require 'lt', we know + # only the left branch is required: + followB = j - ord(kind == CmpKind.lt and cmpRes == 0) + else: + # it's already greater, all others are greater too: + break + of CmpKind.ge, CmpKind.gt: + # want: key >= 10 or key > 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + # Case A: all keys are smaller: + if cmp(key, x.keys[x.m-1]) >= 0: + # --> use the very last branch + followA = x.m-1 + followB = x.m-1 + else: + # also covers case B: all keys are bigger --> use all branches + # we find the key that is bigger or equal to ours and from + # then on, follow every branch: + for j in 1..= 5" + dos(b1.root, CmpKind.ge, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo " <= 5" + dos(b1.root, CmpKind.le, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + + echo " == 5" + dos(b1.root, CmpKind.eq, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo " < 5" + dos(b1.root, CmpKind.lt, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo " > 5" + dos(b1.root, CmpKind.gt, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) main() From 5b436bad5a553054d95a4df8136f2d67b3c8cc40 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 19 May 2017 13:33:19 +0200 Subject: [PATCH 11/12] added non-recursive iteration --- src/karaxdb/btree.nim | 101 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index e64396f..66e87f9 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -169,6 +169,83 @@ proc dos(x: Node; kind: CmpKind; key: Key; withKey: proc(k: Key; v: Val)) = for i in followA..followB: dos(x.links[i], kind, key, withKey) +proc don(x: Node; kind: CmpKind; key: Key; withKey: proc(k: Key; v: Val)) = + var stack = @[x] + while stack.len > 0: + let x = stack.pop() + if not x.isInternal: + for j in 0 ..< x.m: + if cmp(x.keys[j], key) =~ kind: + withKey(x.keys[j], x.vals[j]) + else: + # we compute the range of links to follow first, before + # recursing: + var followA = 0 + var followB = -1 + case kind + of CmpKind.eq: + # want: key == 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + for j in 1..x.m: + if j == x.m or cmp(key, x.keys[j]) < 0: + followA = j-1 + followB = j-1 + break + of CmpKind.le, CmpKind.lt: + # want: key <= 10 or key < 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + + # Case A: all keys are bigger: + if cmp(key, x.keys[1]) < 0: + # --> use the very first branch + followA = 0 + followB = 0 + else: + # Case B: all keys are smaller --> use all branches is covered too + # by this loop. + for j in 1..= 0: + if followB < 0: followA = j-1 + # if the keys are identical and we require 'lt', we know + # only the left branch is required: + followB = j - ord(kind == CmpKind.lt and cmpRes == 0) + else: + # it's already greater, all others are greater too: + break + of CmpKind.ge, CmpKind.gt: + # want: key >= 10 or key > 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + + # Case A: all keys are smaller: + if cmp(key, x.keys[x.m-1]) >= 0: + # --> use the very last branch + followA = x.m-1 + followB = x.m-1 + else: + # also covers case B: all keys are bigger --> use all branches + # we find the key that is bigger or equal to ours and from + # then on, follow every branch: + for j in 1.. 5" dos(b1.root, CmpKind.gt, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo "======================================================================" + echo " >= 5" + don(b1.root, CmpKind.ge, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo " <= 5" + don(b1.root, CmpKind.le, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + + echo " == 5" + don(b1.root, CmpKind.eq, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo " < 5" + don(b1.root, CmpKind.lt, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo " > 5" + don(b1.root, CmpKind.gt, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + main() From 80ef044771e8e729cfc1694aa263188916cdb98e Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 19 May 2017 15:48:48 +0200 Subject: [PATCH 12/12] implemented cursor into BTree --- src/karaxdb/btree.nim | 166 +++++++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 49 deletions(-) diff --git a/src/karaxdb/btree.nim b/src/karaxdb/btree.nim index 66e87f9..eb9353f 100644 --- a/src/karaxdb/btree.nim +++ b/src/karaxdb/btree.nim @@ -41,10 +41,12 @@ type n: int ## number of key-value pairs CmpKind {.pure.} = enum eq, le, lt, ge, gt, neq + CursorState = enum stPop, stLeaf, stEnd Cursor = object n: Node - i, dir: int - up: seq[(Node, int)] + i: int + stack: seq[Node] + state: CursorState proc newBTree(): BTree = BTree(root: Node(m: 0, isInternal: false)) @@ -73,28 +75,6 @@ proc `=~`(i: int; k: CmpKind): bool = of CmpKind.gt: i > 0 of CmpKind.neq: i != 0 -proc follow(i: int; k: CmpKind): bool = - case k - of CmpKind.eq: - # we demand equality so if less than, follow it - i < 0 - of CmpKind.le: - i <= 0 - of CmpKind.lt: i <= 0 - of CmpKind.ge: i >= 0 - of CmpKind.gt: i >= 0 - of CmpKind.neq: true - -proc startingPoint(x: Node; key: Key; kind: CmpKind): Cursor = - if x.isInternal: - for j in 0 ..< x.m: - if j+1 == x.m or cmp(key, x.keys[j+1]) =~ kind: - return startingPoint(x.links[j], key, kind) - else: - for j in 0 ..< x.m: - let cmpRes = cmp(key, x.keys[j]) - if cmpRes =~ kind: return Cursor(n: x, i: j) - proc dos(x: Node; kind: CmpKind; key: Key; withKey: proc(k: Key; v: Val)) = if not x.isInternal: for j in 0 ..< x.m: @@ -246,36 +226,114 @@ proc don(x: Node; kind: CmpKind; key: Key; withKey: proc(k: Key; v: Val)) = for i in countdown(followB, followA): stack.add(x.links[i]) -proc init(x: Node): Cursor = - result.up = @[] +proc initCursor(x: Node): Cursor = + result.stack = @[x] result.i = 0 - var it = x - while it.isInternal: - result.up.add((it, 0)) - it = it.links[0] - result.n = it + result.n = nil + result.state = stPop -proc next(c: var Cursor) = - assert c.n != nil - if c.i >= c.n.m: - # current leaf exhausted, pick the next one: - if c.up.len > 0: - (c.n, c.i) = c.up.pop() - - inc c.i - var u = 1 - while c.i > c.n.m or c.n.isInternal: - if c.up.len > 0: - (c.n, c.i) = c.up[c.up.len - u] - else: - c.n = nil +proc next(c: var Cursor; kind: CmpKind; key: Key) = + case c.state + of stEnd: discard + of stLeaf: + let x = c.n + for j in c.i+1 ..< x.m: + if cmp(x.keys[j], key) =~ kind: + c.state = stLeaf + c.i = j + return + c.state = stPop + next(c, kind, key) + of stPop: + if c.stack.len == 0: + c.state = stEnd return - assert(c.n == nil or not c.n.isInternal) + let x = c.stack.pop() + if not x.isInternal: + c.i = -1 + c.n = x + c.state = stLeaf + next(c, kind, key) + else: + # we compute the range of links to follow first, before + # recursing: + var followA = 0 + var followB = -1 + case kind + of CmpKind.eq: + # want: key == 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 + for j in 1..x.m: + if j == x.m or cmp(key, x.keys[j]) < 0: + followA = j-1 + followB = j-1 + break + of CmpKind.le, CmpKind.lt: + # want: key <= 10 or key < 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 -proc atEnd(c: Cursor): bool = c.n == nil + # Case A: all keys are bigger: + if cmp(key, x.keys[1]) < 0: + # --> use the very first branch + followA = 0 + followB = 0 + else: + # Case B: all keys are smaller --> use all branches is covered too + # by this loop. + for j in 1..= 0: + if followB < 0: followA = j-1 + # if the keys are identical and we require 'lt', we know + # only the left branch is required: + followB = j - ord(kind == CmpKind.lt and cmpRes == 0) + else: + # it's already greater, all others are greater too: + break + of CmpKind.ge, CmpKind.gt: + # want: key >= 10 or key > 10 + # keys: 0 3 4 5 10 20 + # keys: 20 30 40 -proc getKey(c: Cursor): Key = discard -proc getVal(c: Cursor): Val = discard + # Case A: all keys are smaller: + if cmp(key, x.keys[x.m-1]) >= 0: + # --> use the very last branch + followA = x.m-1 + followB = x.m-1 + else: + # also covers case B: all keys are bigger --> use all branches + # we find the key that is bigger or equal to ours and from + # then on, follow every branch: + for j in 1.. 5" don(b1.root, CmpKind.gt, "5", proc(k: Key; v: Val) = echo("k ", k, " = ", v)) + echo "======================================================================" + var c = initCursor(b1.root) + var i = 0 + while true: + next(c, CmpKind.le, "9") + if atEnd(c): break + echo "key ", getKey(c), " ", getVal(c) + if i > 30: break + inc i + main()