first version of ropes.nim with unsafeNew (broken)

compiler/ccgexprs.nim

@@ -933,22 +933,31 @@ proc genNew(p: BProc, e: PNode) =
   var
     a, b: TLoc
     reftype, bt: PType
+    sizeExpr: PRope
   refType = skipTypes(e.sons[1].typ, abstractVarRange)
   InitLocExpr(p, e.sons[1], a)
   initLoc(b, locExpr, a.t, OnHeap)
+  # 'genNew' also handles 'unsafeNew':
+  if e.len == 3:
+    var se: TLoc
+    InitLocExpr(p, e.sons[2], se)
+    sizeExpr = se.rdLoc
+  else:
+    sizeExpr = ropef("sizeof($1)",
+        getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange)))
   let args = [getTypeDesc(p.module, reftype),
               genTypeInfo(p.module, refType),
-              getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange))]
+              sizeExpr]
   if a.s == OnHeap and optRefcGc in gGlobalOptions:
     # use newObjRC1 as an optimization; and we don't need 'keepAlive' either
     if canFormAcycle(a.t):
       lineCg(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc)
     else:
       lineCg(p, cpsStmts, "if ($1) #nimGCunrefNoCycle($1);$n", a.rdLoc)
-    b.r = ropecg(p.module, "($1) #newObjRC1($2, sizeof($3))", args)
+    b.r = ropecg(p.module, "($1) #newObjRC1($2, $3)", args)
     lineCg(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc)
   else:
-    b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", args)
+    b.r = ropecg(p.module, "($1) #newObj($2, $3)", args)
     genAssignment(p, a, b, {needToKeepAlive})  # set the object type:
   bt = skipTypes(refType.sons[0], abstractRange)
   genObjectInit(p, cpsStmts, bt, a, false)

compiler/ccgmerge.nim

@@ -145,6 +145,7 @@ proc atEndMark(buf: cstring, pos: int): bool =
   while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s
   result = s == NimMergeEndMark.len
 
+when false:
   proc readVerbatimSection(L: var TBaseLexer): PRope = 
     var pos = L.bufpos
     var buf = L.buf
@@ -171,6 +172,32 @@ proc readVerbatimSection(L: var TBaseLexer): PRope =
     L.bufpos = pos
     freezeMutableRope(result)
 
+proc readVerbatimSection(L: var TBaseLexer): PRope = 
+  var pos = L.bufpos
+  var buf = L.buf
+  var r = newStringOfCap(30_000)
+  while true:
+    case buf[pos]
+    of CR:
+      pos = lexbase.HandleCR(L, pos)
+      buf = L.buf
+      r.add(tnl)
+    of LF:
+      pos = lexbase.HandleLF(L, pos)
+      buf = L.buf
+      r.add(tnl)
+    of '\0':
+      InternalError("ccgmerge: expected: " & NimMergeEndMark)
+      break
+    else: 
+      if atEndMark(buf, pos):
+        inc pos, NimMergeEndMark.len
+        break
+      r.add(buf[pos])
+      inc pos
+  L.bufpos = pos
+  result = r.toRope
+
 proc readKey(L: var TBaseLexer, result: var string) =
   var pos = L.bufpos
   var buf = L.buf

compiler/docgen.nim

@@ -350,9 +350,11 @@ proc CommandRstAux(filename, outExt: string) =
   var d = newDocumentor(filen, options.gConfigVars)
   var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc,
                      {roSupportRawDirective})
-  d.modDesc = newMutableRope(30_000)
+  var modDesc = newStringOfCap(30_000)
-  renderRstToOut(d[], rst, d.modDesc.data)
+  #d.modDesc = newMutableRope(30_000)
-  freezeMutableRope(d.modDesc)
+  renderRstToOut(d[], rst, modDesc)
+  #freezeMutableRope(d.modDesc)
+  d.modDesc = toRope(modDesc)
   writeOutput(d, filename, outExt)
   generateIndex(d)
 

compiler/ropes.nim

@@ -58,6 +58,9 @@
 import 
   msgs, strutils, platform, hashes, crc, options
 
+const
+  PayloadSize = 64_000 # dummy size for range checking
+
 type
   TFormatStr* = string # later we may change it to CString for better
                        # performance of the code generator (assignments 
@@ -66,9 +69,9 @@ type
   PRope* = ref TRope
   TRope*{.acyclic.} = object of TObject # the empty rope is represented 
                                         # by nil to safe space
-    left*, right*: PRope
+    left, right: PRope
-    length*: int
+    L: int                            # < 0 if a leaf
-    data*: string             # != nil if a leaf
+    d: array [0..PayloadSize, char]             # != nil if a leaf
   
   TRopeSeq* = seq[PRope]
 
@@ -94,14 +97,23 @@ proc RopeInvariant*(r: PRope): bool
 
 proc ropeLen(a: PRope): int = 
   if a == nil: result = 0
-  else: result = a.length
+  else: result = a.L.abs
   
 proc newRope(data: string = nil): PRope =
-  new(result)
   if data != nil:
-    result.length = len(data)
+    unsafeNew(result, sizeof(TRope)-PayloadSize+len(data))
-    result.data = data
+    result.L = -len(data)
+    # copy including '\0':
+    copyMem(addr result.d, cstring(data), len(data))
+  else:
+    unsafeNew(result, sizeof(TRope)-PayloadSize)
 
+proc eqContent(r: PRope, s: string): bool =
+  assert r.L < 0
+  if -r.L == s.len:
+    result = equalMem(addr(r.d), cstring(s), s.len)
+
+when false:
   proc newMutableRope*(capacity = 30): PRope =
     ## creates a new rope that supports direct modifications of the rope's
     ## 'data' and 'length' fields.
@@ -130,7 +142,7 @@ proc RopeInvariant(r: PRope): bool =
 proc insertInCache(s: string): PRope = 
   var h = hash(s) and high(cache)
   result = cache[h]
-  if isNil(result) or result.data != s:
+  if isNil(result) or not eqContent(result, s):
     result = newRope(s)
     cache[h] = result
   
@@ -155,19 +167,19 @@ proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) =
   var stack = @[r]
   while len(stack) > 0: 
     var it = pop(stack)
-    while it.data == nil: 
+    while it.L >= 0:
       add(stack, it.right)
       it = it.left
-    assert(it.data != nil)
+    assert(it.L < 0)
-    CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length)
+    CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L)
-    Inc(resultLen, it.length)
+    Inc(resultLen, -it.L)
     assert(resultLen <= len(result))
 
 proc ropeToStr(p: PRope): string = 
   if p == nil: 
     result = ""
   else:
-    result = newString(p.length)
+    result = newString(p.L.abs)
     var resultLen = 0
     newRecRopeToStr(result, resultLen, p)
 
@@ -176,7 +188,7 @@ proc con(a, b: PRope): PRope =
   elif b == nil: result = a
   else:
     result = newRope()
-    result.length = a.length + b.length
+    result.L = a.L.abs + b.L.abs
     result.left = a
     result.right = b
 
@@ -196,12 +208,12 @@ proc writeRope*(f: TFile, c: PRope) =
   var stack = @[c]
   while len(stack) > 0:
     var it = pop(stack)
-    while it.data == nil: 
+    while it.L >= 0:
       add(stack, it.right)
       it = it.left
       assert(it != nil)
-    assert(it.data != nil)
+    assert(it.L < 0)
-    write(f, it.data)
+    write(f, cstring(it.d), -it.L)
 
 proc WriteRope*(head: PRope, filename: string, useWarning = false) =
   var f: tfile
@@ -261,13 +273,13 @@ const
   bufSize = 1024              # 1 KB is reasonable
 
 proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = 
-  if r.data != nil:
+  if r.L < 0:
-    if r.length > bufSize: 
+    if -r.L > bufSize: 
       internalError("ropes: token too long")
       return
-    var readBytes = readBuffer(bin, buf, r.length)
+    var readBytes = readBuffer(bin, buf, -r.L)
-    result = readBytes == r.length and
+    result = readBytes == -r.L and
-        equalMem(buf, addr(r.data[0]), r.length) # BUGFIX
+        equalMem(buf, addr(r.d[0]), readBytes)
   else:
     result = auxRopeEqualsFile(r.left, bin, buf)
     if result: result = auxRopeEqualsFile(r.right, bin, buf)
@@ -284,29 +296,20 @@ proc RopeEqualsFile(r: PRope, f: string): bool =
   dealloc(buf)
   close(bin)
 
-proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = 
-  if r.data != nil: 
-    result = startVal
-    for i in countup(0, len(r.data) - 1): 
-      result = updateCrc32(r.data[i], result)
-  else: 
-    result = crcFromRopeAux(r.left, startVal)
-    result = crcFromRopeAux(r.right, result)
-
 proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = 
   # XXX profiling shows this is actually expensive
   var stack: TRopeSeq = @[r]
   result = startVal
   while len(stack) > 0: 
     var it = pop(stack)
-    while it.data == nil: 
+    while it.L >= 0:
       add(stack, it.right)
       it = it.left
-    assert(it.data != nil)
+    assert(it.L < 0)
     var i = 0
-    var L = len(it.data)
+    var L = -it.L
     while i < L:
-      result = updateCrc32(it.data[i], result)
+      result = updateCrc32(it.d[i], result)
       inc(i)
 
 proc crcFromRope(r: PRope): TCrc32 = 

lib/system.nim

@@ -112,11 +112,17 @@ proc new*[T](a: var ref T) {.magic: "New", noSideEffect.}
   ## creates a new object of type ``T`` and returns a safe (traced)
   ## reference to it in ``a``.
 
-proc new(T: typedesc): ref T =
+proc new*(T: typedesc): ref T =
   ## creates a new object of type ``T`` and returns a safe (traced)
   ## reference to it as result value
   new(result)
 
+proc unsafeNew*[T](a: var ref T, size: int) {.magic: "New", noSideEffect.}
+  ## creates a new object of type ``T`` and returns a safe (traced)
+  ## reference to it in ``a``. This is **unsafe** as it allocates an object
+  ## of the passed ``size``. This should only be used for optimization
+  ## purposes when you know what you're doing!
+
 proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.}
   ## leaked implementation detail. Do not use.
 

todo.txt

@@ -50,7 +50,7 @@ Concurrency
   provide a ``syncgc`` pragma to trigger compiler injection --> more general:
   an ``injectLoop`` pragma
 - 'writes: []' effect; track reads/writes for shared types
-- use the effect system for static deadlock prevention
+- use the effect system for static deadlock prevention and race detection
 
 
 version 0.9.XX