compiler_ropes: ropeEqualsFile -> equalsFile, no more max rope component length

read file data in bigger blocks, avoiding lots of file api calls

also removed crc step - seems easier to simply compare the bytes rather
than calculating two crc's

compiler/msgs.nim

@@ -494,20 +494,16 @@ proc toCChar*(c: char): string =
   else: result = $(c)
 
 proc makeCString*(s: string): PRope =
-  # BUGFIX: We have to split long strings into many ropes. Otherwise
-  # this could trigger an internalError(). See the ropes module for
-  # further information.
   const
     MaxLineLength = 64
   result = nil
-  var res = "\""
+  var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
+  add(res, "\"")
   for i in countup(0, len(s) - 1):
     if (i + 1) mod MaxLineLength == 0:
       add(res, '\"')
       add(res, tnl)
-      app(result, toRope(res)) # reset:
+      add(res, '\"')
-      setLen(res, 1)
-      res[0] = '\"'
     add(res, toCChar(s[i]))
   add(res, '\"')
   app(result, toRope(res))
@@ -877,8 +873,6 @@ ropes.errorHandler = proc (err: TRopesError, msg: string, useWarning: bool) =
   case err
   of rInvalidFormatStr:
     internalError("ropes: invalid format string: " & msg)
-  of rTokenTooLong:
-    internalError("ropes: token too long: " & msg)
   of rCannotOpenFile:
     rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, msg)
 

compiler/ropes.nim

@@ -56,7 +56,7 @@
 #  To cache them they are inserted in a `cache` array.
 
 import
-  strutils, platform, hashes, crc, options
+  platform, hashes
 
 type
   TFormatStr* = string # later we may change it to CString for better
@@ -75,14 +75,11 @@ type
   TRopesError* = enum
     rCannotOpenFile
     rInvalidFormatStr
-    rTokenTooLong
 
 proc toRope*(s: string): PRope
 proc toRope*(i: BiggestInt): PRope
-proc writeRopeIfNotEqual*(r: PRope, filename: string): bool
 proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope
 proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope])
-proc ropeEqualsFile*(r: PRope, f: string): bool
   # returns true if the rope r is the same as the contents of file f
 proc ropeInvariant*(r: PRope): bool
   # exported for debugging
@@ -310,63 +307,48 @@ proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) =
 const
   bufSize = 1024              # 1 KB is reasonable
 
-proc auxRopeEqualsFile(r: PRope, bin: var File, buf: pointer): bool =
+proc auxEqualsFile(r: PRope, f: File, buf: var array[bufSize, char],
+                   bpos, blen: var int): bool =
   if r.data != nil:
-    if r.length > bufSize:
+    var dpos = 0
-      errorHandler(rTokenTooLong, r.data)
+    let dlen = r.data.len
+    while dpos < dlen:
+      if bpos == blen:
+        # Read more data
+        bpos = 0
+        blen = readBuffer(f, addr(buf[0]), buf.len)
+        if blen == 0:  # no more data in file
+          result = false
           return
-    var readBytes = readBuffer(bin, buf, r.length)
+      let n = min(blen - bpos, dlen - dpos)
-    result = readBytes == r.length and
+      if not equalMem(addr(buf[bpos]), addr(r.data[dpos]), n):
-        equalMem(buf, addr(r.data[0]), r.length) # BUGFIX
+        result = false
+        return
+      dpos += n
+      bpos += n
+    result = true
   else:
-    result = auxRopeEqualsFile(r.left, bin, buf)
+    result = auxEqualsFile(r.left, f, buf, bpos, blen) and
-    if result: result = auxRopeEqualsFile(r.right, bin, buf)
+             auxEqualsFile(r.right, f, buf, bpos, blen)
 
-proc ropeEqualsFile(r: PRope, f: string): bool =
+proc equalsFile*(r: PRope, f: File): bool =
-  var bin: File
+  var
-  result = open(bin, f)
+    buf: array[bufSize, char]
-  if not result:
+    bpos = bufSize
-    return                    # not equal if file does not exist
+    blen = bufSize
-  var buf = alloc(bufSize)
+  result = auxEqualsFile(r, f, buf, bpos, blen) and
-  result = auxRopeEqualsFile(r, bin, buf)
+           readBuffer(f, addr(buf[0]), 1) == 0  # check that we've read all
+
+proc equalsFile*(r: PRope, filename: string): bool =
+  var f: File
+  result = open(f, filename)
   if result:
-    result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file?
+    result = equalsFile(r, f)
-  dealloc(buf)
+    close(f)
-  close(bin)
 
-proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 =
+proc writeRopeIfNotEqual*(r: PRope, filename: string): bool =
-  if r.data != nil:
-    result = startVal
-    for i in countup(0, len(r.data) - 1):
-      result = updateCrc32(r.data[i], result)
-  else:
-    result = crcFromRopeAux(r.left, startVal)
-    result = crcFromRopeAux(r.right, result)
-
-proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 =
-  # XXX profiling shows this is actually expensive
-  var stack: TRopeSeq = @[r]
-  result = startVal
-  while len(stack) > 0:
-    var it = pop(stack)
-    while it.data == nil:
-      add(stack, it.right)
-      it = it.left
-    assert(it.data != nil)
-    var i = 0
-    var L = len(it.data)
-    while i < L:
-      result = updateCrc32(it.data[i], result)
-      inc(i)
-
-proc crcFromRope(r: PRope): TCrc32 =
-  result = newCrcFromRopeAux(r, InitCrc32)
-
-proc writeRopeIfNotEqual(r: PRope, filename: string): bool =
   # returns true if overwritten
-  var c: TCrc32
+  if not equalsFile(r, filename):
-  c = crcFromFile(filename)
-  if c != crcFromRope(r):
     writeRope(r, filename)
     result = true
   else: