system.substr is not implemented with compilerProcs anymore

compiler/ccgexprs.nim

@@ -1902,8 +1902,9 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
   of mIncl, mExcl, mCard, mLtSet, mLeSet, mEqSet, mMulSet, mPlusSet, mMinusSet,
      mInSet:
     genSetOp(p, e, d, op)
-  of mNewString, mNewStringOfCap, mCopyStr, mCopyStrLast, mExit,
+  of mCopyStr, mCopyStrLast:
-      mParseBiggestFloat:
+    genCall(p, e, d)
+  of mNewString, mNewStringOfCap, mExit, mParseBiggestFloat:
     var opr = e.sons[0].sym
     if lfNoDecl notin opr.loc.flags:
       discard cgsym(p.module, $opr.loc.r)

compiler/condsyms.nim

@@ -72,3 +72,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNoZeroTerminator")
   defineSymbol("nimNotNil")
   defineSymbol("nimVmExportFixed")
+  defineSymbol("nimNewRuntime")

compiler/semtypes.nim

@@ -1671,6 +1671,8 @@ proc processMagicType(c: PContext, m: PSym) =
   of mString:
     setMagicType(c.config, m, tyString, c.config.target.ptrSize)
     rawAddSon(m.typ, getSysType(c.graph, m.info, tyChar))
+    if c.config.selectedGc == gcDestructors:
+      incl m.typ.flags, tfHasAsgn
   of mCstring:
     setMagicType(c.config, m, tyCString, c.config.target.ptrSize)
     rawAddSon(m.typ, getSysType(c.graph, m.info, tyChar))
@@ -1710,6 +1712,8 @@ proc processMagicType(c: PContext, m: PSym) =
     setMagicType(c.config, m, tySet, 0)
   of mSeq:
     setMagicType(c.config, m, tySequence, 0)
+    if c.config.selectedGc == gcDestructors:
+      incl m.typ.flags, tfHasAsgn
   of mOpt:
     setMagicType(c.config, m, tyOpt, 0)
   of mOrdinal:

compiler/types.nim

@@ -278,6 +278,8 @@ proc analyseObjectWithTypeField(t: PType): TTypeFieldResult =
 proc isGCRef(t: PType): bool =
   result = t.kind in GcTypeKinds or
     (t.kind == tyProc and t.callConv == ccClosure)
+  if result and t.kind in {tyString, tySequence} and tfHasAsgn in t.flags:
+    result = false
 
 proc containsGarbageCollectedRef*(typ: PType): bool =
   # returns true if typ contains a reference, sequence or string (all the

lib/core/allocators.nim

@@ -8,20 +8,32 @@
 #
 
 type
+  AllocatorFlag* {.pure.} = enum  ## flags describing the properties of the allocator
+    ThreadLocal ## the allocator is thread local only.
+    ZerosMem    ## the allocator always zeros the memory on an allocation
   Allocator* = ptr object {.inheritable.}
     alloc*: proc (a: Allocator; size: int; alignment: int = 8): pointer {.nimcall.}
     dealloc*: proc (a: Allocator; p: pointer; size: int) {.nimcall.}
     realloc*: proc (a: Allocator; p: pointer; oldSize, newSize: int): pointer {.nimcall.}
+    flags*: set[AllocatorFlag]
 
 var
-  currentAllocator {.threadvar.}: Allocator
+  localAllocator {.threadvar.}: Allocator
+  sharedAllocator: Allocator
 
-proc getCurrentAllocator*(): Allocator =
+proc getLocalAllocator*(): Allocator =
-  result = currentAllocator
+  result = localAllocator
 
-proc setCurrentAllocator*(a: Allocator) =
+proc setLocalAllocator*(a: Allocator) =
-  currentAllocator = a
+  localAllocator = a
 
+proc getSharedAllocator*(): Allocator =
+  result = sharedAllocator
+
+proc setSharedAllocator*(a: Allocator) =
+  sharedAllocator = a
+
+when false:
   proc alloc*(size: int; alignment: int = 8): pointer =
     let a = getCurrentAllocator()
     result = a.alloc(a, size, alignment)

lib/core/strs.nim

@@ -7,54 +7,114 @@
 #    distribution, for details about the copyright.
 #
 
-## Default string implementation used by Nim's core.
+## Default new string implementation used by Nim's core.
+
+when false:
+  # these are to be implemented or changed in the code generator.
+
+  #proc rawNewStringNoInit(space: int): NimString {.compilerProc.}
+  # seems to be unused.
+  proc rawNewString(space: int): NimString {.compilerProc.}
+  proc mnewString(len: int): NimString {.compilerProc.}
+  proc copyStrLast(s: NimString, start, last: int): NimString {.compilerProc.}
+  proc nimToCStringConv(s: NimString): cstring {.compilerProc, inline.}
+  proc copyStr(s: NimString, start: int): NimString {.compilerProc.}
+  proc toNimStr(str: cstring, len: int): NimString {.compilerProc.}
+  proc cstrToNimstr(str: cstring): NimString {.compilerRtl.}
+  proc copyString(src: NimString): NimString {.compilerRtl.}
+  proc copyStringRC1(src: NimString): NimString {.compilerRtl.}
+  proc copyDeepString(src: NimString): NimString {.inline.}
+  proc addChar(s: NimString, c: char): NimString
+  proc resizeString(dest: NimString, addlen: int): NimString {.compilerRtl.}
+  proc appendString(dest, src: NimString) {.compilerproc, inline.}
+  proc appendChar(dest: NimString, c: char) {.compilerproc, inline.}
+  proc setLengthStr(s: NimString, newLen: int): NimString {.compilerRtl.}
+  # ----------------- sequences ----------------------------------------------
+
+  proc incrSeqV3(s: PGenericSeq, typ: PNimType): PGenericSeq {.compilerProc.} =
+  proc setLengthSeqV2(s: PGenericSeq, typ: PNimType, newLen: int): PGenericSeq {.
+      compilerRtl.}
+  proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} =
+  proc newSeqRC1(typ: PNimType, len: int): pointer {.compilerRtl.} =
 
 import allocators
 
 type
-  string {.core, exportc: "NimStringV2".} = object
+  StrContent = object
-    len, cap: int
+    cap: int
-    data: ptr UncheckedArray[char]
+    region: Allocator
+    data: UncheckedArray[char]
+
+  NimString {.core.} = object
+    len: int
+    p: ptr StrContent ## invariant. Never nil
 
 const nimStrVersion {.core.} = 2
 
-template frees(s) = dealloc(s.data, s.cap + 1)
+template isLiteral(s): bool = s.len == 0 or s.p.region == nil
 
-proc `=destroy`(s: var string) =
+template contentSize(cap): int = cap + 1 + sizeof(int) + sizeof(Allocator)
-  if s.data != nil:
+
+template frees(s) =
+  if not isLiteral(s):
+    s.p.region.dealloc(s.p, contentSize(s.p.cap))
+
+proc `=destroy`(s: var NimString) =
   frees(s)
-    s.data = nil
   s.len = 0
-    s.cap = 0
 
-proc `=sink`(a: var string, b: string) =
+template lose(a) =
+  frees(a)
+
+proc `=sink`(a: var NimString, b: NimString) =
   # we hope this is optimized away for not yet alive objects:
-  if a.data != nil and a.data != b.data:
+  if unlikely(a.p == b.p): return
-    frees(a)
+  lose(a)
   a.len = b.len
-  a.cap = b.cap
+  a.p = b.p
-  a.data = b.data
 
-proc `=`(a: var string; b: string) =
+proc `=`(a: var NimString; b: NimString) =
-  if a.data != nil and a.data != b.data:
+  if unlikely(a.p == b.p): return
-    frees(a)
+  lose(a)
-    a.data = nil
   a.len = b.len
-  a.cap = b.cap
+  if isLiteral(b):
-  if b.data != nil:
+    # we can shallow copy literals:
-    a.data = cast[type(a.data)](alloc(a.cap + 1))
+    a.p = b.p
-    copyMem(a.data, b.data, a.cap+1)
+  else:
+    let region = if a.p.region != nil: a.p.region else: getLocalAllocator()
+    # we have to allocate the 'cap' here, consider
+    # 'let y = newStringOfCap(); var x = y'
+    # on the other hand... These get turned into moves now.
+    a.p = cast[ptr StrContent](region.alloc(contentSize(b.len)))
+    a.p.region = region
+    a.p.cap = b.len
+    copyMem(unsafeAddr a.p.data[0], unsafeAddr b.p.data[0], b.len+1)
 
-proc resize(s: var string) =
+proc resize(old: int): int {.inline.} =
-  let old = s.cap
+  if old <= 0: result = 4
-  if old == 0: s.cap = 8
+  elif old < 65536: result = old * 2
-  else: s.cap = (s.cap * 3) shr 1
+  else: result = old * 3 div 2 # for large arrays * 3/2 is better
-  s.data = cast[type(s.data)](realloc(s.data, old + 1, s.cap + 1))
 
-proc add*(s: var string; c: char) =
+proc prepareAdd(s: var NimString; addlen: int) =
-  if s.len >= s.cap: resize(s)
+  if isLiteral(s):
-  s.data[s.len] = c
+    let oldP = s.p
-  s.data[s.len+1] = '\0'
+    # can't mutate a literal, so we need a fresh copy here:
+    let region = getLocalAllocator()
+    s.p = cast[ptr StrContent](region.alloc(contentSize(s.len + addlen)))
+    s.p.region = region
+    s.p.cap = s.len + addlen
+    if s.len > 0:
+      # we are about to append, so there is no need to copy the \0 terminator:
+      copyMem(unsafeAddr s.p.data[0], unsafeAddr oldP.data[0], s.len)
+  elif s.len + addlen > s.p.cap:
+    let cap = max(s.len + addlen, resize(s.p.cap))
+    s.p = s.p.region.realloc(s.p, oldSize = contentSize(s.p.cap), newSize = contentSize(cap))
+    s.p.cap = cap
+
+proc nimAddCharV1(s: var NimString; c: char) {.compilerRtl.} =
+  prepareAdd(s, 1)
+  s.p.data[s.len] = c
+  s.p.data[s.len+1] = '\0'
   inc s.len
 
 proc ensure(s: var string; newLen: int) =
@@ -71,8 +131,6 @@ proc add*(s: var string; y: string) =
     copyMem(addr s.data[len], y.data, y.data.len + 1)
     s.len = newLen
 
-proc len*(s: string): int {.inline.} = s.len
-
 proc newString*(len: int): string =
   result.len = len
   result.cap = len

lib/system.nim

@@ -413,7 +413,7 @@ include "system/inclrtl"
 const NoFakeVars* = defined(nimscript) ## true if the backend doesn't support \
   ## "fake variables" like 'var EBADF {.importc.}: cint'.
 
-when not defined(JS):
+when not defined(JS) and not defined(gcDestructors):
   type
     TGenericSeq {.compilerproc, pure, inheritable.} = object
       len, reserved: int
@@ -1702,17 +1702,6 @@ proc addQuitProc*(QuitProc: proc() {.noconv.}) {.
 # In case of an unhandled exeption the exit handlers should
 # not be called explicitly! The user may decide to do this manually though.
 
-proc substr*(s: string, first = 0): string {.
-  magic: "CopyStr", importc: "copyStr", noSideEffect.}
-proc substr*(s: string, first, last: int): string {.
-  magic: "CopyStrLast", importc: "copyStrLast", noSideEffect.}
-  ## copies a slice of `s` into a new string and returns this new
-  ## string. The bounds `first` and `last` denote the indices of
-  ## the first and last characters that shall be copied. If ``last``
-  ## is omitted, it is treated as ``high(s)``. If ``last >= s.len``, ``s.len``
-  ## is used instead: This means ``substr`` can also be used to `cut`:idx:
-  ## or `limit`:idx: a string's length.
-
 when not defined(nimscript) and not defined(JS):
   proc zeroMem*(p: pointer, size: Natural) {.inline, benign.}
     ## overwrites the contents of the memory at ``p`` with the value 0.
@@ -3262,7 +3251,11 @@ when not defined(JS): #and not defined(nimscript):
     when hasAlloc: include "system/mmdisp"
     {.pop.}
     {.push stack_trace: off, profiler:off.}
-    when hasAlloc: include "system/sysstr"
+    when hasAlloc:
+      when defined(gcDestructors):
+        include "core/strs"
+      else:
+        include "system/sysstr"
     {.pop.}
     when hasAlloc: include "system/strmantle"
 
@@ -4171,6 +4164,21 @@ when not defined(js):
   proc toOpenArrayByte*(x: string; first, last: int): openarray[byte] {.
     magic: "Slice".}
 
+proc substr*(s: string, first, last: int): string =
+  let L = max(min(last, high(s)) - first + 1, 0)
+  result = newString(L)
+  for i in 0 .. L-1:
+    result[i] = s[i+first]
+
+proc substr*(s: string, first = 0): string =
+  ## copies a slice of `s` into a new string and returns this new
+  ## string. The bounds `first` and `last` denote the indices of
+  ## the first and last characters that shall be copied. If ``last``
+  ## is omitted, it is treated as ``high(s)``. If ``last >= s.len``, ``s.len``
+  ## is used instead: This means ``substr`` can also be used to `cut`:idx:
+  ## or `limit`:idx: a string's length.
+  result = substr(s, first, high(s))
+
 type
   ForLoopStmt* {.compilerProc.} = object ## special type that marks a macro
                                          ## as a `for-loop macro`:idx:

lib/system/sysstr.nim

@@ -63,6 +63,8 @@ proc mnewString(len: int): NimString {.compilerProc.} =
   result.len = len
 
 proc copyStrLast(s: NimString, start, last: int): NimString {.compilerProc.} =
+  # This is not used by most recent versions of the compiler anymore, but
+  # required for bootstrapping purposes.
   let start = max(start, 0)
   let len = min(last, s.len-1) - start + 1
   if len > 0:
@@ -73,13 +75,15 @@ proc copyStrLast(s: NimString, start, last: int): NimString {.compilerProc.} =
   else:
     result = rawNewString(len)
 
+proc copyStr(s: NimString, start: int): NimString {.compilerProc.} =
+  # This is not used by most recent versions of the compiler anymore, but
+  # required for bootstrapping purposes.
+  result = copyStrLast(s, start, s.len-1)
+
 proc nimToCStringConv(s: NimString): cstring {.compilerProc, inline.} =
   if s == nil or s.len == 0: result = cstring""
   else: result = cstring(addr s.data)
 
-proc copyStr(s: NimString, start: int): NimString {.compilerProc.} =
-  result = copyStrLast(s, start, s.len-1)
-
 proc toNimStr(str: cstring, len: int): NimString {.compilerProc.} =
   result = rawNewStringNoInit(len)
   result.len = len