better implementation for wrapWords

lib/pure/strutils.nim

@@ -1211,7 +1211,8 @@ proc wordWrap*(s: string, maxLineWidth = 80,
                splitLongWords = true,
                seps: set[char] = Whitespace,
                newLine = "\n"): string {.
-               noSideEffect, rtl, extern: "nsuWordWrap".} =
+               noSideEffect, rtl, extern: "nsuWordWrap",
+               deprecated: "use wrapWords in std/wordwrap instead".} =
   ## Word wraps `s`.
   result = newStringOfCap(s.len + s.len shr 6)
   var spaceLeft = maxLineWidth

lib/std/wordwrap.nim

@@ -1,67 +1,88 @@
-import unicode
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2018 Nim contributors
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
 
-proc wordWrap*(s: string, maxLineWidth = 80,
+import strutils, unicode
+
+proc olen(s: string): int =
+  var i = 0
+  result = 0
+  while i < s.len:
+    inc result
+    let L = graphemeLen(s, i)
+    inc i, L
+
+proc wrapWords*(s: string, maxLineWidth = 80,
                splitLongWords = true,
-               newLine = "\n"): string  =
+               seps: set[char] = Whitespace,
-  ## This function breaks all words that reach over `maxLineWidth`
+               newLine = "\n"): string {.noSideEffect.} =
-  ## measured in number of runes. When `splitLongWords` is `true`
+  ## Word wraps `s`.
-  ## words that are longer than `maxLineWidth` are splitted. Multiple
+  result = newStringOfCap(s.len + s.len shr 6)
-  ## spaces and newlines are converted to a single space.  All
+  var spaceLeft = maxLineWidth
-  ## whitespace is treated equally.  Non-breaking whitespace is
+  var lastSep = ""
-  ## ignored.
+  for word, isSep in tokenize(s, seps):
-
+    let wlen = olen(word)
-  var currentWordLength: int = 0
+    if isSep:
-  var currentWord: string = newStringOfCap(32)
+      lastSep = word
-  var currentLineLength: int = 0
+      spaceLeft = spaceLeft - wlen
-  var currentWordLengthAtLineEnd: int = -1
+    elif wlen > spaceLeft:
-  var longWordMode = false
+      if splitLongWords and wlen > maxLineWidth:
-
+        var i = 0
-  template handleWhitespace(): untyped =
+        while i < word.len:
-    if currentWord.len > 0:
+          if spaceLeft <= 0:
-
+            spaceLeft = maxLineWidth
-      if currentLineLength + 1 + currentWordLength > maxLineWidth:
+            result.add newLine
-        result.add newLine
+          dec spaceLeft
-        currentLineLength = 0
+          let L = graphemeLen(word, i)
-
+          for j in 0 ..< L: result.add word[i+j]
-      if currentLineLength > 0:
+          inc i, L
-        result.add ' '
+      else:
-        currentLineLength += 1
+        spaceLeft = maxLineWidth - wlen
-
+        result.add(newLine)
-      result.add currentWord
+        result.add(word)
-      currentLineLength += currentWordLength
-
-      currentWord.setlen 0
-      currentWordLength = 0
-
-  for rune in s.runes:
-    if rune.isWhiteSpace:
-      handleWhitespace()
     else:
-      if splitLongWords and currentWordLength >= maxLineWidth:
+      spaceLeft = spaceLeft - wlen
-        handleWhitespace()
+      result.add(lastSep)
-
+      result.add(word)
-      currentWord.add rune
+      lastSep.setLen(0)
-      inc currentWordLength
-
-  handleWhitespace()
-
 
 when isMainModule:
-  import strutils
 
+  when true:
+    let
+      inp = """ this is a long text --  muchlongerthan10chars and here
+                 it goes"""
+      outp = " this is a\nlong text\n--\nmuchlongerthan10chars\nand here\nit goes"
+    doAssert wrapWords(inp, 10, false) == outp
 
-  proc checkLineLength(arg: string): void =
+    let
-    for line in splitlines(arg):
+      longInp = """ThisIsOneVeryLongStringWhichWeWillSplitIntoEightSeparatePartsNow"""
-      var numRunes = 0
+      longOutp = "ThisIsOn\neVeryLon\ngStringW\nhichWeWi\nllSplitI\nntoEight\nSeparate\nPartsNow"
-      for rune in runes(line):
+    doAssert wrapWords(longInp, 8, true) == longOutp
-        numRunes += 1
 
-      assert numRunes <= 80
+  # test we don't break Umlauts into invalid bytes:
+  let fies = "äöüöäöüöäöüöäöüööäöüöäößßßßüöäößßßßßß"
+  let fiesRes = "ä\nö\nü\nö\nä\nö\nü\nö\nä\nö\nü\nö\nä\nö\nü\nö\nö\nä\nö\nü\nö\nä\nö\nß\nß\nß\nß\nü\nö\nä\nö\nß\nß\nß\nß\nß\nß"
+  doAssert wrapWords(fies, 1, true) == fiesRes
 
-  let longlongword = "abc uitdaeröägfßhydüäpydqfü,träpydqgpmüdträpydföägpydörztdüöäfguiaeowäzjdtrüöäp psnrtuiydrözenrüöäpyfdqazpesnrtulocjtüöäzydgyqgfqfgprtnwjlcydkqgfüöezmäzydydqüüöäpdtrnvwfhgckdumböäpydfgtdgfhtdrntdrntydfogiayqfguiatrnydrntüöärtniaoeydfgaoeiqfglwcßqfgxvlcwgtfhiaoenrsüöäapmböäptdrniaoydfglckqfhouenrtsüöäptrniaoeyqfgulocfqclgwxßqflgcwßqfxglcwrniatrnmüböäpmöäbpümöäbpüöämpbaoestnriaesnrtdiaesrtdniaesdrtnaetdriaoenvlcyfglwckßqfgvwkßqgfvlwkßqfgvlwckßqvlwkgfUIαοιαοιαχολωχσωχνωκψρχκψρτιεαοσηζϵηζιοεννκεωνιαλωσωκνκψρκγτφγτχκγτεκργτιχνκιωχσιλωσλωχξλξλξωχωχξχλωωχαοεοιαεοαεοιαεοαεοιαοεσναοεκνρκψγκψφϵιηαααοε"
+  let longlongword = """abc uitdaeröägfßhydüäpydqfü,träpydqgpmüdträpydföägpydörztdüöäfguiaeowäzjdtrüöäp psnrtuiydrözenrüöäpyfdqazpesnrtulocjtüö
+äzydgyqgfqfgprtnwjlcydkqgfüöezmäzydydqüüöäpdtrnvwfhgckdumböäpydfgtdgfhtdrntdrntydfogiayqfguiatrnydrntüöärtniaoeydfgaoeiqfglwcßqfgxvlcwgtfhiaoen
+rsüöäapmböäptdrniaoydfglckqfhouenrtsüöäptrniaoeyqfgulocfqclgwxßqflgcwßqfxglcwrniatrnmüböäpmöäbpümöäbpüöämpbaoestnriaesnrtdiaesrtdniaesdrtnaetdr
+iaoenvlcyfglwckßqfgvwkßqgfvlwkßqfgvlwckßqvlwkgfUIαοιαοιαχολωχσωχνωκψρχκψρτιεαοσηζϵηζιοεννκεωνιαλωσωκνκψρκγτφγτχκγτεκργτιχνκιωχσιλωσλωχξλξλξωχωχ
+ξχλωωχαοεοιαεοαεοιαεοαεοιαοεσναοεκνρκψγκψφϵιηαααοε"""
+  let longlongwordRes = """
+abc uitdaeröägfßhydüäpydqfü,träpydqgpmüdträpydföägpydörztdüöäfguiaeowäzjdtrüöäp
+psnrtuiydrözenrüöäpyfdqazpesnrtulocjtüöäzydgyqgfqfgprtnwjlcydkqgfüöezmäzydydqüü
+öäpdtrnvwfhgckdumböäpydfgtdgfhtdrntdrntydfogiayqfguiatrnydrntüöärtniaoeydfgaoeiq
+fglwcßqfgxvlcwgtfhiaoenrsüöäapmböäptdrniaoydfglckqfhouenrtsüöäptrniaoeyqfgulocf
+qclgwxßqflgcwßqfxglcwrniatrnmüböäpmöäbpümöäbpüöämpbaoestnriaesnrtdiaesrtdniaesdr
+tnaetdriaoenvlcyfglwckßqfgvwkßqgfvlwkßqfgvlwckßqvlwkgfUIαοιαοιαχολωχσωχνωκψρχκψ
+ρτιεαοσηζϵηζιοεννκεωνιαλωσωκνκψρκγτφγτχκγτεκργτιχνκιωχσιλωσλωχξλξλξωχωχ
+ξχλωωχαοεοιαεοαεοιαεοαεοιαοεσναοεκνρκψγκψφϵιηαααοε"""
+  doAssert wrapWords(longlongword) == longlongwordRes
 
-  checkLineLength(longlongword.wordWrap)
-
-  let tmp ="Наши исследования позволяют сделать вывод о том, что субъект выбирает xxxuiaetudtiraeüöätpghiacodöeronfdquiahgoüöädoiaqofhgiaeotrnuiaßqzfgiaoeurnudtitraenuitenruitarenitarenuitarentduiranetduiranetdruianetrnuiaertnuiatdenruiatdrne институциональный психоз. Важность этой функции подчеркивается тем фактом, что объект вызывает эгоцентризм. Самоактуализация аннигилирует генезис. Анима аннигилирует возрастной код. Закон просветляет аутотренинг. Наши исследования позволяют сделать вывод о том, что воспитание заметно осознаёт инсайт."
-
-  checkLineLength(tmp.wordWrap)