jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[backport] improve unicode docs, fixes #2353 (#10174)

Browse source 
* as instructed in #2353, provides a short description why
there are no specialized procs for seq[Rune]
* adds several examples to better explain what some functions do
* small fixes (double backticks, add missing dots, etc.)
* use `rune` instead of "unicode characer"
This commit is contained in:
Miran 2019-01-04 13:20:12 +01:00 • committed by GitHub
commit 69149a0e92
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 122 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

207
lib/pure/unicode.nim
View file

@ -8,6 +8,11 @@
# #
## This module provides support to handle the Unicode UTF-8 encoding. ## This module provides support to handle the Unicode UTF-8 encoding.
##
## There are no specialized ``insert``, ``delete``, ``add`` and ``contains``
## procedures for ``seq[Rune]`` in this module because the generic variants
## of these procedures in the system module already work with it.
{.deadCodeElim: on.} # dce option deprecated {.deadCodeElim: on.} # dce option deprecated
@ -15,7 +20,7 @@ include "system/inclrtl"
type type
RuneImpl = int32 # underlying type of Rune RuneImpl = int32 # underlying type of Rune
Rune* = distinct RuneImpl ## type that can hold any Unicode character Rune* = distinct RuneImpl ## Unicode code point. Can hold any Unicode character.
Rune16* = distinct int16 ## 16 bit Unicode character Rune16* = distinct int16 ## 16 bit Unicode character
proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b) proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b)
@ -25,7 +30,12 @@ proc `==`*(a, b: Rune): bool = return int(a) == int(b)
template ones(n: untyped): untyped = ((1 shl n)-1) template ones(n: untyped): untyped = ((1 shl n)-1)
proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
## Returns the number of Unicode characters of the string ``s`` ## Returns the number of runes of the string ``s``.
runnableExamples:
let a = "añyóng"
doAssert a.runeLen == 6
## note: a.len == 8
var i = 0 var i = 0
while i < len(s): while i < len(s):
if ord(s[i]) <=% 127: inc(i) if ord(s[i]) <=% 127: inc(i)
@ -38,7 +48,12 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
inc(result) inc(result)
proc runeLenAt*(s: string, i: Natural): int = proc runeLenAt*(s: string, i: Natural): int =
## Returns the number of bytes the rune starting at ``s[i]`` takes ## Returns the number of bytes the rune starting at ``s[i]`` takes.
runnableExamples:
let a = "añyóng"
doAssert a.runeLenAt(0) == 1
doAssert a.runeLenAt(1) == 2
if ord(s[i]) <=% 127: result = 1 if ord(s[i]) <=% 127: result = 1
elif ord(s[i]) shr 5 == 0b110: result = 2 elif ord(s[i]) shr 5 == 0b110: result = 2
elif ord(s[i]) shr 4 == 0b1110: result = 3 elif ord(s[i]) shr 4 == 0b1110: result = 3
@ -50,7 +65,7 @@ proc runeLenAt*(s: string, i: Natural): int =
const replRune = Rune(0xFFFD) const replRune = Rune(0xFFFD)
template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) =
## Returns the Unicode character ``s[i]`` in ``result``. If ``doInc == true`` ## Returns the rune ``s[i]`` in ``result``. If ``doInc == true``
## ``i`` is incremented by the number of bytes that have been processed. ## ``i`` is incremented by the number of bytes that have been processed.
bind ones bind ones
if ord(s[i]) <=% 127: if ord(s[i]) <=% 127:
@ -152,17 +167,21 @@ proc validateUtf8*(s: string): int =
return -1 return -1
proc runeAt*(s: string, i: Natural): Rune = proc runeAt*(s: string, i: Natural): Rune =
## Returns the unicode character in ``s`` at byte index ``i`` ## Returns the rune in ``s`` at **byte index** ``i``.
runnableExamples:
let a = "añyóng"
doAssert a.runeAt(1) == "ñ".runeAt(0)
doAssert a.runeAt(2) == "ñ".runeAt(1)
doAssert a.runeAt(3) == "y".runeAt(0)
fastRuneAt(s, i, result, false) fastRuneAt(s, i, result, false)
template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) = template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) =
## Copies UTF-8 representation of `c` into the preallocated string `s` ## Copies UTF-8 representation of ``c`` into the preallocated string ``s``
## starting at position `pos`. If `doInc == true`, `pos` is incremented ## starting at position ``pos``. If ``doInc == true``, ``pos`` is incremented
## by the number of bytes that have been processed. ## by the number of bytes that have been processed.
## ##
## To be the most efficient, make sure `s` is preallocated ## To be the most efficient, make sure ``s`` is preallocated
## with an additional amount equal to the byte length of ## with an additional amount equal to the byte length of ``c``.
## `c`.
var i = RuneImpl(c) var i = RuneImpl(c)
if i <=% 127: if i <=% 127:
s.setLen(pos+1) s.setLen(pos+1)
@ -207,28 +226,39 @@ template fastToUTF8Copy*(c: Rune, s: var string, pos: int, doInc = true) =
discard # error, exception? discard # error, exception?
proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} = proc toUTF8*(c: Rune): string {.rtl, extern: "nuc$1".} =
## Converts a rune into its UTF-8 representation ## Converts a rune into its UTF-8 representation.
runnableExamples:
let a = "añyóng"
doAssert a.runeAt(1).toUTF8 == "ñ"
result = "" result = ""
fastToUTF8Copy(c, result, 0, false) fastToUTF8Copy(c, result, 0, false)
proc add*(s: var string; c: Rune) = proc add*(s: var string; c: Rune) =
## Adds a rune ``c`` to a string ``s``.
runnableExamples:
var s = "abc"
let c = "ä".runeAt(0)
s.add(c)
doAssert s == "abcä"
let pos = s.len let pos = s.len
fastToUTF8Copy(c, s, pos, false) fastToUTF8Copy(c, s, pos, false)
proc `$`*(rune: Rune): string = proc `$`*(rune: Rune): string =
## Converts a Rune to a string ## An alias for `toUTF8 <#toUTF8%2CRune>`_.
rune.toUTF8 rune.toUTF8
proc `$`*(runes: seq[Rune]): string = proc `$`*(runes: seq[Rune]): string =
## Converts a sequence of Runes to a string ## Converts a sequence of Runes to a string.
result = "" result = ""
for rune in runes: for rune in runes:
result.add rune result.add rune
proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int = proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int =
## Returns the byte position of unicode character ## Returns the byte position of rune
## at position pos in s with an optional start byte position. ## at position ``pos`` in ``s`` with an optional start byte position.
## returns the special value -1 if it runs out of the string ## Returns the special value -1 if it runs out of the string.
## ##
## Beware: This can lead to unoptimized code and slow execution! ## Beware: This can lead to unoptimized code and slow execution!
## Most problems can be solved more efficiently by using an iterator ## Most problems can be solved more efficiently by using an iterator
@ -244,7 +274,7 @@ proc runeOffset*(s: string, pos:Natural, start: Natural = 0): int =
return o return o
proc runeAtPos*(s: string, pos: int): Rune = proc runeAtPos*(s: string, pos: int): Rune =
## Returns the unicode character at position pos ## Returns the rune at position ``pos``.
## ##
## Beware: This can lead to unoptimized code and slow execution! ## Beware: This can lead to unoptimized code and slow execution!
## Most problems can be solved more efficiently by using an iterator ## Most problems can be solved more efficiently by using an iterator
@ -252,7 +282,7 @@ proc runeAtPos*(s: string, pos: int): Rune =
fastRuneAt(s, runeOffset(s, pos), result, false) fastRuneAt(s, runeOffset(s, pos), result, false)
proc runeStrAtPos*(s: string, pos: Natural): string = proc runeStrAtPos*(s: string, pos: Natural): string =
## Returns the unicode character at position pos as UTF8 String ## Returns the rune at position ``pos`` as UTF8 String.
## ##
## Beware: This can lead to unoptimized code and slow execution! ## Beware: This can lead to unoptimized code and slow execution!
## Most problems can be solved more efficiently by using an iterator ## Most problems can be solved more efficiently by using an iterator
@ -262,7 +292,7 @@ proc runeStrAtPos*(s: string, pos: Natural): string =
proc runeReverseOffset*(s: string, rev:Positive): (int, int) = proc runeReverseOffset*(s: string, rev:Positive): (int, int) =
## Returns a tuple with the the byte offset of the ## Returns a tuple with the the byte offset of the
## unicode character at position ``rev`` in s counting ## rune at position ``rev`` in ``s``, counting
## from the end (starting with 1) and the total ## from the end (starting with 1) and the total
## number of runes in the string. Returns a negative value ## number of runes in the string. Returns a negative value
## for offset if there are to few runes in the string to ## for offset if there are to few runes in the string to
@ -286,13 +316,21 @@ proc runeReverseOffset*(s: string, rev:Positive): (int, int) =
return (-a, rev.int-a) return (-a, rev.int-a)
return (x, -a+rev.int) return (x, -a+rev.int)
proc runeSubStr*(s: string, pos:int, len:int = int.high): string = proc runeSubStr*(s: string, pos: int, len: int = int.high): string =
## Returns the UTF-8 substring starting at codepoint pos ## Returns the UTF-8 substring starting at codepoint ``pos``
## with len codepoints. If pos or len is negative they count from ## with ``len`` codepoints. If ``pos`` or ``len`` is negative they count from
## the end of the string. If len is not given it means the longest ## the end of the string. If ``len`` is not given it means the longest
## possible string. ## possible string.
## ##
## (Needs some examples) runnableExamples:
let s = "Hänsel ««: 10,00€"
doAssert(runeSubStr(s, 0, 2) == "Hä")
doAssert(runeSubStr(s, 10, 1) == ":")
doAssert(runeSubStr(s, -6) == "10,00€")
doAssert(runeSubStr(s, 10) == ": 10,00€")
doAssert(runeSubStr(s, 12, 5) == "10,00")
doAssert(runeSubStr(s, -6, 3) == "10,")
if pos < 0: if pos < 0:
let (o, rl) = runeReverseOffset(s, -pos) let (o, rl) = runeReverseOffset(s, -pos)
if len >= rl: if len >= rl:
@ -1321,7 +1359,7 @@ proc binarySearch(c: RuneImpl, tab: openArray[int], len, stride: int): int =
return -1 return -1
proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} =
## Converts ``c`` into lower case. This works for any Unicode character. ## Converts ``c`` into lower case. This works for any rune.
## If possible, prefer ``toLower`` over ``toUpper``. ## If possible, prefer ``toLower`` over ``toUpper``.
var c = RuneImpl(c) var c = RuneImpl(c)
var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3) var p = binarySearch(c, tolowerRanges, len(tolowerRanges) div 3, 3)
@ -1333,7 +1371,7 @@ proc toLower*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} =
return Rune(c) return Rune(c)
proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} =
## Converts ``c`` into upper case. This works for any Unicode character. ## Converts ``c`` into upper case. This works for any rune.
## If possible, prefer ``toLower`` over ``toUpper``. ## If possible, prefer ``toLower`` over ``toUpper``.
var c = RuneImpl(c) var c = RuneImpl(c)
var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3) var p = binarySearch(c, toupperRanges, len(toupperRanges) div 3, 3)
@ -1345,7 +1383,7 @@ proc toUpper*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} =
return Rune(c) return Rune(c)
proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} = proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} =
## Converts ``c`` to title case ## Converts ``c`` to title case.
var c = RuneImpl(c) var c = RuneImpl(c)
var p = binarySearch(c, toTitleSinglets, len(toTitleSinglets) div 2, 2) var p = binarySearch(c, toTitleSinglets, len(toTitleSinglets) div 2, 2)
if p >= 0 and c == toTitleSinglets[p]: if p >= 0 and c == toTitleSinglets[p]:
@ -1353,7 +1391,7 @@ proc toTitle*(c: Rune): Rune {.rtl, extern: "nuc$1", procvar.} =
return Rune(c) return Rune(c)
proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
## Returns true iff ``c`` is a lower case Unicode character. ## Returns true iff ``c`` is a lower case rune.
## If possible, prefer ``isLower`` over ``isUpper``. ## If possible, prefer ``isLower`` over ``isUpper``.
var c = RuneImpl(c) var c = RuneImpl(c)
# Note: toUpperRanges is correct here! # Note: toUpperRanges is correct here!
@ -1365,7 +1403,7 @@ proc isLower*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
return true return true
proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
## Returns true iff ``c`` is a upper case Unicode character. ## Returns true iff ``c`` is a upper case rune.
## If possible, prefer ``isLower`` over ``isUpper``. ## If possible, prefer ``isLower`` over ``isUpper``.
var c = RuneImpl(c) var c = RuneImpl(c)
# Note: toLowerRanges is correct here! # Note: toLowerRanges is correct here!
@ -1377,7 +1415,7 @@ proc isUpper*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
return true return true
proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
## Returns true iff ``c`` is an *alpha* Unicode character (i.e., a letter) ## Returns true iff ``c`` is an *alpha* rune (i.e., a letter)
if isUpper(c) or isLower(c): if isUpper(c) or isLower(c):
return true return true
var c = RuneImpl(c) var c = RuneImpl(c)
@ -1389,18 +1427,18 @@ proc isAlpha*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
return true return true
proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = proc isTitle*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
## Returns true iff ``c`` is a Unicode titlecase character ## Returns true iff ``c`` is a Unicode titlecase character.
return isUpper(c) and isLower(c) return isUpper(c) and isLower(c)
proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = proc isWhiteSpace*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
## Returns true iff ``c`` is a Unicode whitespace character ## Returns true iff ``c`` is a Unicode whitespace character.
var c = RuneImpl(c) var c = RuneImpl(c)
var p = binarySearch(c, spaceRanges, len(spaceRanges) div 2, 2) var p = binarySearch(c, spaceRanges, len(spaceRanges) div 2, 2)
if p >= 0 and c >= spaceRanges[p] and c <= spaceRanges[p+1]: if p >= 0 and c >= spaceRanges[p] and c <= spaceRanges[p+1]:
return true return true
proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} = proc isCombining*(c: Rune): bool {.rtl, extern: "nuc$1", procvar.} =
## Returns true iff ``c`` is a Unicode combining character ## Returns true iff ``c`` is a Unicode combining character.
var c = RuneImpl(c) var c = RuneImpl(c)
# Optimized to return false immediately for ASCII # Optimized to return false immediately for ASCII
@ -1424,12 +1462,12 @@ template runeCheck(s, runeProc) =
proc isAlpha*(s: string): bool {.noSideEffect, procvar, proc isAlpha*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nuc$1Str".} = rtl, extern: "nuc$1Str".} =
## Returns true iff `s` contains all alphabetic unicode characters. ## Returns true iff ``s`` contains all alphabetic runes.
runeCheck(s, isAlpha) runeCheck(s, isAlpha)
proc isSpace*(s: string): bool {.noSideEffect, procvar, proc isSpace*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nuc$1Str".} = rtl, extern: "nuc$1Str".} =
## Returns true iff `s` contains all whitespace unicode characters. ## Returns true iff ``s`` contains all whitespace runes.
runeCheck(s, isWhiteSpace) runeCheck(s, isWhiteSpace)
template runeCaseCheck(s, runeProc, skipNonAlpha) = template runeCaseCheck(s, runeProc, skipNonAlpha) =
@ -1459,7 +1497,7 @@ proc isLower*(s: string, skipNonAlpha: bool): bool {.
## Checks whether ``s`` is lower case. ## Checks whether ``s`` is lower case.
## ##
## If ``skipNonAlpha`` is true, returns true if all alphabetical ## If ``skipNonAlpha`` is true, returns true if all alphabetical
## runes in ``s`` are lower case. Returns false if none of the ## runes in ``s`` are lower case. Returns false if none of the
## runes in ``s`` are alphabetical. ## runes in ``s`` are alphabetical.
## ##
## If ``skipNonAlpha`` is false, returns true only if all runes in ## If ``skipNonAlpha`` is false, returns true only if all runes in
@ -1474,7 +1512,7 @@ proc isUpper*(s: string, skipNonAlpha: bool): bool {.
## Checks whether ``s`` is upper case. ## Checks whether ``s`` is upper case.
## ##
## If ``skipNonAlpha`` is true, returns true if all alphabetical ## If ``skipNonAlpha`` is true, returns true if all alphabetical
## runes in ``s`` are upper case. Returns false if none of the ## runes in ``s`` are upper case. Returns false if none of the
## runes in ``s`` are alphabetical. ## runes in ``s`` are alphabetical.
## ##
## If ``skipNonAlpha`` is false, returns true only if all runes in ## If ``skipNonAlpha`` is false, returns true only if all runes in
@ -1485,7 +1523,7 @@ proc isUpper*(s: string, skipNonAlpha: bool): bool {.
runeCaseCheck(s, isUpper, skipNonAlpha) runeCaseCheck(s, isUpper, skipNonAlpha)
template convertRune(s, runeProc) = template convertRune(s, runeProc) =
## Convert runes in `s` using `runeProc` as the converter. ## Convert runes in ``s`` using ``runeProc`` as the converter.
result = newString(len(s)) result = newString(len(s))
var var
@ -1502,20 +1540,20 @@ template convertRune(s, runeProc) =
proc toUpper*(s: string): string {.noSideEffect, procvar, proc toUpper*(s: string): string {.noSideEffect, procvar,
rtl, extern: "nuc$1Str".} = rtl, extern: "nuc$1Str".} =
## Converts `s` into upper-case unicode characters. ## Converts ``s`` into upper-case runes.
convertRune(s, toUpper) convertRune(s, toUpper)
proc toLower*(s: string): string {.noSideEffect, procvar, proc toLower*(s: string): string {.noSideEffect, procvar,
rtl, extern: "nuc$1Str".} = rtl, extern: "nuc$1Str".} =
## Converts `s` into lower-case unicode characters. ## Converts ``s`` into lower-case runes.
convertRune(s, toLower) convertRune(s, toLower)
proc swapCase*(s: string): string {.noSideEffect, procvar, proc swapCase*(s: string): string {.noSideEffect, procvar,
rtl, extern: "nuc$1".} = rtl, extern: "nuc$1".} =
## Swaps the case of unicode characters in `s` ## Swaps the case of runes in ``s``.
## ##
## Returns a new string such that the cases of all unicode characters ## Returns a new string such that the cases of all runes
## are swapped if possible ## are swapped if possible.
var var
i = 0 i = 0
@ -1538,7 +1576,7 @@ proc swapCase*(s: string): string {.noSideEffect, procvar,
proc capitalize*(s: string): string {.noSideEffect, procvar, proc capitalize*(s: string): string {.noSideEffect, procvar,
rtl, extern: "nuc$1".} = rtl, extern: "nuc$1".} =
## Converts the first character of `s` into an upper-case unicode character. ## Converts the first character of ``s`` into an upper-case rune.
if len(s) == 0: if len(s) == 0:
return s return s
@ -1552,10 +1590,10 @@ proc capitalize*(s: string): string {.noSideEffect, procvar,
proc translate*(s: string, replacements: proc(key: string): string): string {. proc translate*(s: string, replacements: proc(key: string): string): string {.
rtl, extern: "nuc$1".} = rtl, extern: "nuc$1".} =
## Translates words in a string using the `replacements` proc to substitute ## Translates words in a string using the ``replacements`` proc to substitute
## words inside `s` with their replacements ## words inside ``s`` with their replacements.
## ##
## `replacements` is any proc that takes a word and returns ## ``replacements`` is any proc that takes a word and returns
## a new word to fill it's place. ## a new word to fill it's place.
# Allocate memory for the new string based on the old one. # Allocate memory for the new string based on the old one.
@ -1601,10 +1639,10 @@ proc translate*(s: string, replacements: proc(key: string): string): string {.
proc title*(s: string): string {.noSideEffect, procvar, proc title*(s: string): string {.noSideEffect, procvar,
rtl, extern: "nuc$1".} = rtl, extern: "nuc$1".} =
## Converts `s` to a unicode title. ## Converts ``s`` to a unicode title.
## ##
## Returns a new string such that the first character ## Returns a new string such that the first character
## in each word inside `s` is capitalized ## in each word inside ``s`` is capitalized.
var var
i = 0 i = 0
@ -1631,10 +1669,10 @@ proc title*(s: string): string {.noSideEffect, procvar,
proc isTitle*(s: string): bool {.noSideEffect, procvar, proc isTitle*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nuc$1Str", rtl, extern: "nuc$1Str",
deprecated: "Deprecated since version 0.20 since its semantics are unclear".}= deprecated: "Deprecated since version 0.20 since its semantics are unclear".}=
## Checks whether or not `s` is a unicode title. ## Checks whether or not ``s`` is a unicode title.
## ##
## Returns true if the first character in each word inside `s` ## Returns true if the first character in each word inside ``s``
## are upper case and there is at least one character in `s`. ## are upper case and there is at least one character in ``s``.
if s.len == 0: if s.len == 0:
return false return false
@ -1656,7 +1694,7 @@ proc isTitle*(s: string): bool {.noSideEffect, procvar,
firstRune = true firstRune = true
iterator runes*(s: string): Rune = iterator runes*(s: string): Rune =
## Iterates over any unicode character of the string ``s`` returning runes ## Iterates over any rune of the string ``s`` returning runes.
var var
i = 0 i = 0
result: Rune result: Rune
@ -1665,7 +1703,7 @@ iterator runes*(s: string): Rune =
yield result yield result
iterator utf8*(s: string): string = iterator utf8*(s: string): string =
## Iterates over any unicode character of the string ``s`` returning utf8 values ## Iterates over any rune of the string ``s`` returning utf8 values.
var o = 0 var o = 0
while o < s.len: while o < s.len:
let n = runeLenAt(s, o) let n = runeLenAt(s, o)
@ -1673,7 +1711,7 @@ iterator utf8*(s: string): string =
o += n o += n
proc toRunes*(s: string): seq[Rune] = proc toRunes*(s: string): seq[Rune] =
## Obtains a sequence containing the Runes in ``s`` ## Obtains a sequence containing the Runes in ``s``.
result = newSeq[Rune]() result = newSeq[Rune]()
for r in s.runes: for r in s.runes:
result.add(r) result.add(r)
@ -1696,15 +1734,14 @@ proc cmpRunesIgnoreCase*(a, b: string): int {.rtl, extern: "nuc$1", procvar.} =
result = a.len - b.len result = a.len - b.len
proc reversed*(s: string): string = proc reversed*(s: string): string =
## Returns the reverse of ``s``, interpreting it as Unicode characters. ## Returns the reverse of ``s``, interpreting it as runes.
## Unicode combining characters are correctly interpreted as well: ## Unicode combining characters are correctly interpreted as well.
## runnableExamples:
## .. code-block:: nim assert reversed("Reverse this!") == "!siht esreveR"
## assert reversed("先秦兩漢") == "漢兩秦先"
## assert reversed("Reverse this!") == "!siht esreveR" assert reversed("as⃝df̅") == "f̅ds⃝a"
## assert reversed("先秦兩漢") == "漢兩秦先" assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞"
## assert reversed("as⃝df̅") == "f̅ds⃝a"
## assert reversed("a⃞b⃞c⃞") == "c⃞b⃞a⃞"
var var
i = 0 i = 0
lastI = 0 lastI = 0
@ -1731,7 +1768,7 @@ proc reversed*(s: string): string =
reverseUntil(len(s)) reverseUntil(len(s))
proc graphemeLen*(s: string; i: Natural): Natural = proc graphemeLen*(s: string; i: Natural): Natural =
## The number of bytes belonging to 's[i]' including following combining ## The number of bytes belonging to ``s[i]`` including following combining
## characters. ## characters.
var j = i.int var j = i.int
var r, r2: Rune var r, r2: Rune
@ -1744,7 +1781,7 @@ proc graphemeLen*(s: string; i: Natural): Natural =
result = j-i result = j-i
proc lastRune*(s: string; last: int): (Rune, int) = proc lastRune*(s: string; last: int): (Rune, int) =
## length of the last rune in 's[0..last]'. Returns the rune and its length ## Length of the last rune in ``s[0..last]``. Returns the rune and its length
## in bytes. ## in bytes.
if s[last] <= chr(127): if s[last] <= chr(127):
result = (Rune(s[last]), 1) result = (Rune(s[last]), 1)
@ -1778,7 +1815,7 @@ proc stringHasSep(s: string, index: int, sep: Rune): bool =
return sep == rune return sep == rune
template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) = template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) =
## Common code for split procedures ## Common code for split procedures.
var var
last = 0 last = 0
splits = maxsplit splits = maxsplit
@ -1801,9 +1838,9 @@ template splitCommon(s, sep, maxsplit: untyped, sepLen: int = -1) =
iterator split*(s: string, seps: openarray[Rune] = unicodeSpaces, iterator split*(s: string, seps: openarray[Rune] = unicodeSpaces,
maxsplit: int = -1): string = maxsplit: int = -1): string =
## Splits the unicode string `s` into substrings using a group of separators. ## Splits the unicode string ``s`` into substrings using a group of separators.
## ##
## Substrings are separated by a substring containing only `seps`. ## Substrings are separated by a substring containing only ``seps``.
## ##
## .. code-block:: nim ## .. code-block:: nim
## for word in split("this\lis an\texample"): ## for word in split("this\lis an\texample"):
@ -1844,7 +1881,7 @@ iterator split*(s: string, seps: openarray[Rune] = unicodeSpaces,
splitCommon(s, seps, maxsplit) splitCommon(s, seps, maxsplit)
iterator splitWhitespace*(s: string): string = iterator splitWhitespace*(s: string): string =
## Splits a unicode string at whitespace runes ## Splits a unicode string at whitespace runes.
splitCommon(s, unicodeSpaces, -1) splitCommon(s, unicodeSpaces, -1)
template accResult(iter: untyped) = template accResult(iter: untyped) =
@ -1858,9 +1895,9 @@ proc splitWhitespace*(s: string): seq[string] {.noSideEffect,
accResult(splitWhitespace(s)) accResult(splitWhitespace(s))
iterator split*(s: string, sep: Rune, maxsplit: int = -1): string = iterator split*(s: string, sep: Rune, maxsplit: int = -1): string =
## Splits the unicode string `s` into substrings using a single separator. ## Splits the unicode string ``s`` into substrings using a single separator.
## ##
## Substrings are separated by the rune `sep`. ## Substrings are separated by the rune ``sep``.
## The code: ## The code:
## ##
## .. code-block:: nim ## .. code-block:: nim
@ -1898,11 +1935,11 @@ proc split*(s: string, sep: Rune, maxsplit: int = -1): seq[string] {.noSideEffec
proc strip*(s: string, leading = true, trailing = true, proc strip*(s: string, leading = true, trailing = true,
runes: openarray[Rune] = unicodeSpaces): string {.noSideEffect, runes: openarray[Rune] = unicodeSpaces): string {.noSideEffect,
rtl, extern: "nucStrip".} = rtl, extern: "nucStrip".} =
## Strips leading or trailing `runes` from `s` and returns ## Strips leading or trailing ``runes`` from ``s`` and returns
## the resulting string. ## the resulting string.
## ##
## If `leading` is true, leading `runes` are stripped. ## If ``leading`` is true, leading ``runes`` are stripped.
## If `trailing` is true, trailing `runes` are stripped. ## If ``trailing`` is true, trailing ``runes`` are stripped.
## If both are false, the string is returned unchanged. ## If both are false, the string is returned unchanged.
var var
s_i = 0 ## starting index into string ``s`` s_i = 0 ## starting index into string ``s``
@ -1948,9 +1985,9 @@ proc strip*(s: string, leading = true, trailing = true,
proc repeat*(c: Rune, count: Natural): string {.noSideEffect, proc repeat*(c: Rune, count: Natural): string {.noSideEffect,
rtl, extern: "nucRepeatRune".} = rtl, extern: "nucRepeatRune".} =
## Returns a string of `count` Runes `c`. ## Returns a string of ``count`` Runes ``c``.
## ##
## The returned string will have a rune-length of `count`. ## The returned string will have a rune-length of ``count``.
let s = $c let s = $c
result = newStringOfCap(count * s.len) result = newStringOfCap(count * s.len)
for i in 0 ..< count: for i in 0 ..< count:
@ -1958,11 +1995,11 @@ proc repeat*(c: Rune, count: Natural): string {.noSideEffect,
proc align*(s: string, count: Natural, padding = ' '.Rune): string {. proc align*(s: string, count: Natural, padding = ' '.Rune): string {.
noSideEffect, rtl, extern: "nucAlignString".} = noSideEffect, rtl, extern: "nucAlignString".} =
## Aligns a unicode string `s` with `padding`, so that it has a rune-length ## Aligns a unicode string ``s`` with ``padding``, so that it has a rune-length
## of `count`. ## of ``count``.
## ##
## `padding` characters (by default spaces) are added before `s` resulting in ## ``padding`` characters (by default spaces) are added before ``s`` resulting in
## right alignment. If ``s.runelen >= count``, no spaces are added and `s` is ## right alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is
## returned unchanged. If you need to left align a string use the `alignLeft ## returned unchanged. If you need to left align a string use the `alignLeft
## proc <#alignLeft>`_. ## proc <#alignLeft>`_.
runnableExamples: runnableExamples:
@ -1985,11 +2022,11 @@ proc align*(s: string, count: Natural, padding = ' '.Rune): string {.
proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {. proc alignLeft*(s: string, count: Natural, padding = ' '.Rune): string {.
noSideEffect.} = noSideEffect.} =
## Left-Aligns a unicode string `s` with `padding`, so that it has a ## Left-Aligns a unicode string ``s`` with ``padding``, so that it has a
## rune-length of `count`. ## rune-length of ``count``.
## ##
## `padding` characters (by default spaces) are added after `s` resulting in ## ``padding`` characters (by default spaces) are added after ``s`` resulting in
## left alignment. If ``s.runelen >= count``, no spaces are added and `s` is ## left alignment. If ``s.runelen >= count``, no spaces are added and ``s`` is
## returned unchanged. If you need to right align a string use the `align ## returned unchanged. If you need to right align a string use the `align
## proc <#align>`_. ## proc <#align>`_.
runnableExamples: runnableExamples: