jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

right shift is now by default sign preserving (#11322)

Browse source 
* right shift is now by default sign preserving
* fix hashString and semfold
* enable arithmetic shift right globally for CI
* fix typo
* remove xxx
* use oldShiftRight as flag
* apply feedback
* add changelog entry
This commit is contained in:
Arne Döring 2019-05-29 16:48:00 +02:00 • committed by Andreas Rumpf
commit 88b5dd3362
18 changed files with 211 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

132
lib/pure/unicode.nim
View file

@ -48,12 +48,12 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
var i = 0
while i < len(s):
if ord(s[i]) <=% 127: inc(i)
elif ord(s[i]) shr 5 == 0b110: inc(i, 2)
elif ord(s[i]) shr 4 == 0b1110: inc(i, 3)
elif ord(s[i]) shr 3 == 0b11110: inc(i, 4)
elif ord(s[i]) shr 2 == 0b111110: inc(i, 5)
elif ord(s[i]) shr 1 == 0b1111110: inc(i, 6)
if uint(s[i]) <= 127: inc(i)
elif uint(s[i]) shr 5 == 0b110: inc(i, 2)
elif uint(s[i]) shr 4 == 0b1110: inc(i, 3)
elif uint(s[i]) shr 3 == 0b11110: inc(i, 4)
elif uint(s[i]) shr 2 == 0b111110: inc(i, 5)
elif uint(s[i]) shr 1 == 0b1111110: inc(i, 6)
else: inc i
inc(result)
@ -67,12 +67,12 @@ proc runeLenAt*(s: string, i: Natural): int =
doAssert a.runeLenAt(0) == 1
doAssert a.runeLenAt(1) == 2
if ord(s[i]) <=% 127: result = 1
elif ord(s[i]) shr 5 == 0b110: result = 2
elif ord(s[i]) shr 4 == 0b1110: result = 3
elif ord(s[i]) shr 3 == 0b11110: result = 4
elif ord(s[i]) shr 2 == 0b111110: result = 5
elif ord(s[i]) shr 1 == 0b1111110: result = 6
if uint(s[i]) <= 127: result = 1
elif uint(s[i]) shr 5 == 0b110: result = 2
elif uint(s[i]) shr 4 == 0b1110: result = 3
elif uint(s[i]) shr 3 == 0b11110: result = 4
elif uint(s[i]) shr 2 == 0b111110: result = 5
elif uint(s[i]) shr 1 == 0b1111110: result = 6
else: result = 1
const replRune = Rune(0xFFFD)
@ -83,76 +83,76 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) =
## If ``doInc == true`` (default), ``i`` is incremented by the number
## of bytes that have been processed.
bind ones
if ord(s[i]) <=% 127:
result = Rune(ord(s[i]))
if uint(s[i]) <= 127:
result = Rune(uint(s[i]))
when doInc: inc(i)
elif ord(s[i]) shr 5 == 0b110:
# assert(ord(s[i+1]) shr 6 == 0b10)
elif uint(s[i]) shr 5 == 0b110:
# assert(uint(s[i+1]) shr 6 == 0b10)
if i <= s.len - 2:
result = Rune((ord(s[i]) and (ones(5))) shl 6 or
(ord(s[i+1]) and ones(6)))
result = Rune((uint(s[i]) and (ones(5))) shl 6 or
(uint(s[i+1]) and ones(6)))
when doInc: inc(i, 2)
else:
result = replRune
when doInc: inc(i)
elif ord(s[i]) shr 4 == 0b1110:
# assert(ord(s[i+1]) shr 6 == 0b10)
# assert(ord(s[i+2]) shr 6 == 0b10)
elif uint(s[i]) shr 4 == 0b1110:
# assert(uint(s[i+1]) shr 6 == 0b10)
# assert(uint(s[i+2]) shr 6 == 0b10)
if i <= s.len - 3:
result = Rune((ord(s[i]) and ones(4)) shl 12 or
(ord(s[i+1]) and ones(6)) shl 6 or
(ord(s[i+2]) and ones(6)))
result = Rune((uint(s[i]) and ones(4)) shl 12 or
(uint(s[i+1]) and ones(6)) shl 6 or
(uint(s[i+2]) and ones(6)))
when doInc: inc(i, 3)
else:
result = replRune
when doInc: inc(i)
elif ord(s[i]) shr 3 == 0b11110:
# assert(ord(s[i+1]) shr 6 == 0b10)
# assert(ord(s[i+2]) shr 6 == 0b10)
# assert(ord(s[i+3]) shr 6 == 0b10)
elif uint(s[i]) shr 3 == 0b11110:
# assert(uint(s[i+1]) shr 6 == 0b10)
# assert(uint(s[i+2]) shr 6 == 0b10)
# assert(uint(s[i+3]) shr 6 == 0b10)
if i <= s.len - 4:
result = Rune((ord(s[i]) and ones(3)) shl 18 or
(ord(s[i+1]) and ones(6)) shl 12 or
(ord(s[i+2]) and ones(6)) shl 6 or
(ord(s[i+3]) and ones(6)))
result = Rune((uint(s[i]) and ones(3)) shl 18 or
(uint(s[i+1]) and ones(6)) shl 12 or
(uint(s[i+2]) and ones(6)) shl 6 or
(uint(s[i+3]) and ones(6)))
when doInc: inc(i, 4)
else:
result = replRune
when doInc: inc(i)
elif ord(s[i]) shr 2 == 0b111110:
# assert(ord(s[i+1]) shr 6 == 0b10)
# assert(ord(s[i+2]) shr 6 == 0b10)
# assert(ord(s[i+3]) shr 6 == 0b10)
# assert(ord(s[i+4]) shr 6 == 0b10)
elif uint(s[i]) shr 2 == 0b111110:
# assert(uint(s[i+1]) shr 6 == 0b10)
# assert(uint(s[i+2]) shr 6 == 0b10)
# assert(uint(s[i+3]) shr 6 == 0b10)
# assert(uint(s[i+4]) shr 6 == 0b10)
if i <= s.len - 5:
result = Rune((ord(s[i]) and ones(2)) shl 24 or
(ord(s[i+1]) and ones(6)) shl 18 or
(ord(s[i+2]) and ones(6)) shl 12 or
(ord(s[i+3]) and ones(6)) shl 6 or
(ord(s[i+4]) and ones(6)))
result = Rune((uint(s[i]) and ones(2)) shl 24 or
(uint(s[i+1]) and ones(6)) shl 18 or
(uint(s[i+2]) and ones(6)) shl 12 or
(uint(s[i+3]) and ones(6)) shl 6 or
(uint(s[i+4]) and ones(6)))
when doInc: inc(i, 5)
else:
result = replRune
when doInc: inc(i)
elif ord(s[i]) shr 1 == 0b1111110:
# assert(ord(s[i+1]) shr 6 == 0b10)
# assert(ord(s[i+2]) shr 6 == 0b10)
# assert(ord(s[i+3]) shr 6 == 0b10)
# assert(ord(s[i+4]) shr 6 == 0b10)
# assert(ord(s[i+5]) shr 6 == 0b10)
elif uint(s[i]) shr 1 == 0b1111110:
# assert(uint(s[i+1]) shr 6 == 0b10)
# assert(uint(s[i+2]) shr 6 == 0b10)
# assert(uint(s[i+3]) shr 6 == 0b10)
# assert(uint(s[i+4]) shr 6 == 0b10)
# assert(uint(s[i+5]) shr 6 == 0b10)
if i <= s.len - 6:
result = Rune((ord(s[i]) and ones(1)) shl 30 or
(ord(s[i+1]) and ones(6)) shl 24 or
(ord(s[i+2]) and ones(6)) shl 18 or
(ord(s[i+3]) and ones(6)) shl 12 or
(ord(s[i+4]) and ones(6)) shl 6 or
(ord(s[i+5]) and ones(6)))
result = Rune((uint(s[i]) and ones(1)) shl 30 or
(uint(s[i+1]) and ones(6)) shl 24 or
(uint(s[i+2]) and ones(6)) shl 18 or
(uint(s[i+3]) and ones(6)) shl 12 or
(uint(s[i+4]) and ones(6)) shl 6 or
(uint(s[i+5]) and ones(6)))
when doInc: inc(i, 6)
else:
result = replRune
when doInc: inc(i)
else:
result = Rune(ord(s[i]))
result = Rune(uint(s[i]))
when doInc: inc(i)
proc runeAt*(s: string, i: Natural): Rune =
@ -180,20 +180,20 @@ proc validateUTF8*(s: string): int =
var i = 0
let L = s.len
while i < L:
if ord(s[i]) <=% 127:
if uint(s[i]) <= 127:
inc(i)
elif ord(s[i]) shr 5 == 0b110:
if ord(s[i]) < 0xc2: return i # Catch overlong ascii representations.
if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2)
elif uint(s[i]) shr 5 == 0b110:
if uint(s[i]) < 0xc2: return i # Catch overlong ascii representations.
if i+1 < L and uint(s[i+1]) shr 6 == 0b10: inc(i, 2)
else: return i
elif ord(s[i]) shr 4 == 0b1110:
if i+2 < L and ord(s[i+1]) shr 6 == 0b10 and ord(s[i+2]) shr 6 == 0b10:
elif uint(s[i]) shr 4 == 0b1110:
if i+2 < L and uint(s[i+1]) shr 6 == 0b10 and uint(s[i+2]) shr 6 == 0b10:
inc i, 3
else: return i
elif ord(s[i]) shr 3 == 0b11110:
if i+3 < L and ord(s[i+1]) shr 6 == 0b10 and
ord(s[i+2]) shr 6 == 0b10 and
ord(s[i+3]) shr 6 == 0b10:
elif uint(s[i]) shr 3 == 0b11110:
if i+3 < L and uint(s[i+1]) shr 6 == 0b10 and
uint(s[i+2]) shr 6 == 0b10 and
uint(s[i+3]) shr 6 == 0b10:
inc i, 4
else: return i
else:
@ -906,7 +906,7 @@ proc lastRune*(s: string; last: int): (Rune, int) =
result = (Rune(s[last]), 1)
else:
var L = 0
while last-L >= 0 and ord(s[last-L]) shr 6 == 0b10: inc(L)
while last-L >= 0 and uint(s[last-L]) shr 6 == 0b10: inc(L)
var r: Rune
fastRuneAt(s, last-L, r, false)
result = (r, L+1)