right shift is now by default sign preserving (#11322)
* right shift is now by default sign preserving * fix hashString and semfold * enable arithmetic shift right globally for CI * fix typo * remove xxx * use oldShiftRight as flag * apply feedback * add changelog entry
This commit is contained in:
Arne Döring
Andreas Rumpf
parent
897e7c90ac
commit
88b5dd3362
18 changed files with 211 additions and 187 deletions
|
|
@ -23,7 +23,7 @@ import
|
|||
hashes, math
|
||||
|
||||
type
|
||||
BitScalar = int
|
||||
BitScalar = uint
|
||||
|
||||
const
|
||||
InitIntSetSize = 8 # must be a power of two!
|
||||
|
|
@ -102,8 +102,8 @@ proc intSetPut(t: var IntSet, key: int): PTrunk =
|
|||
proc bitincl(s: var IntSet, key: int) {.inline.} =
|
||||
var t = intSetPut(s, `shr`(key, TrunkShift))
|
||||
var u = key and TrunkMask
|
||||
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
|
||||
`shl`(1, u and IntMask)
|
||||
t.bits[u shr IntShift] = t.bits[u shr IntShift] or
|
||||
(BitScalar(1) shl (u and IntMask))
|
||||
|
||||
proc exclImpl(s: var IntSet, key: int) =
|
||||
if s.elems <= s.a.len:
|
||||
|
|
@ -113,11 +113,11 @@ proc exclImpl(s: var IntSet, key: int) =
|
|||
dec s.elems
|
||||
return
|
||||
else:
|
||||
var t = intSetGet(s, `shr`(key, TrunkShift))
|
||||
var t = intSetGet(s, key shr TrunkShift)
|
||||
if t != nil:
|
||||
var u = key and TrunkMask
|
||||
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] and
|
||||
not `shl`(1, u and IntMask)
|
||||
t.bits[u shr IntShift] = t.bits[u shr IntShift] and
|
||||
not(BitScalar(1) shl (u and IntMask))
|
||||
|
||||
template dollarImpl(): untyped =
|
||||
result = "{"
|
||||
|
|
@ -137,7 +137,7 @@ iterator items*(s: IntSet): int {.inline.} =
|
|||
while r != nil:
|
||||
var i = 0
|
||||
while i <= high(r.bits):
|
||||
var w = r.bits[i]
|
||||
var w: uint = r.bits[i]
|
||||
# taking a copy of r.bits[i] here is correct, because
|
||||
# modifying operations are not allowed during traversation
|
||||
var j = 0
|
||||
|
|
@ -186,7 +186,7 @@ proc contains*(s: IntSet, key: int): bool =
|
|||
var t = intSetGet(s, `shr`(key, TrunkShift))
|
||||
if t != nil:
|
||||
var u = key and TrunkMask
|
||||
result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
|
||||
result = (t.bits[u shr IntShift] and (BitScalar(1) shl (u and IntMask))) != 0
|
||||
else:
|
||||
result = false
|
||||
|
||||
|
|
@ -268,10 +268,10 @@ proc containsOrIncl*(s: var IntSet, key: int): bool =
|
|||
var t = intSetGet(s, `shr`(key, TrunkShift))
|
||||
if t != nil:
|
||||
var u = key and TrunkMask
|
||||
result = (t.bits[`shr`(u, IntShift)] and `shl`(1, u and IntMask)) != 0
|
||||
result = (t.bits[u shr IntShift] and BitScalar(1) shl (u and IntMask)) != 0
|
||||
if not result:
|
||||
t.bits[`shr`(u, IntShift)] = t.bits[`shr`(u, IntShift)] or
|
||||
`shl`(1, u and IntMask)
|
||||
t.bits[u shr IntShift] = t.bits[u shr IntShift] or
|
||||
(BitScalar(1) shl (u and IntMask))
|
||||
else:
|
||||
incl(s, key)
|
||||
result = false
|
||||
|
|
|
|||
|
|
@ -60,17 +60,22 @@ proc `!&`*(h: Hash, val: int): Hash {.inline.} =
|
|||
## Mixes a hash value `h` with `val` to produce a new hash value.
|
||||
##
|
||||
## This is only needed if you need to implement a hash proc for a new datatype.
|
||||
result = h +% val
|
||||
result = result +% result shl 10
|
||||
result = result xor (result shr 6)
|
||||
let h = cast[uint](h)
|
||||
let val = cast[uint](val)
|
||||
var res = h + val
|
||||
res = res + res shl 10
|
||||
res = res xor (res shr 6)
|
||||
result = cast[Hash](res)
|
||||
|
||||
proc `!$`*(h: Hash): Hash {.inline.} =
|
||||
## Finishes the computation of the hash value.
|
||||
##
|
||||
## This is only needed if you need to implement a hash proc for a new datatype.
|
||||
result = h +% h shl 3
|
||||
result = result xor (result shr 11)
|
||||
result = result +% result shl 15
|
||||
let h = cast[uint](h) # Hash is practically unsigned.
|
||||
var res = h + h shl 3
|
||||
res = res xor (res shr 11)
|
||||
res = res + res shl 15
|
||||
result = cast[Hash](res)
|
||||
|
||||
proc hashData*(data: pointer, size: int): Hash =
|
||||
## Hashes an array of bytes of size `size`.
|
||||
|
|
@ -105,7 +110,7 @@ proc hash*(x: pointer): Hash {.inline.} =
|
|||
}
|
||||
"""
|
||||
else:
|
||||
result = (cast[Hash](x)) shr 3 # skip the alignment
|
||||
result = cast[Hash](cast[uint](x) shr 3) # skip the alignment
|
||||
|
||||
when not defined(booting):
|
||||
proc hash*[T: proc](x: T): Hash {.inline.} =
|
||||
|
|
|
|||
|
|
@ -48,12 +48,12 @@ proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} =
|
|||
|
||||
var i = 0
|
||||
while i < len(s):
|
||||
if ord(s[i]) <=% 127: inc(i)
|
||||
elif ord(s[i]) shr 5 == 0b110: inc(i, 2)
|
||||
elif ord(s[i]) shr 4 == 0b1110: inc(i, 3)
|
||||
elif ord(s[i]) shr 3 == 0b11110: inc(i, 4)
|
||||
elif ord(s[i]) shr 2 == 0b111110: inc(i, 5)
|
||||
elif ord(s[i]) shr 1 == 0b1111110: inc(i, 6)
|
||||
if uint(s[i]) <= 127: inc(i)
|
||||
elif uint(s[i]) shr 5 == 0b110: inc(i, 2)
|
||||
elif uint(s[i]) shr 4 == 0b1110: inc(i, 3)
|
||||
elif uint(s[i]) shr 3 == 0b11110: inc(i, 4)
|
||||
elif uint(s[i]) shr 2 == 0b111110: inc(i, 5)
|
||||
elif uint(s[i]) shr 1 == 0b1111110: inc(i, 6)
|
||||
else: inc i
|
||||
inc(result)
|
||||
|
||||
|
|
@ -67,12 +67,12 @@ proc runeLenAt*(s: string, i: Natural): int =
|
|||
doAssert a.runeLenAt(0) == 1
|
||||
doAssert a.runeLenAt(1) == 2
|
||||
|
||||
if ord(s[i]) <=% 127: result = 1
|
||||
elif ord(s[i]) shr 5 == 0b110: result = 2
|
||||
elif ord(s[i]) shr 4 == 0b1110: result = 3
|
||||
elif ord(s[i]) shr 3 == 0b11110: result = 4
|
||||
elif ord(s[i]) shr 2 == 0b111110: result = 5
|
||||
elif ord(s[i]) shr 1 == 0b1111110: result = 6
|
||||
if uint(s[i]) <= 127: result = 1
|
||||
elif uint(s[i]) shr 5 == 0b110: result = 2
|
||||
elif uint(s[i]) shr 4 == 0b1110: result = 3
|
||||
elif uint(s[i]) shr 3 == 0b11110: result = 4
|
||||
elif uint(s[i]) shr 2 == 0b111110: result = 5
|
||||
elif uint(s[i]) shr 1 == 0b1111110: result = 6
|
||||
else: result = 1
|
||||
|
||||
const replRune = Rune(0xFFFD)
|
||||
|
|
@ -83,76 +83,76 @@ template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) =
|
|||
## If ``doInc == true`` (default), ``i`` is incremented by the number
|
||||
## of bytes that have been processed.
|
||||
bind ones
|
||||
if ord(s[i]) <=% 127:
|
||||
result = Rune(ord(s[i]))
|
||||
if uint(s[i]) <= 127:
|
||||
result = Rune(uint(s[i]))
|
||||
when doInc: inc(i)
|
||||
elif ord(s[i]) shr 5 == 0b110:
|
||||
# assert(ord(s[i+1]) shr 6 == 0b10)
|
||||
elif uint(s[i]) shr 5 == 0b110:
|
||||
# assert(uint(s[i+1]) shr 6 == 0b10)
|
||||
if i <= s.len - 2:
|
||||
result = Rune((ord(s[i]) and (ones(5))) shl 6 or
|
||||
(ord(s[i+1]) and ones(6)))
|
||||
result = Rune((uint(s[i]) and (ones(5))) shl 6 or
|
||||
(uint(s[i+1]) and ones(6)))
|
||||
when doInc: inc(i, 2)
|
||||
else:
|
||||
result = replRune
|
||||
when doInc: inc(i)
|
||||
elif ord(s[i]) shr 4 == 0b1110:
|
||||
# assert(ord(s[i+1]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+2]) shr 6 == 0b10)
|
||||
elif uint(s[i]) shr 4 == 0b1110:
|
||||
# assert(uint(s[i+1]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+2]) shr 6 == 0b10)
|
||||
if i <= s.len - 3:
|
||||
result = Rune((ord(s[i]) and ones(4)) shl 12 or
|
||||
(ord(s[i+1]) and ones(6)) shl 6 or
|
||||
(ord(s[i+2]) and ones(6)))
|
||||
result = Rune((uint(s[i]) and ones(4)) shl 12 or
|
||||
(uint(s[i+1]) and ones(6)) shl 6 or
|
||||
(uint(s[i+2]) and ones(6)))
|
||||
when doInc: inc(i, 3)
|
||||
else:
|
||||
result = replRune
|
||||
when doInc: inc(i)
|
||||
elif ord(s[i]) shr 3 == 0b11110:
|
||||
# assert(ord(s[i+1]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+2]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+3]) shr 6 == 0b10)
|
||||
elif uint(s[i]) shr 3 == 0b11110:
|
||||
# assert(uint(s[i+1]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+2]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+3]) shr 6 == 0b10)
|
||||
if i <= s.len - 4:
|
||||
result = Rune((ord(s[i]) and ones(3)) shl 18 or
|
||||
(ord(s[i+1]) and ones(6)) shl 12 or
|
||||
(ord(s[i+2]) and ones(6)) shl 6 or
|
||||
(ord(s[i+3]) and ones(6)))
|
||||
result = Rune((uint(s[i]) and ones(3)) shl 18 or
|
||||
(uint(s[i+1]) and ones(6)) shl 12 or
|
||||
(uint(s[i+2]) and ones(6)) shl 6 or
|
||||
(uint(s[i+3]) and ones(6)))
|
||||
when doInc: inc(i, 4)
|
||||
else:
|
||||
result = replRune
|
||||
when doInc: inc(i)
|
||||
elif ord(s[i]) shr 2 == 0b111110:
|
||||
# assert(ord(s[i+1]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+2]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+3]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+4]) shr 6 == 0b10)
|
||||
elif uint(s[i]) shr 2 == 0b111110:
|
||||
# assert(uint(s[i+1]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+2]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+3]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+4]) shr 6 == 0b10)
|
||||
if i <= s.len - 5:
|
||||
result = Rune((ord(s[i]) and ones(2)) shl 24 or
|
||||
(ord(s[i+1]) and ones(6)) shl 18 or
|
||||
(ord(s[i+2]) and ones(6)) shl 12 or
|
||||
(ord(s[i+3]) and ones(6)) shl 6 or
|
||||
(ord(s[i+4]) and ones(6)))
|
||||
result = Rune((uint(s[i]) and ones(2)) shl 24 or
|
||||
(uint(s[i+1]) and ones(6)) shl 18 or
|
||||
(uint(s[i+2]) and ones(6)) shl 12 or
|
||||
(uint(s[i+3]) and ones(6)) shl 6 or
|
||||
(uint(s[i+4]) and ones(6)))
|
||||
when doInc: inc(i, 5)
|
||||
else:
|
||||
result = replRune
|
||||
when doInc: inc(i)
|
||||
elif ord(s[i]) shr 1 == 0b1111110:
|
||||
# assert(ord(s[i+1]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+2]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+3]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+4]) shr 6 == 0b10)
|
||||
# assert(ord(s[i+5]) shr 6 == 0b10)
|
||||
elif uint(s[i]) shr 1 == 0b1111110:
|
||||
# assert(uint(s[i+1]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+2]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+3]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+4]) shr 6 == 0b10)
|
||||
# assert(uint(s[i+5]) shr 6 == 0b10)
|
||||
if i <= s.len - 6:
|
||||
result = Rune((ord(s[i]) and ones(1)) shl 30 or
|
||||
(ord(s[i+1]) and ones(6)) shl 24 or
|
||||
(ord(s[i+2]) and ones(6)) shl 18 or
|
||||
(ord(s[i+3]) and ones(6)) shl 12 or
|
||||
(ord(s[i+4]) and ones(6)) shl 6 or
|
||||
(ord(s[i+5]) and ones(6)))
|
||||
result = Rune((uint(s[i]) and ones(1)) shl 30 or
|
||||
(uint(s[i+1]) and ones(6)) shl 24 or
|
||||
(uint(s[i+2]) and ones(6)) shl 18 or
|
||||
(uint(s[i+3]) and ones(6)) shl 12 or
|
||||
(uint(s[i+4]) and ones(6)) shl 6 or
|
||||
(uint(s[i+5]) and ones(6)))
|
||||
when doInc: inc(i, 6)
|
||||
else:
|
||||
result = replRune
|
||||
when doInc: inc(i)
|
||||
else:
|
||||
result = Rune(ord(s[i]))
|
||||
result = Rune(uint(s[i]))
|
||||
when doInc: inc(i)
|
||||
|
||||
proc runeAt*(s: string, i: Natural): Rune =
|
||||
|
|
@ -180,20 +180,20 @@ proc validateUTF8*(s: string): int =
|
|||
var i = 0
|
||||
let L = s.len
|
||||
while i < L:
|
||||
if ord(s[i]) <=% 127:
|
||||
if uint(s[i]) <= 127:
|
||||
inc(i)
|
||||
elif ord(s[i]) shr 5 == 0b110:
|
||||
if ord(s[i]) < 0xc2: return i # Catch overlong ascii representations.
|
||||
if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2)
|
||||
elif uint(s[i]) shr 5 == 0b110:
|
||||
if uint(s[i]) < 0xc2: return i # Catch overlong ascii representations.
|
||||
if i+1 < L and uint(s[i+1]) shr 6 == 0b10: inc(i, 2)
|
||||
else: return i
|
||||
elif ord(s[i]) shr 4 == 0b1110:
|
||||
if i+2 < L and ord(s[i+1]) shr 6 == 0b10 and ord(s[i+2]) shr 6 == 0b10:
|
||||
elif uint(s[i]) shr 4 == 0b1110:
|
||||
if i+2 < L and uint(s[i+1]) shr 6 == 0b10 and uint(s[i+2]) shr 6 == 0b10:
|
||||
inc i, 3
|
||||
else: return i
|
||||
elif ord(s[i]) shr 3 == 0b11110:
|
||||
if i+3 < L and ord(s[i+1]) shr 6 == 0b10 and
|
||||
ord(s[i+2]) shr 6 == 0b10 and
|
||||
ord(s[i+3]) shr 6 == 0b10:
|
||||
elif uint(s[i]) shr 3 == 0b11110:
|
||||
if i+3 < L and uint(s[i+1]) shr 6 == 0b10 and
|
||||
uint(s[i+2]) shr 6 == 0b10 and
|
||||
uint(s[i+3]) shr 6 == 0b10:
|
||||
inc i, 4
|
||||
else: return i
|
||||
else:
|
||||
|
|
@ -906,7 +906,7 @@ proc lastRune*(s: string; last: int): (Rune, int) =
|
|||
result = (Rune(s[last]), 1)
|
||||
else:
|
||||
var L = 0
|
||||
while last-L >= 0 and ord(s[last-L]) shr 6 == 0b10: inc(L)
|
||||
while last-L >= 0 and uint(s[last-L]) shr 6 == 0b10: inc(L)
|
||||
var r: Rune
|
||||
fastRuneAt(s, last-L, r, false)
|
||||
result = (r, L+1)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue