jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed UTF-16 to UTF-8 conversion in widestrs.nim

Browse source 
the source of problem for issue #3228
This commit is contained in:
jangko 2015-08-20 14:17:46 +07:00
commit 493dbc8932
1 changed files with 18 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

42
lib/system/widestrs.nim
View file

@ -119,36 +119,28 @@ proc `$`*(w: WideCString, estimate: int): string =
var i = 0 var i = 0
while w[i].int16 != 0'i16: while w[i].int16 != 0'i16:
var ch = w[i].int var ch = uint32(cast[uint16](w[i]))
inc i inc i
if ch >=% UNI_SUR_HIGH_START and ch <=% UNI_SUR_HIGH_END: if ch >= uint32(UNI_SUR_HIGH_START) and ch <= uint32(UNI_SUR_HIGH_END):
# If the 16 bits following the high surrogate are in the source buffer... # If the 16 bits following the high surrogate are in the source buffer...
let ch2 = w[i].int let ch2 = uint32(cast[uint16](w[i]))
# If it's a low surrogate, convert to UTF32: ch = (ch shl halfShift) + ch2 + halfBase
if ch2 >=% UNI_SUR_LOW_START and ch2 <=% UNI_SUR_LOW_END: inc i
ch = ((ch -% UNI_SUR_HIGH_START) shr halfShift) +%
(ch2 -% UNI_SUR_LOW_START) +% halfBase
inc i
if ch <=% 127: if ch < 0x80'u32:
result.add chr(ch) result.add chr(ch)
elif ch <=% 0x07FF: elif ch < 0x800'u32:
result.add chr((ch shr 6) or 0b110_00000) result.add chr((ch shr 6) or 0xc0)
result.add chr((ch and ones(6)) or 0b10_000000) result.add chr((ch and 0x3f) or 0x80)
elif ch <=% 0xFFFF: elif ch < 0x10000'u32:
result.add chr(ch shr 12 or 0b1110_0000) result.add chr((ch shr 12) or 0xe0)
result.add chr(ch shr 6 and ones(6) or 0b10_0000_00) result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr(ch and ones(6) or 0b10_0000_00) result.add chr((ch and 0x3f) or 0x80)
elif ch <=% 0x0010FFFF:
result.add chr(ch shr 18 or 0b1111_0000)
result.add chr(ch shr 12 and ones(6) or 0b10_0000_00)
result.add chr(ch shr 6 and ones(6) or 0b10_0000_00)
result.add chr(ch and ones(6) or 0b10_0000_00)
else: else:
# replacement char: result.add chr((ch shr 18) or 0xf0)
result.add chr(0xFFFD shr 12 or 0b1110_0000) result.add chr(((ch shr 12) and 0x3f) or 0x80)
result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00) result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr(0xFFFD and ones(6) or 0b10_0000_00) result.add chr((ch and 0x3f) or 0x80)
proc `$`*(s: WideCString): string = proc `$`*(s: WideCString): string =
result = s $ 80 result = s $ 80