jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed UTF-16 to UTF-8 conversion in widestrs.nim

Browse source 
the source of problem for issue #3228
also add test for entire range of valid UTF-16
and test for invalid UTF-16 sequence
This commit is contained in:
jangko 2015-08-21 10:43:31 +07:00
commit 7c757599f1
2 changed files with 55 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

26
lib/system/widestrs.nim
View file

@ -114,7 +114,7 @@ proc newWideCString*(s: cstring): WideCString =
proc newWideCString*(s: string): WideCString =
result = newWideCString(s, s.len)
proc `$`*(w: WideCString, estimate: int): string =
proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string =
result = newStringOfCap(estimate + estimate shr 2)
var i = 0
@ -124,9 +124,18 @@ proc `$`*(w: WideCString, estimate: int): string =
if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
# If the 16 bits following the high surrogate are in the source buffer...
let ch2 = int(cast[uint16](w[i]))
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
#invalid UTF-16
ch = replacement
elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
#invalid UTF-16
ch = replacement
if ch < 0x80:
result.add chr(ch)
elif ch < 0x800:
@ -136,11 +145,16 @@ proc `$`*(w: WideCString, estimate: int): string =
result.add chr((ch shr 12) or 0xe0)
result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr((ch and 0x3f) or 0x80)
else:
elif ch <= 0x10FFFF:
result.add chr((ch shr 18) or 0xf0)
result.add chr(((ch shr 12) and 0x3f) or 0x80)
result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr((ch and 0x3f) or 0x80)
else:
# replacement char(in case user give very large number):
result.add chr(0xFFFD shr 12 or 0b1110_0000)
result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
result.add chr(0xFFFD and ones(6) or 0b10_0000_00)
proc `$`*(s: WideCString): string =
result = s $ 80