cleaned up twchartoutf8 test

This commit is contained in:
Araq 2015-08-21 21:40:24 +02:00
commit dd2a0ec431

View file

@ -1,106 +1,110 @@
discard """
output: '''OK'''
"""
#assume WideCharToMultiByte always produce correct result #assume WideCharToMultiByte always produce correct result
#windows only #windows only
when not defined(windows): when not defined(windows):
{.error: "windows only".} echo "OK"
else:
{.push gcsafe.} {.push gcsafe.}
const CP_UTF8 = 65001'i32 const CP_UTF8 = 65001'i32
type type
LPBOOL = ptr int32 LPBOOL = ptr int32
LPWCSTR = ptr uint16 LPWCSTR = ptr uint16
proc WideCharToMultiByte*(CodePage: int32, dwFlags: int32,
lpWideCharStr: LPWCSTR, cchWideChar: int32,
lpMultiByteStr: cstring, cchMultiByte: int32,
lpDefaultChar: cstring, lpUsedDefaultChar: LPBOOL): int32{.
stdcall, dynlib: "kernel32", importc: "WideCharToMultiByte".}
{.pop.} proc WideCharToMultiByte*(CodePage: int32, dwFlags: int32,
lpWideCharStr: LPWCSTR, cchWideChar: int32,
lpMultiByteStr: cstring, cchMultiByte: int32,
lpDefaultChar: cstring, lpUsedDefaultChar: LPBOOL): int32{.
stdcall, dynlib: "kernel32", importc: "WideCharToMultiByte".}
proc convertToUTF8(wc: WideCString, wclen: int32): string = {.pop.}
let size = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
cstring(nil), 0'i32, cstring(nil), LPBOOL(nil))
result = newString(size)
let res = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
cstring(result), size, cstring(nil), LPBOOL(nil))
result[size] = chr(0)
assert size == res
proc testCP(wc: WideCString, lo, hi: int) =
var x = 0
let chunk = 1024
for i in lo..hi:
wc[x] = cast[Utf16Char](i)
if (x >= chunk) or (i >= hi):
wc[x] = Utf16Char(0)
var a = convertToUTF8(wc, int32(x))
var b = wc $ chunk
assert a == b
x = 0
inc x
proc testCP2(wc: WideCString, lo, hi: int) = proc convertToUTF8(wc: WideCString, wclen: int32): string =
assert((lo >= 0x10000) and (hi <= 0x10FFFF)) let size = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
var x = 0 cstring(nil), 0'i32, cstring(nil), LPBOOL(nil))
let chunk = 1024 result = newString(size)
for i in lo..hi: let res = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
let ch = i - 0x10000 cstring(result), size, cstring(nil), LPBOOL(nil))
let W1 = 0xD800 or (ch shr 10) result[size] = chr(0)
let W2 = 0xDC00 or (0x3FF and ch) doAssert size == res
wc[x] = cast[Utf16Char](W1)
wc[x+1] = cast[Utf16Char](W2)
inc(x, 2)
if (x >= chunk) or (i >= hi):
wc[x] = Utf16Char(0)
var a = convertToUTF8(wc, int32(x))
var b = wc $ chunk
assert a == b
x = 0
#RFC-2781 "UTF-16, an encoding of ISO 10646" proc testCP(wc: WideCString, lo, hi: int) =
var x = 0
var wc: WideCString let chunk = 1024
unsafeNew(wc, 1024 * 4 + 2) for i in lo..hi:
wc[x] = cast[Utf16Char](i)
if (x >= chunk) or (i >= hi):
wc[x] = Utf16Char(0)
var a = convertToUTF8(wc, int32(x))
var b = wc $ chunk
doAssert a == b
x = 0
inc x
#U+0000 to U+D7FF proc testCP2(wc: WideCString, lo, hi: int) =
#skip the U+0000 doAssert((lo >= 0x10000) and (hi <= 0x10FFFF))
wc.testCP(1, 0xD7FF) var x = 0
let chunk = 1024
for i in lo..hi:
let ch = i - 0x10000
let W1 = 0xD800 or (ch shr 10)
let W2 = 0xDC00 or (0x3FF and ch)
wc[x] = cast[Utf16Char](W1)
wc[x+1] = cast[Utf16Char](W2)
inc(x, 2)
#U+E000 to U+FFFF if (x >= chunk) or (i >= hi):
wc.testCP(0xE000, 0xFFFF) wc[x] = Utf16Char(0)
var a = convertToUTF8(wc, int32(x))
var b = wc $ chunk
doAssert a == b
x = 0
#U+10000 to U+10FFFF #RFC-2781 "UTF-16, an encoding of ISO 10646"
wc.testCP2(0x10000, 0x10FFFF)
#invalid UTF-16 var wc: WideCString
const unsafeNew(wc, 1024 * 4 + 2)
b = "\xEF\xBF\xBD"
c = "\xEF\xBF\xBF"
wc[0] = cast[Utf16Char](0xDC00) #U+0000 to U+D7FF
wc[1] = Utf16Char(0) #skip the U+0000
var a = $wc wc.testCP(1, 0xD7FF)
assert a == b
wc[0] = cast[Utf16Char](0xFFFF) #U+E000 to U+FFFF
wc[1] = cast[Utf16Char](0xDC00) wc.testCP(0xE000, 0xFFFF)
wc[2] = Utf16Char(0)
a = $wc
assert a == c & b
wc[0] = cast[Utf16Char](0xD800) #U+10000 to U+10FFFF
wc[1] = Utf16Char(0) wc.testCP2(0x10000, 0x10FFFF)
a = $wc
assert a == b
wc[0] = cast[Utf16Char](0xD800) #invalid UTF-16
wc[1] = cast[Utf16Char](0xFFFF) const
wc[2] = Utf16Char(0) b = "\xEF\xBF\xBD"
a = $wc c = "\xEF\xBF\xBF"
assert a == b & c
echo "OK" wc[0] = cast[Utf16Char](0xDC00)
wc[1] = Utf16Char(0)
var a = $wc
doAssert a == b
wc[0] = cast[Utf16Char](0xFFFF)
wc[1] = cast[Utf16Char](0xDC00)
wc[2] = Utf16Char(0)
a = $wc
doAssert a == c & b
wc[0] = cast[Utf16Char](0xD800)
wc[1] = Utf16Char(0)
a = $wc
doAssert a == b
wc[0] = cast[Utf16Char](0xD800)
wc[1] = cast[Utf16Char](0xFFFF)
wc[2] = Utf16Char(0)
a = $wc
doAssert a == b & c
echo "OK"