cleaned up twchartoutf8 test
This commit is contained in:
parent
c56a1f8266
commit
dd2a0ec431
1 changed files with 91 additions and 87 deletions
|
|
@ -1,106 +1,110 @@
|
||||||
|
discard """
|
||||||
|
output: '''OK'''
|
||||||
|
"""
|
||||||
|
|
||||||
#assume WideCharToMultiByte always produce correct result
|
#assume WideCharToMultiByte always produce correct result
|
||||||
#windows only
|
#windows only
|
||||||
|
|
||||||
when not defined(windows):
|
when not defined(windows):
|
||||||
{.error: "windows only".}
|
echo "OK"
|
||||||
|
else:
|
||||||
{.push gcsafe.}
|
{.push gcsafe.}
|
||||||
|
|
||||||
const CP_UTF8 = 65001'i32
|
const CP_UTF8 = 65001'i32
|
||||||
|
|
||||||
type
|
type
|
||||||
LPBOOL = ptr int32
|
LPBOOL = ptr int32
|
||||||
LPWCSTR = ptr uint16
|
LPWCSTR = ptr uint16
|
||||||
|
|
||||||
proc WideCharToMultiByte*(CodePage: int32, dwFlags: int32,
|
|
||||||
lpWideCharStr: LPWCSTR, cchWideChar: int32,
|
|
||||||
lpMultiByteStr: cstring, cchMultiByte: int32,
|
|
||||||
lpDefaultChar: cstring, lpUsedDefaultChar: LPBOOL): int32{.
|
|
||||||
stdcall, dynlib: "kernel32", importc: "WideCharToMultiByte".}
|
|
||||||
|
|
||||||
{.pop.}
|
proc WideCharToMultiByte*(CodePage: int32, dwFlags: int32,
|
||||||
|
lpWideCharStr: LPWCSTR, cchWideChar: int32,
|
||||||
|
lpMultiByteStr: cstring, cchMultiByte: int32,
|
||||||
|
lpDefaultChar: cstring, lpUsedDefaultChar: LPBOOL): int32{.
|
||||||
|
stdcall, dynlib: "kernel32", importc: "WideCharToMultiByte".}
|
||||||
|
|
||||||
proc convertToUTF8(wc: WideCString, wclen: int32): string =
|
{.pop.}
|
||||||
let size = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
|
|
||||||
cstring(nil), 0'i32, cstring(nil), LPBOOL(nil))
|
|
||||||
result = newString(size)
|
|
||||||
let res = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
|
|
||||||
cstring(result), size, cstring(nil), LPBOOL(nil))
|
|
||||||
result[size] = chr(0)
|
|
||||||
assert size == res
|
|
||||||
|
|
||||||
proc testCP(wc: WideCString, lo, hi: int) =
|
|
||||||
var x = 0
|
|
||||||
let chunk = 1024
|
|
||||||
for i in lo..hi:
|
|
||||||
wc[x] = cast[Utf16Char](i)
|
|
||||||
if (x >= chunk) or (i >= hi):
|
|
||||||
wc[x] = Utf16Char(0)
|
|
||||||
var a = convertToUTF8(wc, int32(x))
|
|
||||||
var b = wc $ chunk
|
|
||||||
assert a == b
|
|
||||||
x = 0
|
|
||||||
inc x
|
|
||||||
|
|
||||||
proc testCP2(wc: WideCString, lo, hi: int) =
|
proc convertToUTF8(wc: WideCString, wclen: int32): string =
|
||||||
assert((lo >= 0x10000) and (hi <= 0x10FFFF))
|
let size = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
|
||||||
var x = 0
|
cstring(nil), 0'i32, cstring(nil), LPBOOL(nil))
|
||||||
let chunk = 1024
|
result = newString(size)
|
||||||
for i in lo..hi:
|
let res = WideCharToMultiByte(CP_UTF8, 0'i32, cast[LPWCSTR](addr(wc[0])), wclen,
|
||||||
let ch = i - 0x10000
|
cstring(result), size, cstring(nil), LPBOOL(nil))
|
||||||
let W1 = 0xD800 or (ch shr 10)
|
result[size] = chr(0)
|
||||||
let W2 = 0xDC00 or (0x3FF and ch)
|
doAssert size == res
|
||||||
wc[x] = cast[Utf16Char](W1)
|
|
||||||
wc[x+1] = cast[Utf16Char](W2)
|
|
||||||
inc(x, 2)
|
|
||||||
|
|
||||||
if (x >= chunk) or (i >= hi):
|
|
||||||
wc[x] = Utf16Char(0)
|
|
||||||
var a = convertToUTF8(wc, int32(x))
|
|
||||||
var b = wc $ chunk
|
|
||||||
assert a == b
|
|
||||||
x = 0
|
|
||||||
|
|
||||||
#RFC-2781 "UTF-16, an encoding of ISO 10646"
|
proc testCP(wc: WideCString, lo, hi: int) =
|
||||||
|
var x = 0
|
||||||
var wc: WideCString
|
let chunk = 1024
|
||||||
unsafeNew(wc, 1024 * 4 + 2)
|
for i in lo..hi:
|
||||||
|
wc[x] = cast[Utf16Char](i)
|
||||||
|
if (x >= chunk) or (i >= hi):
|
||||||
|
wc[x] = Utf16Char(0)
|
||||||
|
var a = convertToUTF8(wc, int32(x))
|
||||||
|
var b = wc $ chunk
|
||||||
|
doAssert a == b
|
||||||
|
x = 0
|
||||||
|
inc x
|
||||||
|
|
||||||
#U+0000 to U+D7FF
|
proc testCP2(wc: WideCString, lo, hi: int) =
|
||||||
#skip the U+0000
|
doAssert((lo >= 0x10000) and (hi <= 0x10FFFF))
|
||||||
wc.testCP(1, 0xD7FF)
|
var x = 0
|
||||||
|
let chunk = 1024
|
||||||
|
for i in lo..hi:
|
||||||
|
let ch = i - 0x10000
|
||||||
|
let W1 = 0xD800 or (ch shr 10)
|
||||||
|
let W2 = 0xDC00 or (0x3FF and ch)
|
||||||
|
wc[x] = cast[Utf16Char](W1)
|
||||||
|
wc[x+1] = cast[Utf16Char](W2)
|
||||||
|
inc(x, 2)
|
||||||
|
|
||||||
#U+E000 to U+FFFF
|
if (x >= chunk) or (i >= hi):
|
||||||
wc.testCP(0xE000, 0xFFFF)
|
wc[x] = Utf16Char(0)
|
||||||
|
var a = convertToUTF8(wc, int32(x))
|
||||||
|
var b = wc $ chunk
|
||||||
|
doAssert a == b
|
||||||
|
x = 0
|
||||||
|
|
||||||
#U+10000 to U+10FFFF
|
#RFC-2781 "UTF-16, an encoding of ISO 10646"
|
||||||
wc.testCP2(0x10000, 0x10FFFF)
|
|
||||||
|
|
||||||
#invalid UTF-16
|
var wc: WideCString
|
||||||
const
|
unsafeNew(wc, 1024 * 4 + 2)
|
||||||
b = "\xEF\xBF\xBD"
|
|
||||||
c = "\xEF\xBF\xBF"
|
|
||||||
|
|
||||||
wc[0] = cast[Utf16Char](0xDC00)
|
#U+0000 to U+D7FF
|
||||||
wc[1] = Utf16Char(0)
|
#skip the U+0000
|
||||||
var a = $wc
|
wc.testCP(1, 0xD7FF)
|
||||||
assert a == b
|
|
||||||
|
|
||||||
wc[0] = cast[Utf16Char](0xFFFF)
|
#U+E000 to U+FFFF
|
||||||
wc[1] = cast[Utf16Char](0xDC00)
|
wc.testCP(0xE000, 0xFFFF)
|
||||||
wc[2] = Utf16Char(0)
|
|
||||||
a = $wc
|
|
||||||
assert a == c & b
|
|
||||||
|
|
||||||
wc[0] = cast[Utf16Char](0xD800)
|
#U+10000 to U+10FFFF
|
||||||
wc[1] = Utf16Char(0)
|
wc.testCP2(0x10000, 0x10FFFF)
|
||||||
a = $wc
|
|
||||||
assert a == b
|
|
||||||
|
|
||||||
wc[0] = cast[Utf16Char](0xD800)
|
#invalid UTF-16
|
||||||
wc[1] = cast[Utf16Char](0xFFFF)
|
const
|
||||||
wc[2] = Utf16Char(0)
|
b = "\xEF\xBF\xBD"
|
||||||
a = $wc
|
c = "\xEF\xBF\xBF"
|
||||||
assert a == b & c
|
|
||||||
|
|
||||||
echo "OK"
|
wc[0] = cast[Utf16Char](0xDC00)
|
||||||
|
wc[1] = Utf16Char(0)
|
||||||
|
var a = $wc
|
||||||
|
doAssert a == b
|
||||||
|
|
||||||
|
wc[0] = cast[Utf16Char](0xFFFF)
|
||||||
|
wc[1] = cast[Utf16Char](0xDC00)
|
||||||
|
wc[2] = Utf16Char(0)
|
||||||
|
a = $wc
|
||||||
|
doAssert a == c & b
|
||||||
|
|
||||||
|
wc[0] = cast[Utf16Char](0xD800)
|
||||||
|
wc[1] = Utf16Char(0)
|
||||||
|
a = $wc
|
||||||
|
doAssert a == b
|
||||||
|
|
||||||
|
wc[0] = cast[Utf16Char](0xD800)
|
||||||
|
wc[1] = cast[Utf16Char](0xFFFF)
|
||||||
|
wc[2] = Utf16Char(0)
|
||||||
|
a = $wc
|
||||||
|
doAssert a == b & c
|
||||||
|
|
||||||
|
echo "OK"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue