jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3442 from Dhertz/devel

Browse source 
Make sure the json module decodes UTF16 correctly
This commit is contained in:
Andreas Rumpf 2015-10-23 02:02:07 +02:00
commit 1c9f05ef13
1 changed files with 32 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

40
lib/pure/json.nim
View file

@ -203,6 +203,15 @@ proc handleHexChar(c: char, x: var int): bool =
of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
else: result = false # error else: result = false # error
proc parseEscapedUTF16(buf: cstring, pos: var int): int =
result = 0
#UTF-16 escape is always 4 bytes.
for _ in 0..3:
if handleHexChar(buf[pos], result):
inc(pos)
else:
return -1
proc parseString(my: var JsonParser): TokKind = proc parseString(my: var JsonParser): TokKind =
result = tkString result = tkString
var pos = my.bufpos + 1 var pos = my.bufpos + 1
@ -238,11 +247,22 @@ proc parseString(my: var JsonParser): TokKind =
inc(pos, 2) inc(pos, 2)
of 'u': of 'u':
inc(pos, 2) inc(pos, 2)
var r: int var r = parseEscapedUTF16(buf, pos)
if handleHexChar(buf[pos], r): inc(pos) if r < 0:
if handleHexChar(buf[pos], r): inc(pos) my.err = errInvalidToken
if handleHexChar(buf[pos], r): inc(pos) break
if handleHexChar(buf[pos], r): inc(pos) # Deal with surrogates
if (r and 0xfc00) == 0xd800:
if buf[pos] & buf[pos+1] != "\\u":
my.err = errInvalidToken
break
inc(pos, 2)
var s = parseEscapedUTF16(buf, pos)
if (s and 0xfc00) == 0xdc00 and s > 0:
r = 0x10000 + (((r - 0xd800) shl 10) or (s - 0xdc00))
else:
my.err = errInvalidToken
break
add(my.a, toUTF8(Rune(r))) add(my.a, toUTF8(Rune(r)))
else: else:
# don't bother with the error # don't bother with the error
@ -1200,11 +1220,15 @@ when isMainModule:
assert(false) assert(false)
except IndexError: assert(true) except IndexError: assert(true)
let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd" }""" let testJson = parseJson"""{ "a": [1, 2, 3, 4], "b": "asd", "c": "\ud83c\udf83", "d": "\u00E6"}"""
# nil passthrough # nil passthrough
assert(testJson{"doesnt_exist"}{"anything"}.isNil) assert(testJson{"doesnt_exist"}{"anything"}.isNil)
testJson{["c", "d"]} = %true testJson{["e", "f"]} = %true
assert(testJson["c"]["d"].bval) assert(testJson["e"]["f"].bval)
# make sure UTF-16 decoding works.
assert(testJson["c"].str == "🎃")
assert(testJson["d"].str == "æ")
# make sure no memory leek when parsing invalid string # make sure no memory leek when parsing invalid string
let startMemory = getOccupiedMem() let startMemory = getOccupiedMem()