jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

json module changes

Browse source
This commit is contained in:
Araq 2011-01-15 11:50:12 +01:00
commit ff0b0f6b6d
5 changed files with 680 additions and 658 deletions
Download patch file Download diff file Expand all files Collapse all files

841
lib/pure/json.nim Normal file → Executable file
View file

@ -1,25 +1,489 @@
# #
# #
# Nimrod's Runtime Library # Nimrod's Runtime Library
# (c) Copyright 2010 Andreas Rumpf, Dominik Picheta # (c) Copyright 2011 Andreas Rumpf, Dominik Picheta
# #
# See the file "copying.txt", included in this # See the file "copying.txt", included in this
# distribution, for details about the copyright. # distribution, for details about the copyright.
# #
import parsejson, streams, strutils ## This module implements a simple high performance `JSON`:idx:
## parser. JSON (JavaScript Object Notation) is a lightweight
## data-interchange format that is easy for humans to read and write
## (unlike XML). It is easy for machines to parse and generate.
## JSON is based on a subset of the JavaScript Programming Language,
## Standard ECMA-262 3rd Edition - December 1999.
import
hashes, strutils, lexbase, streams, unicode
type
TJsonEventKind* = enum ## enumation of all events that may occur when parsing
jsonError, ## an error ocurred during parsing
jsonEof, ## end of file reached
jsonString, ## a string literal
jsonNumber, ## a number literal
jsonTrue, ## the value ``true``
jsonFalse, ## the value ``false``
jsonNull, ## the value ``null``
jsonObjectStart, ## start of an object: the ``{`` token
jsonObjectEnd, ## end of an object: the ``}`` token
jsonArrayStart, ## start of an array: the ``[`` token
jsonArrayEnd ## start of an array: the ``]`` token
TTokKind = enum # must be synchronized with TJsonEventKind!
tkError,
tkEof,
tkString,
tkNumber,
tkTrue,
tkFalse,
tkNull,
tkCurlyLe,
tkCurlyRi,
tkBracketLe,
tkBracketRi,
tkColon,
tkComma
TJsonError* = enum ## enumeration that lists all errors that can occur
errNone, ## no error
errInvalidToken, ## invalid token
errStringExpected, ## string expected
errColonExpected, ## ``:`` expected
errCommaExpected, ## ``,`` expected
errBracketRiExpected, ## ``]`` expected
errCurlyRiExpected, ## ``}`` expected
errQuoteExpected, ## ``"`` or ``'`` expected
errEOC_Expected, ## ``*/`` expected
errEofExpected, ## EOF expected
errExprExpected ## expr expected
TParserState = enum
stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma,
stateExpectObjectComma, stateExpectColon, stateExpectValue
TJsonParser* = object of TBaseLexer ## the parser object.
a: string
tok: TTokKind
kind: TJsonEventKind
err: TJsonError
state: seq[TParserState]
filename: string
const
errorMessages: array [TJsonError, string] = [
"no error",
"invalid token",
"string expected",
"':' expected",
"',' expected",
"']' expected",
"'}' expected",
"'\"' or \"'\" expected",
"'*/' expected",
"EOF expected",
"expression expected"
]
tokToStr: array [TTokKind, string] = [
"invalid token",
"EOF",
"string literal",
"number literal",
"true",
"false",
"null",
"{", "}", "[", "]", ":", ","
]
proc open*(my: var TJsonParser, input: PStream, filename: string) =
## initializes the parser with an input stream. `Filename` is only used
## for nice error messages.
lexbase.open(my, input)
my.filename = filename
my.state = @[stateStart]
my.kind = jsonError
my.a = ""
proc close*(my: var TJsonParser) {.inline.} =
## closes the parser `my` and its associated input stream.
lexbase.close(my)
proc str*(my: TJsonParser): string {.inline.} =
## returns the character data for the events: ``jsonNumber``,
## ``jsonString``
assert(my.kind in {jsonNumber, jsonString})
return my.a
proc number*(my: TJsonParser): float {.inline.} =
## returns the number for the event: ``jsonNumber``
assert(my.kind == jsonNumber)
return parseFloat(my.a)
proc kind*(my: TJsonParser): TJsonEventKind {.inline.} =
## returns the current event type for the JSON parser
return my.kind
proc getColumn*(my: TJsonParser): int {.inline.} =
## get the current column the parser has arrived at.
result = getColNumber(my, my.bufPos)
proc getLine*(my: TJsonParser): int {.inline.} =
## get the current line the parser has arrived at.
result = my.linenumber
proc getFilename*(my: TJsonParser): string {.inline.} =
## get the filename of the file that the parser processes.
result = my.filename
proc errorMsg*(my: TJsonParser): string =
## returns a helpful error message for the event ``jsonError``
assert(my.kind == jsonError)
result = "$1($2, $3) Error: $4" % [
my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]]
proc errorMsgExpected*(my: TJsonParser, e: string): string =
## returns an error message "`e` expected" in the same format as the
## other error messages
result = "$1($2, $3) Error: $4" % [
my.filename, $getLine(my), $getColumn(my), e & " expected"]
proc handleHexChar(c: Char, x: var int): bool =
result = true # Success
case c
of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
else: result = false # error
proc parseString(my: var TJsonParser): TTokKind =
result = tkString
var pos = my.bufpos + 1
var buf = my.buf
while true:
case buf[pos]
of '\0':
my.err = errQuoteExpected
result = tkError
break
of '"':
inc(pos)
break
of '\\':
case buf[pos+1]
of '\\', '"', '\'', '/':
add(my.a, buf[pos+1])
inc(pos, 2)
of 'b':
add(my.a, '\b')
inc(pos, 2)
of 'f':
add(my.a, '\f')
inc(pos, 2)
of 'n':
add(my.a, '\L')
inc(pos, 2)
of 'r':
add(my.a, '\C')
inc(pos, 2)
of 't':
add(my.a, '\t')
inc(pos, 2)
of 'u':
inc(pos, 2)
var r: int
if handleHexChar(buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos)
add(my.a, toUTF8(TRune(r)))
else:
# don't bother with the error
add(my.a, buf[pos])
inc(pos)
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
add(my.a, '\c')
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
add(my.a, '\L')
else:
add(my.a, buf[pos])
inc(pos)
my.bufpos = pos # store back
proc skip(my: var TJsonParser) =
var pos = my.bufpos
var buf = my.buf
while true:
case buf[pos]
of '/':
if buf[pos+1] == '/':
# skip line comment:
inc(pos, 2)
while true:
case buf[pos]
of '\0':
break
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
break
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
break
else:
inc(pos)
elif buf[pos+1] == '*':
# skip long comment:
inc(pos, 2)
while true:
case buf[pos]
of '\0':
my.err = errEOC_Expected
break
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
of '*':
inc(pos)
if buf[pos] == '/':
inc(pos)
break
else:
inc(pos)
else:
break
of ' ', '\t':
Inc(pos)
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
else:
break
my.bufpos = pos
proc parseNumber(my: var TJsonParser) =
var pos = my.bufpos
var buf = my.buf
if buf[pos] == '-':
add(my.a, '-')
inc(pos)
if buf[pos] == '.':
add(my.a, "0.")
inc(pos)
else:
while buf[pos] in Digits:
add(my.a, buf[pos])
inc(pos)
if buf[pos] == '.':
add(my.a, '.')
inc(pos)
# digits after the dot:
while buf[pos] in Digits:
add(my.a, buf[pos])
inc(pos)
if buf[pos] in {'E', 'e'}:
add(my.a, buf[pos])
inc(pos)
if buf[pos] in {'+', '-'}:
add(my.a, buf[pos])
inc(pos)
while buf[pos] in Digits:
add(my.a, buf[pos])
inc(pos)
my.bufpos = pos
proc parseName(my: var TJsonParser) =
var pos = my.bufpos
var buf = my.buf
if buf[pos] in IdentStartChars:
while buf[pos] in IdentChars:
add(my.a, buf[pos])
inc(pos)
my.bufpos = pos
proc getTok(my: var TJsonParser): TTokKind =
setLen(my.a, 0)
skip(my) # skip whitespace, comments
case my.buf[my.bufpos]
of '-', '.', '0'..'9':
parseNumber(my)
result = tkNumber
of '"':
result = parseString(my)
of '[':
inc(my.bufpos)
result = tkBracketLe
of '{':
inc(my.bufpos)
result = tkCurlyLe
of ']':
inc(my.bufpos)
result = tkBracketRi
of '}':
inc(my.bufpos)
result = tkCurlyRi
of ',':
inc(my.bufpos)
result = tkComma
of ':':
inc(my.bufpos)
result = tkColon
of '\0':
result = tkEof
of 'a'..'z', 'A'..'Z', '_':
parseName(my)
case my.a
of "null": result = tkNull
of "true": result = tkTrue
of "false": result = tkFalse
else: result = tkError
else:
inc(my.bufpos)
result = tkError
my.tok = result
proc next*(my: var TJsonParser) =
## retrieves the first/next event. This controls the parser.
var tk = getTok(my)
var i = my.state.len-1
# the following code is a state machine. If we had proper coroutines,
# the code could be much simpler.
case my.state[i]
of stateEof:
if tk == tkEof:
my.kind = jsonEof
else:
my.kind = jsonError
my.err = errEofExpected
of stateStart:
# tokens allowed?
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state[i] = stateEof # expect EOF next!
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state.add(stateArray) # we expect any
my.kind = jsonArrayStart
of tkCurlyLe:
my.state.add(stateObject)
my.kind = jsonObjectStart
of tkEof:
my.kind = jsonEof
else:
my.kind = jsonError
my.err = errEofExpected
of stateObject:
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state.add(stateExpectColon)
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state.add(stateExpectColon)
my.state.add(stateArray)
my.kind = jsonArrayStart
of tkCurlyLe:
my.state.add(stateExpectColon)
my.state.add(stateObject)
my.kind = jsonObjectStart
of tkCurlyRi:
my.kind = jsonObjectEnd
discard my.state.pop()
else:
my.kind = jsonError
my.err = errCurlyRiExpected
of stateArray:
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state.add(stateExpectArrayComma) # expect value next!
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state.add(stateExpectArrayComma)
my.state.add(stateArray)
my.kind = jsonArrayStart
of tkCurlyLe:
my.state.add(stateExpectArrayComma)
my.state.add(stateObject)
my.kind = jsonObjectStart
of tkBracketRi:
my.kind = jsonArrayEnd
discard my.state.pop()
else:
my.kind = jsonError
my.err = errBracketRiExpected
of stateExpectArrayComma:
case tk
of tkComma:
discard my.state.pop()
next(my)
of tkBracketRi:
my.kind = jsonArrayEnd
discard my.state.pop() # pop stateExpectArrayComma
discard my.state.pop() # pop stateArray
else:
my.kind = jsonError
my.err = errBracketRiExpected
of stateExpectObjectComma:
case tk
of tkComma:
discard my.state.pop()
next(my)
of tkCurlyRi:
my.kind = jsonObjectEnd
discard my.state.pop() # pop stateExpectObjectComma
discard my.state.pop() # pop stateObject
else:
my.kind = jsonError
my.err = errCurlyRiExpected
of stateExpectColon:
case tk
of tkColon:
my.state[i] = stateExpectValue
next(my)
else:
my.kind = jsonError
my.err = errColonExpected
of stateExpectValue:
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state[i] = stateExpectObjectComma
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state[i] = stateExpectObjectComma
my.state.add(stateArray)
my.kind = jsonArrayStart
of tkCurlyLe:
my.state[i] = stateExpectObjectComma
my.state.add(stateObject)
my.kind = jsonObjectStart
else:
my.kind = jsonError
my.err = errExprExpected
# ------------- higher level interface ---------------------------------------
type type
TJsonNodeKind* = enum TJsonNodeKind* = enum ## possible JSON node types
JString,
JNumber,
JBool,
JNull, JNull,
JBool,
JNumber,
JString,
JObject, JObject,
JArray JArray
PJsonNode* = ref TJsonNode PJsonNode* = ref TJsonNode ## JSON node
TJsonNode* = object TJsonNode {.final, pure.} = object
case kind*: TJsonNodeKind case kind*: TJsonNodeKind
of JString: of JString:
str*: String str*: String
@ -30,18 +494,99 @@ type
of JNull: of JNull:
nil nil
of JObject: of JObject:
fields*: seq[tuple[key: string, obj: PJsonNode]] fields*: seq[tuple[key: string, val: PJsonNode]]
of JArray: of JArray:
elems*: seq[PJsonNode] elems*: seq[PJsonNode]
EJsonParsingError* = object of EBase EJsonParsingError* = object of EBase
proc raiseParseErr(parser: TJsonParser, msg: string, line = True) = proc raiseParseErr(p: TJsonParser, msg: string) =
if line: raise newException(EJsonParsingError, errorMsgExpected(p, msg))
raise newException(EJsonParsingError, "(" & $parser.getLine & ", " &
$parser.getColumn & ") " & msg) proc newJString*(s: String): PJsonNode =
else: ## Creates a new `JString PJsonNode`.
raise newException(EJsonParsingError, msg) new(result)
result.kind = JString
result.str = s
proc newJNumber*(n: Float): PJsonNode =
## Creates a new `JNumber PJsonNode`.
new(result)
result.kind = JNumber
result.num = n
proc newJBool*(b: Bool): PJsonNode =
## Creates a new `JBool PJsonNode`.
new(result)
result.kind = JBool
result.bval = b
proc newJNull*(): PJsonNode =
## Creates a new `JNull PJsonNode`.
new(result)
proc newJObject*(): PJsonNode =
## Creates a new `JObject PJsonNode`
new(result)
result.kind = JObject
result.fields = @[]
proc newJArray*(): PJsonNode =
## Creates a new `JArray PJsonNode`
new(result)
result.kind = JArray
result.elems = @[]
proc len*(n: PJsonNode): int =
## If `n` is a `JArray`, it returns the number of elements.
## If `n` is a `JObject`, it returns the number of pairs.
## Else it returns 0.
case n.kind
of JArray: result = n.elems.len
of JObject: result = n.fields.len
else: nil
proc `[]`*(node: PJsonNode, name: String): PJsonNode =
## Gets a field from a `JObject`.
assert(node.kind == JObject)
for key, item in items(node.fields):
if key == name:
return item
return nil
proc `[]`*(node: PJsonNode, index: Int): PJsonNode =
## Gets the node at `index` in an Array.
assert(node.kind == JArray)
return node.elems[index]
proc existsKey*(node: PJsonNode, key: String): Bool =
## Checks if `key` exists in `node`.
assert(node.kind == JObject)
for k, item in items(node.fields):
if k == key: return True
proc add*(father, child: PJsonNode) =
## Adds `child` to a JArray node `father`.
assert father.kind == JArray
father.elems.add(child)
proc add*(obj: PJsonNode, key: string, val: PJsonNode) =
## Adds ``(key, val)`` pair to the JObject node `obj`. For speed
## reasons no check for duplicate keys is performed!
## But ``[]=`` performs the check.
assert obj.kind == JObject
obj.fields.add((key, val))
proc `[]=`*(obj: PJsonNode, key: String, val: PJsonNode) =
## Sets a field from a `JObject`. Performs a check for duplicate keys.
assert(obj.kind == JObject)
for i in 0..obj.fields.len-1:
if obj.fields[i].key == key:
obj.fields[i].val = val
return
obj.fields.add((key, val))
# ------------- pretty printing ----------------------------------------------
proc indent(s: var string, i: int) = proc indent(s: var string, i: int) =
s.add(repeatChar(i)) s.add(repeatChar(i))
@ -53,7 +598,24 @@ proc newIndent(curr, indent: int, ml: bool): Int =
proc nl(s: var string, ml: bool) = proc nl(s: var string, ml: bool) =
if ml: s.add("\n") if ml: s.add("\n")
proc toPretty(result: var string, node: PJsonNode, indent = 2, ml = True, lstArr = False, currIndent = 0) = proc escapeJson*(s: string): string =
## Converts a string `s` to its JSON representation.
result = "\""
for x in runes(s):
var r = int(x)
if r >= 32 and r <= 127:
var c = chr(r)
case c
of '"': result.add("\\\"")
of '\\': result.add("\\\\")
else: result.add(c)
else:
result.add("\\u")
result.add(toHex(r, 4))
result.add("\"")
proc toPretty(result: var string, node: PJsonNode, indent = 2, ml = True,
lstArr = False, currIndent = 0) =
case node.kind case node.kind
of JObject: of JObject:
if currIndent != 0 and not lstArr: result.nl(ml) if currIndent != 0 and not lstArr: result.nl(ml)
@ -64,16 +626,18 @@ proc toPretty(result: var string, node: PJsonNode, indent = 2, ml = True, lstArr
if i > 0: if i > 0:
result.add(", ") result.add(", ")
result.nl(ml) # New Line result.nl(ml) # New Line
var (key, item) = node.fields[i] # Need to indent more than {
result.indent(newIndent(currIndent, indent, ml)) # Need to indent more than { result.indent(newIndent(currIndent, indent, ml))
result.add("\"" & key & "\": ") result.add(escapeJson(node.fields[i].key))
toPretty(result, item, indent, ml, False, newIndent(currIndent, indent, ml)) result.add(": ")
toPretty(result, node.fields[i].val, indent, ml, False,
newIndent(currIndent, indent, ml))
result.nl(ml) result.nl(ml)
result.indent(currIndent) # indent the same as { result.indent(currIndent) # indent the same as {
result.add("}") result.add("}")
of JString: of JString:
if lstArr: result.indent(currIndent) if lstArr: result.indent(currIndent)
result.add("\"" & node.str & "\"") result.add(escapeJson(node.str))
of JNumber: of JNumber:
if lstArr: result.indent(currIndent) if lstArr: result.indent(currIndent)
result.add($node.num) result.add($node.num)
@ -99,167 +663,106 @@ proc toPretty(result: var string, node: PJsonNode, indent = 2, ml = True, lstArr
result.add("null") result.add("null")
proc pretty*(node: PJsonNode, indent = 2): String = proc pretty*(node: PJsonNode, indent = 2): String =
## Converts a `PJsonNode` to its JSON Representation, with indentation and ## Converts `node` to its JSON Representation, with indentation and
## on multiple lines. ## on multiple lines.
result = "" result = ""
toPretty(result, node, indent) toPretty(result, node, indent)
proc `$`*(node: PJsonNode): String = proc `$`*(node: PJsonNode): String =
## Converts a `PJsonNode` to its JSON Representation on one line. ## Converts `node` to its JSON Representation on one line.
result = "" result = ""
toPretty(result, node, 1, False) toPretty(result, node, 1, False)
proc newJString*(s: String): PJsonNode = proc eat(p: var TJsonParser, tok: TTokKind) =
## Creates a new `JString PJsonNode` if p.tok == tok: discard getTok(p)
new(result) else: raiseParseErr(p, tokToStr[tok])
result.kind = JString
result.str = s
proc newJNumber*(n: Float): PJsonNode = proc parseJson(p: var TJsonParser): PJsonNode =
## Creates a new `JNumber PJsonNode` ## Parses JSON from a JSON Parser `p`.
new(result) case p.tok
result.kind = JNumber of tkString:
result.num = n result = newJString(p.a)
discard getTok(p)
proc newJBool*(b: Bool): PJsonNode = of tkNumber:
## Creates a new `JBool PJsonNode` result = newJNumber(parseFloat(p.a))
new(result) discard getTok(p)
result.kind = JBool of tkTrue:
result.bval = b result = newJBool(true)
discard getTok(p)
proc newJNull*(): PJsonNode = of tkFalse:
## Creates a new `JNull PJsonNode` result = newJBool(false)
new(result) discard getTok(p)
result.kind = JNull of tkNull:
proc newJObject*(f: seq[tuple[key: string, obj: PJsonNode]]): PJsonNode =
## Creates a new `JObject PJsonNode`
new(result)
result.kind = JObject
result.fields = f
proc newJArray*(a: seq[PJsonNode]): PJsonNode =
## Creates a new `JArray PJsonNode`
new(result)
result.kind = JArray
result.elems = a
proc parseOther(parser: var TJsonParser): PJsonNode =
# Parses a *single* node which is not an Array or Object.
new(result)
case parser.kind
of jsonString:
result = newJString(parser.str())
of jsonNumber:
result = newJNumber(parser.number())
of jsonTrue, jsonFalse:
result = newJBool((parser.kind == jsonTrue))
of jsonNull:
result = newJNull() result = newJNull()
of jsonError: discard getTok(p)
parser.raiseParseErr(parser.errorMsg(), false) of tkCurlyLe:
else: parser.raiseParseErr("Unexpected " & $parser.kind & " here.") result = newJObject()
discard getTok(p)
while p.tok != tkCurlyRi:
if p.tok != tkString:
raiseParseErr(p, "string literal as key expected")
var key = p.a
discard getTok(p)
eat(p, tkColon)
var val = parseJson(p)
result[key] = val
if p.tok != tkComma: break
discard getTok(p)
eat(p, tkCurlyRi)
of tkBracketLe:
result = newJArray()
discard getTok(p)
while p.tok != tkBracketRi:
result.add(parseJson(p))
if p.tok != tkComma: break
discard getTok(p)
eat(p, tkBracketRi)
of tkError, tkCurlyRi, tkBracketRi, tkColon, tkComma, tkEof:
raiseParseErr(p, "{")
proc parseObj(parser: var TJSonParser, oStart: Bool = False): PJsonNode proc parseJson*(s: PStream, filename: string): PJsonNode =
## Parses from a stream `s` into a `PJsonNode`. `filename` is only needed
## for nice error messages.
var p: TJsonParser
p.open(s, filename)
discard getTok(p) # read first token
result = p.parseJson()
p.close()
proc parseArray(parser: var TJsonParser): PJsonNode = proc parseJson*(buffer: string): PJsonNode =
result = newJArray(@[]) ## Parses JSON from `buffer`.
while True: result = parseJson(newStringStream(buffer), "input")
parser.next()
case parser.kind proc parseFile*(filename: string): PJsonNode =
of jsonArrayStart: ## Parses `file` into a `PJsonNode`.
# Array in an array. var stream = newFileStream(filename, fmRead)
var arr = parser.parseArray() if stream == nil:
result.elems.add(arr) raise newException(EIO, "cannot read from file: " & filename)
of jsonArrayEnd: result = parseJson(stream, filename)
return
of jsonString, jsonNumber, jsonTrue, jsonFalse, jsonNull: when false:
var other = parser.parseOther() import os
result.elems.add(other) var s = newFileStream(ParamStr(1), fmRead)
of jsonObjectStart: if s == nil: quit("cannot open the file" & ParamStr(1))
var obj = parser.parseObj(True) var x: TJsonParser
result.elems.add(obj) open(x, s, ParamStr(1))
of jsonObjectEnd: parser.raiseParseErr("Unexpected }") while true:
of jsonEof: parser.raiseParseErr("Unexpected EOF.") next(x)
of jsonError: parser.raiseParseErr(parser.errorMsg(), false) case x.kind
proc parseObj(parser: var TJSonParser, oStart: Bool = False): PJsonNode =
var key = ""
var objStarted = oStart
result = newJObject(@[])
while True:
parser.next()
case parser.kind
of jsonError: of jsonError:
parser.raiseParseErr(parser.errorMsg(), false) Echo(x.errorMsg())
break break
of jsonEof: break of jsonEof: break
of jsonString, jsonNumber, jsonTrue, jsonFalse, jsonNull: of jsonString, jsonNumber: echo(x.str)
if parser.kind == jsonString and (key == "" and objStarted): of jsonTrue: Echo("!TRUE")
key = parser.str() of jsonFalse: Echo("!FALSE")
elif key == "": of jsonNull: Echo("!NULL")
parser.raiseParseErr("Expected object or array.") of jsonObjectStart: Echo("{")
else: of jsonObjectEnd: Echo("}")
var obj = parser.parseOther() of jsonArrayStart: Echo("[")
result.fields.add((key, obj)) of jsonArrayEnd: Echo("]")
key = ""
of jsonObjectStart:
objStarted = True
if key != "":
# Make sure that parseObj knows that the object has been started
var obj = parser.parseObj(True)
result.fields.add((key, obj))
key = ""
of jsonObjectEnd: return
of jsonArrayStart:
var arr = parser.parseArray()
if key != "":
result.fields.add((key, arr))
key = ""
else:
return arr
of jsonArrayEnd: parser.raiseParseErr("Unexpected ]")
proc parse*(json: string): PJsonNode =
## Parses string `json` into a `PJsonNode`.
var stream = newStringStream(json)
var parser: TJsonParser
parser.open(stream, "")
result = parser.parseObj()
parser.close() close(x)
proc parseFile*(file: String): PJsonNode =
## Parses `file` into a `PJsonNode`.
var stream = newFileStream(file, fmRead)
var parser: TJsonParser
parser.open(stream, file)
result = parser.parseObj()
parser.close()
proc `[]`*(node: PJsonNode, name: String): PJsonNode =
## Gets a field from a `JObject`.
assert(node.kind == JObject)
for key, item in items(node.fields):
if key == name:
return item
return nil
proc `[]`*(node: PJsonNode, index: Int): PJsonNode =
## Gets the node at `index` in an Array.
assert(node.kind == JArray)
return node.elems[index]
proc existsKey*(node: PJsonNode, name: String): Bool =
## Checks if key `name` exists in `node`.
assert(node.kind == JObject)
for key, item in items(node.fields):
if key == name:
return True
return False
# { "json": 5 } # { "json": 5 }
# To get that we shall use, obj["json"] # To get that we shall use, obj["json"]
@ -267,12 +770,10 @@ proc existsKey*(node: PJsonNode, name: String): Bool =
when isMainModule: when isMainModule:
#var node = parse("{ \"test\": null }") #var node = parse("{ \"test\": null }")
#echo(node.existsKey("test56")) #echo(node.existsKey("test56"))
var parsed = parseFile("test2.json") var parsed = parseFile("tests/testdata/jsontest.json")
echo(parsed["commits"][0]["author"]["username"].str) echo(parsed)
echo() echo()
echo(pretty(parsed, 2)) echo(pretty(parsed, 2))
echo()
echo(parsed)
discard """ discard """
while true: while true:

484
lib/pure/parsejson.nim
View file

@ -1,484 +0,0 @@
#
#
# Nimrod's Runtime Library
# (c) Copyright 2010 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## This module implements a simple high performance `JSON`:idx:
## parser. JSON (JavaScript Object Notation) is a lightweight
## data-interchange format that is easy for humans to read and write
## (unlike XML). It is easy for machines to parse and generate.
## JSON is based on a subset of the JavaScript Programming Language,
## Standard ECMA-262 3rd Edition - December 1999.
import
hashes, strutils, lexbase, streams, unicode
type
TJsonEventKind* = enum ## enumation of all events that may occur when parsing
jsonError, ## an error ocurred during parsing
jsonEof, ## end of file reached
jsonString, ## a string literal
jsonNumber, ## a number literal
jsonTrue, ## the value ``true``
jsonFalse, ## the value ``false``
jsonNull, ## the value ``null``
jsonObjectStart, ## start of an object: the ``{`` token
jsonObjectEnd, ## end of an object: the ``}`` token
jsonArrayStart, ## start of an array: the ``[`` token
jsonArrayEnd ## start of an array: the ``]`` token
TTokKind = enum # must be synchronized with TJsonEventKind!
tkError,
tkEof,
tkString,
tkNumber,
tkTrue,
tkFalse,
tkNull,
tkCurlyLe,
tkCurlyRi,
tkBracketLe,
tkBracketRi,
tkColon,
tkComma
TJsonError* = enum ## enumeration that lists all errors that can occur
errNone, ## no error
errInvalidToken, ## invalid token
errStringExpected, ## string expected
errColonExpected, ## ``:`` expected
errCommaExpected, ## ``,`` expected
errBracketRiExpected, ## ``]`` expected
errCurlyRiExpected, ## ``}`` expected
errQuoteExpected, ## ``"`` or ``'`` expected
errEOC_Expected, ## ``*/`` expected
errEofExpected, ## EOF expected
errExprExpected ## expr expected
TParserState = enum
stateEof, stateStart, stateObject, stateArray, stateExpectArrayComma,
stateExpectObjectComma, stateExpectColon, stateExpectValue
TJsonParser* = object of TBaseLexer ## the parser object.
a: string
kind: TJsonEventKind
err: TJsonError
state: seq[TParserState]
filename: string
const
errorMessages: array [TJsonError, string] = [
"no error",
"invalid token",
"string expected",
"':' expected",
"',' expected",
"']' expected",
"'}' expected",
"'\"' or \"'\" expected",
"'*/' expected",
"EOF expected",
"expression expected"
]
proc open*(my: var TJsonParser, input: PStream, filename: string) =
## initializes the parser with an input stream. `Filename` is only used
## for nice error messages.
lexbase.open(my, input)
my.filename = filename
my.state = @[stateStart]
my.kind = jsonError
my.a = ""
proc close*(my: var TJsonParser) {.inline.} =
## closes the parser `my` and its associated input stream.
lexbase.close(my)
proc str*(my: TJsonParser): string {.inline.} =
## returns the character data for the events: ``jsonNumber``,
## ``jsonString``
assert(my.kind in {jsonNumber, jsonString})
return my.a
proc number*(my: TJsonParser): float {.inline.} =
## returns the number for the event: ``jsonNumber``
assert(my.kind == jsonNumber)
return parseFloat(my.a)
proc kind*(my: TJsonParser): TJsonEventKind {.inline.} =
## returns the current event type for the JSON parser
return my.kind
proc getColumn*(my: TJsonParser): int {.inline.} =
## get the current column the parser has arrived at.
result = getColNumber(my, my.bufPos)
proc getLine*(my: TJsonParser): int {.inline.} =
## get the current line the parser has arrived at.
result = my.linenumber
proc getFilename*(my: TJsonParser): string {.inline.} =
## get the filename of the file that the parser processes.
result = my.filename
proc errorMsg*(my: TJsonParser): string =
## returns a helpful error message for the event ``jsonError``
assert(my.kind == jsonError)
result = "$1($2, $3) Error: $4" % [
my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]]
proc errorMsgExpected*(my: TJsonParser, e: string): string =
## returns an error message "`e` expected" in the same format as the
## other error messages
result = "$1($2, $3) Error: $4" % [
my.filename, $getLine(my), $getColumn(my), e & " expected"]
proc handleHexChar(c: Char, x: var int): bool =
result = true # Success
case c
of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
else: result = false # error
proc parseString(my: var TJsonParser): TTokKind =
result = tkString
var pos = my.bufpos + 1
var buf = my.buf
while true:
case buf[pos]
of '\0':
my.err = errQuoteExpected
result = tkError
break
of '"':
inc(pos)
break
of '\\':
case buf[pos+1]
of '\\', '"', '\'', '/':
add(my.a, buf[pos+1])
inc(pos, 2)
of 'b':
add(my.a, '\b')
inc(pos, 2)
of 'f':
add(my.a, '\f')
inc(pos, 2)
of 'n':
add(my.a, '\L')
inc(pos, 2)
of 'r':
add(my.a, '\C')
inc(pos, 2)
of 't':
add(my.a, '\t')
inc(pos, 2)
of 'u':
inc(pos, 2)
var r: int
if handleHexChar(buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos)
if handleHexChar(buf[pos], r): inc(pos)
add(my.a, toUTF8(TRune(r)))
else:
# don't bother with the error
add(my.a, buf[pos])
inc(pos)
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
add(my.a, '\c')
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
add(my.a, '\L')
else:
add(my.a, buf[pos])
inc(pos)
my.bufpos = pos # store back
proc skip(my: var TJsonParser) =
var pos = my.bufpos
var buf = my.buf
while true:
case buf[pos]
of '/':
if buf[pos+1] == '/':
# skip line comment:
inc(pos, 2)
while true:
case buf[pos]
of '\0':
break
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
break
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
break
else:
inc(pos)
elif buf[pos+1] == '*':
# skip long comment:
inc(pos, 2)
while true:
case buf[pos]
of '\0':
my.err = errEOC_Expected
break
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
of '*':
inc(pos)
if buf[pos] == '/':
inc(pos)
break
else:
inc(pos)
else:
break
of ' ', '\t':
Inc(pos)
of '\c':
pos = lexbase.HandleCR(my, pos)
buf = my.buf
of '\L':
pos = lexbase.HandleLF(my, pos)
buf = my.buf
else:
break
my.bufpos = pos
proc parseNumber(my: var TJsonParser) =
var pos = my.bufpos
var buf = my.buf
if buf[pos] == '-':
add(my.a, '-')
inc(pos)
if buf[pos] == '.':
add(my.a, "0.")
inc(pos)
else:
while buf[pos] in Digits:
add(my.a, buf[pos])
inc(pos)
if buf[pos] == '.':
add(my.a, '.')
inc(pos)
# digits after the dot:
while buf[pos] in Digits:
add(my.a, buf[pos])
inc(pos)
if buf[pos] in {'E', 'e'}:
add(my.a, buf[pos])
inc(pos)
if buf[pos] in {'+', '-'}:
add(my.a, buf[pos])
inc(pos)
while buf[pos] in Digits:
add(my.a, buf[pos])
inc(pos)
my.bufpos = pos
proc parseName(my: var TJsonParser) =
var pos = my.bufpos
var buf = my.buf
if buf[pos] in IdentStartChars:
while buf[pos] in IdentChars:
add(my.a, buf[pos])
inc(pos)
my.bufpos = pos
proc getTok(my: var TJsonParser): TTokKind =
setLen(my.a, 0)
skip(my) # skip whitespace, comments
case my.buf[my.bufpos]
of '-', '.', '0'..'9':
parseNumber(my)
result = tkNumber
of '"':
result = parseString(my)
of '[':
inc(my.bufpos)
result = tkBracketLe
of '{':
inc(my.bufpos)
result = tkCurlyLe
of ']':
inc(my.bufpos)
result = tkBracketRi
of '}':
inc(my.bufpos)
result = tkCurlyRi
of ',':
inc(my.bufpos)
result = tkComma
of ':':
inc(my.bufpos)
result = tkColon
of '\0':
result = tkEof
of 'a'..'z', 'A'..'Z', '_':
parseName(my)
case my.a
of "null": result = tkNull
of "true": result = tkTrue
of "false": result = tkFalse
else: result = tkError
else:
inc(my.bufpos)
result = tkError
proc next*(my: var TJsonParser) =
## retrieves the first/next event. This controls the parser.
var tk = getTok(my)
var i = my.state.len-1
# the following code is a state machine. If we had proper coroutines,
# the code could be much simpler.
case my.state[i]
of stateEof:
if tk == tkEof:
my.kind = jsonEof
else:
my.kind = jsonError
my.err = errEofExpected
of stateStart:
# tokens allowed?
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state[i] = stateEof # expect EOF next!
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state.add(stateArray) # we expect any
my.kind = jsonArrayStart
of tkCurlyLe:
my.state.add(stateObject)
my.kind = jsonObjectStart
of tkEof:
my.kind = jsonEof
else:
my.kind = jsonError
my.err = errEofExpected
of stateObject:
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state.add(stateExpectColon)
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state.add(stateExpectColon)
my.state.add(stateArray)
my.kind = jsonArrayStart
of tkCurlyLe:
my.state.add(stateExpectColon)
my.state.add(stateObject)
my.kind = jsonObjectStart
of tkCurlyRi:
my.kind = jsonObjectEnd
discard my.state.pop()
else:
my.kind = jsonError
my.err = errCurlyRiExpected
of stateArray:
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state.add(stateExpectArrayComma) # expect value next!
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state.add(stateExpectArrayComma)
my.state.add(stateArray)
my.kind = jsonArrayStart
of tkCurlyLe:
my.state.add(stateExpectArrayComma)
my.state.add(stateObject)
my.kind = jsonObjectStart
of tkBracketRi:
my.kind = jsonArrayEnd
discard my.state.pop()
else:
my.kind = jsonError
my.err = errBracketRiExpected
of stateExpectArrayComma:
case tk
of tkComma:
discard my.state.pop()
next(my)
of tkBracketRi:
my.kind = jsonArrayEnd
discard my.state.pop() # pop stateExpectArrayComma
discard my.state.pop() # pop stateArray
else:
my.kind = jsonError
my.err = errBracketRiExpected
of stateExpectObjectComma:
case tk
of tkComma:
discard my.state.pop()
next(my)
of tkCurlyRi:
my.kind = jsonObjectEnd
discard my.state.pop() # pop stateExpectObjectComma
discard my.state.pop() # pop stateObject
else:
my.kind = jsonError
my.err = errCurlyRiExpected
of stateExpectColon:
case tk
of tkColon:
my.state[i] = stateExpectValue
next(my)
else:
my.kind = jsonError
my.err = errColonExpected
of stateExpectValue:
case tk
of tkString, tkNumber, tkTrue, tkFalse, tkNull:
my.state[i] = stateExpectObjectComma
my.kind = TJsonEventKind(ord(tk))
of tkBracketLe:
my.state[i] = stateExpectObjectComma
my.state.add(stateArray)
my.kind = jsonArrayStart
of tkCurlyLe:
my.state[i] = stateExpectObjectComma
my.state.add(stateObject)
my.kind = jsonObjectStart
else:
my.kind = jsonError
my.err = errExprExpected
when isMainModule:
import os
var s = newFileStream(ParamStr(1), fmRead)
if s == nil: quit("cannot open the file" & ParamStr(1))
var x: TJsonParser
open(x, s, ParamStr(1))
while true:
next(x)
case x.kind
of jsonError:
Echo(x.errorMsg())
break
of jsonEof: break
of jsonString, jsonNumber: echo(x.str)
of jsonTrue: Echo("!TRUE")
of jsonFalse: Echo("!FALSE")
of jsonNull: Echo("!NULL")
of jsonObjectStart: Echo("{")
of jsonObjectEnd: Echo("}")
of jsonArrayStart: Echo("[")
of jsonArrayEnd: Echo("]")
close(x)

6
tests/testdata/jsontest.json vendored
View file

@ -1,11 +1,11 @@
// Simple JSON test file // Simple JSON test file
// (c) 2009 Andreas Rumpf // (c) 2011 Andreas Rumpf
/* a long comment */ /* a long comment */
{ {
"key1": null, "key1": null,
{1: 2, {}: 4}: 12, "obj": {"1": 2, "5": 4},
"key2": [ "key2": [
{}, {},
@ -19,7 +19,7 @@
] , ] ,
"key3": false "keyÄÖöoßß": false
} }
// [{}, {899: 12, "x": "y"}, [], 123, 89, 89, "xyz", null, [], [], [1, 2, 3]] // [{}, {899: 12, "x": "y"}, [], 123, 89, 89, "xyz", null, [], [], [1, 2, 3]]

5
todo.txt
View file

@ -1,7 +1,12 @@
- test new JSON module
- we need a way to disable tests - we need a way to disable tests
- deprecate ^ and make it available as operator - deprecate ^ and make it available as operator
- test branch coverage - test branch coverage
- checked exceptions - checked exceptions
- explicit indices for array literals
- built-in serialization
- do not ambiguity error for methods if amibiguity only affects the same
dispatcher anyway
High priority (version 0.9.0) High priority (version 0.9.0)

2
web/question.txt
View file

@ -45,7 +45,7 @@ How stable is Nimrod?
The compiler is in development and some important features are still missing. The compiler is in development and some important features are still missing.
However, the compiler is quite stable already: It is able to compile itself However, the compiler is quite stable already: It is able to compile itself
and a substantial body of other code. Until version 1.0.0 is released, and a substantial body of other code. Until version 1.0.0 is released,
incompabilities with older versions of the compiler will be introduced. The incompatibilities with older versions of the compiler will be introduced. The
semantic details of overloading, macros/templates/generics and iterators semantic details of overloading, macros/templates/generics and iterators
and their interactions are subject to change. and their interactions are subject to change.