Update pegs.nim to work at compiletime. No range errors. (#13459)
This commit is contained in:
solo989
GitHub
parent
6a0e87eb38
commit
e84e01cb8c
1 changed files with 175 additions and 153 deletions
|
|
@ -1420,7 +1420,7 @@ type
|
||||||
|
|
||||||
PegLexer {.inheritable.} = object ## the lexer object.
|
PegLexer {.inheritable.} = object ## the lexer object.
|
||||||
bufpos: int ## the current position within the buffer
|
bufpos: int ## the current position within the buffer
|
||||||
buf: cstring ## the buffer itself
|
buf: string ## the buffer itself
|
||||||
lineNumber: int ## the current line number
|
lineNumber: int ## the current line number
|
||||||
lineStart: int ## index of last line start in buffer
|
lineStart: int ## index of last line start in buffer
|
||||||
colOffset: int ## column to add
|
colOffset: int ## column to add
|
||||||
|
|
@ -1481,6 +1481,9 @@ proc handleHexChar(c: var PegLexer, xi: var int) =
|
||||||
|
|
||||||
proc getEscapedChar(c: var PegLexer, tok: var Token) =
|
proc getEscapedChar(c: var PegLexer, tok: var Token) =
|
||||||
inc(c.bufpos)
|
inc(c.bufpos)
|
||||||
|
if c.bufpos >= len(c.buf):
|
||||||
|
tok.kind = tkInvalid
|
||||||
|
return
|
||||||
case c.buf[c.bufpos]
|
case c.buf[c.bufpos]
|
||||||
of 'r', 'R', 'c', 'C':
|
of 'r', 'R', 'c', 'C':
|
||||||
add(tok.literal, '\c')
|
add(tok.literal, '\c')
|
||||||
|
|
@ -1508,6 +1511,9 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) =
|
||||||
inc(c.bufpos)
|
inc(c.bufpos)
|
||||||
of 'x', 'X':
|
of 'x', 'X':
|
||||||
inc(c.bufpos)
|
inc(c.bufpos)
|
||||||
|
if c.bufpos >= len(c.buf):
|
||||||
|
tok.kind = tkInvalid
|
||||||
|
return
|
||||||
var xi = 0
|
var xi = 0
|
||||||
handleHexChar(c, xi)
|
handleHexChar(c, xi)
|
||||||
handleHexChar(c, xi)
|
handleHexChar(c, xi)
|
||||||
|
|
@ -1517,7 +1523,7 @@ proc getEscapedChar(c: var PegLexer, tok: var Token) =
|
||||||
var val = ord(c.buf[c.bufpos]) - ord('0')
|
var val = ord(c.buf[c.bufpos]) - ord('0')
|
||||||
inc(c.bufpos)
|
inc(c.bufpos)
|
||||||
var i = 1
|
var i = 1
|
||||||
while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}):
|
while (c.bufpos < len(c.buf)) and (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}):
|
||||||
val = val * 10 + ord(c.buf[c.bufpos]) - ord('0')
|
val = val * 10 + ord(c.buf[c.bufpos]) - ord('0')
|
||||||
inc(c.bufpos)
|
inc(c.bufpos)
|
||||||
inc(i)
|
inc(i)
|
||||||
|
|
@ -1571,7 +1577,7 @@ proc getString(c: var PegLexer, tok: var Token) =
|
||||||
|
|
||||||
proc getDollar(c: var PegLexer, tok: var Token) =
|
proc getDollar(c: var PegLexer, tok: var Token) =
|
||||||
var pos = c.bufpos + 1
|
var pos = c.bufpos + 1
|
||||||
if c.buf[pos] in {'0'..'9'}:
|
if pos < c.buf.len and c.buf[pos] in {'0'..'9'}:
|
||||||
tok.kind = tkBackref
|
tok.kind = tkBackref
|
||||||
tok.index = 0
|
tok.index = 0
|
||||||
while pos < c.buf.len and c.buf[pos] in {'0'..'9'}:
|
while pos < c.buf.len and c.buf[pos] in {'0'..'9'}:
|
||||||
|
|
@ -1586,54 +1592,55 @@ proc getCharSet(c: var PegLexer, tok: var Token) =
|
||||||
tok.charset = {}
|
tok.charset = {}
|
||||||
var pos = c.bufpos + 1
|
var pos = c.bufpos + 1
|
||||||
var caret = false
|
var caret = false
|
||||||
if c.buf[pos] == '^':
|
if pos < c.buf.len:
|
||||||
inc(pos)
|
if c.buf[pos] == '^':
|
||||||
caret = true
|
|
||||||
while pos < c.buf.len:
|
|
||||||
var ch: char
|
|
||||||
case c.buf[pos]
|
|
||||||
of ']':
|
|
||||||
if pos < c.buf.len: inc(pos)
|
|
||||||
break
|
|
||||||
of '\\':
|
|
||||||
c.bufpos = pos
|
|
||||||
getEscapedChar(c, tok)
|
|
||||||
pos = c.bufpos
|
|
||||||
ch = tok.literal[tok.literal.len-1]
|
|
||||||
of '\C', '\L', '\0':
|
|
||||||
tok.kind = tkInvalid
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
ch = c.buf[pos]
|
|
||||||
inc(pos)
|
inc(pos)
|
||||||
incl(tok.charset, ch)
|
caret = true
|
||||||
if c.buf[pos] == '-':
|
while pos < c.buf.len:
|
||||||
if pos+1 < c.buf.len and c.buf[pos+1] == ']':
|
var ch: char
|
||||||
incl(tok.charset, '-')
|
case c.buf[pos]
|
||||||
inc(pos)
|
of ']':
|
||||||
|
if pos < c.buf.len: inc(pos)
|
||||||
|
break
|
||||||
|
of '\\':
|
||||||
|
c.bufpos = pos
|
||||||
|
getEscapedChar(c, tok)
|
||||||
|
pos = c.bufpos
|
||||||
|
ch = tok.literal[tok.literal.len-1]
|
||||||
|
of '\C', '\L', '\0':
|
||||||
|
tok.kind = tkInvalid
|
||||||
|
break
|
||||||
else:
|
else:
|
||||||
if pos+1 < c.buf.len:
|
ch = c.buf[pos]
|
||||||
|
inc(pos)
|
||||||
|
incl(tok.charset, ch)
|
||||||
|
if c.buf[pos] == '-':
|
||||||
|
if pos+1 < c.buf.len and c.buf[pos+1] == ']':
|
||||||
|
incl(tok.charset, '-')
|
||||||
inc(pos)
|
inc(pos)
|
||||||
else:
|
|
||||||
break
|
|
||||||
var ch2: char
|
|
||||||
case c.buf[pos]
|
|
||||||
of '\\':
|
|
||||||
c.bufpos = pos
|
|
||||||
getEscapedChar(c, tok)
|
|
||||||
pos = c.bufpos
|
|
||||||
ch2 = tok.literal[tok.literal.len-1]
|
|
||||||
of '\C', '\L', '\0':
|
|
||||||
tok.kind = tkInvalid
|
|
||||||
break
|
|
||||||
else:
|
else:
|
||||||
if pos+1 < c.buf.len:
|
if pos+1 < c.buf.len:
|
||||||
ch2 = c.buf[pos]
|
|
||||||
inc(pos)
|
inc(pos)
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
for i in ord(ch)+1 .. ord(ch2):
|
var ch2: char
|
||||||
incl(tok.charset, chr(i))
|
case c.buf[pos]
|
||||||
|
of '\\':
|
||||||
|
c.bufpos = pos
|
||||||
|
getEscapedChar(c, tok)
|
||||||
|
pos = c.bufpos
|
||||||
|
ch2 = tok.literal[tok.literal.len-1]
|
||||||
|
of '\C', '\L', '\0':
|
||||||
|
tok.kind = tkInvalid
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
if pos+1 < c.buf.len:
|
||||||
|
ch2 = c.buf[pos]
|
||||||
|
inc(pos)
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
for i in ord(ch)+1 .. ord(ch2):
|
||||||
|
incl(tok.charset, chr(i))
|
||||||
c.bufpos = pos
|
c.bufpos = pos
|
||||||
if caret: tok.charset = {'\1'..'\xFF'} - tok.charset
|
if caret: tok.charset = {'\1'..'\xFF'} - tok.charset
|
||||||
|
|
||||||
|
|
@ -1661,6 +1668,13 @@ proc getTok(c: var PegLexer, tok: var Token) =
|
||||||
setLen(tok.literal, 0)
|
setLen(tok.literal, 0)
|
||||||
skip(c)
|
skip(c)
|
||||||
|
|
||||||
|
if c.bufpos >= c.buf.len:
|
||||||
|
tok.kind = tkEof
|
||||||
|
tok.literal = "[EOF]"
|
||||||
|
add(tok.literal, '\0')
|
||||||
|
inc(c.bufpos)
|
||||||
|
return
|
||||||
|
|
||||||
case c.buf[c.bufpos]
|
case c.buf[c.bufpos]
|
||||||
of '{':
|
of '{':
|
||||||
inc(c.bufpos)
|
inc(c.bufpos)
|
||||||
|
|
@ -1700,6 +1714,8 @@ proc getTok(c: var PegLexer, tok: var Token) =
|
||||||
of '$': getDollar(c, tok)
|
of '$': getDollar(c, tok)
|
||||||
of 'a'..'z', 'A'..'Z', '\128'..'\255':
|
of 'a'..'z', 'A'..'Z', '\128'..'\255':
|
||||||
getSymbol(c, tok)
|
getSymbol(c, tok)
|
||||||
|
if c.bufpos >= c.buf.len:
|
||||||
|
return
|
||||||
if c.buf[c.bufpos] in {'\'', '"'} or
|
if c.buf[c.bufpos] in {'\'', '"'} or
|
||||||
c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and
|
c.buf[c.bufpos] == '$' and c.bufpos+1 < c.buf.len and
|
||||||
c.buf[c.bufpos+1] in {'0'..'9'}:
|
c.buf[c.bufpos+1] in {'0'..'9'}:
|
||||||
|
|
@ -1768,7 +1784,9 @@ proc arrowIsNextTok(c: PegLexer): bool =
|
||||||
# the only look ahead we need
|
# the only look ahead we need
|
||||||
var pos = c.bufpos
|
var pos = c.bufpos
|
||||||
while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos)
|
while pos < c.buf.len and c.buf[pos] in {'\t', ' '}: inc(pos)
|
||||||
result = c.buf[pos] == '<' and (pos+1 < c.buf.len) and c.buf[pos+1] == '-'
|
if pos+1 >= c.buf.len:
|
||||||
|
return
|
||||||
|
result = c.buf[pos] == '<' and c.buf[pos+1] == '-'
|
||||||
|
|
||||||
# ----------------------------- parser ----------------------------------------
|
# ----------------------------- parser ----------------------------------------
|
||||||
|
|
||||||
|
|
@ -2038,141 +2056,145 @@ proc escapePeg*(s: string): string =
|
||||||
if inQuote: result.add('\'')
|
if inQuote: result.add('\'')
|
||||||
|
|
||||||
when isMainModule:
|
when isMainModule:
|
||||||
assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27"
|
proc pegsTest() =
|
||||||
assert match("(a b c)", peg"'(' @ ')'")
|
assert escapePeg("abc''def'") == r"'abc'\x27\x27'def'\x27"
|
||||||
assert match("W_HI_Le", peg"\y 'while'")
|
assert match("(a b c)", peg"'(' @ ')'")
|
||||||
assert(not match("W_HI_L", peg"\y 'while'"))
|
assert match("W_HI_Le", peg"\y 'while'")
|
||||||
assert(not match("W_HI_Le", peg"\y v'while'"))
|
assert(not match("W_HI_L", peg"\y 'while'"))
|
||||||
assert match("W_HI_Le", peg"y'while'")
|
assert(not match("W_HI_Le", peg"\y v'while'"))
|
||||||
|
assert match("W_HI_Le", peg"y'while'")
|
||||||
|
|
||||||
assert($ +digits == $peg"\d+")
|
assert($ +digits == $peg"\d+")
|
||||||
assert "0158787".match(peg"\d+")
|
assert "0158787".match(peg"\d+")
|
||||||
assert "ABC 0232".match(peg"\w+\s+\d+")
|
assert "ABC 0232".match(peg"\w+\s+\d+")
|
||||||
assert "ABC".match(peg"\d+ / \w+")
|
assert "ABC".match(peg"\d+ / \w+")
|
||||||
|
|
||||||
var accum: seq[string] = @[]
|
var accum: seq[string] = @[]
|
||||||
for word in split("00232this02939is39an22example111", peg"\d+"):
|
for word in split("00232this02939is39an22example111", peg"\d+"):
|
||||||
accum.add(word)
|
accum.add(word)
|
||||||
assert(accum == @["this", "is", "an", "example"])
|
assert(accum == @["this", "is", "an", "example"])
|
||||||
|
|
||||||
assert matchLen("key", ident) == 3
|
assert matchLen("key", ident) == 3
|
||||||
|
|
||||||
var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident)
|
var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident)
|
||||||
assert matchLen("key1= cal9", pattern) == 11
|
assert matchLen("key1= cal9", pattern) == 11
|
||||||
|
|
||||||
var ws = newNonTerminal("ws", 1, 1)
|
var ws = newNonTerminal("ws", 1, 1)
|
||||||
ws.rule = *whitespace
|
ws.rule = *whitespace
|
||||||
|
|
||||||
var expr = newNonTerminal("expr", 1, 1)
|
var expr = newNonTerminal("expr", 1, 1)
|
||||||
expr.rule = sequence(capture(ident), *sequence(
|
expr.rule = sequence(capture(ident), *sequence(
|
||||||
nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr)))
|
nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr)))
|
||||||
|
|
||||||
var c: Captures
|
var c: Captures
|
||||||
var s = "a+b + c +d+e+f"
|
var s = "a+b + c +d+e+f"
|
||||||
assert rawMatch(s, expr.rule, 0, c) == len(s)
|
assert rawMatch(s, expr.rule, 0, c) == len(s)
|
||||||
var a = ""
|
var a = ""
|
||||||
for i in 0..c.ml-1:
|
for i in 0..c.ml-1:
|
||||||
a.add(substr(s, c.matches[i][0], c.matches[i][1]))
|
a.add(substr(s, c.matches[i][0], c.matches[i][1]))
|
||||||
assert a == "abcdef"
|
assert a == "abcdef"
|
||||||
#echo expr.rule
|
#echo expr.rule
|
||||||
|
|
||||||
#const filename = "lib/devel/peg/grammar.txt"
|
#const filename = "lib/devel/peg/grammar.txt"
|
||||||
#var grammar = parsePeg(newFileStream(filename, fmRead), filename)
|
#var grammar = parsePeg(newFileStream(filename, fmRead), filename)
|
||||||
#echo "a <- [abc]*?".match(grammar)
|
#echo "a <- [abc]*?".match(grammar)
|
||||||
assert find("_____abc_______", term("abc"), 2) == 5
|
assert find("_____abc_______", term("abc"), 2) == 5
|
||||||
assert match("_______ana", peg"A <- 'ana' / . A")
|
assert match("_______ana", peg"A <- 'ana' / . A")
|
||||||
assert match("abcs%%%", peg"A <- ..A / .A / '%'")
|
assert match("abcs%%%", peg"A <- ..A / .A / '%'")
|
||||||
|
|
||||||
var matches: array[0..MaxSubpatterns-1, string]
|
var matches: array[0..MaxSubpatterns-1, string]
|
||||||
if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}":
|
if "abc" =~ peg"{'a'}'bc' 'xyz' / {\ident}":
|
||||||
assert matches[0] == "abc"
|
assert matches[0] == "abc"
|
||||||
else:
|
else:
|
||||||
assert false
|
assert false
|
||||||
|
|
||||||
var g2 = peg"""S <- A B / C D
|
var g2 = peg"""S <- A B / C D
|
||||||
A <- 'a'+
|
A <- 'a'+
|
||||||
B <- 'b'+
|
B <- 'b'+
|
||||||
C <- 'c'+
|
C <- 'c'+
|
||||||
D <- 'd'+
|
D <- 'd'+
|
||||||
"""
|
"""
|
||||||
assert($g2 == "((A B) / (C D))")
|
assert($g2 == "((A B) / (C D))")
|
||||||
assert match("cccccdddddd", g2)
|
assert match("cccccdddddd", g2)
|
||||||
assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") ==
|
assert("var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2") ==
|
||||||
"var1<-keykey; var2<-key2key2")
|
"var1<-keykey; var2<-key2key2")
|
||||||
assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") ==
|
assert("var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") ==
|
||||||
"$1<-$2$2; $1<-$2$2")
|
"$1<-$2$2; $1<-$2$2")
|
||||||
assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}")
|
assert "var1=key; var2=key2".endsWith(peg"{\ident}'='{\ident}")
|
||||||
|
|
||||||
if "aaaaaa" =~ peg"'aa' !. / ({'a'})+":
|
if "aaaaaa" =~ peg"'aa' !. / ({'a'})+":
|
||||||
assert matches[0] == "a"
|
assert matches[0] == "a"
|
||||||
else:
|
else:
|
||||||
assert false
|
assert false
|
||||||
|
|
||||||
if match("abcdefg", peg"c {d} ef {g}", matches, 2):
|
if match("abcdefg", peg"c {d} ef {g}", matches, 2):
|
||||||
assert matches[0] == "d"
|
assert matches[0] == "d"
|
||||||
assert matches[1] == "g"
|
assert matches[1] == "g"
|
||||||
else:
|
else:
|
||||||
assert false
|
assert false
|
||||||
|
|
||||||
accum = @[]
|
accum = @[]
|
||||||
for x in findAll("abcdef", peg".", 3):
|
for x in findAll("abcdef", peg".", 3):
|
||||||
accum.add(x)
|
accum.add(x)
|
||||||
assert(accum == @["d", "e", "f"])
|
assert(accum == @["d", "e", "f"])
|
||||||
|
|
||||||
for x in findAll("abcdef", peg"^{.}", 3):
|
for x in findAll("abcdef", peg"^{.}", 3):
|
||||||
assert x == "d"
|
assert x == "d"
|
||||||
|
|
||||||
if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')":
|
if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')":
|
||||||
assert matches[0] == "f"
|
assert matches[0] == "f"
|
||||||
assert matches[1] == "a, b"
|
assert matches[1] == "a, b"
|
||||||
else:
|
else:
|
||||||
assert false
|
assert false
|
||||||
|
|
||||||
assert match("eine übersicht und außerdem", peg"(\letter \white*)+")
|
assert match("eine übersicht und außerdem", peg"(\letter \white*)+")
|
||||||
# ß is not a lower cased letter?!
|
# ß is not a lower cased letter?!
|
||||||
assert match("eine übersicht und auerdem", peg"(\lower \white*)+")
|
assert match("eine übersicht und auerdem", peg"(\lower \white*)+")
|
||||||
assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
|
assert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
|
||||||
assert(not match("456678", peg"(\letter)+"))
|
assert(not match("456678", peg"(\letter)+"))
|
||||||
|
|
||||||
assert("var1 = key; var2 = key2".replacef(
|
assert("var1 = key; var2 = key2".replacef(
|
||||||
peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
|
peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
|
||||||
"var1<-keykey;var2<-key2key2")
|
"var1<-keykey;var2<-key2key2")
|
||||||
|
|
||||||
assert match("prefix/start", peg"^start$", 7)
|
assert match("prefix/start", peg"^start$", 7)
|
||||||
|
|
||||||
if "foo" =~ peg"{'a'}?.*":
|
if "foo" =~ peg"{'a'}?.*":
|
||||||
assert matches[0].len == 0
|
assert matches[0].len == 0
|
||||||
else: assert false
|
else: assert false
|
||||||
|
|
||||||
if "foo" =~ peg"{''}.*":
|
if "foo" =~ peg"{''}.*":
|
||||||
assert matches[0] == ""
|
assert matches[0] == ""
|
||||||
else: assert false
|
else: assert false
|
||||||
|
|
||||||
if "foo" =~ peg"{'foo'}":
|
if "foo" =~ peg"{'foo'}":
|
||||||
assert matches[0] == "foo"
|
assert matches[0] == "foo"
|
||||||
else: assert false
|
else: assert false
|
||||||
|
|
||||||
let empty_test = peg"^\d*"
|
let empty_test = peg"^\d*"
|
||||||
let str = "XYZ"
|
let str = "XYZ"
|
||||||
|
|
||||||
assert(str.find(empty_test) == 0)
|
assert(str.find(empty_test) == 0)
|
||||||
assert(str.match(empty_test))
|
assert(str.match(empty_test))
|
||||||
|
|
||||||
proc handleMatches*(m: int, n: int, c: openArray[string]): string =
|
proc handleMatches(m: int, n: int, c: openArray[string]): string =
|
||||||
result = ""
|
result = ""
|
||||||
|
|
||||||
if m > 0:
|
if m > 0:
|
||||||
result.add ", "
|
result.add ", "
|
||||||
|
|
||||||
result.add case n:
|
result.add case n:
|
||||||
of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'"
|
of 2: toLowerAscii(c[0]) & ": '" & c[1] & "'"
|
||||||
of 1: toLowerAscii(c[0]) & ": ''"
|
of 1: toLowerAscii(c[0]) & ": ''"
|
||||||
else: ""
|
else: ""
|
||||||
|
|
||||||
assert("Var1=key1;var2=Key2; VAR3".
|
assert("Var1=key1;var2=Key2; VAR3".
|
||||||
replace(peg"{\ident}('='{\ident})* ';'* \s*",
|
replace(peg"{\ident}('='{\ident})* ';'* \s*",
|
||||||
handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''")
|
handleMatches) == "var1: 'key1', var2: 'Key2', var3: ''")
|
||||||
|
|
||||||
|
|
||||||
doAssert "test1".match(peg"""{@}$""")
|
doAssert "test1".match(peg"""{@}$""")
|
||||||
doAssert "test2".match(peg"""{(!$ .)*} $""")
|
doAssert "test2".match(peg"""{(!$ .)*} $""")
|
||||||
|
pegsTest()
|
||||||
|
static:
|
||||||
|
pegsTest()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue