jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixes #2205

Browse source
This commit is contained in:
Araq 2015-03-06 12:14:42 +01:00
commit c914532c26
1 changed files with 156 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

312
lib/pure/pegs.nim
View file

@ -29,7 +29,7 @@ when useUnicode:
const const
InlineThreshold = 5 ## number of leaves; -1 to disable inlining InlineThreshold = 5 ## number of leaves; -1 to disable inlining
MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that
## can be captured. More subpatterns cannot be captured! ## can be captured. More subpatterns cannot be captured!
type type
PegKind = enum PegKind = enum
@ -85,14 +85,14 @@ type
of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns] of pkBackRef..pkBackRefIgnoreStyle: index: range[0..MaxSubpatterns]
else: sons: seq[TNode] else: sons: seq[TNode]
NonTerminal* = ref NonTerminalObj NonTerminal* = ref NonTerminalObj
Peg* = TNode ## type that represents a PEG Peg* = TNode ## type that represents a PEG
{.deprecated: [TPeg: Peg].} {.deprecated: [TPeg: Peg].}
proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} = proc term*(t: string): Peg {.nosideEffect, rtl, extern: "npegs$1Str".} =
## constructs a PEG from a terminal string ## constructs a PEG from a terminal string
if t.len != 1: if t.len != 1:
result.kind = pkTerminal result.kind = pkTerminal
result.term = t result.term = t
else: else:
@ -116,7 +116,7 @@ proc term*(t: char): Peg {.nosideEffect, rtl, extern: "npegs$1Char".} =
assert t != '\0' assert t != '\0'
result.kind = pkChar result.kind = pkChar
result.ch = t result.ch = t
proc charSet*(s: set[char]): Peg {.nosideEffect, rtl, extern: "npegs$1".} = proc charSet*(s: set[char]): Peg {.nosideEffect, rtl, extern: "npegs$1".} =
## constructs a PEG from a character set `s` ## constructs a PEG from a character set `s`
assert '\0' notin s assert '\0' notin s
@ -129,12 +129,12 @@ proc add(d: var Peg, s: Peg) {.inline.} = add(d.sons, s)
proc addChoice(dest: var Peg, elem: Peg) = proc addChoice(dest: var Peg, elem: Peg) =
var L = dest.len-1 var L = dest.len-1
if L >= 0 and dest.sons[L].kind == pkCharChoice: if L >= 0 and dest.sons[L].kind == pkCharChoice:
# caution! Do not introduce false aliasing here! # caution! Do not introduce false aliasing here!
case elem.kind case elem.kind
of pkCharChoice: of pkCharChoice:
dest.sons[L] = charSet(dest.sons[L].charChoice[] + elem.charChoice[]) dest.sons[L] = charSet(dest.sons[L].charChoice[] + elem.charChoice[])
of pkChar: of pkChar:
dest.sons[L] = charSet(dest.sons[L].charChoice[] + {elem.ch}) dest.sons[L] = charSet(dest.sons[L].charChoice[] + {elem.ch})
else: add(dest, elem) else: add(dest, elem)
else: add(dest, elem) else: add(dest, elem)
@ -158,12 +158,12 @@ proc `/`*(a: varargs[Peg]): Peg {.
proc addSequence(dest: var Peg, elem: Peg) = proc addSequence(dest: var Peg, elem: Peg) =
var L = dest.len-1 var L = dest.len-1
if L >= 0 and dest.sons[L].kind == pkTerminal: if L >= 0 and dest.sons[L].kind == pkTerminal:
# caution! Do not introduce false aliasing here! # caution! Do not introduce false aliasing here!
case elem.kind case elem.kind
of pkTerminal: of pkTerminal:
dest.sons[L] = term(dest.sons[L].term & elem.term) dest.sons[L] = term(dest.sons[L].term & elem.term)
of pkChar: of pkChar:
dest.sons[L] = term(dest.sons[L].term & elem.ch) dest.sons[L] = term(dest.sons[L].term & elem.ch)
else: add(dest, elem) else: add(dest, elem)
else: add(dest, elem) else: add(dest, elem)
@ -172,7 +172,7 @@ proc sequence*(a: varargs[Peg]): Peg {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## constructs a sequence with all the PEGs from `a` ## constructs a sequence with all the PEGs from `a`
multipleOp(pkSequence, addSequence) multipleOp(pkSequence, addSequence)
proc `?`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsOptional".} = proc `?`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsOptional".} =
## constructs an optional for the PEG `a` ## constructs an optional for the PEG `a`
if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar, if a.kind in {pkOption, pkGreedyRep, pkGreedyAny, pkGreedyRepChar,
@ -207,7 +207,7 @@ proc `!*`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsSearch".} =
result.kind = pkSearch result.kind = pkSearch
result.sons = @[a] result.sons = @[a]
proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl, proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl,
extern: "npgegsCapturedSearch".} = extern: "npgegsCapturedSearch".} =
## constructs a "captured search" for the PEG `a` ## constructs a "captured search" for the PEG `a`
result.kind = pkCapturedSearch result.kind = pkCapturedSearch
@ -216,7 +216,7 @@ proc `!*\`*(a: Peg): Peg {.noSideEffect, rtl,
proc `+`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsGreedyPosRep".} = proc `+`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsGreedyPosRep".} =
## constructs a "greedy positive repetition" with the PEG `a` ## constructs a "greedy positive repetition" with the PEG `a`
return sequence(a, *a) return sequence(a, *a)
proc `&`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsAndPredicate".} = proc `&`*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsAndPredicate".} =
## constructs an "and predicate" with the PEG `a` ## constructs an "and predicate" with the PEG `a`
result.kind = pkAndPredicate result.kind = pkAndPredicate
@ -239,33 +239,33 @@ proc newLine*: Peg {.inline.} =
## constructs the PEG `newline`:idx: (``\n``) ## constructs the PEG `newline`:idx: (``\n``)
result.kind = pkNewLine result.kind = pkNewLine
proc unicodeLetter*: Peg {.inline.} = proc unicodeLetter*: Peg {.inline.} =
## constructs the PEG ``\letter`` which matches any Unicode letter. ## constructs the PEG ``\letter`` which matches any Unicode letter.
result.kind = pkLetter result.kind = pkLetter
proc unicodeLower*: Peg {.inline.} =
## constructs the PEG ``\lower`` which matches any Unicode lowercase letter.
result.kind = pkLower
proc unicodeUpper*: Peg {.inline.} = proc unicodeLower*: Peg {.inline.} =
## constructs the PEG ``\lower`` which matches any Unicode lowercase letter.
result.kind = pkLower
proc unicodeUpper*: Peg {.inline.} =
## constructs the PEG ``\upper`` which matches any Unicode uppercase letter. ## constructs the PEG ``\upper`` which matches any Unicode uppercase letter.
result.kind = pkUpper result.kind = pkUpper
proc unicodeTitle*: Peg {.inline.} = proc unicodeTitle*: Peg {.inline.} =
## constructs the PEG ``\title`` which matches any Unicode title letter. ## constructs the PEG ``\title`` which matches any Unicode title letter.
result.kind = pkTitle result.kind = pkTitle
proc unicodeWhitespace*: Peg {.inline.} = proc unicodeWhitespace*: Peg {.inline.} =
## constructs the PEG ``\white`` which matches any Unicode ## constructs the PEG ``\white`` which matches any Unicode
## whitespace character. ## whitespace character.
result.kind = pkWhitespace result.kind = pkWhitespace
proc startAnchor*: Peg {.inline.} = proc startAnchor*: Peg {.inline.} =
## constructs the PEG ``^`` which matches the start of the input. ## constructs the PEG ``^`` which matches the start of the input.
result.kind = pkStartAnchor result.kind = pkStartAnchor
proc endAnchor*: Peg {.inline.} = proc endAnchor*: Peg {.inline.} =
## constructs the PEG ``$`` which matches the end of the input. ## constructs the PEG ``$`` which matches the end of the input.
result = !any() result = !any()
proc capture*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsCapture".} = proc capture*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsCapture".} =
@ -274,21 +274,21 @@ proc capture*(a: Peg): Peg {.nosideEffect, rtl, extern: "npegsCapture".} =
result.sons = @[a] result.sons = @[a]
proc backref*(index: range[1..MaxSubpatterns]): Peg {. proc backref*(index: range[1..MaxSubpatterns]): Peg {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## constructs a back reference of the given `index`. `index` starts counting ## constructs a back reference of the given `index`. `index` starts counting
## from 1. ## from 1.
result.kind = pkBackRef result.kind = pkBackRef
result.index = index-1 result.index = index-1
proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {. proc backrefIgnoreCase*(index: range[1..MaxSubpatterns]): Peg {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## constructs a back reference of the given `index`. `index` starts counting ## constructs a back reference of the given `index`. `index` starts counting
## from 1. Ignores case for matching. ## from 1. Ignores case for matching.
result.kind = pkBackRefIgnoreCase result.kind = pkBackRefIgnoreCase
result.index = index-1 result.index = index-1
proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {. proc backrefIgnoreStyle*(index: range[1..MaxSubpatterns]): Peg {.
nosideEffect, rtl, extern: "npegs$1".}= nosideEffect, rtl, extern: "npegs$1".}=
## constructs a back reference of the given `index`. `index` starts counting ## constructs a back reference of the given `index`. `index` starts counting
## from 1. Ignores style for matching. ## from 1. Ignores style for matching.
result.kind = pkBackRefIgnoreStyle result.kind = pkBackRefIgnoreStyle
@ -298,7 +298,7 @@ proc spaceCost(n: Peg): int =
case n.kind case n.kind
of pkEmpty: discard of pkEmpty: discard
of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar, of pkTerminal, pkTerminalIgnoreCase, pkTerminalIgnoreStyle, pkChar,
pkGreedyRepChar, pkCharChoice, pkGreedyRepSet, pkGreedyRepChar, pkCharChoice, pkGreedyRepSet,
pkAny..pkWhitespace, pkGreedyAny: pkAny..pkWhitespace, pkGreedyAny:
result = 1 result = 1
of pkNonTerminal: of pkNonTerminal:
@ -310,7 +310,7 @@ proc spaceCost(n: Peg): int =
if result >= InlineThreshold: break if result >= InlineThreshold: break
proc nonterminal*(n: NonTerminal): Peg {. proc nonterminal*(n: NonTerminal): Peg {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## constructs a PEG that consists of the nonterminal symbol ## constructs a PEG that consists of the nonterminal symbol
assert n != nil assert n != nil
if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold: if ntDeclared in n.flags and spaceCost(n.rule) < InlineThreshold:
@ -331,7 +331,7 @@ proc newNonTerminal*(name: string, line, column: int): NonTerminal {.
template letters*: expr = template letters*: expr =
## expands to ``charset({'A'..'Z', 'a'..'z'})`` ## expands to ``charset({'A'..'Z', 'a'..'z'})``
charSet({'A'..'Z', 'a'..'z'}) charSet({'A'..'Z', 'a'..'z'})
template digits*: expr = template digits*: expr =
## expands to ``charset({'0'..'9'})`` ## expands to ``charset({'0'..'9'})``
charSet({'0'..'9'}) charSet({'0'..'9'})
@ -339,11 +339,11 @@ template digits*: expr =
template whitespace*: expr = template whitespace*: expr =
## expands to ``charset({' ', '\9'..'\13'})`` ## expands to ``charset({' ', '\9'..'\13'})``
charSet({' ', '\9'..'\13'}) charSet({' ', '\9'..'\13'})
template identChars*: expr = template identChars*: expr =
## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})`` ## expands to ``charset({'a'..'z', 'A'..'Z', '0'..'9', '_'})``
charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}) charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})
template identStartChars*: expr = template identStartChars*: expr =
## expands to ``charset({'A'..'Z', 'a'..'z', '_'})`` ## expands to ``charset({'A'..'Z', 'a'..'z', '_'})``
charSet({'a'..'z', 'A'..'Z', '_'}) charSet({'a'..'z', 'A'..'Z', '_'})
@ -352,14 +352,14 @@ template ident*: expr =
## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier ## same as ``[a-zA-Z_][a-zA-z_0-9]*``; standard identifier
sequence(charSet({'a'..'z', 'A'..'Z', '_'}), sequence(charSet({'a'..'z', 'A'..'Z', '_'}),
*charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'})) *charSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}))
template natural*: expr = template natural*: expr =
## same as ``\d+`` ## same as ``\d+``
+digits +digits
# ------------------------- debugging ----------------------------------------- # ------------------------- debugging -----------------------------------------
proc esc(c: char, reserved = {'\0'..'\255'}): string = proc esc(c: char, reserved = {'\0'..'\255'}): string =
case c case c
of '\b': result = "\\b" of '\b': result = "\\b"
of '\t': result = "\\t" of '\t': result = "\\t"
@ -374,38 +374,38 @@ proc esc(c: char, reserved = {'\0'..'\255'}): string =
elif c < ' ' or c >= '\128': result = '\\' & $ord(c) elif c < ' ' or c >= '\128': result = '\\' & $ord(c)
elif c in reserved: result = '\\' & c elif c in reserved: result = '\\' & c
else: result = $c else: result = $c
proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'" proc singleQuoteEsc(c: char): string = return "'" & esc(c, {'\''}) & "'"
proc singleQuoteEsc(str: string): string = proc singleQuoteEsc(str: string): string =
result = "'" result = "'"
for c in items(str): add result, esc(c, {'\''}) for c in items(str): add result, esc(c, {'\''})
add result, '\'' add result, '\''
proc charSetEscAux(cc: set[char]): string = proc charSetEscAux(cc: set[char]): string =
const reserved = {'^', '-', ']'} const reserved = {'^', '-', ']'}
result = "" result = ""
var c1 = 0 var c1 = 0
while c1 <= 0xff: while c1 <= 0xff:
if chr(c1) in cc: if chr(c1) in cc:
var c2 = c1 var c2 = c1
while c2 < 0xff and chr(succ(c2)) in cc: inc(c2) while c2 < 0xff and chr(succ(c2)) in cc: inc(c2)
if c1 == c2: if c1 == c2:
add result, esc(chr(c1), reserved) add result, esc(chr(c1), reserved)
elif c2 == succ(c1): elif c2 == succ(c1):
add result, esc(chr(c1), reserved) & esc(chr(c2), reserved) add result, esc(chr(c1), reserved) & esc(chr(c2), reserved)
else: else:
add result, esc(chr(c1), reserved) & '-' & esc(chr(c2), reserved) add result, esc(chr(c1), reserved) & '-' & esc(chr(c2), reserved)
c1 = c2 c1 = c2
inc(c1) inc(c1)
proc charSetEsc(cc: set[char]): string = proc charSetEsc(cc: set[char]): string =
if card(cc) >= 128+64: if card(cc) >= 128+64:
result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']' result = "[^" & charSetEscAux({'\1'..'\xFF'} - cc) & ']'
else: else:
result = '[' & charSetEscAux(cc) & ']' result = '[' & charSetEscAux(cc) & ']'
proc toStrAux(r: Peg, res: var string) = proc toStrAux(r: Peg, res: var string) =
case r.kind case r.kind
of pkEmpty: add(res, "()") of pkEmpty: add(res, "()")
of pkAny: add(res, '.') of pkAny: add(res, '.')
@ -469,25 +469,25 @@ proc toStrAux(r: Peg, res: var string) =
toStrAux(r.sons[0], res) toStrAux(r.sons[0], res)
of pkCapture: of pkCapture:
add(res, '{') add(res, '{')
toStrAux(r.sons[0], res) toStrAux(r.sons[0], res)
add(res, '}') add(res, '}')
of pkBackRef: of pkBackRef:
add(res, '$') add(res, '$')
add(res, $r.index) add(res, $r.index)
of pkBackRefIgnoreCase: of pkBackRefIgnoreCase:
add(res, "i$") add(res, "i$")
add(res, $r.index) add(res, $r.index)
of pkBackRefIgnoreStyle: of pkBackRefIgnoreStyle:
add(res, "y$") add(res, "y$")
add(res, $r.index) add(res, $r.index)
of pkRule: of pkRule:
toStrAux(r.sons[0], res) toStrAux(r.sons[0], res)
add(res, " <- ") add(res, " <- ")
toStrAux(r.sons[1], res) toStrAux(r.sons[1], res)
of pkList: of pkList:
for i in 0 .. high(r.sons): for i in 0 .. high(r.sons):
toStrAux(r.sons[i], res) toStrAux(r.sons[i], res)
add(res, "\n") add(res, "\n")
of pkStartAnchor: of pkStartAnchor:
add(res, '^') add(res, '^')
@ -506,8 +506,8 @@ type
{.deprecated: [TCaptures: Captures].} {.deprecated: [TCaptures: Captures].}
proc bounds*(c: Captures, proc bounds*(c: Captures,
i: range[0..MaxSubpatterns-1]): tuple[first, last: int] = i: range[0..MaxSubpatterns-1]): tuple[first, last: int] =
## returns the bounds ``[first..last]`` of the `i`'th capture. ## returns the bounds ``[first..last]`` of the `i`'th capture.
result = c.matches[i] result = c.matches[i]
@ -527,7 +527,7 @@ when not useUnicode:
proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {. proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## low-level matching proc that implements the PEG interpreter. Use this ## low-level matching proc that implements the PEG interpreter. Use this
## for maximum efficiency (every other PEG operation ends up calling this ## for maximum efficiency (every other PEG operation ends up calling this
## proc). ## proc).
## Returns -1 if it does not match, else the length of the match ## Returns -1 if it does not match, else the length of the match
@ -541,7 +541,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
result = runeLenAt(s, start) result = runeLenAt(s, start)
else: else:
result = -1 result = -1
of pkLetter: of pkLetter:
if s[start] != '\0': if s[start] != '\0':
var a: Rune var a: Rune
result = start result = start
@ -550,7 +550,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
else: result = -1 else: result = -1
else: else:
result = -1 result = -1
of pkLower: of pkLower:
if s[start] != '\0': if s[start] != '\0':
var a: Rune var a: Rune
result = start result = start
@ -559,7 +559,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
else: result = -1 else: result = -1
else: else:
result = -1 result = -1
of pkUpper: of pkUpper:
if s[start] != '\0': if s[start] != '\0':
var a: Rune var a: Rune
result = start result = start
@ -568,16 +568,16 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
else: result = -1 else: result = -1
else: else:
result = -1 result = -1
of pkTitle: of pkTitle:
if s[start] != '\0': if s[start] != '\0':
var a: Rune var a: Rune
result = start result = start
fastRuneAt(s, result, a) fastRuneAt(s, result, a)
if isTitle(a): dec(result, start) if isTitle(a): dec(result, start)
else: result = -1 else: result = -1
else: else:
result = -1 result = -1
of pkWhitespace: of pkWhitespace:
if s[start] != '\0': if s[start] != '\0':
var a: Rune var a: Rune
result = start result = start
@ -641,7 +641,7 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
when false: echo "leave: ", p.nt.name when false: echo "leave: ", p.nt.name
if result < 0: c.ml = oldMl if result < 0: c.ml = oldMl
of pkSequence: of pkSequence:
var oldMl = c.ml var oldMl = c.ml
result = 0 result = 0
for i in 0..high(p.sons): for i in 0..high(p.sons):
var x = rawMatch(s, p.sons[i], start+result, c) var x = rawMatch(s, p.sons[i], start+result, c)
@ -723,11 +723,11 @@ proc rawMatch*(s: string, p: Peg, start: int, c: var Captures): int {.
#else: silently ignore the capture #else: silently ignore the capture
else: else:
c.ml = idx c.ml = idx
of pkBackRef..pkBackRefIgnoreStyle: of pkBackRef..pkBackRefIgnoreStyle:
if p.index >= c.ml: return -1 if p.index >= c.ml: return -1
var (a, b) = c.matches[p.index] var (a, b) = c.matches[p.index]
var n: Peg var n: Peg
n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef)) n.kind = succ(pkTerminal, ord(p.kind)-ord(pkBackRef))
n.term = s.substr(a, b) n.term = s.substr(a, b)
result = rawMatch(s, n, start, c) result = rawMatch(s, n, start, c)
of pkStartAnchor: of pkStartAnchor:
@ -755,7 +755,7 @@ proc match*(s: string, pattern: Peg, matches: var openArray[string],
result = rawMatch(s, pattern, start, c) == len(s) - start result = rawMatch(s, pattern, start, c) == len(s) - start
if result: fillMatches(s, matches, c) if result: fillMatches(s, matches, c)
proc match*(s: string, pattern: Peg, proc match*(s: string, pattern: Peg,
start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} = start = 0): bool {.nosideEffect, rtl, extern: "npegs$1".} =
## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``. ## returns ``true`` if ``s`` matches the ``pattern`` beginning from ``start``.
var c: Captures var c: Captures
@ -773,7 +773,7 @@ proc matchLen*(s: string, pattern: Peg, matches: var openArray[string],
result = rawMatch(s, pattern, start, c) result = rawMatch(s, pattern, start, c)
if result >= 0: fillMatches(s, matches, c) if result >= 0: fillMatches(s, matches, c)
proc matchLen*(s: string, pattern: Peg, proc matchLen*(s: string, pattern: Peg,
start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} =
## the same as ``match``, but it returns the length of the match, ## the same as ``match``, but it returns the length of the match,
## if there is no match, -1 is returned. Note that a match length ## if there is no match, -1 is returned. Note that a match length
@ -797,11 +797,11 @@ proc find*(s: string, pattern: Peg, matches: var openArray[string],
return i return i
return -1 return -1
# could also use the pattern here: (!P .)* P # could also use the pattern here: (!P .)* P
proc findBounds*(s: string, pattern: Peg, matches: var openArray[string], proc findBounds*(s: string, pattern: Peg, matches: var openArray[string],
start = 0): tuple[first, last: int] {. start = 0): tuple[first, last: int] {.
nosideEffect, rtl, extern: "npegs$1Capture".} = nosideEffect, rtl, extern: "npegs$1Capture".} =
## returns the starting position and end position of ``pattern`` in ``s`` ## returns the starting position and end position of ``pattern`` in ``s``
## and the captured ## and the captured
## substrings in the array ``matches``. If it does not match, nothing ## substrings in the array ``matches``. If it does not match, nothing
## is written into ``matches`` and (-1,0) is returned. ## is written into ``matches`` and (-1,0) is returned.
@ -814,8 +814,8 @@ proc findBounds*(s: string, pattern: Peg, matches: var openArray[string],
fillMatches(s, matches, c) fillMatches(s, matches, c)
return (i, i+L-1) return (i, i+L-1)
return (-1, 0) return (-1, 0)
proc find*(s: string, pattern: Peg, proc find*(s: string, pattern: Peg,
start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} = start = 0): int {.nosideEffect, rtl, extern: "npegs$1".} =
## returns the starting position of ``pattern`` in ``s``. If it does not ## returns the starting position of ``pattern`` in ``s``. If it does not
## match, -1 is returned. ## match, -1 is returned.
@ -824,8 +824,8 @@ proc find*(s: string, pattern: Peg,
for i in start .. s.len-1: for i in start .. s.len-1:
if rawMatch(s, pattern, i, c) >= 0: return i if rawMatch(s, pattern, i, c) >= 0: return i
return -1 return -1
iterator findAll*(s: string, pattern: Peg, start = 0): string = iterator findAll*(s: string, pattern: Peg, start = 0): string =
## yields all matching *substrings* of `s` that match `pattern`. ## yields all matching *substrings* of `s` that match `pattern`.
var c: Captures var c: Captures
c.origStart = start c.origStart = start
@ -838,23 +838,23 @@ iterator findAll*(s: string, pattern: Peg, start = 0): string =
else: else:
yield substr(s, i, i+L-1) yield substr(s, i, i+L-1)
inc(i, L) inc(i, L)
proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {. proc findAll*(s: string, pattern: Peg, start = 0): seq[string] {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## returns all matching *substrings* of `s` that match `pattern`. ## returns all matching *substrings* of `s` that match `pattern`.
## If it does not match, @[] is returned. ## If it does not match, @[] is returned.
accumulateResult(findAll(s, pattern, start)) accumulateResult(findAll(s, pattern, start))
when not defined(nimhygiene): when not defined(nimhygiene):
{.pragma: inject.} {.pragma: inject.}
template `=~`*(s: string, pattern: Peg): bool = template `=~`*(s: string, pattern: Peg): bool =
## This calls ``match`` with an implicit declared ``matches`` array that ## This calls ``match`` with an implicit declared ``matches`` array that
## can be used in the scope of the ``=~`` call: ## can be used in the scope of the ``=~`` call:
## ##
## .. code-block:: nim ## .. code-block:: nim
## ##
## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}": ## if line =~ peg"\s* {\w+} \s* '=' \s* {\w+}":
## # matches a key=value pair: ## # matches a key=value pair:
## echo("Key: ", matches[0]) ## echo("Key: ", matches[0])
## echo("Value: ", matches[1]) ## echo("Value: ", matches[1])
@ -865,7 +865,7 @@ template `=~`*(s: string, pattern: Peg): bool =
## echo("comment: ", matches[0]) ## echo("comment: ", matches[0])
## else: ## else:
## echo("syntax error") ## echo("syntax error")
## ##
bind MaxSubpatterns bind MaxSubpatterns
when not declaredInScope(matches): when not declaredInScope(matches):
var matches {.inject.}: array[0..MaxSubpatterns-1, string] var matches {.inject.}: array[0..MaxSubpatterns-1, string]
@ -902,7 +902,7 @@ proc replacef*(s: string, sub: Peg, by: string): string {.
## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples:
## ##
## .. code-block:: nim ## .. code-block:: nim
## "var1=key; var2=key2".replace(peg"{\ident}'='{\ident}", "$1<-$2$2") ## "var1=key; var2=key2".replacef(peg"{\ident}'='{\ident}", "$1<-$2$2")
## ##
## Results in: ## Results in:
## ##
@ -941,10 +941,10 @@ proc replace*(s: string, sub: Peg, by = ""): string {.
add(result, by) add(result, by)
inc(i, x) inc(i, x)
add(result, substr(s, i)) add(result, substr(s, i))
proc parallelReplace*(s: string, subs: varargs[ proc parallelReplace*(s: string, subs: varargs[
tuple[pattern: Peg, repl: string]]): string {. tuple[pattern: Peg, repl: string]]): string {.
nosideEffect, rtl, extern: "npegs$1".} = nosideEffect, rtl, extern: "npegs$1".} =
## Returns a modified copy of `s` with the substitutions in `subs` ## Returns a modified copy of `s` with the substitutions in `subs`
## applied in parallel. ## applied in parallel.
result = "" result = ""
@ -964,8 +964,8 @@ proc parallelReplace*(s: string, subs: varargs[
add(result, s[i]) add(result, s[i])
inc(i) inc(i)
# copy the rest: # copy the rest:
add(result, substr(s, i)) add(result, substr(s, i))
proc transformFile*(infile, outfile: string, proc transformFile*(infile, outfile: string,
subs: varargs[tuple[pattern: Peg, repl: string]]) {. subs: varargs[tuple[pattern: Peg, repl: string]]) {.
rtl, extern: "npegs$1".} = rtl, extern: "npegs$1".} =
@ -974,7 +974,7 @@ proc transformFile*(infile, outfile: string,
## error occurs. This is supposed to be used for quick scripting. ## error occurs. This is supposed to be used for quick scripting.
var x = readFile(infile).string var x = readFile(infile).string
writeFile(outfile, x.parallelReplace(subs)) writeFile(outfile, x.parallelReplace(subs))
iterator split*(s: string, sep: Peg): string = iterator split*(s: string, sep: Peg): string =
## Splits the string `s` into substrings. ## Splits the string `s` into substrings.
## ##
@ -1049,14 +1049,14 @@ type
tkBackref, ## '$' tkBackref, ## '$'
tkDollar, ## '$' tkDollar, ## '$'
tkHat ## '^' tkHat ## '^'
TToken {.final.} = object ## a token TToken {.final.} = object ## a token
kind: TTokKind ## the type of the token kind: TTokKind ## the type of the token
modifier: TModifier modifier: TModifier
literal: string ## the parsed (string) literal literal: string ## the parsed (string) literal
charset: set[char] ## if kind == tkCharSet charset: set[char] ## if kind == tkCharSet
index: int ## if kind == tkBackref index: int ## if kind == tkBackref
PegLexer {.inheritable.} = object ## the lexer object. PegLexer {.inheritable.} = object ## the lexer object.
bufpos: int ## the current position within the buffer bufpos: int ## the current position within the buffer
buf: cstring ## the buffer itself buf: cstring ## the buffer itself
@ -1086,7 +1086,7 @@ proc handleLF(L: var PegLexer, pos: int): int =
result = pos+1 result = pos+1
L.lineStart = result L.lineStart = result
proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) = proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) =
L.buf = input L.buf = input
L.bufpos = 0 L.bufpos = 0
L.lineNumber = line L.lineNumber = line
@ -1094,69 +1094,69 @@ proc init(L: var PegLexer, input, filename: string, line = 1, col = 0) =
L.lineStart = 0 L.lineStart = 0
L.filename = filename L.filename = filename
proc getColumn(L: PegLexer): int {.inline.} = proc getColumn(L: PegLexer): int {.inline.} =
result = abs(L.bufpos - L.lineStart) + L.colOffset result = abs(L.bufpos - L.lineStart) + L.colOffset
proc getLine(L: PegLexer): int {.inline.} = proc getLine(L: PegLexer): int {.inline.} =
result = L.lineNumber result = L.lineNumber
proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = proc errorStr(L: PegLexer, msg: string, line = -1, col = -1): string =
var line = if line < 0: getLine(L) else: line var line = if line < 0: getLine(L) else: line
var col = if col < 0: getColumn(L) else: col var col = if col < 0: getColumn(L) else: col
result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg]
proc handleHexChar(c: var PegLexer, xi: var int) = proc handleHexChar(c: var PegLexer, xi: var int) =
case c.buf[c.bufpos] case c.buf[c.bufpos]
of '0'..'9': of '0'..'9':
xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0')) xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('0'))
inc(c.bufpos) inc(c.bufpos)
of 'a'..'f': of 'a'..'f':
xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10) xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('a') + 10)
inc(c.bufpos) inc(c.bufpos)
of 'A'..'F': of 'A'..'F':
xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10) xi = (xi shl 4) or (ord(c.buf[c.bufpos]) - ord('A') + 10)
inc(c.bufpos) inc(c.bufpos)
else: discard else: discard
proc getEscapedChar(c: var PegLexer, tok: var TToken) = proc getEscapedChar(c: var PegLexer, tok: var TToken) =
inc(c.bufpos) inc(c.bufpos)
case c.buf[c.bufpos] case c.buf[c.bufpos]
of 'r', 'R', 'c', 'C': of 'r', 'R', 'c', 'C':
add(tok.literal, '\c') add(tok.literal, '\c')
inc(c.bufpos) inc(c.bufpos)
of 'l', 'L': of 'l', 'L':
add(tok.literal, '\L') add(tok.literal, '\L')
inc(c.bufpos) inc(c.bufpos)
of 'f', 'F': of 'f', 'F':
add(tok.literal, '\f') add(tok.literal, '\f')
inc(c.bufpos) inc(c.bufpos)
of 'e', 'E': of 'e', 'E':
add(tok.literal, '\e') add(tok.literal, '\e')
inc(c.bufpos) inc(c.bufpos)
of 'a', 'A': of 'a', 'A':
add(tok.literal, '\a') add(tok.literal, '\a')
inc(c.bufpos) inc(c.bufpos)
of 'b', 'B': of 'b', 'B':
add(tok.literal, '\b') add(tok.literal, '\b')
inc(c.bufpos) inc(c.bufpos)
of 'v', 'V': of 'v', 'V':
add(tok.literal, '\v') add(tok.literal, '\v')
inc(c.bufpos) inc(c.bufpos)
of 't', 'T': of 't', 'T':
add(tok.literal, '\t') add(tok.literal, '\t')
inc(c.bufpos) inc(c.bufpos)
of 'x', 'X': of 'x', 'X':
inc(c.bufpos) inc(c.bufpos)
var xi = 0 var xi = 0
handleHexChar(c, xi) handleHexChar(c, xi)
handleHexChar(c, xi) handleHexChar(c, xi)
if xi == 0: tok.kind = tkInvalid if xi == 0: tok.kind = tkInvalid
else: add(tok.literal, chr(xi)) else: add(tok.literal, chr(xi))
of '0'..'9': of '0'..'9':
var val = ord(c.buf[c.bufpos]) - ord('0') var val = ord(c.buf[c.bufpos]) - ord('0')
inc(c.bufpos) inc(c.bufpos)
var i = 1 var i = 1
while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}): while (i <= 3) and (c.buf[c.bufpos] in {'0'..'9'}):
val = val * 10 + ord(c.buf[c.bufpos]) - ord('0') val = val * 10 + ord(c.buf[c.bufpos]) - ord('0')
inc(c.bufpos) inc(c.bufpos)
inc(i) inc(i)
@ -1169,32 +1169,32 @@ proc getEscapedChar(c: var PegLexer, tok: var TToken) =
else: else:
add(tok.literal, c.buf[c.bufpos]) add(tok.literal, c.buf[c.bufpos])
inc(c.bufpos) inc(c.bufpos)
proc skip(c: var PegLexer) = proc skip(c: var PegLexer) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf var buf = c.buf
while true: while true:
case buf[pos] case buf[pos]
of ' ', '\t': of ' ', '\t':
inc(pos) inc(pos)
of '#': of '#':
while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos) while not (buf[pos] in {'\c', '\L', '\0'}): inc(pos)
of '\c': of '\c':
pos = handleCR(c, pos) pos = handleCR(c, pos)
buf = c.buf buf = c.buf
of '\L': of '\L':
pos = handleLF(c, pos) pos = handleLF(c, pos)
buf = c.buf buf = c.buf
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
c.bufpos = pos c.bufpos = pos
proc getString(c: var PegLexer, tok: var TToken) = proc getString(c: var PegLexer, tok: var TToken) =
tok.kind = tkStringLit tok.kind = tkStringLit
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf var buf = c.buf
var quote = buf[pos-1] var quote = buf[pos-1]
while true: while true:
case buf[pos] case buf[pos]
of '\\': of '\\':
c.bufpos = pos c.bufpos = pos
@ -1205,13 +1205,13 @@ proc getString(c: var PegLexer, tok: var TToken) =
break break
elif buf[pos] == quote: elif buf[pos] == quote:
inc(pos) inc(pos)
break break
else: else:
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
inc(pos) inc(pos)
c.bufpos = pos c.bufpos = pos
proc getDollar(c: var PegLexer, tok: var TToken) = proc getDollar(c: var PegLexer, tok: var TToken) =
var pos = c.bufpos + 1 var pos = c.bufpos + 1
var buf = c.buf var buf = c.buf
if buf[pos] in {'0'..'9'}: if buf[pos] in {'0'..'9'}:
@ -1223,8 +1223,8 @@ proc getDollar(c: var PegLexer, tok: var TToken) =
else: else:
tok.kind = tkDollar tok.kind = tkDollar
c.bufpos = pos c.bufpos = pos
proc getCharSet(c: var PegLexer, tok: var TToken) = proc getCharSet(c: var PegLexer, tok: var TToken) =
tok.kind = tkCharSet tok.kind = tkCharSet
tok.charset = {} tok.charset = {}
var pos = c.bufpos + 1 var pos = c.bufpos + 1
@ -1247,7 +1247,7 @@ proc getCharSet(c: var PegLexer, tok: var TToken) =
of '\C', '\L', '\0': of '\C', '\L', '\0':
tok.kind = tkInvalid tok.kind = tkInvalid
break break
else: else:
ch = buf[pos] ch = buf[pos]
inc(pos) inc(pos)
incl(tok.charset, ch) incl(tok.charset, ch)
@ -1267,18 +1267,18 @@ proc getCharSet(c: var PegLexer, tok: var TToken) =
of '\C', '\L', '\0': of '\C', '\L', '\0':
tok.kind = tkInvalid tok.kind = tkInvalid
break break
else: else:
ch2 = buf[pos] ch2 = buf[pos]
inc(pos) inc(pos)
for i in ord(ch)+1 .. ord(ch2): for i in ord(ch)+1 .. ord(ch2):
incl(tok.charset, chr(i)) incl(tok.charset, chr(i))
c.bufpos = pos c.bufpos = pos
if caret: tok.charset = {'\1'..'\xFF'} - tok.charset if caret: tok.charset = {'\1'..'\xFF'} - tok.charset
proc getSymbol(c: var PegLexer, tok: var TToken) = proc getSymbol(c: var PegLexer, tok: var TToken) =
var pos = c.bufpos var pos = c.bufpos
var buf = c.buf var buf = c.buf
while true: while true:
add(tok.literal, buf[pos]) add(tok.literal, buf[pos])
inc(pos) inc(pos)
if buf[pos] notin strutils.IdentChars: break if buf[pos] notin strutils.IdentChars: break
@ -1294,7 +1294,7 @@ proc getBuiltin(c: var PegLexer, tok: var TToken) =
tok.kind = tkEscaped tok.kind = tkEscaped
getEscapedChar(c, tok) # may set tok.kind to tkInvalid getEscapedChar(c, tok) # may set tok.kind to tkInvalid
proc getTok(c: var PegLexer, tok: var TToken) = proc getTok(c: var PegLexer, tok: var TToken) =
tok.kind = tkInvalid tok.kind = tkInvalid
tok.modifier = modNone tok.modifier = modNone
setLen(tok.literal, 0) setLen(tok.literal, 0)
@ -1309,11 +1309,11 @@ proc getTok(c: var PegLexer, tok: var TToken) =
else: else:
tok.kind = tkCurlyLe tok.kind = tkCurlyLe
add(tok.literal, '{') add(tok.literal, '{')
of '}': of '}':
tok.kind = tkCurlyRi tok.kind = tkCurlyRi
inc(c.bufpos) inc(c.bufpos)
add(tok.literal, '}') add(tok.literal, '}')
of '[': of '[':
getCharSet(c, tok) getCharSet(c, tok)
of '(': of '(':
tok.kind = tkParLe tok.kind = tkParLe
@ -1323,7 +1323,7 @@ proc getTok(c: var PegLexer, tok: var TToken) =
tok.kind = tkParRi tok.kind = tkParRi
inc(c.bufpos) inc(c.bufpos)
add(tok.literal, ')') add(tok.literal, ')')
of '.': of '.':
tok.kind = tkAny tok.kind = tkAny
inc(c.bufpos) inc(c.bufpos)
add(tok.literal, '.') add(tok.literal, '.')
@ -1331,16 +1331,16 @@ proc getTok(c: var PegLexer, tok: var TToken) =
tok.kind = tkAnyRune tok.kind = tkAnyRune
inc(c.bufpos) inc(c.bufpos)
add(tok.literal, '_') add(tok.literal, '_')
of '\\': of '\\':
getBuiltin(c, tok) getBuiltin(c, tok)
of '\'', '"': getString(c, tok) of '\'', '"': getString(c, tok)
of '$': getDollar(c, tok) of '$': getDollar(c, tok)
of '\0': of '\0':
tok.kind = tkEof tok.kind = tkEof
tok.literal = "[EOF]" tok.literal = "[EOF]"
of 'a'..'z', 'A'..'Z', '\128'..'\255': of 'a'..'z', 'A'..'Z', '\128'..'\255':
getSymbol(c, tok) getSymbol(c, tok)
if c.buf[c.bufpos] in {'\'', '"'} or if c.buf[c.bufpos] in {'\'', '"'} or
c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}: c.buf[c.bufpos] == '$' and c.buf[c.bufpos+1] in {'0'..'9'}:
case tok.literal case tok.literal
of "i": tok.modifier = modIgnoreCase of "i": tok.modifier = modIgnoreCase
@ -1388,7 +1388,7 @@ proc getTok(c: var PegLexer, tok: var TToken) =
tok.kind = tkAt tok.kind = tkAt
inc(c.bufpos) inc(c.bufpos)
add(tok.literal, '@') add(tok.literal, '@')
if c.buf[c.bufpos] == '@': if c.buf[c.bufpos] == '@':
tok.kind = tkCurlyAt tok.kind = tkCurlyAt
inc(c.bufpos) inc(c.bufpos)
add(tok.literal, '@') add(tok.literal, '@')
@ -1407,7 +1407,7 @@ proc arrowIsNextTok(c: PegLexer): bool =
result = c.buf[pos] == '<' and c.buf[pos+1] == '-' result = c.buf[pos] == '<' and c.buf[pos+1] == '-'
# ----------------------------- parser ---------------------------------------- # ----------------------------- parser ----------------------------------------
type type
EInvalidPeg* = object of ValueError ## raised if an invalid EInvalidPeg* = object of ValueError ## raised if an invalid
## PEG has been detected ## PEG has been detected
@ -1425,7 +1425,7 @@ proc pegError(p: PegParser, msg: string, line = -1, col = -1) =
e.msg = errorStr(p, msg, line, col) e.msg = errorStr(p, msg, line, col)
raise e raise e
proc getTok(p: var PegParser) = proc getTok(p: var PegParser) =
getTok(p, p.tok) getTok(p, p.tok)
if p.tok.kind == tkInvalid: pegError(p, "invalid token") if p.tok.kind == tkInvalid: pegError(p, "invalid token")
@ -1475,7 +1475,7 @@ proc builtin(p: var PegParser): Peg =
of "white": result = unicodeWhitespace() of "white": result = unicodeWhitespace()
else: pegError(p, "unknown built-in: " & p.tok.literal) else: pegError(p, "unknown built-in: " & p.tok.literal)
proc token(terminal: Peg, p: PegParser): Peg = proc token(terminal: Peg, p: PegParser): Peg =
if p.skip.kind == pkEmpty: result = terminal if p.skip.kind == pkEmpty: result = terminal
else: result = sequence(p.skip, terminal) else: result = sequence(p.skip, terminal)
@ -1496,7 +1496,7 @@ proc primary(p: var PegParser): Peg =
else: discard else: discard
case p.tok.kind case p.tok.kind
of tkIdentifier: of tkIdentifier:
if p.identIsVerbatim: if p.identIsVerbatim:
var m = p.tok.modifier var m = p.tok.modifier
if m == modNone: m = p.modifier if m == modNone: m = p.modifier
result = modifiedTerm(p.tok.literal, m).token(p) result = modifiedTerm(p.tok.literal, m).token(p)
@ -1539,17 +1539,17 @@ proc primary(p: var PegParser): Peg =
of tkEscaped: of tkEscaped:
result = term(p.tok.literal[0]).token(p) result = term(p.tok.literal[0]).token(p)
getTok(p) getTok(p)
of tkDollar: of tkDollar:
result = endAnchor() result = endAnchor()
getTok(p) getTok(p)
of tkHat: of tkHat:
result = startAnchor() result = startAnchor()
getTok(p) getTok(p)
of tkBackref: of tkBackref:
var m = p.tok.modifier var m = p.tok.modifier
if m == modNone: m = p.modifier if m == modNone: m = p.modifier
result = modifiedBackref(p.tok.index, m).token(p) result = modifiedBackref(p.tok.index, m).token(p)
if p.tok.index < 0 or p.tok.index > p.captures: if p.tok.index < 0 or p.tok.index > p.captures:
pegError(p, "invalid back reference index: " & $p.tok.index) pegError(p, "invalid back reference index: " & $p.tok.index)
getTok(p) getTok(p)
else: else:
@ -1573,7 +1573,7 @@ proc seqExpr(p: var PegParser): Peg =
while true: while true:
case p.tok.kind case p.tok.kind
of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe, of tkAmp, tkNot, tkAt, tkStringLit, tkCharSet, tkParLe, tkCurlyLe,
tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref, tkAny, tkAnyRune, tkBuiltin, tkEscaped, tkDollar, tkBackref,
tkHat, tkCurlyAt: tkHat, tkCurlyAt:
result = sequence(result, primary(p)) result = sequence(result, primary(p))
of tkIdentifier: of tkIdentifier:
@ -1587,7 +1587,7 @@ proc parseExpr(p: var PegParser): Peg =
while p.tok.kind == tkBar: while p.tok.kind == tkBar:
getTok(p) getTok(p)
result = result / seqExpr(p) result = result / seqExpr(p)
proc parseRule(p: var PegParser): NonTerminal = proc parseRule(p: var PegParser): NonTerminal =
if p.tok.kind == tkIdentifier and arrowIsNextTok(p): if p.tok.kind == tkIdentifier and arrowIsNextTok(p):
result = getNonTerminal(p, p.tok.literal) result = getNonTerminal(p, p.tok.literal)
@ -1601,7 +1601,7 @@ proc parseRule(p: var PegParser): NonTerminal =
incl(result.flags, ntDeclared) # NOW inlining may be attempted incl(result.flags, ntDeclared) # NOW inlining may be attempted
else: else:
pegError(p, "rule expected, but found: " & p.tok.literal) pegError(p, "rule expected, but found: " & p.tok.literal)
proc rawParse(p: var PegParser): Peg = proc rawParse(p: var PegParser): Peg =
## parses a rule or a PEG expression ## parses a rule or a PEG expression
while p.tok.kind == tkBuiltin: while p.tok.kind == tkBuiltin:
@ -1680,7 +1680,7 @@ when isMainModule:
assert(not match("W_HI_L", peg"\y 'while'")) assert(not match("W_HI_L", peg"\y 'while'"))
assert(not match("W_HI_Le", peg"\y v'while'")) assert(not match("W_HI_Le", peg"\y v'while'"))
assert match("W_HI_Le", peg"y'while'") assert match("W_HI_Le", peg"y'while'")
assert($ +digits == $peg"\d+") assert($ +digits == $peg"\d+")
assert "0158787".match(peg"\d+") assert "0158787".match(peg"\d+")
assert "ABC 0232".match(peg"\w+\s+\d+") assert "ABC 0232".match(peg"\w+\s+\d+")
@ -1693,14 +1693,14 @@ when isMainModule:
var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident) var pattern = sequence(ident, *whitespace, term('='), *whitespace, ident)
assert matchLen("key1= cal9", pattern) == 11 assert matchLen("key1= cal9", pattern) == 11
var ws = newNonTerminal("ws", 1, 1) var ws = newNonTerminal("ws", 1, 1)
ws.rule = *whitespace ws.rule = *whitespace
var expr = newNonTerminal("expr", 1, 1) var expr = newNonTerminal("expr", 1, 1)
expr.rule = sequence(capture(ident), *sequence( expr.rule = sequence(capture(ident), *sequence(
nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr))) nonterminal(ws), term('+'), nonterminal(ws), nonterminal(expr)))
var c: Captures var c: Captures
var s = "a+b + c +d+e+f" var s = "a+b + c +d+e+f"
assert rawMatch(s, expr.rule, 0, c) == len(s) assert rawMatch(s, expr.rule, 0, c) == len(s)
@ -1722,7 +1722,7 @@ when isMainModule:
assert matches[0] == "abc" assert matches[0] == "abc"
else: else:
assert false assert false
var g2 = peg"""S <- A B / C D var g2 = peg"""S <- A B / C D
A <- 'a'+ A <- 'a'+
B <- 'b'+ B <- 'b'+
@ -1753,13 +1753,13 @@ when isMainModule:
for x in findAll("abcdef", peg"^{.}", 3): for x in findAll("abcdef", peg"^{.}", 3):
assert x == "d" assert x == "d"
if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')": if "f(a, b)" =~ peg"{[0-9]+} / ({\ident} '(' {@} ')')":
assert matches[0] == "f" assert matches[0] == "f"
assert matches[1] == "a, b" assert matches[1] == "a, b"
else: else:
assert false assert false
assert match("eine übersicht und außerdem", peg"(\letter \white*)+") assert match("eine übersicht und außerdem", peg"(\letter \white*)+")
# ß is not a lower cased letter?! # ß is not a lower cased letter?!
assert match("eine übersicht und auerdem", peg"(\lower \white*)+") assert match("eine übersicht und auerdem", peg"(\lower \white*)+")