jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #14 from BlaXpirit/inline-options

Browse source 
Inline options
This commit is contained in:
Flaviu Tamas 2015-05-08 13:56:25 -04:00
commit 0056ebdd15
5 changed files with 119 additions and 123 deletions
Download patch file Download diff file Expand all files Collapse all files

193
src/nre.nim
View file

@ -47,14 +47,7 @@ from unicode import runeLenAt
type type
Regex* = ref object Regex* = ref object
## Represents the pattern that things are matched against, constructed with ## Represents the pattern that things are matched against, constructed with
## ``re(string, string)``. Examples: ``re"foo"``, ``re(r"foo # comment", ## ``re(string)``. Examples: ``re"foo"``, ``re(r"(*ANYCRLF)(?x)foo # comment".
## "x<anycrlf>")``, ``re"(?x)(*ANYCRLF)foo # comment"``. For more details
## on the leading option groups, see the `Option
## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`__
## and the `Newline
## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`__
## sections of the `PCRE syntax
## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`__.
## ##
## ``pattern: string`` ## ``pattern: string``
## the string that was used to create the pattern. ## the string that was used to create the pattern.
@ -66,33 +59,36 @@ type
## a table from the capture names to their numeric id. ## a table from the capture names to their numeric id.
## ##
## ##
## Flags ## Options
## ..... ## .......
## ##
## - ``8``, ``u``, ``<utf8>`` - treat both the pattern and subject as UTF8 ## The following options may appear anywhere in the pattern, and they affect
## - ``9``, ``<no_utf8>`` - prevents the pattern from being interpreted as UTF, no matter ## the rest of it.
## what ##
## - ``A``, ``<anchored>`` - as if the pattern had a ``^`` at the beginning ## - ``(?i)`` - case insensitive
## - ``E``, ``<dollar_endonly>`` - DOLLAR\_ENDONLY ## - ``(?m)`` - multi-line: ``^`` and ``$`` match the beginning and end of
## - ``f``, ``<firstline>`` - fails if there is not a match on the first line
## - ``i``, ``<case_insensitive>`` - case insensitive
## - ``m``, ``<multiline>`` - multi-line, ``^`` and ``$`` match the beginning and end of
## lines, not of the subject string ## lines, not of the subject string
## - ``N``, ``<no_auto_capture>`` - turn off auto-capture, ``(?foo)`` is necessary to capture. ## - ``(?s)`` - ``.`` also matches newline (*dotall*)
## - ``s``, ``<dotall>`` - ``.`` matches newline ## - ``(?U)`` - expressions are not greedy by default. ``?`` can be added
## - ``U``, ``<ungreedy>`` - expressions are not greedy by default. ``?`` can be added to ## to a qualifier to make it greedy
## a qualifier to make it greedy. ## - ``(?x)`` - whitespace and comments (``#``) are ignored (*extended*)
## - ``W``, ``<ucp>`` - Unicode character properties; ``\w`` matches ``к``. ## - ``(?X)`` - character escapes without special meaning (``\w`` vs.
## - ``X``, ``<extra>`` - "Extra", character escapes without special meaning (``\w`` ## ``\a``) are errors (*extra*)
## vs. ``\a``) are errors ##
## - ``x``, ``<extended>`` - extended, comments (``#``) and newlines are ignored ## One or a combination of these options may appear only at the beginning
## (extended) ## of the pattern:
## - ``Y``, ``<no_start_optimize>`` - pcre.NO\_START\_OPTIMIZE, ##
## - ``<cr>`` - newlines are separated by ``\r`` ## - ``(*UTF8)`` - treat both the pattern and subject as UTF-8
## - ``<crlf>`` - newlines are separated by ``\r\n`` (Windows default) ## - ``(*UCP)`` - Unicode character properties; ``\w`` matches ``я``
## - ``<lf>`` - newlines are separated by ``\n`` (UNIX default) ## - ``(*U)`` - a combination of the two options above
## - ``<anycrlf>`` - newlines are separated by any of the above ## - ``(*FIRSTLINE*)`` - fails if there is not a match on the first line
## - ``<any>`` - newlines are separated by any of the above and Unicode ## - ``(*NO_AUTO_CAPTURE)`` - turn off auto-capture for groups;
## ``(?<name>...)`` can be used to capture
## - ``(*CR)`` - newlines are separated by ``\r``
## - ``(*LF)`` - newlines are separated by ``\n`` (UNIX default)
## - ``(*CRLF)`` - newlines are separated by ``\r\n`` (Windows default)
## - ``(*ANYCRLF)`` - newlines are separated by any of the above
## - ``(*ANY)`` - newlines are separated by any of the above and Unicode
## newlines: ## newlines:
## ##
## single characters VT (vertical tab, U+000B), FF (form feed, U+000C), ## single characters VT (vertical tab, U+000B), FF (form feed, U+000C),
@ -101,10 +97,15 @@ type
## are recognized only in UTF-8 mode. ## are recognized only in UTF-8 mode.
## — man pcre ## — man pcre
## ##
## - ``<bsr_anycrlf>`` - ``\R`` matches CR, LF, or CRLF ## - ``(*JAVASCRIPT_COMPAT)`` - JavaScript compatibility
## - ``<bsr_unicode>`` - ``\R`` matches any unicode newline ## - ``(*NO_STUDY)`` - turn off studying; study is enabled by default
## - ``<js>`` - Javascript compatibility ##
## - ``<no_study>`` - turn off studying; study is enabled by deafault ## For more details on the leading option groups, see the `Option
## Setting <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#OPTION_SETTING>`__
## and the `Newline
## Convention <http://man7.org/linux/man-pages/man3/pcresyntax.3.html#NEWLINE_CONVENTION>`__
## sections of the `PCRE syntax
## manual <http://man7.org/linux/man-pages/man3/pcresyntax.3.html>`__.
pattern*: string ## not nil pattern*: string ## not nil
pcreObj: ptr pcre.Pcre ## not nil pcreObj: ptr pcre.Pcre ## not nil
pcreExtra: ptr pcre.ExtraData ## nil pcreExtra: ptr pcre.ExtraData ## nil
@ -316,71 +317,61 @@ proc `==`*(a, b: RegexMatch): bool =
# Creation & Destruction {{{ # Creation & Destruction {{{
# PCRE Options {{{ # PCRE Options {{{
let Options: Table[string, int] = { const PcreOptions = {
"8" : pcre.UTF8, "NEVER_UTF": pcre.NEVER_UTF,
"utf8" : pcre.UTF8, "ANCHORED": pcre.ANCHORED,
"9" : pcre.NEVER_UTF, "DOLLAR_ENDONLY": pcre.DOLLAR_ENDONLY,
"no_utf8" : pcre.NEVER_UTF, "FIRSTLINE": pcre.FIRSTLINE,
"A" : pcre.ANCHORED, "NO_AUTO_CAPTURE": pcre.NO_AUTO_CAPTURE,
"anchored" : pcre.ANCHORED, "JAVASCRIPT_COMPAT": pcre.JAVASCRIPT_COMPAT,
# "C" : pcre.AUTO_CALLOUT, unsuported XXX "U": pcre.UTF8 or pcre.UCP
"E" : pcre.DOLLAR_ENDONLY,
"dollar_endonly" : pcre.DOLLAR_ENDONLY,
"f" : pcre.FIRSTLINE,
"firstline" : pcre.FIRSTLINE,
"i" : pcre.CASELESS,
"case_insensitive" : pcre.CASELESS,
"m" : pcre.MULTILINE,
"multiline" : pcre.MULTILINE,
"N" : pcre.NO_AUTO_CAPTURE,
"no_auto_capture" : pcre.NO_AUTO_CAPTURE,
"s" : pcre.DOTALL,
"dotall" : pcre.DOTALL,
"U" : pcre.UNGREEDY,
"ungreedy" : pcre.UNGREEDY,
"u" : pcre.UTF8,
"W" : pcre.UCP,
"ucp" : pcre.UCP,
"X" : pcre.EXTRA,
"extra" : pcre.EXTRA,
"x" : pcre.EXTENDED,
"extended" : pcre.EXTENDED,
"Y" : pcre.NO_START_OPTIMIZE,
"no_start_optimize" : pcre.NO_START_OPTIMIZE,
"any" : pcre.NEWLINE_ANY,
"anycrlf" : pcre.NEWLINE_ANYCRLF,
"cr" : pcre.NEWLINE_CR,
"crlf" : pcre.NEWLINE_CRLF,
"lf" : pcre.NEWLINE_LF,
"bsr_anycrlf" : pcre.BSR_ANYCRLF,
"bsr_unicode" : pcre.BSR_UNICODE,
"js" : pcre.JAVASCRIPT_COMPAT,
}.toTable }.toTable
proc tokenizeOptions(opts: string): tuple[flags: int, study: bool] = # Options that are supported inside regular expressions themselves
result = (0, true) const SkipOptions = [
"LIMIT_MATCH=", "LIMIT_RECURSION=", "NO_AUTO_POSSESS", "NO_START_OPT",
"UTF8", "UTF16", "UTF32", "UTF", "UCP",
"CR", "LF", "CRLF", "ANYCRLF", "ANY", "BSR_ANYCRLF", "BSR_UNICODE"
]
var longOpt: string = nil proc extractOptions(pattern: string): tuple[pattern: string, flags: int, study: bool] =
for i, c in opts: result = ("", 0, true)
# Handle long options {{{
if c == '<':
longOpt = ""
continue
if longOpt != nil: var optionStart = 0
if c == '>': var equals = false
if longOpt == "no_study": for i, c in pattern:
if optionStart == i:
if c != '(':
break
optionStart = i
elif optionStart == i-1:
if c != '*':
break
elif c == ')':
let name = pattern[optionStart+2 .. i-1]
if equals or name in SkipOptions:
result.pattern.add pattern[optionStart .. i]
elif PcreOptions.hasKey name:
result.flags = result.flags or PcreOptions[name]
elif name == "NO_STUDY":
result.study = false result.study = false
else: else:
result.flags = result.flags or Options.fget(longOpt) break
longOpt = nil optionStart = i+1
else: equals = false
longOpt.add(c.toLower)
continue elif not equals:
# }}} if c == '=':
equals = true
if pattern[optionStart+2 .. i] notin SkipOptions:
break
elif c notin {'A'..'Z', '0'..'9', '_'}:
break
result.pattern.add pattern[optionStart .. pattern.high]
result.flags = result.flags or Options.fget($c)
# }}} # }}}
type UncheckedArray {.unchecked.}[T] = array[0 .. 0, T] type UncheckedArray {.unchecked.}[T] = array[0 .. 0, T]
@ -411,24 +402,22 @@ proc getNameToNumberTable(pattern: Regex): Table[string, int] =
result[name] = num result[name] = num
proc initRegex(pattern: string, options: string): Regex = proc initRegex(pattern: string, flags: int, study = true): Regex =
new(result, destroyRegex) new(result, destroyRegex)
result.pattern = pattern result.pattern = pattern
var errorMsg: cstring var errorMsg: cstring
var errOffset: cint var errOffset: cint
let opts = tokenizeOptions(options)
result.pcreObj = pcre.compile(cstring(pattern), result.pcreObj = pcre.compile(cstring(pattern),
# better hope int is at least 4 bytes.. # better hope int is at least 4 bytes..
cint(opts.flags), addr errorMsg, cint(flags), addr errorMsg,
addr errOffset, nil) addr errOffset, nil)
if result.pcreObj == nil: if result.pcreObj == nil:
# failed to compile # failed to compile
raise SyntaxError(msg: $errorMsg, pos: errOffset, pattern: pattern) raise SyntaxError(msg: $errorMsg, pos: errOffset, pattern: pattern)
if opts.study: if study:
# XXX investigate JIT # XXX investigate JIT
result.pcreExtra = pcre.study(result.pcreObj, 0x0, addr errorMsg) result.pcreExtra = pcre.study(result.pcreObj, 0x0, addr errorMsg)
if errorMsg != nil: if errorMsg != nil:
@ -436,7 +425,9 @@ proc initRegex(pattern: string, options: string): Regex =
result.captureNameToId = result.getNameToNumberTable() result.captureNameToId = result.getNameToNumberTable()
proc re*(pattern: string, options = ""): Regex = initRegex(pattern, options) proc re*(pattern: string): Regex =
let (pattern, flags, study) = extractOptions(pattern)
initRegex(pattern, flags, study)
# }}} # }}}
# Operations {{{ # Operations {{{

4
test/find.nim
View file

@ -21,5 +21,5 @@ suite "find":
check("".findAll(re"") == @[""]) check("".findAll(re"") == @[""])
check("abc".findAll(re"") == @["", "", "", ""]) check("abc".findAll(re"") == @["", "", "", ""])
check("word word".findAll(re"\b") == @["", "", "", ""]) check("word word".findAll(re"\b") == @["", "", "", ""])
check("word\r\lword".findAll(re(r"$", "m<anycrlf>")) == @["", ""]) check("word\r\lword".findAll(re"(*ANYCRLF)(?m)$") == @["", ""])
check("слово слово".findAll(re(r"\b", "uW")) == @["", "", "", ""]) check("слово слово".findAll(re"(*U)\b") == @["", "", "", ""])

35
test/init.nim
View file

@ -1,24 +1,30 @@
import unittest import unittest, private/pcre
import nre include nre
suite "Test NRE initialization": suite "Test NRE initialization":
test "correct intialization": test "correct intialization":
check(re("[0-9]+") != nil) check(re("[0-9]+") != nil)
check(re("[0-9]+", "i") != nil) check(re("(?i)[0-9]+") != nil)
test "correct options": test "options":
expect(SyntaxError): # ValueError would be bad check(extractOptions("(*NEVER_UTF)") ==
discard re("[0-9]+", ("", pcre.NEVER_UTF, true))
"89AEfimNsUWXxY<any><anycrlf><cr><crlf><lf><bsr_anycrlf><bsr_unicode><js><no_study>") check(extractOptions("(*UTF8)(*ANCHORED)(*UCP)z") ==
expect(SyntaxError): ("(*UTF8)(*UCP)z", pcre.ANCHORED, true))
discard re("[0-9]+", check(extractOptions("(*ANCHORED)(*UTF8)(*JAVASCRIPT_COMPAT)z") ==
"<utf8><no_utf8><anchored><dollar_endonly><firstline>" & ("(*UTF8)z", pcre.ANCHORED or pcre.JAVASCRIPT_COMPAT, true))
"<case_insensitive><multiline><no_auto_capture><dotall><ungreedy>" &
"<ucp><extra><extended><no_start_optimize>") check(extractOptions("(*NO_STUDY)(") == ("(", 0, false))
check(extractOptions("(*LIMIT_MATCH=6)(*ANCHORED)z") ==
("(*LIMIT_MATCH=6)z", pcre.ANCHORED, true))
test "incorrect options": test "incorrect options":
expect(KeyError): discard re("[0-9]+", "a") for s in ["CR", "(CR", "(*CR", "(*abc)", "(*abc)CR",
expect(KeyError): discard re("[0-9]+", "<does_not_exist>") "(?i)",
"(*LIMIT_MATCH=5", "(*NO_AUTO_POSSESS=5)"]:
let ss = s & "(*NEVER_UTF)"
check(extractOptions(ss) == (ss, 0, true))
test "invalid regex": test "invalid regex":
expect(SyntaxError): discard re("[0-9") expect(SyntaxError): discard re("[0-9")
@ -28,4 +34,3 @@ suite "Test NRE initialization":
let ex = SyntaxError(getCurrentException()) let ex = SyntaxError(getCurrentException())
check(ex.pos == 4) check(ex.pos == 4)
check(ex.pattern == "[0-9") check(ex.pattern == "[0-9")

4
test/misc.nim
View file

@ -2,8 +2,8 @@ import unittest, nre, strutils, optional_t.nonstrict
suite "Misc tests": suite "Misc tests":
test "unicode": test "unicode":
check("".find(re("", "8")).match == "") check("".find(re"(*UTF8)").match == "")
check("перевірка".replace(re(r"\w", "uW"), "") == "") check("перевірка".replace(re"(*U)\w", "") == "")
test "empty or non-empty match": test "empty or non-empty match":
check("abc".findall(re"|.").join(":") == ":a::b::c:") check("abc".findall(re"|.").join(":") == ":a::b::c:")

4
test/split.nim
View file

@ -21,8 +21,8 @@ suite "string splitting":
check("12345".split(re("")) == @["1", "2", "3", "4", "5"]) check("12345".split(re("")) == @["1", "2", "3", "4", "5"])
check("".split(re"") == newSeq[string]()) check("".split(re"") == newSeq[string]())
check("word word".split(re"\b") == @["word", " ", "word"]) check("word word".split(re"\b") == @["word", " ", "word"])
check("word\r\lword".split(re(r"$", "m<anycrlf>")) == @["word", "\r\lword"]) check("word\r\lword".split(re"(*ANYCRLF)(?m)$") == @["word", "\r\lword"])
check("слово слово".split(re(r"(\b)", "uW")) == @["", "слово", "", " ", "", "слово", ""]) check("слово слово".split(re"(*U)(\b)") == @["", "слово", "", " ", "", "слово", ""])
test "perl split tests": test "perl split tests":
check("forty-two" .split(re"") .join(",") == "f,o,r,t,y,-,t,w,o") check("forty-two" .split(re"") .join(",") == "f,o,r,t,y,-,t,w,o")