Base re off of nre

lib/impure/re.nim

@@ -2,16 +2,13 @@
 #
 #            Nim's Runtime Library
 #        (c) Copyright 2012 Andreas Rumpf
+#        (c) Copyright 2015 Oleh Prypin
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
 
-## Regular expression support for Nim. Consider using the pegs module instead.
+## Regular expression support for Nim.
-##
-## There is an alternative regular expressions library with a more unified API:
-## `nre <https://github.com/flaviut/nre>`_. It may be added to the standard
-## library in the future, instead of `re`.
 ##
 ## **Note:** The 're' proc defaults to the **extended regular expression
 ## syntax** which lets you use whitespace freely to make your regexes readable.
@@ -28,13 +25,15 @@
 ## .. include:: ../doc/regexprs.txt
 ##
 
-import
+
-  pcre, strutils, rtarrays
+import strutils
+import nre, options
+
+type Regex = nre.Regex
 
 const
-  MaxSubpatterns* = 20
+  MaxSubpatterns* {.deprecated.} = 1000
-    ## defines the maximum number of subpatterns that can be captured.
+    ## This is obsolete. There is no limit on the number of subpatterns.
-    ## This limit still exists for ``replacef`` and ``parallelReplace``.
 
 type
   RegexFlag* = enum     ## options for regular expressions
@@ -44,66 +43,61 @@ type
     reExtended = 3,      ## ignore whitespace and ``#`` comments
     reStudy = 4          ## study the expression (may be omitted if the
                          ## expression will be used only once)
-  
-  RegexDesc = object 
-    h: ptr Pcre
-    e: ptr ExtraData
-  
-  Regex* = ref RegexDesc ## a compiled regular expression
 
   RegexError* = object of ValueError
     ## is raised if the pattern is no valid regular expression.
 
-{.deprecated: [TRegexFlag: RegexFlag, TRegexDesc: RegexDesc, TRegex: Regex,
-    EInvalidRegEx: RegexError].}
-
-proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} =
-  var e: ref RegexError
-  new(e)
-  e.msg = msg
-  raise e
-
-proc rawCompile(pattern: string, flags: cint): ptr Pcre =
-  var
-    msg: cstring
-    offset: cint
-  result = pcre.compile(pattern, flags, addr(msg), addr(offset), nil)
-  if result == nil:
-    raiseInvalidRegex($msg & "\n" & pattern & "\n" & spaces(offset) & "^\n")
-
-proc finalizeRegEx(x: Regex) =
-  # XXX This is a hack, but PCRE does not export its "free" function properly.
-  # Sigh. The hack relies on PCRE's implementation (see ``pcre_get.c``).
-  # Fortunately the implementation is unlikely to change.
-  pcre.free_substring(cast[cstring](x.h))
-  if not isNil(x.e):
-    pcre.free_substring(cast[cstring](x.e))
 
 proc re*(s: string, flags = {reExtended, reStudy}): Regex =
   ## Constructor of regular expressions. Note that Nim's
   ## extended raw string literals support this syntax ``re"[abc]"`` as
   ## a short form for ``re(r"[abc]")``.
-  new(result, finalizeRegEx)
+  var options = ""
-  result.h = rawCompile(s, cast[cint](flags - {reStudy}))
+  if reStudy notin flags:
-  if reStudy in flags:
+    options.add "(*NO_STUDY)"
-    var msg: cstring
+  for t in [
-    result.e = pcre.study(result.h, 0, addr msg)
+    (reIgnoreCase, "(?i)"),
-    if not isNil(msg): raiseInvalidRegex($msg)
+    (reMultiLine, "(?m)"),
+    (reDotAll, "(?s)"),
+    (reExtended, "(?x)")
+  ]:
+    if t[0] in flags:
+      options.add t[1]
+  try:
+    return nre.re(options & s)
+  except nre.SyntaxError:
+    let e = nre.SyntaxError(getCurrentException())
+    raise newException(RegexError, e.msg & "\n" & e.pattern & "\n" &
+                                   repeat(' ', e.pos) & "^\n")
+  except nre.StudyError:
+    let e = nre.StudyError(getCurrentException())
+    raise newException(RegexError, "Study error: " & e.msg)
 
-proc matchOrFind(s: string, pattern: Regex, matches: var openArray[string],
+
-                 start, flags: cint): cint =
+template execute(f: expr, notFound: stmt): stmt {.immediate, dirty.} =
-  var
+  let mm = f(s, pattern, start)
-    rtarray = initRtArray[cint]((matches.len+1)*3)
+  var m: RegexMatch
-    rawMatches = rtarray.getRawData
+  if mm.isSome:
-    res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start, flags,
+    m = mm.unsafeGet()
-      cast[ptr cint](rawMatches), (matches.len+1).cint*3)
+  else:
-  if res < 0'i32: return res
+    notFound
-  for i in 1..int(res)-1:
+
-    var a = rawMatches[i * 2]
+template stringCaptures(): stmt {.immediate, dirty.} =
-    var b = rawMatches[i * 2 + 1]
+  for i in 0 .. <pattern.captureCount:
-    if a >= 0'i32: matches[i-1] = substr(s, int(a), int(b)-1)
+    matches[i] = m.captures[i]
-    else: matches[i-1] = nil
+
-  return rawMatches[1] - rawMatches[0]
+template boundCaptures(): stmt {.immediate, dirty.} =
+  for i in 0 .. <pattern.captureCount:
+    let b = m.captureBounds[i]
+    matches[i] =
+      if b.isSome: (b.unsafeGet().a, <b.unsafeGet().b)
+      else: (-1, 0)
+
+template returnBounds(): stmt {.immediate, dirty.} =
+  return (m.matchBounds.a, <m.matchBounds.b)
+
+template returnLength(): stmt {.immediate, dirty.} =
+  return m.matchBounds.b - m.matchBounds.a
 
 proc findBounds*(s: string, pattern: Regex, matches: var openArray[string],
                  start = 0): tuple[first, last: int] =
@@ -111,18 +105,10 @@ proc findBounds*(s: string, pattern: Regex, matches: var openArray[string],
   ## and the captured
   ## substrings in the array `matches`. If it does not match, nothing
   ## is written into `matches` and ``(-1,0)`` is returned.
-  var
+  execute nre.find:
-    rtarray = initRtArray[cint]((matches.len+1)*3)
+    return (-1, 0)
-    rawMatches = rtarray.getRawData
+  stringCaptures()
-    res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32,
+  returnBounds()
-      cast[ptr cint](rawMatches), (matches.len+1).cint*3)
-  if res < 0'i32: return (-1, 0)
-  for i in 1..int(res)-1:
-    var a = rawMatches[i * 2]
-    var b = rawMatches[i * 2 + 1]
-    if a >= 0'i32: matches[i-1] = substr(s, int(a), int(b)-1)
-    else: matches[i-1] = nil
-  return (rawMatches[0].int, rawMatches[1].int - 1)
 
 proc findBounds*(s: string, pattern: Regex,
                  matches: var openArray[tuple[first, last: int]],
@@ -131,56 +117,42 @@ proc findBounds*(s: string, pattern: Regex,
   ## and the captured substrings in the array `matches`.
   ## If it does not match, nothing is written into `matches` and
   ## ``(-1,0)`` is returned.
-  var
+  execute nre.find:
-    rtarray = initRtArray[cint]((matches.len+1)*3)
+    return (-1, 0)
-    rawMatches = rtarray.getRawData
+  boundCaptures()
-    res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32,
+  returnBounds()
-      cast[ptr cint](rawMatches), (matches.len+1).cint*3)
-  if res < 0'i32: return (-1, 0)
-  for i in 1..int(res)-1:
-    var a = rawMatches[i * 2]
-    var b = rawMatches[i * 2 + 1]
-    if a >= 0'i32: matches[i-1] = (int(a), int(b)-1)
-    else: matches[i-1] = (-1,0)
-  return (rawMatches[0].int, rawMatches[1].int - 1)
 
 proc findBounds*(s: string, pattern: Regex,
                  start = 0): tuple[first, last: int] =
-  ## returns the starting position and end position of ``pattern`` in ``s``.
+  ## returns the starting position of `pattern` in `s`. If it does not
-  ## If it does not match, ``(-1,0)`` is returned.
+  ## match, ``(-1,0)`` is returned.
-  var
+  execute nre.find:
-    rtarray = initRtArray[cint](3)
+    return (-1, 0)
-    rawMatches = rtarray.getRawData
+  returnBounds()
-    res = pcre.exec(pattern.h, nil, s, len(s).cint, start.cint, 0'i32,
-      cast[ptr cint](rawMatches), 3)
-  if res < 0'i32: return (int(res), 0)
-  return (int(rawMatches[0]), int(rawMatches[1]-1))
-
-proc matchOrFind(s: string, pattern: Regex, start, flags: cint): cint =
-  var
-    rtarray = initRtArray[cint](3)
-    rawMatches = rtarray.getRawData
-  result = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start, flags,
-                    cast[ptr cint](rawMatches), 3)
-  if result >= 0'i32:
-    result = rawMatches[1] - rawMatches[0]
 
 proc matchLen*(s: string, pattern: Regex, matches: var openArray[string],
-              start = 0): int =
+               start = 0): int =
   ## the same as ``match``, but it returns the length of the match,
   ## if there is no match, -1 is returned. Note that a match length
   ## of zero can happen.
-  return matchOrFind(s, pattern, matches, start.cint, pcre.ANCHORED)
+  execute nre.match:
+    return -1
+  stringCaptures()
+  returnLength()
 
 proc matchLen*(s: string, pattern: Regex, start = 0): int =
   ## the same as ``match``, but it returns the length of the match,
   ## if there is no match, -1 is returned. Note that a match length
   ## of zero can happen.
-  return matchOrFind(s, pattern, start.cint, pcre.ANCHORED)
+  execute nre.match:
+    return -1
+  returnLength()
 
 proc match*(s: string, pattern: Regex, start = 0): bool =
   ## returns ``true`` if ``s[start..]`` matches the ``pattern``.
-  result = matchLen(s, pattern, start) != -1
+  execute nre.match:
+    return false
+  return true
 
 proc match*(s: string, pattern: Regex, matches: var openArray[string],
            start = 0): bool =
@@ -188,65 +160,45 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string],
   ## the captured substrings in the array ``matches``. If it does not
   ## match, nothing is written into ``matches`` and ``false`` is
   ## returned.
-  result = matchLen(s, pattern, matches, start) != -1
+  execute nre.match:
+    return false
+  stringCaptures()
+  return true
 
 proc find*(s: string, pattern: Regex, matches: var openArray[string],
            start = 0): int =
   ## returns the starting position of ``pattern`` in ``s`` and the captured
   ## substrings in the array ``matches``. If it does not match, nothing
   ## is written into ``matches`` and -1 is returned.
-  var
+  execute nre.find:
-    rtarray = initRtArray[cint]((matches.len+1)*3)
+    return -1
-    rawMatches = rtarray.getRawData
+  stringCaptures()
-    res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32,
+  return m.matchBounds.a
-      cast[ptr cint](rawMatches), (matches.len+1).cint*3)
-  if res < 0'i32: return res
-  for i in 1..int(res)-1:
-    var a = rawMatches[i * 2]
-    var b = rawMatches[i * 2 + 1]
-    if a >= 0'i32: matches[i-1] = substr(s, int(a), int(b)-1)
-    else: matches[i-1] = nil
-  return rawMatches[0]
 
 proc find*(s: string, pattern: Regex, start = 0): int =
   ## returns the starting position of ``pattern`` in ``s``. If it does not
   ## match, -1 is returned.
-  var
+  execute nre.find:
-    rtarray = initRtArray[cint](3)
+    return -1
-    rawMatches = rtarray.getRawData
+  return m.matchBounds.a
-    res = pcre.exec(pattern.h, nil, s, len(s).cint, start.cint, 0'i32,
-      cast[ptr cint](rawMatches), 3)
-  if res < 0'i32: return res
-  return rawMatches[0]
 
 iterator findAll*(s: string, pattern: Regex, start = 0): string =
   ## Yields all matching *substrings* of `s` that match `pattern`.
   ##
   ## Note that since this is an iterator you should not modify the string you
   ## are iterating over: bad things could happen.
-  var
+  for m in nre.findIter(s, pattern, start):
-    i = int32(start)
+    yield $m
-    rtarray = initRtArray[cint](3)
-    rawMatches = rtarray.getRawData
-  while true:
-    let res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, i, 0'i32,
-      cast[ptr cint](rawMatches), 3)
-    if res < 0'i32: break
-    let a = rawMatches[0]
-    let b = rawMatches[1]
-    if a == b and a == i: break
-    yield substr(s, int(a), int(b)-1)
-    i = b
 
 proc findAll*(s: string, pattern: Regex, start = 0): seq[string] =
   ## returns all matching *substrings* of `s` that match `pattern`.
   ## If it does not match, @[] is returned.
-  accumulateResult(findAll(s, pattern, start))
+  accumulateResult findAll(s, pattern, start)
 
 when not defined(nimhygiene):
   {.pragma: inject.}
 
-template `=~` *(s: string, pattern: Regex): expr =
+template `=~`*(s: string, pattern: Regex, start = 0): expr =
   ## This calls ``match`` with an implicit declared ``matches`` array that
   ## can be used in the scope of the ``=~`` call:
   ##
@@ -264,10 +216,11 @@ template `=~` *(s: string, pattern: Regex): expr =
   ##   else:
   ##     echo("syntax error")
   ##
-  bind MaxSubpatterns
+  let mm = nre.match(s, pattern, start)
-  when not declaredInScope(matches):
+  var matches {.inject.}: seq[string]
-    var matches {.inject.}: array[MaxSubpatterns, string]
+  if mm.isSome:
-  match(s, pattern, matches)
+    matches = mm.unsafeGet().captures.toSeq()
+  mm.isSome
 
 # ------------------------- more string handling ------------------------------
 
@@ -286,8 +239,9 @@ proc startsWith*(s: string, prefix: Regex): bool =
 
 proc endsWith*(s: string, suffix: Regex): bool =
   ## returns true if `s` ends with the pattern `prefix`
-  for i in 0 .. s.len-1:
+  for i in 0 .. s.high:
-    if matchLen(s, suffix, i) == s.len - i: return true
+    if matchLen(s, suffix, i) == s.len - i:
+      return true
 
 proc replace*(s: string, sub: Regex, by = ""): string =
   ## Replaces `sub` in `s` by the string `by`. Captures cannot be
@@ -301,15 +255,7 @@ proc replace*(s: string, sub: Regex, by = ""): string =
   ## .. code-block:: nim
   ##
   ##   "; "
-  result = ""
+  return nre.replace(s, sub, by.replace("$", "$$"))
-  var prev = 0
-  while true:
-    var match = findBounds(s, sub, prev)
-    if match.first < 0: break
-    add(result, substr(s, prev, match.first-1))
-    add(result, by)
-    prev = match.last + 1
-  add(result, substr(s, prev))
 
 proc replacef*(s: string, sub: Regex, by: string): string =
   ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by`
@@ -323,18 +269,7 @@ proc replacef*(s: string, sub: Regex, by: string): string =
   ## .. code-block:: nim
   ##
   ## "var1<-keykey; val2<-key2key2"
-  result = ""
+  return nre.replace(s, sub, by)
-  var caps: array[MaxSubpatterns, string]
-  var prev = 0
-  while true:
-    var match = findBounds(s, sub, caps, prev)
-    if match.first < 0: break
-    assert result != nil
-    assert s != nil
-    add(result, substr(s, prev, match.first-1))
-    addf(result, by, caps)
-    prev = match.last + 1
-  add(result, substr(s, prev))
 
 proc parallelReplace*(s: string, subs: openArray[
                       tuple[pattern: Regex, repl: string]]): string =
@@ -342,19 +277,20 @@ proc parallelReplace*(s: string, subs: openArray[
   ## applied in parallel.
   result = ""
   var i = 0
-  var caps: array[MaxSubpatterns, string]
   while i < s.len:
     block searchSubs:
-      for j in 0..high(subs):
+      for sub in subs:
-        var x = matchLen(s, subs[j][0], caps, i)
+        let pattern = sub.pattern
-        if x > 0:
+        let start = i
-          addf(result, subs[j][1], caps)
+        execute nre.match:
-          inc(i, x)
+          continue
-          break searchSubs
+        result.addf sub.repl, m.captures.toSeq()
-      add(result, s[i])
+        i += m.matchBounds.b - m.matchBounds.a
-      inc(i)
+        break searchSubs
+      result.add s[i]
+      inc i
   # copy the rest:
-  add(result, substr(s, i))
+  result.add s.substr(i)
 
 proc transformFile*(infile, outfile: string,
                     subs: openArray[tuple[pattern: Regex, repl: string]]) =
@@ -377,67 +313,42 @@ iterator split*(s: string, sep: Regex): string =
   ## Results in:
   ##
   ## .. code-block:: nim
-  ##   ""
   ##   "this"
   ##   "is"
   ##   "an"
   ##   "example"
-  ##   ""
   ##
-  var
+  for m in nre.split(s, sep):
-    first = -1
+    yield m
-    last = -1
-  while last < len(s):
-    var x = matchLen(s, sep, last)
-    if x > 0: inc(last, x)
-    first = last
-    if x == 0: inc(last)
-    while last < len(s):
-      x = matchLen(s, sep, last)
-      if x >= 0: break
-      inc(last)
-    if first <= last:
-      yield substr(s, first, last-1)
 
 proc split*(s: string, sep: Regex): seq[string] =
   ## Splits the string `s` into substrings.
-  accumulateResult(split(s, sep))
+  return nre.split(s, sep)
 
 proc escapeRe*(s: string): string =
   ## escapes `s` so that it is matched verbatim when used as a regular
   ## expression.
-  result = ""
+  return nre.escapeRe(s)
-  for c in items(s):
-    case c
-    of 'a'..'z', 'A'..'Z', '0'..'9', '_':
-      result.add(c)
-    else:
-      result.add("\\x")
-      result.add(toHex(ord(c), 2))
 
 const ## common regular expressions
-  reIdentifier* {.deprecated.} = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b"
+  reIdentifier* = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b"  ## describes an identifier
-    ## describes an identifier
+  reNatural* = r"\b\d+\b" ## describes a natural number
-  reNatural* {.deprecated.} = r"\b\d+\b"
+  reInteger* = r"\b[-+]?\d+\b" ## describes an integer
-    ## describes a natural number
+  reHex* = r"\b0[xX][0-9a-fA-F]+\b" ## describes a hexadecimal number
-  reInteger* {.deprecated.} = r"\b[-+]?\d+\b"
+  reBinary* = r"\b0[bB][01]+\b" ## describes a binary number (example: 0b11101)
-    ## describes an integer
+  reOctal* = r"\b0[oO][0-7]+\b" ## describes an octal number (example: 0o777)
-  reHex* {.deprecated.} = r"\b0[xX][0-9a-fA-F]+\b"
+  reFloat* = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b"
-    ## describes a hexadecimal number
-  reBinary* {.deprecated.} = r"\b0[bB][01]+\b"
-    ## describes a binary number (example: 0b11101)
-  reOctal* {.deprecated.} = r"\b0[oO][0-7]+\b"
-    ## describes an octal number (example: 0o777)
-  reFloat* {.deprecated.} = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b"
     ## describes a floating point number
-  reEmail* {.deprecated.} = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" &
+  reEmail* {.deprecated.} =
-                            r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@" &
+    r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" &
-                            r"(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" &
+    r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)" &
-                            r"(?:[a-zA-Z]{2}|com|org|net|gov|mil|biz|" &
+    r"*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" &
-                            r"info|mobi|name|aero|jobs|museum)\b"
+    r"(?:[a-zA-Z]{2}|com|org|" &
+    r"net|gov|mil|biz|info|mobi|name|aero|jobs|museum)\b"
     ## describes a common email address
-  reURL* {.deprecated.} = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms-help)" &
+  reURL* {.deprecated.} =
-                          r":((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b"
+    r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms\-help):" &
+    r"((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b"
     ## describes an URL
 
 when isMainModule:
@@ -455,7 +366,7 @@ when isMainModule:
 
   assert find("_____abc_______", re"abc") == 5
 
-  var matches: array[6, string]
+  var matches: array[0..5, string]
   if match("abcdefg", re"c(d)ef(g)", matches, 2):
     assert matches[0] == "d"
     assert matches[1] == "g"
@@ -464,8 +375,7 @@ when isMainModule:
 
   if "abc" =~ re"(a)bcxyz|(\w+)":
     assert matches[1] == "abc"
-  else:
+  else: assert false
-    assert false
 
   if "abc" =~ re"(cba)?.*":
     assert matches[0] == nil
@@ -481,16 +391,9 @@ when isMainModule:
   assert("var1=key; var2=key2".replace(re"(\w+)=(\w+)", "$1<-$2$2") ==
          "$1<-$2$2; $1<-$2$2")
 
-  var accum: seq[string] = @[]
+  var accum: seq[string] = split("00232this02939is39an22example111", re"\d+")
-  for word in split("00232this02939is39an22example111", re"\d+"):
-    accum.add(word)
   assert(accum == @["", "this", "is", "an", "example", ""])
 
-  accum = @[]
-  for word in split("AAA :   : BBB", re"\s*:\s*"):
-    accum.add(word)
-  assert(accum == @["AAA", "", "BBB"])
-
   for x in findAll("abcdef", re"^{.}", 3):
     assert x == "d"
   accum = @[]
@@ -502,9 +405,10 @@ when isMainModule:
   assert("XYZ".match(re"^\d*") == true)
 
   block:
-    var matches: array[16, string]
+    var matches: array[0..15, string]
-    if match("abcdefghijklmnop", re"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)(p)", matches):
+    if match("abcdefghijklmnop",
-      for i in 0..matches.high:
+             re"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)(p)", matches):
+      for i in 0 .. matches.high:
         assert matches[i] == $chr(i + 'a'.ord)
     else:
       assert false