New re version from AMK
diff --git a/Lib/re.py b/Lib/re.py
index bf74808..b3c266c 100644
--- a/Lib/re.py
+++ b/Lib/re.py
@@ -1,7 +1,5 @@
#!/usr/bin/env python
# -*- mode: python -*-
-# $Id$
-
import sys
import string
@@ -12,11 +10,12 @@
#
# pcre.error and re.error should be the same, since exceptions can be
-# raised from either module.
+# raised from either module.
# compilation flags
I = IGNORECASE
+L = LOCALE
M = MULTILINE
S = DOTALL
X = VERBOSE
@@ -61,8 +60,25 @@
pattern = _cachecompile(pattern)
return pattern.split(string, maxsplit)
+def escape(pattern):
+ "Escape all non-alphanumeric characters in pattern."
+ result = []
+ alphanum=string.letters+'_'+string.digits
+ for char in pattern:
+ if char not in alphanum:
+ result.append('\\')
+ result.append(char)
+ return string.join(result, '')
+
+def compile(pattern, flags=0):
+ "Compile a regular expression pattern, returning a RegexObject."
+ groupindex={}
+ code=pcre_compile(pattern, flags, groupindex)
+ return RegexObject(pattern, flags, code, groupindex)
+
+
#
-#
+# Class definitions
#
class RegexObject:
@@ -71,31 +87,54 @@
self.flags = flags
self.pattern = pattern
self.groupindex = groupindex
- def search(self, string, pos=0):
- regs = self.code.match(string, pos, 0)
+
+ def search(self, string, pos=0, endpos=None):
+ """Scan through string looking for a match to the pattern, returning
+ a MatchObject instance, or None if no match was found."""
+
+ if endpos is None or endpos>len(string):
+ endpos=len(string)
+ if endpos<pos: endpos=pos
+ regs = self.code.match(string, pos, endpos, 0)
if regs is None:
return None
- self.num_regs=len(regs)
+ self._num_regs=len(regs)
return MatchObject(self,
string,
- pos,
+ pos, endpos,
regs)
- def match(self, string, pos=0):
- regs = self.code.match(string, pos, ANCHORED)
+ def match(self, string, pos=0, endpos=None):
+ """Try to apply the pattern at the start of the string, returning
+ a MatchObject instance, or None if no match was found."""
+
+ if endpos is None or endpos>len(string):
+ endpos=len(string)
+ if endpos<pos: endpos=pos
+ regs = self.code.match(string, pos, endpos, ANCHORED)
if regs is None:
return None
- self.num_regs=len(regs)
+ self._num_regs=len(regs)
return MatchObject(self,
string,
- pos,
+ pos, endpos,
regs)
def sub(self, repl, string, count=0):
+ """Return the string obtained by replacing the leftmost
+ non-overlapping occurrences of the pattern in string by the
+ replacement repl"""
+
return self.subn(repl, string, count)[0]
- def subn(self, repl, source, count=0):
+ def subn(self, repl, source, count=0):
+ """Return a 2-tuple containing (new_string, number).
+ new_string is the string obtained by replacing the leftmost
+ non-overlapping occurrences of the pattern in string by the
+ replacement repl. number is the number of substitutions that
+ were made."""
+
if count < 0:
raise error, "negative substitution count"
if count == 0:
@@ -134,6 +173,9 @@
return (string.join(results, ''), n)
def split(self, source, maxsplit=0):
+ """Split \var{string} by the occurrences of the pattern,
+ returning a list containing the resulting substrings."""
+
if maxsplit < 0:
raise error, "negative split count"
if maxsplit == 0:
@@ -156,21 +198,39 @@
pos = pos+1
continue
results.append(source[lastmatch:i])
- g = m.group()
+ g = m.groups()
if g:
+ if type(g)==type( "" ): g = [g]
results[len(results):] = list(g)
pos = lastmatch = j
results.append(source[lastmatch:])
return results
+ # The following 3 functions were contributed by Mike Fletcher, and
+ # allow pickling and unpickling of RegexObject instances.
+ def __getinitargs__(self):
+ return (None,None,None,None) # any 4 elements, to work around
+ # problems with the
+ # pickle/cPickle modules not yet
+ # ignoring the __init__ function
+ def __getstate__(self):
+ return self.pattern, self.flags, self.groupindex
+ def __setstate__(self, statetuple):
+ self.pattern = statetuple[0]
+ self.flags = statetuple[1]
+ self.groupindex = statetuple[2]
+ self.code = apply(pcre_compile, statetuple)
+
class MatchObject:
- def __init__(self, re, string, pos, regs):
+ def __init__(self, re, string, pos, endpos, regs):
self.re = re
self.string = string
- self.pos = pos
+ self.pos = pos
+ self.endpos = endpos
self.regs = regs
- def start(self, g):
+ def start(self, g = 0):
+ "Return the start of the substring matched by group g"
if type(g) == type(''):
try:
g = self.re.groupindex[g]
@@ -178,7 +238,8 @@
raise IndexError, ('group "' + g + '" is undefined')
return self.regs[g][0]
- def end(self, g):
+ def end(self, g = 0):
+ "Return the end of the substring matched by group g"
if type(g) == type(''):
try:
g = self.re.groupindex[g]
@@ -186,7 +247,9 @@
raise IndexError, ('group "' + g + '" is undefined')
return self.regs[g][1]
- def span(self, g):
+ def span(self, g = 0):
+ """Return a tuple containing the start,end of the substring
+ matched by group g"""
if type(g) == type(''):
try:
g = self.re.groupindex[g]
@@ -194,12 +257,18 @@
raise IndexError, ('group "' + g + '" is undefined')
return self.regs[g]
+ def groups(self):
+ "Return a tuple containing all subgroups of the match object"
+
+ # If _num_regs==1, we don't want to call self.group with an
+ # empty tuple.
+ if self.re._num_regs == 1: return ()
+ return apply(self.group, tuple(range(1, self.re._num_regs) ) )
+
def group(self, *groups):
+ "Return one or more groups of the match."
if len(groups) == 0:
- groups = range(1, self.re.num_regs)
- use_all = 1
- else:
- use_all = 0
+ groups = (0,)
result = []
for g in groups:
if type(g) == type(''):
@@ -212,25 +281,10 @@
result.append(None)
else:
result.append(self.string[self.regs[g][0]:self.regs[g][1]])
- if use_all or len(result) > 1:
+ if len(result) > 1:
return tuple(result)
elif len(result) == 1:
return result[0]
else:
return ()
-def escape(pattern):
- result = []
- alphanum=string.letters+'_'+string.digits
- for char in pattern:
- if char not in alphanum:
- result.append('\\')
- result.append(char)
- return string.join(result, '')
-
-def compile(pattern, flags=0):
- groupindex={}
- code=pcre_compile(pattern, flags, groupindex)
- return RegexObject(pattern, flags, code, groupindex)
-
-
diff --git a/Lib/test/re_tests.py b/Lib/test/re_tests.py
index 9447dd2..9bbfa54 100755
--- a/Lib/test/re_tests.py
+++ b/Lib/test/re_tests.py
@@ -2,7 +2,7 @@
# -*- mode: python -*-
# $Id$
-# Re test suite and benchmark suite v1.5a2
+# Re test suite and benchmark suite v1.5b2
# The 3 possible outcomes for each pattern
[SUCCEED, FAIL, SYNTAX_ERROR] = range(3)
@@ -47,7 +47,62 @@
#
# If the regex isn't expected to work, the latter two elements can be omitted.
-tests = [
+tests = [
+ # Test ?P< and ?P= extensions
+ ('(?P<foo_123', '', SYNTAX_ERROR), # Unterminated group identifier
+ ('(?P<1>a)', '', SYNTAX_ERROR), # Begins with a digit
+ ('(?P<!>a)', '', SYNTAX_ERROR), # Begins with an illegal char
+ ('(?P<foo!>a)', '', SYNTAX_ERROR), # Begins with an illegal char
+
+ # Same tests, for the ?P= form
+ ('(?P<foo_123>a)(?P=foo_123', 'aa', SYNTAX_ERROR),
+ ('(?P<foo_123>a)(?P=1)', 'aa', SYNTAX_ERROR),
+ ('(?P<foo_123>a)(?P=!)', 'aa', SYNTAX_ERROR),
+ ('(?P<foo_123>a)(?P=foo_124', 'aa', SYNTAX_ERROR), # Backref to undefined group
+
+ ('(?P<foo_123>a)', 'a', SUCCEED, 'g1', 'a'),
+ ('(?P<foo_123>a)(?P=foo_123)', 'aa', SUCCEED, 'g1', 'a'),
+
+ # Test octal escapes
+ ('\\1', 'a', SYNTAX_ERROR),
+ ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'),
+ ('\\141', 'a', SUCCEED, 'found', 'a'),
+ ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'),
+
+ # Test that a literal \0 is handled everywhere
+ ('\0', '\0', SUCCEED, 'found', '\0'),
+ (r'\0', '\0', SUCCEED, 'found', '\0'),
+ ('[\0a]', '\0', SUCCEED, 'found', '\0'),
+ ('[a\0]', '\0', SUCCEED, 'found', '\0'),
+ ('[^a\0]', '\0', FAIL),
+ (r'[\0a]', '\0', SUCCEED, 'found', '\0'),
+ (r'[a\0]', '\0', SUCCEED, 'found', '\0'),
+ (r'[^a\0]', '\0', FAIL),
+
+ # Test various letter escapes
+ (r'\a[\b]\f\n\r\t\v', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'),
+ (r'[\a][\b][\f][\n][\r][\t][\v]', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'),
+ (r'\u', '', SYNTAX_ERROR), # A Perl escape
+ (r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'),
+ (r'\xff', '\377', SUCCEED, 'found', chr(255)),
+ (r'\x00ffffffffffffff', '\377', SUCCEED, 'found', chr(255)),
+ (r'\x00f', '\017', SUCCEED, 'found', chr(15)),
+ (r'\x00fe', '\376', SUCCEED, 'found', chr(254)),
+
+ (r"^\w+=(\\[\000-\277]|[^\n\\])*", "SRC=eval.c g.c blah blah blah \\\\\n\tapes.c",
+ SUCCEED, 'found', "SRC=eval.c g.c blah blah blah \\\\"),
+
+ # Test that . only matches \n in DOTALL mode
+ ('a.b', 'acb', SUCCEED, 'found', 'acb'),
+ ('a.b', 'a\nb', FAIL),
+ ('a.*b', 'acc\nccb', FAIL),
+ ('a.{4,5}b', 'acc\nccb', FAIL),
+ ('a.b', 'a\rb', SUCCEED, 'found', 'a\rb'),
+ ('a.b(?s)', 'a\nb', SUCCEED, 'found', 'a\nb'),
+ ('a.*(?s)b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'),
+ ('(?s)a.{4,5}b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'),
+ ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'),
+
('abc', 'abc', SUCCEED, 'found', 'abc'),
('abc', 'xbc', FAIL),
('abc', 'axc', FAIL),
@@ -338,8 +393,9 @@
('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'),
('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'),
('[k]', 'ab', FAIL),
- ##('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'),
- ##('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'),
+# XXX
+# ('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'),
+# ('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'),
('a[-]?c', 'ac', SUCCEED, 'found', 'ac'),
('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'),
@@ -470,15 +526,14 @@
('(?i)(.*)c(.*)', 'ABCDE', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCDE-AB-DE'),
('(?i)\\((.*), (.*)\\)', '(A, B)', SUCCEED, 'g2+"-"+g1', 'B-A'),
('(?i)[k]', 'AB', FAIL),
- ##('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'),
- ##('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'),
+# ('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'),
+# ('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'),
('(?i)a[-]?c', 'AC', SUCCEED, 'found', 'AC'),
('(?i)(abc)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'),
('(?i)([a-c]*)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'),
- # these zero-width assertions are not supported
- #('a(?!b).', 'abad', SUCCEED, 'found', 'ad'),
- #('a(?=d).', 'abad', SUCCEED, 'found', 'ad'),
- #('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'),
+ ('a(?!b).', 'abad', SUCCEED, 'found', 'ad'),
+ ('a(?=d).', 'abad', SUCCEED, 'found', 'ad'),
+ ('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'),
('a(?:b|c|d)(.)', 'ace', SUCCEED, 'g1', 'e'),
('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'),
('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'),
@@ -535,5 +590,5 @@
(r'\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'),
('\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'),
(r'\t\n\v\r\f\a', '\t\n\v\r\f\a', SUCCEED, 'found', chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)),
- (r'[\t][\n][\v][\r][\f][\a][\A][\b][\B][\Z][\g]', '\t\n\v\r\f\aA\bBZg', SUCCEED, 'found', '\t\n\v\r\f\aA\bBZg'),
+ (r'[\t][\n][\v][\r][\f][\b]', '\t\n\v\r\f\b', SUCCEED, 'found', '\t\n\v\r\f\b'),
]
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
index d90e2a8..83eab9f 100644
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -4,7 +4,6 @@
from test_support import verbose, TestFailed
import re
-import reop
import sys, os, string, traceback
# Misc tests from Tim Peters' re.doc
@@ -23,12 +22,13 @@
assert re.sub('.', lambda m: r"\n", 'x') == '\\n'
assert re.sub('.', r"\n", 'x') == '\n'
-
+
s = r"\1\1"
assert re.sub('(.)', s, 'x') == 'xx'
assert re.sub('(.)', re.escape(s), 'x') == s
assert re.sub('(.)', lambda m: s, 'x') == s
+ assert re.sub('(?P<a>x)', '\g<a>\g<a>', 'xx') == 'xxxx'
assert re.sub('(?P<unk>x)', '\g<unk>\g<unk>', 'xx') == 'xxxx'
assert re.sub('a', r'\t\n\v\r\f\a\b\B\Z\a\A\w\W\s\S\d\D', 'a') == '\t\n\v\r\f\a\bBZ\aAwWsSdD'
@@ -98,13 +98,9 @@
assert re.subn("b+", "x", "bbbb BBBB") == ('x BBBB', 1)
assert re.subn("b+", "x", "xyz") == ('xyz', 0)
assert re.subn("b*", "x", "xyz") == ('xxxyxzx', 4)
-
except AssertionError:
raise TestFailed, "re.subn"
-if verbose:
- print 'Running tests on re.split'
-
try:
assert re.split(":", ":a:b::c") == ['', 'a', 'b', '', 'c']
assert re.split(":*", ":a:b::c") == ['', 'a', 'b', 'c']
@@ -119,12 +115,37 @@
except AssertionError:
raise TestFailed, "re.split"
+if verbose:
+ print 'Pickling a RegexObject instance'
+ import pickle
+ pat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)')
+ s = pickle.dumps(pat)
+ pat = pickle.loads(s)
+
+if verbose:
+ print 'Running tests on re.split'
+
+try:
+ assert re.I == re.IGNORECASE
+ assert re.L == re.LOCALE
+ assert re.M == re.MULTILINE
+ assert re.S == re.DOTALL
+ assert re.X == re.VERBOSE
+except AssertionError:
+ raise TestFailed, 're module constants'
+
+for flags in [re.I, re.M, re.X, re.S]:
+ try:
+ r = re.compile('^pattern$', flags)
+ except:
+ print 'Exception raised on flag', flags
+
from re_tests import *
if verbose:
print 'Running re_tests test suite'
else:
# To save time, only run the first and last 10 tests
- tests = tests[:10] + tests[-10:]
+ pass #tests = tests[:10] + tests[-10:]
for t in tests:
sys.stdout.flush()
@@ -150,7 +171,7 @@
else:
try:
result=obj.search(s)
- except (re.error, reop.error), msg:
+ except (re.error), msg:
print '=== Unexpected exception', t, repr(msg)
if outcome==SYNTAX_ERROR:
# This should have been a syntax error; forget it.
@@ -190,9 +211,22 @@
else:
print '=== Failed incorrectly', t
+ # Try the match with the search area limited to the extent
+ # of the match and see if it still succeeds. \B will
+ # break (because it won't match at the end or start of a
+ # string), so we'll ignore patterns that feature it.
+
+ if pattern[:2]!='\\B' and pattern[-2:]!='\\B':
+ obj=re.compile(pattern)
+ result=obj.search(s, pos=result.start(0), endpos=result.end(0)+1)
+ if result==None:
+ print '=== Failed on range-limited match', t
+
# Try the match with IGNORECASE enabled, and check that it
# still succeeds.
obj=re.compile(pattern, re.IGNORECASE)
result=obj.search(s)
if result==None:
print '=== Fails on case-insensitive match', t
+
+