Fredrik Lundh: here's the 96.6% version of SRE
diff --git a/Lib/sre.py b/Lib/sre.py
index 79878b3..455cd27 100644
--- a/Lib/sre.py
+++ b/Lib/sre.py
@@ -1,4 +1,3 @@
-# -*- Mode: Python; tab-width: 4 -*-
#
# Secret Labs' Regular Expression Engine
# $Id$
@@ -7,39 +6,127 @@
#
# Copyright (c) 1998-2000 by Secret Labs AB. All rights reserved.
#
-# This code can only be used for 1.6 alpha testing. All other use
-# require explicit permission from Secret Labs AB.
-#
# Portions of this engine have been developed in cooperation with
# CNRI. Hewlett-Packard provided funding for 1.6 integration and
# other compatibility work.
#
-"""
-this is a long string
-"""
-
import sre_compile
+# flags
+I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE
+L = LOCALE = sre_compile.SRE_FLAG_LOCALE
+M = MULTILINE = sre_compile.SRE_FLAG_MULTILINE
+S = DOTALL = sre_compile.SRE_FLAG_DOTALL
+X = VERBOSE = sre_compile.SRE_FLAG_VERBOSE
+
# --------------------------------------------------------------------
# public interface
-def compile(pattern, flags=0):
- return sre_compile.compile(pattern, _fixflags(flags))
+# FIXME: add docstrings
def match(pattern, string, flags=0):
- return compile(pattern, _fixflags(flags)).match(string)
+ return _compile(pattern, flags).match(string)
def search(pattern, string, flags=0):
- return compile(pattern, _fixflags(flags)).search(string)
+ return _compile(pattern, flags).search(string)
-# FIXME: etc
+def sub(pattern, repl, string, count=0):
+ return _compile(pattern).sub(repl, string, count)
+
+def subn(pattern, repl, string, count=0):
+ return _compile(pattern).subn(repl, string, count)
+
+def split(pattern, string, maxsplit=0):
+ return _compile(pattern).split(string, maxsplit)
+
+def findall(pattern, string, maxsplit=0):
+ return _compile(pattern).findall(string, maxsplit)
+
+def compile(pattern, flags=0):
+ return _compile(pattern, flags)
+
+def escape(pattern):
+ s = list(pattern)
+ for i in range(len(pattern)):
+ c = pattern[i]
+ if not ("a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9"):
+ if c == "\000":
+ s[i] = "\\000"
+ else:
+ s[i] = "\\" + c
+ return pattern[:0].join(s)
# --------------------------------------------------------------------
-# helpers
+# internals
-def _fixflags(flags):
- # convert flag bitmask to sequence
- assert not flags
- return ()
+_cache = {}
+_MAXCACHE = 100
+def _compile(pattern, flags=0):
+ # internal: compile pattern
+ tp = type(pattern)
+ if tp not in (type(""), type(u"")):
+ return pattern
+ key = (tp, pattern, flags)
+ try:
+ return _cache[key]
+ except KeyError:
+ pass
+ p = sre_compile.compile(pattern, flags)
+ if len(_cache) >= _MAXCACHE:
+ _cache.clear()
+ _cache[key] = p
+ return p
+
+def _sub(pattern, template, string, count=0):
+ # internal: pattern.sub implementation hook
+ return _subn(pattern, template, string, count)[0]
+
+def _expand(match, template):
+ # internal: expand template
+ return template # FIXME
+
+def _subn(pattern, template, string, count=0):
+ # internal: pattern.subn implementation hook
+ if callable(template):
+ filter = callable
+ else:
+ # FIXME: prepare template
+ def filter(match, template=template):
+ return _expand(match, template)
+ n = i = 0
+ s = []
+ append = s.append
+ c = pattern.cursor(string)
+ while not count or n < count:
+ m = c.search()
+ if not m:
+ break
+ j = m.start()
+ if j > i:
+ append(string[i:j])
+ append(filter(m))
+ i = m.end()
+ n = n + 1
+ if i < len(string):
+ append(string[i:])
+ return string[:0].join(s), n
+
+def _split(pattern, string, maxsplit=0):
+ # internal: pattern.split implementation hook
+ n = i = 0
+ s = []
+ append = s.append
+ c = pattern.cursor(string)
+ while not maxsplit or n < maxsplit:
+ m = c.search()
+ if not m:
+ break
+ j = m.start()
+ append(string[i:j])
+ i = m.end()
+ n = n + 1
+ if i < len(string):
+ append(string[i:])
+ return s
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
index 8738061..53da005 100644
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -14,9 +14,6 @@
# other compatibility work.
#
-# FIXME: <fl> formalize (objectify?) and document the compiler code
-# format, so that other frontends can use this compiler
-
import array, string, sys
import _sre
@@ -45,64 +42,70 @@
self.data.append(code)
def todata(self):
# print self.data
- return array.array(WORDSIZE, self.data).tostring()
+ try:
+ return array.array(WORDSIZE, self.data).tostring()
+ except OverflowError:
+ print self.data
+ raise
-def _lower(literal):
- # return _sre._lower(literal) # FIXME
- return string.lower(literal)
-
-def _compile(code, pattern, flags):
+def _compile(code, pattern, flags, level=0):
append = code.append
for op, av in pattern:
if op is ANY:
- if "s" in flags:
- append(CODES[op]) # any character at all!
+ if flags & SRE_FLAG_DOTALL:
+ append(OPCODES[op]) # any character at all!
else:
- append(CODES[NOT_LITERAL])
- append(10)
+ append(OPCODES[CATEGORY])
+ append(CHCODES[CATEGORY_NOT_LINEBREAK])
elif op in (SUCCESS, FAILURE):
- append(CODES[op])
+ append(OPCODES[op])
elif op is AT:
- append(CODES[op])
- append(POSITIONS[av])
+ append(OPCODES[op])
+ if flags & SRE_FLAG_MULTILINE:
+ append(ATCODES[AT_MULTILINE[av]])
+ else:
+ append(ATCODES[av])
elif op is BRANCH:
- append(CODES[op])
+ append(OPCODES[op])
tail = []
for av in av[1]:
skip = len(code); append(0)
- _compile(code, av, flags)
- append(CODES[JUMP])
+ _compile(code, av, flags, level)
+ append(OPCODES[JUMP])
tail.append(len(code)); append(0)
code[skip] = len(code) - skip
append(0) # end of branch
for tail in tail:
code[tail] = len(code) - tail
elif op is CALL:
- append(CODES[op])
+ append(OPCODES[op])
skip = len(code); append(0)
- _compile(code, av, flags)
- append(CODES[SUCCESS])
+ _compile(code, av, flags, level+1)
+ append(OPCODES[SUCCESS])
code[skip] = len(code) - skip
elif op is CATEGORY: # not used by current parser
- append(CODES[op])
- append(CATEGORIES[av])
+ append(OPCODES[op])
+ if flags & SRE_FLAG_LOCALE:
+ append(CH_LOCALE[CHCODES[av]])
+ else:
+ append(CHCODES[av])
elif op is GROUP:
- if "i" in flags:
- append(CODES[MAP_IGNORE[op]])
+ if flags & SRE_FLAG_IGNORECASE:
+ append(OPCODES[OP_IGNORE[op]])
else:
- append(CODES[op])
- append(av)
+ append(OPCODES[op])
+ append(av-1)
elif op is IN:
- if "i" in flags:
- append(CODES[MAP_IGNORE[op]])
+ if flags & SRE_FLAG_IGNORECASE:
+ append(OPCODES[OP_IGNORE[op]])
def fixup(literal):
- return ord(_lower(literal))
+ return ord(literal.lower())
else:
- append(CODES[op])
+ append(OPCODES[op])
fixup = ord
skip = len(code); append(0)
for op, av in av:
- append(CODES[op])
+ append(OPCODES[op])
if op is NEGATE:
pass
elif op is LITERAL:
@@ -111,58 +114,60 @@
append(fixup(av[0]))
append(fixup(av[1]))
elif op is CATEGORY:
- append(CATEGORIES[av])
+ if flags & SRE_FLAG_LOCALE:
+ append(CH_LOCALE[CHCODES[av]])
+ else:
+ append(CHCODES[av])
else:
raise ValueError, "unsupported set operator"
- append(CODES[FAILURE])
+ append(OPCODES[FAILURE])
code[skip] = len(code) - skip
elif op in (LITERAL, NOT_LITERAL):
- if "i" in flags:
- append(CODES[MAP_IGNORE[op]])
- append(ord(_lower(av)))
+ if flags & SRE_FLAG_IGNORECASE:
+ append(OPCODES[OP_IGNORE[op]])
+ append(ord(av.lower()))
else:
- append(CODES[op])
+ append(OPCODES[op])
append(ord(av))
elif op is MARK:
- append(CODES[op])
+ append(OPCODES[op])
append(av)
elif op in (REPEAT, MIN_REPEAT, MAX_REPEAT):
lo, hi = av[2].getwidth()
if lo == 0:
raise SyntaxError, "cannot repeat zero-width items"
if lo == hi == 1 and op is MAX_REPEAT:
- append(CODES[MAX_REPEAT_ONE])
+ append(OPCODES[MAX_REPEAT_ONE])
skip = len(code); append(0)
append(av[0])
append(av[1])
- _compile(code, av[2], flags)
- append(CODES[SUCCESS])
+ _compile(code, av[2], flags, level+1)
+ append(OPCODES[SUCCESS])
code[skip] = len(code) - skip
else:
- append(CODES[op])
+ append(OPCODES[op])
skip = len(code); append(0)
append(av[0])
append(av[1])
- _compile(code, av[2], flags)
+ _compile(code, av[2], flags, level+1)
if op is MIN_REPEAT:
- append(CODES[MIN_UNTIL])
+ append(OPCODES[MIN_UNTIL])
else:
- # FIXME: MAX_REPEAT PROBABLY DOESN'T WORK (?)
- append(CODES[MAX_UNTIL])
+ append(OPCODES[MAX_UNTIL])
code[skip] = len(code) - skip
elif op is SUBPATTERN:
-## group = av[0]
-## if group:
-## append(CODES[MARK])
-## append((group-1)*2)
- _compile(code, av[1], flags)
-## if group:
-## append(CODES[MARK])
-## append((group-1)*2+1)
+ group = av[0]
+ if group:
+ append(OPCODES[MARK])
+ append((group-1)*2)
+ _compile(code, av[1], flags, level+1)
+ if group:
+ append(OPCODES[MARK])
+ append((group-1)*2+1)
else:
raise ValueError, ("unsupported operand type", op)
-def compile(p, flags=()):
+def compile(p, flags=0):
# convert pattern list to internal format
if type(p) in (type(""), type(u"")):
import sre_parse
@@ -170,12 +175,10 @@
p = sre_parse.parse(p)
else:
pattern = None
- # print p.getwidth()
- # print p
+ flags = p.pattern.flags | flags
code = Code()
- _compile(code, p.data, p.pattern.flags)
- code.append(CODES[SUCCESS])
- # print list(code.data)
+ _compile(code, p.data, flags)
+ code.append(OPCODES[SUCCESS])
data = code.todata()
if 0: # debugging
print
@@ -183,5 +186,8 @@
import sre_disasm
sre_disasm.disasm(data)
print "-" * 68
- # print len(data), p.pattern.groups, len(p.pattern.groupdict)
- return _sre.compile(pattern, data, p.pattern.groups-1, p.pattern.groupdict)
+ return _sre.compile(
+ pattern, flags,
+ data,
+ p.pattern.groups-1, p.pattern.groupdict
+ )
diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py
index af88309..531dc31 100644
--- a/Lib/sre_constants.py
+++ b/Lib/sre_constants.py
@@ -48,20 +48,31 @@
# positions
AT_BEGINNING = "at_beginning"
+AT_BEGINNING_LINE = "at_beginning_line"
AT_BOUNDARY = "at_boundary"
AT_NON_BOUNDARY = "at_non_boundary"
AT_END = "at_end"
+AT_END_LINE = "at_end_line"
# categories
-
CATEGORY_DIGIT = "category_digit"
CATEGORY_NOT_DIGIT = "category_not_digit"
CATEGORY_SPACE = "category_space"
CATEGORY_NOT_SPACE = "category_not_space"
CATEGORY_WORD = "category_word"
CATEGORY_NOT_WORD = "category_not_word"
+CATEGORY_LINEBREAK = "category_linebreak"
+CATEGORY_NOT_LINEBREAK = "category_not_linebreak"
+CATEGORY_LOC_DIGIT = "category_loc_digit"
+CATEGORY_LOC_NOT_DIGIT = "category_loc_not_digit"
+CATEGORY_LOC_SPACE = "category_loc_space"
+CATEGORY_LOC_NOT_SPACE = "category_loc_not_space"
+CATEGORY_LOC_WORD = "category_loc_word"
+CATEGORY_LOC_NOT_WORD = "category_loc_not_word"
+CATEGORY_LOC_LINEBREAK = "category_loc_linebreak"
+CATEGORY_LOC_NOT_LINEBREAK = "category_loc_not_linebreak"
-CODES = [
+OPCODES = [
# failure=0 success=1 (just because it looks better that way :-)
FAILURE, SUCCESS,
@@ -87,45 +98,75 @@
]
-# convert to dictionary
-c = {}
-i = 0
-for code in CODES:
- c[code] = i
- i = i + 1
-CODES = c
+ATCODES = [
+ AT_BEGINNING, AT_BEGINNING_LINE, AT_BOUNDARY,
+ AT_NON_BOUNDARY, AT_END, AT_END_LINE
+]
+
+CHCODES = [
+ CATEGORY_DIGIT, CATEGORY_NOT_DIGIT, CATEGORY_SPACE,
+ CATEGORY_NOT_SPACE, CATEGORY_WORD, CATEGORY_NOT_WORD,
+ CATEGORY_LINEBREAK, CATEGORY_NOT_LINEBREAK, CATEGORY_LOC_DIGIT,
+ CATEGORY_LOC_NOT_DIGIT, CATEGORY_LOC_SPACE,
+ CATEGORY_LOC_NOT_SPACE, CATEGORY_LOC_WORD, CATEGORY_LOC_NOT_WORD,
+ CATEGORY_LOC_LINEBREAK, CATEGORY_LOC_NOT_LINEBREAK
+]
+
+def makedict(list):
+ d = {}
+ i = 0
+ for item in list:
+ d[item] = i
+ i = i + 1
+ return d
+
+OPCODES = makedict(OPCODES)
+ATCODES = makedict(ATCODES)
+CHCODES = makedict(CHCODES)
# replacement operations for "ignore case" mode
-MAP_IGNORE = {
+OP_IGNORE = {
GROUP: GROUP_IGNORE,
IN: IN_IGNORE,
LITERAL: LITERAL_IGNORE,
NOT_LITERAL: NOT_LITERAL_IGNORE
}
-POSITIONS = {
- AT_BEGINNING: ord("a"),
- AT_BOUNDARY: ord("b"),
- AT_NON_BOUNDARY: ord("B"),
- AT_END: ord("z"),
+AT_MULTILINE = {
+ AT_BEGINNING: AT_BEGINNING_LINE,
+ AT_END: AT_END_LINE
}
-CATEGORIES = {
- CATEGORY_DIGIT: ord("d"),
- CATEGORY_NOT_DIGIT: ord("D"),
- CATEGORY_SPACE: ord("s"),
- CATEGORY_NOT_SPACE: ord("S"),
- CATEGORY_WORD: ord("w"),
- CATEGORY_NOT_WORD: ord("W"),
+CH_LOCALE = {
+ CATEGORY_DIGIT: CATEGORY_LOC_DIGIT,
+ CATEGORY_NOT_DIGIT: CATEGORY_LOC_NOT_DIGIT,
+ CATEGORY_SPACE: CATEGORY_LOC_SPACE,
+ CATEGORY_NOT_SPACE: CATEGORY_LOC_NOT_SPACE,
+ CATEGORY_WORD: CATEGORY_LOC_WORD,
+ CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD,
+ CATEGORY_LINEBREAK: CATEGORY_LOC_LINEBREAK,
+ CATEGORY_NOT_LINEBREAK: CATEGORY_LOC_NOT_LINEBREAK
}
+# flags
+SRE_FLAG_TEMPLATE = 1 # NYI
+SRE_FLAG_IGNORECASE = 2
+SRE_FLAG_LOCALE = 4
+SRE_FLAG_MULTILINE = 8
+SRE_FLAG_DOTALL = 16
+SRE_FLAG_VERBOSE = 32
+
if __name__ == "__main__":
import string
- items = CODES.items()
- items.sort(lambda a, b: cmp(a[1], b[1]))
+ def dump(f, d, prefix):
+ items = d.items()
+ items.sort(lambda a, b: cmp(a[1], b[1]))
+ for k, v in items:
+ f.write("#define %s_%s %s\n" % (prefix, string.upper(k), v))
f = open("sre_constants.h", "w")
- f.write("/* generated by sre_constants.py */\n")
- for k, v in items:
- f.write("#define SRE_OP_" + string.upper(k) + " " + str(v) + "\n")
+ f.write("/* generated from sre_constants.py */\n")
+ dump(f, OPCODES, "SRE_OP")
+ dump(f, ATCODES, "SRE")
+ dump(f, CHCODES, "SRE")
f.close()
print "done"