rewrote the pattern.sub and pattern.subn methods in C
removed (conceptually flawed) getliteral helper; the new sub/subn code
uses a faster code path for literal replacement strings, but doesn't
(yet) look for literal patterns.
added STATE_OFFSET macro, and use it to convert state.start/ptr to
char indexes
diff --git a/Lib/sre.py b/Lib/sre.py
index 70d0148..9c3f4b3 100644
--- a/Lib/sre.py
+++ b/Lib/sre.py
@@ -104,7 +104,7 @@
"U", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",
"UNICODE", "error" ]
-__version__ = "2.2.0"
+__version__ = "2.2.1"
# this module works under 1.5.2 and later. don't use string methods
import string
@@ -244,26 +244,33 @@
template = sre_parse.parse_template(template, pattern)
return sre_parse.expand_template(template, match)
-def _sub(pattern, template, text, count=0):
- # internal: pattern.sub implementation hook
- return _subn(pattern, template, text, count, 1)[0]
-
-def _subn(pattern, template, text, count=0, sub=0):
- # internal: pattern.subn implementation hook
+def _subx(pattern, template):
+ # internal: pattern.sub/subn implementation helper
if callable(template):
filter = template
else:
template = _compile_repl(template, pattern)
- literals = template[1]
- if sub and not count:
- literal = pattern._getliteral()
- if literal and "\\" in literal:
- literal = None # may contain untranslated escapes
- if literal is not None and len(literals) == 1 and literals[0]:
- # shortcut: both pattern and string are literals
- return string.replace(text, pattern.pattern, literals[0]), 0
- def filter(match, template=template):
- return sre_parse.expand_template(template, match)
+ if not template[0] and len(template[1]) == 1:
+ # literal replacement
+ filter = template[1][0]
+ else:
+ def filter(match, template=template):
+ return sre_parse.expand_template(template, match)
+ return filter
+
+def _sub(pattern, template, text, count=0):
+ # internal: pattern.sub implementation hook
+ # FIXME: not used in SRE 2.2.1 and later; will be removed soon
+ return _subn(pattern, template, text, count)[0]
+
+def _subn(pattern, template, text, count=0):
+ # internal: pattern.subn implementation hook
+ # FIXME: not used in SRE 2.2.1 and later; will be removed soon
+ filter = _subx(pattern, template)
+ if not callable(filter):
+ # literal replacement
+ def filter(match, literal=filter):
+ return literal
n = i = 0
s = []
append = s.append
@@ -286,6 +293,7 @@
def _split(pattern, text, maxsplit=0):
# internal: pattern.split implementation hook
+ # FIXME: not used in SRE 2.2.1 and later; will be removed soon
n = i = 0
s = []
append = s.append
diff --git a/Modules/_sre.c b/Modules/_sre.c
index fadcc57..be93d93 100644
--- a/Modules/_sre.c
+++ b/Modules/_sre.c
@@ -31,9 +31,9 @@
* 2001-04-28 fl added __copy__ methods (work in progress)
* 2001-05-14 fl fixes for 1.5.2
* 2001-07-01 fl added BIGCHARSET support (from Martin von Loewis)
- * 2001-09-18 fl added _getliteral helper
* 2001-10-18 fl fixed group reset issue (from Matthew Mueller)
* 2001-10-20 fl added split primitive; reenable unicode for 1.6/2.0/2.1
+ * 2001-10-21 fl added sub/subn primitive
*
* Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved.
*
@@ -49,7 +49,7 @@
#ifndef SRE_RECURSIVE
static char copyright[] =
- " SRE 2.2.0 Copyright (c) 1997-2001 by Secret Labs AB ";
+ " SRE 2.2.1 Copyright (c) 1997-2001 by Secret Labs AB ";
#include "Python.h"
#include "structmember.h" /* offsetof */
@@ -76,8 +76,9 @@
/* -------------------------------------------------------------------- */
/* optional features */
-/* test: define to use sre._split helper instead of C code */
+/* test: define to use sre.py helpers instead of C code */
#undef USE_PYTHON_SPLIT
+#undef USE_PYTHON_SUB
/* prevent run-away recursion (bad patterns on long strings) */
@@ -1493,6 +1494,10 @@
mark_fini(state);
}
+/* calculate offset from start of string */
+#define STATE_OFFSET(state, member)\
+ (((char*)(member) - (char*)(state)->beginning) / (state)->charsize)
+
LOCAL(PyObject*)
state_getslice(SRE_STATE* state, int index, PyObject* string, int empty)
{
@@ -1509,10 +1514,8 @@
return Py_None;
}
} else {
- i = ((char*)state->mark[index] - (char*)state->beginning) /
- state->charsize;
- j = ((char*)state->mark[index+1] - (char*)state->beginning) /
- state->charsize;
+ i = STATE_OFFSET(state, state->mark[index]);
+ j = STATE_OFFSET(state, state->mark[index+1]);
}
return PySequence_GetSlice(string, i, j);
@@ -1721,6 +1724,8 @@
PyObject* func;
PyObject* result;
+ if (!args)
+ return NULL;
name = PyString_FromString(module);
if (!name)
return NULL;
@@ -1759,6 +1764,60 @@
#endif
static PyObject*
+join(PyObject* list, PyObject* pattern)
+{
+ /* join list elements */
+
+ PyObject* joiner;
+#if PY_VERSION_HEX >= 0x01060000
+ PyObject* function;
+ PyObject* args;
+#endif
+ PyObject* result;
+
+ switch (PyList_GET_SIZE(list)) {
+ case 0:
+ Py_DECREF(list);
+ return PyString_FromString("");
+ case 1:
+ result = PyList_GET_ITEM(list, 0);
+ Py_INCREF(result);
+ Py_DECREF(list);
+ return result;
+ }
+
+ /* two or more elements: slice out a suitable separator from the
+ first member, and use that to join the entire list */
+
+ joiner = PySequence_GetSlice(pattern, 0, 0);
+ if (!joiner)
+ return NULL;
+
+#if PY_VERSION_HEX >= 0x01060000
+ function = PyObject_GetAttrString(joiner, "join");
+ if (!function) {
+ Py_DECREF(joiner);
+ return NULL;
+ }
+ args = PyTuple_New(1);
+ PyTuple_SET_ITEM(args, 0, list);
+ result = PyObject_CallObject(function, args);
+ Py_DECREF(args); /* also removes list */
+ Py_DECREF(function);
+#else
+ result = call(
+ "string", "join",
+ Py_BuildValue("OO", list, joiner)
+ );
+#endif
+ Py_DECREF(joiner);
+
+ return result;
+}
+
+
+#ifdef USE_PYTHON_SUB
+static PyObject*
pattern_sub(PatternObject* self, PyObject* args, PyObject* kw)
{
PyObject* template;
@@ -1775,7 +1834,9 @@
Py_BuildValue("OOOO", self, template, string, count)
);
}
+#endif
+#ifdef USE_PYTHON_SUB
static PyObject*
pattern_subn(PatternObject* self, PyObject* args, PyObject* kw)
{
@@ -1793,6 +1854,7 @@
Py_BuildValue("OOOO", self, template, string, count)
);
}
+#endif
#if defined(USE_PYTHON_SPLIT)
static PyObject*
@@ -1851,59 +1913,52 @@
#endif
}
- if (status > 0) {
-
- /* don't bother to build a match object */
- switch (self->groups) {
- case 0:
- b = ((char*) state.start - (char*) state.beginning) /
- state.charsize;
- e = ((char*) state.ptr - (char*) state.beginning) /
- state.charsize;
- item = PySequence_GetSlice(string, b, e);
- if (!item)
- goto error;
- break;
- case 1:
- item = state_getslice(&state, 1, string, 1);
- if (!item)
- goto error;
- break;
- default:
- item = PyTuple_New(self->groups);
- if (!item)
- goto error;
- for (i = 0; i < self->groups; i++) {
- PyObject* o = state_getslice(&state, i+1, string, 1);
- if (!o) {
- Py_DECREF(item);
- goto error;
- }
- PyTuple_SET_ITEM(item, i, o);
- }
- break;
- }
-
- status = PyList_Append(list, item);
- Py_DECREF(item);
-
- if (status < 0)
- goto error;
-
- if (state.ptr == state.start)
- state.start = (void*) ((char*) state.ptr + state.charsize);
- else
- state.start = state.ptr;
-
- } else {
-
+ if (status <= 0) {
if (status == 0)
break;
-
pattern_error(status);
goto error;
-
}
+
+ /* don't bother to build a match object */
+ switch (self->groups) {
+ case 0:
+ b = STATE_OFFSET(&state, state.start);
+ e = STATE_OFFSET(&state, state.ptr);
+ item = PySequence_GetSlice(string, b, e);
+ if (!item)
+ goto error;
+ break;
+ case 1:
+ item = state_getslice(&state, 1, string, 1);
+ if (!item)
+ goto error;
+ break;
+ default:
+ item = PyTuple_New(self->groups);
+ if (!item)
+ goto error;
+ for (i = 0; i < self->groups; i++) {
+ PyObject* o = state_getslice(&state, i+1, string, 1);
+ if (!o) {
+ Py_DECREF(item);
+ goto error;
+ }
+ PyTuple_SET_ITEM(item, i, o);
+ }
+ break;
+ }
+
+ status = PyList_Append(list, item);
+ Py_DECREF(item);
+ if (status < 0)
+ goto error;
+
+ if (state.ptr == state.start)
+ state.start = (void*) ((char*) state.ptr + state.charsize);
+ else
+ state.start = state.ptr;
+
}
state_fini(&state);
@@ -1925,8 +1980,8 @@
PyObject* item;
int status;
int n;
- int i, b, e;
- int g;
+ int i;
+ void* last;
PyObject* string;
int maxsplit = 0;
@@ -1941,9 +1996,10 @@
list = PyList_New(0);
- i = n = 0;
+ n = 0;
+ last = state.start;
- while (maxsplit == 0 || n < maxsplit) {
+ while (!maxsplit || n < maxsplit) {
state_reset(&state);
@@ -1957,58 +2013,54 @@
#endif
}
- if (status > 0) {
+ if (status <= 0) {
+ if (status == 0)
+ break;
+ pattern_error(status);
+ goto error;
+ }
+
+ if (state.start == state.ptr) {
+ if (last == state.end)
+ break;
+ /* skip one character */
+ state.start = (void*) ((char*) state.ptr + state.charsize);
+ continue;
+ }
- if (state.start == state.ptr) {
- if (i >= state.endpos)
- break;
- /* skip one character */
- state.start = (void*) ((char*) state.ptr + state.charsize);
- continue;
- }
+ /* get segment before this match */
+ item = PySequence_GetSlice(
+ string, STATE_OFFSET(&state, last),
+ STATE_OFFSET(&state, state.start)
+ );
+ if (!item)
+ goto error;
+ status = PyList_Append(list, item);
+ Py_DECREF(item);
+ if (status < 0)
+ goto error;
- b = ((char*) state.start - (char*) state.beginning) /
- state.charsize;
- e = ((char*) state.ptr - (char*) state.beginning) /
- state.charsize;
-
- /* get segment before this match */
- item = PySequence_GetSlice(string, i, b);
+ /* add groups (if any) */
+ for (i = 0; i < self->groups; i++) {
+ item = state_getslice(&state, i+1, string, 0);
if (!item)
goto error;
status = PyList_Append(list, item);
Py_DECREF(item);
if (status < 0)
goto error;
-
- for (g = 0; g < self->groups; g++) {
- item = state_getslice(&state, g+1, string, 0);
- if (!item)
- goto error;
- status = PyList_Append(list, item);
- Py_DECREF(item);
- if (status < 0)
- goto error;
- }
-
- i = e;
- n = n + 1;
-
- state.start = state.ptr;
-
- } else {
-
- if (status == 0)
- break;
-
- pattern_error(status);
- goto error;
-
}
+
+ n = n + 1;
+
+ last = state.start = state.ptr;
+
}
/* get segment following last match */
- item = PySequence_GetSlice(string, i, state.endpos);
+ item = PySequence_GetSlice(
+ string, STATE_OFFSET(&state, last), state.endpos
+ );
if (!item)
goto error;
status = PyList_Append(list, item);
@@ -2027,6 +2079,174 @@
}
#endif
+#if !defined(USE_PYTHON_SUB)
+static PyObject*
+pattern_subx(PatternObject* self, PyObject* template, PyObject* string,
+ int count, int subn)
+{
+ SRE_STATE state;
+ PyObject* list;
+ PyObject* item;
+ PyObject* filter;
+ PyObject* args;
+ PyObject* match;
+ int status;
+ int n;
+ int i, b, e;
+ int filter_is_callable;
+
+ /* call subx helper to get the filter */
+ filter = call(
+ SRE_MODULE, "_subx",
+ Py_BuildValue("OO", self, template)
+ );
+ if (!filter)
+ return NULL;
+
+ filter_is_callable = PyCallable_Check(filter);
+
+ string = state_init(&state, self, string, 0, INT_MAX);
+ if (!string)
+ return NULL;
+
+ list = PyList_New(0);
+
+ n = i = 0;
+
+ while (!count || n < count) {
+
+ state_reset(&state);
+
+ state.ptr = state.start;
+
+ if (state.charsize == 1) {
+ status = sre_search(&state, PatternObject_GetCode(self));
+ } else {
+#if defined(HAVE_UNICODE)
+ status = sre_usearch(&state, PatternObject_GetCode(self));
+#endif
+ }
+
+ if (status <= 0) {
+ if (status == 0)
+ break;
+ pattern_error(status);
+ goto error;
+ }
+
+ b = STATE_OFFSET(&state, state.start);
+ e = STATE_OFFSET(&state, state.ptr);
+
+ if (i < b) {
+ /* get segment before this match */
+ item = PySequence_GetSlice(string, i, b);
+ if (!item)
+ goto error;
+ status = PyList_Append(list, item);
+ Py_DECREF(item);
+ if (status < 0)
+ goto error;
+
+ } else if (i == b && i == e && n > 0)
+ /* ignore empty match on latest position */
+ goto next;
+
+ if (filter_is_callable) {
+ /* filter match */
+ match = pattern_new_match(self, &state, 1);
+ if (!match)
+ goto error;
+ args = Py_BuildValue("(O)", match);
+ if (!args) {
+ Py_DECREF(args);
+ goto error;
+ }
+ item = PyObject_CallObject(filter, args);
+ Py_DECREF(args);
+ Py_DECREF(match);
+ if (!item)
+ goto error;
+ } else {
+ /* filter is literal string */
+ item = filter;
+ Py_INCREF(filter);
+ }
+
+ /* add to list */
+ status = PyList_Append(list, item);
+ Py_DECREF(item);
+ if (status < 0)
+ goto error;
+
+ i = e;
+ n = n + 1;
+
+next:
+ /* move on */
+ if (state.ptr == state.start)
+ state.start = (void*) ((char*) state.ptr + state.charsize);
+ else
+ state.start = state.ptr;
+
+ }
+
+ /* get segment following last match */
+ item = PySequence_GetSlice(string, i, state.endpos);
+ if (!item)
+ goto error;
+ status = PyList_Append(list, item);
+ Py_DECREF(item);
+ if (status < 0)
+ goto error;
+
+ state_fini(&state);
+
+ /* convert list to single string */
+ item = join(list, self->pattern);
+ if (!item)
+ return NULL;
+
+ if (subn)
+ return Py_BuildValue("Ni", item, n);
+
+ return item;
+
+error:
+ Py_DECREF(list);
+ state_fini(&state);
+ return NULL;
+
+}
+
+static PyObject*
+pattern_sub(PatternObject* self, PyObject* args, PyObject* kw)
+{
+ PyObject* template;
+ PyObject* string;
+ int count = 0;
+ static char* kwlist[] = { "repl", "string", "count", NULL };
+ if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|i:sub", kwlist,
+ &template, &string, &count))
+ return NULL;
+
+ return pattern_subx(self, template, string, count, 0);
+}
+
+static PyObject*
+pattern_subn(PatternObject* self, PyObject* args, PyObject* kw)
+{
+ PyObject* template;
+ PyObject* string;
+ int count = 0;
+ static char* kwlist[] = { "repl", "string", "count", NULL };
+ if (!PyArg_ParseTupleAndKeywords(args, kw, "OO|i:subn", kwlist,
+ &template, &string, &count))
+ return NULL;
+
+ return pattern_subx(self, template, string, count, 1);
+}
+#endif
+
static PyObject*
pattern_copy(PatternObject* self, PyObject* args)
{
@@ -2084,32 +2304,6 @@
#endif
}
-static PyObject*
-pattern_getliteral(PatternObject* self, PyObject* args)
-{
- /* internal: if the pattern is a literal string, return that
- string. otherwise, return None */
-
- SRE_CODE* code;
- PyObject* literal;
-
- if (!PyArg_ParseTuple(args, ":_getliteral"))
- return NULL;
-
- code = PatternObject_GetCode(self);
-
- if (code[0] == SRE_OP_INFO && code[2] & SRE_INFO_LITERAL) {
- /* FIXME: extract literal string from code buffer. we can't
- use the pattern member, since it may contain untranslated
- escape codes (see SF bug 449000) */
- literal = Py_None;
- } else
- literal = Py_None; /* no literal */
-
- Py_INCREF(literal);
- return literal;
-}
-
static PyMethodDef pattern_methods[] = {
{"match", (PyCFunction) pattern_match, METH_VARARGS|METH_KEYWORDS},
{"search", (PyCFunction) pattern_search, METH_VARARGS|METH_KEYWORDS},
@@ -2120,7 +2314,6 @@
{"scanner", (PyCFunction) pattern_scanner, METH_VARARGS},
{"__copy__", (PyCFunction) pattern_copy, METH_VARARGS},
{"__deepcopy__", (PyCFunction) pattern_deepcopy, METH_VARARGS},
- {"_getliteral", (PyCFunction) pattern_getliteral, METH_VARARGS},
{NULL, NULL}
};