Change the treatment of positions returned by PEP293
error handers in the Unicode codecs: Negative
positions are treated as being relative to the end of
the input and out of bounds positions result in an
IndexError.
Also update the PEP and include an explanation of
this in the documentation for codecs.register_error.
Fixes a small bug in iconv_codecs: if the position
from the callback is negative *add* it to the size
instead of substracting it.
From SF patch #677429.
diff --git a/Doc/lib/libcodecs.tex b/Doc/lib/libcodecs.tex
index 355ac5d..caaaaf4 100644
--- a/Doc/lib/libcodecs.tex
+++ b/Doc/lib/libcodecs.tex
@@ -103,11 +103,22 @@
Register the error handling function \var{error_handler} under the
name \var{name}. \var{error_handler} will be called during encoding
and decoding in case of an error, when \var{name} is specified as the
-errors parameter. \var{error_handler} will be called with an
-\exception{UnicodeEncodeError}, \exception{UnicodeDecodeError} or
-\exception{UnicodeTranslateError} instance and must return a tuple
-with a replacement for the unencodable/undecodable part of the input
-and a position where encoding/decoding should continue.
+errors parameter.
+
+For encoding \var{error_handler} will be called with a
+\exception{UnicodeEncodeError} instance, which contains information about
+the location of the error. The error handler must either raise this or
+a different exception or return a tuple with a replacement for the
+unencodable part of the input and a position where encoding should
+continue. The encoder will encode the replacement and continue encoding
+the original input at the specified position. Negative position values
+will be treated as being relative to the end of the input string. If the
+resulting position is out of bound an IndexError will be raised.
+
+Decoding and translating works similar, except \exception{UnicodeDecodeError}
+or \exception{UnicodeTranslateError} will be passed to the handler and
+that the replacement from the error handler will be put into the output
+directly.
\end{funcdesc}
\begin{funcdesc}{lookup_error}{name}
diff --git a/Doc/lib/libfuncs.tex b/Doc/lib/libfuncs.tex
index d5b565f..323a516 100644
--- a/Doc/lib/libfuncs.tex
+++ b/Doc/lib/libfuncs.tex
@@ -572,7 +572,7 @@
\var{classinfo} argument, or of a (direct or indirect) subclass
thereof. Also return true if \var{classinfo} is a type object and
\var{object} is an object of that type. If \var{object} is not a
- class instance or a object of the given type, the function always
+ class instance or an object of the given type, the function always
returns false. If \var{classinfo} is neither a class object nor a
type object, it may be a tuple of class or type objects, or may
recursively contain other such tuples (other sequence types are not
diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
index b51b489..bf583c2 100644
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -1,6 +1,23 @@
import test.test_support, unittest
import sys, codecs, htmlentitydefs, unicodedata
+class PosReturn:
+ # this can be used for configurable callbacks
+
+ def __init__(self):
+ self.pos = 0
+
+ def handle(self, exc):
+ oldpos = self.pos
+ realpos = oldpos
+ if realpos<0:
+ realpos = len(exc.object) + realpos
+ # if we don't advance this time, terminate on the next call
+ # otherwise we'd get an endless loop
+ if realpos <= exc.start:
+ self.pos = len(exc.object)
+ return (u"<?>", oldpos)
+
class CodecCallbackTest(unittest.TestCase):
def test_xmlcharrefreplace(self):
@@ -543,18 +560,36 @@
codecs.register_error("test.baddecodereturn2", baddecodereturn2)
self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn2")
- pos = [-42]
- def negposreturn(exc):
- pos[0] += 1 # use list to work around scoping problem
- return (u"?", pos[0])
- codecs.register_error("test.negposreturn", negposreturn)
- "\xff".decode("ascii", "test.negposreturn")
+ handler = PosReturn()
+ codecs.register_error("test.posreturn", handler.handle)
- def hugeposreturn(exc):
- return (u"?", 424242)
- codecs.register_error("test.hugeposreturn", hugeposreturn)
- "\xff".decode("ascii", "test.hugeposreturn")
- "\\uyyyy".decode("raw-unicode-escape", "test.hugeposreturn")
+ # Valid negative position
+ handler.pos = -1
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
+
+ # Valid negative position
+ handler.pos = -2
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?><?>")
+
+ # Negative position out of bounds
+ handler.pos = -3
+ self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
+
+ # Valid positive position
+ handler.pos = 1
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
+
+ # Largest valid positive position (one beyond end of input
+ handler.pos = 2
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>")
+
+ # Invalid positive position
+ handler.pos = 3
+ self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
+
+ # Restart at the "0"
+ handler.pos = 6
+ self.assertEquals("\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), u"<?>0")
class D(dict):
def __getitem__(self, key):
@@ -579,22 +614,39 @@
codecs.register_error("test.badencodereturn2", badencodereturn2)
self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn2")
- pos = [-42]
- def negposreturn(exc):
- pos[0] += 1 # use list to work around scoping problem
- return (u"?", pos[0])
- codecs.register_error("test.negposreturn", negposreturn)
- u"\xff".encode("ascii", "test.negposreturn")
+ handler = PosReturn()
+ codecs.register_error("test.posreturn", handler.handle)
- def hugeposreturn(exc):
- return (u"?", 424242)
- codecs.register_error("test.hugeposreturn", hugeposreturn)
- u"\xff".encode("ascii", "test.hugeposreturn")
+ # Valid negative position
+ handler.pos = -1
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
+
+ # Valid negative position
+ handler.pos = -2
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?><?>")
+
+ # Negative position out of bounds
+ handler.pos = -3
+ self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
+
+ # Valid positive position
+ handler.pos = 1
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
+
+ # Largest valid positive position (one beyond end of input
+ handler.pos = 2
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>")
+
+ # Invalid positive position
+ handler.pos = 3
+ self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
+
+ handler.pos = 0
class D(dict):
def __getitem__(self, key):
raise ValueError
- for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.hugeposreturn"):
+ for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.posreturn"):
self.assertRaises(UnicodeError, codecs.charmap_encode, u"\xff", err, {0xff: None})
self.assertRaises(ValueError, codecs.charmap_encode, u"\xff", err, D())
self.assertRaises(TypeError, codecs.charmap_encode, u"\xff", err, {0xff: 300})
diff --git a/Modules/_iconv_codec.c b/Modules/_iconv_codec.c
index d61adbc..3f2a72a 100644
--- a/Modules/_iconv_codec.c
+++ b/Modules/_iconv_codec.c
@@ -247,8 +247,13 @@
Py_DECREF(retobj);
if (newpos < 0)
- newpos = inputlen - newpos;
- if (newpos < 0 || newpos >= inputlen)
+ newpos = inputlen + newpos;
+ if (newpos < 0 || newpos > inputlen) {
+ PyErr_Format(PyExc_IndexError, "position %ld from error handler"
+ " out of bounds", newpos);
+ goto errorexit;
+ }
+ if (newpos == inputlen)
break;
inp = inp_top + Py_UNICODE_SIZE * newpos;
inplen = inplen_total - Py_UNICODE_SIZE * newpos;
@@ -471,8 +476,13 @@
Py_DECREF(retobj);
if (newpos < 0)
- newpos = inplen_total - newpos;
- if (newpos < 0 || newpos >= inplen_total)
+ newpos = inplen_total + newpos;
+ if (newpos < 0 || newpos > inplen_total) {
+ PyErr_Format(PyExc_IndexError, "position %ld from error handler"
+ " out of bounds", newpos);
+ goto errorexit;
+ }
+ if (newpos == inplen_total)
break;
inp = inp_top + newpos;
inplen = inplen_total - newpos;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 1abef89..dfeabf5 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -728,9 +728,11 @@
if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
goto onError;
if (newpos<0)
- newpos = 0;
- else if (newpos>insize)
- newpos = insize;
+ newpos = insize+newpos;
+ if (newpos<0 || newpos>insize) {
+ PyErr_Format(PyExc_IndexError, "position %d from error handler out of bounds", newpos);
+ goto onError;
+ }
/* need more space? (at least enough for what we
have+the replacement+the rest of the string (starting
@@ -2246,9 +2248,12 @@
return NULL;
}
if (*newpos<0)
- *newpos = 0;
- else if (*newpos>size)
- *newpos = size;
+ *newpos = size+*newpos;
+ if (*newpos<0 || *newpos>size) {
+ PyErr_Format(PyExc_IndexError, "position %d from error handler out of bounds", *newpos);
+ Py_DECREF(restuple);
+ return NULL;
+ }
Py_INCREF(resunicode);
Py_DECREF(restuple);
return resunicode;
@@ -3084,9 +3089,12 @@
return NULL;
}
if (*newpos<0)
- *newpos = 0;
- else if (*newpos>size)
- *newpos = size;
+ *newpos = size+*newpos;
+ if (*newpos<0 || *newpos>size) {
+ PyErr_Format(PyExc_IndexError, "position %d from error handler out of bounds", *newpos);
+ Py_DECREF(restuple);
+ return NULL;
+ }
Py_INCREF(resunicode);
Py_DECREF(restuple);
return resunicode;