bpo-32583: Fix possible crashing in builtin Unicode decoders (#5325)
When using customized decode error handlers, it is possible for builtin decoders
to write out-of-bounds and then crash.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 775bd15..3d9e09d 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4190,7 +4190,10 @@
Py_ssize_t insize;
Py_ssize_t newpos;
Py_ssize_t replen;
+ Py_ssize_t remain;
PyObject *inputobj = NULL;
+ int need_to_grow = 0;
+ const char *new_inptr;
if (*errorHandler == NULL) {
*errorHandler = PyCodec_LookupError(errors);
@@ -4221,6 +4224,7 @@
inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject);
if (!inputobj)
goto onError;
+ remain = *inend - *input - *endinpos;
*input = PyBytes_AS_STRING(inputobj);
insize = PyBytes_GET_SIZE(inputobj);
*inend = *input + insize;
@@ -4238,6 +4242,19 @@
replen = PyUnicode_GET_LENGTH(repunicode);
if (replen > 1) {
writer->min_length += replen - 1;
+ need_to_grow = 1;
+ }
+ new_inptr = *input + newpos;
+ if (*inend - new_inptr > remain) {
+ /* We don't know the decoding algorithm here so we make the worst
+ assumption that one byte decodes to one unicode character.
+ If unfortunately one byte could decode to more unicode characters,
+ the decoder may write out-of-bound then. Is it possible for the
+ algorithms using this function? */
+ writer->min_length += *inend - new_inptr - remain;
+ need_to_grow = 1;
+ }
+ if (need_to_grow) {
writer->overallocate = 1;
if (_PyUnicodeWriter_Prepare(writer, writer->min_length,
PyUnicode_MAX_CHAR_VALUE(repunicode)) == -1)
@@ -4247,7 +4264,7 @@
goto onError;
*endinpos = newpos;
- *inptr = *input + newpos;
+ *inptr = new_inptr;
/* we made it! */
Py_DECREF(restuple);
@@ -5572,7 +5589,8 @@
#endif
/* Note: size will always be longer than the resulting Unicode
- character count */
+ character count normally. Error handler will take care of
+ resizing when needed. */
_PyUnicodeWriter_Init(&writer);
writer.min_length = (e - q + 1) / 2;
if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1)