bpo-36311: Fixes decoding multibyte characters around chunk boundaries and improves decoding performance (GH-15083)
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 5545eae..aa93377 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7186,6 +7186,12 @@
#define NEED_RETRY
#endif
+/* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when
+ transcoding from UTF-16), but INT_MAX / 4 perfoms better in
+ both cases also and avoids partial characters overrunning the
+ length limit in MultiByteToWideChar on Windows */
+#define DECODING_CHUNK_SIZE (INT_MAX/4)
+
#ifndef WC_ERR_INVALID_CHARS
# define WC_ERR_INVALID_CHARS 0x0080
#endif
@@ -7422,8 +7428,8 @@
do
{
#ifdef NEED_RETRY
- if (size > INT_MAX) {
- chunk_size = INT_MAX;
+ if (size > DECODING_CHUNK_SIZE) {
+ chunk_size = DECODING_CHUNK_SIZE;
final = 0;
done = 0;
}
@@ -7827,10 +7833,8 @@
do
{
#ifdef NEED_RETRY
- /* UTF-16 encoding may double the size, so use only INT_MAX/2
- chunks. */
- if (len > INT_MAX/2) {
- chunk_len = INT_MAX/2;
+ if (len > DECODING_CHUNK_SIZE) {
+ chunk_len = DECODING_CHUNK_SIZE;
done = 0;
}
else