bpo-36311: Fixes decoding multibyte characters around chunk boundaries and improves decoding performance (GH-15083)
(cherry picked from commit 7ebdda0dbee7df6f0c945a7e1e623e47676e112d)
Co-authored-by: Steve Dower <steve.dower@python.org>
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 132f57a..c28f36f 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7118,6 +7118,12 @@
#define NEED_RETRY
#endif
+/* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when
+ transcoding from UTF-16), but INT_MAX / 4 perfoms better in
+ both cases also and avoids partial characters overrunning the
+ length limit in MultiByteToWideChar on Windows */
+#define DECODING_CHUNK_SIZE (INT_MAX/4)
+
#ifndef WC_ERR_INVALID_CHARS
# define WC_ERR_INVALID_CHARS 0x0080
#endif
@@ -7354,8 +7360,8 @@
do
{
#ifdef NEED_RETRY
- if (size > INT_MAX) {
- chunk_size = INT_MAX;
+ if (size > DECODING_CHUNK_SIZE) {
+ chunk_size = DECODING_CHUNK_SIZE;
final = 0;
done = 0;
}
@@ -7759,10 +7765,8 @@
do
{
#ifdef NEED_RETRY
- /* UTF-16 encoding may double the size, so use only INT_MAX/2
- chunks. */
- if (len > INT_MAX/2) {
- chunk_len = INT_MAX/2;
+ if (len > DECODING_CHUNK_SIZE) {
+ chunk_len = DECODING_CHUNK_SIZE;
done = 0;
}
else