Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
Amaury Forgeot d'Arc. Added tests for partial decoding of non-BMP
characters.
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index f2a1ae3..15cd687 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -330,7 +330,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read
@@ -352,6 +352,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -386,7 +390,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -404,6 +408,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -426,7 +434,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"",
@@ -444,6 +452,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -494,7 +506,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"", # first byte of BOM read
"", # second byte of BOM read => byteorder known
@@ -506,6 +518,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -543,7 +559,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -553,6 +569,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -582,7 +602,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
[
"",
"\x00",
@@ -592,6 +612,10 @@
"\x00\xff\u0100",
"\x00\xff\u0100",
"\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff",
+ "\x00\xff\u0100\uffff\U00010000",
]
)
@@ -621,7 +645,7 @@
def test_partial(self):
self.check_partial(
- "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
[
"\x00",
"\x00",
@@ -634,6 +658,10 @@
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800",
"\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff",
+ "\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
@@ -816,7 +844,7 @@
def test_partial(self):
self.check_partial(
- "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
[
"",
"",
@@ -835,6 +863,10 @@
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800",
"\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff",
+ "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
]
)
diff --git a/Misc/NEWS b/Misc/NEWS
index b40d4c7..8a5d14d 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@
Core and Builtins
-----------------
+- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
+ Amaury Forgeot d'Arc.
+
- Issue #16881: Fix Py_ARRAY_LENGTH macro for GCC < 3.1.
- Issue #16856: Fix a segmentation fault from calling repr() on a dict with
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 1522a16..b4fc004 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -5284,8 +5284,11 @@
/* The remaining input chars are ignored if the callback
chooses to skip the input */
case 1:
+ q -= 2;
+ if (consumed)
+ goto End;
errmsg = "unexpected end of data";
- startinpos = ((const char *)q) - 2 - starts;
+ startinpos = ((const char *)q) - starts;
endinpos = ((const char *)e) - starts;
break;
case 2: