Merged revisions 81908 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
................
r81908 | antoine.pitrou | 2010-06-11 23:46:32 +0200 (ven., 11 juin 2010) | 11 lines
Merged revisions 81907 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r81907 | antoine.pitrou | 2010-06-11 23:42:26 +0200 (ven., 11 juin 2010) | 5 lines
Issue #8941: decoding big endian UTF-32 data in UCS-2 builds could crash
the interpreter with characters outside the Basic Multilingual Plane
(higher than 0x10000).
........
................
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 5db5dcb..5ee2d9f 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -354,6 +354,16 @@
self.check_state_handling_decode(self.encoding,
"spamspam", self.spambe)
+ def test_issue8941(self):
+ # Issue #8941: insufficient result allocation when decoding into
+ # surrogate pairs on UCS-2 builds.
+ encoded_le = b'\xff\xfe\x00\x00' + b'\x00\x00\x01\x00' * 1024
+ self.assertEqual('\U00010000' * 1024,
+ codecs.utf_32_decode(encoded_le)[0])
+ encoded_be = b'\x00\x00\xfe\xff' + b'\x00\x01\x00\x00' * 1024
+ self.assertEqual('\U00010000' * 1024,
+ codecs.utf_32_decode(encoded_be)[0])
+
class UTF32LETest(ReadTest):
encoding = "utf-32-le"
@@ -387,6 +397,13 @@
self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode,
b"\xff", "strict", True)
+ def test_issue8941(self):
+ # Issue #8941: insufficient result allocation when decoding into
+ # surrogate pairs on UCS-2 builds.
+ encoded = b'\x00\x00\x01\x00' * 1024
+ self.assertEqual('\U00010000' * 1024,
+ codecs.utf_32_le_decode(encoded)[0])
+
class UTF32BETest(ReadTest):
encoding = "utf-32-be"
@@ -420,6 +437,14 @@
self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
b"\xff", "strict", True)
+ def test_issue8941(self):
+ # Issue #8941: insufficient result allocation when decoding into
+ # surrogate pairs on UCS-2 builds.
+ encoded = b'\x00\x01\x00\x00' * 1024
+ self.assertEqual('\U00010000' * 1024,
+ codecs.utf_32_be_decode(encoded)[0])
+
+
class UTF16Test(ReadTest):
encoding = "utf-16"
diff --git a/Misc/NEWS b/Misc/NEWS
index 9638ebd..6f9f76f 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@
Core and Builtins
-----------------
+- Issue #8941: decoding big endian UTF-32 data in UCS-2 builds could crash
+ the interpreter with characters outside the Basic Multilingual Plane
+ (higher than 0x10000).
+
- In the str.format(), raise a ValueError when indexes to arguments are too
large.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index a9ff3b3..cc0bbec 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -2618,11 +2618,11 @@
PyUnicodeObject *unicode;
Py_UNICODE *p;
#ifndef Py_UNICODE_WIDE
- int i, pairs;
+ int pairs = 0;
#else
const int pairs = 0;
#endif
- const unsigned char *q, *e;
+ const unsigned char *q, *e, *qq;
int bo = 0; /* assume native ordering by default */
const char *errmsg = "";
/* Offsets from q for retrieving bytes in the right order. */
@@ -2633,23 +2633,7 @@
#endif
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
- /* On narrow builds we split characters outside the BMP into two
- codepoints => count how much extra space we need. */
-#ifndef Py_UNICODE_WIDE
- for (i = pairs = 0; i < size/4; i++)
- if (((Py_UCS4 *)s)[i] >= 0x10000)
- pairs++;
-#endif
-
- /* This might be one to much, because of a BOM */
- unicode = _PyUnicode_New((size+3)/4+pairs);
- if (!unicode)
- return NULL;
- if (size == 0)
- return (PyObject *)unicode;
-
- /* Unpack UTF-32 encoded data */
- p = unicode->str;
+
q = (unsigned char *)s;
e = q + size;
@@ -2701,6 +2685,24 @@
iorder[3] = 0;
}
+ /* On narrow builds we split characters outside the BMP into two
+ codepoints => count how much extra space we need. */
+#ifndef Py_UNICODE_WIDE
+ for (qq = q; qq < e; qq += 4)
+ if (qq[iorder[2]] != 0 || qq[iorder[3]] != 0)
+ pairs++;
+#endif
+
+ /* This might be one to much, because of a BOM */
+ unicode = _PyUnicode_New((size+3)/4+pairs);
+ if (!unicode)
+ return NULL;
+ if (size == 0)
+ return (PyObject *)unicode;
+
+ /* Unpack UTF-32 encoded data */
+ p = unicode->str;
+
while (q < e) {
Py_UCS4 ch;
/* remaining bytes at the end? (size should be divisible by 4) */