Issue #15379: Fix passing of non-BMP characters as integers for the charmap decoder (already working as unicode strings).
Patch by Serhiy Storchaka.
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 59179c4..4e808ec 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1693,6 +1693,15 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict", "\U0010FFFFbc"),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
)
@@ -1718,6 +1727,113 @@
("", len(allbytes))
)
+ def test_decode_with_int2str_map(self):
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: 'c'}),
+ ("abc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'Aa', 1: 'Bb', 2: 'Cc'}),
+ ("AaBbCc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: '\U0010FFFF', 1: 'b', 2: 'c'}),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 2: ''}),
+ ("ab", 3)
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b'}
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b'}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b'}),
+ ("ab", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab", 3)
+ )
+
+ allbytes = bytes(range(256))
+ self.assertEqual(
+ codecs.charmap_decode(allbytes, "ignore", {}),
+ ("", len(allbytes))
+ )
+
+ def test_decode_with_int2int_map(self):
+ a = ord('a')
+ b = ord('b')
+ c = ord('c')
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: c}),
+ ("abc", 3)
+ )
+
+ # Issue #15379
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: 0x10FFFF, 1: b, 2: c}),
+ ("\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "strict",
+ {0: sys.maxunicode, 1: b, 2: c}),
+ (chr(sys.maxunicode) + "bc", 3)
+ )
+
+ self.assertRaises(TypeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: sys.maxunicode + 1, 1: b, 2: c}
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: a, 1: b},
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: a, 1: b}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: a, 1: b}),
+ ("ab", 3)
+ )
+
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
f = io.BytesIO(b"\xc3\xbc")
diff --git a/Misc/NEWS b/Misc/NEWS
index 29fd307..49d6834 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
Core and Builtins
-----------------
+- Issue #15379: Fix passing of non-BMP characters as integers for the charmap
+ decoder (already working as unicode strings). Patch by Serhiy Storchaka.
+
- Issue #15144: Fix possible integer overflow when handling pointers as
integer values, by using Py_uintptr_t instead of size_t. Patch by
Serhiy Storchaka.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 748508b..0da565a 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -7525,9 +7525,10 @@
/* Apply mapping */
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
- if (value < 0 || value > 65535) {
- PyErr_SetString(PyExc_TypeError,
- "character mapping must be in range(65536)");
+ if (value < 0 || value > MAX_UNICODE) {
+ PyErr_Format(PyExc_TypeError,
+ "character mapping must be in range(0x%lx)",
+ (unsigned long)MAX_UNICODE + 1);
Py_DECREF(x);
goto onError;
}