Issue #22286: The "backslashreplace" error handlers now works with
decoding and translating.
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 7439b9f..c2b5d6c 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -127,7 +127,8 @@
'surrogateescape' - replace with private code points U+DCnn.
'xmlcharrefreplace' - Replace with the appropriate XML
character reference (only for encoding).
- 'backslashreplace' - Replace with backslashed escape sequences
+ 'backslashreplace' - Replace with backslashed escape sequences.
+ 'namereplace' - Replace with \\N{...} escape sequences
(only for encoding).
The set of allowed values can be extended via register_error.
@@ -359,7 +360,8 @@
'xmlcharrefreplace' - Replace with the appropriate XML
character reference.
'backslashreplace' - Replace with backslashed escape
- sequences (only for encoding).
+ sequences.
+ 'namereplace' - Replace with \\N{...} escape sequences.
The set of allowed parameter values can be extended via
register_error.
@@ -429,7 +431,8 @@
'strict' - raise a ValueError (or a subclass)
'ignore' - ignore the character and continue with the next
- 'replace'- replace with a suitable replacement character;
+ 'replace'- replace with a suitable replacement character
+ 'backslashreplace' - Replace with backslashed escape sequences;
The set of allowed parameter values can be extended via
register_error.
diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
index 9743791..e29ac53 100644
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -246,6 +246,11 @@
"\u0000\ufffd"
)
+ self.assertEqual(
+ b"\x00\x00\x00\x00\x00".decode("unicode-internal", "backslashreplace"),
+ "\u0000\\x00"
+ )
+
codecs.register_error("test.hui", handler_unicodeinternal)
self.assertEqual(
@@ -565,17 +570,6 @@
codecs.backslashreplace_errors,
UnicodeError("ouch")
)
- # "backslashreplace" can only be used for encoding
- self.assertRaises(
- TypeError,
- codecs.backslashreplace_errors,
- UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")
- )
- self.assertRaises(
- TypeError,
- codecs.backslashreplace_errors,
- UnicodeTranslateError("\u3042", 0, 1, "ouch")
- )
# Use the correct exception
self.assertEqual(
codecs.backslashreplace_errors(
@@ -701,6 +695,16 @@
UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")),
("\\udfff", 1)
)
+ self.assertEqual(
+ codecs.backslashreplace_errors(
+ UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")),
+ ("\\xff", 1)
+ )
+ self.assertEqual(
+ codecs.backslashreplace_errors(
+ UnicodeTranslateError("\u3042", 0, 1, "ouch")),
+ ("\\u3042", 1)
+ )
def test_badhandlerresults(self):
results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 7fed1f7..353a850 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -378,6 +378,10 @@
before + after)
self.assertEqual(test_sequence.decode(self.encoding, "replace"),
before + self.ill_formed_sequence_replace + after)
+ backslashreplace = ''.join('\\x%02x' % b
+ for b in self.ill_formed_sequence)
+ self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
+ before + backslashreplace + after)
class UTF32Test(ReadTest, unittest.TestCase):
encoding = "utf-32"
@@ -1300,14 +1304,19 @@
"unicode_internal")
if sys.byteorder == "little":
invalid = b"\x00\x00\x11\x00"
+ invalid_backslashreplace = r"\x00\x00\x11\x00"
else:
invalid = b"\x00\x11\x00\x00"
+ invalid_backslashreplace = r"\x00\x11\x00\x00"
with support.check_warnings():
self.assertRaises(UnicodeDecodeError,
invalid.decode, "unicode_internal")
with support.check_warnings():
self.assertEqual(invalid.decode("unicode_internal", "replace"),
'\ufffd')
+ with support.check_warnings():
+ self.assertEqual(invalid.decode("unicode_internal", "backslashreplace"),
+ invalid_backslashreplace)
@unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
def test_decode_error_attributes(self):
@@ -2043,6 +2052,16 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab"),
+ ("ab\\x02", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab\ufffe"),
+ ("ab\\x02", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore", "ab"),
("ab", 3)
)
@@ -2119,6 +2138,25 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+ {0: 'a', 1: 'b'}),
+ ("ab\\x02", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+ {0: 'a', 1: 'b', 2: None}),
+ ("ab\\x02", 3)
+ )
+
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab\\x02", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: 'a', 1: 'b'}),
("ab", 3)
@@ -2195,6 +2233,18 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+ {0: a, 1: b}),
+ ("ab\\x02", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab\\x02", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: a, 1: b}),
("ab", 3)
@@ -2253,9 +2303,13 @@
self.assertRaises(UnicodeDecodeError, codecs.unicode_escape_decode, br"\U00110000")
self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
+ self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "backslashreplace"),
+ (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
self.assertRaises(UnicodeDecodeError, codecs.raw_unicode_escape_decode, br"\U00110000")
self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
+ self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "backslashreplace"),
+ (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
class UnicodeEscapeTest(unittest.TestCase):
@@ -2894,11 +2948,13 @@
(b'[\xff]', 'strict', None),
(b'[\xff]', 'ignore', '[]'),
(b'[\xff]', 'replace', '[\ufffd]'),
+ (b'[\xff]', 'backslashreplace', '[\\xff]'),
(b'[\xff]', 'surrogateescape', '[\udcff]'),
(b'[\xff]', 'surrogatepass', None),
(b'\x81\x00abc', 'strict', None),
(b'\x81\x00abc', 'ignore', '\x00abc'),
(b'\x81\x00abc', 'replace', '\ufffd\x00abc'),
+ (b'\x81\x00abc', 'backslashreplace', '\\xff\x00abc'),
))
def test_cp1252(self):