bpo-37388: Development mode check encoding and errors (GH-14341)
In development mode and in debug build, encoding and errors arguments
are now checked on string encoding and decoding operations. Examples:
open(), str.encode() and bytes.decode().
By default, for best performances, the errors argument is only
checked at the first encoding/decoding error, and the encoding
argument is sometimes ignored for empty strings.
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index 0b6493b..c355164 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -36,6 +36,8 @@
# Does io.IOBase finalizer log the exception if the close() method fails?
# The exception is ignored silently by default in release build.
_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
+# Does open() check its 'errors' argument?
+_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
@@ -2022,6 +2024,8 @@
else:
if not isinstance(errors, str):
raise ValueError("invalid errors: %r" % errors)
+ if _CHECK_ERRORS:
+ codecs.lookup_error(errors)
self._buffer = buffer
self._decoded_chars = '' # buffer for text returned from decoder
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index bbd45c7..b5eeb2b 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -12,12 +12,14 @@
import functools
import pickle
import tempfile
+import textwrap
import unittest
import test.support
import test.string_tests
import test.list_tests
from test.support import bigaddrspacetest, MAX_Py_ssize_t
+from test.support.script_helper import assert_python_failure
if sys.flags.bytes_warning:
@@ -315,6 +317,62 @@
# Default encoding is utf-8
self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603')
+ def test_check_encoding_errors(self):
+ # bpo-37388: bytes(str) and bytes.encode() must check encoding
+ # and errors arguments in dev mode
+ invalid = 'Boom, Shaka Laka, Boom!'
+ encodings = ('ascii', 'utf8', 'latin1')
+ code = textwrap.dedent(f'''
+ import sys
+ type2test = {self.type2test.__name__}
+ encodings = {encodings!r}
+
+ for data in ('', 'short string'):
+ try:
+ type2test(data, encoding={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(21)
+
+ for encoding in encodings:
+ try:
+ type2test(data, encoding=encoding, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(22)
+
+ for data in (b'', b'short string'):
+ data = type2test(data)
+ print(repr(data))
+ try:
+ data.decode(encoding={invalid!r})
+ except LookupError:
+ sys.exit(10)
+ else:
+ sys.exit(23)
+
+ try:
+ data.decode(errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(24)
+
+ for encoding in encodings:
+ try:
+ data.decode(encoding=encoding, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(25)
+
+ sys.exit(10)
+ ''')
+ proc = assert_python_failure('-X', 'dev', '-c', code)
+ self.assertEqual(proc.rc, 10, proc)
+
def test_from_int(self):
b = self.type2test(0)
self.assertEqual(b, self.type2test())
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index fc474c9..1fe1cba 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -29,6 +29,7 @@
import signal
import sys
import sysconfig
+import textwrap
import threading
import time
import unittest
@@ -37,7 +38,8 @@
from collections import deque, UserList
from itertools import cycle, count
from test import support
-from test.support.script_helper import assert_python_ok, run_python_until_end
+from test.support.script_helper import (
+ assert_python_ok, assert_python_failure, run_python_until_end)
from test.support import FakePath
import codecs
@@ -4130,6 +4132,51 @@
# there used to be a buffer overflow in the parser for rawmode
self.assertRaises(ValueError, self.open, support.TESTFN, 'rwax+')
+ def test_check_encoding_errors(self):
+ # bpo-37388: open() and TextIOWrapper must check encoding and errors
+ # arguments in dev mode
+ mod = self.io.__name__
+ filename = __file__
+ invalid = 'Boom, Shaka Laka, Boom!'
+ code = textwrap.dedent(f'''
+ import sys
+ from {mod} import open, TextIOWrapper
+
+ try:
+ open({filename!r}, encoding={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(21)
+
+ try:
+ open({filename!r}, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(22)
+
+ fp = open({filename!r}, "rb")
+ with fp:
+ try:
+ TextIOWrapper(fp, encoding={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(23)
+
+ try:
+ TextIOWrapper(fp, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(24)
+
+ sys.exit(10)
+ ''')
+ proc = assert_python_failure('-X', 'dev', '-c', code)
+ self.assertEqual(proc.rc, 10, proc)
+
class CMiscIOTest(MiscIOTest):
io = io
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 36b72e4..177d80d 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -11,9 +11,11 @@
import operator
import struct
import sys
+import textwrap
import unittest
import warnings
from test import support, string_tests
+from test.support.script_helper import assert_python_failure
# Error handling (bad decoder return)
def search_function(encoding):
@@ -2436,6 +2438,66 @@
support.check_free_after_iterating(self, iter, str)
support.check_free_after_iterating(self, reversed, str)
+ def test_check_encoding_errors(self):
+ # bpo-37388: str(bytes) and str.decode() must check encoding and errors
+ # arguments in dev mode
+ encodings = ('ascii', 'utf8', 'latin1')
+ invalid = 'Boom, Shaka Laka, Boom!'
+ code = textwrap.dedent(f'''
+ import sys
+ encodings = {encodings!r}
+
+ for data in (b'', b'short string'):
+ try:
+ str(data, encoding={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(21)
+
+ try:
+ str(data, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(22)
+
+ for encoding in encodings:
+ try:
+ str(data, encoding, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(22)
+
+ for data in ('', 'short string'):
+ try:
+ data.encode(encoding={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(23)
+
+ try:
+ data.encode(errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(24)
+
+ for encoding in encodings:
+ try:
+ data.encode(encoding, errors={invalid!r})
+ except LookupError:
+ pass
+ else:
+ sys.exit(24)
+
+ sys.exit(10)
+ ''')
+ proc = assert_python_failure('-X', 'dev', '-c', code)
+ self.assertEqual(proc.rc, 10, proc)
+
class CAPITest(unittest.TestCase):