[3.7] bpo-34589: Add -X coerce_c_locale option; C locale coercion off by default (GH-9379)
* bpo-34589: Make _PyCoreConfig.coerce_c_locale private (GH-9371)
_PyCoreConfig:
* Rename coerce_c_locale to _coerce_c_locale
* Rename coerce_c_locale_warn to _coerce_c_locale_warn
These fields are now private (name prefixed by "_").
(cherry picked from commit 188ebfa475a6f6aa2d0ea14ca8e1fbe7865b6d27)
* bpo-34589: C locale coercion off by default (GH-9073)
Py_Initialize() and Py_Main() cannot enable the C locale coercion
(PEP 538) anymore: it is always disabled. It can now only be enabled
by the Python program ("python3).
test_embed: get_filesystem_encoding() doesn't have to set PYTHONUTF8
nor PYTHONCOERCECLOCALE, these variables are already set in the
parent.
(cherry picked from commit 7a0791b6992d420dc52536257f2f093851ed7215)
* bpo-34589: Add -X coerce_c_locale command line option (GH-9378)
Add a new -X coerce_c_locale command line option to control C locale
coercion (PEP 538).
(cherry picked from commit dbdee0073cf0b88fe541980ace1f650900f455cc)diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py
index 1db293b..f62208a 100644
--- a/Lib/test/test_c_locale_coercion.py
+++ b/Lib/test/test_c_locale_coercion.py
@@ -139,7 +139,7 @@
return data
@classmethod
- def get_child_details(cls, env_vars):
+ def get_child_details(cls, env_vars, xoption=None):
"""Retrieves fsencoding and standard stream details from a child process
Returns (encoding_details, stderr_lines):
@@ -150,10 +150,11 @@
The child is run in isolated mode if the current interpreter supports
that.
"""
- result, py_cmd = run_python_until_end(
- "-X", "utf8=0", "-c", cls.CHILD_PROCESS_SCRIPT,
- **env_vars
- )
+ args = []
+ if xoption:
+ args.extend(("-X", f"coerce_c_locale={xoption}"))
+ args.extend(("-X", "utf8=0", "-c", cls.CHILD_PROCESS_SCRIPT))
+ result, py_cmd = run_python_until_end(*args, **env_vars)
if not result.rc == 0:
result.fail(py_cmd)
# All subprocess outputs in this test case should be pure ASCII
@@ -212,7 +213,8 @@
expected_fs_encoding,
expected_stream_encoding,
expected_warnings,
- coercion_expected):
+ coercion_expected,
+ xoption=None):
"""Check the C locale handling for the given process environment
Parameters:
@@ -220,7 +222,7 @@
expected_stream_encoding: expected encoding for standard streams
expected_warning: stderr output to expect (if any)
"""
- result = EncodingDetails.get_child_details(env_vars)
+ result = EncodingDetails.get_child_details(env_vars, xoption)
encoding_details, stderr_lines = result
expected_details = EncodingDetails.get_expected_details(
coercion_expected,
@@ -290,6 +292,7 @@
coerce_c_locale,
expected_warnings=None,
coercion_expected=True,
+ use_xoption=False,
**extra_vars):
"""Check the C locale handling for various configurations
@@ -319,8 +322,12 @@
"PYTHONCOERCECLOCALE": "",
}
base_var_dict.update(extra_vars)
+ xoption = None
if coerce_c_locale is not None:
- base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
+ if use_xoption:
+ xoption = coerce_c_locale
+ else:
+ base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
# Check behaviour for the default locale
with self.subTest(default_locale=True,
@@ -342,7 +349,8 @@
fs_encoding,
stream_encoding,
_expected_warnings,
- _coercion_expected)
+ _coercion_expected,
+ xoption=xoption)
# Check behaviour for explicitly configured locales
for locale_to_set in EXPECTED_C_LOCALE_EQUIVALENTS:
@@ -357,7 +365,8 @@
fs_encoding,
stream_encoding,
expected_warnings,
- coercion_expected)
+ coercion_expected,
+ xoption=xoption)
def test_PYTHONCOERCECLOCALE_not_set(self):
# This should coerce to the first available target locale by default
@@ -404,6 +413,32 @@
expected_warnings=[LEGACY_LOCALE_WARNING],
coercion_expected=False)
+ def test_xoption_set_to_1(self):
+ self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale="1",
+ use_xoption=True)
+
+ def test_xoption_set_to_zero(self):
+ # The setting "0" should result in the locale coercion being disabled
+ self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
+ EXPECTED_C_LOCALE_STREAM_ENCODING,
+ coerce_c_locale="0",
+ coercion_expected=False,
+ use_xoption=True)
+ # Setting LC_ALL=C shouldn't make any difference to the behaviour
+ self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
+ EXPECTED_C_LOCALE_STREAM_ENCODING,
+ coerce_c_locale="0",
+ LC_ALL="C",
+ coercion_expected=False,
+ use_xoption=True)
+
+ def test_xoption_set_to_warn(self):
+ # -X coerce_c_locale=warn enables runtime warnings for legacy locales
+ self._check_c_locale_coercion("utf-8", "utf-8",
+ coerce_c_locale="warn",
+ expected_warnings=[CLI_COERCION_WARNING],
+ use_xoption=True)
+
def test_main():
test.support.run_unittest(
LocaleConfigurationTests,
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 95cdc8d..0424f75 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -159,13 +159,16 @@
env = os.environ.copy()
# Use C locale to get ascii for the locale encoding
env['LC_ALL'] = 'C'
- env['PYTHONCOERCECLOCALE'] = '0'
code = (
b'import locale; '
b'print(ascii("' + undecodable + b'"), '
b'locale.getpreferredencoding())')
p = subprocess.Popen(
- [sys.executable, "-c", code],
+ [sys.executable,
+ # Disable C locale coercion and UTF-8 Mode to not use UTF-8
+ "-X", "coerce_c_locale=0",
+ "-X", "utf8=0",
+ "-c", code],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
env=env)
stdout, stderr = p.communicate()
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 2927458..79c34ec 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -267,9 +267,6 @@
'malloc_stats': 0,
'utf8_mode': 0,
- 'coerce_c_locale': 0,
- 'coerce_c_locale_warn': 0,
-
'program_name': './_testembed',
'argc': 0,
'argv': '[]',
@@ -290,6 +287,8 @@
'_disable_importlib': 0,
'Py_FrozenFlag': 0,
+ '_coerce_c_locale': 0,
+ '_coerce_c_locale_warn': 0,
}
def check_config(self, testname, expected):
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 27f7590..9155afc 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -656,9 +656,8 @@
def c_locale_get_error_handler(self, locale, isolated=False, encoding=None):
# Force the POSIX locale
- env = os.environ.copy()
+ env = dict(os.environ)
env["LC_ALL"] = locale
- env["PYTHONCOERCECLOCALE"] = "0"
code = '\n'.join((
'import sys',
'def dump(name):',
@@ -668,7 +667,10 @@
'dump("stdout")',
'dump("stderr")',
))
- args = [sys.executable, "-c", code]
+ args = [sys.executable,
+ "-X", "utf8=0",
+ "-X", "coerce_c_locale=0",
+ "-c", code]
if isolated:
args.append("-I")
if encoding is not None:
diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py
index 554abfa..8c64276 100644
--- a/Lib/test/test_utf8_mode.py
+++ b/Lib/test/test_utf8_mode.py
@@ -27,6 +27,8 @@
return (loc in POSIX_LOCALES)
def get_output(self, *args, failure=False, **kw):
+ # Always disable the C locale coercion (PEP 538)
+ args = ('-X', 'coerce_c_locale=0', *args)
kw = dict(self.DEFAULT_ENV, **kw)
if failure:
out = assert_python_failure(*args, **kw)
@@ -116,7 +118,6 @@
# PYTHONLEGACYWINDOWSFSENCODING disables the UTF-8 mode
# and has the priority over -X utf8 and PYTHONUTF8
out = self.get_output('-X', 'utf8', '-c', code,
- PYTHONUTF8='strict',
PYTHONLEGACYWINDOWSFSENCODING='1')
self.assertEqual(out, 'mbcs/replace')