bpo-29240: readline now ignores the UTF-8 Mode (#5145)
Add new fuctions ignoring the UTF-8 mode:
* _Py_DecodeCurrentLocale()
* _Py_EncodeCurrentLocale()
* _PyUnicode_DecodeCurrentLocaleAndSize()
* _PyUnicode_EncodeCurrentLocale()
Modify the readline module to use these functions.
Re-enable test_readline.test_nonascii().
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 92a6ad6..1a230e0 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -3395,8 +3395,8 @@
}
}
-PyObject *
-PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
+static PyObject *
+unicode_encode_locale(PyObject *unicode, const char *errors, int current_locale)
{
Py_ssize_t wlen, wlen2;
wchar_t *wstr;
@@ -3423,7 +3423,12 @@
/* "surrogateescape" error handler */
char *str;
- str = Py_EncodeLocale(wstr, &error_pos);
+ if (current_locale) {
+ str = _Py_EncodeCurrentLocale(wstr, &error_pos);
+ }
+ else {
+ str = Py_EncodeLocale(wstr, &error_pos);
+ }
if (str == NULL) {
if (error_pos == (size_t)-1) {
PyErr_NoMemory();
@@ -3437,7 +3442,12 @@
PyMem_Free(wstr);
bytes = PyBytes_FromString(str);
- PyMem_Free(str);
+ if (current_locale) {
+ PyMem_RawFree(str);
+ }
+ else {
+ PyMem_Free(str);
+ }
}
else {
/* strict mode */
@@ -3503,6 +3513,18 @@
}
PyObject *
+PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
+{
+ return unicode_encode_locale(unicode, errors, 0);
+}
+
+PyObject *
+_PyUnicode_EncodeCurrentLocale(PyObject *unicode, const char *errors)
+{
+ return unicode_encode_locale(unicode, errors, 1);
+}
+
+PyObject *
PyUnicode_EncodeFSDefault(PyObject *unicode)
{
#if defined(__APPLE__)
@@ -3524,7 +3546,8 @@
Py_FileSystemDefaultEncodeErrors);
}
else {
- return PyUnicode_EncodeLocale(unicode, Py_FileSystemDefaultEncodeErrors);
+ return unicode_encode_locale(unicode,
+ Py_FileSystemDefaultEncodeErrors, 0);
}
#endif
}
@@ -3695,9 +3718,9 @@
return 0;
}
-PyObject*
-PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
- const char *errors)
+static PyObject*
+unicode_decode_locale(const char *str, Py_ssize_t len, const char *errors,
+ int current_locale)
{
wchar_t smallbuf[256];
size_t smallbuf_len = Py_ARRAY_LENGTH(smallbuf);
@@ -3719,7 +3742,12 @@
if (surrogateescape) {
/* "surrogateescape" error handler */
- wstr = Py_DecodeLocale(str, &wlen);
+ if (current_locale) {
+ wstr = _Py_DecodeCurrentLocale(str, &wlen);
+ }
+ else {
+ wstr = Py_DecodeLocale(str, &wlen);
+ }
if (wstr == NULL) {
if (wlen == (size_t)-1)
PyErr_NoMemory();
@@ -3795,10 +3823,24 @@
}
PyObject*
+PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
+ const char *errors)
+{
+ return unicode_decode_locale(str, len, errors, 0);
+}
+
+PyObject*
+_PyUnicode_DecodeCurrentLocaleAndSize(const char *str, Py_ssize_t len,
+ const char *errors)
+{
+ return unicode_decode_locale(str, len, errors, 1);
+}
+
+PyObject*
PyUnicode_DecodeLocale(const char *str, const char *errors)
{
Py_ssize_t size = (Py_ssize_t)strlen(str);
- return PyUnicode_DecodeLocaleAndSize(str, size, errors);
+ return unicode_decode_locale(str, size, errors, 0);
}