commit c17ff5cad2952d9abd63aeae060cae5658257b0c
author Miss Skeleton (bot) <31488909+miss-islington@users.noreply.github.com> Tue Oct 20 05:07:14 2020 -0700
committer GitHub <noreply@github.com> Tue Oct 20 05:07:14 2020 -0700
tree 5a63ed2acf3f7805e4d9f13872830d852d6cc5df
parent 3fc7080220b8dd2e1b067b3224879133d895ea80

bpo-38324: Fix test__locale.py Windows failures (GH-20529)


Use wide-char _W_* fields of lconv structure on Windows
Remove "ps_AF" from test__locale.known_numerics on Windows
(cherry picked from commit f2312037e3a974d26ed3e23884f94c6af111a27a)

Co-authored-by: TIGirardi <tiagoigirardi@gmail.com>

Lib/test/test__locale.py

diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index cda0ee9..59a00ba 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -72,6 +72,10 @@
     'ps_AF': ('\u066b', '\u066c'),
 }
 
+if sys.platform == 'win32':
+    # ps_AF doesn't work on Windows: see bpo-38324 (msg361830)
+    del known_numerics['ps_AF']
+
 class _LocaleTests(unittest.TestCase):
 
     def setUp(self):

Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst

diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
new file mode 100644
index 0000000..c45aa13
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
@@ -0,0 +1 @@
+Avoid Unicode errors when accessing certain locale data on Windows.

Modules/_localemodule.c

diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index 0819d0e..2e353bb 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -144,6 +144,7 @@
 static int
 locale_decode_monetary(PyObject *dict, struct lconv *lc)
 {
+#ifndef MS_WINDOWS
     int change_locale;
     change_locale = (!locale_is_ascii(lc->int_curr_symbol)
                      || !locale_is_ascii(lc->currency_symbol)
@@ -179,12 +180,18 @@
         }
     }
 
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else  /* MS_WINDOWS */
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
     int res = -1;
 
 #define RESULT_STRING(ATTR) \
     do { \
         PyObject *obj; \
-        obj = PyUnicode_DecodeLocale(lc->ATTR, NULL); \
+        obj = GET_LOCALE_STRING(ATTR); \
         if (obj == NULL) { \
             goto done; \
         } \
@@ -200,14 +207,17 @@
     RESULT_STRING(mon_decimal_point);
     RESULT_STRING(mon_thousands_sep);
 #undef RESULT_STRING
+#undef GET_LOCALE_STRING
 
     res = 0;
 
 done:
+#ifndef MS_WINDOWS
     if (loc != NULL) {
         setlocale(LC_CTYPE, oldloc);
     }
     PyMem_Free(oldloc);
+#endif
     return res;
 }
 
@@ -243,9 +253,15 @@
         Py_DECREF(obj); \
     } while (0)
 
+#ifdef MS_WINDOWS
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#else
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#endif
 #define RESULT_STRING(s)\
     do { \
-        x = PyUnicode_DecodeLocale(lc->s, NULL); \
+        x = GET_LOCALE_STRING(s); \
         RESULT(#s, x); \
     } while (0)
 
@@ -274,8 +290,10 @@
     RESULT_INT(n_sign_posn);
 
     /* Numeric information: LC_NUMERIC encoding */
-    PyObject *decimal_point, *thousands_sep;
+    PyObject *decimal_point = NULL, *thousands_sep = NULL;
     if (_Py_GetLocaleconvNumeric(lc, &decimal_point, &thousands_sep) < 0) {
+        Py_XDECREF(decimal_point);
+        Py_XDECREF(thousands_sep);
         goto failed;
     }
 
@@ -304,6 +322,7 @@
 #undef RESULT
 #undef RESULT_STRING
 #undef RESULT_INT
+#undef GET_LOCALE_STRING
 }
 
 #if defined(HAVE_WCSCOLL)

Python/fileutils.c

diff --git a/Python/fileutils.c b/Python/fileutils.c
index 2c86828..397ac34 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -2032,6 +2032,7 @@
     assert(decimal_point != NULL);
     assert(thousands_sep != NULL);
 
+#ifndef MS_WINDOWS
     int change_locale = 0;
     if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) {
         change_locale = 1;
@@ -2070,14 +2071,20 @@
         }
     }
 
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows strcut lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
     int res = -1;
 
-    *decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL);
+    *decimal_point = GET_LOCALE_STRING(decimal_point);
     if (*decimal_point == NULL) {
         goto done;
     }
 
-    *thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL);
+    *thousands_sep = GET_LOCALE_STRING(thousands_sep);
     if (*thousands_sep == NULL) {
         goto done;
     }
@@ -2085,9 +2092,13 @@
     res = 0;
 
 done:
+#ifndef MS_WINDOWS
     if (loc != NULL) {
         setlocale(LC_CTYPE, oldloc);
     }
     PyMem_Free(oldloc);
+#endif
     return res;
+
+#undef GET_LOCALE_STRING
 }