[Python-checkins] bpo-38324: Fix test__locale.py Windows failures (GH-20529)

Tue Oct 20 07:39:58 EDT 2020

https://github.com/python/cpython/commit/f2312037e3a974d26ed3e23884f94c6af111a27a
commit: f2312037e3a974d26ed3e23884f94c6af111a27a
branch: master
author: TIGirardi <tiagoigirardi at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020年10月20日T12:39:52+01:00
summary:
bpo-38324: Fix test__locale.py Windows failures (GH-20529)
Use wide-char _W_* fields of lconv structure on Windows
Remove "ps_AF" from test__locale.known_numerics on Windows
files:
A Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
M Lib/test/test__locale.py
M Modules/_localemodule.c
M Python/fileutils.c

diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py
index cda0ee91b700f..59a00bad7d98a 100644
--- a/Lib/test/test__locale.py
+++ b/Lib/test/test__locale.py
@@ -72,6 +72,10 @@ def accept(loc):
 'ps_AF': ('\u066b', '\u066c'),
 }
 
+if sys.platform == 'win32':
+ # ps_AF doesn't work on Windows: see bpo-38324 (msg361830)
+ del known_numerics['ps_AF']
+
 class _LocaleTests(unittest.TestCase):
 
 def setUp(self):
diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
new file mode 100644
index 0000000000000..c45aa13091429
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst
@@ -0,0 +1 @@
+Avoid Unicode errors when accessing certain locale data on Windows.
diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c
index 0fe2e08b41f9f..9c7ce876e4059 100644
--- a/Modules/_localemodule.c
+++ b/Modules/_localemodule.c
@@ -155,6 +155,7 @@ locale_is_ascii(const char *str)
 static int
 locale_decode_monetary(PyObject *dict, struct lconv *lc)
 {
+#ifndef MS_WINDOWS
 int change_locale;
 change_locale = (!locale_is_ascii(lc->int_curr_symbol)
 || !locale_is_ascii(lc->currency_symbol)
@@ -190,12 +191,18 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
 }
 }
 
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
 int res = -1;
 
 #define RESULT_STRING(ATTR) \
 do { \
 PyObject *obj; \
- obj = PyUnicode_DecodeLocale(lc->ATTR, NULL); \
+ obj = GET_LOCALE_STRING(ATTR); \
 if (obj == NULL) { \
 goto done; \
 } \
@@ -211,14 +218,17 @@ locale_decode_monetary(PyObject *dict, struct lconv *lc)
 RESULT_STRING(mon_decimal_point);
 RESULT_STRING(mon_thousands_sep);
 #undef RESULT_STRING
+#undef GET_LOCALE_STRING
 
 res = 0;
 
 done:
+#ifndef MS_WINDOWS
 if (loc != NULL) {
 setlocale(LC_CTYPE, oldloc);
 }
 PyMem_Free(oldloc);
+#endif
 return res;
 }
 
@@ -258,9 +268,15 @@ _locale_localeconv_impl(PyObject *module)
 Py_DECREF(obj); \
 } while (0)
 
+#ifdef MS_WINDOWS
+/* Use _W_* fields of Windows struct lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#else
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#endif
 #define RESULT_STRING(s)\
 do { \
- x = PyUnicode_DecodeLocale(lc->s, NULL); \
+ x = GET_LOCALE_STRING(s); \
 RESULT(#s, x); \
 } while (0)
 
@@ -289,8 +305,10 @@ _locale_localeconv_impl(PyObject *module)
 RESULT_INT(n_sign_posn);
 
 /* Numeric information: LC_NUMERIC encoding */
- PyObject *decimal_point, *thousands_sep;
+ PyObject *decimal_point = NULL, *thousands_sep = NULL;
 if (_Py_GetLocaleconvNumeric(lc, &decimal_point, &thousands_sep) < 0) {
+ Py_XDECREF(decimal_point);
+ Py_XDECREF(thousands_sep);
 goto failed;
 }
 
@@ -319,6 +337,7 @@ _locale_localeconv_impl(PyObject *module)
 #undef RESULT
 #undef RESULT_STRING
 #undef RESULT_INT
+#undef GET_LOCALE_STRING
 }
 
 #if defined(HAVE_WCSCOLL)
diff --git a/Python/fileutils.c b/Python/fileutils.c
index b79067f2b5d38..e125ba46c21ba 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -2047,6 +2047,7 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
 assert(decimal_point != NULL);
 assert(thousands_sep != NULL);
 
+#ifndef MS_WINDOWS
 int change_locale = 0;
 if ((strlen(lc->decimal_point) > 1 || ((unsigned char)lc->decimal_point[0]) > 127)) {
 change_locale = 1;
@@ -2085,14 +2086,20 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
 }
 }
 
+#define GET_LOCALE_STRING(ATTR) PyUnicode_DecodeLocale(lc->ATTR, NULL)
+#else /* MS_WINDOWS */
+/* Use _W_* fields of Windows strcut lconv */
+#define GET_LOCALE_STRING(ATTR) PyUnicode_FromWideChar(lc->_W_ ## ATTR, -1)
+#endif /* MS_WINDOWS */
+
 int res = -1;
 
- *decimal_point = PyUnicode_DecodeLocale(lc->decimal_point, NULL);
+ *decimal_point = GET_LOCALE_STRING(decimal_point);
 if (*decimal_point == NULL) {
 goto done;
 }
 
- *thousands_sep = PyUnicode_DecodeLocale(lc->thousands_sep, NULL);
+ *thousands_sep = GET_LOCALE_STRING(thousands_sep);
 if (*thousands_sep == NULL) {
 goto done;
 }
@@ -2100,11 +2107,15 @@ _Py_GetLocaleconvNumeric(struct lconv *lc,
 res = 0;
 
 done:
+#ifndef MS_WINDOWS
 if (loc != NULL) {
 setlocale(LC_CTYPE, oldloc);
 }
 PyMem_Free(oldloc);
+#endif
 return res;
+
+#undef GET_LOCALE_STRING
 }
 
 /* Our selection logic for which function to use is as follows:
