bpo-36346: Make using the legacy Unicode C API optional (GH-21437)
Add compile time option USE_UNICODE_WCHAR_CACHE. Setting it to 0
makes the interpreter not using the wchar_t cache and the legacy Unicode C API.
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index 7c8ba37..b9856b3 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -270,7 +270,14 @@
if (!PyUnicode_FSDecoder(nameobj, &stringobj)) {
return -1;
}
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
widename = PyUnicode_AsUnicode(stringobj);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ widename = PyUnicode_AsWideCharString(stringobj, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (widename == NULL)
return -1;
#else
@@ -491,6 +498,11 @@
internal_close(self);
done:
+#ifdef MS_WINDOWS
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(widename);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+#endif
Py_CLEAR(stringobj);
return ret;
}
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 1e4c31f..fca94a8 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -1668,6 +1668,7 @@
static volatile int x;
+#if USE_UNICODE_WCHAR_CACHE
/* Ignore use of deprecated APIs */
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
@@ -1772,6 +1773,8 @@
Py_DECREF(tuple);
Py_RETURN_NONE;
}
+_Py_COMP_DIAG_POP
+#endif /* USE_UNICODE_WCHAR_CACHE */
static PyObject *
test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored))
@@ -1824,6 +1827,10 @@
return raiseTestError("test_widechar",
"PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail");
+#if USE_UNICODE_WCHAR_CACHE
+/* Ignore use of deprecated APIs */
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
wide = PyUnicode_FromUnicode(invalid, 1);
if (wide == NULL)
PyErr_Clear();
@@ -1844,11 +1851,12 @@
return raiseTestError("test_widechar",
"PyUnicode_Ready() didn't fail");
}
+_Py_COMP_DIAG_POP
+#endif /* USE_UNICODE_WCHAR_CACHE */
#endif
Py_RETURN_NONE;
}
-_Py_COMP_DIAG_POP
static PyObject *
unicode_aswidechar(PyObject *self, PyObject *args)
@@ -2024,6 +2032,7 @@
return Py_BuildValue("(Nn)", to_copy, copied);
}
+#if USE_UNICODE_WCHAR_CACHE
/* Ignore use of deprecated APIs */
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
@@ -2096,6 +2105,7 @@
return u;
}
_Py_COMP_DIAG_POP
+#endif /* USE_UNICODE_WCHAR_CACHE */
static PyObject *
getargs_w_star(PyObject *self, PyObject *args)
@@ -5398,8 +5408,10 @@
{"codec_incrementaldecoder",
(PyCFunction)codec_incrementaldecoder, METH_VARARGS},
{"test_s_code", test_s_code, METH_NOARGS},
+#if USE_UNICODE_WCHAR_CACHE
{"test_u_code", test_u_code, METH_NOARGS},
{"test_Z_code", test_Z_code, METH_NOARGS},
+#endif /* USE_UNICODE_WCHAR_CACHE */
{"test_widechar", test_widechar, METH_NOARGS},
{"unicode_aswidechar", unicode_aswidechar, METH_VARARGS},
{"unicode_aswidecharstring",unicode_aswidecharstring, METH_VARARGS},
@@ -5408,9 +5420,11 @@
{"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS},
{"unicode_findchar", unicode_findchar, METH_VARARGS},
{"unicode_copycharacters", unicode_copycharacters, METH_VARARGS},
+#if USE_UNICODE_WCHAR_CACHE
{"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS},
{"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS},
{"unicode_legacy_string", unicode_legacy_string, METH_VARARGS},
+#endif /* USE_UNICODE_WCHAR_CACHE */
{"_test_thread_state", test_thread_state, METH_VARARGS},
{"_pending_threadfunc", pending_threadfunc, METH_VARARGS},
#ifdef HAVE_GETTIMEOFDAY
diff --git a/Modules/_winapi.c b/Modules/_winapi.c
index e1672c4..ddb11aa 100644
--- a/Modules/_winapi.c
+++ b/Modules/_winapi.c
@@ -164,10 +164,11 @@
create_converter('BOOL', 'i') # F_BOOL used previously (always 'i')
create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter)
create_converter('LPCTSTR', 's')
-create_converter('LPCWSTR', 'u')
-create_converter('LPWSTR', 'u')
create_converter('UINT', 'I') # F_UINT used previously (always 'I')
+class LPCWSTR_converter(Py_UNICODE_converter):
+ type = 'LPCWSTR'
+
class HANDLE_return_converter(CReturnConverter):
type = 'HANDLE'
@@ -197,7 +198,7 @@
data.return_conversion.append(
'return_value = HANDLE_TO_PYNUM(_return_value);\n')
[python start generated code]*/
-/*[python end generated code: output=da39a3ee5e6b4b0d input=79464c61a31ae932]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=011ee0c3a2244bfe]*/
#include "clinic/_winapi.c.h"
@@ -520,15 +521,15 @@
/*[clinic input]
_winapi.CreateJunction
- src_path: LPWSTR
- dst_path: LPWSTR
+ src_path: LPCWSTR
+ dst_path: LPCWSTR
/
[clinic start generated code]*/
static PyObject *
-_winapi_CreateJunction_impl(PyObject *module, LPWSTR src_path,
- LPWSTR dst_path)
-/*[clinic end generated code: output=66b7eb746e1dfa25 input=8cd1f9964b6e3d36]*/
+_winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path,
+ LPCWSTR dst_path)
+/*[clinic end generated code: output=44b3f5e9bbcc4271 input=963d29b44b9384a7]*/
{
/* Privilege adjustment */
HANDLE token = NULL;
diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h
index 6022dfe..a9630d5 100644
--- a/Modules/clinic/_winapi.c.h
+++ b/Modules/clinic/_winapi.c.h
@@ -195,8 +195,8 @@
LPCWSTR name;
HANDLE _return_value;
- if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkku:CreateFileMapping",
- &file_handle, &security_attributes, &protect, &max_size_high, &max_size_low, &name)) {
+ if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkkO&:CreateFileMapping",
+ &file_handle, &security_attributes, &protect, &max_size_high, &max_size_low, _PyUnicode_WideCharString_Converter, &name)) {
goto exit;
}
_return_value = _winapi_CreateFileMapping_impl(module, file_handle, security_attributes, protect, max_size_high, max_size_low, name);
@@ -209,6 +209,11 @@
return_value = HANDLE_TO_PYNUM(_return_value);
exit:
+ /* Cleanup for name */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)name);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+
return return_value;
}
@@ -221,23 +226,55 @@
{"CreateJunction", (PyCFunction)(void(*)(void))_winapi_CreateJunction, METH_FASTCALL, _winapi_CreateJunction__doc__},
static PyObject *
-_winapi_CreateJunction_impl(PyObject *module, LPWSTR src_path,
- LPWSTR dst_path);
+_winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path,
+ LPCWSTR dst_path);
static PyObject *
_winapi_CreateJunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
- LPWSTR src_path;
- LPWSTR dst_path;
+ LPCWSTR src_path;
+ LPCWSTR dst_path;
- if (!_PyArg_ParseStack(args, nargs, "uu:CreateJunction",
- &src_path, &dst_path)) {
+ if (!_PyArg_CheckPositional("CreateJunction", nargs, 2, 2)) {
+ goto exit;
+ }
+ if (!PyUnicode_Check(args[0])) {
+ _PyArg_BadArgument("CreateJunction", "argument 1", "str", args[0]);
+ goto exit;
+ }
+ #if USE_UNICODE_WCHAR_CACHE
+ src_path = _PyUnicode_AsUnicode(args[0]);
+ #else /* USE_UNICODE_WCHAR_CACHE */
+ src_path = PyUnicode_AsWideCharString(args[0], NULL);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+ if (src_path == NULL) {
+ goto exit;
+ }
+ if (!PyUnicode_Check(args[1])) {
+ _PyArg_BadArgument("CreateJunction", "argument 2", "str", args[1]);
+ goto exit;
+ }
+ #if USE_UNICODE_WCHAR_CACHE
+ dst_path = _PyUnicode_AsUnicode(args[1]);
+ #else /* USE_UNICODE_WCHAR_CACHE */
+ dst_path = PyUnicode_AsWideCharString(args[1], NULL);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+ if (dst_path == NULL) {
goto exit;
}
return_value = _winapi_CreateJunction_impl(module, src_path, dst_path);
exit:
+ /* Cleanup for src_path */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)src_path);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+ /* Cleanup for dst_path */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)dst_path);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+
return return_value;
}
@@ -715,8 +752,8 @@
LPCWSTR name;
HANDLE _return_value;
- if (!_PyArg_ParseStack(args, nargs, "kiu:OpenFileMapping",
- &desired_access, &inherit_handle, &name)) {
+ if (!_PyArg_ParseStack(args, nargs, "kiO&:OpenFileMapping",
+ &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) {
goto exit;
}
_return_value = _winapi_OpenFileMapping_impl(module, desired_access, inherit_handle, name);
@@ -729,6 +766,11 @@
return_value = HANDLE_TO_PYNUM(_return_value);
exit:
+ /* Cleanup for name */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)name);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+
return return_value;
}
@@ -1106,4 +1148,4 @@
exit:
return return_value;
}
-/*[clinic end generated code: output=db87076a32fa7abe input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1f10e03f64ff9777 input=a9049054013a1b77]*/
diff --git a/Modules/overlapped.c b/Modules/overlapped.c
index 9c4e2da..4f0ba85 100644
--- a/Modules/overlapped.c
+++ b/Modules/overlapped.c
@@ -1291,6 +1291,7 @@
static int
parse_address(PyObject *obj, SOCKADDR *Address, int Length)
{
+ PyObject *Host_obj;
Py_UNICODE *Host;
unsigned short Port;
unsigned long FlowInfo;
@@ -1298,33 +1299,66 @@
memset(Address, 0, Length);
- if (PyArg_ParseTuple(obj, "uH", &Host, &Port))
- {
+ switch (PyTuple_GET_SIZE(obj)) {
+ case 2: {
+ if (!PyArg_ParseTuple(obj, "UH", &Host_obj, &Port)) {
+ return -1;
+ }
+#if USE_UNICODE_WCHAR_CACHE
+ Host = (wchar_t *)_PyUnicode_AsUnicode(Host_obj);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ Host = PyUnicode_AsWideCharString(Host_obj, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ if (Host == NULL) {
+ return -1;
+ }
Address->sa_family = AF_INET;
if (WSAStringToAddressW(Host, AF_INET, NULL, Address, &Length) < 0) {
SetFromWindowsErr(WSAGetLastError());
- return -1;
+ Length = -1;
}
- ((SOCKADDR_IN*)Address)->sin_port = htons(Port);
+ else {
+ ((SOCKADDR_IN*)Address)->sin_port = htons(Port);
+ }
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(Host);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return Length;
}
- else if (PyArg_ParseTuple(obj,
- "uHkk;ConnectEx(): illegal address_as_bytes "
- "argument", &Host, &Port, &FlowInfo, &ScopeId))
- {
- PyErr_Clear();
+ case 4: {
+ if (!PyArg_ParseTuple(obj,
+ "UHkk;ConnectEx(): illegal address_as_bytes argument",
+ &Host_obj, &Port, &FlowInfo, &ScopeId))
+ {
+ return -1;
+ }
+#if USE_UNICODE_WCHAR_CACHE
+ Host = (wchar_t *)_PyUnicode_AsUnicode(Host_obj);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ Host = PyUnicode_AsWideCharString(Host_obj, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ if (Host == NULL) {
+ return -1;
+ }
Address->sa_family = AF_INET6;
if (WSAStringToAddressW(Host, AF_INET6, NULL, Address, &Length) < 0) {
SetFromWindowsErr(WSAGetLastError());
- return -1;
+ Length = -1;
}
- ((SOCKADDR_IN6*)Address)->sin6_port = htons(Port);
- ((SOCKADDR_IN6*)Address)->sin6_flowinfo = FlowInfo;
- ((SOCKADDR_IN6*)Address)->sin6_scope_id = ScopeId;
+ else {
+ ((SOCKADDR_IN6*)Address)->sin6_port = htons(Port);
+ ((SOCKADDR_IN6*)Address)->sin6_flowinfo = FlowInfo;
+ ((SOCKADDR_IN6*)Address)->sin6_scope_id = ScopeId;
+ }
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(Host);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return Length;
}
-
- return -1;
+ default:
+ PyErr_SetString(PyExc_ValueError, "illegal address_as_bytes argument");
+ return -1;
+ }
}
/*[clinic input]
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index a411f28..efd9954 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -988,6 +988,11 @@
static void
path_cleanup(path_t *path)
{
+#if !USE_UNICODE_WCHAR_CACHE
+ wchar_t *wide = (wchar_t *)path->wide;
+ path->wide = NULL;
+ PyMem_Free(wide);
+#endif /* USE_UNICODE_WCHAR_CACHE */
Py_CLEAR(path->object);
Py_CLEAR(path->cleanup);
}
@@ -1002,7 +1007,7 @@
const char *narrow;
#ifdef MS_WINDOWS
PyObject *wo = NULL;
- const wchar_t *wide;
+ wchar_t *wide = NULL;
#endif
#define FORMAT_EXCEPTION(exc, fmt) \
@@ -1075,7 +1080,14 @@
if (is_unicode) {
#ifdef MS_WINDOWS
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
wide = PyUnicode_AsUnicodeAndSize(o, &length);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wide = PyUnicode_AsWideCharString(o, &length);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (!wide) {
goto error_exit;
}
@@ -1091,6 +1103,9 @@
path->wide = wide;
path->narrow = FALSE;
path->fd = -1;
+#if !USE_UNICODE_WCHAR_CACHE
+ wide = NULL;
+#endif /* USE_UNICODE_WCHAR_CACHE */
goto success_exit;
#else
if (!PyUnicode_FSConverter(o, &bytes)) {
@@ -1166,7 +1181,15 @@
goto error_exit;
}
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
wide = PyUnicode_AsUnicodeAndSize(wo, &length);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wide = PyUnicode_AsWideCharString(wo, &length);
+ Py_DECREF(wo);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (!wide) {
goto error_exit;
}
@@ -1180,8 +1203,12 @@
}
path->wide = wide;
path->narrow = TRUE;
- path->cleanup = wo;
Py_DECREF(bytes);
+#if USE_UNICODE_WCHAR_CACHE
+ path->cleanup = wo;
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wide = NULL;
+#endif /* USE_UNICODE_WCHAR_CACHE */
#else
path->wide = NULL;
path->narrow = narrow;
@@ -1205,7 +1232,11 @@
Py_XDECREF(o);
Py_XDECREF(bytes);
#ifdef MS_WINDOWS
+#if USE_UNICODE_WCHAR_CACHE
Py_XDECREF(wo);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ PyMem_Free(wide);
+#endif /* USE_UNICODE_WCHAR_CACHE */
#endif
return 0;
}
@@ -12824,7 +12855,15 @@
#ifdef MS_WINDOWS
if (!PyUnicode_FSDecoder(self->path, &ub))
return NULL;
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
const wchar_t *path = PyUnicode_AsUnicode(ub);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *path = PyUnicode_AsWideCharString(ub, NULL);
+ Py_DECREF(ub);
+#endif /* USE_UNICODE_WCHAR_CACHE */
#else /* POSIX */
if (!PyUnicode_FSConverter(self->path, &ub))
return NULL;
@@ -12834,6 +12873,7 @@
result = fstatat(self->dir_fd, path, &st,
follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW);
#else
+ Py_DECREF(ub);
PyErr_SetString(PyExc_NotImplementedError, "can't fetch stat");
return NULL;
#endif /* HAVE_FSTATAT */
@@ -12846,7 +12886,11 @@
else
result = LSTAT(path, &st);
}
+#if defined(MS_WINDOWS) && !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(path);
+#else /* USE_UNICODE_WCHAR_CACHE */
Py_DECREF(ub);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (result != 0)
return path_object_error(self->path);
@@ -13035,15 +13079,24 @@
#ifdef MS_WINDOWS
if (!self->got_file_index) {
PyObject *unicode;
- const wchar_t *path;
STRUCT_STAT stat;
int result;
if (!PyUnicode_FSDecoder(self->path, &unicode))
return NULL;
- path = PyUnicode_AsUnicode(unicode);
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+ const wchar_t *path = PyUnicode_AsUnicode(unicode);
result = LSTAT(path, &stat);
Py_DECREF(unicode);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *path = PyUnicode_AsWideCharString(unicode, NULL);
+ Py_DECREF(unicode);
+ result = LSTAT(path, &stat);
+ PyMem_Free(path);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (result != 0)
return path_object_error(self->path);
@@ -13597,10 +13650,9 @@
iterator->dirp = NULL;
#endif
- memcpy(&iterator->path, path, sizeof(path_t));
/* Move the ownership to iterator->path */
- path->object = NULL;
- path->cleanup = NULL;
+ memcpy(&iterator->path, path, sizeof(path_t));
+ memset(path, 0, sizeof(path_t));
#ifdef MS_WINDOWS
iterator->first_time = 1;
@@ -13622,9 +13674,9 @@
#else /* POSIX */
errno = 0;
#ifdef HAVE_FDOPENDIR
- if (path->fd != -1) {
+ if (iterator->path.fd != -1) {
/* closedir() closes the FD, so we duplicate it */
- fd = _Py_dup(path->fd);
+ fd = _Py_dup(iterator->path.fd);
if (fd == -1)
goto error;