bpo-36346: Make using the legacy Unicode C API optional (GH-21437)
Add compile time option USE_UNICODE_WCHAR_CACHE. Setting it to 0
makes the interpreter not using the wchar_t cache and the legacy Unicode C API.
diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
index 49ad32d..615b4a9 100644
--- a/Include/cpython/unicodeobject.h
+++ b/Include/cpython/unicodeobject.h
@@ -11,7 +11,9 @@
/* --- Internal Unicode Operations ---------------------------------------- */
-#define USE_UNICODE_WCHAR_CACHE 1
+#ifndef USE_UNICODE_WCHAR_CACHE
+# define USE_UNICODE_WCHAR_CACHE 1
+#endif /* USE_UNICODE_WCHAR_CACHE */
/* Since splitting on whitespace is an important use case, and
whitespace in most situations is solely ASCII whitespace, we
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index b21978a..1ce3a78 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -36,6 +36,11 @@
from .testresult import get_test_runner
+try:
+ from _testcapi import unicode_legacy_string
+except ImportError:
+ unicode_legacy_string = None
+
__all__ = [
# globals
"PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast",
@@ -426,6 +431,9 @@
lzma = None
return unittest.skipUnless(lzma, reason)
+requires_legacy_unicode_capi = unittest.skipUnless(unicode_legacy_string,
+ 'requires legacy Unicode C API')
+
is_jython = sys.platform.startswith('java')
is_android = hasattr(sys, 'getandroidapilevel')
diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
index d421be0..a92870c 100644
--- a/Lib/test/test_csv.py
+++ b/Lib/test/test_csv.py
@@ -250,9 +250,9 @@
self.assertRaises(OSError, writer.writerows, BadIterable())
@support.cpython_only
+ @support.requires_legacy_unicode_capi
def test_writerows_legacy_strings(self):
import _testcapi
-
c = _testcapi.unicode_legacy_string('a')
with TemporaryFile("w+", newline='') as fileobj:
writer = csv.writer(fileobj)
diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py
index 716e6eb..9dbae44 100644
--- a/Lib/test/test_decimal.py
+++ b/Lib/test/test_decimal.py
@@ -33,7 +33,8 @@
import numbers
import locale
from test.support import (run_unittest, run_doctest, is_resource_enabled,
- requires_IEEE_754, requires_docstrings)
+ requires_IEEE_754, requires_docstrings,
+ requires_legacy_unicode_capi)
from test.support import (TestFailed,
run_with_locale, cpython_only)
from test.support.import_helper import import_fresh_module
@@ -582,6 +583,7 @@
self.assertRaises(InvalidOperation, Decimal, "1_2_\u00003")
@cpython_only
+ @requires_legacy_unicode_capi
def test_from_legacy_strings(self):
import _testcapi
Decimal = self.decimal.Decimal
@@ -2817,6 +2819,7 @@
Overflow])
@cpython_only
+ @requires_legacy_unicode_capi
def test_from_legacy_strings(self):
import _testcapi
c = self.decimal.Context()
diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py
index d39ea56..0956019 100644
--- a/Lib/test/test_getargs2.py
+++ b/Lib/test/test_getargs2.py
@@ -976,6 +976,7 @@
buf = bytearray()
self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf)
+ @support.requires_legacy_unicode_capi
def test_u(self):
from _testcapi import getargs_u
self.assertEqual(getargs_u('abc\xe9'), 'abc\xe9')
@@ -985,6 +986,7 @@
self.assertRaises(TypeError, getargs_u, memoryview(b'memoryview'))
self.assertRaises(TypeError, getargs_u, None)
+ @support.requires_legacy_unicode_capi
def test_u_hash(self):
from _testcapi import getargs_u_hash
self.assertEqual(getargs_u_hash('abc\xe9'), 'abc\xe9')
@@ -994,6 +996,7 @@
self.assertRaises(TypeError, getargs_u_hash, memoryview(b'memoryview'))
self.assertRaises(TypeError, getargs_u_hash, None)
+ @support.requires_legacy_unicode_capi
def test_Z(self):
from _testcapi import getargs_Z
self.assertEqual(getargs_Z('abc\xe9'), 'abc\xe9')
@@ -1003,6 +1006,7 @@
self.assertRaises(TypeError, getargs_Z, memoryview(b'memoryview'))
self.assertIsNone(getargs_Z(None))
+ @support.requires_legacy_unicode_capi
def test_Z_hash(self):
from _testcapi import getargs_Z_hash
self.assertEqual(getargs_Z_hash('abc\xe9'), 'abc\xe9')
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index afc9555..d485bc7 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -723,6 +723,7 @@
self.assertFalse("0".isidentifier())
@support.cpython_only
+ @support.requires_legacy_unicode_capi
def test_isidentifier_legacy(self):
import _testcapi
u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊'
@@ -2350,6 +2351,7 @@
self.assertEqual(len(args), 1)
@support.cpython_only
+ @support.requires_legacy_unicode_capi
def test_resize(self):
from _testcapi import getargs_u
for length in range(1, 100, 7):
@@ -2920,6 +2922,7 @@
self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0)
@support.cpython_only
+ @support.requires_legacy_unicode_capi
def test_encode_decimal(self):
from _testcapi import unicode_encodedecimal
self.assertEqual(unicode_encodedecimal('123'),
@@ -2936,6 +2939,7 @@
unicode_encodedecimal, "123\u20ac", "replace")
@support.cpython_only
+ @support.requires_legacy_unicode_capi
def test_transform_decimal(self):
from _testcapi import unicode_transformdecimaltoascii as transform_decimal
self.assertEqual(transform_decimal('123'),
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index 7c8ba37..b9856b3 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -270,7 +270,14 @@
if (!PyUnicode_FSDecoder(nameobj, &stringobj)) {
return -1;
}
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
widename = PyUnicode_AsUnicode(stringobj);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ widename = PyUnicode_AsWideCharString(stringobj, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (widename == NULL)
return -1;
#else
@@ -491,6 +498,11 @@
internal_close(self);
done:
+#ifdef MS_WINDOWS
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(widename);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+#endif
Py_CLEAR(stringobj);
return ret;
}
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 1e4c31f..fca94a8 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -1668,6 +1668,7 @@
static volatile int x;
+#if USE_UNICODE_WCHAR_CACHE
/* Ignore use of deprecated APIs */
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
@@ -1772,6 +1773,8 @@
Py_DECREF(tuple);
Py_RETURN_NONE;
}
+_Py_COMP_DIAG_POP
+#endif /* USE_UNICODE_WCHAR_CACHE */
static PyObject *
test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored))
@@ -1824,6 +1827,10 @@
return raiseTestError("test_widechar",
"PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail");
+#if USE_UNICODE_WCHAR_CACHE
+/* Ignore use of deprecated APIs */
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
wide = PyUnicode_FromUnicode(invalid, 1);
if (wide == NULL)
PyErr_Clear();
@@ -1844,11 +1851,12 @@
return raiseTestError("test_widechar",
"PyUnicode_Ready() didn't fail");
}
+_Py_COMP_DIAG_POP
+#endif /* USE_UNICODE_WCHAR_CACHE */
#endif
Py_RETURN_NONE;
}
-_Py_COMP_DIAG_POP
static PyObject *
unicode_aswidechar(PyObject *self, PyObject *args)
@@ -2024,6 +2032,7 @@
return Py_BuildValue("(Nn)", to_copy, copied);
}
+#if USE_UNICODE_WCHAR_CACHE
/* Ignore use of deprecated APIs */
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
@@ -2096,6 +2105,7 @@
return u;
}
_Py_COMP_DIAG_POP
+#endif /* USE_UNICODE_WCHAR_CACHE */
static PyObject *
getargs_w_star(PyObject *self, PyObject *args)
@@ -5398,8 +5408,10 @@
{"codec_incrementaldecoder",
(PyCFunction)codec_incrementaldecoder, METH_VARARGS},
{"test_s_code", test_s_code, METH_NOARGS},
+#if USE_UNICODE_WCHAR_CACHE
{"test_u_code", test_u_code, METH_NOARGS},
{"test_Z_code", test_Z_code, METH_NOARGS},
+#endif /* USE_UNICODE_WCHAR_CACHE */
{"test_widechar", test_widechar, METH_NOARGS},
{"unicode_aswidechar", unicode_aswidechar, METH_VARARGS},
{"unicode_aswidecharstring",unicode_aswidecharstring, METH_VARARGS},
@@ -5408,9 +5420,11 @@
{"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS},
{"unicode_findchar", unicode_findchar, METH_VARARGS},
{"unicode_copycharacters", unicode_copycharacters, METH_VARARGS},
+#if USE_UNICODE_WCHAR_CACHE
{"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS},
{"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS},
{"unicode_legacy_string", unicode_legacy_string, METH_VARARGS},
+#endif /* USE_UNICODE_WCHAR_CACHE */
{"_test_thread_state", test_thread_state, METH_VARARGS},
{"_pending_threadfunc", pending_threadfunc, METH_VARARGS},
#ifdef HAVE_GETTIMEOFDAY
diff --git a/Modules/_winapi.c b/Modules/_winapi.c
index e1672c4..ddb11aa 100644
--- a/Modules/_winapi.c
+++ b/Modules/_winapi.c
@@ -164,10 +164,11 @@
create_converter('BOOL', 'i') # F_BOOL used previously (always 'i')
create_converter('DWORD', 'k') # F_DWORD is always "k" (which is much shorter)
create_converter('LPCTSTR', 's')
-create_converter('LPCWSTR', 'u')
-create_converter('LPWSTR', 'u')
create_converter('UINT', 'I') # F_UINT used previously (always 'I')
+class LPCWSTR_converter(Py_UNICODE_converter):
+ type = 'LPCWSTR'
+
class HANDLE_return_converter(CReturnConverter):
type = 'HANDLE'
@@ -197,7 +198,7 @@
data.return_conversion.append(
'return_value = HANDLE_TO_PYNUM(_return_value);\n')
[python start generated code]*/
-/*[python end generated code: output=da39a3ee5e6b4b0d input=79464c61a31ae932]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=011ee0c3a2244bfe]*/
#include "clinic/_winapi.c.h"
@@ -520,15 +521,15 @@
/*[clinic input]
_winapi.CreateJunction
- src_path: LPWSTR
- dst_path: LPWSTR
+ src_path: LPCWSTR
+ dst_path: LPCWSTR
/
[clinic start generated code]*/
static PyObject *
-_winapi_CreateJunction_impl(PyObject *module, LPWSTR src_path,
- LPWSTR dst_path)
-/*[clinic end generated code: output=66b7eb746e1dfa25 input=8cd1f9964b6e3d36]*/
+_winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path,
+ LPCWSTR dst_path)
+/*[clinic end generated code: output=44b3f5e9bbcc4271 input=963d29b44b9384a7]*/
{
/* Privilege adjustment */
HANDLE token = NULL;
diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h
index 6022dfe..a9630d5 100644
--- a/Modules/clinic/_winapi.c.h
+++ b/Modules/clinic/_winapi.c.h
@@ -195,8 +195,8 @@
LPCWSTR name;
HANDLE _return_value;
- if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkku:CreateFileMapping",
- &file_handle, &security_attributes, &protect, &max_size_high, &max_size_low, &name)) {
+ if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "" F_POINTER "kkkO&:CreateFileMapping",
+ &file_handle, &security_attributes, &protect, &max_size_high, &max_size_low, _PyUnicode_WideCharString_Converter, &name)) {
goto exit;
}
_return_value = _winapi_CreateFileMapping_impl(module, file_handle, security_attributes, protect, max_size_high, max_size_low, name);
@@ -209,6 +209,11 @@
return_value = HANDLE_TO_PYNUM(_return_value);
exit:
+ /* Cleanup for name */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)name);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+
return return_value;
}
@@ -221,23 +226,55 @@
{"CreateJunction", (PyCFunction)(void(*)(void))_winapi_CreateJunction, METH_FASTCALL, _winapi_CreateJunction__doc__},
static PyObject *
-_winapi_CreateJunction_impl(PyObject *module, LPWSTR src_path,
- LPWSTR dst_path);
+_winapi_CreateJunction_impl(PyObject *module, LPCWSTR src_path,
+ LPCWSTR dst_path);
static PyObject *
_winapi_CreateJunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
- LPWSTR src_path;
- LPWSTR dst_path;
+ LPCWSTR src_path;
+ LPCWSTR dst_path;
- if (!_PyArg_ParseStack(args, nargs, "uu:CreateJunction",
- &src_path, &dst_path)) {
+ if (!_PyArg_CheckPositional("CreateJunction", nargs, 2, 2)) {
+ goto exit;
+ }
+ if (!PyUnicode_Check(args[0])) {
+ _PyArg_BadArgument("CreateJunction", "argument 1", "str", args[0]);
+ goto exit;
+ }
+ #if USE_UNICODE_WCHAR_CACHE
+ src_path = _PyUnicode_AsUnicode(args[0]);
+ #else /* USE_UNICODE_WCHAR_CACHE */
+ src_path = PyUnicode_AsWideCharString(args[0], NULL);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+ if (src_path == NULL) {
+ goto exit;
+ }
+ if (!PyUnicode_Check(args[1])) {
+ _PyArg_BadArgument("CreateJunction", "argument 2", "str", args[1]);
+ goto exit;
+ }
+ #if USE_UNICODE_WCHAR_CACHE
+ dst_path = _PyUnicode_AsUnicode(args[1]);
+ #else /* USE_UNICODE_WCHAR_CACHE */
+ dst_path = PyUnicode_AsWideCharString(args[1], NULL);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+ if (dst_path == NULL) {
goto exit;
}
return_value = _winapi_CreateJunction_impl(module, src_path, dst_path);
exit:
+ /* Cleanup for src_path */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)src_path);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+ /* Cleanup for dst_path */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)dst_path);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+
return return_value;
}
@@ -715,8 +752,8 @@
LPCWSTR name;
HANDLE _return_value;
- if (!_PyArg_ParseStack(args, nargs, "kiu:OpenFileMapping",
- &desired_access, &inherit_handle, &name)) {
+ if (!_PyArg_ParseStack(args, nargs, "kiO&:OpenFileMapping",
+ &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) {
goto exit;
}
_return_value = _winapi_OpenFileMapping_impl(module, desired_access, inherit_handle, name);
@@ -729,6 +766,11 @@
return_value = HANDLE_TO_PYNUM(_return_value);
exit:
+ /* Cleanup for name */
+ #if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free((void *)name);
+ #endif /* USE_UNICODE_WCHAR_CACHE */
+
return return_value;
}
@@ -1106,4 +1148,4 @@
exit:
return return_value;
}
-/*[clinic end generated code: output=db87076a32fa7abe input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1f10e03f64ff9777 input=a9049054013a1b77]*/
diff --git a/Modules/overlapped.c b/Modules/overlapped.c
index 9c4e2da..4f0ba85 100644
--- a/Modules/overlapped.c
+++ b/Modules/overlapped.c
@@ -1291,6 +1291,7 @@
static int
parse_address(PyObject *obj, SOCKADDR *Address, int Length)
{
+ PyObject *Host_obj;
Py_UNICODE *Host;
unsigned short Port;
unsigned long FlowInfo;
@@ -1298,33 +1299,66 @@
memset(Address, 0, Length);
- if (PyArg_ParseTuple(obj, "uH", &Host, &Port))
- {
+ switch (PyTuple_GET_SIZE(obj)) {
+ case 2: {
+ if (!PyArg_ParseTuple(obj, "UH", &Host_obj, &Port)) {
+ return -1;
+ }
+#if USE_UNICODE_WCHAR_CACHE
+ Host = (wchar_t *)_PyUnicode_AsUnicode(Host_obj);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ Host = PyUnicode_AsWideCharString(Host_obj, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ if (Host == NULL) {
+ return -1;
+ }
Address->sa_family = AF_INET;
if (WSAStringToAddressW(Host, AF_INET, NULL, Address, &Length) < 0) {
SetFromWindowsErr(WSAGetLastError());
- return -1;
+ Length = -1;
}
- ((SOCKADDR_IN*)Address)->sin_port = htons(Port);
+ else {
+ ((SOCKADDR_IN*)Address)->sin_port = htons(Port);
+ }
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(Host);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return Length;
}
- else if (PyArg_ParseTuple(obj,
- "uHkk;ConnectEx(): illegal address_as_bytes "
- "argument", &Host, &Port, &FlowInfo, &ScopeId))
- {
- PyErr_Clear();
+ case 4: {
+ if (!PyArg_ParseTuple(obj,
+ "UHkk;ConnectEx(): illegal address_as_bytes argument",
+ &Host_obj, &Port, &FlowInfo, &ScopeId))
+ {
+ return -1;
+ }
+#if USE_UNICODE_WCHAR_CACHE
+ Host = (wchar_t *)_PyUnicode_AsUnicode(Host_obj);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ Host = PyUnicode_AsWideCharString(Host_obj, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ if (Host == NULL) {
+ return -1;
+ }
Address->sa_family = AF_INET6;
if (WSAStringToAddressW(Host, AF_INET6, NULL, Address, &Length) < 0) {
SetFromWindowsErr(WSAGetLastError());
- return -1;
+ Length = -1;
}
- ((SOCKADDR_IN6*)Address)->sin6_port = htons(Port);
- ((SOCKADDR_IN6*)Address)->sin6_flowinfo = FlowInfo;
- ((SOCKADDR_IN6*)Address)->sin6_scope_id = ScopeId;
+ else {
+ ((SOCKADDR_IN6*)Address)->sin6_port = htons(Port);
+ ((SOCKADDR_IN6*)Address)->sin6_flowinfo = FlowInfo;
+ ((SOCKADDR_IN6*)Address)->sin6_scope_id = ScopeId;
+ }
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(Host);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return Length;
}
-
- return -1;
+ default:
+ PyErr_SetString(PyExc_ValueError, "illegal address_as_bytes argument");
+ return -1;
+ }
}
/*[clinic input]
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index a411f28..efd9954 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -988,6 +988,11 @@
static void
path_cleanup(path_t *path)
{
+#if !USE_UNICODE_WCHAR_CACHE
+ wchar_t *wide = (wchar_t *)path->wide;
+ path->wide = NULL;
+ PyMem_Free(wide);
+#endif /* USE_UNICODE_WCHAR_CACHE */
Py_CLEAR(path->object);
Py_CLEAR(path->cleanup);
}
@@ -1002,7 +1007,7 @@
const char *narrow;
#ifdef MS_WINDOWS
PyObject *wo = NULL;
- const wchar_t *wide;
+ wchar_t *wide = NULL;
#endif
#define FORMAT_EXCEPTION(exc, fmt) \
@@ -1075,7 +1080,14 @@
if (is_unicode) {
#ifdef MS_WINDOWS
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
wide = PyUnicode_AsUnicodeAndSize(o, &length);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wide = PyUnicode_AsWideCharString(o, &length);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (!wide) {
goto error_exit;
}
@@ -1091,6 +1103,9 @@
path->wide = wide;
path->narrow = FALSE;
path->fd = -1;
+#if !USE_UNICODE_WCHAR_CACHE
+ wide = NULL;
+#endif /* USE_UNICODE_WCHAR_CACHE */
goto success_exit;
#else
if (!PyUnicode_FSConverter(o, &bytes)) {
@@ -1166,7 +1181,15 @@
goto error_exit;
}
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
wide = PyUnicode_AsUnicodeAndSize(wo, &length);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wide = PyUnicode_AsWideCharString(wo, &length);
+ Py_DECREF(wo);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (!wide) {
goto error_exit;
}
@@ -1180,8 +1203,12 @@
}
path->wide = wide;
path->narrow = TRUE;
- path->cleanup = wo;
Py_DECREF(bytes);
+#if USE_UNICODE_WCHAR_CACHE
+ path->cleanup = wo;
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wide = NULL;
+#endif /* USE_UNICODE_WCHAR_CACHE */
#else
path->wide = NULL;
path->narrow = narrow;
@@ -1205,7 +1232,11 @@
Py_XDECREF(o);
Py_XDECREF(bytes);
#ifdef MS_WINDOWS
+#if USE_UNICODE_WCHAR_CACHE
Py_XDECREF(wo);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ PyMem_Free(wide);
+#endif /* USE_UNICODE_WCHAR_CACHE */
#endif
return 0;
}
@@ -12824,7 +12855,15 @@
#ifdef MS_WINDOWS
if (!PyUnicode_FSDecoder(self->path, &ub))
return NULL;
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
const wchar_t *path = PyUnicode_AsUnicode(ub);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *path = PyUnicode_AsWideCharString(ub, NULL);
+ Py_DECREF(ub);
+#endif /* USE_UNICODE_WCHAR_CACHE */
#else /* POSIX */
if (!PyUnicode_FSConverter(self->path, &ub))
return NULL;
@@ -12834,6 +12873,7 @@
result = fstatat(self->dir_fd, path, &st,
follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW);
#else
+ Py_DECREF(ub);
PyErr_SetString(PyExc_NotImplementedError, "can't fetch stat");
return NULL;
#endif /* HAVE_FSTATAT */
@@ -12846,7 +12886,11 @@
else
result = LSTAT(path, &st);
}
+#if defined(MS_WINDOWS) && !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(path);
+#else /* USE_UNICODE_WCHAR_CACHE */
Py_DECREF(ub);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (result != 0)
return path_object_error(self->path);
@@ -13035,15 +13079,24 @@
#ifdef MS_WINDOWS
if (!self->got_file_index) {
PyObject *unicode;
- const wchar_t *path;
STRUCT_STAT stat;
int result;
if (!PyUnicode_FSDecoder(self->path, &unicode))
return NULL;
- path = PyUnicode_AsUnicode(unicode);
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+ const wchar_t *path = PyUnicode_AsUnicode(unicode);
result = LSTAT(path, &stat);
Py_DECREF(unicode);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *path = PyUnicode_AsWideCharString(unicode, NULL);
+ Py_DECREF(unicode);
+ result = LSTAT(path, &stat);
+ PyMem_Free(path);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (result != 0)
return path_object_error(self->path);
@@ -13597,10 +13650,9 @@
iterator->dirp = NULL;
#endif
- memcpy(&iterator->path, path, sizeof(path_t));
/* Move the ownership to iterator->path */
- path->object = NULL;
- path->cleanup = NULL;
+ memcpy(&iterator->path, path, sizeof(path_t));
+ memset(path, 0, sizeof(path_t));
#ifdef MS_WINDOWS
iterator->first_time = 1;
@@ -13622,9 +13674,9 @@
#else /* POSIX */
errno = 0;
#ifdef HAVE_FDOPENDIR
- if (path->fd != -1) {
+ if (iterator->path.fd != -1) {
/* closedir() closes the FD, so we duplicate it */
- fd = _Py_dup(path->fd);
+ fd = _Py_dup(iterator->path.fd);
if (fd == -1)
goto error;
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 648dd15..2e1045a 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -3150,9 +3150,11 @@
assert(unicode != NULL);
assert(_PyUnicode_CHECK(unicode));
+#if USE_UNICODE_WCHAR_CACHE
if (_PyUnicode_WSTR(unicode) != NULL) {
return PyUnicode_WSTR_LENGTH(unicode);
}
+#endif /* USE_UNICODE_WCHAR_CACHE */
assert(PyUnicode_IS_READY(unicode));
res = _PyUnicode_LENGTH(unicode);
@@ -3173,16 +3175,21 @@
static void
unicode_copy_as_widechar(PyObject *unicode, wchar_t *w, Py_ssize_t size)
{
- const wchar_t *wstr;
-
assert(unicode != NULL);
assert(_PyUnicode_CHECK(unicode));
- wstr = _PyUnicode_WSTR(unicode);
+#if USE_UNICODE_WCHAR_CACHE
+ const wchar_t *wstr = _PyUnicode_WSTR(unicode);
if (wstr != NULL) {
memcpy(w, wstr, size * sizeof(wchar_t));
return;
}
+#else /* USE_UNICODE_WCHAR_CACHE */
+ if (PyUnicode_KIND(unicode) == sizeof(wchar_t)) {
+ memcpy(w, PyUnicode_DATA(unicode), size * sizeof(wchar_t));
+ return;
+ }
+#endif /* USE_UNICODE_WCHAR_CACHE */
assert(PyUnicode_IS_READY(unicode));
if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) {
@@ -4378,7 +4385,6 @@
Py_ssize_t requiredsize;
Py_ssize_t newpos;
PyObject *inputobj = NULL;
- wchar_t *repwstr;
Py_ssize_t repwlen;
if (*errorHandler == NULL) {
@@ -4424,9 +4430,19 @@
goto onError;
}
- repwstr = PyUnicode_AsUnicodeAndSize(repunicode, &repwlen);
- if (repwstr == NULL)
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+ repwlen = PyUnicode_GetSize(repunicode);
+ if (repwlen < 0)
goto onError;
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ repwlen = PyUnicode_AsWideChar(repunicode, NULL, 0);
+ if (repwlen < 0)
+ goto onError;
+ repwlen--;
+#endif /* USE_UNICODE_WCHAR_CACHE */
/* need more space? (at least enough for what we
have+the replacement+the rest of the string (starting
at the new input position), so we won't have to check space
@@ -4446,7 +4462,7 @@
goto onError;
}
}
- wcsncpy(*buf + *outpos, repwstr, repwlen);
+ PyUnicode_AsWideChar(repunicode, *buf + *outpos, repwlen);
*outpos += repwlen;
*endinpos = newpos;
*inptr = *input + newpos;
@@ -7748,6 +7764,7 @@
/* Create a substring so that we can get the UTF-16 representation
of just the slice under consideration. */
PyObject *substring;
+ int ret = -1;
assert(len > 0);
@@ -7759,11 +7776,22 @@
substring = PyUnicode_Substring(unicode, offset, offset+len);
if (substring == NULL)
return -1;
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
p = PyUnicode_AsUnicodeAndSize(substring, &size);
if (p == NULL) {
Py_DECREF(substring);
return -1;
}
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ p = PyUnicode_AsWideCharString(substring, &size);
+ Py_CLEAR(substring);
+ if (p == NULL) {
+ return -1;
+ }
+#endif /* USE_UNICODE_WCHAR_CACHE */
assert(size <= INT_MAX);
/* First get the size of the result */
@@ -7775,16 +7803,15 @@
goto error;
/* If we used a default char, then we failed! */
if (pusedDefaultChar && *pusedDefaultChar) {
- Py_DECREF(substring);
- return -2;
+ ret = -2;
+ goto done;
}
if (*outbytes == NULL) {
/* Create string object */
*outbytes = PyBytes_FromStringAndSize(NULL, outsize);
if (*outbytes == NULL) {
- Py_DECREF(substring);
- return -1;
+ goto done;
}
out = PyBytes_AS_STRING(*outbytes);
}
@@ -7793,12 +7820,10 @@
const Py_ssize_t n = PyBytes_Size(*outbytes);
if (outsize > PY_SSIZE_T_MAX - n) {
PyErr_NoMemory();
- Py_DECREF(substring);
- return -1;
+ goto done;
}
if (_PyBytes_Resize(outbytes, n + outsize) < 0) {
- Py_DECREF(substring);
- return -1;
+ goto done;
}
out = PyBytes_AS_STRING(*outbytes) + n;
}
@@ -7808,19 +7833,29 @@
p, (int)size,
out, outsize,
NULL, pusedDefaultChar);
- Py_CLEAR(substring);
if (outsize <= 0)
goto error;
- if (pusedDefaultChar && *pusedDefaultChar)
- return -2;
- return 0;
+ if (pusedDefaultChar && *pusedDefaultChar) {
+ ret = -2;
+ goto done;
+ }
+ ret = 0;
+
+done:
+#if USE_UNICODE_WCHAR_CACHE
+ Py_DECREF(substring);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ PyMem_Free(p);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ return ret;
error:
- Py_XDECREF(substring);
- if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION)
- return -2;
+ if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) {
+ ret = -2;
+ goto done;
+ }
PyErr_SetFromWindowsErr(0);
- return -1;
+ goto done;
}
/*
diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h
index 3301bed..183301f 100644
--- a/PC/clinic/winreg.c.h
+++ b/PC/clinic/winreg.c.h
@@ -1143,8 +1143,7 @@
static PyObject *
winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key,
- DWORD type, const Py_UNICODE *value,
- Py_ssize_clean_t value_length);
+ DWORD type, PyObject *value_obj);
static PyObject *
winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
@@ -1153,14 +1152,13 @@
HKEY key;
const Py_UNICODE *sub_key;
DWORD type;
- const Py_UNICODE *value;
- Py_ssize_clean_t value_length;
+ PyObject *value_obj;
- if (!_PyArg_ParseStack(args, nargs, "O&O&ku#:SetValue",
- clinic_HKEY_converter, &key, _PyUnicode_WideCharString_Opt_Converter, &sub_key, &type, &value, &value_length)) {
+ if (!_PyArg_ParseStack(args, nargs, "O&O&kU:SetValue",
+ clinic_HKEY_converter, &key, _PyUnicode_WideCharString_Opt_Converter, &sub_key, &type, &value_obj)) {
goto exit;
}
- return_value = winreg_SetValue_impl(module, key, sub_key, type, value, value_length);
+ return_value = winreg_SetValue_impl(module, key, sub_key, type, value_obj);
exit:
/* Cleanup for sub_key */
@@ -1348,4 +1346,4 @@
exit:
return return_value;
}
-/*[clinic end generated code: output=30b1311886c13907 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=497a2e804821d5c9 input=a9049054013a1b77]*/
diff --git a/PC/winreg.c b/PC/winreg.c
index b2725b8..a24d784 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -640,16 +640,25 @@
for (j = 0; j < i; j++)
{
PyObject *t;
- wchar_t *wstr;
Py_ssize_t len;
t = PyList_GET_ITEM(value, j);
if (!PyUnicode_Check(t))
return FALSE;
- wstr = PyUnicode_AsUnicodeAndSize(t, &len);
- if (wstr == NULL)
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+ len = PyUnicode_GetSize(t);
+ if (len < 0)
return FALSE;
- size += Py_SAFE_DOWNCAST((len + 1) * sizeof(wchar_t),
+ len++;
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ len = PyUnicode_AsWideChar(t, NULL, 0);
+ if (len < 0)
+ return FALSE;
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ size += Py_SAFE_DOWNCAST(len * sizeof(wchar_t),
size_t, DWORD);
}
@@ -665,17 +674,18 @@
for (j = 0; j < i; j++)
{
PyObject *t;
- wchar_t *wstr;
Py_ssize_t len;
t = PyList_GET_ITEM(value, j);
- wstr = PyUnicode_AsUnicodeAndSize(t, &len);
- assert(wstr);
- wcscpy(P, wstr);
- P += (len + 1);
+ assert(size > 0);
+ len = PyUnicode_AsWideChar(t, P, size);
+ assert(len >= 0);
+ assert(len < size);
+ size -= (DWORD)len + 1;
+ P += len + 1;
}
/* And doubly-terminate the list... */
- *P = '\0';
+ *P = L'\0';
break;
}
case REG_BINARY:
@@ -1669,7 +1679,7 @@
type: DWORD
An integer that specifies the type of the data. Currently this must
be REG_SZ, meaning only strings are supported.
- value: Py_UNICODE(zeroes=True)
+ value as value_obj: unicode
A string that specifies the new value.
/
@@ -1688,30 +1698,51 @@
static PyObject *
winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key,
- DWORD type, const Py_UNICODE *value,
- Py_ssize_clean_t value_length)
-/*[clinic end generated code: output=686bedb1cbb4367b input=2cd2adab79339c53]*/
+ DWORD type, PyObject *value_obj)
+/*[clinic end generated code: output=d4773dc9c372311a input=bf088494ae2d24fd]*/
{
+ Py_ssize_t value_length;
long rc;
if (type != REG_SZ) {
PyErr_SetString(PyExc_TypeError, "type must be winreg.REG_SZ");
return NULL;
}
- if ((size_t)value_length >= PY_DWORD_MAX) {
+
+#if USE_UNICODE_WCHAR_CACHE
+_Py_COMP_DIAG_PUSH
+_Py_COMP_DIAG_IGNORE_DEPR_DECLS
+ const wchar_t *value = PyUnicode_AsUnicodeAndSize(value_obj, &value_length);
+_Py_COMP_DIAG_POP
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *value = PyUnicode_AsWideCharString(value_obj, &value_length);
+#endif /* USE_UNICODE_WCHAR_CACHE */
+ if (value == NULL) {
+ return NULL;
+ }
+ if ((Py_ssize_t)(DWORD)value_length != value_length) {
PyErr_SetString(PyExc_OverflowError, "value is too long");
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(value);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return NULL;
}
if (PySys_Audit("winreg.SetValue", "nunu#",
(Py_ssize_t)key, sub_key, (Py_ssize_t)type,
value, value_length) < 0) {
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(value);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return NULL;
}
Py_BEGIN_ALLOW_THREADS
rc = RegSetValueW(key, sub_key, REG_SZ, value, (DWORD)(value_length + 1));
Py_END_ALLOW_THREADS
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(value);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (rc != ERROR_SUCCESS)
return PyErr_SetFromWindowsErrWithFunction(rc, "RegSetValue");
Py_RETURN_NONE;
diff --git a/Python/dynload_win.c b/Python/dynload_win.c
index 8431c5b..5702ab2 100644
--- a/Python/dynload_win.c
+++ b/Python/dynload_win.c
@@ -166,11 +166,14 @@
{
dl_funcptr p;
char funcname[258], *import_python;
- const wchar_t *wpathname;
_Py_CheckPython3();
- wpathname = _PyUnicode_AsUnicode(pathname);
+#if USE_UNICODE_WCHAR_CACHE
+ const wchar_t *wpathname = _PyUnicode_AsUnicode(pathname);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (wpathname == NULL)
return NULL;
@@ -192,6 +195,9 @@
LOAD_LIBRARY_SEARCH_DEFAULT_DIRS |
LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR);
Py_END_ALLOW_THREADS
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(wpathname);
+#endif /* USE_UNICODE_WCHAR_CACHE */
/* restore old error mode settings */
SetErrorMode(old_mode);
diff --git a/Python/fileutils.c b/Python/fileutils.c
index 2c86828..50ef3c1 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -1008,15 +1008,21 @@
#ifdef MS_WINDOWS
int err;
struct _stat wstatbuf;
- const wchar_t *wpath;
- wpath = _PyUnicode_AsUnicode(path);
+#if USE_UNICODE_WCHAR_CACHE
+ const wchar_t *wpath = _PyUnicode_AsUnicode(path);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (wpath == NULL)
return -2;
err = _wstat(wpath, &wstatbuf);
if (!err)
statbuf->st_mode = wstatbuf.st_mode;
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(wpath);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return err;
#else
int ret;
@@ -1433,7 +1439,6 @@
FILE *f;
int async_err = 0;
#ifdef MS_WINDOWS
- const wchar_t *wpath;
wchar_t wmode[10];
int usize;
@@ -1448,7 +1453,11 @@
Py_TYPE(path));
return NULL;
}
- wpath = _PyUnicode_AsUnicode(path);
+#if USE_UNICODE_WCHAR_CACHE
+ const wchar_t *wpath = _PyUnicode_AsUnicode(path);
+#else /* USE_UNICODE_WCHAR_CACHE */
+ wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
+#endif /* USE_UNICODE_WCHAR_CACHE */
if (wpath == NULL)
return NULL;
@@ -1456,6 +1465,9 @@
wmode, Py_ARRAY_LENGTH(wmode));
if (usize == 0) {
PyErr_SetFromWindowsErr(0);
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(wpath);
+#endif /* USE_UNICODE_WCHAR_CACHE */
return NULL;
}
@@ -1465,6 +1477,9 @@
Py_END_ALLOW_THREADS
} while (f == NULL
&& errno == EINTR && !(async_err = PyErr_CheckSignals()));
+#if !USE_UNICODE_WCHAR_CACHE
+ PyMem_Free(wpath);
+#endif /* USE_UNICODE_WCHAR_CACHE */
#else
PyObject *bytes;
const char *path_bytes;