bpo-32030: Rework memory allocators (#4625)
* Fix _PyMem_SetupAllocators("debug"): always restore allocators to
the defaults, rather than only caling _PyMem_SetupDebugHooks().
* Add _PyMem_SetDefaultAllocator() helper to set the "default"
allocator.
* Add _PyMem_GetAllocatorsName(): get the name of the allocators
* main() now uses debug hooks on memory allocators if Py_DEBUG is
defined, rather than calling directly malloc()
* Document default memory allocators in C API documentation
* _Py_InitializeCore() now fails with a fatal user error if
PYTHONMALLOC value is an unknown memory allocator, instead of
failing with a fatal internal error.
* Add new tests on the PYTHONMALLOC environment variable
* Add support.with_pymalloc()
* Add the _testcapi.WITH_PYMALLOC constant and expose it as
support.with_pymalloc().
* sysconfig.get_config_var('WITH_PYMALLOC') doesn't work on Windows, so
replace it with support.with_pymalloc().
* pythoninfo: add _testcapi collector for pymemdiff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index 85e32a9..7ad076d 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -56,6 +56,14 @@
info_add(name, value)
+def copy_attr(info_add, name, mod, attr_name):
+ try:
+ value = getattr(mod, attr_name)
+ except AttributeError:
+ return
+ info_add(name, value)
+
+
def call_func(info_add, name, mod, func_name, *, formatter=None):
try:
func = getattr(mod, func_name)
@@ -168,11 +176,10 @@
call_func(info_add, 'os.gid', os, 'getgid')
call_func(info_add, 'os.uname', os, 'uname')
- if hasattr(os, 'getgroups'):
- groups = os.getgroups()
- groups = map(str, groups)
- groups = ', '.join(groups)
- info_add("os.groups", groups)
+ def format_groups(groups):
+ return ', '.join(map(str, groups))
+
+ call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups)
if hasattr(os, 'getlogin'):
try:
@@ -184,11 +191,7 @@
else:
info_add("os.login", login)
- if hasattr(os, 'cpu_count'):
- cpu_count = os.cpu_count()
- if cpu_count:
- info_add('os.cpu_count', cpu_count)
-
+ call_func(info_add, 'os.cpu_count', os, 'cpu_count')
call_func(info_add, 'os.loadavg', os, 'getloadavg')
# Get environment variables: filter to list
@@ -219,7 +222,9 @@
)
for name, value in os.environ.items():
uname = name.upper()
- if (uname in ENV_VARS or uname.startswith(("PYTHON", "LC_"))
+ if (uname in ENV_VARS
+ # Copy PYTHON* and LC_* variables
+ or uname.startswith(("PYTHON", "LC_"))
# Visual Studio: VS140COMNTOOLS
or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))):
info_add('os.environ[%s]' % name, value)
@@ -313,12 +318,10 @@
)
copy_attributes(info_add, time, 'time.%s', attributes)
- if not hasattr(time, 'get_clock_info'):
- return
-
- for clock in ('time', 'perf_counter'):
- tinfo = time.get_clock_info(clock)
- info_add('time.%s' % clock, tinfo)
+ if hasattr(time, 'get_clock_info'):
+ for clock in ('time', 'perf_counter'):
+ tinfo = time.get_clock_info(clock)
+ info_add('time.%s' % clock, tinfo)
def collect_sysconfig(info_add):
@@ -331,7 +334,6 @@
'CCSHARED',
'CFLAGS',
'CFLAGSFORSHARED',
- 'PY_LDFLAGS',
'CONFIG_ARGS',
'HOST_GNU_TYPE',
'MACHDEP',
@@ -339,6 +341,7 @@
'OPT',
'PY_CFLAGS',
'PY_CFLAGS_NODIST',
+ 'PY_LDFLAGS',
'Py_DEBUG',
'Py_ENABLE_SHARED',
'SHELL',
@@ -422,6 +425,16 @@
copy_attributes(info_add, _decimal, '_decimal.%s', attributes)
+def collect_testcapi(info_add):
+ try:
+ import _testcapi
+ except ImportError:
+ return
+
+ call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname')
+ copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC')
+
+
def collect_info(info):
error = False
info_add = info.add
@@ -444,6 +457,7 @@
collect_zlib,
collect_expat,
collect_decimal,
+ collect_testcapi,
):
try:
collect_func(info_add)
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 42c41ff..f0e1507 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -2848,3 +2848,8 @@
def restore(self):
for signum, handler in self.handlers.items():
self.signal.signal(signum, handler)
+
+
+def with_pymalloc():
+ import _testcapi
+ return _testcapi.WITH_PYMALLOC
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 7a10cda..2a6de3c 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -654,8 +654,7 @@
PYTHONMALLOC = 'malloc_debug'
-@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1,
- 'need pymalloc')
+@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
class PyMemPymallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'pymalloc_debug'
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 7f95fcc..96405e7 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -5,6 +5,7 @@
import os
import subprocess
import sys
+import sysconfig
import tempfile
import unittest
from test import support
@@ -559,10 +560,14 @@
except ImportError:
pass
else:
- code = "import _testcapi; _testcapi.pymem_api_misuse()"
+ code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())"
with support.SuppressCrashReport():
out = self.run_xdev("-c", code, check_exitcode=False)
- self.assertIn("Debug memory block at address p=", out)
+ if support.with_pymalloc():
+ alloc_name = "pymalloc_debug"
+ else:
+ alloc_name = "malloc_debug"
+ self.assertEqual(out, alloc_name)
try:
import faulthandler
@@ -573,6 +578,49 @@
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
+ def check_pythonmalloc(self, env_var, name):
+ code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())'
+ env = dict(os.environ)
+ if env_var is not None:
+ env['PYTHONMALLOC'] = env_var
+ else:
+ env.pop('PYTHONMALLOC', None)
+ args = (sys.executable, '-c', code)
+ proc = subprocess.run(args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ env=env)
+ self.assertEqual(proc.stdout.rstrip(), name)
+ self.assertEqual(proc.returncode, 0)
+
+ def test_pythonmalloc(self):
+ # Test the PYTHONMALLOC environment variable
+ pydebug = hasattr(sys, "gettotalrefcount")
+ pymalloc = support.with_pymalloc()
+ if pymalloc:
+ default_name = 'pymalloc_debug' if pydebug else 'pymalloc'
+ default_name_debug = 'pymalloc_debug'
+ else:
+ default_name = 'malloc_debug' if pydebug else 'malloc'
+ default_name_debug = 'malloc_debug'
+
+ tests = [
+ (None, default_name),
+ ('debug', default_name_debug),
+ ('malloc', 'malloc'),
+ ('malloc_debug', 'malloc_debug'),
+ ]
+ if pymalloc:
+ tests.extend((
+ ('pymalloc', 'pymalloc'),
+ ('pymalloc_debug', 'pymalloc_debug'),
+ ))
+
+ for env_var, name in tests:
+ with self.subTest(env_var=env_var, name=name):
+ self.check_pythonmalloc(env_var, name)
+
class IgnoreEnvironmentTest(unittest.TestCase):
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 20965b9..4b8fcb9 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -753,8 +753,15 @@
@unittest.skipUnless(hasattr(sys, "getallocatedblocks"),
"sys.getallocatedblocks unavailable on this build")
def test_getallocatedblocks(self):
+ try:
+ import _testcapi
+ except ImportError:
+ with_pymalloc = support.with_pymalloc()
+ else:
+ alloc_name = _testcapi.pymem_getallocatorsname()
+ with_pymalloc = (alloc_name in ('pymalloc', 'pymalloc_debug'))
+
# Some sanity checks
- with_pymalloc = sysconfig.get_config_var('WITH_PYMALLOC')
a = sys.getallocatedblocks()
self.assertIs(type(a), int)
if with_pymalloc: