bpo-41100: Support macOS 11 and Apple Silicon (GH-22855)
Co-authored-by: Lawrence D’Anna <lawrence_danna@apple.com>
* Add support for macOS 11 and Apple Silicon (aka arm64)
As a side effect of this work use the system copy of libffi on macOS, and remove the vendored copy
* Support building on recent versions of macOS while deploying to older versions
This allows building installers on macOS 11 while still supporting macOS 10.9.
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py
index e9efce7..8a696ee 100644
--- a/Lib/_osx_support.py
+++ b/Lib/_osx_support.py
@@ -110,6 +110,26 @@ def _get_system_version():
return _SYSTEM_VERSION
+_SYSTEM_VERSION_TUPLE = None
+def _get_system_version_tuple():
+ """
+ Return the macOS system version as a tuple
+
+ The return value is safe to use to compare
+ two version numbers.
+ """
+ global _SYSTEM_VERSION_TUPLE
+ if _SYSTEM_VERSION_TUPLE is None:
+ osx_version = _get_system_version()
+ if osx_version:
+ try:
+ _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
+ except ValueError:
+ _SYSTEM_VERSION_TUPLE = ()
+
+ return _SYSTEM_VERSION_TUPLE
+
+
def _remove_original_values(_config_vars):
"""Remove original unmodified values for testing"""
# This is needed for higher-level cross-platform tests of get_platform.
@@ -132,14 +152,18 @@ def _supports_universal_builds():
# builds, in particular -isysroot and -arch arguments to the compiler. This
# is in support of allowing 10.4 universal builds to run on 10.3.x systems.
- osx_version = _get_system_version()
- if osx_version:
- try:
- osx_version = tuple(int(i) for i in osx_version.split('.'))
- except ValueError:
- osx_version = ''
+ osx_version = _get_system_version_tuple()
return bool(osx_version >= (10, 4)) if osx_version else False
+def _supports_arm64_builds():
+ """Returns True if arm64 builds are supported on this system"""
+ # There are two sets of systems supporting macOS/arm64 builds:
+ # 1. macOS 11 and later, unconditionally
+ # 2. macOS 10.15 with Xcode 12.2 or later
+ # For now the second category is ignored.
+ osx_version = _get_system_version_tuple()
+ return osx_version >= (11, 0) if osx_version else False
+
def _find_appropriate_compiler(_config_vars):
"""Find appropriate C compiler for extension module builds"""
@@ -331,6 +355,12 @@ def compiler_fixup(compiler_so, cc_args):
except ValueError:
break
+ elif not _supports_arm64_builds():
+ # Look for "-arch arm64" and drop that
+ for idx in range(len(compiler_so)):
+ if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
+ del compiler_so[idx:idx+2]
+
if 'ARCHFLAGS' in os.environ and not stripArch:
# User specified different -arch flags in the environ,
# see also distutils.sysconfig
@@ -481,6 +511,8 @@ def get_platform_osx(_config_vars, osname, release, machine):
if len(archs) == 1:
machine = archs[0]
+ elif archs == ('arm64', 'x86_64'):
+ machine = 'universal2'
elif archs == ('i386', 'ppc'):
machine = 'fat'
elif archs == ('i386', 'x86_64'):
diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py
index 9d86b05..1c3f8fd 100644
--- a/Lib/ctypes/macholib/dyld.py
+++ b/Lib/ctypes/macholib/dyld.py
@@ -6,6 +6,11 @@
from ctypes.macholib.framework import framework_info
from ctypes.macholib.dylib import dylib_info
from itertools import *
+try:
+ from _ctypes import _dyld_shared_cache_contains_path
+except ImportError:
+ def _dyld_shared_cache_contains_path(*args):
+ raise NotImplementedError
__all__ = [
'dyld_find', 'framework_find',
@@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None):
dyld_executable_path_search(name, executable_path),
dyld_default_search(name, env),
), env):
+
if os.path.isfile(path):
return path
+ try:
+ if _dyld_shared_cache_contains_path(path):
+ return path
+ except NotImplementedError:
+ pass
+
raise ValueError("dylib %s could not be found" % (name,))
def framework_find(fn, executable_path=None, env=None):
diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py
index 6b35269..a1bac26 100644
--- a/Lib/ctypes/test/test_macholib.py
+++ b/Lib/ctypes/test/test_macholib.py
@@ -45,19 +45,22 @@ def find_lib(name):
class MachOTest(unittest.TestCase):
@unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
def test_find(self):
-
- self.assertEqual(find_lib('pthread'),
- '/usr/lib/libSystem.B.dylib')
+ # On Mac OS 11, system dylibs are only present in the shared cache,
+ # so symlinks like libpthread.dylib -> libSystem.B.dylib will not
+ # be resolved by dyld_find
+ self.assertIn(find_lib('pthread'),
+ ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))
result = find_lib('z')
# Issue #21093: dyld default search path includes $HOME/lib and
# /usr/local/lib before /usr/lib, which caused test failures if
# a local copy of libz exists in one of them. Now ignore the head
# of the path.
- self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
+ self.assertRegex(result, r".*/lib/libz.*\.dylib")
- self.assertEqual(find_lib('IOKit'),
- '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
+ self.assertIn(find_lib('IOKit'),
+ ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
+ '/System/Library/Frameworks/IOKit.framework/IOKit'))
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py
index f9e0d76..6bb009a 100644
--- a/Lib/distutils/tests/test_build_ext.py
+++ b/Lib/distutils/tests/test_build_ext.py
@@ -493,7 +493,7 @@ def _try_compile_deployment_target(self, operator, target):
# format the target value as defined in the Apple
# Availability Macros. We can't use the macro names since
# at least one value we test with will not exist yet.
- if target[1] < 10:
+ if target[:2] < (10, 10):
# for 10.1 through 10.9.x -> "10n0"
target = '%02d%01d0' % target
else:
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index e61228d..d550abf 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -1036,6 +1036,7 @@ def test_from_format(self):
c_char_p)
PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
+ PyBytes_FromFormat.argtypes = (c_char_p,)
PyBytes_FromFormat.restype = py_object
# basic tests
diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py
index b5d21e5..9b6d93c 100644
--- a/Lib/test/test_platform.py
+++ b/Lib/test/test_platform.py
@@ -246,7 +246,7 @@ def test_mac_ver(self):
self.assertEqual(res[1], ('', '', ''))
if sys.byteorder == 'little':
- self.assertIn(res[2], ('i386', 'x86_64'))
+ self.assertIn(res[2], ('i386', 'x86_64', 'arm64'))
else:
self.assertEqual(res[2], 'PowerPC')
diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
index f57c882..a522717 100644
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -1925,6 +1925,233 @@ def test_posix_spawnp(self):
assert_python_ok(*args, PATH=path)
+@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS")
+class TestPosixWeaklinking(unittest.TestCase):
+ # These test cases verify that weak linking support on macOS works
+ # as expected. These cases only test new behaviour introduced by weak linking,
+ # regular behaviour is tested by the normal test cases.
+ #
+ # See the section on Weak Linking in Mac/README.txt for more information.
+ def setUp(self):
+ import sysconfig
+ import platform
+
+ config_vars = sysconfig.get_config_vars()
+ self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] }
+ self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split("."))
+
+ def _verify_available(self, name):
+ if name not in self.available:
+ raise unittest.SkipTest(f"{name} not weak-linked")
+
+ def test_pwritev(self):
+ self._verify_available("HAVE_PWRITEV")
+ if self.mac_ver >= (10, 16):
+ self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available")
+ self.assertTrue(hasattr(os, "preadv"), "os.readv is not available")
+
+ else:
+ self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available")
+ self.assertFalse(hasattr(os, "preadv"), "os.readv is available")
+
+ def test_stat(self):
+ self._verify_available("HAVE_FSTATAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_FSTATAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_FSTATAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.stat("file", dir_fd=0)
+
+ def test_access(self):
+ self._verify_available("HAVE_FACCESSAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_FACCESSAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_FACCESSAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.access("file", os.R_OK, dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"):
+ os.access("file", os.R_OK, follow_symlinks=False)
+
+ with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"):
+ os.access("file", os.R_OK, effective_ids=True)
+
+ def test_chmod(self):
+ self._verify_available("HAVE_FCHMODAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_FCHMODAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_FCHMODAT", posix._have_functions)
+ self.assertIn("HAVE_LCHMOD", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.chmod("file", 0o644, dir_fd=0)
+
+ def test_chown(self):
+ self._verify_available("HAVE_FCHOWNAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_FCHOWNAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions)
+ self.assertIn("HAVE_LCHOWN", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.chown("file", 0, 0, dir_fd=0)
+
+ def test_link(self):
+ self._verify_available("HAVE_LINKAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_LINKAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_LINKAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"):
+ os.link("source", "target", src_dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"):
+ os.link("source", "target", dst_dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"):
+ os.link("source", "target", src_dir_fd=0, dst_dir_fd=0)
+
+ # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag
+ with os_helper.temp_dir() as base_path:
+ link_path = os.path.join(base_path, "link")
+ target_path = os.path.join(base_path, "target")
+ source_path = os.path.join(base_path, "source")
+
+ with open(source_path, "w") as fp:
+ fp.write("data")
+
+ os.symlink("target", link_path)
+
+ # Calling os.link should fail in the link(2) call, and
+ # should not reject *follow_symlinks* (to match the
+ # behaviour you'd get when building on a platform without
+ # linkat)
+ with self.assertRaises(FileExistsError):
+ os.link(source_path, link_path, follow_symlinks=True)
+
+ with self.assertRaises(FileExistsError):
+ os.link(source_path, link_path, follow_symlinks=False)
+
+
+ def test_listdir_scandir(self):
+ self._verify_available("HAVE_FDOPENDIR")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_FDOPENDIR", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions)
+
+ with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"):
+ os.listdir(0)
+
+ with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"):
+ os.scandir(0)
+
+ def test_mkdir(self):
+ self._verify_available("HAVE_MKDIRAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_MKDIRAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_MKDIRAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.mkdir("dir", dir_fd=0)
+
+ def test_rename_replace(self):
+ self._verify_available("HAVE_RENAMEAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_RENAMEAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_RENAMEAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+ os.rename("a", "b", src_dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+ os.rename("a", "b", dst_dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+ os.replace("a", "b", src_dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+ os.replace("a", "b", dst_dir_fd=0)
+
+ def test_unlink_rmdir(self):
+ self._verify_available("HAVE_UNLINKAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_UNLINKAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_UNLINKAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.unlink("path", dir_fd=0)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.rmdir("path", dir_fd=0)
+
+ def test_open(self):
+ self._verify_available("HAVE_OPENAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_OPENAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_OPENAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.open("path", os.O_RDONLY, dir_fd=0)
+
+ def test_readlink(self):
+ self._verify_available("HAVE_READLINKAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_READLINKAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_READLINKAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.readlink("path", dir_fd=0)
+
+ def test_symlink(self):
+ self._verify_available("HAVE_SYMLINKAT")
+ if self.mac_ver >= (10, 10):
+ self.assertIn("HAVE_SYMLINKAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.symlink("a", "b", dir_fd=0)
+
+ def test_utime(self):
+ self._verify_available("HAVE_FUTIMENS")
+ self._verify_available("HAVE_UTIMENSAT")
+ if self.mac_ver >= (10, 13):
+ self.assertIn("HAVE_FUTIMENS", posix._have_functions)
+ self.assertIn("HAVE_UTIMENSAT", posix._have_functions)
+
+ else:
+ self.assertNotIn("HAVE_FUTIMENS", posix._have_functions)
+ self.assertNotIn("HAVE_UTIMENSAT", posix._have_functions)
+
+ with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+ os.utime("path", dir_fd=0)
+
+
def test_main():
try:
support.run_unittest(
@@ -1932,6 +2159,7 @@ def test_main():
PosixGroupsTester,
TestPosixSpawn,
TestPosixSpawnP,
+ TestPosixWeaklinking
)
finally:
support.reap_children()
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
index 6ced047..3258298 100644
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -1041,6 +1041,36 @@ def test_object_to_timespec(self):
with self.assertRaises(ValueError):
pytime_object_to_timespec(float('nan'), time_rnd)
+@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS")
+class TestTimeWeaklinking(unittest.TestCase):
+ # These test cases verify that weak linking support on macOS works
+ # as expected. These cases only test new behaviour introduced by weak linking,
+ # regular behaviour is tested by the normal test cases.
+ #
+ # See the section on Weak Linking in Mac/README.txt for more information.
+ def test_clock_functions(self):
+ import sysconfig
+ import platform
+
+ config_vars = sysconfig.get_config_vars()
+ var_name = "HAVE_CLOCK_GETTIME"
+ if var_name not in config_vars or not config_vars[var_name]:
+ raise unittest.SkipTest(f"{var_name} is not available")
+
+ mac_ver = tuple(int(x) for x in platform.mac_ver()[0].split("."))
+
+ clock_names = [
+ "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime",
+ "clock_settime_ns", "clock_getres"]
+
+ if mac_ver >= (10, 12):
+ for name in clock_names:
+ self.assertTrue(hasattr(time, name), f"time.{name} is not available")
+
+ else:
+ for name in clock_names:
+ self.assertFalse(hasattr(time, name), f"time.{name} is available")
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index 90b0965..4f5636e 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -2516,11 +2516,13 @@ class CAPITest(unittest.TestCase):
def test_from_format(self):
import_helper.import_module('ctypes')
from ctypes import (
+ c_char_p,
pythonapi, py_object, sizeof,
c_int, c_long, c_longlong, c_ssize_t,
c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
name = "PyUnicode_FromFormat"
_PyUnicode_FromFormat = getattr(pythonapi, name)
+ _PyUnicode_FromFormat.argtypes = (c_char_p,)
_PyUnicode_FromFormat.restype = py_object
def PyUnicode_FromFormat(format, *args):