bpo-29585: optimize site.py startup time (GH-136)
Avoid importing `sysconfig` from `site` by copying minimum code.
Python startup is 5% faster on Linux and 30% faster on macOS
diff --git a/Lib/site.py b/Lib/site.py
index 8797938..fcf7dde 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -124,7 +124,7 @@
# if they only differ in case); turn relative paths into absolute
# paths.
dir, dircase = makepath(dir)
- if not dircase in known_paths:
+ if dircase not in known_paths:
L.append(dir)
known_paths.add(dircase)
sys.path[:] = L
@@ -234,6 +234,46 @@
return True
+
+# NOTE: sysconfig and it's dependencies are relatively large but site module
+# needs very limited part of them.
+# To speedup startup time, we have copy of them.
+#
+# See https://bugs.python.org/issue29585
+
+# Copy of sysconfig._getuserbase()
+def _getuserbase():
+ env_base = os.environ.get("PYTHONUSERBASE", None)
+ if env_base:
+ return env_base
+
+ def joinuser(*args):
+ return os.path.expanduser(os.path.join(*args))
+
+ if os.name == "nt":
+ base = os.environ.get("APPDATA") or "~"
+ return joinuser(base, "Python")
+
+ if sys.platform == "darwin" and sys._framework:
+ return joinuser("~", "Library", sys._framework,
+ "%d.%d" % sys.version_info[:2])
+
+ return joinuser("~", ".local")
+
+
+# Same to sysconfig.get_path('purelib', os.name+'_user')
+def _get_path(userbase):
+ version = sys.version_info
+
+ if os.name == 'nt':
+ return f'{userbase}/Python{version[0]}{version[1]}/site-packages'
+
+ if sys.platform == 'darwin' and sys._framework:
+ return f'{userbase}/lib/python/site-packages'
+
+ return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages'
+
+
def getuserbase():
"""Returns the `user base` directory path.
@@ -242,12 +282,11 @@
it.
"""
global USER_BASE
- if USER_BASE is not None:
- return USER_BASE
- from sysconfig import get_config_var
- USER_BASE = get_config_var('userbase')
+ if USER_BASE is None:
+ USER_BASE = _getuserbase()
return USER_BASE
+
def getusersitepackages():
"""Returns the user-specific site-packages directory path.
@@ -255,20 +294,11 @@
function will also set it.
"""
global USER_SITE
- user_base = getuserbase() # this will also set USER_BASE
+ userbase = getuserbase() # this will also set USER_BASE
- if USER_SITE is not None:
- return USER_SITE
+ if USER_SITE is None:
+ USER_SITE = _get_path(userbase)
- from sysconfig import get_path
-
- if sys.platform == 'darwin':
- from sysconfig import get_config_var
- if get_config_var('PYTHONFRAMEWORK'):
- USER_SITE = get_path('purelib', 'osx_framework_user')
- return USER_SITE
-
- USER_SITE = get_path('purelib', '%s_user' % os.name)
return USER_SITE
def addusersitepackages(known_paths):
@@ -310,15 +340,11 @@
else:
sitepackages.append(prefix)
sitepackages.append(os.path.join(prefix, "lib", "site-packages"))
- if sys.platform == "darwin":
- # for framework builds *only* we add the standard Apple
- # locations.
- from sysconfig import get_config_var
- framework = get_config_var("PYTHONFRAMEWORK")
- if framework:
- sitepackages.append(
- os.path.join("/Library", framework,
- '%d.%d' % sys.version_info[:2], "site-packages"))
+ # for framework builds *only* we add the standard Apple locations.
+ if sys.platform == "darwin" and sys._framework:
+ sitepackages.append(
+ os.path.join("/Library", framework,
+ '%d.%d' % sys.version_info[:2], "site-packages"))
return sitepackages
def addsitepackages(known_paths, prefixes=None):
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index ed0a34d..e6618b1 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -51,6 +51,7 @@
'scripts': '{base}/Scripts',
'data': '{base}',
},
+ # NOTE: When modifying "purelib" scheme, update site._get_path() too.
'nt_user': {
'stdlib': '{userbase}/Python{py_version_nodot}',
'platstdlib': '{userbase}/Python{py_version_nodot}',
@@ -177,32 +178,25 @@
return os.name
+# NOTE: site.py has copy of this function.
+# Sync it when modify this function.
def _getuserbase():
env_base = os.environ.get("PYTHONUSERBASE", None)
+ if env_base:
+ return env_base
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
- if env_base:
- return env_base
- else:
- return joinuser(base, "Python")
+ return joinuser(base, "Python")
- if sys.platform == "darwin":
- framework = get_config_var("PYTHONFRAMEWORK")
- if framework:
- if env_base:
- return env_base
- else:
- return joinuser("~", "Library", framework, "%d.%d" %
- sys.version_info[:2])
+ if sys.platform == "darwin" and sys._framework:
+ return joinuser("~", "Library", sys._framework,
+ "%d.%d" % sys.version_info[:2])
- if env_base:
- return env_base
- else:
- return joinuser("~", ".local")
+ return joinuser("~", ".local")
def _parse_makefile(filename, vars=None):
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index 1501622..bf7be4e 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -180,6 +180,13 @@
finally:
pth_file.cleanup()
+ def test_getuserbase(self):
+ self.assertEqual(site._getuserbase(), sysconfig._getuserbase())
+
+ def test_get_path(self):
+ self.assertEqual(site._get_path(site._getuserbase()),
+ sysconfig.get_path('purelib', os.name + '_user'))
+
@unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
"user-site (site.ENABLE_USER_SITE)")
def test_s_option(self):
diff --git a/Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst b/Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst
new file mode 100644
index 0000000..c841f1b
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst
@@ -0,0 +1,2 @@
+Avoid importing ``sysconfig`` from ``site`` to improve startup speed. Python
+startup is about 5% faster on Linux and 30% faster on macOS.
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 424a88f..84673e3 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1965,6 +1965,7 @@
SET_SYS_FROM_STRING("_git",
Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(),
_Py_gitversion()));
+ SET_SYS_FROM_STRING("_framework", PyUnicode_FromString(PYTHONFRAMEWORK));
SET_SYS_FROM_STRING("api_version",
PyLong_FromLong(PYTHON_API_VERSION));
SET_SYS_FROM_STRING("copyright",
diff --git a/configure b/configure
index ec42e08..ff8152d 100755
--- a/configure
+++ b/configure
@@ -783,6 +783,7 @@
docdir
oldincludedir
includedir
+runstatedir
localstatedir
sharedstatedir
sysconfdir
@@ -896,6 +897,7 @@
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1148,6 +1150,15 @@
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
+ -runstatedir | --runstatedir | --runstatedi | --runstated \
+ | --runstate | --runstat | --runsta | --runst | --runs \
+ | --run | --ru | --r)
+ ac_prev=runstatedir ;;
+ -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+ | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+ | --run=* | --ru=* | --r=*)
+ runstatedir=$ac_optarg ;;
+
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1285,7 +1296,7 @@
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
- libdir localedir mandir
+ libdir localedir mandir runstatedir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
@@ -1438,6 +1449,7 @@
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
+ --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
@@ -3231,6 +3243,12 @@
+
+cat >>confdefs.h <<_ACEOF
+#define PYTHONFRAMEWORK "${PYTHONFRAMEWORK}"
+_ACEOF
+
+
##AC_ARG_WITH(dyld,
## AS_HELP_STRING([--with-dyld],
## [Use (OpenStep|Rhapsody) dynamic linker]))
diff --git a/configure.ac b/configure.ac
index 18b940a..815769f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,8 @@
AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX)
AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX)
+AC_DEFINE_UNQUOTED(PYTHONFRAMEWORK, "${PYTHONFRAMEWORK}", [framework name])
+
##AC_ARG_WITH(dyld,
## AS_HELP_STRING([--with-dyld],
## [Use (OpenStep|Rhapsody) dynamic linker]))
diff --git a/pyconfig.h.in b/pyconfig.h.in
index fa2792b..f7c50ea 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -1247,6 +1247,9 @@
/* Define as the preferred size in bits of long digits */
#undef PYLONG_BITS_IN_DIGIT
+/* framework name */
+#undef PYTHONFRAMEWORK
+
/* Define if you want to coerce the C locale to a UTF-8 based locale */
#undef PY_COERCE_C_LOCALE