Issue #5309: distutils' build and build_ext commands now accept a ``-j``
option to enable parallel building of extension modules.
diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py
index cfc15cf..337dd0b 100644
--- a/Lib/distutils/command/build.py
+++ b/Lib/distutils/command/build.py
@@ -36,6 +36,8 @@
"(default: %s)" % get_platform()),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('debug', 'g',
"compile extensions and libraries with debugging information"),
('force', 'f',
@@ -65,6 +67,7 @@
self.debug = None
self.force = 0
self.executable = None
+ self.parallel = None
def finalize_options(self):
if self.plat_name is None:
@@ -116,6 +119,12 @@
if self.executable is None:
self.executable = os.path.normpath(sys.executable)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
# Run all relevant sub-commands. This will be some subset of:
# - build_py - pure Python modules
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
index 3ab2d04..08449e1 100644
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -4,7 +4,10 @@
modules (currently limited to C extensions, should accommodate C++
extensions ASAP)."""
-import sys, os, re
+import contextlib
+import os
+import re
+import sys
from distutils.core import Command
from distutils.errors import *
from distutils.sysconfig import customize_compiler, get_python_version
@@ -85,6 +88,8 @@
"forcibly build everything (ignore file timestamps)"),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('swig-cpp', None,
"make SWIG create C++ files (default is C)"),
('swig-opts=', None,
@@ -124,6 +129,7 @@
self.swig_cpp = None
self.swig_opts = None
self.user = None
+ self.parallel = None
def finalize_options(self):
from distutils import sysconfig
@@ -134,6 +140,7 @@
('compiler', 'compiler'),
('debug', 'debug'),
('force', 'force'),
+ ('parallel', 'parallel'),
('plat_name', 'plat_name'),
)
@@ -274,6 +281,12 @@
self.library_dirs.append(user_lib)
self.rpath.append(user_lib)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
from distutils.ccompiler import new_compiler
@@ -442,15 +455,45 @@
def build_extensions(self):
# First, sanity-check the 'extensions' list
self.check_extensions_list(self.extensions)
+ if self.parallel:
+ self._build_extensions_parallel()
+ else:
+ self._build_extensions_serial()
+ def _build_extensions_parallel(self):
+ workers = self.parallel
+ if self.parallel is True:
+ workers = os.cpu_count() # may return None
+ try:
+ from concurrent.futures import ThreadPoolExecutor
+ except ImportError:
+ workers = None
+
+ if workers is None:
+ self._build_extensions_serial()
+ return
+
+ with ThreadPoolExecutor(max_workers=workers) as executor:
+ futures = [executor.submit(self.build_extension, ext)
+ for ext in self.extensions]
+ for ext, fut in zip(self.extensions, futures):
+ with self._filter_build_errors(ext):
+ fut.result()
+
+ def _build_extensions_serial(self):
for ext in self.extensions:
- try:
+ with self._filter_build_errors(ext):
self.build_extension(ext)
- except (CCompilerError, DistutilsError, CompileError) as e:
- if not ext.optional:
- raise
- self.warn('building extension "%s" failed: %s' %
- (ext.name, e))
+
+ @contextlib.contextmanager
+ def _filter_build_errors(self, ext):
+ try:
+ yield
+ except (CCompilerError, DistutilsError, CompileError) as e:
+ if not ext.optional:
+ raise
+ self.warn('building extension "%s" failed: %s' %
+ (ext.name, e))
def build_extension(self, ext):
sources = ext.sources