bpo-26903: Limit ProcessPoolExecutor to 61 workers on Windows (GH-13132)
Co-Authored-By: brianquinlan <brian@sweetapp.com>
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index d7e2478..dd14eae 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -57,6 +57,7 @@
import weakref
from functools import partial
import itertools
+import sys
import traceback
# Workers are created as daemon threads and processes. This is done to allow the
@@ -109,6 +110,12 @@
EXTRA_QUEUED_CALLS = 1
+# On Windows, WaitForMultipleObjects is used to wait for processes to finish.
+# It can wait on, at most, 63 objects. There is an overhead of two objects:
+# - the result queue reader
+# - the thread wakeup reader
+_MAX_WINDOWS_WORKERS = 63 - 2
+
# Hack to embed stringification of remote traceback in local traceback
class _RemoteTraceback(Exception):
@@ -505,9 +512,16 @@
if max_workers is None:
self._max_workers = os.cpu_count() or 1
+ if sys.platform == 'win32':
+ self._max_workers = min(_MAX_WINDOWS_WORKERS,
+ self._max_workers)
else:
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
+ elif (sys.platform == 'win32' and
+ max_workers > _MAX_WINDOWS_WORKERS):
+ raise ValueError(
+ f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
self._max_workers = max_workers