bpo-41586: Add pipesize parameter to subprocess & F_GETPIPE_SZ and F_SETPIPE_SZ to fcntl. (GH-21921)
* Add F_SETPIPE_SZ and F_GETPIPE_SZ to fcntl module
* Add pipesize parameter for subprocess.Popen class
This will allow the user to control the size of the pipes.
On linux the default is 64K. When a pipe is full it blocks for writing.
When a pipe is empty it blocks for reading. On processes that are
very fast this can lead to a lot of wasted CPU cycles. On a typical
Linux system the max pipe size is 1024K which is much better.
For high performance-oriented libraries such as xopen it is nice to
be able to set the pipe size.
The workaround without this feature is to use my_popen_process.stdout.fileno() in
conjuction with fcntl and 1031 (value of F_SETPIPE_SZ) to acquire this behavior.
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 86fdf27..6a6c2fc 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -62,6 +62,11 @@
import grp
except ImportError:
grp = None
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
+
__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
"getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
@@ -756,7 +761,7 @@
startupinfo=None, creationflags=0,
restore_signals=True, start_new_session=False,
pass_fds=(), *, user=None, group=None, extra_groups=None,
- encoding=None, errors=None, text=None, umask=-1):
+ encoding=None, errors=None, text=None, umask=-1, pipesize=-1):
"""Create new Popen instance."""
_cleanup()
# Held while anything is calling waitpid before returncode has been
@@ -773,6 +778,11 @@
if not isinstance(bufsize, int):
raise TypeError("bufsize must be an integer")
+ if pipesize is None:
+ pipesize = -1 # Restore default
+ if not isinstance(pipesize, int):
+ raise TypeError("pipesize must be an integer")
+
if _mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
@@ -797,6 +807,7 @@
self.returncode = None
self.encoding = encoding
self.errors = errors
+ self.pipesize = pipesize
# Validate the combinations of text and universal_newlines
if (text is not None and universal_newlines is not None
@@ -1575,6 +1586,8 @@
pass
elif stdin == PIPE:
p2cread, p2cwrite = os.pipe()
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
elif stdin == DEVNULL:
p2cread = self._get_devnull()
elif isinstance(stdin, int):
@@ -1587,6 +1600,8 @@
pass
elif stdout == PIPE:
c2pread, c2pwrite = os.pipe()
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
elif stdout == DEVNULL:
c2pwrite = self._get_devnull()
elif isinstance(stdout, int):
@@ -1599,6 +1614,8 @@
pass
elif stderr == PIPE:
errread, errwrite = os.pipe()
+ if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"):
+ fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize)
elif stderr == STDOUT:
if c2pwrite != -1:
errwrite = c2pwrite