bpo-41586: Add pipesize parameter to subprocess & F_GETPIPE_SZ and F_SETPIPE_SZ to fcntl. (GH-21921)
* Add F_SETPIPE_SZ and F_GETPIPE_SZ to fcntl module
* Add pipesize parameter for subprocess.Popen class
This will allow the user to control the size of the pipes.
On linux the default is 64K. When a pipe is full it blocks for writing.
When a pipe is empty it blocks for reading. On processes that are
very fast this can lead to a lot of wasted CPU cycles. On a typical
Linux system the max pipe size is 1024K which is much better.
For high performance-oriented libraries such as xopen it is nice to
be able to set the pipe size.
The workaround without this feature is to use my_popen_process.stdout.fileno() in
conjuction with fcntl and 1031 (value of F_SETPIPE_SZ) to acquire this behavior.
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 434ba56..8b576c0 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -39,6 +39,11 @@
except ImportError:
grp = None
+try:
+ import fcntl
+except:
+ fcntl = None
+
if support.PGO:
raise unittest.SkipTest("test is not helpful for PGO")
@@ -661,6 +666,46 @@ def test_stdin_devnull(self):
p.wait()
self.assertEqual(p.stdin, None)
+ def test_pipesizes(self):
+ # stdin redirection
+ pipesize = 16 * 1024
+ p = subprocess.Popen([sys.executable, "-c",
+ 'import sys; sys.stdin.read(); sys.stdout.write("out"); sys.stderr.write("error!")'],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ pipesize=pipesize)
+ # We only assert pipe size has changed on platforms that support it.
+ if sys.platform != "win32" and hasattr(fcntl, "F_GETPIPE_SZ"):
+ for fifo in [p.stdin, p.stdout, p.stderr]:
+ self.assertEqual(fcntl.fcntl(fifo.fileno(), fcntl.F_GETPIPE_SZ), pipesize)
+ # Windows pipe size can be acquired with the GetNamedPipeInfoFunction
+ # https://docs.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-getnamedpipeinfo
+ # However, this function is not yet in _winapi.
+ p.stdin.write(b"pear")
+ p.stdin.close()
+ p.wait()
+
+ def test_pipesize_default(self):
+ p = subprocess.Popen([sys.executable, "-c",
+ 'import sys; sys.stdin.read(); sys.stdout.write("out");'
+ ' sys.stderr.write("error!")'],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ pipesize=-1)
+ # UNIX tests using fcntl
+ if sys.platform != "win32" and hasattr(fcntl, "F_GETPIPE_SZ"):
+ fp_r, fp_w = os.pipe()
+ default_pipesize = fcntl.fcntl(fp_w, fcntl.F_GETPIPE_SZ)
+ for fifo in [p.stdin, p.stdout, p.stderr]:
+ self.assertEqual(
+ fcntl.fcntl(fifo.fileno(), fcntl.F_GETPIPE_SZ), default_pipesize)
+ # On other platforms we cannot test the pipe size (yet). But above code
+ # using pipesize=-1 should not crash.
+ p.stdin.close()
+ p.wait()
+
def test_env(self):
newenv = os.environ.copy()
newenv["FRUIT"] = "orange"
@@ -3503,7 +3548,7 @@ def test_getoutput(self):
def test__all__(self):
"""Ensure that __all__ is populated properly."""
- intentionally_excluded = {"list2cmdline", "Handle", "pwd", "grp"}
+ intentionally_excluded = {"list2cmdline", "Handle", "pwd", "grp", "fcntl"}
exported = set(subprocess.__all__)
possible_exports = set()
import types