blob: 3dea089e6a751a85bb4ace04b39daee8ae9d0bb5 [file] [log] [blame]
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001# subprocess - Subprocesses with accessible I/O streams
2#
Tim Peterse718f612004-10-12 21:51:32 +00003# For more information about this module, see PEP 324.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00004#
Peter Astrand3a708df2005-09-23 17:37:29 +00005# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00006#
Peter Astrand69bf13f2005-02-14 08:56:32 +00007# Licensed to PSF under a Contributor Agreement.
Peter Astrand3a708df2005-09-23 17:37:29 +00008# See http://www.python.org/2.4/license for licensing details.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00009
Raymond Hettinger837dd932004-10-17 16:36:53 +000010r"""subprocess - Subprocesses with accessible I/O streams
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000011
Fredrik Lundh15aaacc2004-10-17 14:47:05 +000012This module allows you to spawn processes, connect to their
13input/output/error pipes, and obtain their return codes. This module
Benjamin Peterson5eea8a72014-03-12 21:41:35 -050014intends to replace several older modules and functions:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000015
16os.system
17os.spawn*
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000018
19Information about how the subprocess module can be used to replace these
20modules and functions can be found below.
21
22
23
24Using the subprocess module
25===========================
26This module defines one class called Popen:
27
Gregory P. Smitha1ed5392013-03-23 11:44:25 -070028class Popen(args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000029 stdin=None, stdout=None, stderr=None,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +000030 preexec_fn=None, close_fds=True, shell=False,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000031 cwd=None, env=None, universal_newlines=False,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000032 startupinfo=None, creationflags=0,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +000033 restore_signals=True, start_new_session=False, pass_fds=()):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000034
35
36Arguments are:
37
38args should be a string, or a sequence of program arguments. The
39program to execute is normally the first item in the args sequence or
40string, but can be explicitly set by using the executable argument.
41
Gregory P. Smithf5604852010-12-13 06:45:02 +000042On POSIX, with shell=False (default): In this case, the Popen class
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000043uses os.execvp() to execute the child program. args should normally
44be a sequence. A string will be treated as a sequence with the string
45as the only item (the program to execute).
46
Gregory P. Smithf5604852010-12-13 06:45:02 +000047On POSIX, with shell=True: If args is a string, it specifies the
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000048command string to execute through the shell. If args is a sequence,
49the first item specifies the command string, and any additional items
50will be treated as additional shell arguments.
51
52On Windows: the Popen class uses CreateProcess() to execute the child
53program, which operates on strings. If args is a sequence, it will be
54converted to a string using the list2cmdline method. Please note that
55not all MS Windows applications interpret the command line the same
56way: The list2cmdline is designed for applications using the same
57rules as the MS C runtime.
58
Gregory P. Smitha1ed5392013-03-23 11:44:25 -070059bufsize will be supplied as the corresponding argument to the io.open()
60function when creating the stdin/stdout/stderr pipe file objects:
610 means unbuffered (read & write are one system call and can return short),
621 means line buffered, any other positive value means use a buffer of
63approximately that size. A negative bufsize, the default, means the system
64default of io.DEFAULT_BUFFER_SIZE will be used.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000065
66stdin, stdout and stderr specify the executed programs' standard
67input, standard output and standard error file handles, respectively.
68Valid values are PIPE, an existing file descriptor (a positive
69integer), an existing file object, and None. PIPE indicates that a
70new pipe to the child should be created. With None, no redirection
71will occur; the child's file handles will be inherited from the
72parent. Additionally, stderr can be STDOUT, which indicates that the
73stderr data from the applications should be captured into the same
74file handle as for stdout.
75
Gregory P. Smithf5604852010-12-13 06:45:02 +000076On POSIX, if preexec_fn is set to a callable object, this object will be
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000077called in the child process just before the child is executed. The use
78of preexec_fn is not thread safe, using it in the presence of threads
79could lead to a deadlock in the child process before the new executable
80is executed.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000081
82If close_fds is true, all file descriptors except 0, 1 and 2 will be
Gregory P. Smithf5604852010-12-13 06:45:02 +000083closed before the child process is executed. The default for close_fds
Gregory P. Smith8edd99d2010-12-14 13:43:30 +000084varies by platform: Always true on POSIX. True when stdin/stdout/stderr
85are None on Windows, false otherwise.
86
87pass_fds is an optional sequence of file descriptors to keep open between the
88parent and child. Providing any pass_fds implicitly sets close_fds to true.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000089
90if shell is true, the specified command will be executed through the
91shell.
92
93If cwd is not None, the current directory will be changed to cwd
94before the child is executed.
95
Gregory P. Smithf5604852010-12-13 06:45:02 +000096On POSIX, if restore_signals is True all signals that Python sets to
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000097SIG_IGN are restored to SIG_DFL in the child process before the exec.
98Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This
99parameter does nothing on Windows.
100
Gregory P. Smithf5604852010-12-13 06:45:02 +0000101On POSIX, if start_new_session is True, the setsid() system call will be made
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000102in the child process prior to executing the command.
103
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000104If env is not None, it defines the environment variables for the new
105process.
106
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400107If universal_newlines is False, the file objects stdin, stdout and stderr
Ronald Oussoren385521c2013-07-07 09:26:45 +0200108are opened as binary files, and no line ending conversion is done.
109
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400110If universal_newlines is True, the file objects stdout and stderr are
111opened as a text file, but lines may be terminated by any of '\n',
Gregory P. Smithf5604852010-12-13 06:45:02 +0000112the Unix end-of-line convention, '\r', the old Macintosh convention or
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000113'\r\n', the Windows convention. All of these external representations
Gregory P. Smith1f8a40b2013-03-20 18:32:03 -0700114are seen as '\n' by the Python program. Also, the newlines attribute
115of the file objects stdout, stdin and stderr are not updated by the
116communicate() method.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000117
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400118In either case, the process being communicated with should start up
119expecting to receive bytes on its standard input and decode them with
120the same encoding they are sent in.
121
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000122The startupinfo and creationflags, if given, will be passed to the
123underlying CreateProcess() function. They can specify things such as
124appearance of the main window and priority for the new process.
125(Windows only)
126
127
Georg Brandlf9734072008-12-07 15:30:06 +0000128This module also defines some shortcut functions:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000129
Peter Astrand5f5e1412004-12-05 20:15:36 +0000130call(*popenargs, **kwargs):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000131 Run command with arguments. Wait for command to complete, then
132 return the returncode attribute.
133
134 The arguments are the same as for the Popen constructor. Example:
135
Florent Xicluna4886d242010-03-08 13:27:26 +0000136 >>> retcode = subprocess.call(["ls", "-l"])
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000137
Peter Astrand454f7672005-01-01 09:36:35 +0000138check_call(*popenargs, **kwargs):
139 Run command with arguments. Wait for command to complete. If the
140 exit code was zero then return, otherwise raise
141 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000142 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000143
144 The arguments are the same as for the Popen constructor. Example:
145
Florent Xicluna4886d242010-03-08 13:27:26 +0000146 >>> subprocess.check_call(["ls", "-l"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000147 0
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000148
Brett Cannona23810f2008-05-26 19:04:21 +0000149getstatusoutput(cmd):
150 Return (status, output) of executing cmd in a shell.
151
Tim Golden60798142013-11-05 12:57:25 +0000152 Execute the string 'cmd' in a shell with 'check_output' and
153 return a 2-tuple (status, output). Universal newlines mode is used,
154 meaning that the result with be decoded to a string.
155
156 A trailing newline is stripped from the output.
157 The exit status for the command can be interpreted
158 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000159
Brett Cannona23810f2008-05-26 19:04:21 +0000160 >>> subprocess.getstatusoutput('ls /bin/ls')
161 (0, '/bin/ls')
162 >>> subprocess.getstatusoutput('cat /bin/junk')
163 (256, 'cat: /bin/junk: No such file or directory')
164 >>> subprocess.getstatusoutput('/bin/junk')
165 (256, 'sh: /bin/junk: not found')
166
167getoutput(cmd):
168 Return output (stdout or stderr) of executing cmd in a shell.
169
170 Like getstatusoutput(), except the exit status is ignored and the return
171 value is a string containing the command's output. Example:
172
Brett Cannona23810f2008-05-26 19:04:21 +0000173 >>> subprocess.getoutput('ls /bin/ls')
174 '/bin/ls'
175
Georg Brandlf9734072008-12-07 15:30:06 +0000176check_output(*popenargs, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700177 Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000178
Georg Brandl2708f3a2009-12-20 14:38:23 +0000179 If the exit code was non-zero it raises a CalledProcessError. The
180 CalledProcessError object will have the return code in the returncode
181 attribute and output in the output attribute.
Georg Brandlf9734072008-12-07 15:30:06 +0000182
Georg Brandl2708f3a2009-12-20 14:38:23 +0000183 The arguments are the same as for the Popen constructor. Example:
Georg Brandlf9734072008-12-07 15:30:06 +0000184
Georg Brandl2708f3a2009-12-20 14:38:23 +0000185 >>> output = subprocess.check_output(["ls", "-l", "/dev/null"])
Georg Brandlf9734072008-12-07 15:30:06 +0000186
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300187 There is an additional optional argument, "input", allowing you to
188 pass a string to the subprocess's stdin. If you use this argument
189 you may not also use the Popen constructor's "stdin" argument.
Brett Cannona23810f2008-05-26 19:04:21 +0000190
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400191 If universal_newlines is set to True, the "input" argument must
192 be a string rather than bytes, and the return value will be a string.
193
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000194Exceptions
195----------
196Exceptions raised in the child process, before the new program has
197started to execute, will be re-raised in the parent. Additionally,
198the exception object will have one extra attribute called
199'child_traceback', which is a string containing traceback information
Ezio Melotti30b9d5d2013-08-17 15:50:46 +0300200from the child's point of view.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000201
202The most common exception raised is OSError. This occurs, for
203example, when trying to execute a non-existent file. Applications
204should prepare for OSErrors.
205
206A ValueError will be raised if Popen is called with invalid arguments.
207
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400208Exceptions defined within this module inherit from SubprocessError.
209check_call() and check_output() will raise CalledProcessError if the
Gregory P. Smithb4039aa2011-03-14 14:16:20 -0400210called process returns a non-zero return code. TimeoutExpired
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400211be raised if a timeout was specified and expired.
Peter Astrand454f7672005-01-01 09:36:35 +0000212
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000213
214Security
215--------
216Unlike some other popen functions, this implementation will never call
217/bin/sh implicitly. This means that all characters, including shell
218metacharacters, can safely be passed to child processes.
219
220
221Popen objects
222=============
223Instances of the Popen class have the following methods:
224
225poll()
226 Check if child process has terminated. Returns returncode
227 attribute.
228
229wait()
230 Wait for child process to terminate. Returns returncode attribute.
231
232communicate(input=None)
233 Interact with process: Send data to stdin. Read data from stdout
234 and stderr, until end-of-file is reached. Wait for process to
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400235 terminate. The optional input argument should be data to be
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000236 sent to the child process, or None, if no data should be sent to
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400237 the child. If the Popen instance was constructed with universal_newlines
238 set to True, the input argument should be a string and will be encoded
239 using the preferred system encoding (see locale.getpreferredencoding);
240 if universal_newlines is False, the input argument should be a
241 byte string.
Tim Peterse718f612004-10-12 21:51:32 +0000242
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000243 communicate() returns a tuple (stdout, stderr).
244
245 Note: The data read is buffered in memory, so do not use this
246 method if the data size is large or unlimited.
247
248The following attributes are also available:
249
250stdin
251 If the stdin argument is PIPE, this attribute is a file object
252 that provides input to the child process. Otherwise, it is None.
253
254stdout
255 If the stdout argument is PIPE, this attribute is a file object
256 that provides output from the child process. Otherwise, it is
257 None.
258
259stderr
260 If the stderr argument is PIPE, this attribute is file object that
261 provides error output from the child process. Otherwise, it is
262 None.
263
264pid
265 The process ID of the child process.
266
267returncode
268 The child return code. A None value indicates that the process
269 hasn't terminated yet. A negative value -N indicates that the
Gregory P. Smithf5604852010-12-13 06:45:02 +0000270 child was terminated by signal N (POSIX only).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000271
272
273Replacing older functions with the subprocess module
274====================================================
275In this section, "a ==> b" means that b can be used as a replacement
276for a.
277
278Note: All functions in this section fail (more or less) silently if
279the executed program cannot be found; this module raises an OSError
280exception.
281
282In the following examples, we assume that the subprocess module is
283imported with "from subprocess import *".
284
285
286Replacing /bin/sh shell backquote
287---------------------------------
288output=`mycmd myarg`
289==>
290output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
291
292
293Replacing shell pipe line
294-------------------------
295output=`dmesg | grep hda`
296==>
297p1 = Popen(["dmesg"], stdout=PIPE)
Peter Astrand6fdf3cb2004-11-30 18:06:42 +0000298p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000299output = p2.communicate()[0]
300
301
302Replacing os.system()
303---------------------
304sts = os.system("mycmd" + " myarg")
305==>
306p = Popen("mycmd" + " myarg", shell=True)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000307pid, sts = os.waitpid(p.pid, 0)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000308
309Note:
310
311* Calling the program through the shell is usually not required.
312
313* It's easier to look at the returncode attribute than the
314 exitstatus.
315
316A more real-world example would look like this:
317
318try:
319 retcode = call("mycmd" + " myarg", shell=True)
320 if retcode < 0:
Guido van Rossumc2f93dc2007-05-24 00:50:02 +0000321 print("Child was terminated by signal", -retcode, file=sys.stderr)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000322 else:
Guido van Rossumc2f93dc2007-05-24 00:50:02 +0000323 print("Child returned", retcode, file=sys.stderr)
324except OSError as e:
325 print("Execution failed:", e, file=sys.stderr)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000326
327
328Replacing os.spawn*
329-------------------
Tim Peterse718f612004-10-12 21:51:32 +0000330P_NOWAIT example:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000331
332pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
333==>
334pid = Popen(["/bin/mycmd", "myarg"]).pid
335
336
337P_WAIT example:
338
339retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
340==>
341retcode = call(["/bin/mycmd", "myarg"])
342
343
Tim Peterse718f612004-10-12 21:51:32 +0000344Vector example:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000345
346os.spawnvp(os.P_NOWAIT, path, args)
347==>
348Popen([path] + args[1:])
349
350
Tim Peterse718f612004-10-12 21:51:32 +0000351Environment example:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000352
353os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
354==>
355Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000356"""
357
358import sys
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700359_mswindows = (sys.platform == "win32")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000360
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000361import io
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000362import os
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400363import time
Christian Heimesa342c012008-04-20 21:01:16 +0000364import signal
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000365import builtins
Gregory P. Smithd23047b2010-12-04 09:10:44 +0000366import warnings
Ross Lagerwall4f61b022011-04-05 15:34:00 +0200367import errno
Victor Stinnerae586492014-09-02 23:18:25 +0200368from time import monotonic as _time
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000369
Peter Astrand454f7672005-01-01 09:36:35 +0000370# Exception classes used by this module.
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400371class SubprocessError(Exception): pass
372
373
374class CalledProcessError(SubprocessError):
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +0000375 """Raised when a check_call() or check_output() process returns non-zero.
376
377 The exit status will be stored in the returncode attribute, negative
378 if it represents a signal number.
379
Georg Brandlf9734072008-12-07 15:30:06 +0000380 check_output() will also store the output in the output attribute.
381 """
Gregory P. Smith6e730002015-04-14 16:14:25 -0700382 def __init__(self, returncode, cmd, output=None, stderr=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000383 self.returncode = returncode
384 self.cmd = cmd
Georg Brandlf9734072008-12-07 15:30:06 +0000385 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -0700386 self.stderr = stderr
387
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000388 def __str__(self):
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +0000389 if self.returncode and self.returncode < 0:
390 try:
391 return "Command '%s' died with %r." % (
392 self.cmd, signal.Signals(-self.returncode))
393 except ValueError:
394 return "Command '%s' died with unknown signal %d." % (
395 self.cmd, -self.returncode)
396 else:
397 return "Command '%s' returned non-zero exit status %d." % (
398 self.cmd, self.returncode)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000399
Gregory P. Smith6e730002015-04-14 16:14:25 -0700400 @property
401 def stdout(self):
402 """Alias for output attribute, to match stderr"""
403 return self.output
404
405 @stdout.setter
406 def stdout(self, value):
407 # There's no obvious reason to set this, but allow it anyway so
408 # .stdout is a transparent alias for .output
409 self.output = value
410
Peter Astrand454f7672005-01-01 09:36:35 +0000411
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400412class TimeoutExpired(SubprocessError):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400413 """This exception is raised when the timeout expires while waiting for a
414 child process.
415 """
Gregory P. Smith6e730002015-04-14 16:14:25 -0700416 def __init__(self, cmd, timeout, output=None, stderr=None):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400417 self.cmd = cmd
Reid Kleckner2b228f02011-03-16 16:57:54 -0400418 self.timeout = timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400419 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -0700420 self.stderr = stderr
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400421
422 def __str__(self):
423 return ("Command '%s' timed out after %s seconds" %
424 (self.cmd, self.timeout))
425
Gregory P. Smith6e730002015-04-14 16:14:25 -0700426 @property
427 def stdout(self):
428 return self.output
429
430 @stdout.setter
431 def stdout(self, value):
432 # There's no obvious reason to set this, but allow it anyway so
433 # .stdout is a transparent alias for .output
434 self.output = value
435
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400436
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700437if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000438 import threading
439 import msvcrt
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200440 import _winapi
Brian Curtin1ce6b582010-04-24 16:19:22 +0000441 class STARTUPINFO:
442 dwFlags = 0
443 hStdInput = None
444 hStdOutput = None
445 hStdError = None
446 wShowWindow = 0
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000447else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -0700448 import _posixsubprocess
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100449 import select
450 import selectors
Gregory P. Smithd65ba512014-04-23 00:27:17 -0700451 try:
452 import threading
453 except ImportError:
454 import dummy_threading as threading
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000455
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000456 # When select or poll has indicated that the file is writable,
457 # we can write up to _PIPE_BUF bytes without risk of blocking.
458 # POSIX defines PIPE_BUF as >= 512.
459 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
460
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100461 # poll/select have the advantage of not requiring any extra file
462 # descriptor, contrarily to epoll/kqueue (also, they require a single
463 # syscall).
464 if hasattr(selectors, 'PollSelector'):
465 _PopenSelector = selectors.PollSelector
466 else:
467 _PopenSelector = selectors.SelectSelector
468
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000469
Brett Cannona23810f2008-05-26 19:04:21 +0000470__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
Gregory P. Smith6e730002015-04-14 16:14:25 -0700471 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
472 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
Gregory P. Smithace55862015-04-07 15:57:54 -0700473 # NOTE: We intentionally exclude list2cmdline as it is
474 # considered an internal implementation detail. issue10838.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000475
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700476if _mswindows:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200477 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
478 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
479 STD_ERROR_HANDLE, SW_HIDE,
480 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
Brian Curtin5d9deaa2011-04-29 16:24:07 -0500481
Brian Curtin08fd8d92011-04-29 16:11:30 -0500482 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
Brian Curtin8b8e7f42011-04-29 15:48:13 -0500483 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
484 "STD_ERROR_HANDLE", "SW_HIDE",
Martin Panter528619b2016-04-16 23:42:37 +0000485 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
486 "STARTUPINFO"])
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200487
488 class Handle(int):
489 closed = False
490
491 def Close(self, CloseHandle=_winapi.CloseHandle):
492 if not self.closed:
493 self.closed = True
494 CloseHandle(self)
495
496 def Detach(self):
497 if not self.closed:
498 self.closed = True
499 return int(self)
500 raise ValueError("already closed")
501
502 def __repr__(self):
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300503 return "%s(%d)" % (self.__class__.__name__, int(self))
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200504
505 __del__ = Close
506 __str__ = __repr__
507
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000508
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200509# This lists holds Popen instances for which the underlying process had not
510# exited at the time its __del__ method got called: those processes are wait()ed
511# for synchronously from _cleanup() when a new Popen object is created, to avoid
512# zombie processes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000513_active = []
514
515def _cleanup():
516 for inst in _active[:]:
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000517 res = inst._internal_poll(_deadstate=sys.maxsize)
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200518 if res is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000519 try:
520 _active.remove(inst)
521 except ValueError:
522 # This can happen if two threads create a new Popen instance.
523 # It's harmless that it was already removed, so ignore.
524 pass
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000525
526PIPE = -1
527STDOUT = -2
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200528DEVNULL = -3
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000529
530
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200531# XXX This function is only used by multiprocessing and the test suite,
532# but it's here so that it can be imported when Python is compiled without
533# threads.
534
Victor Stinner9def2842016-01-18 12:15:08 +0100535def _optim_args_from_interpreter_flags():
536 """Return a list of command-line arguments reproducing the current
537 optimization settings in sys.flags."""
538 args = []
539 value = sys.flags.optimize
540 if value > 0:
541 args.append('-' + 'O' * value)
542 return args
543
544
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200545def _args_from_interpreter_flags():
546 """Return a list of command-line arguments reproducing the current
547 settings in sys.flags and sys.warnoptions."""
548 flag_opt_map = {
549 'debug': 'd',
550 # 'inspect': 'i',
551 # 'interactive': 'i',
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200552 'dont_write_bytecode': 'B',
553 'no_user_site': 's',
554 'no_site': 'S',
555 'ignore_environment': 'E',
556 'verbose': 'v',
557 'bytes_warning': 'b',
558 'quiet': 'q',
Victor Stinner9def2842016-01-18 12:15:08 +0100559 # -O is handled in _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200560 }
Victor Stinner9def2842016-01-18 12:15:08 +0100561 args = _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200562 for flag, opt in flag_opt_map.items():
563 v = getattr(sys.flags, flag)
564 if v > 0:
565 args.append('-' + opt * v)
566 for opt in sys.warnoptions:
567 args.append('-W' + opt)
568 return args
569
570
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400571def call(*popenargs, timeout=None, **kwargs):
572 """Run command with arguments. Wait for command to complete or
573 timeout, then return the returncode attribute.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000574
575 The arguments are the same as for the Popen constructor. Example:
576
577 retcode = call(["ls", "-l"])
578 """
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200579 with Popen(*popenargs, **kwargs) as p:
580 try:
581 return p.wait(timeout=timeout)
582 except:
583 p.kill()
584 p.wait()
585 raise
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000586
587
Peter Astrand454f7672005-01-01 09:36:35 +0000588def check_call(*popenargs, **kwargs):
589 """Run command with arguments. Wait for command to complete. If
590 the exit code was zero then return, otherwise raise
591 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000592 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000593
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400594 The arguments are the same as for the call function. Example:
Peter Astrand454f7672005-01-01 09:36:35 +0000595
596 check_call(["ls", "-l"])
597 """
598 retcode = call(*popenargs, **kwargs)
Peter Astrand454f7672005-01-01 09:36:35 +0000599 if retcode:
Georg Brandlf9734072008-12-07 15:30:06 +0000600 cmd = kwargs.get("args")
601 if cmd is None:
602 cmd = popenargs[0]
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000603 raise CalledProcessError(retcode, cmd)
Georg Brandlf9734072008-12-07 15:30:06 +0000604 return 0
605
606
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400607def check_output(*popenargs, timeout=None, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700608 r"""Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000609
610 If the exit code was non-zero it raises a CalledProcessError. The
611 CalledProcessError object will have the return code in the returncode
612 attribute and output in the output attribute.
613
614 The arguments are the same as for the Popen constructor. Example:
615
616 >>> check_output(["ls", "-l", "/dev/null"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000617 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
Georg Brandlf9734072008-12-07 15:30:06 +0000618
619 The stdout argument is not allowed as it is used internally.
Georg Brandl127d4702009-12-28 08:10:38 +0000620 To capture standard error in the result, use stderr=STDOUT.
Georg Brandlf9734072008-12-07 15:30:06 +0000621
622 >>> check_output(["/bin/sh", "-c",
Georg Brandl2708f3a2009-12-20 14:38:23 +0000623 ... "ls -l non_existent_file ; exit 0"],
Georg Brandl127d4702009-12-28 08:10:38 +0000624 ... stderr=STDOUT)
Georg Brandl2708f3a2009-12-20 14:38:23 +0000625 b'ls: non_existent_file: No such file or directory\n'
Gregory P. Smith91110f52013-03-19 23:25:16 -0700626
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300627 There is an additional optional argument, "input", allowing you to
628 pass a string to the subprocess's stdin. If you use this argument
629 you may not also use the Popen constructor's "stdin" argument, as
630 it too will be used internally. Example:
631
632 >>> check_output(["sed", "-e", "s/foo/bar/"],
633 ... input=b"when in the course of fooman events\n")
634 b'when in the course of barman events\n'
635
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400636 If universal_newlines=True is passed, the "input" argument must be a
637 string and the return value will be a string rather than bytes.
Georg Brandlf9734072008-12-07 15:30:06 +0000638 """
639 if 'stdout' in kwargs:
640 raise ValueError('stdout argument not allowed, it will be overridden.')
Gregory P. Smith6e730002015-04-14 16:14:25 -0700641
642 if 'input' in kwargs and kwargs['input'] is None:
643 # Explicitly passing input=None was previously equivalent to passing an
644 # empty string. That is maintained here for backwards compatibility.
645 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
646
647 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
648 **kwargs).stdout
649
650
651class CompletedProcess(object):
652 """A process that has finished running.
653
654 This is returned by run().
655
656 Attributes:
657 args: The list or str args passed to run().
658 returncode: The exit code of the process, negative for signals.
659 stdout: The standard output (None if not captured).
660 stderr: The standard error (None if not captured).
661 """
662 def __init__(self, args, returncode, stdout=None, stderr=None):
663 self.args = args
664 self.returncode = returncode
665 self.stdout = stdout
666 self.stderr = stderr
667
668 def __repr__(self):
669 args = ['args={!r}'.format(self.args),
670 'returncode={!r}'.format(self.returncode)]
671 if self.stdout is not None:
672 args.append('stdout={!r}'.format(self.stdout))
673 if self.stderr is not None:
674 args.append('stderr={!r}'.format(self.stderr))
675 return "{}({})".format(type(self).__name__, ', '.join(args))
676
677 def check_returncode(self):
678 """Raise CalledProcessError if the exit code is non-zero."""
679 if self.returncode:
680 raise CalledProcessError(self.returncode, self.args, self.stdout,
681 self.stderr)
682
683
684def run(*popenargs, input=None, timeout=None, check=False, **kwargs):
685 """Run command with arguments and return a CompletedProcess instance.
686
687 The returned instance will have attributes args, returncode, stdout and
688 stderr. By default, stdout and stderr are not captured, and those attributes
689 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
690
691 If check is True and the exit code was non-zero, it raises a
692 CalledProcessError. The CalledProcessError object will have the return code
693 in the returncode attribute, and output & stderr attributes if those streams
694 were captured.
695
696 If timeout is given, and the process takes too long, a TimeoutExpired
697 exception will be raised.
698
699 There is an optional argument "input", allowing you to
700 pass a string to the subprocess's stdin. If you use this argument
701 you may not also use the Popen constructor's "stdin" argument, as
702 it will be used internally.
703
704 The other arguments are the same as for the Popen constructor.
705
706 If universal_newlines=True is passed, the "input" argument must be a
707 string and stdout/stderr in the returned object will be strings rather than
708 bytes.
709 """
710 if input is not None:
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300711 if 'stdin' in kwargs:
712 raise ValueError('stdin and input arguments may not both be used.')
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300713 kwargs['stdin'] = PIPE
Gregory P. Smith6e730002015-04-14 16:14:25 -0700714
715 with Popen(*popenargs, **kwargs) as process:
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200716 try:
Gregory P. Smith6e730002015-04-14 16:14:25 -0700717 stdout, stderr = process.communicate(input, timeout=timeout)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200718 except TimeoutExpired:
719 process.kill()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700720 stdout, stderr = process.communicate()
721 raise TimeoutExpired(process.args, timeout, output=stdout,
722 stderr=stderr)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200723 except:
724 process.kill()
725 process.wait()
726 raise
727 retcode = process.poll()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700728 if check and retcode:
729 raise CalledProcessError(retcode, process.args,
730 output=stdout, stderr=stderr)
731 return CompletedProcess(process.args, retcode, stdout, stderr)
Peter Astrand454f7672005-01-01 09:36:35 +0000732
733
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000734def list2cmdline(seq):
735 """
736 Translate a sequence of arguments into a command line
737 string, using the same rules as the MS C runtime:
738
739 1) Arguments are delimited by white space, which is either a
740 space or a tab.
741
742 2) A string surrounded by double quotation marks is
743 interpreted as a single argument, regardless of white space
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000744 contained within. A quoted string can be embedded in an
745 argument.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000746
747 3) A double quotation mark preceded by a backslash is
748 interpreted as a literal double quotation mark.
749
750 4) Backslashes are interpreted literally, unless they
751 immediately precede a double quotation mark.
752
753 5) If backslashes immediately precede a double quotation mark,
754 every pair of backslashes is interpreted as a literal
755 backslash. If the number of backslashes is odd, the last
756 backslash escapes the next double quotation mark as
757 described in rule 3.
758 """
759
760 # See
Eric Smith3c573af2009-11-09 15:23:15 +0000761 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
762 # or search http://msdn.microsoft.com for
763 # "Parsing C++ Command-Line Arguments"
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000764 result = []
765 needquote = False
766 for arg in seq:
767 bs_buf = []
768
769 # Add a space to separate this argument from the others
770 if result:
771 result.append(' ')
772
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000773 needquote = (" " in arg) or ("\t" in arg) or not arg
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000774 if needquote:
775 result.append('"')
776
777 for c in arg:
778 if c == '\\':
Tim Peterse718f612004-10-12 21:51:32 +0000779 # Don't know if we need to double yet.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000780 bs_buf.append(c)
781 elif c == '"':
Christian Heimesfdab48e2008-01-20 09:06:41 +0000782 # Double backslashes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000783 result.append('\\' * len(bs_buf)*2)
784 bs_buf = []
785 result.append('\\"')
786 else:
787 # Normal char
788 if bs_buf:
789 result.extend(bs_buf)
790 bs_buf = []
791 result.append(c)
792
Christian Heimesfdab48e2008-01-20 09:06:41 +0000793 # Add remaining backslashes, if any.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000794 if bs_buf:
795 result.extend(bs_buf)
796
797 if needquote:
Peter Astrand7e78ade2005-03-03 21:10:23 +0000798 result.extend(bs_buf)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000799 result.append('"')
800
801 return ''.join(result)
802
803
Brett Cannona23810f2008-05-26 19:04:21 +0000804# Various tools for executing commands and looking at their output and status.
805#
Brett Cannona23810f2008-05-26 19:04:21 +0000806
807def getstatusoutput(cmd):
Tim Golden60798142013-11-05 12:57:25 +0000808 """ Return (status, output) of executing cmd in a shell.
Brett Cannona23810f2008-05-26 19:04:21 +0000809
Tim Golden60798142013-11-05 12:57:25 +0000810 Execute the string 'cmd' in a shell with 'check_output' and
811 return a 2-tuple (status, output). Universal newlines mode is used,
812 meaning that the result with be decoded to a string.
813
814 A trailing newline is stripped from the output.
815 The exit status for the command can be interpreted
816 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000817
818 >>> import subprocess
819 >>> subprocess.getstatusoutput('ls /bin/ls')
820 (0, '/bin/ls')
821 >>> subprocess.getstatusoutput('cat /bin/junk')
822 (256, 'cat: /bin/junk: No such file or directory')
823 >>> subprocess.getstatusoutput('/bin/junk')
824 (256, 'sh: /bin/junk: not found')
825 """
Tim Goldene0041752013-11-03 12:53:17 +0000826 try:
827 data = check_output(cmd, shell=True, universal_newlines=True, stderr=STDOUT)
828 status = 0
829 except CalledProcessError as ex:
830 data = ex.output
831 status = ex.returncode
832 if data[-1:] == '\n':
833 data = data[:-1]
834 return status, data
Brett Cannona23810f2008-05-26 19:04:21 +0000835
836def getoutput(cmd):
837 """Return output (stdout or stderr) of executing cmd in a shell.
838
839 Like getstatusoutput(), except the exit status is ignored and the return
840 value is a string containing the command's output. Example:
841
842 >>> import subprocess
843 >>> subprocess.getoutput('ls /bin/ls')
844 '/bin/ls'
845 """
846 return getstatusoutput(cmd)[1]
847
848
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000849_PLATFORM_DEFAULT_CLOSE_FDS = object()
Gregory P. Smithf5604852010-12-13 06:45:02 +0000850
851
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000852class Popen(object):
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200853
854 _child_created = False # Set here since __del__ checks it
855
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700856 def __init__(self, args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000857 stdin=None, stdout=None, stderr=None,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000858 preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
859 shell=False, cwd=None, env=None, universal_newlines=False,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000860 startupinfo=None, creationflags=0,
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000861 restore_signals=True, start_new_session=False,
862 pass_fds=()):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000863 """Create new Popen instance."""
864 _cleanup()
Gregory P. Smithd65ba512014-04-23 00:27:17 -0700865 # Held while anything is calling waitpid before returncode has been
866 # updated to prevent clobbering returncode if wait() or poll() are
867 # called from multiple threads at once. After acquiring the lock,
868 # code must re-check self.returncode to see if another thread just
869 # finished a waitpid() call.
870 self._waitpid_lock = threading.Lock()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000871
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400872 self._input = None
873 self._communication_started = False
Guido van Rossum46a05a72007-06-07 21:56:45 +0000874 if bufsize is None:
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700875 bufsize = -1 # Restore default
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000876 if not isinstance(bufsize, int):
Peter Astrand738131d2004-11-30 21:04:45 +0000877 raise TypeError("bufsize must be an integer")
878
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700879 if _mswindows:
Tim Peterse8374a52004-10-13 03:15:00 +0000880 if preexec_fn is not None:
881 raise ValueError("preexec_fn is not supported on Windows "
882 "platforms")
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000883 any_stdio_set = (stdin is not None or stdout is not None or
884 stderr is not None)
885 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
886 if any_stdio_set:
887 close_fds = False
888 else:
889 close_fds = True
890 elif close_fds and any_stdio_set:
891 raise ValueError(
892 "close_fds is not supported on Windows platforms"
893 " if you redirect stdin/stdout/stderr")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000894 else:
895 # POSIX
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000896 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
897 close_fds = True
898 if pass_fds and not close_fds:
899 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
900 close_fds = True
Tim Peterse8374a52004-10-13 03:15:00 +0000901 if startupinfo is not None:
902 raise ValueError("startupinfo is only supported on Windows "
903 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000904 if creationflags != 0:
Tim Peterse8374a52004-10-13 03:15:00 +0000905 raise ValueError("creationflags is only supported on Windows "
906 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000907
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400908 self.args = args
Tim Peterse718f612004-10-12 21:51:32 +0000909 self.stdin = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000910 self.stdout = None
911 self.stderr = None
912 self.pid = None
913 self.returncode = None
914 self.universal_newlines = universal_newlines
915
916 # Input and output objects. The general principle is like
917 # this:
918 #
919 # Parent Child
920 # ------ -----
921 # p2cwrite ---stdin---> p2cread
922 # c2pread <--stdout--- c2pwrite
923 # errread <--stderr--- errwrite
924 #
925 # On POSIX, the child objects are file descriptors. On
926 # Windows, these are Windows file handles. The parent objects
927 # are file descriptors on both platforms. The parent objects
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000928 # are -1 when not using PIPEs. The child objects are -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000929 # when not redirecting.
Tim Peterse718f612004-10-12 21:51:32 +0000930
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000931 (p2cread, p2cwrite,
932 c2pread, c2pwrite,
933 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
934
Antoine Pitrouc9982322011-01-04 19:07:07 +0000935 # We wrap OS handles *before* launching the child, otherwise a
936 # quickly terminating child could make our fds unwrappable
937 # (see #8458).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000938
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700939 if _mswindows:
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000940 if p2cwrite != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000941 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000942 if c2pread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000943 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000944 if errread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000945 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
Thomas Wouterscf297e42007-02-23 15:07:44 +0000946
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000947 if p2cwrite != -1:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000948 self.stdin = io.open(p2cwrite, 'wb', bufsize)
Andrew Svetlov592df202012-08-15 17:36:15 +0300949 if universal_newlines:
Antoine Pitrouafe8d062014-09-21 21:10:56 +0200950 self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
951 line_buffering=(bufsize == 1))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000952 if c2pread != -1:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000953 self.stdout = io.open(c2pread, 'rb', bufsize)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000954 if universal_newlines:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000955 self.stdout = io.TextIOWrapper(self.stdout)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000956 if errread != -1:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000957 self.stderr = io.open(errread, 'rb', bufsize)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000958 if universal_newlines:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000959 self.stderr = io.TextIOWrapper(self.stderr)
Tim Peterse718f612004-10-12 21:51:32 +0000960
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700961 self._closed_child_pipe_fds = False
Antoine Pitrouc9982322011-01-04 19:07:07 +0000962 try:
963 self._execute_child(args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300964 pass_fds, cwd, env,
Antoine Pitrouc9982322011-01-04 19:07:07 +0000965 startupinfo, creationflags, shell,
966 p2cread, p2cwrite,
967 c2pread, c2pwrite,
968 errread, errwrite,
969 restore_signals, start_new_session)
970 except:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800971 # Cleanup if the child failed starting.
972 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
Antoine Pitrouc9982322011-01-04 19:07:07 +0000973 try:
974 f.close()
Andrew Svetlov3438fa42012-12-17 23:35:18 +0200975 except OSError:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800976 pass # Ignore EBADF or other errors.
977
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700978 if not self._closed_child_pipe_fds:
979 to_close = []
980 if stdin == PIPE:
981 to_close.append(p2cread)
982 if stdout == PIPE:
983 to_close.append(c2pwrite)
984 if stderr == PIPE:
985 to_close.append(errwrite)
986 if hasattr(self, '_devnull'):
987 to_close.append(self._devnull)
988 for fd in to_close:
989 try:
990 os.close(fd)
Gregory P. Smith22ba31a2013-06-15 18:14:56 -0700991 except OSError:
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700992 pass
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800993
Antoine Pitrouc9982322011-01-04 19:07:07 +0000994 raise
995
Guido van Rossum98297ee2007-11-06 21:34:58 +0000996 def _translate_newlines(self, data, encoding):
Andrew Svetlov82860712012-08-19 22:13:41 +0300997 data = data.decode(encoding)
998 return data.replace("\r\n", "\n").replace("\r", "\n")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000999
Brian Curtin79cdb662010-12-03 02:46:02 +00001000 def __enter__(self):
1001 return self
1002
1003 def __exit__(self, type, value, traceback):
1004 if self.stdout:
1005 self.stdout.close()
1006 if self.stderr:
1007 self.stderr.close()
Serhiy Storchakaab900c22015-02-28 12:43:08 +02001008 try: # Flushing a BufferedWriter may raise an error
1009 if self.stdin:
1010 self.stdin.close()
1011 finally:
1012 # Wait for the process to terminate, to avoid zombies.
1013 self.wait()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001014
Serhiy Storchaka72e77612014-02-10 19:20:22 +02001015 def __del__(self, _maxsize=sys.maxsize):
1016 if not self._child_created:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001017 # We didn't get to successfully create a child process.
1018 return
Victor Stinner5a48e212016-05-20 12:11:15 +02001019 if self.returncode is None:
Victor Stinnerc206f1e2016-06-14 16:42:59 +02001020 # Not reading subprocess exit status creates a zombi process which
1021 # is only destroyed at the parent python process exit
1022 warnings.warn("subprocess %s is still running" % self.pid,
1023 ResourceWarning, source=self)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001024 # In case the child hasn't been waited on, check if it's done.
Brett Cannon84df1e62010-05-14 00:33:40 +00001025 self._internal_poll(_deadstate=_maxsize)
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001026 if self.returncode is None and _active is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001027 # Child is still running, keep us alive until we can wait on it.
1028 _active.append(self)
1029
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001030 def _get_devnull(self):
1031 if not hasattr(self, '_devnull'):
1032 self._devnull = os.open(os.devnull, os.O_RDWR)
1033 return self._devnull
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001034
Victor Stinnera5e881d2015-01-14 17:07:59 +01001035 def _stdin_write(self, input):
1036 if input:
1037 try:
1038 self.stdin.write(input)
1039 except BrokenPipeError:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001040 pass # communicate() must ignore broken pipe errors.
Victor Stinnera5e881d2015-01-14 17:07:59 +01001041 except OSError as e:
1042 if e.errno == errno.EINVAL and self.poll() is not None:
1043 # Issue #19612: On Windows, stdin.write() fails with EINVAL
1044 # if the process already exited before the write
1045 pass
1046 else:
1047 raise
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001048 try:
1049 self.stdin.close()
1050 except BrokenPipeError:
1051 pass # communicate() must ignore broken pipe errors.
1052 except OSError as e:
1053 if e.errno == errno.EINVAL and self.poll() is not None:
1054 pass
1055 else:
1056 raise
Victor Stinnera5e881d2015-01-14 17:07:59 +01001057
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001058 def communicate(self, input=None, timeout=None):
Peter Astrand23109f02005-03-03 20:28:59 +00001059 """Interact with process: Send data to stdin. Read data from
1060 stdout and stderr, until end-of-file is reached. Wait for
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -04001061 process to terminate.
Tim Peterseba28be2005-03-28 01:08:02 +00001062
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -04001063 The optional "input" argument should be data to be sent to the
1064 child process (if self.universal_newlines is True, this should
1065 be a string; if it is False, "input" should be bytes), or
1066 None, if no data should be sent to the child.
1067
1068 communicate() returns a tuple (stdout, stderr). These will be
1069 bytes or, if self.universal_newlines was True, a string.
1070 """
Peter Astrand23109f02005-03-03 20:28:59 +00001071
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001072 if self._communication_started and input:
1073 raise ValueError("Cannot send input after starting communication")
1074
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001075 # Optimization: If we are not worried about timeouts, we haven't
1076 # started communicating, and we have one or zero pipes, using select()
1077 # or threads is unnecessary.
Victor Stinner7a8d0812011-04-05 13:13:08 +02001078 if (timeout is None and not self._communication_started and
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001079 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
Tim Peterseba28be2005-03-28 01:08:02 +00001080 stdout = None
1081 stderr = None
Peter Astrand23109f02005-03-03 20:28:59 +00001082 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +01001083 self._stdin_write(input)
Peter Astrand23109f02005-03-03 20:28:59 +00001084 elif self.stdout:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001085 stdout = self.stdout.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +00001086 self.stdout.close()
Peter Astrand23109f02005-03-03 20:28:59 +00001087 elif self.stderr:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001088 stderr = self.stderr.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +00001089 self.stderr.close()
Peter Astrand23109f02005-03-03 20:28:59 +00001090 self.wait()
Victor Stinner7a8d0812011-04-05 13:13:08 +02001091 else:
1092 if timeout is not None:
Victor Stinner949d8c92012-05-30 13:30:32 +02001093 endtime = _time() + timeout
Victor Stinner7a8d0812011-04-05 13:13:08 +02001094 else:
1095 endtime = None
Tim Peterseba28be2005-03-28 01:08:02 +00001096
Victor Stinner7a8d0812011-04-05 13:13:08 +02001097 try:
1098 stdout, stderr = self._communicate(input, endtime, timeout)
1099 finally:
1100 self._communication_started = True
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001101
Victor Stinner7a8d0812011-04-05 13:13:08 +02001102 sts = self.wait(timeout=self._remaining_time(endtime))
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001103
1104 return (stdout, stderr)
Peter Astrand23109f02005-03-03 20:28:59 +00001105
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001106
Georg Brandl6aa2d1f2008-08-12 08:35:52 +00001107 def poll(self):
1108 return self._internal_poll()
1109
1110
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001111 def _remaining_time(self, endtime):
1112 """Convenience for _communicate when computing timeouts."""
1113 if endtime is None:
1114 return None
1115 else:
Victor Stinner949d8c92012-05-30 13:30:32 +02001116 return endtime - _time()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001117
1118
Reid Kleckner2b228f02011-03-16 16:57:54 -04001119 def _check_timeout(self, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001120 """Convenience for checking if a timeout has expired."""
1121 if endtime is None:
1122 return
Victor Stinner949d8c92012-05-30 13:30:32 +02001123 if _time() > endtime:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001124 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001125
1126
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -07001127 if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001128 #
1129 # Windows methods
1130 #
1131 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001132 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001133 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1134 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001135 if stdin is None and stdout is None and stderr is None:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001136 return (-1, -1, -1, -1, -1, -1)
Tim Peterse718f612004-10-12 21:51:32 +00001137
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001138 p2cread, p2cwrite = -1, -1
1139 c2pread, c2pwrite = -1, -1
1140 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001141
Peter Astrandd38ddf42005-02-10 08:32:50 +00001142 if stdin is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001143 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001144 if p2cread is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001145 p2cread, _ = _winapi.CreatePipe(None, 0)
1146 p2cread = Handle(p2cread)
1147 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001148 elif stdin == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001149 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1150 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001151 elif stdin == DEVNULL:
1152 p2cread = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001153 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001154 p2cread = msvcrt.get_osfhandle(stdin)
1155 else:
1156 # Assuming file-like object
1157 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1158 p2cread = self._make_inheritable(p2cread)
1159
Peter Astrandd38ddf42005-02-10 08:32:50 +00001160 if stdout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001161 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001162 if c2pwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001163 _, c2pwrite = _winapi.CreatePipe(None, 0)
1164 c2pwrite = Handle(c2pwrite)
1165 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001166 elif stdout == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001167 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1168 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001169 elif stdout == DEVNULL:
1170 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001171 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001172 c2pwrite = msvcrt.get_osfhandle(stdout)
1173 else:
1174 # Assuming file-like object
1175 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1176 c2pwrite = self._make_inheritable(c2pwrite)
1177
Peter Astrandd38ddf42005-02-10 08:32:50 +00001178 if stderr is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001179 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001180 if errwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001181 _, errwrite = _winapi.CreatePipe(None, 0)
1182 errwrite = Handle(errwrite)
1183 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001184 elif stderr == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001185 errread, errwrite = _winapi.CreatePipe(None, 0)
1186 errread, errwrite = Handle(errread), Handle(errwrite)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001187 elif stderr == STDOUT:
1188 errwrite = c2pwrite
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001189 elif stderr == DEVNULL:
1190 errwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001191 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001192 errwrite = msvcrt.get_osfhandle(stderr)
1193 else:
1194 # Assuming file-like object
1195 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1196 errwrite = self._make_inheritable(errwrite)
1197
1198 return (p2cread, p2cwrite,
1199 c2pread, c2pwrite,
1200 errread, errwrite)
1201
1202
1203 def _make_inheritable(self, handle):
1204 """Return a duplicate of handle, which is inheritable"""
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001205 h = _winapi.DuplicateHandle(
1206 _winapi.GetCurrentProcess(), handle,
1207 _winapi.GetCurrentProcess(), 0, 1,
1208 _winapi.DUPLICATE_SAME_ACCESS)
1209 return Handle(h)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001210
1211
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001212 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001213 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001214 startupinfo, creationflags, shell,
1215 p2cread, p2cwrite,
1216 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001217 errread, errwrite,
1218 unused_restore_signals, unused_start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001219 """Execute program (MS Windows version)"""
1220
Gregory P. Smith8edd99d2010-12-14 13:43:30 +00001221 assert not pass_fds, "pass_fds not supported on Windows."
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +00001222
Guido van Rossum3172c5d2007-10-16 18:12:55 +00001223 if not isinstance(args, str):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001224 args = list2cmdline(args)
1225
Peter Astrandc1d65362004-11-07 14:30:34 +00001226 # Process startup details
Peter Astrandd38ddf42005-02-10 08:32:50 +00001227 if startupinfo is None:
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001228 startupinfo = STARTUPINFO()
Victor Stinnerb3693582010-05-21 20:13:12 +00001229 if -1 not in (p2cread, c2pwrite, errwrite):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001230 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
Peter Astrandc1d65362004-11-07 14:30:34 +00001231 startupinfo.hStdInput = p2cread
1232 startupinfo.hStdOutput = c2pwrite
1233 startupinfo.hStdError = errwrite
1234
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001235 if shell:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001236 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1237 startupinfo.wShowWindow = _winapi.SW_HIDE
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001238 comspec = os.environ.get("COMSPEC", "cmd.exe")
Tim Golden126c2962010-08-11 14:20:40 +00001239 args = '{} /c "{}"'.format (comspec, args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001240
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001241 # Start the process
1242 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001243 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
Tim Peterse8374a52004-10-13 03:15:00 +00001244 # no special security
1245 None, None,
Guido van Rossume7ba4952007-06-06 23:52:48 +00001246 int(not close_fds),
Tim Peterse8374a52004-10-13 03:15:00 +00001247 creationflags,
1248 env,
1249 cwd,
1250 startupinfo)
Tim Goldenad537f22010-08-08 11:18:16 +00001251 finally:
1252 # Child is launched. Close the parent's copy of those pipe
1253 # handles that only the child should have open. You need
1254 # to make sure that no handles to the write end of the
1255 # output pipe are maintained in this process or else the
1256 # pipe will not close when the child process exits and the
1257 # ReadFile will hang.
1258 if p2cread != -1:
1259 p2cread.Close()
1260 if c2pwrite != -1:
1261 c2pwrite.Close()
1262 if errwrite != -1:
1263 errwrite.Close()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001264 if hasattr(self, '_devnull'):
1265 os.close(self._devnull)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001266
1267 # Retain the process handle, but close the thread handle
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001268 self._child_created = True
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001269 self._handle = Handle(hp)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001270 self.pid = pid
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001271 _winapi.CloseHandle(ht)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001272
Brett Cannon84df1e62010-05-14 00:33:40 +00001273 def _internal_poll(self, _deadstate=None,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001274 _WaitForSingleObject=_winapi.WaitForSingleObject,
1275 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1276 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001277 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001278 attribute.
1279
1280 This method is called by __del__, so it can only refer to objects
1281 in its local scope.
1282
1283 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001284 if self.returncode is None:
Brett Cannon84df1e62010-05-14 00:33:40 +00001285 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1286 self.returncode = _GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001287 return self.returncode
1288
1289
Reid Kleckner2b228f02011-03-16 16:57:54 -04001290 def wait(self, timeout=None, endtime=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001291 """Wait for child process to terminate. Returns returncode
1292 attribute."""
Reid Kleckner2b228f02011-03-16 16:57:54 -04001293 if endtime is not None:
1294 timeout = self._remaining_time(endtime)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001295 if timeout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001296 timeout_millis = _winapi.INFINITE
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001297 else:
Reid Kleckner91156ff2011-03-21 10:06:10 -07001298 timeout_millis = int(timeout * 1000)
Peter Astrandd38ddf42005-02-10 08:32:50 +00001299 if self.returncode is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001300 result = _winapi.WaitForSingleObject(self._handle,
1301 timeout_millis)
1302 if result == _winapi.WAIT_TIMEOUT:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001303 raise TimeoutExpired(self.args, timeout)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001304 self.returncode = _winapi.GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001305 return self.returncode
1306
1307
1308 def _readerthread(self, fh, buffer):
1309 buffer.append(fh.read())
Victor Stinner667d4b52010-12-25 22:40:32 +00001310 fh.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001311
1312
Reid Kleckner2b228f02011-03-16 16:57:54 -04001313 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001314 # Start reader threads feeding into a list hanging off of this
1315 # object, unless they've already been started.
1316 if self.stdout and not hasattr(self, "_stdout_buff"):
1317 self._stdout_buff = []
1318 self.stdout_thread = \
1319 threading.Thread(target=self._readerthread,
1320 args=(self.stdout, self._stdout_buff))
1321 self.stdout_thread.daemon = True
1322 self.stdout_thread.start()
1323 if self.stderr and not hasattr(self, "_stderr_buff"):
1324 self._stderr_buff = []
1325 self.stderr_thread = \
1326 threading.Thread(target=self._readerthread,
1327 args=(self.stderr, self._stderr_buff))
1328 self.stderr_thread.daemon = True
1329 self.stderr_thread.start()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001330
1331 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +01001332 self._stdin_write(input)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001333
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001334 # Wait for the reader threads, or time out. If we time out, the
1335 # threads remain reading and the fds left open in case the user
1336 # calls communicate again.
1337 if self.stdout is not None:
1338 self.stdout_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001339 if self.stdout_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001340 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001341 if self.stderr is not None:
1342 self.stderr_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001343 if self.stderr_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001344 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001345
1346 # Collect the output from and close both pipes, now that we know
1347 # both have been read successfully.
1348 stdout = None
1349 stderr = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001350 if self.stdout:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001351 stdout = self._stdout_buff
1352 self.stdout.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001353 if self.stderr:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001354 stderr = self._stderr_buff
1355 self.stderr.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001356
1357 # All data exchanged. Translate lists into strings.
Peter Astrandd38ddf42005-02-10 08:32:50 +00001358 if stdout is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001359 stdout = stdout[0]
Peter Astrandd38ddf42005-02-10 08:32:50 +00001360 if stderr is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001361 stderr = stderr[0]
1362
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001363 return (stdout, stderr)
1364
Christian Heimesa342c012008-04-20 21:01:16 +00001365 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001366 """Send a signal to the process."""
1367 # Don't signal a process that we know has already died.
1368 if self.returncode is not None:
1369 return
Christian Heimesa342c012008-04-20 21:01:16 +00001370 if sig == signal.SIGTERM:
1371 self.terminate()
Brian Curtineb24d742010-04-12 17:16:38 +00001372 elif sig == signal.CTRL_C_EVENT:
1373 os.kill(self.pid, signal.CTRL_C_EVENT)
1374 elif sig == signal.CTRL_BREAK_EVENT:
1375 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
Christian Heimesa342c012008-04-20 21:01:16 +00001376 else:
Brian Curtin19651362010-09-07 13:24:38 +00001377 raise ValueError("Unsupported signal: {}".format(sig))
Christian Heimesa342c012008-04-20 21:01:16 +00001378
1379 def terminate(self):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001380 """Terminates the process."""
1381 # Don't terminate a process that we know has already died.
1382 if self.returncode is not None:
1383 return
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001384 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001385 _winapi.TerminateProcess(self._handle, 1)
Antoine Pitroub69ef162012-03-11 19:33:29 +01001386 except PermissionError:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001387 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1388 # process already died.
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001389 rc = _winapi.GetExitCodeProcess(self._handle)
1390 if rc == _winapi.STILL_ACTIVE:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001391 raise
1392 self.returncode = rc
Christian Heimesa342c012008-04-20 21:01:16 +00001393
1394 kill = terminate
1395
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001396 else:
1397 #
1398 # POSIX methods
1399 #
1400 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001401 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001402 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1403 """
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001404 p2cread, p2cwrite = -1, -1
1405 c2pread, c2pwrite = -1, -1
1406 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001407
Peter Astrandd38ddf42005-02-10 08:32:50 +00001408 if stdin is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001409 pass
1410 elif stdin == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001411 p2cread, p2cwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001412 elif stdin == DEVNULL:
1413 p2cread = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001414 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001415 p2cread = stdin
1416 else:
1417 # Assuming file-like object
1418 p2cread = stdin.fileno()
1419
Peter Astrandd38ddf42005-02-10 08:32:50 +00001420 if stdout is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001421 pass
1422 elif stdout == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001423 c2pread, c2pwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001424 elif stdout == DEVNULL:
1425 c2pwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001426 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001427 c2pwrite = stdout
1428 else:
1429 # Assuming file-like object
Tim Peterse718f612004-10-12 21:51:32 +00001430 c2pwrite = stdout.fileno()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001431
Peter Astrandd38ddf42005-02-10 08:32:50 +00001432 if stderr is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001433 pass
1434 elif stderr == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001435 errread, errwrite = os.pipe()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001436 elif stderr == STDOUT:
Martin Panterc7635892016-05-13 01:54:44 +00001437 if c2pwrite != -1:
1438 errwrite = c2pwrite
1439 else: # child's stdout is not set, use parent's stdout
1440 errwrite = sys.__stdout__.fileno()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001441 elif stderr == DEVNULL:
1442 errwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001443 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001444 errwrite = stderr
1445 else:
1446 # Assuming file-like object
1447 errwrite = stderr.fileno()
1448
1449 return (p2cread, p2cwrite,
1450 c2pread, c2pwrite,
1451 errread, errwrite)
1452
1453
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001454 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001455 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001456 startupinfo, creationflags, shell,
1457 p2cread, p2cwrite,
1458 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001459 errread, errwrite,
1460 restore_signals, start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001461 """Execute program (POSIX version)"""
1462
Victor Stinner7b3b20a2011-03-03 12:54:05 +00001463 if isinstance(args, (str, bytes)):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001464 args = [args]
Thomas Wouters89f507f2006-12-13 04:49:30 +00001465 else:
1466 args = list(args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001467
1468 if shell:
1469 args = ["/bin/sh", "-c"] + args
Stefan Krah9542cc62010-07-19 14:20:53 +00001470 if executable:
1471 args[0] = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001472
Peter Astrandd38ddf42005-02-10 08:32:50 +00001473 if executable is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001474 executable = args[0]
Gregory P. Smith5591b022012-10-10 03:34:47 -07001475 orig_executable = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001476
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001477 # For transferring possible exec failure from child to parent.
1478 # Data format: "exception name:hex errno:description"
1479 # Pickle is not used; it is complex and involves memory allocation.
Victor Stinnerdaf45552013-08-28 00:53:59 +02001480 errpipe_read, errpipe_write = os.pipe()
Gregory P. Smith53dd8162013-12-01 16:03:24 -08001481 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1482 low_fds_to_close = []
1483 while errpipe_write < 3:
1484 low_fds_to_close.append(errpipe_write)
1485 errpipe_write = os.dup(errpipe_write)
1486 for low_fd in low_fds_to_close:
1487 os.close(low_fd)
Christian Heimesfdab48e2008-01-20 09:06:41 +00001488 try:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001489 try:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001490 # We must avoid complex work that could involve
1491 # malloc or free in the child process to avoid
1492 # potential deadlocks, thus we do all this here.
1493 # and pass it to fork_exec()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001494
Victor Stinner372b8382011-06-21 17:24:21 +02001495 if env is not None:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001496 env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
1497 for k, v in env.items()]
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001498 else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001499 env_list = None # Use execv instead of execve.
1500 executable = os.fsencode(executable)
1501 if os.path.dirname(executable):
1502 executable_list = (executable,)
1503 else:
1504 # This matches the behavior of os._execvpe().
1505 executable_list = tuple(
1506 os.path.join(os.fsencode(dir), executable)
1507 for dir in os.get_exec_path(env))
Gregory P. Smith361e30c2013-12-01 00:12:24 -08001508 fds_to_keep = set(pass_fds)
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001509 fds_to_keep.add(errpipe_write)
1510 self.pid = _posixsubprocess.fork_exec(
1511 args, executable_list,
1512 close_fds, sorted(fds_to_keep), cwd, env_list,
1513 p2cread, p2cwrite, c2pread, c2pwrite,
1514 errread, errwrite,
1515 errpipe_read, errpipe_write,
1516 restore_signals, start_new_session, preexec_fn)
Charles-François Natali558639f2011-08-18 19:11:29 +02001517 self._child_created = True
Facundo Batista10706e22009-06-19 20:34:30 +00001518 finally:
1519 # be sure the FD is closed no matter what
1520 os.close(errpipe_write)
1521
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001522 # self._devnull is not always defined.
1523 devnull_fd = getattr(self, '_devnull', None)
1524 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001525 os.close(p2cread)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001526 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001527 os.close(c2pwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001528 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001529 os.close(errwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001530 if devnull_fd is not None:
1531 os.close(devnull_fd)
1532 # Prevent a double close of these fds from __init__ on error.
1533 self._closed_child_pipe_fds = True
Facundo Batista10706e22009-06-19 20:34:30 +00001534
1535 # Wait for exec to fail or succeed; possibly raising an
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001536 # exception (limited in size)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001537 errpipe_data = bytearray()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001538 while True:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001539 part = os.read(errpipe_read, 50000)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001540 errpipe_data += part
1541 if not part or len(errpipe_data) > 50000:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001542 break
Facundo Batista10706e22009-06-19 20:34:30 +00001543 finally:
1544 # be sure the FD is closed no matter what
1545 os.close(errpipe_read)
1546
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001547 if errpipe_data:
Gregory P. Smithe85db2b2010-12-14 14:38:00 +00001548 try:
Victor Stinnera58e2c52016-05-20 12:08:12 +02001549 pid, sts = os.waitpid(self.pid, 0)
1550 if pid == self.pid:
1551 self._handle_exitstatus(sts)
1552 else:
1553 self.returncode = sys.maxsize
Victor Stinnera5e881d2015-01-14 17:07:59 +01001554 except ChildProcessError:
1555 pass
Victor Stinnera58e2c52016-05-20 12:08:12 +02001556
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001557 try:
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001558 exception_name, hex_errno, err_msg = (
1559 errpipe_data.split(b':', 2))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001560 except ValueError:
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001561 exception_name = b'SubprocessError'
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001562 hex_errno = b'0'
Gregory P. Smith3aee2222012-11-11 00:04:13 -08001563 err_msg = (b'Bad exception data from child: ' +
1564 repr(errpipe_data))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001565 child_exception_type = getattr(
1566 builtins, exception_name.decode('ascii'),
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001567 SubprocessError)
Victor Stinner4d078042010-04-23 19:28:32 +00001568 err_msg = err_msg.decode(errors="surrogatepass")
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001569 if issubclass(child_exception_type, OSError) and hex_errno:
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001570 errno_num = int(hex_errno, 16)
Gregory P. Smith5591b022012-10-10 03:34:47 -07001571 child_exec_never_called = (err_msg == "noexec")
1572 if child_exec_never_called:
1573 err_msg = ""
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001574 if errno_num != 0:
1575 err_msg = os.strerror(errno_num)
1576 if errno_num == errno.ENOENT:
Gregory P. Smith5591b022012-10-10 03:34:47 -07001577 if child_exec_never_called:
1578 # The error must be from chdir(cwd).
1579 err_msg += ': ' + repr(cwd)
1580 else:
1581 err_msg += ': ' + repr(orig_executable)
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001582 raise child_exception_type(errno_num, err_msg)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001583 raise child_exception_type(err_msg)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001584
1585
Brett Cannon84df1e62010-05-14 00:33:40 +00001586 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
1587 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
1588 _WEXITSTATUS=os.WEXITSTATUS):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001589 """All callers to this function MUST hold self._waitpid_lock."""
Brett Cannon84df1e62010-05-14 00:33:40 +00001590 # This method is called (indirectly) by __del__, so it cannot
Serhiy Storchaka72e77612014-02-10 19:20:22 +02001591 # refer to anything outside of its local scope.
Brett Cannon84df1e62010-05-14 00:33:40 +00001592 if _WIFSIGNALED(sts):
1593 self.returncode = -_WTERMSIG(sts)
1594 elif _WIFEXITED(sts):
1595 self.returncode = _WEXITSTATUS(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001596 else:
1597 # Should never happen
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001598 raise SubprocessError("Unknown child exit status!")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001599
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001600
Brett Cannon84df1e62010-05-14 00:33:40 +00001601 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
Andrew Svetlov1d960fe2012-12-24 20:08:53 +02001602 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001603 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001604 attribute.
1605
1606 This method is called by __del__, so it cannot reference anything
1607 outside of the local scope (nor can any methods it calls).
1608
1609 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001610 if self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001611 if not self._waitpid_lock.acquire(False):
1612 # Something else is busy calling waitpid. Don't allow two
1613 # at once. We know nothing yet.
1614 return None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001615 try:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001616 if self.returncode is not None:
1617 return self.returncode # Another thread waited.
Brett Cannon84df1e62010-05-14 00:33:40 +00001618 pid, sts = _waitpid(self.pid, _WNOHANG)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001619 if pid == self.pid:
1620 self._handle_exitstatus(sts)
Andrew Svetlovad28c7f2012-12-18 22:02:39 +02001621 except OSError as e:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001622 if _deadstate is not None:
1623 self.returncode = _deadstate
Andrew Svetlov08bab072012-12-24 20:06:35 +02001624 elif e.errno == _ECHILD:
Gregory P. Smith39051712012-09-29 11:40:38 -07001625 # This happens if SIGCLD is set to be ignored or
1626 # waiting for child processes has otherwise been
1627 # disabled for our process. This child is dead, we
1628 # can't get the status.
1629 # http://bugs.python.org/issue15756
1630 self.returncode = 0
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001631 finally:
1632 self._waitpid_lock.release()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001633 return self.returncode
1634
1635
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001636 def _try_wait(self, wait_flags):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001637 """All callers to this function MUST hold self._waitpid_lock."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001638 try:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001639 (pid, sts) = os.waitpid(self.pid, wait_flags)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001640 except ChildProcessError:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001641 # This happens if SIGCLD is set to be ignored or waiting
1642 # for child processes has otherwise been disabled for our
1643 # process. This child is dead, we can't get the status.
1644 pid = self.pid
1645 sts = 0
1646 return (pid, sts)
1647
1648
1649 def wait(self, timeout=None, endtime=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001650 """Wait for child process to terminate. Returns returncode
1651 attribute."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001652 if self.returncode is not None:
1653 return self.returncode
Reid Kleckner2b228f02011-03-16 16:57:54 -04001654
1655 # endtime is preferred to timeout. timeout is only used for
1656 # printing.
1657 if endtime is not None or timeout is not None:
1658 if endtime is None:
Victor Stinner949d8c92012-05-30 13:30:32 +02001659 endtime = _time() + timeout
Reid Kleckner2b228f02011-03-16 16:57:54 -04001660 elif timeout is None:
1661 timeout = self._remaining_time(endtime)
1662
1663 if endtime is not None:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001664 # Enter a busy loop if we have a timeout. This busy loop was
1665 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1666 delay = 0.0005 # 500 us -> initial delay of 1 ms
1667 while True:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001668 if self._waitpid_lock.acquire(False):
1669 try:
1670 if self.returncode is not None:
1671 break # Another thread waited.
1672 (pid, sts) = self._try_wait(os.WNOHANG)
1673 assert pid == self.pid or pid == 0
1674 if pid == self.pid:
1675 self._handle_exitstatus(sts)
1676 break
1677 finally:
1678 self._waitpid_lock.release()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001679 remaining = self._remaining_time(endtime)
1680 if remaining <= 0:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001681 raise TimeoutExpired(self.args, timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001682 delay = min(delay * 2, remaining, .05)
1683 time.sleep(delay)
Gregory P. Smithf328d792012-11-10 21:06:18 -08001684 else:
1685 while self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001686 with self._waitpid_lock:
1687 if self.returncode is not None:
1688 break # Another thread waited.
1689 (pid, sts) = self._try_wait(0)
1690 # Check the pid and loop as waitpid has been known to
1691 # return 0 even without WNOHANG in odd situations.
1692 # http://bugs.python.org/issue14396.
1693 if pid == self.pid:
1694 self._handle_exitstatus(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001695 return self.returncode
1696
1697
Reid Kleckner2b228f02011-03-16 16:57:54 -04001698 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001699 if self.stdin and not self._communication_started:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001700 # Flush stdio buffer. This might block, if the user has
1701 # been writing to .stdin in an uncontrolled fashion.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001702 try:
1703 self.stdin.flush()
1704 except BrokenPipeError:
1705 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001706 if not input:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001707 try:
1708 self.stdin.close()
1709 except BrokenPipeError:
1710 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001711
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001712 stdout = None
1713 stderr = None
1714
1715 # Only create this mapping if we haven't already.
1716 if not self._communication_started:
1717 self._fileobj2output = {}
1718 if self.stdout:
1719 self._fileobj2output[self.stdout] = []
1720 if self.stderr:
1721 self._fileobj2output[self.stderr] = []
1722
1723 if self.stdout:
1724 stdout = self._fileobj2output[self.stdout]
1725 if self.stderr:
1726 stderr = self._fileobj2output[self.stderr]
1727
1728 self._save_input(input)
1729
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001730 if self._input:
1731 input_view = memoryview(self._input)
1732
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001733 with _PopenSelector() as selector:
1734 if self.stdin and input:
1735 selector.register(self.stdin, selectors.EVENT_WRITE)
1736 if self.stdout:
1737 selector.register(self.stdout, selectors.EVENT_READ)
1738 if self.stderr:
1739 selector.register(self.stderr, selectors.EVENT_READ)
1740
1741 while selector.get_map():
1742 timeout = self._remaining_time(endtime)
1743 if timeout is not None and timeout < 0:
1744 raise TimeoutExpired(self.args, orig_timeout)
1745
1746 ready = selector.select(timeout)
1747 self._check_timeout(endtime, orig_timeout)
1748
1749 # XXX Rewrite these to use non-blocking I/O on the file
1750 # objects; they are no longer using C stdio!
1751
1752 for key, events in ready:
1753 if key.fileobj is self.stdin:
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001754 chunk = input_view[self._input_offset :
1755 self._input_offset + _PIPE_BUF]
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001756 try:
1757 self._input_offset += os.write(key.fd, chunk)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001758 except BrokenPipeError:
1759 selector.unregister(key.fileobj)
1760 key.fileobj.close()
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001761 else:
1762 if self._input_offset >= len(self._input):
1763 selector.unregister(key.fileobj)
1764 key.fileobj.close()
1765 elif key.fileobj in (self.stdout, self.stderr):
Gregory P. Smith7b83b182013-12-08 10:58:28 -08001766 data = os.read(key.fd, 32768)
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001767 if not data:
1768 selector.unregister(key.fileobj)
1769 key.fileobj.close()
1770 self._fileobj2output[key.fileobj].append(data)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001771
1772 self.wait(timeout=self._remaining_time(endtime))
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001773
1774 # All data exchanged. Translate lists into strings.
1775 if stdout is not None:
1776 stdout = b''.join(stdout)
1777 if stderr is not None:
1778 stderr = b''.join(stderr)
1779
1780 # Translate newlines, if requested.
1781 # This also turns bytes into strings.
1782 if self.universal_newlines:
1783 if stdout is not None:
1784 stdout = self._translate_newlines(stdout,
1785 self.stdout.encoding)
1786 if stderr is not None:
1787 stderr = self._translate_newlines(stderr,
1788 self.stderr.encoding)
1789
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001790 return (stdout, stderr)
1791
1792
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001793 def _save_input(self, input):
1794 # This method is called from the _communicate_with_*() methods
1795 # so that if we time out while communicating, we can continue
1796 # sending input if we retry.
1797 if self.stdin and self._input is None:
1798 self._input_offset = 0
1799 self._input = input
1800 if self.universal_newlines and input is not None:
1801 self._input = self._input.encode(self.stdin.encoding)
1802
1803
Christian Heimesa342c012008-04-20 21:01:16 +00001804 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001805 """Send a signal to the process."""
1806 # Skip signalling a process that we know has already died.
1807 if self.returncode is None:
1808 os.kill(self.pid, sig)
Christian Heimesa342c012008-04-20 21:01:16 +00001809
1810 def terminate(self):
1811 """Terminate the process with SIGTERM
1812 """
1813 self.send_signal(signal.SIGTERM)
1814
1815 def kill(self):
1816 """Kill the process with SIGKILL
1817 """
1818 self.send_signal(signal.SIGKILL)