blob: d629bdfc93ea6ddc3987e4c8363d1959dbfebb5e [file] [log] [blame]
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001# subprocess - Subprocesses with accessible I/O streams
2#
Tim Peterse718f612004-10-12 21:51:32 +00003# For more information about this module, see PEP 324.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00004#
Peter Astrand3a708df2005-09-23 17:37:29 +00005# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00006#
Peter Astrand69bf13f2005-02-14 08:56:32 +00007# Licensed to PSF under a Contributor Agreement.
Peter Astrand3a708df2005-09-23 17:37:29 +00008# See http://www.python.org/2.4/license for licensing details.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00009
Raymond Hettinger837dd932004-10-17 16:36:53 +000010r"""subprocess - Subprocesses with accessible I/O streams
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000011
Fredrik Lundh15aaacc2004-10-17 14:47:05 +000012This module allows you to spawn processes, connect to their
13input/output/error pipes, and obtain their return codes. This module
Benjamin Peterson5eea8a72014-03-12 21:41:35 -050014intends to replace several older modules and functions:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000015
16os.system
17os.spawn*
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000018
19Information about how the subprocess module can be used to replace these
20modules and functions can be found below.
21
22
23
24Using the subprocess module
25===========================
26This module defines one class called Popen:
27
Gregory P. Smitha1ed5392013-03-23 11:44:25 -070028class Popen(args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000029 stdin=None, stdout=None, stderr=None,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +000030 preexec_fn=None, close_fds=True, shell=False,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000031 cwd=None, env=None, universal_newlines=False,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000032 startupinfo=None, creationflags=0,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +000033 restore_signals=True, start_new_session=False, pass_fds=()):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000034
35
36Arguments are:
37
38args should be a string, or a sequence of program arguments. The
39program to execute is normally the first item in the args sequence or
40string, but can be explicitly set by using the executable argument.
41
Gregory P. Smithf5604852010-12-13 06:45:02 +000042On POSIX, with shell=False (default): In this case, the Popen class
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000043uses os.execvp() to execute the child program. args should normally
44be a sequence. A string will be treated as a sequence with the string
45as the only item (the program to execute).
46
Gregory P. Smithf5604852010-12-13 06:45:02 +000047On POSIX, with shell=True: If args is a string, it specifies the
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000048command string to execute through the shell. If args is a sequence,
49the first item specifies the command string, and any additional items
50will be treated as additional shell arguments.
51
52On Windows: the Popen class uses CreateProcess() to execute the child
53program, which operates on strings. If args is a sequence, it will be
54converted to a string using the list2cmdline method. Please note that
55not all MS Windows applications interpret the command line the same
56way: The list2cmdline is designed for applications using the same
57rules as the MS C runtime.
58
Gregory P. Smitha1ed5392013-03-23 11:44:25 -070059bufsize will be supplied as the corresponding argument to the io.open()
60function when creating the stdin/stdout/stderr pipe file objects:
610 means unbuffered (read & write are one system call and can return short),
621 means line buffered, any other positive value means use a buffer of
63approximately that size. A negative bufsize, the default, means the system
64default of io.DEFAULT_BUFFER_SIZE will be used.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000065
66stdin, stdout and stderr specify the executed programs' standard
67input, standard output and standard error file handles, respectively.
68Valid values are PIPE, an existing file descriptor (a positive
69integer), an existing file object, and None. PIPE indicates that a
70new pipe to the child should be created. With None, no redirection
71will occur; the child's file handles will be inherited from the
72parent. Additionally, stderr can be STDOUT, which indicates that the
73stderr data from the applications should be captured into the same
74file handle as for stdout.
75
Gregory P. Smithf5604852010-12-13 06:45:02 +000076On POSIX, if preexec_fn is set to a callable object, this object will be
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000077called in the child process just before the child is executed. The use
78of preexec_fn is not thread safe, using it in the presence of threads
79could lead to a deadlock in the child process before the new executable
80is executed.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000081
82If close_fds is true, all file descriptors except 0, 1 and 2 will be
Gregory P. Smithf5604852010-12-13 06:45:02 +000083closed before the child process is executed. The default for close_fds
Gregory P. Smith8edd99d2010-12-14 13:43:30 +000084varies by platform: Always true on POSIX. True when stdin/stdout/stderr
85are None on Windows, false otherwise.
86
87pass_fds is an optional sequence of file descriptors to keep open between the
88parent and child. Providing any pass_fds implicitly sets close_fds to true.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000089
90if shell is true, the specified command will be executed through the
91shell.
92
93If cwd is not None, the current directory will be changed to cwd
94before the child is executed.
95
Gregory P. Smithf5604852010-12-13 06:45:02 +000096On POSIX, if restore_signals is True all signals that Python sets to
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000097SIG_IGN are restored to SIG_DFL in the child process before the exec.
98Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This
99parameter does nothing on Windows.
100
Gregory P. Smithf5604852010-12-13 06:45:02 +0000101On POSIX, if start_new_session is True, the setsid() system call will be made
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000102in the child process prior to executing the command.
103
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000104If env is not None, it defines the environment variables for the new
105process.
106
Ronald Oussoren385521c2013-07-07 09:26:45 +0200107If universal_newlines is false, the file objects stdin, stdout and stderr
108are opened as binary files, and no line ending conversion is done.
109
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000110If universal_newlines is true, the file objects stdout and stderr are
111opened as a text files, but lines may be terminated by any of '\n',
Gregory P. Smithf5604852010-12-13 06:45:02 +0000112the Unix end-of-line convention, '\r', the old Macintosh convention or
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000113'\r\n', the Windows convention. All of these external representations
Gregory P. Smith1f8a40b2013-03-20 18:32:03 -0700114are seen as '\n' by the Python program. Also, the newlines attribute
115of the file objects stdout, stdin and stderr are not updated by the
116communicate() method.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000117
118The startupinfo and creationflags, if given, will be passed to the
119underlying CreateProcess() function. They can specify things such as
120appearance of the main window and priority for the new process.
121(Windows only)
122
123
Georg Brandlf9734072008-12-07 15:30:06 +0000124This module also defines some shortcut functions:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000125
Peter Astrand5f5e1412004-12-05 20:15:36 +0000126call(*popenargs, **kwargs):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000127 Run command with arguments. Wait for command to complete, then
128 return the returncode attribute.
129
130 The arguments are the same as for the Popen constructor. Example:
131
Florent Xicluna4886d242010-03-08 13:27:26 +0000132 >>> retcode = subprocess.call(["ls", "-l"])
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000133
Peter Astrand454f7672005-01-01 09:36:35 +0000134check_call(*popenargs, **kwargs):
135 Run command with arguments. Wait for command to complete. If the
136 exit code was zero then return, otherwise raise
137 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000138 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000139
140 The arguments are the same as for the Popen constructor. Example:
141
Florent Xicluna4886d242010-03-08 13:27:26 +0000142 >>> subprocess.check_call(["ls", "-l"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000143 0
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000144
Brett Cannona23810f2008-05-26 19:04:21 +0000145getstatusoutput(cmd):
146 Return (status, output) of executing cmd in a shell.
147
Tim Golden60798142013-11-05 12:57:25 +0000148 Execute the string 'cmd' in a shell with 'check_output' and
149 return a 2-tuple (status, output). Universal newlines mode is used,
150 meaning that the result with be decoded to a string.
151
152 A trailing newline is stripped from the output.
153 The exit status for the command can be interpreted
154 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000155
Brett Cannona23810f2008-05-26 19:04:21 +0000156 >>> subprocess.getstatusoutput('ls /bin/ls')
157 (0, '/bin/ls')
158 >>> subprocess.getstatusoutput('cat /bin/junk')
159 (256, 'cat: /bin/junk: No such file or directory')
160 >>> subprocess.getstatusoutput('/bin/junk')
161 (256, 'sh: /bin/junk: not found')
162
163getoutput(cmd):
164 Return output (stdout or stderr) of executing cmd in a shell.
165
166 Like getstatusoutput(), except the exit status is ignored and the return
167 value is a string containing the command's output. Example:
168
Brett Cannona23810f2008-05-26 19:04:21 +0000169 >>> subprocess.getoutput('ls /bin/ls')
170 '/bin/ls'
171
Georg Brandlf9734072008-12-07 15:30:06 +0000172check_output(*popenargs, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700173 Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000174
Georg Brandl2708f3a2009-12-20 14:38:23 +0000175 If the exit code was non-zero it raises a CalledProcessError. The
176 CalledProcessError object will have the return code in the returncode
177 attribute and output in the output attribute.
Georg Brandlf9734072008-12-07 15:30:06 +0000178
Georg Brandl2708f3a2009-12-20 14:38:23 +0000179 The arguments are the same as for the Popen constructor. Example:
Georg Brandlf9734072008-12-07 15:30:06 +0000180
Georg Brandl2708f3a2009-12-20 14:38:23 +0000181 >>> output = subprocess.check_output(["ls", "-l", "/dev/null"])
Georg Brandlf9734072008-12-07 15:30:06 +0000182
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300183 There is an additional optional argument, "input", allowing you to
184 pass a string to the subprocess's stdin. If you use this argument
185 you may not also use the Popen constructor's "stdin" argument.
Brett Cannona23810f2008-05-26 19:04:21 +0000186
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000187Exceptions
188----------
189Exceptions raised in the child process, before the new program has
190started to execute, will be re-raised in the parent. Additionally,
191the exception object will have one extra attribute called
192'child_traceback', which is a string containing traceback information
Ezio Melotti30b9d5d2013-08-17 15:50:46 +0300193from the child's point of view.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000194
195The most common exception raised is OSError. This occurs, for
196example, when trying to execute a non-existent file. Applications
197should prepare for OSErrors.
198
199A ValueError will be raised if Popen is called with invalid arguments.
200
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400201Exceptions defined within this module inherit from SubprocessError.
202check_call() and check_output() will raise CalledProcessError if the
Gregory P. Smithb4039aa2011-03-14 14:16:20 -0400203called process returns a non-zero return code. TimeoutExpired
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400204be raised if a timeout was specified and expired.
Peter Astrand454f7672005-01-01 09:36:35 +0000205
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000206
207Security
208--------
209Unlike some other popen functions, this implementation will never call
210/bin/sh implicitly. This means that all characters, including shell
211metacharacters, can safely be passed to child processes.
212
213
214Popen objects
215=============
216Instances of the Popen class have the following methods:
217
218poll()
219 Check if child process has terminated. Returns returncode
220 attribute.
221
222wait()
223 Wait for child process to terminate. Returns returncode attribute.
224
225communicate(input=None)
226 Interact with process: Send data to stdin. Read data from stdout
227 and stderr, until end-of-file is reached. Wait for process to
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000228 terminate. The optional input argument should be a string to be
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000229 sent to the child process, or None, if no data should be sent to
230 the child.
Tim Peterse718f612004-10-12 21:51:32 +0000231
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000232 communicate() returns a tuple (stdout, stderr).
233
234 Note: The data read is buffered in memory, so do not use this
235 method if the data size is large or unlimited.
236
237The following attributes are also available:
238
239stdin
240 If the stdin argument is PIPE, this attribute is a file object
241 that provides input to the child process. Otherwise, it is None.
242
243stdout
244 If the stdout argument is PIPE, this attribute is a file object
245 that provides output from the child process. Otherwise, it is
246 None.
247
248stderr
249 If the stderr argument is PIPE, this attribute is file object that
250 provides error output from the child process. Otherwise, it is
251 None.
252
253pid
254 The process ID of the child process.
255
256returncode
257 The child return code. A None value indicates that the process
258 hasn't terminated yet. A negative value -N indicates that the
Gregory P. Smithf5604852010-12-13 06:45:02 +0000259 child was terminated by signal N (POSIX only).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000260
261
262Replacing older functions with the subprocess module
263====================================================
264In this section, "a ==> b" means that b can be used as a replacement
265for a.
266
267Note: All functions in this section fail (more or less) silently if
268the executed program cannot be found; this module raises an OSError
269exception.
270
271In the following examples, we assume that the subprocess module is
272imported with "from subprocess import *".
273
274
275Replacing /bin/sh shell backquote
276---------------------------------
277output=`mycmd myarg`
278==>
279output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
280
281
282Replacing shell pipe line
283-------------------------
284output=`dmesg | grep hda`
285==>
286p1 = Popen(["dmesg"], stdout=PIPE)
Peter Astrand6fdf3cb2004-11-30 18:06:42 +0000287p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000288output = p2.communicate()[0]
289
290
291Replacing os.system()
292---------------------
293sts = os.system("mycmd" + " myarg")
294==>
295p = Popen("mycmd" + " myarg", shell=True)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000296pid, sts = os.waitpid(p.pid, 0)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000297
298Note:
299
300* Calling the program through the shell is usually not required.
301
302* It's easier to look at the returncode attribute than the
303 exitstatus.
304
305A more real-world example would look like this:
306
307try:
308 retcode = call("mycmd" + " myarg", shell=True)
309 if retcode < 0:
Guido van Rossumc2f93dc2007-05-24 00:50:02 +0000310 print("Child was terminated by signal", -retcode, file=sys.stderr)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000311 else:
Guido van Rossumc2f93dc2007-05-24 00:50:02 +0000312 print("Child returned", retcode, file=sys.stderr)
313except OSError as e:
314 print("Execution failed:", e, file=sys.stderr)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000315
316
317Replacing os.spawn*
318-------------------
Tim Peterse718f612004-10-12 21:51:32 +0000319P_NOWAIT example:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000320
321pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
322==>
323pid = Popen(["/bin/mycmd", "myarg"]).pid
324
325
326P_WAIT example:
327
328retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
329==>
330retcode = call(["/bin/mycmd", "myarg"])
331
332
Tim Peterse718f612004-10-12 21:51:32 +0000333Vector example:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000334
335os.spawnvp(os.P_NOWAIT, path, args)
336==>
337Popen([path] + args[1:])
338
339
Tim Peterse718f612004-10-12 21:51:32 +0000340Environment example:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000341
342os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
343==>
344Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000345"""
346
347import sys
348mswindows = (sys.platform == "win32")
349
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000350import io
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000351import os
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400352import time
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000353import traceback
Christian Heimesfdab48e2008-01-20 09:06:41 +0000354import gc
Christian Heimesa342c012008-04-20 21:01:16 +0000355import signal
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000356import builtins
Gregory P. Smithd23047b2010-12-04 09:10:44 +0000357import warnings
Ross Lagerwall4f61b022011-04-05 15:34:00 +0200358import errno
Victor Stinner949d8c92012-05-30 13:30:32 +0200359try:
360 from time import monotonic as _time
Brett Cannoncd171c82013-07-04 17:43:24 -0400361except ImportError:
Victor Stinner949d8c92012-05-30 13:30:32 +0200362 from time import time as _time
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000363
Peter Astrand454f7672005-01-01 09:36:35 +0000364# Exception classes used by this module.
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400365class SubprocessError(Exception): pass
366
367
368class CalledProcessError(SubprocessError):
Georg Brandlf9734072008-12-07 15:30:06 +0000369 """This exception is raised when a process run by check_call() or
370 check_output() returns a non-zero exit status.
371 The exit status will be stored in the returncode attribute;
372 check_output() will also store the output in the output attribute.
373 """
374 def __init__(self, returncode, cmd, output=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000375 self.returncode = returncode
376 self.cmd = cmd
Georg Brandlf9734072008-12-07 15:30:06 +0000377 self.output = output
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000378 def __str__(self):
379 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
380
Peter Astrand454f7672005-01-01 09:36:35 +0000381
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400382class TimeoutExpired(SubprocessError):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400383 """This exception is raised when the timeout expires while waiting for a
384 child process.
385 """
Reid Kleckner2b228f02011-03-16 16:57:54 -0400386 def __init__(self, cmd, timeout, output=None):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400387 self.cmd = cmd
Reid Kleckner2b228f02011-03-16 16:57:54 -0400388 self.timeout = timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400389 self.output = output
390
391 def __str__(self):
392 return ("Command '%s' timed out after %s seconds" %
393 (self.cmd, self.timeout))
394
395
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000396if mswindows:
397 import threading
398 import msvcrt
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200399 import _winapi
Brian Curtin1ce6b582010-04-24 16:19:22 +0000400 class STARTUPINFO:
401 dwFlags = 0
402 hStdInput = None
403 hStdOutput = None
404 hStdError = None
405 wShowWindow = 0
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000406else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -0700407 import _posixsubprocess
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100408 import select
409 import selectors
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000410
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000411 # When select or poll has indicated that the file is writable,
412 # we can write up to _PIPE_BUF bytes without risk of blocking.
413 # POSIX defines PIPE_BUF as >= 512.
414 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
415
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100416 # poll/select have the advantage of not requiring any extra file
417 # descriptor, contrarily to epoll/kqueue (also, they require a single
418 # syscall).
419 if hasattr(selectors, 'PollSelector'):
420 _PopenSelector = selectors.PollSelector
421 else:
422 _PopenSelector = selectors.SelectSelector
423
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000424
Brett Cannona23810f2008-05-26 19:04:21 +0000425__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200426 "getoutput", "check_output", "CalledProcessError", "DEVNULL"]
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000427
Brian Curtin1ce6b582010-04-24 16:19:22 +0000428if mswindows:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200429 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
430 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
431 STD_ERROR_HANDLE, SW_HIDE,
432 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
Brian Curtin5d9deaa2011-04-29 16:24:07 -0500433
Brian Curtin08fd8d92011-04-29 16:11:30 -0500434 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
Brian Curtin8b8e7f42011-04-29 15:48:13 -0500435 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
436 "STD_ERROR_HANDLE", "SW_HIDE",
437 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200438
439 class Handle(int):
440 closed = False
441
442 def Close(self, CloseHandle=_winapi.CloseHandle):
443 if not self.closed:
444 self.closed = True
445 CloseHandle(self)
446
447 def Detach(self):
448 if not self.closed:
449 self.closed = True
450 return int(self)
451 raise ValueError("already closed")
452
453 def __repr__(self):
454 return "Handle(%d)" % int(self)
455
456 __del__ = Close
457 __str__ = __repr__
458
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000459try:
460 MAXFD = os.sysconf("SC_OPEN_MAX")
461except:
462 MAXFD = 256
463
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200464# This lists holds Popen instances for which the underlying process had not
465# exited at the time its __del__ method got called: those processes are wait()ed
466# for synchronously from _cleanup() when a new Popen object is created, to avoid
467# zombie processes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000468_active = []
469
470def _cleanup():
471 for inst in _active[:]:
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000472 res = inst._internal_poll(_deadstate=sys.maxsize)
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200473 if res is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000474 try:
475 _active.remove(inst)
476 except ValueError:
477 # This can happen if two threads create a new Popen instance.
478 # It's harmless that it was already removed, so ignore.
479 pass
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000480
481PIPE = -1
482STDOUT = -2
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200483DEVNULL = -3
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000484
485
Gregory P. Smitha59c59f2010-03-01 00:17:40 +0000486def _eintr_retry_call(func, *args):
487 while True:
488 try:
489 return func(*args)
Antoine Pitrou24d659d2011-10-23 23:49:42 +0200490 except InterruptedError:
491 continue
Gregory P. Smitha59c59f2010-03-01 00:17:40 +0000492
493
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200494# XXX This function is only used by multiprocessing and the test suite,
495# but it's here so that it can be imported when Python is compiled without
496# threads.
497
498def _args_from_interpreter_flags():
499 """Return a list of command-line arguments reproducing the current
500 settings in sys.flags and sys.warnoptions."""
501 flag_opt_map = {
502 'debug': 'd',
503 # 'inspect': 'i',
504 # 'interactive': 'i',
505 'optimize': 'O',
506 'dont_write_bytecode': 'B',
507 'no_user_site': 's',
508 'no_site': 'S',
509 'ignore_environment': 'E',
510 'verbose': 'v',
511 'bytes_warning': 'b',
512 'quiet': 'q',
513 'hash_randomization': 'R',
514 }
515 args = []
516 for flag, opt in flag_opt_map.items():
517 v = getattr(sys.flags, flag)
518 if v > 0:
Nick Coghlanac1a2482013-10-18 22:39:50 +1000519 if flag == 'hash_randomization':
520 v = 1 # Handle specification of an exact seed
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200521 args.append('-' + opt * v)
522 for opt in sys.warnoptions:
523 args.append('-W' + opt)
524 return args
525
526
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400527def call(*popenargs, timeout=None, **kwargs):
528 """Run command with arguments. Wait for command to complete or
529 timeout, then return the returncode attribute.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000530
531 The arguments are the same as for the Popen constructor. Example:
532
533 retcode = call(["ls", "-l"])
534 """
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200535 with Popen(*popenargs, **kwargs) as p:
536 try:
537 return p.wait(timeout=timeout)
538 except:
539 p.kill()
540 p.wait()
541 raise
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000542
543
Peter Astrand454f7672005-01-01 09:36:35 +0000544def check_call(*popenargs, **kwargs):
545 """Run command with arguments. Wait for command to complete. If
546 the exit code was zero then return, otherwise raise
547 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000548 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000549
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400550 The arguments are the same as for the call function. Example:
Peter Astrand454f7672005-01-01 09:36:35 +0000551
552 check_call(["ls", "-l"])
553 """
554 retcode = call(*popenargs, **kwargs)
Peter Astrand454f7672005-01-01 09:36:35 +0000555 if retcode:
Georg Brandlf9734072008-12-07 15:30:06 +0000556 cmd = kwargs.get("args")
557 if cmd is None:
558 cmd = popenargs[0]
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000559 raise CalledProcessError(retcode, cmd)
Georg Brandlf9734072008-12-07 15:30:06 +0000560 return 0
561
562
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400563def check_output(*popenargs, timeout=None, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700564 r"""Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000565
566 If the exit code was non-zero it raises a CalledProcessError. The
567 CalledProcessError object will have the return code in the returncode
568 attribute and output in the output attribute.
569
570 The arguments are the same as for the Popen constructor. Example:
571
572 >>> check_output(["ls", "-l", "/dev/null"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000573 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
Georg Brandlf9734072008-12-07 15:30:06 +0000574
575 The stdout argument is not allowed as it is used internally.
Georg Brandl127d4702009-12-28 08:10:38 +0000576 To capture standard error in the result, use stderr=STDOUT.
Georg Brandlf9734072008-12-07 15:30:06 +0000577
578 >>> check_output(["/bin/sh", "-c",
Georg Brandl2708f3a2009-12-20 14:38:23 +0000579 ... "ls -l non_existent_file ; exit 0"],
Georg Brandl127d4702009-12-28 08:10:38 +0000580 ... stderr=STDOUT)
Georg Brandl2708f3a2009-12-20 14:38:23 +0000581 b'ls: non_existent_file: No such file or directory\n'
Gregory P. Smith91110f52013-03-19 23:25:16 -0700582
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300583 There is an additional optional argument, "input", allowing you to
584 pass a string to the subprocess's stdin. If you use this argument
585 you may not also use the Popen constructor's "stdin" argument, as
586 it too will be used internally. Example:
587
588 >>> check_output(["sed", "-e", "s/foo/bar/"],
589 ... input=b"when in the course of fooman events\n")
590 b'when in the course of barman events\n'
591
Gregory P. Smith91110f52013-03-19 23:25:16 -0700592 If universal_newlines=True is passed, the return value will be a
593 string rather than bytes.
Georg Brandlf9734072008-12-07 15:30:06 +0000594 """
595 if 'stdout' in kwargs:
596 raise ValueError('stdout argument not allowed, it will be overridden.')
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300597 if 'input' in kwargs:
598 if 'stdin' in kwargs:
599 raise ValueError('stdin and input arguments may not both be used.')
600 inputdata = kwargs['input']
601 del kwargs['input']
602 kwargs['stdin'] = PIPE
603 else:
604 inputdata = None
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200605 with Popen(*popenargs, stdout=PIPE, **kwargs) as process:
606 try:
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300607 output, unused_err = process.communicate(inputdata, timeout=timeout)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200608 except TimeoutExpired:
609 process.kill()
610 output, unused_err = process.communicate()
611 raise TimeoutExpired(process.args, timeout, output=output)
612 except:
613 process.kill()
614 process.wait()
615 raise
616 retcode = process.poll()
617 if retcode:
618 raise CalledProcessError(retcode, process.args, output=output)
Georg Brandlf9734072008-12-07 15:30:06 +0000619 return output
Peter Astrand454f7672005-01-01 09:36:35 +0000620
621
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000622def list2cmdline(seq):
623 """
624 Translate a sequence of arguments into a command line
625 string, using the same rules as the MS C runtime:
626
627 1) Arguments are delimited by white space, which is either a
628 space or a tab.
629
630 2) A string surrounded by double quotation marks is
631 interpreted as a single argument, regardless of white space
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000632 contained within. A quoted string can be embedded in an
633 argument.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000634
635 3) A double quotation mark preceded by a backslash is
636 interpreted as a literal double quotation mark.
637
638 4) Backslashes are interpreted literally, unless they
639 immediately precede a double quotation mark.
640
641 5) If backslashes immediately precede a double quotation mark,
642 every pair of backslashes is interpreted as a literal
643 backslash. If the number of backslashes is odd, the last
644 backslash escapes the next double quotation mark as
645 described in rule 3.
646 """
647
648 # See
Eric Smith3c573af2009-11-09 15:23:15 +0000649 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
650 # or search http://msdn.microsoft.com for
651 # "Parsing C++ Command-Line Arguments"
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000652 result = []
653 needquote = False
654 for arg in seq:
655 bs_buf = []
656
657 # Add a space to separate this argument from the others
658 if result:
659 result.append(' ')
660
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000661 needquote = (" " in arg) or ("\t" in arg) or not arg
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000662 if needquote:
663 result.append('"')
664
665 for c in arg:
666 if c == '\\':
Tim Peterse718f612004-10-12 21:51:32 +0000667 # Don't know if we need to double yet.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000668 bs_buf.append(c)
669 elif c == '"':
Christian Heimesfdab48e2008-01-20 09:06:41 +0000670 # Double backslashes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000671 result.append('\\' * len(bs_buf)*2)
672 bs_buf = []
673 result.append('\\"')
674 else:
675 # Normal char
676 if bs_buf:
677 result.extend(bs_buf)
678 bs_buf = []
679 result.append(c)
680
Christian Heimesfdab48e2008-01-20 09:06:41 +0000681 # Add remaining backslashes, if any.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000682 if bs_buf:
683 result.extend(bs_buf)
684
685 if needquote:
Peter Astrand7e78ade2005-03-03 21:10:23 +0000686 result.extend(bs_buf)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000687 result.append('"')
688
689 return ''.join(result)
690
691
Brett Cannona23810f2008-05-26 19:04:21 +0000692# Various tools for executing commands and looking at their output and status.
693#
Brett Cannona23810f2008-05-26 19:04:21 +0000694
695def getstatusoutput(cmd):
Tim Golden60798142013-11-05 12:57:25 +0000696 """ Return (status, output) of executing cmd in a shell.
Brett Cannona23810f2008-05-26 19:04:21 +0000697
Tim Golden60798142013-11-05 12:57:25 +0000698 Execute the string 'cmd' in a shell with 'check_output' and
699 return a 2-tuple (status, output). Universal newlines mode is used,
700 meaning that the result with be decoded to a string.
701
702 A trailing newline is stripped from the output.
703 The exit status for the command can be interpreted
704 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000705
706 >>> import subprocess
707 >>> subprocess.getstatusoutput('ls /bin/ls')
708 (0, '/bin/ls')
709 >>> subprocess.getstatusoutput('cat /bin/junk')
710 (256, 'cat: /bin/junk: No such file or directory')
711 >>> subprocess.getstatusoutput('/bin/junk')
712 (256, 'sh: /bin/junk: not found')
713 """
Tim Goldene0041752013-11-03 12:53:17 +0000714 try:
715 data = check_output(cmd, shell=True, universal_newlines=True, stderr=STDOUT)
716 status = 0
717 except CalledProcessError as ex:
718 data = ex.output
719 status = ex.returncode
720 if data[-1:] == '\n':
721 data = data[:-1]
722 return status, data
Brett Cannona23810f2008-05-26 19:04:21 +0000723
724def getoutput(cmd):
725 """Return output (stdout or stderr) of executing cmd in a shell.
726
727 Like getstatusoutput(), except the exit status is ignored and the return
728 value is a string containing the command's output. Example:
729
730 >>> import subprocess
731 >>> subprocess.getoutput('ls /bin/ls')
732 '/bin/ls'
733 """
734 return getstatusoutput(cmd)[1]
735
736
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000737_PLATFORM_DEFAULT_CLOSE_FDS = object()
Gregory P. Smithf5604852010-12-13 06:45:02 +0000738
739
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000740class Popen(object):
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200741
742 _child_created = False # Set here since __del__ checks it
743
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700744 def __init__(self, args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000745 stdin=None, stdout=None, stderr=None,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000746 preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
747 shell=False, cwd=None, env=None, universal_newlines=False,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000748 startupinfo=None, creationflags=0,
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000749 restore_signals=True, start_new_session=False,
750 pass_fds=()):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000751 """Create new Popen instance."""
752 _cleanup()
753
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400754 self._input = None
755 self._communication_started = False
Guido van Rossum46a05a72007-06-07 21:56:45 +0000756 if bufsize is None:
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700757 bufsize = -1 # Restore default
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000758 if not isinstance(bufsize, int):
Peter Astrand738131d2004-11-30 21:04:45 +0000759 raise TypeError("bufsize must be an integer")
760
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000761 if mswindows:
Tim Peterse8374a52004-10-13 03:15:00 +0000762 if preexec_fn is not None:
763 raise ValueError("preexec_fn is not supported on Windows "
764 "platforms")
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000765 any_stdio_set = (stdin is not None or stdout is not None or
766 stderr is not None)
767 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
768 if any_stdio_set:
769 close_fds = False
770 else:
771 close_fds = True
772 elif close_fds and any_stdio_set:
773 raise ValueError(
774 "close_fds is not supported on Windows platforms"
775 " if you redirect stdin/stdout/stderr")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000776 else:
777 # POSIX
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000778 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
779 close_fds = True
780 if pass_fds and not close_fds:
781 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
782 close_fds = True
Tim Peterse8374a52004-10-13 03:15:00 +0000783 if startupinfo is not None:
784 raise ValueError("startupinfo is only supported on Windows "
785 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000786 if creationflags != 0:
Tim Peterse8374a52004-10-13 03:15:00 +0000787 raise ValueError("creationflags is only supported on Windows "
788 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000789
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400790 self.args = args
Tim Peterse718f612004-10-12 21:51:32 +0000791 self.stdin = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000792 self.stdout = None
793 self.stderr = None
794 self.pid = None
795 self.returncode = None
796 self.universal_newlines = universal_newlines
797
798 # Input and output objects. The general principle is like
799 # this:
800 #
801 # Parent Child
802 # ------ -----
803 # p2cwrite ---stdin---> p2cread
804 # c2pread <--stdout--- c2pwrite
805 # errread <--stderr--- errwrite
806 #
807 # On POSIX, the child objects are file descriptors. On
808 # Windows, these are Windows file handles. The parent objects
809 # are file descriptors on both platforms. The parent objects
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000810 # are -1 when not using PIPEs. The child objects are -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000811 # when not redirecting.
Tim Peterse718f612004-10-12 21:51:32 +0000812
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000813 (p2cread, p2cwrite,
814 c2pread, c2pwrite,
815 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
816
Antoine Pitrouc9982322011-01-04 19:07:07 +0000817 # We wrap OS handles *before* launching the child, otherwise a
818 # quickly terminating child could make our fds unwrappable
819 # (see #8458).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000820
Thomas Wouterscf297e42007-02-23 15:07:44 +0000821 if mswindows:
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000822 if p2cwrite != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000823 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000824 if c2pread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000825 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000826 if errread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000827 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
Thomas Wouterscf297e42007-02-23 15:07:44 +0000828
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000829 if p2cwrite != -1:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000830 self.stdin = io.open(p2cwrite, 'wb', bufsize)
Andrew Svetlov592df202012-08-15 17:36:15 +0300831 if universal_newlines:
Antoine Pitrouab85ff32011-07-23 22:03:45 +0200832 self.stdin = io.TextIOWrapper(self.stdin, write_through=True)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000833 if c2pread != -1:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000834 self.stdout = io.open(c2pread, 'rb', bufsize)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000835 if universal_newlines:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000836 self.stdout = io.TextIOWrapper(self.stdout)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000837 if errread != -1:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000838 self.stderr = io.open(errread, 'rb', bufsize)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000839 if universal_newlines:
Guido van Rossumfa0054a2007-05-24 04:05:35 +0000840 self.stderr = io.TextIOWrapper(self.stderr)
Tim Peterse718f612004-10-12 21:51:32 +0000841
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700842 self._closed_child_pipe_fds = False
Antoine Pitrouc9982322011-01-04 19:07:07 +0000843 try:
844 self._execute_child(args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300845 pass_fds, cwd, env,
Antoine Pitrouc9982322011-01-04 19:07:07 +0000846 startupinfo, creationflags, shell,
847 p2cread, p2cwrite,
848 c2pread, c2pwrite,
849 errread, errwrite,
850 restore_signals, start_new_session)
851 except:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800852 # Cleanup if the child failed starting.
853 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
Antoine Pitrouc9982322011-01-04 19:07:07 +0000854 try:
855 f.close()
Andrew Svetlov3438fa42012-12-17 23:35:18 +0200856 except OSError:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800857 pass # Ignore EBADF or other errors.
858
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700859 if not self._closed_child_pipe_fds:
860 to_close = []
861 if stdin == PIPE:
862 to_close.append(p2cread)
863 if stdout == PIPE:
864 to_close.append(c2pwrite)
865 if stderr == PIPE:
866 to_close.append(errwrite)
867 if hasattr(self, '_devnull'):
868 to_close.append(self._devnull)
869 for fd in to_close:
870 try:
871 os.close(fd)
Gregory P. Smith22ba31a2013-06-15 18:14:56 -0700872 except OSError:
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700873 pass
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800874
Antoine Pitrouc9982322011-01-04 19:07:07 +0000875 raise
876
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000877
Guido van Rossum98297ee2007-11-06 21:34:58 +0000878 def _translate_newlines(self, data, encoding):
Andrew Svetlov82860712012-08-19 22:13:41 +0300879 data = data.decode(encoding)
880 return data.replace("\r\n", "\n").replace("\r", "\n")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000881
Brian Curtin79cdb662010-12-03 02:46:02 +0000882 def __enter__(self):
883 return self
884
885 def __exit__(self, type, value, traceback):
886 if self.stdout:
887 self.stdout.close()
888 if self.stderr:
889 self.stderr.close()
890 if self.stdin:
891 self.stdin.close()
Gregory P. Smith6b657452011-05-11 21:42:08 -0700892 # Wait for the process to terminate, to avoid zombies.
893 self.wait()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000894
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200895 def __del__(self, _maxsize=sys.maxsize):
896 if not self._child_created:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000897 # We didn't get to successfully create a child process.
898 return
899 # In case the child hasn't been waited on, check if it's done.
Brett Cannon84df1e62010-05-14 00:33:40 +0000900 self._internal_poll(_deadstate=_maxsize)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000901 if self.returncode is None and _active is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000902 # Child is still running, keep us alive until we can wait on it.
903 _active.append(self)
904
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200905 def _get_devnull(self):
906 if not hasattr(self, '_devnull'):
907 self._devnull = os.open(os.devnull, os.O_RDWR)
908 return self._devnull
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000909
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400910 def communicate(self, input=None, timeout=None):
Peter Astrand23109f02005-03-03 20:28:59 +0000911 """Interact with process: Send data to stdin. Read data from
912 stdout and stderr, until end-of-file is reached. Wait for
Gregory P. Smitha454ef62011-05-22 22:29:49 -0700913 process to terminate. The optional input argument should be
914 bytes to be sent to the child process, or None, if no data
Peter Astrand23109f02005-03-03 20:28:59 +0000915 should be sent to the child.
Tim Peterseba28be2005-03-28 01:08:02 +0000916
Peter Astrand23109f02005-03-03 20:28:59 +0000917 communicate() returns a tuple (stdout, stderr)."""
918
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400919 if self._communication_started and input:
920 raise ValueError("Cannot send input after starting communication")
921
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400922 # Optimization: If we are not worried about timeouts, we haven't
923 # started communicating, and we have one or zero pipes, using select()
924 # or threads is unnecessary.
Victor Stinner7a8d0812011-04-05 13:13:08 +0200925 if (timeout is None and not self._communication_started and
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400926 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
Tim Peterseba28be2005-03-28 01:08:02 +0000927 stdout = None
928 stderr = None
Peter Astrand23109f02005-03-03 20:28:59 +0000929 if self.stdin:
930 if input:
Ross Lagerwall4f61b022011-04-05 15:34:00 +0200931 try:
932 self.stdin.write(input)
Andrew Svetlovf7a17b42012-12-25 16:47:37 +0200933 except OSError as e:
Ross Lagerwall4f61b022011-04-05 15:34:00 +0200934 if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
935 raise
Peter Astrand23109f02005-03-03 20:28:59 +0000936 self.stdin.close()
937 elif self.stdout:
Victor Stinner2cfb6f32011-07-05 14:00:56 +0200938 stdout = _eintr_retry_call(self.stdout.read)
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000939 self.stdout.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000940 elif self.stderr:
Victor Stinner2cfb6f32011-07-05 14:00:56 +0200941 stderr = _eintr_retry_call(self.stderr.read)
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000942 self.stderr.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000943 self.wait()
Victor Stinner7a8d0812011-04-05 13:13:08 +0200944 else:
945 if timeout is not None:
Victor Stinner949d8c92012-05-30 13:30:32 +0200946 endtime = _time() + timeout
Victor Stinner7a8d0812011-04-05 13:13:08 +0200947 else:
948 endtime = None
Tim Peterseba28be2005-03-28 01:08:02 +0000949
Victor Stinner7a8d0812011-04-05 13:13:08 +0200950 try:
951 stdout, stderr = self._communicate(input, endtime, timeout)
952 finally:
953 self._communication_started = True
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400954
Victor Stinner7a8d0812011-04-05 13:13:08 +0200955 sts = self.wait(timeout=self._remaining_time(endtime))
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400956
957 return (stdout, stderr)
Peter Astrand23109f02005-03-03 20:28:59 +0000958
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000959
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000960 def poll(self):
961 return self._internal_poll()
962
963
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400964 def _remaining_time(self, endtime):
965 """Convenience for _communicate when computing timeouts."""
966 if endtime is None:
967 return None
968 else:
Victor Stinner949d8c92012-05-30 13:30:32 +0200969 return endtime - _time()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400970
971
Reid Kleckner2b228f02011-03-16 16:57:54 -0400972 def _check_timeout(self, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400973 """Convenience for checking if a timeout has expired."""
974 if endtime is None:
975 return
Victor Stinner949d8c92012-05-30 13:30:32 +0200976 if _time() > endtime:
Reid Kleckner2b228f02011-03-16 16:57:54 -0400977 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400978
979
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000980 if mswindows:
981 #
982 # Windows methods
983 #
984 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +0000985 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000986 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
987 """
Peter Astrandd38ddf42005-02-10 08:32:50 +0000988 if stdin is None and stdout is None and stderr is None:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000989 return (-1, -1, -1, -1, -1, -1)
Tim Peterse718f612004-10-12 21:51:32 +0000990
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000991 p2cread, p2cwrite = -1, -1
992 c2pread, c2pwrite = -1, -1
993 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000994
Peter Astrandd38ddf42005-02-10 08:32:50 +0000995 if stdin is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200996 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000997 if p2cread is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200998 p2cread, _ = _winapi.CreatePipe(None, 0)
999 p2cread = Handle(p2cread)
1000 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001001 elif stdin == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001002 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1003 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001004 elif stdin == DEVNULL:
1005 p2cread = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001006 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001007 p2cread = msvcrt.get_osfhandle(stdin)
1008 else:
1009 # Assuming file-like object
1010 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1011 p2cread = self._make_inheritable(p2cread)
1012
Peter Astrandd38ddf42005-02-10 08:32:50 +00001013 if stdout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001014 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001015 if c2pwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001016 _, c2pwrite = _winapi.CreatePipe(None, 0)
1017 c2pwrite = Handle(c2pwrite)
1018 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001019 elif stdout == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001020 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1021 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001022 elif stdout == DEVNULL:
1023 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001024 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001025 c2pwrite = msvcrt.get_osfhandle(stdout)
1026 else:
1027 # Assuming file-like object
1028 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1029 c2pwrite = self._make_inheritable(c2pwrite)
1030
Peter Astrandd38ddf42005-02-10 08:32:50 +00001031 if stderr is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001032 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001033 if errwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001034 _, errwrite = _winapi.CreatePipe(None, 0)
1035 errwrite = Handle(errwrite)
1036 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001037 elif stderr == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001038 errread, errwrite = _winapi.CreatePipe(None, 0)
1039 errread, errwrite = Handle(errread), Handle(errwrite)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001040 elif stderr == STDOUT:
1041 errwrite = c2pwrite
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001042 elif stderr == DEVNULL:
1043 errwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001044 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001045 errwrite = msvcrt.get_osfhandle(stderr)
1046 else:
1047 # Assuming file-like object
1048 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1049 errwrite = self._make_inheritable(errwrite)
1050
1051 return (p2cread, p2cwrite,
1052 c2pread, c2pwrite,
1053 errread, errwrite)
1054
1055
1056 def _make_inheritable(self, handle):
1057 """Return a duplicate of handle, which is inheritable"""
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001058 h = _winapi.DuplicateHandle(
1059 _winapi.GetCurrentProcess(), handle,
1060 _winapi.GetCurrentProcess(), 0, 1,
1061 _winapi.DUPLICATE_SAME_ACCESS)
1062 return Handle(h)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001063
1064
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001065 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001066 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001067 startupinfo, creationflags, shell,
1068 p2cread, p2cwrite,
1069 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001070 errread, errwrite,
1071 unused_restore_signals, unused_start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001072 """Execute program (MS Windows version)"""
1073
Gregory P. Smith8edd99d2010-12-14 13:43:30 +00001074 assert not pass_fds, "pass_fds not supported on Windows."
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +00001075
Guido van Rossum3172c5d2007-10-16 18:12:55 +00001076 if not isinstance(args, str):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001077 args = list2cmdline(args)
1078
Peter Astrandc1d65362004-11-07 14:30:34 +00001079 # Process startup details
Peter Astrandd38ddf42005-02-10 08:32:50 +00001080 if startupinfo is None:
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001081 startupinfo = STARTUPINFO()
Victor Stinnerb3693582010-05-21 20:13:12 +00001082 if -1 not in (p2cread, c2pwrite, errwrite):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001083 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
Peter Astrandc1d65362004-11-07 14:30:34 +00001084 startupinfo.hStdInput = p2cread
1085 startupinfo.hStdOutput = c2pwrite
1086 startupinfo.hStdError = errwrite
1087
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001088 if shell:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001089 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1090 startupinfo.wShowWindow = _winapi.SW_HIDE
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001091 comspec = os.environ.get("COMSPEC", "cmd.exe")
Tim Golden126c2962010-08-11 14:20:40 +00001092 args = '{} /c "{}"'.format (comspec, args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001093
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001094 # Start the process
1095 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001096 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
Tim Peterse8374a52004-10-13 03:15:00 +00001097 # no special security
1098 None, None,
Guido van Rossume7ba4952007-06-06 23:52:48 +00001099 int(not close_fds),
Tim Peterse8374a52004-10-13 03:15:00 +00001100 creationflags,
1101 env,
1102 cwd,
1103 startupinfo)
Tim Goldenad537f22010-08-08 11:18:16 +00001104 finally:
1105 # Child is launched. Close the parent's copy of those pipe
1106 # handles that only the child should have open. You need
1107 # to make sure that no handles to the write end of the
1108 # output pipe are maintained in this process or else the
1109 # pipe will not close when the child process exits and the
1110 # ReadFile will hang.
1111 if p2cread != -1:
1112 p2cread.Close()
1113 if c2pwrite != -1:
1114 c2pwrite.Close()
1115 if errwrite != -1:
1116 errwrite.Close()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001117 if hasattr(self, '_devnull'):
1118 os.close(self._devnull)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001119
1120 # Retain the process handle, but close the thread handle
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001121 self._child_created = True
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001122 self._handle = Handle(hp)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001123 self.pid = pid
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001124 _winapi.CloseHandle(ht)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001125
Brett Cannon84df1e62010-05-14 00:33:40 +00001126 def _internal_poll(self, _deadstate=None,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001127 _WaitForSingleObject=_winapi.WaitForSingleObject,
1128 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1129 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001130 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001131 attribute.
1132
1133 This method is called by __del__, so it can only refer to objects
1134 in its local scope.
1135
1136 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001137 if self.returncode is None:
Brett Cannon84df1e62010-05-14 00:33:40 +00001138 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1139 self.returncode = _GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001140 return self.returncode
1141
1142
Reid Kleckner2b228f02011-03-16 16:57:54 -04001143 def wait(self, timeout=None, endtime=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001144 """Wait for child process to terminate. Returns returncode
1145 attribute."""
Reid Kleckner2b228f02011-03-16 16:57:54 -04001146 if endtime is not None:
1147 timeout = self._remaining_time(endtime)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001148 if timeout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001149 timeout_millis = _winapi.INFINITE
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001150 else:
Reid Kleckner91156ff2011-03-21 10:06:10 -07001151 timeout_millis = int(timeout * 1000)
Peter Astrandd38ddf42005-02-10 08:32:50 +00001152 if self.returncode is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001153 result = _winapi.WaitForSingleObject(self._handle,
1154 timeout_millis)
1155 if result == _winapi.WAIT_TIMEOUT:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001156 raise TimeoutExpired(self.args, timeout)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001157 self.returncode = _winapi.GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001158 return self.returncode
1159
1160
1161 def _readerthread(self, fh, buffer):
1162 buffer.append(fh.read())
Victor Stinner667d4b52010-12-25 22:40:32 +00001163 fh.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001164
1165
Reid Kleckner2b228f02011-03-16 16:57:54 -04001166 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001167 # Start reader threads feeding into a list hanging off of this
1168 # object, unless they've already been started.
1169 if self.stdout and not hasattr(self, "_stdout_buff"):
1170 self._stdout_buff = []
1171 self.stdout_thread = \
1172 threading.Thread(target=self._readerthread,
1173 args=(self.stdout, self._stdout_buff))
1174 self.stdout_thread.daemon = True
1175 self.stdout_thread.start()
1176 if self.stderr and not hasattr(self, "_stderr_buff"):
1177 self._stderr_buff = []
1178 self.stderr_thread = \
1179 threading.Thread(target=self._readerthread,
1180 args=(self.stderr, self._stderr_buff))
1181 self.stderr_thread.daemon = True
1182 self.stderr_thread.start()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001183
1184 if self.stdin:
Peter Astrandd38ddf42005-02-10 08:32:50 +00001185 if input is not None:
Ross Lagerwall4f61b022011-04-05 15:34:00 +02001186 try:
1187 self.stdin.write(input)
Andrew Svetlovf7a17b42012-12-25 16:47:37 +02001188 except OSError as e:
Victor Stinnerd5c8ce72014-02-18 22:00:53 +01001189 if e.errno == errno.EPIPE:
Victor Stinner5f47ac22014-02-18 22:06:35 +01001190 # communicate() should ignore pipe full error
Victor Stinnerd5c8ce72014-02-18 22:00:53 +01001191 pass
1192 elif (e.errno == errno.EINVAL
1193 and self.poll() is not None):
1194 # Issue #19612: stdin.write() fails with EINVAL
1195 # if the process already exited before the write
1196 pass
1197 else:
Ross Lagerwall4f61b022011-04-05 15:34:00 +02001198 raise
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001199 self.stdin.close()
1200
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001201 # Wait for the reader threads, or time out. If we time out, the
1202 # threads remain reading and the fds left open in case the user
1203 # calls communicate again.
1204 if self.stdout is not None:
1205 self.stdout_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001206 if self.stdout_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001207 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001208 if self.stderr is not None:
1209 self.stderr_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001210 if self.stderr_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001211 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001212
1213 # Collect the output from and close both pipes, now that we know
1214 # both have been read successfully.
1215 stdout = None
1216 stderr = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001217 if self.stdout:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001218 stdout = self._stdout_buff
1219 self.stdout.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001220 if self.stderr:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001221 stderr = self._stderr_buff
1222 self.stderr.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001223
1224 # All data exchanged. Translate lists into strings.
Peter Astrandd38ddf42005-02-10 08:32:50 +00001225 if stdout is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001226 stdout = stdout[0]
Peter Astrandd38ddf42005-02-10 08:32:50 +00001227 if stderr is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001228 stderr = stderr[0]
1229
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001230 return (stdout, stderr)
1231
Christian Heimesa342c012008-04-20 21:01:16 +00001232 def send_signal(self, sig):
1233 """Send a signal to the process
1234 """
1235 if sig == signal.SIGTERM:
1236 self.terminate()
Brian Curtineb24d742010-04-12 17:16:38 +00001237 elif sig == signal.CTRL_C_EVENT:
1238 os.kill(self.pid, signal.CTRL_C_EVENT)
1239 elif sig == signal.CTRL_BREAK_EVENT:
1240 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
Christian Heimesa342c012008-04-20 21:01:16 +00001241 else:
Brian Curtin19651362010-09-07 13:24:38 +00001242 raise ValueError("Unsupported signal: {}".format(sig))
Christian Heimesa342c012008-04-20 21:01:16 +00001243
1244 def terminate(self):
1245 """Terminates the process
1246 """
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001247 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001248 _winapi.TerminateProcess(self._handle, 1)
Antoine Pitroub69ef162012-03-11 19:33:29 +01001249 except PermissionError:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001250 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1251 # process already died.
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001252 rc = _winapi.GetExitCodeProcess(self._handle)
1253 if rc == _winapi.STILL_ACTIVE:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001254 raise
1255 self.returncode = rc
Christian Heimesa342c012008-04-20 21:01:16 +00001256
1257 kill = terminate
1258
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001259 else:
1260 #
1261 # POSIX methods
1262 #
1263 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001264 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001265 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1266 """
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001267 p2cread, p2cwrite = -1, -1
1268 c2pread, c2pwrite = -1, -1
1269 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001270
Peter Astrandd38ddf42005-02-10 08:32:50 +00001271 if stdin is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001272 pass
1273 elif stdin == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001274 p2cread, p2cwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001275 elif stdin == DEVNULL:
1276 p2cread = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001277 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001278 p2cread = stdin
1279 else:
1280 # Assuming file-like object
1281 p2cread = stdin.fileno()
1282
Peter Astrandd38ddf42005-02-10 08:32:50 +00001283 if stdout is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001284 pass
1285 elif stdout == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001286 c2pread, c2pwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001287 elif stdout == DEVNULL:
1288 c2pwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001289 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001290 c2pwrite = stdout
1291 else:
1292 # Assuming file-like object
Tim Peterse718f612004-10-12 21:51:32 +00001293 c2pwrite = stdout.fileno()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001294
Peter Astrandd38ddf42005-02-10 08:32:50 +00001295 if stderr is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001296 pass
1297 elif stderr == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001298 errread, errwrite = os.pipe()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001299 elif stderr == STDOUT:
1300 errwrite = c2pwrite
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001301 elif stderr == DEVNULL:
1302 errwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001303 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001304 errwrite = stderr
1305 else:
1306 # Assuming file-like object
1307 errwrite = stderr.fileno()
1308
1309 return (p2cread, p2cwrite,
1310 c2pread, c2pwrite,
1311 errread, errwrite)
1312
1313
Antoine Pitrou47f14ba2011-01-03 23:42:01 +00001314 def _close_fds(self, fds_to_keep):
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +00001315 start_fd = 3
Antoine Pitrou47f14ba2011-01-03 23:42:01 +00001316 for fd in sorted(fds_to_keep):
Gregory P. Smith8edd99d2010-12-14 13:43:30 +00001317 if fd >= start_fd:
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +00001318 os.closerange(start_fd, fd)
1319 start_fd = fd + 1
1320 if start_fd <= MAXFD:
1321 os.closerange(start_fd, MAXFD)
1322
1323
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001324 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001325 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001326 startupinfo, creationflags, shell,
1327 p2cread, p2cwrite,
1328 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001329 errread, errwrite,
1330 restore_signals, start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001331 """Execute program (POSIX version)"""
1332
Victor Stinner7b3b20a2011-03-03 12:54:05 +00001333 if isinstance(args, (str, bytes)):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001334 args = [args]
Thomas Wouters89f507f2006-12-13 04:49:30 +00001335 else:
1336 args = list(args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001337
1338 if shell:
1339 args = ["/bin/sh", "-c"] + args
Stefan Krah9542cc62010-07-19 14:20:53 +00001340 if executable:
1341 args[0] = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001342
Peter Astrandd38ddf42005-02-10 08:32:50 +00001343 if executable is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001344 executable = args[0]
Gregory P. Smith5591b022012-10-10 03:34:47 -07001345 orig_executable = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001346
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001347 # For transferring possible exec failure from child to parent.
1348 # Data format: "exception name:hex errno:description"
1349 # Pickle is not used; it is complex and involves memory allocation.
Victor Stinnerdaf45552013-08-28 00:53:59 +02001350 errpipe_read, errpipe_write = os.pipe()
Gregory P. Smith53dd8162013-12-01 16:03:24 -08001351 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1352 low_fds_to_close = []
1353 while errpipe_write < 3:
1354 low_fds_to_close.append(errpipe_write)
1355 errpipe_write = os.dup(errpipe_write)
1356 for low_fd in low_fds_to_close:
1357 os.close(low_fd)
Christian Heimesfdab48e2008-01-20 09:06:41 +00001358 try:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001359 try:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001360 # We must avoid complex work that could involve
1361 # malloc or free in the child process to avoid
1362 # potential deadlocks, thus we do all this here.
1363 # and pass it to fork_exec()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001364
Victor Stinner372b8382011-06-21 17:24:21 +02001365 if env is not None:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001366 env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
1367 for k, v in env.items()]
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001368 else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001369 env_list = None # Use execv instead of execve.
1370 executable = os.fsencode(executable)
1371 if os.path.dirname(executable):
1372 executable_list = (executable,)
1373 else:
1374 # This matches the behavior of os._execvpe().
1375 executable_list = tuple(
1376 os.path.join(os.fsencode(dir), executable)
1377 for dir in os.get_exec_path(env))
Gregory P. Smith361e30c2013-12-01 00:12:24 -08001378 fds_to_keep = set(pass_fds)
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001379 fds_to_keep.add(errpipe_write)
1380 self.pid = _posixsubprocess.fork_exec(
1381 args, executable_list,
1382 close_fds, sorted(fds_to_keep), cwd, env_list,
1383 p2cread, p2cwrite, c2pread, c2pwrite,
1384 errread, errwrite,
1385 errpipe_read, errpipe_write,
1386 restore_signals, start_new_session, preexec_fn)
Charles-François Natali558639f2011-08-18 19:11:29 +02001387 self._child_created = True
Facundo Batista10706e22009-06-19 20:34:30 +00001388 finally:
1389 # be sure the FD is closed no matter what
1390 os.close(errpipe_write)
1391
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001392 # self._devnull is not always defined.
1393 devnull_fd = getattr(self, '_devnull', None)
1394 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001395 os.close(p2cread)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001396 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001397 os.close(c2pwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001398 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001399 os.close(errwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001400 if devnull_fd is not None:
1401 os.close(devnull_fd)
1402 # Prevent a double close of these fds from __init__ on error.
1403 self._closed_child_pipe_fds = True
Facundo Batista10706e22009-06-19 20:34:30 +00001404
1405 # Wait for exec to fail or succeed; possibly raising an
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001406 # exception (limited in size)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001407 errpipe_data = bytearray()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001408 while True:
1409 part = _eintr_retry_call(os.read, errpipe_read, 50000)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001410 errpipe_data += part
1411 if not part or len(errpipe_data) > 50000:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001412 break
Facundo Batista10706e22009-06-19 20:34:30 +00001413 finally:
1414 # be sure the FD is closed no matter what
1415 os.close(errpipe_read)
1416
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001417 if errpipe_data:
Gregory P. Smithe85db2b2010-12-14 14:38:00 +00001418 try:
1419 _eintr_retry_call(os.waitpid, self.pid, 0)
1420 except OSError as e:
1421 if e.errno != errno.ECHILD:
1422 raise
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001423 try:
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001424 exception_name, hex_errno, err_msg = (
1425 errpipe_data.split(b':', 2))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001426 except ValueError:
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001427 exception_name = b'SubprocessError'
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001428 hex_errno = b'0'
Gregory P. Smith3aee2222012-11-11 00:04:13 -08001429 err_msg = (b'Bad exception data from child: ' +
1430 repr(errpipe_data))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001431 child_exception_type = getattr(
1432 builtins, exception_name.decode('ascii'),
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001433 SubprocessError)
Victor Stinner4d078042010-04-23 19:28:32 +00001434 err_msg = err_msg.decode(errors="surrogatepass")
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001435 if issubclass(child_exception_type, OSError) and hex_errno:
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001436 errno_num = int(hex_errno, 16)
Gregory P. Smith5591b022012-10-10 03:34:47 -07001437 child_exec_never_called = (err_msg == "noexec")
1438 if child_exec_never_called:
1439 err_msg = ""
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001440 if errno_num != 0:
1441 err_msg = os.strerror(errno_num)
1442 if errno_num == errno.ENOENT:
Gregory P. Smith5591b022012-10-10 03:34:47 -07001443 if child_exec_never_called:
1444 # The error must be from chdir(cwd).
1445 err_msg += ': ' + repr(cwd)
1446 else:
1447 err_msg += ': ' + repr(orig_executable)
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001448 raise child_exception_type(errno_num, err_msg)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001449 raise child_exception_type(err_msg)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001450
1451
Brett Cannon84df1e62010-05-14 00:33:40 +00001452 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
1453 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
1454 _WEXITSTATUS=os.WEXITSTATUS):
1455 # This method is called (indirectly) by __del__, so it cannot
Serhiy Storchaka72e77612014-02-10 19:20:22 +02001456 # refer to anything outside of its local scope.
Brett Cannon84df1e62010-05-14 00:33:40 +00001457 if _WIFSIGNALED(sts):
1458 self.returncode = -_WTERMSIG(sts)
1459 elif _WIFEXITED(sts):
1460 self.returncode = _WEXITSTATUS(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001461 else:
1462 # Should never happen
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001463 raise SubprocessError("Unknown child exit status!")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001464
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001465
Brett Cannon84df1e62010-05-14 00:33:40 +00001466 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
Andrew Svetlov1d960fe2012-12-24 20:08:53 +02001467 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001468 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001469 attribute.
1470
1471 This method is called by __del__, so it cannot reference anything
1472 outside of the local scope (nor can any methods it calls).
1473
1474 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001475 if self.returncode is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001476 try:
Brett Cannon84df1e62010-05-14 00:33:40 +00001477 pid, sts = _waitpid(self.pid, _WNOHANG)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001478 if pid == self.pid:
1479 self._handle_exitstatus(sts)
Andrew Svetlovad28c7f2012-12-18 22:02:39 +02001480 except OSError as e:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001481 if _deadstate is not None:
1482 self.returncode = _deadstate
Andrew Svetlov08bab072012-12-24 20:06:35 +02001483 elif e.errno == _ECHILD:
Gregory P. Smith39051712012-09-29 11:40:38 -07001484 # This happens if SIGCLD is set to be ignored or
1485 # waiting for child processes has otherwise been
1486 # disabled for our process. This child is dead, we
1487 # can't get the status.
1488 # http://bugs.python.org/issue15756
1489 self.returncode = 0
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001490 return self.returncode
1491
1492
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001493 def _try_wait(self, wait_flags):
1494 try:
1495 (pid, sts) = _eintr_retry_call(os.waitpid, self.pid, wait_flags)
1496 except OSError as e:
1497 if e.errno != errno.ECHILD:
1498 raise
1499 # This happens if SIGCLD is set to be ignored or waiting
1500 # for child processes has otherwise been disabled for our
1501 # process. This child is dead, we can't get the status.
1502 pid = self.pid
1503 sts = 0
1504 return (pid, sts)
1505
1506
1507 def wait(self, timeout=None, endtime=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001508 """Wait for child process to terminate. Returns returncode
1509 attribute."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001510 if self.returncode is not None:
1511 return self.returncode
Reid Kleckner2b228f02011-03-16 16:57:54 -04001512
1513 # endtime is preferred to timeout. timeout is only used for
1514 # printing.
1515 if endtime is not None or timeout is not None:
1516 if endtime is None:
Victor Stinner949d8c92012-05-30 13:30:32 +02001517 endtime = _time() + timeout
Reid Kleckner2b228f02011-03-16 16:57:54 -04001518 elif timeout is None:
1519 timeout = self._remaining_time(endtime)
1520
1521 if endtime is not None:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001522 # Enter a busy loop if we have a timeout. This busy loop was
1523 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1524 delay = 0.0005 # 500 us -> initial delay of 1 ms
1525 while True:
1526 (pid, sts) = self._try_wait(os.WNOHANG)
1527 assert pid == self.pid or pid == 0
1528 if pid == self.pid:
1529 self._handle_exitstatus(sts)
1530 break
1531 remaining = self._remaining_time(endtime)
1532 if remaining <= 0:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001533 raise TimeoutExpired(self.args, timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001534 delay = min(delay * 2, remaining, .05)
1535 time.sleep(delay)
Gregory P. Smithf328d792012-11-10 21:06:18 -08001536 else:
1537 while self.returncode is None:
1538 (pid, sts) = self._try_wait(0)
1539 # Check the pid and loop as waitpid has been known to return
1540 # 0 even without WNOHANG in odd situations. issue14396.
1541 if pid == self.pid:
1542 self._handle_exitstatus(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001543 return self.returncode
1544
1545
Reid Kleckner2b228f02011-03-16 16:57:54 -04001546 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001547 if self.stdin and not self._communication_started:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001548 # Flush stdio buffer. This might block, if the user has
1549 # been writing to .stdin in an uncontrolled fashion.
1550 self.stdin.flush()
1551 if not input:
1552 self.stdin.close()
1553
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001554 stdout = None
1555 stderr = None
1556
1557 # Only create this mapping if we haven't already.
1558 if not self._communication_started:
1559 self._fileobj2output = {}
1560 if self.stdout:
1561 self._fileobj2output[self.stdout] = []
1562 if self.stderr:
1563 self._fileobj2output[self.stderr] = []
1564
1565 if self.stdout:
1566 stdout = self._fileobj2output[self.stdout]
1567 if self.stderr:
1568 stderr = self._fileobj2output[self.stderr]
1569
1570 self._save_input(input)
1571
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001572 if self._input:
1573 input_view = memoryview(self._input)
1574
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001575 with _PopenSelector() as selector:
1576 if self.stdin and input:
1577 selector.register(self.stdin, selectors.EVENT_WRITE)
1578 if self.stdout:
1579 selector.register(self.stdout, selectors.EVENT_READ)
1580 if self.stderr:
1581 selector.register(self.stderr, selectors.EVENT_READ)
1582
1583 while selector.get_map():
1584 timeout = self._remaining_time(endtime)
1585 if timeout is not None and timeout < 0:
1586 raise TimeoutExpired(self.args, orig_timeout)
1587
1588 ready = selector.select(timeout)
1589 self._check_timeout(endtime, orig_timeout)
1590
1591 # XXX Rewrite these to use non-blocking I/O on the file
1592 # objects; they are no longer using C stdio!
1593
1594 for key, events in ready:
1595 if key.fileobj is self.stdin:
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001596 chunk = input_view[self._input_offset :
1597 self._input_offset + _PIPE_BUF]
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001598 try:
1599 self._input_offset += os.write(key.fd, chunk)
1600 except OSError as e:
1601 if e.errno == errno.EPIPE:
1602 selector.unregister(key.fileobj)
1603 key.fileobj.close()
1604 else:
1605 raise
1606 else:
1607 if self._input_offset >= len(self._input):
1608 selector.unregister(key.fileobj)
1609 key.fileobj.close()
1610 elif key.fileobj in (self.stdout, self.stderr):
Gregory P. Smith7b83b182013-12-08 10:58:28 -08001611 data = os.read(key.fd, 32768)
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001612 if not data:
1613 selector.unregister(key.fileobj)
1614 key.fileobj.close()
1615 self._fileobj2output[key.fileobj].append(data)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001616
1617 self.wait(timeout=self._remaining_time(endtime))
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001618
1619 # All data exchanged. Translate lists into strings.
1620 if stdout is not None:
1621 stdout = b''.join(stdout)
1622 if stderr is not None:
1623 stderr = b''.join(stderr)
1624
1625 # Translate newlines, if requested.
1626 # This also turns bytes into strings.
1627 if self.universal_newlines:
1628 if stdout is not None:
1629 stdout = self._translate_newlines(stdout,
1630 self.stdout.encoding)
1631 if stderr is not None:
1632 stderr = self._translate_newlines(stderr,
1633 self.stderr.encoding)
1634
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001635 return (stdout, stderr)
1636
1637
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001638 def _save_input(self, input):
1639 # This method is called from the _communicate_with_*() methods
1640 # so that if we time out while communicating, we can continue
1641 # sending input if we retry.
1642 if self.stdin and self._input is None:
1643 self._input_offset = 0
1644 self._input = input
1645 if self.universal_newlines and input is not None:
1646 self._input = self._input.encode(self.stdin.encoding)
1647
1648
Christian Heimesa342c012008-04-20 21:01:16 +00001649 def send_signal(self, sig):
1650 """Send a signal to the process
1651 """
1652 os.kill(self.pid, sig)
1653
1654 def terminate(self):
1655 """Terminate the process with SIGTERM
1656 """
1657 self.send_signal(signal.SIGTERM)
1658
1659 def kill(self):
1660 """Kill the process with SIGKILL
1661 """
1662 self.send_signal(signal.SIGKILL)