blob: 25e569849337e91eb53bf03decdc93442746dbdb [file] [log] [blame]
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001# subprocess - Subprocesses with accessible I/O streams
2#
Tim Peterse718f612004-10-12 21:51:32 +00003# For more information about this module, see PEP 324.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00004#
Peter Astrand3a708df2005-09-23 17:37:29 +00005# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00006#
Peter Astrand69bf13f2005-02-14 08:56:32 +00007# Licensed to PSF under a Contributor Agreement.
Peter Astrand3a708df2005-09-23 17:37:29 +00008# See http://www.python.org/2.4/license for licensing details.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00009
Martin Panter4afdca02016-10-25 22:20:48 +000010r"""Subprocesses with accessible I/O streams
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000011
Fredrik Lundh15aaacc2004-10-17 14:47:05 +000012This module allows you to spawn processes, connect to their
Martin Panter4afdca02016-10-25 22:20:48 +000013input/output/error pipes, and obtain their return codes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000014
Martin Panter4afdca02016-10-25 22:20:48 +000015For a complete description of this module see the Python documentation.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000016
Martin Panter4afdca02016-10-25 22:20:48 +000017Main API
18========
19run(...): Runs a command, waits for it to complete, then returns a
20 CompletedProcess instance.
21Popen(...): A class for flexibly executing a command in a new process
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000022
Martin Panter4afdca02016-10-25 22:20:48 +000023Constants
24---------
25DEVNULL: Special value that indicates that os.devnull should be used
26PIPE: Special value that indicates a pipe should be created
27STDOUT: Special value that indicates that stderr should go to stdout
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000028
29
Martin Panter4afdca02016-10-25 22:20:48 +000030Older API
31=========
32call(...): Runs a command, waits for it to complete, then returns
33 the return code.
34check_call(...): Same as call() but raises CalledProcessError()
35 if return code is not 0
36check_output(...): Same as check_call() but returns the contents of
37 stdout instead of a return code
38getoutput(...): Runs a command in the shell, waits for it to complete,
39 then returns the output
40getstatusoutput(...): Runs a command in the shell, waits for it to complete,
41 then returns a (status, output) tuple
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000042"""
43
44import sys
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -070045_mswindows = (sys.platform == "win32")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000046
Guido van Rossumfa0054a2007-05-24 04:05:35 +000047import io
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000048import os
Reid Kleckner31aa7dd2011-03-14 12:02:10 -040049import time
Christian Heimesa342c012008-04-20 21:01:16 +000050import signal
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000051import builtins
Gregory P. Smithd23047b2010-12-04 09:10:44 +000052import warnings
Ross Lagerwall4f61b022011-04-05 15:34:00 +020053import errno
Victor Stinnerae586492014-09-02 23:18:25 +020054from time import monotonic as _time
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000055
Peter Astrand454f7672005-01-01 09:36:35 +000056# Exception classes used by this module.
Gregory P. Smith54d412e2011-03-14 14:08:43 -040057class SubprocessError(Exception): pass
58
59
60class CalledProcessError(SubprocessError):
Martin Panter4afdca02016-10-25 22:20:48 +000061 """Raised when run() is called with check=True and the process
62 returns a non-zero exit status.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +000063
Martin Panter4afdca02016-10-25 22:20:48 +000064 Attributes:
65 cmd, returncode, stdout, stderr, output
Georg Brandlf9734072008-12-07 15:30:06 +000066 """
Gregory P. Smith6e730002015-04-14 16:14:25 -070067 def __init__(self, returncode, cmd, output=None, stderr=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +000068 self.returncode = returncode
69 self.cmd = cmd
Georg Brandlf9734072008-12-07 15:30:06 +000070 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -070071 self.stderr = stderr
72
Thomas Wouters0e3f5912006-08-11 14:57:12 +000073 def __str__(self):
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +000074 if self.returncode and self.returncode < 0:
75 try:
76 return "Command '%s' died with %r." % (
77 self.cmd, signal.Signals(-self.returncode))
78 except ValueError:
79 return "Command '%s' died with unknown signal %d." % (
80 self.cmd, -self.returncode)
81 else:
82 return "Command '%s' returned non-zero exit status %d." % (
83 self.cmd, self.returncode)
Thomas Wouters0e3f5912006-08-11 14:57:12 +000084
Gregory P. Smith6e730002015-04-14 16:14:25 -070085 @property
86 def stdout(self):
87 """Alias for output attribute, to match stderr"""
88 return self.output
89
90 @stdout.setter
91 def stdout(self, value):
92 # There's no obvious reason to set this, but allow it anyway so
93 # .stdout is a transparent alias for .output
94 self.output = value
95
Peter Astrand454f7672005-01-01 09:36:35 +000096
Gregory P. Smith54d412e2011-03-14 14:08:43 -040097class TimeoutExpired(SubprocessError):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -040098 """This exception is raised when the timeout expires while waiting for a
99 child process.
Martin Panter4afdca02016-10-25 22:20:48 +0000100
101 Attributes:
102 cmd, output, stdout, stderr, timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400103 """
Gregory P. Smith6e730002015-04-14 16:14:25 -0700104 def __init__(self, cmd, timeout, output=None, stderr=None):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400105 self.cmd = cmd
Reid Kleckner2b228f02011-03-16 16:57:54 -0400106 self.timeout = timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400107 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -0700108 self.stderr = stderr
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400109
110 def __str__(self):
111 return ("Command '%s' timed out after %s seconds" %
112 (self.cmd, self.timeout))
113
Gregory P. Smith6e730002015-04-14 16:14:25 -0700114 @property
115 def stdout(self):
116 return self.output
117
118 @stdout.setter
119 def stdout(self, value):
120 # There's no obvious reason to set this, but allow it anyway so
121 # .stdout is a transparent alias for .output
122 self.output = value
123
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400124
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700125if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000126 import threading
127 import msvcrt
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200128 import _winapi
Brian Curtin1ce6b582010-04-24 16:19:22 +0000129 class STARTUPINFO:
Subhendu Ghoshae160bb2017-02-25 20:29:05 +0530130 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None,
131 hStdError=None, wShowWindow=0):
132 self.dwFlags = dwFlags
133 self.hStdInput = hStdInput
134 self.hStdOutput = hStdOutput
135 self.hStdError = hStdError
136 self.wShowWindow = wShowWindow
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000137else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -0700138 import _posixsubprocess
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100139 import select
140 import selectors
Gregory P. Smithd65ba512014-04-23 00:27:17 -0700141 try:
142 import threading
143 except ImportError:
144 import dummy_threading as threading
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000145
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000146 # When select or poll has indicated that the file is writable,
147 # we can write up to _PIPE_BUF bytes without risk of blocking.
148 # POSIX defines PIPE_BUF as >= 512.
149 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
150
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100151 # poll/select have the advantage of not requiring any extra file
152 # descriptor, contrarily to epoll/kqueue (also, they require a single
153 # syscall).
154 if hasattr(selectors, 'PollSelector'):
155 _PopenSelector = selectors.PollSelector
156 else:
157 _PopenSelector = selectors.SelectSelector
158
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000159
Brett Cannona23810f2008-05-26 19:04:21 +0000160__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
Gregory P. Smith6e730002015-04-14 16:14:25 -0700161 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
162 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
Gregory P. Smithace55862015-04-07 15:57:54 -0700163 # NOTE: We intentionally exclude list2cmdline as it is
164 # considered an internal implementation detail. issue10838.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000165
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700166if _mswindows:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200167 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
168 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
169 STD_ERROR_HANDLE, SW_HIDE,
170 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
Brian Curtin5d9deaa2011-04-29 16:24:07 -0500171
Brian Curtin08fd8d92011-04-29 16:11:30 -0500172 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
Brian Curtin8b8e7f42011-04-29 15:48:13 -0500173 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
174 "STD_ERROR_HANDLE", "SW_HIDE",
Martin Panter528619b2016-04-16 23:42:37 +0000175 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
176 "STARTUPINFO"])
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200177
178 class Handle(int):
179 closed = False
180
181 def Close(self, CloseHandle=_winapi.CloseHandle):
182 if not self.closed:
183 self.closed = True
184 CloseHandle(self)
185
186 def Detach(self):
187 if not self.closed:
188 self.closed = True
189 return int(self)
190 raise ValueError("already closed")
191
192 def __repr__(self):
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300193 return "%s(%d)" % (self.__class__.__name__, int(self))
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200194
195 __del__ = Close
196 __str__ = __repr__
197
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000198
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200199# This lists holds Popen instances for which the underlying process had not
200# exited at the time its __del__ method got called: those processes are wait()ed
201# for synchronously from _cleanup() when a new Popen object is created, to avoid
202# zombie processes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000203_active = []
204
205def _cleanup():
206 for inst in _active[:]:
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000207 res = inst._internal_poll(_deadstate=sys.maxsize)
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200208 if res is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000209 try:
210 _active.remove(inst)
211 except ValueError:
212 # This can happen if two threads create a new Popen instance.
213 # It's harmless that it was already removed, so ignore.
214 pass
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000215
216PIPE = -1
217STDOUT = -2
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200218DEVNULL = -3
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000219
220
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200221# XXX This function is only used by multiprocessing and the test suite,
222# but it's here so that it can be imported when Python is compiled without
223# threads.
224
Victor Stinner9def2842016-01-18 12:15:08 +0100225def _optim_args_from_interpreter_flags():
226 """Return a list of command-line arguments reproducing the current
227 optimization settings in sys.flags."""
228 args = []
229 value = sys.flags.optimize
230 if value > 0:
231 args.append('-' + 'O' * value)
232 return args
233
234
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200235def _args_from_interpreter_flags():
236 """Return a list of command-line arguments reproducing the current
237 settings in sys.flags and sys.warnoptions."""
238 flag_opt_map = {
239 'debug': 'd',
240 # 'inspect': 'i',
241 # 'interactive': 'i',
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200242 'dont_write_bytecode': 'B',
243 'no_user_site': 's',
244 'no_site': 'S',
245 'ignore_environment': 'E',
246 'verbose': 'v',
247 'bytes_warning': 'b',
248 'quiet': 'q',
Victor Stinner9def2842016-01-18 12:15:08 +0100249 # -O is handled in _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200250 }
Victor Stinner9def2842016-01-18 12:15:08 +0100251 args = _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200252 for flag, opt in flag_opt_map.items():
253 v = getattr(sys.flags, flag)
254 if v > 0:
255 args.append('-' + opt * v)
256 for opt in sys.warnoptions:
257 args.append('-W' + opt)
258 return args
259
260
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400261def call(*popenargs, timeout=None, **kwargs):
262 """Run command with arguments. Wait for command to complete or
263 timeout, then return the returncode attribute.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000264
265 The arguments are the same as for the Popen constructor. Example:
266
267 retcode = call(["ls", "-l"])
268 """
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200269 with Popen(*popenargs, **kwargs) as p:
270 try:
271 return p.wait(timeout=timeout)
272 except:
273 p.kill()
274 p.wait()
275 raise
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000276
277
Peter Astrand454f7672005-01-01 09:36:35 +0000278def check_call(*popenargs, **kwargs):
279 """Run command with arguments. Wait for command to complete. If
280 the exit code was zero then return, otherwise raise
281 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000282 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000283
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400284 The arguments are the same as for the call function. Example:
Peter Astrand454f7672005-01-01 09:36:35 +0000285
286 check_call(["ls", "-l"])
287 """
288 retcode = call(*popenargs, **kwargs)
Peter Astrand454f7672005-01-01 09:36:35 +0000289 if retcode:
Georg Brandlf9734072008-12-07 15:30:06 +0000290 cmd = kwargs.get("args")
291 if cmd is None:
292 cmd = popenargs[0]
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000293 raise CalledProcessError(retcode, cmd)
Georg Brandlf9734072008-12-07 15:30:06 +0000294 return 0
295
296
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400297def check_output(*popenargs, timeout=None, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700298 r"""Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000299
300 If the exit code was non-zero it raises a CalledProcessError. The
301 CalledProcessError object will have the return code in the returncode
302 attribute and output in the output attribute.
303
304 The arguments are the same as for the Popen constructor. Example:
305
306 >>> check_output(["ls", "-l", "/dev/null"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000307 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
Georg Brandlf9734072008-12-07 15:30:06 +0000308
309 The stdout argument is not allowed as it is used internally.
Georg Brandl127d4702009-12-28 08:10:38 +0000310 To capture standard error in the result, use stderr=STDOUT.
Georg Brandlf9734072008-12-07 15:30:06 +0000311
312 >>> check_output(["/bin/sh", "-c",
Georg Brandl2708f3a2009-12-20 14:38:23 +0000313 ... "ls -l non_existent_file ; exit 0"],
Georg Brandl127d4702009-12-28 08:10:38 +0000314 ... stderr=STDOUT)
Georg Brandl2708f3a2009-12-20 14:38:23 +0000315 b'ls: non_existent_file: No such file or directory\n'
Gregory P. Smith91110f52013-03-19 23:25:16 -0700316
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300317 There is an additional optional argument, "input", allowing you to
318 pass a string to the subprocess's stdin. If you use this argument
319 you may not also use the Popen constructor's "stdin" argument, as
320 it too will be used internally. Example:
321
322 >>> check_output(["sed", "-e", "s/foo/bar/"],
323 ... input=b"when in the course of fooman events\n")
324 b'when in the course of barman events\n'
325
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400326 If universal_newlines=True is passed, the "input" argument must be a
327 string and the return value will be a string rather than bytes.
Georg Brandlf9734072008-12-07 15:30:06 +0000328 """
329 if 'stdout' in kwargs:
330 raise ValueError('stdout argument not allowed, it will be overridden.')
Gregory P. Smith6e730002015-04-14 16:14:25 -0700331
332 if 'input' in kwargs and kwargs['input'] is None:
333 # Explicitly passing input=None was previously equivalent to passing an
334 # empty string. That is maintained here for backwards compatibility.
335 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
336
337 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
338 **kwargs).stdout
339
340
341class CompletedProcess(object):
342 """A process that has finished running.
343
344 This is returned by run().
345
346 Attributes:
347 args: The list or str args passed to run().
348 returncode: The exit code of the process, negative for signals.
349 stdout: The standard output (None if not captured).
350 stderr: The standard error (None if not captured).
351 """
352 def __init__(self, args, returncode, stdout=None, stderr=None):
353 self.args = args
354 self.returncode = returncode
355 self.stdout = stdout
356 self.stderr = stderr
357
358 def __repr__(self):
359 args = ['args={!r}'.format(self.args),
360 'returncode={!r}'.format(self.returncode)]
361 if self.stdout is not None:
362 args.append('stdout={!r}'.format(self.stdout))
363 if self.stderr is not None:
364 args.append('stderr={!r}'.format(self.stderr))
365 return "{}({})".format(type(self).__name__, ', '.join(args))
366
367 def check_returncode(self):
368 """Raise CalledProcessError if the exit code is non-zero."""
369 if self.returncode:
370 raise CalledProcessError(self.returncode, self.args, self.stdout,
371 self.stderr)
372
373
374def run(*popenargs, input=None, timeout=None, check=False, **kwargs):
375 """Run command with arguments and return a CompletedProcess instance.
376
377 The returned instance will have attributes args, returncode, stdout and
378 stderr. By default, stdout and stderr are not captured, and those attributes
379 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
380
381 If check is True and the exit code was non-zero, it raises a
382 CalledProcessError. The CalledProcessError object will have the return code
383 in the returncode attribute, and output & stderr attributes if those streams
384 were captured.
385
386 If timeout is given, and the process takes too long, a TimeoutExpired
387 exception will be raised.
388
389 There is an optional argument "input", allowing you to
390 pass a string to the subprocess's stdin. If you use this argument
391 you may not also use the Popen constructor's "stdin" argument, as
392 it will be used internally.
393
394 The other arguments are the same as for the Popen constructor.
395
396 If universal_newlines=True is passed, the "input" argument must be a
397 string and stdout/stderr in the returned object will be strings rather than
398 bytes.
399 """
400 if input is not None:
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300401 if 'stdin' in kwargs:
402 raise ValueError('stdin and input arguments may not both be used.')
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300403 kwargs['stdin'] = PIPE
Gregory P. Smith6e730002015-04-14 16:14:25 -0700404
405 with Popen(*popenargs, **kwargs) as process:
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200406 try:
Gregory P. Smith6e730002015-04-14 16:14:25 -0700407 stdout, stderr = process.communicate(input, timeout=timeout)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200408 except TimeoutExpired:
409 process.kill()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700410 stdout, stderr = process.communicate()
411 raise TimeoutExpired(process.args, timeout, output=stdout,
412 stderr=stderr)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200413 except:
414 process.kill()
415 process.wait()
416 raise
417 retcode = process.poll()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700418 if check and retcode:
419 raise CalledProcessError(retcode, process.args,
420 output=stdout, stderr=stderr)
421 return CompletedProcess(process.args, retcode, stdout, stderr)
Peter Astrand454f7672005-01-01 09:36:35 +0000422
423
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000424def list2cmdline(seq):
425 """
426 Translate a sequence of arguments into a command line
427 string, using the same rules as the MS C runtime:
428
429 1) Arguments are delimited by white space, which is either a
430 space or a tab.
431
432 2) A string surrounded by double quotation marks is
433 interpreted as a single argument, regardless of white space
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000434 contained within. A quoted string can be embedded in an
435 argument.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000436
437 3) A double quotation mark preceded by a backslash is
438 interpreted as a literal double quotation mark.
439
440 4) Backslashes are interpreted literally, unless they
441 immediately precede a double quotation mark.
442
443 5) If backslashes immediately precede a double quotation mark,
444 every pair of backslashes is interpreted as a literal
445 backslash. If the number of backslashes is odd, the last
446 backslash escapes the next double quotation mark as
447 described in rule 3.
448 """
449
450 # See
Eric Smith3c573af2009-11-09 15:23:15 +0000451 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
452 # or search http://msdn.microsoft.com for
453 # "Parsing C++ Command-Line Arguments"
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000454 result = []
455 needquote = False
456 for arg in seq:
457 bs_buf = []
458
459 # Add a space to separate this argument from the others
460 if result:
461 result.append(' ')
462
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000463 needquote = (" " in arg) or ("\t" in arg) or not arg
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000464 if needquote:
465 result.append('"')
466
467 for c in arg:
468 if c == '\\':
Tim Peterse718f612004-10-12 21:51:32 +0000469 # Don't know if we need to double yet.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000470 bs_buf.append(c)
471 elif c == '"':
Christian Heimesfdab48e2008-01-20 09:06:41 +0000472 # Double backslashes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000473 result.append('\\' * len(bs_buf)*2)
474 bs_buf = []
475 result.append('\\"')
476 else:
477 # Normal char
478 if bs_buf:
479 result.extend(bs_buf)
480 bs_buf = []
481 result.append(c)
482
Christian Heimesfdab48e2008-01-20 09:06:41 +0000483 # Add remaining backslashes, if any.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000484 if bs_buf:
485 result.extend(bs_buf)
486
487 if needquote:
Peter Astrand7e78ade2005-03-03 21:10:23 +0000488 result.extend(bs_buf)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000489 result.append('"')
490
491 return ''.join(result)
492
493
Brett Cannona23810f2008-05-26 19:04:21 +0000494# Various tools for executing commands and looking at their output and status.
495#
Brett Cannona23810f2008-05-26 19:04:21 +0000496
497def getstatusoutput(cmd):
Tim Golden60798142013-11-05 12:57:25 +0000498 """ Return (status, output) of executing cmd in a shell.
Brett Cannona23810f2008-05-26 19:04:21 +0000499
Tim Golden60798142013-11-05 12:57:25 +0000500 Execute the string 'cmd' in a shell with 'check_output' and
Steve Dower050acae2016-09-06 20:16:17 -0700501 return a 2-tuple (status, output). The locale encoding is used
502 to decode the output and process newlines.
Tim Golden60798142013-11-05 12:57:25 +0000503
504 A trailing newline is stripped from the output.
505 The exit status for the command can be interpreted
506 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000507
508 >>> import subprocess
509 >>> subprocess.getstatusoutput('ls /bin/ls')
510 (0, '/bin/ls')
511 >>> subprocess.getstatusoutput('cat /bin/junk')
512 (256, 'cat: /bin/junk: No such file or directory')
513 >>> subprocess.getstatusoutput('/bin/junk')
514 (256, 'sh: /bin/junk: not found')
515 """
Tim Goldene0041752013-11-03 12:53:17 +0000516 try:
517 data = check_output(cmd, shell=True, universal_newlines=True, stderr=STDOUT)
518 status = 0
519 except CalledProcessError as ex:
520 data = ex.output
521 status = ex.returncode
522 if data[-1:] == '\n':
523 data = data[:-1]
524 return status, data
Brett Cannona23810f2008-05-26 19:04:21 +0000525
526def getoutput(cmd):
527 """Return output (stdout or stderr) of executing cmd in a shell.
528
529 Like getstatusoutput(), except the exit status is ignored and the return
530 value is a string containing the command's output. Example:
531
532 >>> import subprocess
533 >>> subprocess.getoutput('ls /bin/ls')
534 '/bin/ls'
535 """
536 return getstatusoutput(cmd)[1]
537
538
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000539_PLATFORM_DEFAULT_CLOSE_FDS = object()
Gregory P. Smithf5604852010-12-13 06:45:02 +0000540
541
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000542class Popen(object):
Martin Panter4afdca02016-10-25 22:20:48 +0000543 """ Execute a child program in a new process.
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200544
Martin Panter4afdca02016-10-25 22:20:48 +0000545 For a complete description of the arguments see the Python documentation.
546
547 Arguments:
548 args: A string, or a sequence of program arguments.
549
550 bufsize: supplied as the buffering argument to the open() function when
551 creating the stdin/stdout/stderr pipe file objects
552
553 executable: A replacement program to execute.
554
555 stdin, stdout and stderr: These specify the executed programs' standard
556 input, standard output and standard error file handles, respectively.
557
558 preexec_fn: (POSIX only) An object to be called in the child process
559 just before the child is executed.
560
561 close_fds: Controls closing or inheriting of file descriptors.
562
563 shell: If true, the command will be executed through the shell.
564
565 cwd: Sets the current directory before the child is executed.
566
567 env: Defines the environment variables for the new process.
568
569 universal_newlines: If true, use universal line endings for file
570 objects stdin, stdout and stderr.
571
572 startupinfo and creationflags (Windows only)
573
574 restore_signals (POSIX only)
575
576 start_new_session (POSIX only)
577
578 pass_fds (POSIX only)
579
Martin Panter3dca6242016-10-25 23:41:42 +0000580 encoding and errors: Text mode encoding and error handling to use for
581 file objects stdin, stdout and stderr.
582
Martin Panter4afdca02016-10-25 22:20:48 +0000583 Attributes:
584 stdin, stdout, stderr, pid, returncode
585 """
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200586 _child_created = False # Set here since __del__ checks it
587
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700588 def __init__(self, args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000589 stdin=None, stdout=None, stderr=None,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000590 preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
591 shell=False, cwd=None, env=None, universal_newlines=False,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000592 startupinfo=None, creationflags=0,
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000593 restore_signals=True, start_new_session=False,
Steve Dower050acae2016-09-06 20:16:17 -0700594 pass_fds=(), *, encoding=None, errors=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000595 """Create new Popen instance."""
596 _cleanup()
Gregory P. Smithd65ba512014-04-23 00:27:17 -0700597 # Held while anything is calling waitpid before returncode has been
598 # updated to prevent clobbering returncode if wait() or poll() are
599 # called from multiple threads at once. After acquiring the lock,
600 # code must re-check self.returncode to see if another thread just
601 # finished a waitpid() call.
602 self._waitpid_lock = threading.Lock()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000603
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400604 self._input = None
605 self._communication_started = False
Guido van Rossum46a05a72007-06-07 21:56:45 +0000606 if bufsize is None:
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700607 bufsize = -1 # Restore default
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000608 if not isinstance(bufsize, int):
Peter Astrand738131d2004-11-30 21:04:45 +0000609 raise TypeError("bufsize must be an integer")
610
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700611 if _mswindows:
Tim Peterse8374a52004-10-13 03:15:00 +0000612 if preexec_fn is not None:
613 raise ValueError("preexec_fn is not supported on Windows "
614 "platforms")
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000615 any_stdio_set = (stdin is not None or stdout is not None or
616 stderr is not None)
617 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
618 if any_stdio_set:
619 close_fds = False
620 else:
621 close_fds = True
622 elif close_fds and any_stdio_set:
623 raise ValueError(
624 "close_fds is not supported on Windows platforms"
625 " if you redirect stdin/stdout/stderr")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000626 else:
627 # POSIX
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000628 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
629 close_fds = True
630 if pass_fds and not close_fds:
631 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
632 close_fds = True
Tim Peterse8374a52004-10-13 03:15:00 +0000633 if startupinfo is not None:
634 raise ValueError("startupinfo is only supported on Windows "
635 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000636 if creationflags != 0:
Tim Peterse8374a52004-10-13 03:15:00 +0000637 raise ValueError("creationflags is only supported on Windows "
638 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000639
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400640 self.args = args
Tim Peterse718f612004-10-12 21:51:32 +0000641 self.stdin = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000642 self.stdout = None
643 self.stderr = None
644 self.pid = None
645 self.returncode = None
646 self.universal_newlines = universal_newlines
Steve Dower050acae2016-09-06 20:16:17 -0700647 self.encoding = encoding
648 self.errors = errors
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000649
650 # Input and output objects. The general principle is like
651 # this:
652 #
653 # Parent Child
654 # ------ -----
655 # p2cwrite ---stdin---> p2cread
656 # c2pread <--stdout--- c2pwrite
657 # errread <--stderr--- errwrite
658 #
659 # On POSIX, the child objects are file descriptors. On
660 # Windows, these are Windows file handles. The parent objects
661 # are file descriptors on both platforms. The parent objects
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000662 # are -1 when not using PIPEs. The child objects are -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000663 # when not redirecting.
Tim Peterse718f612004-10-12 21:51:32 +0000664
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000665 (p2cread, p2cwrite,
666 c2pread, c2pwrite,
667 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
668
Antoine Pitrouc9982322011-01-04 19:07:07 +0000669 # We wrap OS handles *before* launching the child, otherwise a
670 # quickly terminating child could make our fds unwrappable
671 # (see #8458).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000672
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700673 if _mswindows:
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000674 if p2cwrite != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000675 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000676 if c2pread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000677 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000678 if errread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000679 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
Thomas Wouterscf297e42007-02-23 15:07:44 +0000680
Steve Dower050acae2016-09-06 20:16:17 -0700681 text_mode = encoding or errors or universal_newlines
Tim Peterse718f612004-10-12 21:51:32 +0000682
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700683 self._closed_child_pipe_fds = False
Steve Dower050acae2016-09-06 20:16:17 -0700684
Antoine Pitrouc9982322011-01-04 19:07:07 +0000685 try:
Steve Dower050acae2016-09-06 20:16:17 -0700686 if p2cwrite != -1:
687 self.stdin = io.open(p2cwrite, 'wb', bufsize)
688 if text_mode:
689 self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
690 line_buffering=(bufsize == 1),
691 encoding=encoding, errors=errors)
692 if c2pread != -1:
693 self.stdout = io.open(c2pread, 'rb', bufsize)
694 if text_mode:
695 self.stdout = io.TextIOWrapper(self.stdout,
696 encoding=encoding, errors=errors)
697 if errread != -1:
698 self.stderr = io.open(errread, 'rb', bufsize)
699 if text_mode:
700 self.stderr = io.TextIOWrapper(self.stderr,
701 encoding=encoding, errors=errors)
702
Antoine Pitrouc9982322011-01-04 19:07:07 +0000703 self._execute_child(args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300704 pass_fds, cwd, env,
Antoine Pitrouc9982322011-01-04 19:07:07 +0000705 startupinfo, creationflags, shell,
706 p2cread, p2cwrite,
707 c2pread, c2pwrite,
708 errread, errwrite,
709 restore_signals, start_new_session)
710 except:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800711 # Cleanup if the child failed starting.
712 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
Antoine Pitrouc9982322011-01-04 19:07:07 +0000713 try:
714 f.close()
Andrew Svetlov3438fa42012-12-17 23:35:18 +0200715 except OSError:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800716 pass # Ignore EBADF or other errors.
717
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700718 if not self._closed_child_pipe_fds:
719 to_close = []
720 if stdin == PIPE:
721 to_close.append(p2cread)
722 if stdout == PIPE:
723 to_close.append(c2pwrite)
724 if stderr == PIPE:
725 to_close.append(errwrite)
726 if hasattr(self, '_devnull'):
727 to_close.append(self._devnull)
728 for fd in to_close:
729 try:
Segev Finer4d385172017-08-18 16:18:13 +0300730 if _mswindows and isinstance(fd, Handle):
731 fd.Close()
732 else:
733 os.close(fd)
Gregory P. Smith22ba31a2013-06-15 18:14:56 -0700734 except OSError:
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700735 pass
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800736
Antoine Pitrouc9982322011-01-04 19:07:07 +0000737 raise
738
Steve Dower050acae2016-09-06 20:16:17 -0700739 def _translate_newlines(self, data, encoding, errors):
740 data = data.decode(encoding, errors)
Andrew Svetlov82860712012-08-19 22:13:41 +0300741 return data.replace("\r\n", "\n").replace("\r", "\n")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000742
Brian Curtin79cdb662010-12-03 02:46:02 +0000743 def __enter__(self):
744 return self
745
746 def __exit__(self, type, value, traceback):
747 if self.stdout:
748 self.stdout.close()
749 if self.stderr:
750 self.stderr.close()
Serhiy Storchakaab900c22015-02-28 12:43:08 +0200751 try: # Flushing a BufferedWriter may raise an error
752 if self.stdin:
753 self.stdin.close()
754 finally:
755 # Wait for the process to terminate, to avoid zombies.
756 self.wait()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000757
Victor Stinner9505b032017-01-06 10:44:44 +0100758 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn):
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200759 if not self._child_created:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000760 # We didn't get to successfully create a child process.
761 return
Victor Stinner5a48e212016-05-20 12:11:15 +0200762 if self.returncode is None:
Victor Stinnerc206f1e2016-06-14 16:42:59 +0200763 # Not reading subprocess exit status creates a zombi process which
764 # is only destroyed at the parent python process exit
Victor Stinner9505b032017-01-06 10:44:44 +0100765 _warn("subprocess %s is still running" % self.pid,
766 ResourceWarning, source=self)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000767 # In case the child hasn't been waited on, check if it's done.
Brett Cannon84df1e62010-05-14 00:33:40 +0000768 self._internal_poll(_deadstate=_maxsize)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000769 if self.returncode is None and _active is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000770 # Child is still running, keep us alive until we can wait on it.
771 _active.append(self)
772
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200773 def _get_devnull(self):
774 if not hasattr(self, '_devnull'):
775 self._devnull = os.open(os.devnull, os.O_RDWR)
776 return self._devnull
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000777
Victor Stinnera5e881d2015-01-14 17:07:59 +0100778 def _stdin_write(self, input):
779 if input:
780 try:
781 self.stdin.write(input)
782 except BrokenPipeError:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000783 pass # communicate() must ignore broken pipe errors.
Victor Stinnerd52aa312017-06-08 17:30:39 +0200784 except OSError as exc:
785 if exc.errno == errno.EINVAL:
786 # bpo-19612, bpo-30418: On Windows, stdin.write() fails
787 # with EINVAL if the child process exited or if the child
788 # process is still running but closed the pipe.
Victor Stinnera5e881d2015-01-14 17:07:59 +0100789 pass
790 else:
791 raise
Victor Stinnerd52aa312017-06-08 17:30:39 +0200792
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000793 try:
794 self.stdin.close()
795 except BrokenPipeError:
796 pass # communicate() must ignore broken pipe errors.
Victor Stinnerd52aa312017-06-08 17:30:39 +0200797 except OSError as exc:
798 if exc.errno == errno.EINVAL:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000799 pass
800 else:
801 raise
Victor Stinnera5e881d2015-01-14 17:07:59 +0100802
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400803 def communicate(self, input=None, timeout=None):
Peter Astrand23109f02005-03-03 20:28:59 +0000804 """Interact with process: Send data to stdin. Read data from
805 stdout and stderr, until end-of-file is reached. Wait for
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400806 process to terminate.
Tim Peterseba28be2005-03-28 01:08:02 +0000807
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400808 The optional "input" argument should be data to be sent to the
809 child process (if self.universal_newlines is True, this should
810 be a string; if it is False, "input" should be bytes), or
811 None, if no data should be sent to the child.
812
813 communicate() returns a tuple (stdout, stderr). These will be
814 bytes or, if self.universal_newlines was True, a string.
815 """
Peter Astrand23109f02005-03-03 20:28:59 +0000816
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400817 if self._communication_started and input:
818 raise ValueError("Cannot send input after starting communication")
819
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400820 # Optimization: If we are not worried about timeouts, we haven't
821 # started communicating, and we have one or zero pipes, using select()
822 # or threads is unnecessary.
Victor Stinner7a8d0812011-04-05 13:13:08 +0200823 if (timeout is None and not self._communication_started and
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400824 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
Tim Peterseba28be2005-03-28 01:08:02 +0000825 stdout = None
826 stderr = None
Peter Astrand23109f02005-03-03 20:28:59 +0000827 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +0100828 self._stdin_write(input)
Peter Astrand23109f02005-03-03 20:28:59 +0000829 elif self.stdout:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +0000830 stdout = self.stdout.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000831 self.stdout.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000832 elif self.stderr:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +0000833 stderr = self.stderr.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000834 self.stderr.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000835 self.wait()
Victor Stinner7a8d0812011-04-05 13:13:08 +0200836 else:
837 if timeout is not None:
Victor Stinner949d8c92012-05-30 13:30:32 +0200838 endtime = _time() + timeout
Victor Stinner7a8d0812011-04-05 13:13:08 +0200839 else:
840 endtime = None
Tim Peterseba28be2005-03-28 01:08:02 +0000841
Victor Stinner7a8d0812011-04-05 13:13:08 +0200842 try:
843 stdout, stderr = self._communicate(input, endtime, timeout)
844 finally:
845 self._communication_started = True
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400846
Victor Stinner7a8d0812011-04-05 13:13:08 +0200847 sts = self.wait(timeout=self._remaining_time(endtime))
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400848
849 return (stdout, stderr)
Peter Astrand23109f02005-03-03 20:28:59 +0000850
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000851
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000852 def poll(self):
Martin Panter4afdca02016-10-25 22:20:48 +0000853 """Check if child process has terminated. Set and return returncode
854 attribute."""
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000855 return self._internal_poll()
856
857
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400858 def _remaining_time(self, endtime):
859 """Convenience for _communicate when computing timeouts."""
860 if endtime is None:
861 return None
862 else:
Victor Stinner949d8c92012-05-30 13:30:32 +0200863 return endtime - _time()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400864
865
Reid Kleckner2b228f02011-03-16 16:57:54 -0400866 def _check_timeout(self, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400867 """Convenience for checking if a timeout has expired."""
868 if endtime is None:
869 return
Victor Stinner949d8c92012-05-30 13:30:32 +0200870 if _time() > endtime:
Reid Kleckner2b228f02011-03-16 16:57:54 -0400871 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400872
873
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700874 if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000875 #
876 # Windows methods
877 #
878 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +0000879 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000880 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
881 """
Peter Astrandd38ddf42005-02-10 08:32:50 +0000882 if stdin is None and stdout is None and stderr is None:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000883 return (-1, -1, -1, -1, -1, -1)
Tim Peterse718f612004-10-12 21:51:32 +0000884
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000885 p2cread, p2cwrite = -1, -1
886 c2pread, c2pwrite = -1, -1
887 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000888
Peter Astrandd38ddf42005-02-10 08:32:50 +0000889 if stdin is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200890 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000891 if p2cread is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200892 p2cread, _ = _winapi.CreatePipe(None, 0)
893 p2cread = Handle(p2cread)
894 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000895 elif stdin == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200896 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
897 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200898 elif stdin == DEVNULL:
899 p2cread = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +0000900 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000901 p2cread = msvcrt.get_osfhandle(stdin)
902 else:
903 # Assuming file-like object
904 p2cread = msvcrt.get_osfhandle(stdin.fileno())
905 p2cread = self._make_inheritable(p2cread)
906
Peter Astrandd38ddf42005-02-10 08:32:50 +0000907 if stdout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200908 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000909 if c2pwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200910 _, c2pwrite = _winapi.CreatePipe(None, 0)
911 c2pwrite = Handle(c2pwrite)
912 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000913 elif stdout == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200914 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
915 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200916 elif stdout == DEVNULL:
917 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +0000918 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000919 c2pwrite = msvcrt.get_osfhandle(stdout)
920 else:
921 # Assuming file-like object
922 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
923 c2pwrite = self._make_inheritable(c2pwrite)
924
Peter Astrandd38ddf42005-02-10 08:32:50 +0000925 if stderr is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200926 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000927 if errwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200928 _, errwrite = _winapi.CreatePipe(None, 0)
929 errwrite = Handle(errwrite)
930 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000931 elif stderr == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200932 errread, errwrite = _winapi.CreatePipe(None, 0)
933 errread, errwrite = Handle(errread), Handle(errwrite)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000934 elif stderr == STDOUT:
935 errwrite = c2pwrite
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200936 elif stderr == DEVNULL:
937 errwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +0000938 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000939 errwrite = msvcrt.get_osfhandle(stderr)
940 else:
941 # Assuming file-like object
942 errwrite = msvcrt.get_osfhandle(stderr.fileno())
943 errwrite = self._make_inheritable(errwrite)
944
945 return (p2cread, p2cwrite,
946 c2pread, c2pwrite,
947 errread, errwrite)
948
949
950 def _make_inheritable(self, handle):
951 """Return a duplicate of handle, which is inheritable"""
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200952 h = _winapi.DuplicateHandle(
953 _winapi.GetCurrentProcess(), handle,
954 _winapi.GetCurrentProcess(), 0, 1,
955 _winapi.DUPLICATE_SAME_ACCESS)
956 return Handle(h)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000957
958
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000959 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300960 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000961 startupinfo, creationflags, shell,
962 p2cread, p2cwrite,
963 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000964 errread, errwrite,
965 unused_restore_signals, unused_start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000966 """Execute program (MS Windows version)"""
967
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000968 assert not pass_fds, "pass_fds not supported on Windows."
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000969
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000970 if not isinstance(args, str):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000971 args = list2cmdline(args)
972
Peter Astrandc1d65362004-11-07 14:30:34 +0000973 # Process startup details
Peter Astrandd38ddf42005-02-10 08:32:50 +0000974 if startupinfo is None:
Thomas Wouters73e5a5b2006-06-08 15:35:45 +0000975 startupinfo = STARTUPINFO()
Victor Stinnerb3693582010-05-21 20:13:12 +0000976 if -1 not in (p2cread, c2pwrite, errwrite):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200977 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
Peter Astrandc1d65362004-11-07 14:30:34 +0000978 startupinfo.hStdInput = p2cread
979 startupinfo.hStdOutput = c2pwrite
980 startupinfo.hStdError = errwrite
981
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000982 if shell:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200983 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
984 startupinfo.wShowWindow = _winapi.SW_HIDE
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000985 comspec = os.environ.get("COMSPEC", "cmd.exe")
Tim Golden126c2962010-08-11 14:20:40 +0000986 args = '{} /c "{}"'.format (comspec, args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000987
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000988 # Start the process
989 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200990 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
Tim Peterse8374a52004-10-13 03:15:00 +0000991 # no special security
992 None, None,
Guido van Rossume7ba4952007-06-06 23:52:48 +0000993 int(not close_fds),
Tim Peterse8374a52004-10-13 03:15:00 +0000994 creationflags,
995 env,
Benjamin Petersoncb90f262017-03-02 00:03:41 -0800996 os.fspath(cwd) if cwd is not None else None,
Tim Peterse8374a52004-10-13 03:15:00 +0000997 startupinfo)
Tim Goldenad537f22010-08-08 11:18:16 +0000998 finally:
999 # Child is launched. Close the parent's copy of those pipe
1000 # handles that only the child should have open. You need
1001 # to make sure that no handles to the write end of the
1002 # output pipe are maintained in this process or else the
1003 # pipe will not close when the child process exits and the
1004 # ReadFile will hang.
1005 if p2cread != -1:
1006 p2cread.Close()
1007 if c2pwrite != -1:
1008 c2pwrite.Close()
1009 if errwrite != -1:
1010 errwrite.Close()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001011 if hasattr(self, '_devnull'):
1012 os.close(self._devnull)
Segev Finer4d385172017-08-18 16:18:13 +03001013 # Prevent a double close of these handles/fds from __init__
1014 # on error.
1015 self._closed_child_pipe_fds = True
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001016
1017 # Retain the process handle, but close the thread handle
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001018 self._child_created = True
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001019 self._handle = Handle(hp)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001020 self.pid = pid
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001021 _winapi.CloseHandle(ht)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001022
Brett Cannon84df1e62010-05-14 00:33:40 +00001023 def _internal_poll(self, _deadstate=None,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001024 _WaitForSingleObject=_winapi.WaitForSingleObject,
1025 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1026 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001027 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001028 attribute.
1029
1030 This method is called by __del__, so it can only refer to objects
1031 in its local scope.
1032
1033 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001034 if self.returncode is None:
Brett Cannon84df1e62010-05-14 00:33:40 +00001035 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1036 self.returncode = _GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001037 return self.returncode
1038
1039
Gregory P. Smith82604e02016-11-20 16:31:07 -08001040 def wait(self, timeout=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001041 """Wait for child process to terminate. Returns returncode
1042 attribute."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001043 if timeout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001044 timeout_millis = _winapi.INFINITE
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001045 else:
Reid Kleckner91156ff2011-03-21 10:06:10 -07001046 timeout_millis = int(timeout * 1000)
Peter Astrandd38ddf42005-02-10 08:32:50 +00001047 if self.returncode is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001048 result = _winapi.WaitForSingleObject(self._handle,
1049 timeout_millis)
1050 if result == _winapi.WAIT_TIMEOUT:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001051 raise TimeoutExpired(self.args, timeout)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001052 self.returncode = _winapi.GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001053 return self.returncode
1054
1055
1056 def _readerthread(self, fh, buffer):
1057 buffer.append(fh.read())
Victor Stinner667d4b52010-12-25 22:40:32 +00001058 fh.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001059
1060
Reid Kleckner2b228f02011-03-16 16:57:54 -04001061 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001062 # Start reader threads feeding into a list hanging off of this
1063 # object, unless they've already been started.
1064 if self.stdout and not hasattr(self, "_stdout_buff"):
1065 self._stdout_buff = []
1066 self.stdout_thread = \
1067 threading.Thread(target=self._readerthread,
1068 args=(self.stdout, self._stdout_buff))
1069 self.stdout_thread.daemon = True
1070 self.stdout_thread.start()
1071 if self.stderr and not hasattr(self, "_stderr_buff"):
1072 self._stderr_buff = []
1073 self.stderr_thread = \
1074 threading.Thread(target=self._readerthread,
1075 args=(self.stderr, self._stderr_buff))
1076 self.stderr_thread.daemon = True
1077 self.stderr_thread.start()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001078
1079 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +01001080 self._stdin_write(input)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001081
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001082 # Wait for the reader threads, or time out. If we time out, the
1083 # threads remain reading and the fds left open in case the user
1084 # calls communicate again.
1085 if self.stdout is not None:
1086 self.stdout_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001087 if self.stdout_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001088 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001089 if self.stderr is not None:
1090 self.stderr_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001091 if self.stderr_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001092 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001093
1094 # Collect the output from and close both pipes, now that we know
1095 # both have been read successfully.
1096 stdout = None
1097 stderr = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001098 if self.stdout:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001099 stdout = self._stdout_buff
1100 self.stdout.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001101 if self.stderr:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001102 stderr = self._stderr_buff
1103 self.stderr.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001104
1105 # All data exchanged. Translate lists into strings.
Peter Astrandd38ddf42005-02-10 08:32:50 +00001106 if stdout is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001107 stdout = stdout[0]
Peter Astrandd38ddf42005-02-10 08:32:50 +00001108 if stderr is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001109 stderr = stderr[0]
1110
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001111 return (stdout, stderr)
1112
Christian Heimesa342c012008-04-20 21:01:16 +00001113 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001114 """Send a signal to the process."""
1115 # Don't signal a process that we know has already died.
1116 if self.returncode is not None:
1117 return
Christian Heimesa342c012008-04-20 21:01:16 +00001118 if sig == signal.SIGTERM:
1119 self.terminate()
Brian Curtineb24d742010-04-12 17:16:38 +00001120 elif sig == signal.CTRL_C_EVENT:
1121 os.kill(self.pid, signal.CTRL_C_EVENT)
1122 elif sig == signal.CTRL_BREAK_EVENT:
1123 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
Christian Heimesa342c012008-04-20 21:01:16 +00001124 else:
Brian Curtin19651362010-09-07 13:24:38 +00001125 raise ValueError("Unsupported signal: {}".format(sig))
Christian Heimesa342c012008-04-20 21:01:16 +00001126
1127 def terminate(self):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001128 """Terminates the process."""
1129 # Don't terminate a process that we know has already died.
1130 if self.returncode is not None:
1131 return
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001132 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001133 _winapi.TerminateProcess(self._handle, 1)
Antoine Pitroub69ef162012-03-11 19:33:29 +01001134 except PermissionError:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001135 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1136 # process already died.
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001137 rc = _winapi.GetExitCodeProcess(self._handle)
1138 if rc == _winapi.STILL_ACTIVE:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001139 raise
1140 self.returncode = rc
Christian Heimesa342c012008-04-20 21:01:16 +00001141
1142 kill = terminate
1143
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001144 else:
1145 #
1146 # POSIX methods
1147 #
1148 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001149 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001150 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1151 """
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001152 p2cread, p2cwrite = -1, -1
1153 c2pread, c2pwrite = -1, -1
1154 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001155
Peter Astrandd38ddf42005-02-10 08:32:50 +00001156 if stdin is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001157 pass
1158 elif stdin == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001159 p2cread, p2cwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001160 elif stdin == DEVNULL:
1161 p2cread = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001162 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001163 p2cread = stdin
1164 else:
1165 # Assuming file-like object
1166 p2cread = stdin.fileno()
1167
Peter Astrandd38ddf42005-02-10 08:32:50 +00001168 if stdout is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001169 pass
1170 elif stdout == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001171 c2pread, c2pwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001172 elif stdout == DEVNULL:
1173 c2pwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001174 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001175 c2pwrite = stdout
1176 else:
1177 # Assuming file-like object
Tim Peterse718f612004-10-12 21:51:32 +00001178 c2pwrite = stdout.fileno()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001179
Peter Astrandd38ddf42005-02-10 08:32:50 +00001180 if stderr is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001181 pass
1182 elif stderr == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001183 errread, errwrite = os.pipe()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001184 elif stderr == STDOUT:
Martin Panterc7635892016-05-13 01:54:44 +00001185 if c2pwrite != -1:
1186 errwrite = c2pwrite
1187 else: # child's stdout is not set, use parent's stdout
1188 errwrite = sys.__stdout__.fileno()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001189 elif stderr == DEVNULL:
1190 errwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001191 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001192 errwrite = stderr
1193 else:
1194 # Assuming file-like object
1195 errwrite = stderr.fileno()
1196
1197 return (p2cread, p2cwrite,
1198 c2pread, c2pwrite,
1199 errread, errwrite)
1200
1201
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001202 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001203 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001204 startupinfo, creationflags, shell,
1205 p2cread, p2cwrite,
1206 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001207 errread, errwrite,
1208 restore_signals, start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001209 """Execute program (POSIX version)"""
1210
Victor Stinner7b3b20a2011-03-03 12:54:05 +00001211 if isinstance(args, (str, bytes)):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001212 args = [args]
Thomas Wouters89f507f2006-12-13 04:49:30 +00001213 else:
1214 args = list(args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001215
1216 if shell:
Xavier de Gayeb35fc622016-12-13 16:32:21 +01001217 # On Android the default shell is at '/system/bin/sh'.
1218 unix_shell = ('/system/bin/sh' if
1219 hasattr(sys, 'getandroidapilevel') else '/bin/sh')
1220 args = [unix_shell, "-c"] + args
Stefan Krah9542cc62010-07-19 14:20:53 +00001221 if executable:
1222 args[0] = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001223
Peter Astrandd38ddf42005-02-10 08:32:50 +00001224 if executable is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001225 executable = args[0]
Gregory P. Smith5591b022012-10-10 03:34:47 -07001226 orig_executable = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001227
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001228 # For transferring possible exec failure from child to parent.
1229 # Data format: "exception name:hex errno:description"
1230 # Pickle is not used; it is complex and involves memory allocation.
Victor Stinnerdaf45552013-08-28 00:53:59 +02001231 errpipe_read, errpipe_write = os.pipe()
Gregory P. Smith53dd8162013-12-01 16:03:24 -08001232 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1233 low_fds_to_close = []
1234 while errpipe_write < 3:
1235 low_fds_to_close.append(errpipe_write)
1236 errpipe_write = os.dup(errpipe_write)
1237 for low_fd in low_fds_to_close:
1238 os.close(low_fd)
Christian Heimesfdab48e2008-01-20 09:06:41 +00001239 try:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001240 try:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001241 # We must avoid complex work that could involve
1242 # malloc or free in the child process to avoid
1243 # potential deadlocks, thus we do all this here.
1244 # and pass it to fork_exec()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001245
Victor Stinner372b8382011-06-21 17:24:21 +02001246 if env is not None:
Serhiy Storchakad174d242017-06-23 19:39:27 +03001247 env_list = []
1248 for k, v in env.items():
1249 k = os.fsencode(k)
1250 if b'=' in k:
1251 raise ValueError("illegal environment variable name")
1252 env_list.append(k + b'=' + os.fsencode(v))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001253 else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001254 env_list = None # Use execv instead of execve.
1255 executable = os.fsencode(executable)
1256 if os.path.dirname(executable):
1257 executable_list = (executable,)
1258 else:
1259 # This matches the behavior of os._execvpe().
1260 executable_list = tuple(
1261 os.path.join(os.fsencode(dir), executable)
1262 for dir in os.get_exec_path(env))
Gregory P. Smith361e30c2013-12-01 00:12:24 -08001263 fds_to_keep = set(pass_fds)
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001264 fds_to_keep.add(errpipe_write)
1265 self.pid = _posixsubprocess.fork_exec(
1266 args, executable_list,
Serhiy Storchaka66bffd12017-04-19 21:12:46 +03001267 close_fds, tuple(sorted(map(int, fds_to_keep))),
1268 cwd, env_list,
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001269 p2cread, p2cwrite, c2pread, c2pwrite,
1270 errread, errwrite,
1271 errpipe_read, errpipe_write,
1272 restore_signals, start_new_session, preexec_fn)
Charles-François Natali558639f2011-08-18 19:11:29 +02001273 self._child_created = True
Facundo Batista10706e22009-06-19 20:34:30 +00001274 finally:
1275 # be sure the FD is closed no matter what
1276 os.close(errpipe_write)
1277
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001278 # self._devnull is not always defined.
1279 devnull_fd = getattr(self, '_devnull', None)
1280 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001281 os.close(p2cread)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001282 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001283 os.close(c2pwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001284 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001285 os.close(errwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001286 if devnull_fd is not None:
1287 os.close(devnull_fd)
1288 # Prevent a double close of these fds from __init__ on error.
1289 self._closed_child_pipe_fds = True
Facundo Batista10706e22009-06-19 20:34:30 +00001290
1291 # Wait for exec to fail or succeed; possibly raising an
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001292 # exception (limited in size)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001293 errpipe_data = bytearray()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001294 while True:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001295 part = os.read(errpipe_read, 50000)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001296 errpipe_data += part
1297 if not part or len(errpipe_data) > 50000:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001298 break
Facundo Batista10706e22009-06-19 20:34:30 +00001299 finally:
1300 # be sure the FD is closed no matter what
1301 os.close(errpipe_read)
1302
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001303 if errpipe_data:
Gregory P. Smithe85db2b2010-12-14 14:38:00 +00001304 try:
Victor Stinnera58e2c52016-05-20 12:08:12 +02001305 pid, sts = os.waitpid(self.pid, 0)
1306 if pid == self.pid:
1307 self._handle_exitstatus(sts)
1308 else:
1309 self.returncode = sys.maxsize
Victor Stinnera5e881d2015-01-14 17:07:59 +01001310 except ChildProcessError:
1311 pass
Victor Stinnera58e2c52016-05-20 12:08:12 +02001312
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001313 try:
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001314 exception_name, hex_errno, err_msg = (
1315 errpipe_data.split(b':', 2))
Ammar Askar3fc499b2017-09-06 02:41:30 -04001316 # The encoding here should match the encoding
1317 # written in by the subprocess implementations
1318 # like _posixsubprocess
1319 err_msg = err_msg.decode()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001320 except ValueError:
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001321 exception_name = b'SubprocessError'
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001322 hex_errno = b'0'
Ammar Askar3fc499b2017-09-06 02:41:30 -04001323 err_msg = 'Bad exception data from child: {!r}'.format(
1324 bytes(errpipe_data))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001325 child_exception_type = getattr(
1326 builtins, exception_name.decode('ascii'),
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001327 SubprocessError)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001328 if issubclass(child_exception_type, OSError) and hex_errno:
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001329 errno_num = int(hex_errno, 16)
Gregory P. Smith5591b022012-10-10 03:34:47 -07001330 child_exec_never_called = (err_msg == "noexec")
1331 if child_exec_never_called:
1332 err_msg = ""
Gregory P. Smith8621bb52017-08-24 14:58:25 -07001333 # The error must be from chdir(cwd).
1334 err_filename = cwd
1335 else:
1336 err_filename = orig_executable
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001337 if errno_num != 0:
1338 err_msg = os.strerror(errno_num)
1339 if errno_num == errno.ENOENT:
Gregory P. Smith8621bb52017-08-24 14:58:25 -07001340 err_msg += ': ' + repr(err_filename)
1341 raise child_exception_type(errno_num, err_msg, err_filename)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001342 raise child_exception_type(err_msg)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001343
1344
Brett Cannon84df1e62010-05-14 00:33:40 +00001345 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
1346 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
Gregory P. Smith50e16e32017-01-22 17:28:38 -08001347 _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED,
1348 _WSTOPSIG=os.WSTOPSIG):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001349 """All callers to this function MUST hold self._waitpid_lock."""
Brett Cannon84df1e62010-05-14 00:33:40 +00001350 # This method is called (indirectly) by __del__, so it cannot
Serhiy Storchaka72e77612014-02-10 19:20:22 +02001351 # refer to anything outside of its local scope.
Brett Cannon84df1e62010-05-14 00:33:40 +00001352 if _WIFSIGNALED(sts):
1353 self.returncode = -_WTERMSIG(sts)
1354 elif _WIFEXITED(sts):
1355 self.returncode = _WEXITSTATUS(sts)
Gregory P. Smith50e16e32017-01-22 17:28:38 -08001356 elif _WIFSTOPPED(sts):
1357 self.returncode = -_WSTOPSIG(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001358 else:
1359 # Should never happen
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001360 raise SubprocessError("Unknown child exit status!")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001361
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001362
Brett Cannon84df1e62010-05-14 00:33:40 +00001363 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
Andrew Svetlov1d960fe2012-12-24 20:08:53 +02001364 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001365 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001366 attribute.
1367
1368 This method is called by __del__, so it cannot reference anything
1369 outside of the local scope (nor can any methods it calls).
1370
1371 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001372 if self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001373 if not self._waitpid_lock.acquire(False):
1374 # Something else is busy calling waitpid. Don't allow two
1375 # at once. We know nothing yet.
1376 return None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001377 try:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001378 if self.returncode is not None:
1379 return self.returncode # Another thread waited.
Brett Cannon84df1e62010-05-14 00:33:40 +00001380 pid, sts = _waitpid(self.pid, _WNOHANG)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001381 if pid == self.pid:
1382 self._handle_exitstatus(sts)
Andrew Svetlovad28c7f2012-12-18 22:02:39 +02001383 except OSError as e:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001384 if _deadstate is not None:
1385 self.returncode = _deadstate
Andrew Svetlov08bab072012-12-24 20:06:35 +02001386 elif e.errno == _ECHILD:
Gregory P. Smith39051712012-09-29 11:40:38 -07001387 # This happens if SIGCLD is set to be ignored or
1388 # waiting for child processes has otherwise been
1389 # disabled for our process. This child is dead, we
1390 # can't get the status.
1391 # http://bugs.python.org/issue15756
1392 self.returncode = 0
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001393 finally:
1394 self._waitpid_lock.release()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001395 return self.returncode
1396
1397
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001398 def _try_wait(self, wait_flags):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001399 """All callers to this function MUST hold self._waitpid_lock."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001400 try:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001401 (pid, sts) = os.waitpid(self.pid, wait_flags)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001402 except ChildProcessError:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001403 # This happens if SIGCLD is set to be ignored or waiting
1404 # for child processes has otherwise been disabled for our
1405 # process. This child is dead, we can't get the status.
1406 pid = self.pid
1407 sts = 0
1408 return (pid, sts)
1409
1410
Gregory P. Smith82604e02016-11-20 16:31:07 -08001411 def wait(self, timeout=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001412 """Wait for child process to terminate. Returns returncode
1413 attribute."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001414 if self.returncode is not None:
1415 return self.returncode
Reid Kleckner2b228f02011-03-16 16:57:54 -04001416
Gregory P. Smith82604e02016-11-20 16:31:07 -08001417 if timeout is not None:
1418 endtime = _time() + timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001419 # Enter a busy loop if we have a timeout. This busy loop was
1420 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1421 delay = 0.0005 # 500 us -> initial delay of 1 ms
1422 while True:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001423 if self._waitpid_lock.acquire(False):
1424 try:
1425 if self.returncode is not None:
1426 break # Another thread waited.
1427 (pid, sts) = self._try_wait(os.WNOHANG)
1428 assert pid == self.pid or pid == 0
1429 if pid == self.pid:
1430 self._handle_exitstatus(sts)
1431 break
1432 finally:
1433 self._waitpid_lock.release()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001434 remaining = self._remaining_time(endtime)
1435 if remaining <= 0:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001436 raise TimeoutExpired(self.args, timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001437 delay = min(delay * 2, remaining, .05)
1438 time.sleep(delay)
Gregory P. Smithf328d792012-11-10 21:06:18 -08001439 else:
1440 while self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001441 with self._waitpid_lock:
1442 if self.returncode is not None:
1443 break # Another thread waited.
1444 (pid, sts) = self._try_wait(0)
1445 # Check the pid and loop as waitpid has been known to
1446 # return 0 even without WNOHANG in odd situations.
1447 # http://bugs.python.org/issue14396.
1448 if pid == self.pid:
1449 self._handle_exitstatus(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001450 return self.returncode
1451
1452
Reid Kleckner2b228f02011-03-16 16:57:54 -04001453 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001454 if self.stdin and not self._communication_started:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001455 # Flush stdio buffer. This might block, if the user has
1456 # been writing to .stdin in an uncontrolled fashion.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001457 try:
1458 self.stdin.flush()
1459 except BrokenPipeError:
1460 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001461 if not input:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001462 try:
1463 self.stdin.close()
1464 except BrokenPipeError:
1465 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001466
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001467 stdout = None
1468 stderr = None
1469
1470 # Only create this mapping if we haven't already.
1471 if not self._communication_started:
1472 self._fileobj2output = {}
1473 if self.stdout:
1474 self._fileobj2output[self.stdout] = []
1475 if self.stderr:
1476 self._fileobj2output[self.stderr] = []
1477
1478 if self.stdout:
1479 stdout = self._fileobj2output[self.stdout]
1480 if self.stderr:
1481 stderr = self._fileobj2output[self.stderr]
1482
1483 self._save_input(input)
1484
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001485 if self._input:
1486 input_view = memoryview(self._input)
1487
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001488 with _PopenSelector() as selector:
1489 if self.stdin and input:
1490 selector.register(self.stdin, selectors.EVENT_WRITE)
1491 if self.stdout:
1492 selector.register(self.stdout, selectors.EVENT_READ)
1493 if self.stderr:
1494 selector.register(self.stderr, selectors.EVENT_READ)
1495
1496 while selector.get_map():
1497 timeout = self._remaining_time(endtime)
1498 if timeout is not None and timeout < 0:
1499 raise TimeoutExpired(self.args, orig_timeout)
1500
1501 ready = selector.select(timeout)
1502 self._check_timeout(endtime, orig_timeout)
1503
1504 # XXX Rewrite these to use non-blocking I/O on the file
1505 # objects; they are no longer using C stdio!
1506
1507 for key, events in ready:
1508 if key.fileobj is self.stdin:
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001509 chunk = input_view[self._input_offset :
1510 self._input_offset + _PIPE_BUF]
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001511 try:
1512 self._input_offset += os.write(key.fd, chunk)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001513 except BrokenPipeError:
1514 selector.unregister(key.fileobj)
1515 key.fileobj.close()
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001516 else:
1517 if self._input_offset >= len(self._input):
1518 selector.unregister(key.fileobj)
1519 key.fileobj.close()
1520 elif key.fileobj in (self.stdout, self.stderr):
Gregory P. Smith7b83b182013-12-08 10:58:28 -08001521 data = os.read(key.fd, 32768)
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001522 if not data:
1523 selector.unregister(key.fileobj)
1524 key.fileobj.close()
1525 self._fileobj2output[key.fileobj].append(data)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001526
1527 self.wait(timeout=self._remaining_time(endtime))
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001528
1529 # All data exchanged. Translate lists into strings.
1530 if stdout is not None:
1531 stdout = b''.join(stdout)
1532 if stderr is not None:
1533 stderr = b''.join(stderr)
1534
1535 # Translate newlines, if requested.
1536 # This also turns bytes into strings.
Steve Dower050acae2016-09-06 20:16:17 -07001537 if self.encoding or self.errors or self.universal_newlines:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001538 if stdout is not None:
1539 stdout = self._translate_newlines(stdout,
Steve Dower050acae2016-09-06 20:16:17 -07001540 self.stdout.encoding,
1541 self.stdout.errors)
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001542 if stderr is not None:
1543 stderr = self._translate_newlines(stderr,
Steve Dower050acae2016-09-06 20:16:17 -07001544 self.stderr.encoding,
1545 self.stderr.errors)
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001546
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001547 return (stdout, stderr)
1548
1549
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001550 def _save_input(self, input):
1551 # This method is called from the _communicate_with_*() methods
1552 # so that if we time out while communicating, we can continue
1553 # sending input if we retry.
1554 if self.stdin and self._input is None:
1555 self._input_offset = 0
1556 self._input = input
Steve Dower050acae2016-09-06 20:16:17 -07001557 if input is not None and (
1558 self.encoding or self.errors or self.universal_newlines):
1559 self._input = self._input.encode(self.stdin.encoding,
1560 self.stdin.errors)
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001561
1562
Christian Heimesa342c012008-04-20 21:01:16 +00001563 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001564 """Send a signal to the process."""
1565 # Skip signalling a process that we know has already died.
1566 if self.returncode is None:
1567 os.kill(self.pid, sig)
Christian Heimesa342c012008-04-20 21:01:16 +00001568
1569 def terminate(self):
1570 """Terminate the process with SIGTERM
1571 """
1572 self.send_signal(signal.SIGTERM)
1573
1574 def kill(self):
1575 """Kill the process with SIGKILL
1576 """
1577 self.send_signal(signal.SIGKILL)