blob: dd994e2aaf152c2c7567b4902710767621acb67a [file] [log] [blame]
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001# subprocess - Subprocesses with accessible I/O streams
2#
Tim Peterse718f612004-10-12 21:51:32 +00003# For more information about this module, see PEP 324.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00004#
Peter Astrand3a708df2005-09-23 17:37:29 +00005# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00006#
Peter Astrand69bf13f2005-02-14 08:56:32 +00007# Licensed to PSF under a Contributor Agreement.
Peter Astrand3a708df2005-09-23 17:37:29 +00008# See http://www.python.org/2.4/license for licensing details.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00009
Martin Panter4afdca02016-10-25 22:20:48 +000010r"""Subprocesses with accessible I/O streams
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000011
Fredrik Lundh15aaacc2004-10-17 14:47:05 +000012This module allows you to spawn processes, connect to their
Martin Panter4afdca02016-10-25 22:20:48 +000013input/output/error pipes, and obtain their return codes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000014
Martin Panter4afdca02016-10-25 22:20:48 +000015For a complete description of this module see the Python documentation.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000016
Martin Panter4afdca02016-10-25 22:20:48 +000017Main API
18========
19run(...): Runs a command, waits for it to complete, then returns a
20 CompletedProcess instance.
21Popen(...): A class for flexibly executing a command in a new process
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000022
Martin Panter4afdca02016-10-25 22:20:48 +000023Constants
24---------
25DEVNULL: Special value that indicates that os.devnull should be used
26PIPE: Special value that indicates a pipe should be created
27STDOUT: Special value that indicates that stderr should go to stdout
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000028
29
Martin Panter4afdca02016-10-25 22:20:48 +000030Older API
31=========
32call(...): Runs a command, waits for it to complete, then returns
33 the return code.
34check_call(...): Same as call() but raises CalledProcessError()
35 if return code is not 0
36check_output(...): Same as check_call() but returns the contents of
37 stdout instead of a return code
38getoutput(...): Runs a command in the shell, waits for it to complete,
39 then returns the output
40getstatusoutput(...): Runs a command in the shell, waits for it to complete,
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -070041 then returns a (exitcode, output) tuple
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000042"""
43
44import sys
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -070045_mswindows = (sys.platform == "win32")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000046
Guido van Rossumfa0054a2007-05-24 04:05:35 +000047import io
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000048import os
Reid Kleckner31aa7dd2011-03-14 12:02:10 -040049import time
Christian Heimesa342c012008-04-20 21:01:16 +000050import signal
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +000051import builtins
Gregory P. Smithd23047b2010-12-04 09:10:44 +000052import warnings
Ross Lagerwall4f61b022011-04-05 15:34:00 +020053import errno
Victor Stinnerae586492014-09-02 23:18:25 +020054from time import monotonic as _time
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000055
Peter Astrand454f7672005-01-01 09:36:35 +000056# Exception classes used by this module.
Gregory P. Smith54d412e2011-03-14 14:08:43 -040057class SubprocessError(Exception): pass
58
59
60class CalledProcessError(SubprocessError):
Martin Panter4afdca02016-10-25 22:20:48 +000061 """Raised when run() is called with check=True and the process
62 returns a non-zero exit status.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +000063
Martin Panter4afdca02016-10-25 22:20:48 +000064 Attributes:
65 cmd, returncode, stdout, stderr, output
Georg Brandlf9734072008-12-07 15:30:06 +000066 """
Gregory P. Smith6e730002015-04-14 16:14:25 -070067 def __init__(self, returncode, cmd, output=None, stderr=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +000068 self.returncode = returncode
69 self.cmd = cmd
Georg Brandlf9734072008-12-07 15:30:06 +000070 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -070071 self.stderr = stderr
72
Thomas Wouters0e3f5912006-08-11 14:57:12 +000073 def __str__(self):
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +000074 if self.returncode and self.returncode < 0:
75 try:
76 return "Command '%s' died with %r." % (
77 self.cmd, signal.Signals(-self.returncode))
78 except ValueError:
79 return "Command '%s' died with unknown signal %d." % (
80 self.cmd, -self.returncode)
81 else:
82 return "Command '%s' returned non-zero exit status %d." % (
83 self.cmd, self.returncode)
Thomas Wouters0e3f5912006-08-11 14:57:12 +000084
Gregory P. Smith6e730002015-04-14 16:14:25 -070085 @property
86 def stdout(self):
87 """Alias for output attribute, to match stderr"""
88 return self.output
89
90 @stdout.setter
91 def stdout(self, value):
92 # There's no obvious reason to set this, but allow it anyway so
93 # .stdout is a transparent alias for .output
94 self.output = value
95
Peter Astrand454f7672005-01-01 09:36:35 +000096
Gregory P. Smith54d412e2011-03-14 14:08:43 -040097class TimeoutExpired(SubprocessError):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -040098 """This exception is raised when the timeout expires while waiting for a
99 child process.
Martin Panter4afdca02016-10-25 22:20:48 +0000100
101 Attributes:
102 cmd, output, stdout, stderr, timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400103 """
Gregory P. Smith6e730002015-04-14 16:14:25 -0700104 def __init__(self, cmd, timeout, output=None, stderr=None):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400105 self.cmd = cmd
Reid Kleckner2b228f02011-03-16 16:57:54 -0400106 self.timeout = timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400107 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -0700108 self.stderr = stderr
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400109
110 def __str__(self):
111 return ("Command '%s' timed out after %s seconds" %
112 (self.cmd, self.timeout))
113
Gregory P. Smith6e730002015-04-14 16:14:25 -0700114 @property
115 def stdout(self):
116 return self.output
117
118 @stdout.setter
119 def stdout(self, value):
120 # There's no obvious reason to set this, but allow it anyway so
121 # .stdout is a transparent alias for .output
122 self.output = value
123
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400124
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700125if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000126 import threading
127 import msvcrt
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200128 import _winapi
Brian Curtin1ce6b582010-04-24 16:19:22 +0000129 class STARTUPINFO:
Subhendu Ghoshae160bb2017-02-25 20:29:05 +0530130 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None,
131 hStdError=None, wShowWindow=0):
132 self.dwFlags = dwFlags
133 self.hStdInput = hStdInput
134 self.hStdOutput = hStdOutput
135 self.hStdError = hStdError
136 self.wShowWindow = wShowWindow
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000137else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -0700138 import _posixsubprocess
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100139 import select
140 import selectors
Antoine Pitroua6a4dc82017-09-07 18:56:24 +0200141 import threading
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000142
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000143 # When select or poll has indicated that the file is writable,
144 # we can write up to _PIPE_BUF bytes without risk of blocking.
145 # POSIX defines PIPE_BUF as >= 512.
146 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
147
Charles-François Natali3a4586a2013-11-08 19:56:59 +0100148 # poll/select have the advantage of not requiring any extra file
149 # descriptor, contrarily to epoll/kqueue (also, they require a single
150 # syscall).
151 if hasattr(selectors, 'PollSelector'):
152 _PopenSelector = selectors.PollSelector
153 else:
154 _PopenSelector = selectors.SelectSelector
155
Amaury Forgeot d'Arcace31022009-07-09 22:44:11 +0000156
Brett Cannona23810f2008-05-26 19:04:21 +0000157__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
Gregory P. Smith6e730002015-04-14 16:14:25 -0700158 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
159 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
Gregory P. Smithace55862015-04-07 15:57:54 -0700160 # NOTE: We intentionally exclude list2cmdline as it is
161 # considered an internal implementation detail. issue10838.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000162
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700163if _mswindows:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200164 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
165 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
166 STD_ERROR_HANDLE, SW_HIDE,
167 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
Brian Curtin5d9deaa2011-04-29 16:24:07 -0500168
Brian Curtin08fd8d92011-04-29 16:11:30 -0500169 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
Brian Curtin8b8e7f42011-04-29 15:48:13 -0500170 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
171 "STD_ERROR_HANDLE", "SW_HIDE",
Martin Panter528619b2016-04-16 23:42:37 +0000172 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
173 "STARTUPINFO"])
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200174
175 class Handle(int):
176 closed = False
177
178 def Close(self, CloseHandle=_winapi.CloseHandle):
179 if not self.closed:
180 self.closed = True
181 CloseHandle(self)
182
183 def Detach(self):
184 if not self.closed:
185 self.closed = True
186 return int(self)
187 raise ValueError("already closed")
188
189 def __repr__(self):
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300190 return "%s(%d)" % (self.__class__.__name__, int(self))
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200191
192 __del__ = Close
193 __str__ = __repr__
194
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000195
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200196# This lists holds Popen instances for which the underlying process had not
197# exited at the time its __del__ method got called: those processes are wait()ed
198# for synchronously from _cleanup() when a new Popen object is created, to avoid
199# zombie processes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000200_active = []
201
202def _cleanup():
203 for inst in _active[:]:
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000204 res = inst._internal_poll(_deadstate=sys.maxsize)
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200205 if res is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000206 try:
207 _active.remove(inst)
208 except ValueError:
209 # This can happen if two threads create a new Popen instance.
210 # It's harmless that it was already removed, so ignore.
211 pass
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000212
213PIPE = -1
214STDOUT = -2
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200215DEVNULL = -3
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000216
217
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200218# XXX This function is only used by multiprocessing and the test suite,
219# but it's here so that it can be imported when Python is compiled without
220# threads.
221
Victor Stinner9def2842016-01-18 12:15:08 +0100222def _optim_args_from_interpreter_flags():
223 """Return a list of command-line arguments reproducing the current
224 optimization settings in sys.flags."""
225 args = []
226 value = sys.flags.optimize
227 if value > 0:
228 args.append('-' + 'O' * value)
229 return args
230
231
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200232def _args_from_interpreter_flags():
233 """Return a list of command-line arguments reproducing the current
234 settings in sys.flags and sys.warnoptions."""
235 flag_opt_map = {
236 'debug': 'd',
237 # 'inspect': 'i',
238 # 'interactive': 'i',
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200239 'dont_write_bytecode': 'B',
240 'no_user_site': 's',
241 'no_site': 'S',
242 'ignore_environment': 'E',
243 'verbose': 'v',
244 'bytes_warning': 'b',
245 'quiet': 'q',
Victor Stinner9def2842016-01-18 12:15:08 +0100246 # -O is handled in _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200247 }
Victor Stinner9def2842016-01-18 12:15:08 +0100248 args = _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200249 for flag, opt in flag_opt_map.items():
250 v = getattr(sys.flags, flag)
251 if v > 0:
252 args.append('-' + opt * v)
253 for opt in sys.warnoptions:
254 args.append('-W' + opt)
255 return args
256
257
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400258def call(*popenargs, timeout=None, **kwargs):
259 """Run command with arguments. Wait for command to complete or
260 timeout, then return the returncode attribute.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000261
262 The arguments are the same as for the Popen constructor. Example:
263
264 retcode = call(["ls", "-l"])
265 """
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200266 with Popen(*popenargs, **kwargs) as p:
267 try:
268 return p.wait(timeout=timeout)
269 except:
270 p.kill()
271 p.wait()
272 raise
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000273
274
Peter Astrand454f7672005-01-01 09:36:35 +0000275def check_call(*popenargs, **kwargs):
276 """Run command with arguments. Wait for command to complete. If
277 the exit code was zero then return, otherwise raise
278 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000279 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000280
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400281 The arguments are the same as for the call function. Example:
Peter Astrand454f7672005-01-01 09:36:35 +0000282
283 check_call(["ls", "-l"])
284 """
285 retcode = call(*popenargs, **kwargs)
Peter Astrand454f7672005-01-01 09:36:35 +0000286 if retcode:
Georg Brandlf9734072008-12-07 15:30:06 +0000287 cmd = kwargs.get("args")
288 if cmd is None:
289 cmd = popenargs[0]
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000290 raise CalledProcessError(retcode, cmd)
Georg Brandlf9734072008-12-07 15:30:06 +0000291 return 0
292
293
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400294def check_output(*popenargs, timeout=None, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700295 r"""Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000296
297 If the exit code was non-zero it raises a CalledProcessError. The
298 CalledProcessError object will have the return code in the returncode
299 attribute and output in the output attribute.
300
301 The arguments are the same as for the Popen constructor. Example:
302
303 >>> check_output(["ls", "-l", "/dev/null"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000304 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
Georg Brandlf9734072008-12-07 15:30:06 +0000305
306 The stdout argument is not allowed as it is used internally.
Georg Brandl127d4702009-12-28 08:10:38 +0000307 To capture standard error in the result, use stderr=STDOUT.
Georg Brandlf9734072008-12-07 15:30:06 +0000308
309 >>> check_output(["/bin/sh", "-c",
Georg Brandl2708f3a2009-12-20 14:38:23 +0000310 ... "ls -l non_existent_file ; exit 0"],
Georg Brandl127d4702009-12-28 08:10:38 +0000311 ... stderr=STDOUT)
Georg Brandl2708f3a2009-12-20 14:38:23 +0000312 b'ls: non_existent_file: No such file or directory\n'
Gregory P. Smith91110f52013-03-19 23:25:16 -0700313
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300314 There is an additional optional argument, "input", allowing you to
315 pass a string to the subprocess's stdin. If you use this argument
316 you may not also use the Popen constructor's "stdin" argument, as
317 it too will be used internally. Example:
318
319 >>> check_output(["sed", "-e", "s/foo/bar/"],
320 ... input=b"when in the course of fooman events\n")
321 b'when in the course of barman events\n'
322
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400323 If universal_newlines=True is passed, the "input" argument must be a
324 string and the return value will be a string rather than bytes.
Georg Brandlf9734072008-12-07 15:30:06 +0000325 """
326 if 'stdout' in kwargs:
327 raise ValueError('stdout argument not allowed, it will be overridden.')
Gregory P. Smith6e730002015-04-14 16:14:25 -0700328
329 if 'input' in kwargs and kwargs['input'] is None:
330 # Explicitly passing input=None was previously equivalent to passing an
331 # empty string. That is maintained here for backwards compatibility.
332 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
333
334 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
335 **kwargs).stdout
336
337
338class CompletedProcess(object):
339 """A process that has finished running.
340
341 This is returned by run().
342
343 Attributes:
344 args: The list or str args passed to run().
345 returncode: The exit code of the process, negative for signals.
346 stdout: The standard output (None if not captured).
347 stderr: The standard error (None if not captured).
348 """
349 def __init__(self, args, returncode, stdout=None, stderr=None):
350 self.args = args
351 self.returncode = returncode
352 self.stdout = stdout
353 self.stderr = stderr
354
355 def __repr__(self):
356 args = ['args={!r}'.format(self.args),
357 'returncode={!r}'.format(self.returncode)]
358 if self.stdout is not None:
359 args.append('stdout={!r}'.format(self.stdout))
360 if self.stderr is not None:
361 args.append('stderr={!r}'.format(self.stderr))
362 return "{}({})".format(type(self).__name__, ', '.join(args))
363
364 def check_returncode(self):
365 """Raise CalledProcessError if the exit code is non-zero."""
366 if self.returncode:
367 raise CalledProcessError(self.returncode, self.args, self.stdout,
368 self.stderr)
369
370
371def run(*popenargs, input=None, timeout=None, check=False, **kwargs):
372 """Run command with arguments and return a CompletedProcess instance.
373
374 The returned instance will have attributes args, returncode, stdout and
375 stderr. By default, stdout and stderr are not captured, and those attributes
376 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
377
378 If check is True and the exit code was non-zero, it raises a
379 CalledProcessError. The CalledProcessError object will have the return code
380 in the returncode attribute, and output & stderr attributes if those streams
381 were captured.
382
383 If timeout is given, and the process takes too long, a TimeoutExpired
384 exception will be raised.
385
386 There is an optional argument "input", allowing you to
387 pass a string to the subprocess's stdin. If you use this argument
388 you may not also use the Popen constructor's "stdin" argument, as
389 it will be used internally.
390
391 The other arguments are the same as for the Popen constructor.
392
393 If universal_newlines=True is passed, the "input" argument must be a
394 string and stdout/stderr in the returned object will be strings rather than
395 bytes.
396 """
397 if input is not None:
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300398 if 'stdin' in kwargs:
399 raise ValueError('stdin and input arguments may not both be used.')
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300400 kwargs['stdin'] = PIPE
Gregory P. Smith6e730002015-04-14 16:14:25 -0700401
402 with Popen(*popenargs, **kwargs) as process:
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200403 try:
Gregory P. Smith6e730002015-04-14 16:14:25 -0700404 stdout, stderr = process.communicate(input, timeout=timeout)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200405 except TimeoutExpired:
406 process.kill()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700407 stdout, stderr = process.communicate()
408 raise TimeoutExpired(process.args, timeout, output=stdout,
409 stderr=stderr)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200410 except:
411 process.kill()
412 process.wait()
413 raise
414 retcode = process.poll()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700415 if check and retcode:
416 raise CalledProcessError(retcode, process.args,
417 output=stdout, stderr=stderr)
418 return CompletedProcess(process.args, retcode, stdout, stderr)
Peter Astrand454f7672005-01-01 09:36:35 +0000419
420
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000421def list2cmdline(seq):
422 """
423 Translate a sequence of arguments into a command line
424 string, using the same rules as the MS C runtime:
425
426 1) Arguments are delimited by white space, which is either a
427 space or a tab.
428
429 2) A string surrounded by double quotation marks is
430 interpreted as a single argument, regardless of white space
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000431 contained within. A quoted string can be embedded in an
432 argument.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000433
434 3) A double quotation mark preceded by a backslash is
435 interpreted as a literal double quotation mark.
436
437 4) Backslashes are interpreted literally, unless they
438 immediately precede a double quotation mark.
439
440 5) If backslashes immediately precede a double quotation mark,
441 every pair of backslashes is interpreted as a literal
442 backslash. If the number of backslashes is odd, the last
443 backslash escapes the next double quotation mark as
444 described in rule 3.
445 """
446
447 # See
Eric Smith3c573af2009-11-09 15:23:15 +0000448 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
449 # or search http://msdn.microsoft.com for
450 # "Parsing C++ Command-Line Arguments"
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000451 result = []
452 needquote = False
453 for arg in seq:
454 bs_buf = []
455
456 # Add a space to separate this argument from the others
457 if result:
458 result.append(' ')
459
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000460 needquote = (" " in arg) or ("\t" in arg) or not arg
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000461 if needquote:
462 result.append('"')
463
464 for c in arg:
465 if c == '\\':
Tim Peterse718f612004-10-12 21:51:32 +0000466 # Don't know if we need to double yet.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000467 bs_buf.append(c)
468 elif c == '"':
Christian Heimesfdab48e2008-01-20 09:06:41 +0000469 # Double backslashes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000470 result.append('\\' * len(bs_buf)*2)
471 bs_buf = []
472 result.append('\\"')
473 else:
474 # Normal char
475 if bs_buf:
476 result.extend(bs_buf)
477 bs_buf = []
478 result.append(c)
479
Christian Heimesfdab48e2008-01-20 09:06:41 +0000480 # Add remaining backslashes, if any.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000481 if bs_buf:
482 result.extend(bs_buf)
483
484 if needquote:
Peter Astrand7e78ade2005-03-03 21:10:23 +0000485 result.extend(bs_buf)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000486 result.append('"')
487
488 return ''.join(result)
489
490
Brett Cannona23810f2008-05-26 19:04:21 +0000491# Various tools for executing commands and looking at their output and status.
492#
Brett Cannona23810f2008-05-26 19:04:21 +0000493
494def getstatusoutput(cmd):
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700495 """Return (exitcode, output) of executing cmd in a shell.
Brett Cannona23810f2008-05-26 19:04:21 +0000496
Tim Golden60798142013-11-05 12:57:25 +0000497 Execute the string 'cmd' in a shell with 'check_output' and
Steve Dower050acae2016-09-06 20:16:17 -0700498 return a 2-tuple (status, output). The locale encoding is used
499 to decode the output and process newlines.
Tim Golden60798142013-11-05 12:57:25 +0000500
501 A trailing newline is stripped from the output.
502 The exit status for the command can be interpreted
503 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000504
505 >>> import subprocess
506 >>> subprocess.getstatusoutput('ls /bin/ls')
507 (0, '/bin/ls')
508 >>> subprocess.getstatusoutput('cat /bin/junk')
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700509 (1, 'cat: /bin/junk: No such file or directory')
Brett Cannona23810f2008-05-26 19:04:21 +0000510 >>> subprocess.getstatusoutput('/bin/junk')
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700511 (127, 'sh: /bin/junk: not found')
512 >>> subprocess.getstatusoutput('/bin/kill $$')
513 (-15, '')
Brett Cannona23810f2008-05-26 19:04:21 +0000514 """
Tim Goldene0041752013-11-03 12:53:17 +0000515 try:
516 data = check_output(cmd, shell=True, universal_newlines=True, stderr=STDOUT)
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700517 exitcode = 0
Tim Goldene0041752013-11-03 12:53:17 +0000518 except CalledProcessError as ex:
519 data = ex.output
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700520 exitcode = ex.returncode
Tim Goldene0041752013-11-03 12:53:17 +0000521 if data[-1:] == '\n':
522 data = data[:-1]
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700523 return exitcode, data
Brett Cannona23810f2008-05-26 19:04:21 +0000524
525def getoutput(cmd):
526 """Return output (stdout or stderr) of executing cmd in a shell.
527
528 Like getstatusoutput(), except the exit status is ignored and the return
529 value is a string containing the command's output. Example:
530
531 >>> import subprocess
532 >>> subprocess.getoutput('ls /bin/ls')
533 '/bin/ls'
534 """
535 return getstatusoutput(cmd)[1]
536
537
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000538_PLATFORM_DEFAULT_CLOSE_FDS = object()
Gregory P. Smithf5604852010-12-13 06:45:02 +0000539
540
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000541class Popen(object):
Martin Panter4afdca02016-10-25 22:20:48 +0000542 """ Execute a child program in a new process.
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200543
Martin Panter4afdca02016-10-25 22:20:48 +0000544 For a complete description of the arguments see the Python documentation.
545
546 Arguments:
547 args: A string, or a sequence of program arguments.
548
549 bufsize: supplied as the buffering argument to the open() function when
550 creating the stdin/stdout/stderr pipe file objects
551
552 executable: A replacement program to execute.
553
554 stdin, stdout and stderr: These specify the executed programs' standard
555 input, standard output and standard error file handles, respectively.
556
557 preexec_fn: (POSIX only) An object to be called in the child process
558 just before the child is executed.
559
560 close_fds: Controls closing or inheriting of file descriptors.
561
562 shell: If true, the command will be executed through the shell.
563
564 cwd: Sets the current directory before the child is executed.
565
566 env: Defines the environment variables for the new process.
567
568 universal_newlines: If true, use universal line endings for file
569 objects stdin, stdout and stderr.
570
571 startupinfo and creationflags (Windows only)
572
573 restore_signals (POSIX only)
574
575 start_new_session (POSIX only)
576
577 pass_fds (POSIX only)
578
Martin Panter3dca6242016-10-25 23:41:42 +0000579 encoding and errors: Text mode encoding and error handling to use for
580 file objects stdin, stdout and stderr.
581
Martin Panter4afdca02016-10-25 22:20:48 +0000582 Attributes:
583 stdin, stdout, stderr, pid, returncode
584 """
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200585 _child_created = False # Set here since __del__ checks it
586
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700587 def __init__(self, args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000588 stdin=None, stdout=None, stderr=None,
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000589 preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
590 shell=False, cwd=None, env=None, universal_newlines=False,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000591 startupinfo=None, creationflags=0,
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000592 restore_signals=True, start_new_session=False,
Steve Dower050acae2016-09-06 20:16:17 -0700593 pass_fds=(), *, encoding=None, errors=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000594 """Create new Popen instance."""
595 _cleanup()
Gregory P. Smithd65ba512014-04-23 00:27:17 -0700596 # Held while anything is calling waitpid before returncode has been
597 # updated to prevent clobbering returncode if wait() or poll() are
598 # called from multiple threads at once. After acquiring the lock,
599 # code must re-check self.returncode to see if another thread just
600 # finished a waitpid() call.
601 self._waitpid_lock = threading.Lock()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000602
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400603 self._input = None
604 self._communication_started = False
Guido van Rossum46a05a72007-06-07 21:56:45 +0000605 if bufsize is None:
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700606 bufsize = -1 # Restore default
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000607 if not isinstance(bufsize, int):
Peter Astrand738131d2004-11-30 21:04:45 +0000608 raise TypeError("bufsize must be an integer")
609
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700610 if _mswindows:
Tim Peterse8374a52004-10-13 03:15:00 +0000611 if preexec_fn is not None:
612 raise ValueError("preexec_fn is not supported on Windows "
613 "platforms")
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000614 any_stdio_set = (stdin is not None or stdout is not None or
615 stderr is not None)
616 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
617 if any_stdio_set:
618 close_fds = False
619 else:
620 close_fds = True
621 elif close_fds and any_stdio_set:
622 raise ValueError(
623 "close_fds is not supported on Windows platforms"
624 " if you redirect stdin/stdout/stderr")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000625 else:
626 # POSIX
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000627 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
628 close_fds = True
629 if pass_fds and not close_fds:
630 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
631 close_fds = True
Tim Peterse8374a52004-10-13 03:15:00 +0000632 if startupinfo is not None:
633 raise ValueError("startupinfo is only supported on Windows "
634 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000635 if creationflags != 0:
Tim Peterse8374a52004-10-13 03:15:00 +0000636 raise ValueError("creationflags is only supported on Windows "
637 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000638
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400639 self.args = args
Tim Peterse718f612004-10-12 21:51:32 +0000640 self.stdin = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000641 self.stdout = None
642 self.stderr = None
643 self.pid = None
644 self.returncode = None
645 self.universal_newlines = universal_newlines
Steve Dower050acae2016-09-06 20:16:17 -0700646 self.encoding = encoding
647 self.errors = errors
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000648
649 # Input and output objects. The general principle is like
650 # this:
651 #
652 # Parent Child
653 # ------ -----
654 # p2cwrite ---stdin---> p2cread
655 # c2pread <--stdout--- c2pwrite
656 # errread <--stderr--- errwrite
657 #
658 # On POSIX, the child objects are file descriptors. On
659 # Windows, these are Windows file handles. The parent objects
660 # are file descriptors on both platforms. The parent objects
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000661 # are -1 when not using PIPEs. The child objects are -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000662 # when not redirecting.
Tim Peterse718f612004-10-12 21:51:32 +0000663
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000664 (p2cread, p2cwrite,
665 c2pread, c2pwrite,
666 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
667
Antoine Pitrouc9982322011-01-04 19:07:07 +0000668 # We wrap OS handles *before* launching the child, otherwise a
669 # quickly terminating child could make our fds unwrappable
670 # (see #8458).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000671
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700672 if _mswindows:
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000673 if p2cwrite != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000674 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000675 if c2pread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000676 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000677 if errread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000678 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
Thomas Wouterscf297e42007-02-23 15:07:44 +0000679
Steve Dower050acae2016-09-06 20:16:17 -0700680 text_mode = encoding or errors or universal_newlines
Tim Peterse718f612004-10-12 21:51:32 +0000681
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700682 self._closed_child_pipe_fds = False
Steve Dower050acae2016-09-06 20:16:17 -0700683
Antoine Pitrouc9982322011-01-04 19:07:07 +0000684 try:
Steve Dower050acae2016-09-06 20:16:17 -0700685 if p2cwrite != -1:
686 self.stdin = io.open(p2cwrite, 'wb', bufsize)
687 if text_mode:
688 self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
689 line_buffering=(bufsize == 1),
690 encoding=encoding, errors=errors)
691 if c2pread != -1:
692 self.stdout = io.open(c2pread, 'rb', bufsize)
693 if text_mode:
694 self.stdout = io.TextIOWrapper(self.stdout,
695 encoding=encoding, errors=errors)
696 if errread != -1:
697 self.stderr = io.open(errread, 'rb', bufsize)
698 if text_mode:
699 self.stderr = io.TextIOWrapper(self.stderr,
700 encoding=encoding, errors=errors)
701
Antoine Pitrouc9982322011-01-04 19:07:07 +0000702 self._execute_child(args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300703 pass_fds, cwd, env,
Antoine Pitrouc9982322011-01-04 19:07:07 +0000704 startupinfo, creationflags, shell,
705 p2cread, p2cwrite,
706 c2pread, c2pwrite,
707 errread, errwrite,
708 restore_signals, start_new_session)
709 except:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800710 # Cleanup if the child failed starting.
711 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
Antoine Pitrouc9982322011-01-04 19:07:07 +0000712 try:
713 f.close()
Andrew Svetlov3438fa42012-12-17 23:35:18 +0200714 except OSError:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800715 pass # Ignore EBADF or other errors.
716
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700717 if not self._closed_child_pipe_fds:
718 to_close = []
719 if stdin == PIPE:
720 to_close.append(p2cread)
721 if stdout == PIPE:
722 to_close.append(c2pwrite)
723 if stderr == PIPE:
724 to_close.append(errwrite)
725 if hasattr(self, '_devnull'):
726 to_close.append(self._devnull)
727 for fd in to_close:
728 try:
Segev Finer4d385172017-08-18 16:18:13 +0300729 if _mswindows and isinstance(fd, Handle):
730 fd.Close()
731 else:
732 os.close(fd)
Gregory P. Smith22ba31a2013-06-15 18:14:56 -0700733 except OSError:
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700734 pass
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800735
Antoine Pitrouc9982322011-01-04 19:07:07 +0000736 raise
737
Steve Dower050acae2016-09-06 20:16:17 -0700738 def _translate_newlines(self, data, encoding, errors):
739 data = data.decode(encoding, errors)
Andrew Svetlov82860712012-08-19 22:13:41 +0300740 return data.replace("\r\n", "\n").replace("\r", "\n")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000741
Brian Curtin79cdb662010-12-03 02:46:02 +0000742 def __enter__(self):
743 return self
744
745 def __exit__(self, type, value, traceback):
746 if self.stdout:
747 self.stdout.close()
748 if self.stderr:
749 self.stderr.close()
Serhiy Storchakaab900c22015-02-28 12:43:08 +0200750 try: # Flushing a BufferedWriter may raise an error
751 if self.stdin:
752 self.stdin.close()
753 finally:
754 # Wait for the process to terminate, to avoid zombies.
755 self.wait()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000756
Victor Stinner9505b032017-01-06 10:44:44 +0100757 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn):
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200758 if not self._child_created:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000759 # We didn't get to successfully create a child process.
760 return
Victor Stinner5a48e212016-05-20 12:11:15 +0200761 if self.returncode is None:
Victor Stinnerc206f1e2016-06-14 16:42:59 +0200762 # Not reading subprocess exit status creates a zombi process which
763 # is only destroyed at the parent python process exit
Victor Stinner9505b032017-01-06 10:44:44 +0100764 _warn("subprocess %s is still running" % self.pid,
765 ResourceWarning, source=self)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000766 # In case the child hasn't been waited on, check if it's done.
Brett Cannon84df1e62010-05-14 00:33:40 +0000767 self._internal_poll(_deadstate=_maxsize)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000768 if self.returncode is None and _active is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000769 # Child is still running, keep us alive until we can wait on it.
770 _active.append(self)
771
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200772 def _get_devnull(self):
773 if not hasattr(self, '_devnull'):
774 self._devnull = os.open(os.devnull, os.O_RDWR)
775 return self._devnull
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000776
Victor Stinnera5e881d2015-01-14 17:07:59 +0100777 def _stdin_write(self, input):
778 if input:
779 try:
780 self.stdin.write(input)
781 except BrokenPipeError:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000782 pass # communicate() must ignore broken pipe errors.
Victor Stinnerd52aa312017-06-08 17:30:39 +0200783 except OSError as exc:
784 if exc.errno == errno.EINVAL:
785 # bpo-19612, bpo-30418: On Windows, stdin.write() fails
786 # with EINVAL if the child process exited or if the child
787 # process is still running but closed the pipe.
Victor Stinnera5e881d2015-01-14 17:07:59 +0100788 pass
789 else:
790 raise
Victor Stinnerd52aa312017-06-08 17:30:39 +0200791
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000792 try:
793 self.stdin.close()
794 except BrokenPipeError:
795 pass # communicate() must ignore broken pipe errors.
Victor Stinnerd52aa312017-06-08 17:30:39 +0200796 except OSError as exc:
797 if exc.errno == errno.EINVAL:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000798 pass
799 else:
800 raise
Victor Stinnera5e881d2015-01-14 17:07:59 +0100801
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400802 def communicate(self, input=None, timeout=None):
Joel Schaerer88031a92017-09-13 21:11:20 +0200803 """Interact with process: Send data to stdin and close it.
804 Read data from stdout and stderr, until end-of-file is
805 reached. Wait for process to terminate.
Tim Peterseba28be2005-03-28 01:08:02 +0000806
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400807 The optional "input" argument should be data to be sent to the
808 child process (if self.universal_newlines is True, this should
809 be a string; if it is False, "input" should be bytes), or
810 None, if no data should be sent to the child.
811
812 communicate() returns a tuple (stdout, stderr). These will be
813 bytes or, if self.universal_newlines was True, a string.
814 """
Peter Astrand23109f02005-03-03 20:28:59 +0000815
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400816 if self._communication_started and input:
817 raise ValueError("Cannot send input after starting communication")
818
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400819 # Optimization: If we are not worried about timeouts, we haven't
820 # started communicating, and we have one or zero pipes, using select()
821 # or threads is unnecessary.
Victor Stinner7a8d0812011-04-05 13:13:08 +0200822 if (timeout is None and not self._communication_started and
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400823 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
Tim Peterseba28be2005-03-28 01:08:02 +0000824 stdout = None
825 stderr = None
Peter Astrand23109f02005-03-03 20:28:59 +0000826 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +0100827 self._stdin_write(input)
Peter Astrand23109f02005-03-03 20:28:59 +0000828 elif self.stdout:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +0000829 stdout = self.stdout.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000830 self.stdout.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000831 elif self.stderr:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +0000832 stderr = self.stderr.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000833 self.stderr.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000834 self.wait()
Victor Stinner7a8d0812011-04-05 13:13:08 +0200835 else:
836 if timeout is not None:
Victor Stinner949d8c92012-05-30 13:30:32 +0200837 endtime = _time() + timeout
Victor Stinner7a8d0812011-04-05 13:13:08 +0200838 else:
839 endtime = None
Tim Peterseba28be2005-03-28 01:08:02 +0000840
Victor Stinner7a8d0812011-04-05 13:13:08 +0200841 try:
842 stdout, stderr = self._communicate(input, endtime, timeout)
843 finally:
844 self._communication_started = True
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400845
Victor Stinner7a8d0812011-04-05 13:13:08 +0200846 sts = self.wait(timeout=self._remaining_time(endtime))
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400847
848 return (stdout, stderr)
Peter Astrand23109f02005-03-03 20:28:59 +0000849
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000850
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000851 def poll(self):
Martin Panter4afdca02016-10-25 22:20:48 +0000852 """Check if child process has terminated. Set and return returncode
853 attribute."""
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000854 return self._internal_poll()
855
856
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400857 def _remaining_time(self, endtime):
858 """Convenience for _communicate when computing timeouts."""
859 if endtime is None:
860 return None
861 else:
Victor Stinner949d8c92012-05-30 13:30:32 +0200862 return endtime - _time()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400863
864
Reid Kleckner2b228f02011-03-16 16:57:54 -0400865 def _check_timeout(self, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400866 """Convenience for checking if a timeout has expired."""
867 if endtime is None:
868 return
Victor Stinner949d8c92012-05-30 13:30:32 +0200869 if _time() > endtime:
Reid Kleckner2b228f02011-03-16 16:57:54 -0400870 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400871
872
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700873 if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000874 #
875 # Windows methods
876 #
877 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +0000878 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000879 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
880 """
Peter Astrandd38ddf42005-02-10 08:32:50 +0000881 if stdin is None and stdout is None and stderr is None:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000882 return (-1, -1, -1, -1, -1, -1)
Tim Peterse718f612004-10-12 21:51:32 +0000883
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000884 p2cread, p2cwrite = -1, -1
885 c2pread, c2pwrite = -1, -1
886 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000887
Peter Astrandd38ddf42005-02-10 08:32:50 +0000888 if stdin is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200889 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000890 if p2cread is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200891 p2cread, _ = _winapi.CreatePipe(None, 0)
892 p2cread = Handle(p2cread)
893 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000894 elif stdin == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200895 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
896 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200897 elif stdin == DEVNULL:
898 p2cread = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +0000899 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000900 p2cread = msvcrt.get_osfhandle(stdin)
901 else:
902 # Assuming file-like object
903 p2cread = msvcrt.get_osfhandle(stdin.fileno())
904 p2cread = self._make_inheritable(p2cread)
905
Peter Astrandd38ddf42005-02-10 08:32:50 +0000906 if stdout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200907 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000908 if c2pwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200909 _, c2pwrite = _winapi.CreatePipe(None, 0)
910 c2pwrite = Handle(c2pwrite)
911 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000912 elif stdout == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200913 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
914 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200915 elif stdout == DEVNULL:
916 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +0000917 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000918 c2pwrite = msvcrt.get_osfhandle(stdout)
919 else:
920 # Assuming file-like object
921 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
922 c2pwrite = self._make_inheritable(c2pwrite)
923
Peter Astrandd38ddf42005-02-10 08:32:50 +0000924 if stderr is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200925 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000926 if errwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200927 _, errwrite = _winapi.CreatePipe(None, 0)
928 errwrite = Handle(errwrite)
929 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000930 elif stderr == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200931 errread, errwrite = _winapi.CreatePipe(None, 0)
932 errread, errwrite = Handle(errread), Handle(errwrite)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000933 elif stderr == STDOUT:
934 errwrite = c2pwrite
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200935 elif stderr == DEVNULL:
936 errwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +0000937 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000938 errwrite = msvcrt.get_osfhandle(stderr)
939 else:
940 # Assuming file-like object
941 errwrite = msvcrt.get_osfhandle(stderr.fileno())
942 errwrite = self._make_inheritable(errwrite)
943
944 return (p2cread, p2cwrite,
945 c2pread, c2pwrite,
946 errread, errwrite)
947
948
949 def _make_inheritable(self, handle):
950 """Return a duplicate of handle, which is inheritable"""
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200951 h = _winapi.DuplicateHandle(
952 _winapi.GetCurrentProcess(), handle,
953 _winapi.GetCurrentProcess(), 0, 1,
954 _winapi.DUPLICATE_SAME_ACCESS)
955 return Handle(h)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000956
957
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000958 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300959 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000960 startupinfo, creationflags, shell,
961 p2cread, p2cwrite,
962 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000963 errread, errwrite,
964 unused_restore_signals, unused_start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000965 """Execute program (MS Windows version)"""
966
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000967 assert not pass_fds, "pass_fds not supported on Windows."
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000968
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000969 if not isinstance(args, str):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000970 args = list2cmdline(args)
971
Peter Astrandc1d65362004-11-07 14:30:34 +0000972 # Process startup details
Peter Astrandd38ddf42005-02-10 08:32:50 +0000973 if startupinfo is None:
Thomas Wouters73e5a5b2006-06-08 15:35:45 +0000974 startupinfo = STARTUPINFO()
Victor Stinnerb3693582010-05-21 20:13:12 +0000975 if -1 not in (p2cread, c2pwrite, errwrite):
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200976 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
Peter Astrandc1d65362004-11-07 14:30:34 +0000977 startupinfo.hStdInput = p2cread
978 startupinfo.hStdOutput = c2pwrite
979 startupinfo.hStdError = errwrite
980
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000981 if shell:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200982 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
983 startupinfo.wShowWindow = _winapi.SW_HIDE
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000984 comspec = os.environ.get("COMSPEC", "cmd.exe")
Tim Golden126c2962010-08-11 14:20:40 +0000985 args = '{} /c "{}"'.format (comspec, args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000986
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000987 # Start the process
988 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200989 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
Tim Peterse8374a52004-10-13 03:15:00 +0000990 # no special security
991 None, None,
Guido van Rossume7ba4952007-06-06 23:52:48 +0000992 int(not close_fds),
Tim Peterse8374a52004-10-13 03:15:00 +0000993 creationflags,
994 env,
Benjamin Petersoncb90f262017-03-02 00:03:41 -0800995 os.fspath(cwd) if cwd is not None else None,
Tim Peterse8374a52004-10-13 03:15:00 +0000996 startupinfo)
Tim Goldenad537f22010-08-08 11:18:16 +0000997 finally:
998 # Child is launched. Close the parent's copy of those pipe
999 # handles that only the child should have open. You need
1000 # to make sure that no handles to the write end of the
1001 # output pipe are maintained in this process or else the
1002 # pipe will not close when the child process exits and the
1003 # ReadFile will hang.
1004 if p2cread != -1:
1005 p2cread.Close()
1006 if c2pwrite != -1:
1007 c2pwrite.Close()
1008 if errwrite != -1:
1009 errwrite.Close()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001010 if hasattr(self, '_devnull'):
1011 os.close(self._devnull)
Segev Finer4d385172017-08-18 16:18:13 +03001012 # Prevent a double close of these handles/fds from __init__
1013 # on error.
1014 self._closed_child_pipe_fds = True
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001015
1016 # Retain the process handle, but close the thread handle
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001017 self._child_created = True
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001018 self._handle = Handle(hp)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001019 self.pid = pid
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001020 _winapi.CloseHandle(ht)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001021
Brett Cannon84df1e62010-05-14 00:33:40 +00001022 def _internal_poll(self, _deadstate=None,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001023 _WaitForSingleObject=_winapi.WaitForSingleObject,
1024 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1025 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001026 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001027 attribute.
1028
1029 This method is called by __del__, so it can only refer to objects
1030 in its local scope.
1031
1032 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001033 if self.returncode is None:
Brett Cannon84df1e62010-05-14 00:33:40 +00001034 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1035 self.returncode = _GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001036 return self.returncode
1037
1038
Gregory P. Smith82604e02016-11-20 16:31:07 -08001039 def wait(self, timeout=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001040 """Wait for child process to terminate. Returns returncode
1041 attribute."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001042 if timeout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001043 timeout_millis = _winapi.INFINITE
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001044 else:
Reid Kleckner91156ff2011-03-21 10:06:10 -07001045 timeout_millis = int(timeout * 1000)
Peter Astrandd38ddf42005-02-10 08:32:50 +00001046 if self.returncode is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001047 result = _winapi.WaitForSingleObject(self._handle,
1048 timeout_millis)
1049 if result == _winapi.WAIT_TIMEOUT:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001050 raise TimeoutExpired(self.args, timeout)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001051 self.returncode = _winapi.GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001052 return self.returncode
1053
1054
1055 def _readerthread(self, fh, buffer):
1056 buffer.append(fh.read())
Victor Stinner667d4b52010-12-25 22:40:32 +00001057 fh.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001058
1059
Reid Kleckner2b228f02011-03-16 16:57:54 -04001060 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001061 # Start reader threads feeding into a list hanging off of this
1062 # object, unless they've already been started.
1063 if self.stdout and not hasattr(self, "_stdout_buff"):
1064 self._stdout_buff = []
1065 self.stdout_thread = \
1066 threading.Thread(target=self._readerthread,
1067 args=(self.stdout, self._stdout_buff))
1068 self.stdout_thread.daemon = True
1069 self.stdout_thread.start()
1070 if self.stderr and not hasattr(self, "_stderr_buff"):
1071 self._stderr_buff = []
1072 self.stderr_thread = \
1073 threading.Thread(target=self._readerthread,
1074 args=(self.stderr, self._stderr_buff))
1075 self.stderr_thread.daemon = True
1076 self.stderr_thread.start()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001077
1078 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +01001079 self._stdin_write(input)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001080
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001081 # Wait for the reader threads, or time out. If we time out, the
1082 # threads remain reading and the fds left open in case the user
1083 # calls communicate again.
1084 if self.stdout is not None:
1085 self.stdout_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001086 if self.stdout_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001087 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001088 if self.stderr is not None:
1089 self.stderr_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001090 if self.stderr_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001091 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001092
1093 # Collect the output from and close both pipes, now that we know
1094 # both have been read successfully.
1095 stdout = None
1096 stderr = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001097 if self.stdout:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001098 stdout = self._stdout_buff
1099 self.stdout.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001100 if self.stderr:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001101 stderr = self._stderr_buff
1102 self.stderr.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001103
1104 # All data exchanged. Translate lists into strings.
Peter Astrandd38ddf42005-02-10 08:32:50 +00001105 if stdout is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001106 stdout = stdout[0]
Peter Astrandd38ddf42005-02-10 08:32:50 +00001107 if stderr is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001108 stderr = stderr[0]
1109
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001110 return (stdout, stderr)
1111
Christian Heimesa342c012008-04-20 21:01:16 +00001112 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001113 """Send a signal to the process."""
1114 # Don't signal a process that we know has already died.
1115 if self.returncode is not None:
1116 return
Christian Heimesa342c012008-04-20 21:01:16 +00001117 if sig == signal.SIGTERM:
1118 self.terminate()
Brian Curtineb24d742010-04-12 17:16:38 +00001119 elif sig == signal.CTRL_C_EVENT:
1120 os.kill(self.pid, signal.CTRL_C_EVENT)
1121 elif sig == signal.CTRL_BREAK_EVENT:
1122 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
Christian Heimesa342c012008-04-20 21:01:16 +00001123 else:
Brian Curtin19651362010-09-07 13:24:38 +00001124 raise ValueError("Unsupported signal: {}".format(sig))
Christian Heimesa342c012008-04-20 21:01:16 +00001125
1126 def terminate(self):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001127 """Terminates the process."""
1128 # Don't terminate a process that we know has already died.
1129 if self.returncode is not None:
1130 return
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001131 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001132 _winapi.TerminateProcess(self._handle, 1)
Antoine Pitroub69ef162012-03-11 19:33:29 +01001133 except PermissionError:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001134 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1135 # process already died.
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001136 rc = _winapi.GetExitCodeProcess(self._handle)
1137 if rc == _winapi.STILL_ACTIVE:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001138 raise
1139 self.returncode = rc
Christian Heimesa342c012008-04-20 21:01:16 +00001140
1141 kill = terminate
1142
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001143 else:
1144 #
1145 # POSIX methods
1146 #
1147 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001148 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001149 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1150 """
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001151 p2cread, p2cwrite = -1, -1
1152 c2pread, c2pwrite = -1, -1
1153 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001154
Peter Astrandd38ddf42005-02-10 08:32:50 +00001155 if stdin is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001156 pass
1157 elif stdin == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001158 p2cread, p2cwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001159 elif stdin == DEVNULL:
1160 p2cread = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001161 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001162 p2cread = stdin
1163 else:
1164 # Assuming file-like object
1165 p2cread = stdin.fileno()
1166
Peter Astrandd38ddf42005-02-10 08:32:50 +00001167 if stdout is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001168 pass
1169 elif stdout == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001170 c2pread, c2pwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001171 elif stdout == DEVNULL:
1172 c2pwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001173 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001174 c2pwrite = stdout
1175 else:
1176 # Assuming file-like object
Tim Peterse718f612004-10-12 21:51:32 +00001177 c2pwrite = stdout.fileno()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001178
Peter Astrandd38ddf42005-02-10 08:32:50 +00001179 if stderr is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001180 pass
1181 elif stderr == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001182 errread, errwrite = os.pipe()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001183 elif stderr == STDOUT:
Martin Panterc7635892016-05-13 01:54:44 +00001184 if c2pwrite != -1:
1185 errwrite = c2pwrite
1186 else: # child's stdout is not set, use parent's stdout
1187 errwrite = sys.__stdout__.fileno()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001188 elif stderr == DEVNULL:
1189 errwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001190 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001191 errwrite = stderr
1192 else:
1193 # Assuming file-like object
1194 errwrite = stderr.fileno()
1195
1196 return (p2cread, p2cwrite,
1197 c2pread, c2pwrite,
1198 errread, errwrite)
1199
1200
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001201 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001202 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001203 startupinfo, creationflags, shell,
1204 p2cread, p2cwrite,
1205 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001206 errread, errwrite,
1207 restore_signals, start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001208 """Execute program (POSIX version)"""
1209
Victor Stinner7b3b20a2011-03-03 12:54:05 +00001210 if isinstance(args, (str, bytes)):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001211 args = [args]
Thomas Wouters89f507f2006-12-13 04:49:30 +00001212 else:
1213 args = list(args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001214
1215 if shell:
Xavier de Gayeb35fc622016-12-13 16:32:21 +01001216 # On Android the default shell is at '/system/bin/sh'.
1217 unix_shell = ('/system/bin/sh' if
1218 hasattr(sys, 'getandroidapilevel') else '/bin/sh')
1219 args = [unix_shell, "-c"] + args
Stefan Krah9542cc62010-07-19 14:20:53 +00001220 if executable:
1221 args[0] = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001222
Peter Astrandd38ddf42005-02-10 08:32:50 +00001223 if executable is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001224 executable = args[0]
Gregory P. Smith5591b022012-10-10 03:34:47 -07001225 orig_executable = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001226
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001227 # For transferring possible exec failure from child to parent.
1228 # Data format: "exception name:hex errno:description"
1229 # Pickle is not used; it is complex and involves memory allocation.
Victor Stinnerdaf45552013-08-28 00:53:59 +02001230 errpipe_read, errpipe_write = os.pipe()
Gregory P. Smith53dd8162013-12-01 16:03:24 -08001231 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1232 low_fds_to_close = []
1233 while errpipe_write < 3:
1234 low_fds_to_close.append(errpipe_write)
1235 errpipe_write = os.dup(errpipe_write)
1236 for low_fd in low_fds_to_close:
1237 os.close(low_fd)
Christian Heimesfdab48e2008-01-20 09:06:41 +00001238 try:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001239 try:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001240 # We must avoid complex work that could involve
1241 # malloc or free in the child process to avoid
1242 # potential deadlocks, thus we do all this here.
1243 # and pass it to fork_exec()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001244
Victor Stinner372b8382011-06-21 17:24:21 +02001245 if env is not None:
Serhiy Storchakad174d242017-06-23 19:39:27 +03001246 env_list = []
1247 for k, v in env.items():
1248 k = os.fsencode(k)
1249 if b'=' in k:
1250 raise ValueError("illegal environment variable name")
1251 env_list.append(k + b'=' + os.fsencode(v))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001252 else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001253 env_list = None # Use execv instead of execve.
1254 executable = os.fsencode(executable)
1255 if os.path.dirname(executable):
1256 executable_list = (executable,)
1257 else:
1258 # This matches the behavior of os._execvpe().
1259 executable_list = tuple(
1260 os.path.join(os.fsencode(dir), executable)
1261 for dir in os.get_exec_path(env))
Gregory P. Smith361e30c2013-12-01 00:12:24 -08001262 fds_to_keep = set(pass_fds)
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001263 fds_to_keep.add(errpipe_write)
1264 self.pid = _posixsubprocess.fork_exec(
1265 args, executable_list,
Serhiy Storchaka66bffd12017-04-19 21:12:46 +03001266 close_fds, tuple(sorted(map(int, fds_to_keep))),
1267 cwd, env_list,
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001268 p2cread, p2cwrite, c2pread, c2pwrite,
1269 errread, errwrite,
1270 errpipe_read, errpipe_write,
1271 restore_signals, start_new_session, preexec_fn)
Charles-François Natali558639f2011-08-18 19:11:29 +02001272 self._child_created = True
Facundo Batista10706e22009-06-19 20:34:30 +00001273 finally:
1274 # be sure the FD is closed no matter what
1275 os.close(errpipe_write)
1276
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001277 # self._devnull is not always defined.
1278 devnull_fd = getattr(self, '_devnull', None)
1279 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001280 os.close(p2cread)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001281 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001282 os.close(c2pwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001283 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001284 os.close(errwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001285 if devnull_fd is not None:
1286 os.close(devnull_fd)
1287 # Prevent a double close of these fds from __init__ on error.
1288 self._closed_child_pipe_fds = True
Facundo Batista10706e22009-06-19 20:34:30 +00001289
1290 # Wait for exec to fail or succeed; possibly raising an
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001291 # exception (limited in size)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001292 errpipe_data = bytearray()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001293 while True:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001294 part = os.read(errpipe_read, 50000)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001295 errpipe_data += part
1296 if not part or len(errpipe_data) > 50000:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001297 break
Facundo Batista10706e22009-06-19 20:34:30 +00001298 finally:
1299 # be sure the FD is closed no matter what
1300 os.close(errpipe_read)
1301
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001302 if errpipe_data:
Gregory P. Smithe85db2b2010-12-14 14:38:00 +00001303 try:
Victor Stinnera58e2c52016-05-20 12:08:12 +02001304 pid, sts = os.waitpid(self.pid, 0)
1305 if pid == self.pid:
1306 self._handle_exitstatus(sts)
1307 else:
1308 self.returncode = sys.maxsize
Victor Stinnera5e881d2015-01-14 17:07:59 +01001309 except ChildProcessError:
1310 pass
Victor Stinnera58e2c52016-05-20 12:08:12 +02001311
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001312 try:
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001313 exception_name, hex_errno, err_msg = (
1314 errpipe_data.split(b':', 2))
Ammar Askar3fc499b2017-09-06 02:41:30 -04001315 # The encoding here should match the encoding
1316 # written in by the subprocess implementations
1317 # like _posixsubprocess
1318 err_msg = err_msg.decode()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001319 except ValueError:
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001320 exception_name = b'SubprocessError'
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001321 hex_errno = b'0'
Ammar Askar3fc499b2017-09-06 02:41:30 -04001322 err_msg = 'Bad exception data from child: {!r}'.format(
1323 bytes(errpipe_data))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001324 child_exception_type = getattr(
1325 builtins, exception_name.decode('ascii'),
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001326 SubprocessError)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001327 if issubclass(child_exception_type, OSError) and hex_errno:
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001328 errno_num = int(hex_errno, 16)
Gregory P. Smith5591b022012-10-10 03:34:47 -07001329 child_exec_never_called = (err_msg == "noexec")
1330 if child_exec_never_called:
1331 err_msg = ""
Gregory P. Smith8621bb52017-08-24 14:58:25 -07001332 # The error must be from chdir(cwd).
1333 err_filename = cwd
1334 else:
1335 err_filename = orig_executable
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001336 if errno_num != 0:
1337 err_msg = os.strerror(errno_num)
1338 if errno_num == errno.ENOENT:
Gregory P. Smith8621bb52017-08-24 14:58:25 -07001339 err_msg += ': ' + repr(err_filename)
1340 raise child_exception_type(errno_num, err_msg, err_filename)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001341 raise child_exception_type(err_msg)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001342
1343
Brett Cannon84df1e62010-05-14 00:33:40 +00001344 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
1345 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
Gregory P. Smith50e16e32017-01-22 17:28:38 -08001346 _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED,
1347 _WSTOPSIG=os.WSTOPSIG):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001348 """All callers to this function MUST hold self._waitpid_lock."""
Brett Cannon84df1e62010-05-14 00:33:40 +00001349 # This method is called (indirectly) by __del__, so it cannot
Serhiy Storchaka72e77612014-02-10 19:20:22 +02001350 # refer to anything outside of its local scope.
Brett Cannon84df1e62010-05-14 00:33:40 +00001351 if _WIFSIGNALED(sts):
1352 self.returncode = -_WTERMSIG(sts)
1353 elif _WIFEXITED(sts):
1354 self.returncode = _WEXITSTATUS(sts)
Gregory P. Smith50e16e32017-01-22 17:28:38 -08001355 elif _WIFSTOPPED(sts):
1356 self.returncode = -_WSTOPSIG(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001357 else:
1358 # Should never happen
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001359 raise SubprocessError("Unknown child exit status!")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001360
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001361
Brett Cannon84df1e62010-05-14 00:33:40 +00001362 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
Andrew Svetlov1d960fe2012-12-24 20:08:53 +02001363 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001364 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001365 attribute.
1366
1367 This method is called by __del__, so it cannot reference anything
1368 outside of the local scope (nor can any methods it calls).
1369
1370 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001371 if self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001372 if not self._waitpid_lock.acquire(False):
1373 # Something else is busy calling waitpid. Don't allow two
1374 # at once. We know nothing yet.
1375 return None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001376 try:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001377 if self.returncode is not None:
1378 return self.returncode # Another thread waited.
Brett Cannon84df1e62010-05-14 00:33:40 +00001379 pid, sts = _waitpid(self.pid, _WNOHANG)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001380 if pid == self.pid:
1381 self._handle_exitstatus(sts)
Andrew Svetlovad28c7f2012-12-18 22:02:39 +02001382 except OSError as e:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001383 if _deadstate is not None:
1384 self.returncode = _deadstate
Andrew Svetlov08bab072012-12-24 20:06:35 +02001385 elif e.errno == _ECHILD:
Gregory P. Smith39051712012-09-29 11:40:38 -07001386 # This happens if SIGCLD is set to be ignored or
1387 # waiting for child processes has otherwise been
1388 # disabled for our process. This child is dead, we
1389 # can't get the status.
1390 # http://bugs.python.org/issue15756
1391 self.returncode = 0
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001392 finally:
1393 self._waitpid_lock.release()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001394 return self.returncode
1395
1396
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001397 def _try_wait(self, wait_flags):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001398 """All callers to this function MUST hold self._waitpid_lock."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001399 try:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001400 (pid, sts) = os.waitpid(self.pid, wait_flags)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001401 except ChildProcessError:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001402 # This happens if SIGCLD is set to be ignored or waiting
1403 # for child processes has otherwise been disabled for our
1404 # process. This child is dead, we can't get the status.
1405 pid = self.pid
1406 sts = 0
1407 return (pid, sts)
1408
1409
Gregory P. Smith82604e02016-11-20 16:31:07 -08001410 def wait(self, timeout=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001411 """Wait for child process to terminate. Returns returncode
1412 attribute."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001413 if self.returncode is not None:
1414 return self.returncode
Reid Kleckner2b228f02011-03-16 16:57:54 -04001415
Gregory P. Smith82604e02016-11-20 16:31:07 -08001416 if timeout is not None:
1417 endtime = _time() + timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001418 # Enter a busy loop if we have a timeout. This busy loop was
1419 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1420 delay = 0.0005 # 500 us -> initial delay of 1 ms
1421 while True:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001422 if self._waitpid_lock.acquire(False):
1423 try:
1424 if self.returncode is not None:
1425 break # Another thread waited.
1426 (pid, sts) = self._try_wait(os.WNOHANG)
1427 assert pid == self.pid or pid == 0
1428 if pid == self.pid:
1429 self._handle_exitstatus(sts)
1430 break
1431 finally:
1432 self._waitpid_lock.release()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001433 remaining = self._remaining_time(endtime)
1434 if remaining <= 0:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001435 raise TimeoutExpired(self.args, timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001436 delay = min(delay * 2, remaining, .05)
1437 time.sleep(delay)
Gregory P. Smithf328d792012-11-10 21:06:18 -08001438 else:
1439 while self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001440 with self._waitpid_lock:
1441 if self.returncode is not None:
1442 break # Another thread waited.
1443 (pid, sts) = self._try_wait(0)
1444 # Check the pid and loop as waitpid has been known to
1445 # return 0 even without WNOHANG in odd situations.
1446 # http://bugs.python.org/issue14396.
1447 if pid == self.pid:
1448 self._handle_exitstatus(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001449 return self.returncode
1450
1451
Reid Kleckner2b228f02011-03-16 16:57:54 -04001452 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001453 if self.stdin and not self._communication_started:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001454 # Flush stdio buffer. This might block, if the user has
1455 # been writing to .stdin in an uncontrolled fashion.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001456 try:
1457 self.stdin.flush()
1458 except BrokenPipeError:
1459 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001460 if not input:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001461 try:
1462 self.stdin.close()
1463 except BrokenPipeError:
1464 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001465
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001466 stdout = None
1467 stderr = None
1468
1469 # Only create this mapping if we haven't already.
1470 if not self._communication_started:
1471 self._fileobj2output = {}
1472 if self.stdout:
1473 self._fileobj2output[self.stdout] = []
1474 if self.stderr:
1475 self._fileobj2output[self.stderr] = []
1476
1477 if self.stdout:
1478 stdout = self._fileobj2output[self.stdout]
1479 if self.stderr:
1480 stderr = self._fileobj2output[self.stderr]
1481
1482 self._save_input(input)
1483
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001484 if self._input:
1485 input_view = memoryview(self._input)
1486
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001487 with _PopenSelector() as selector:
1488 if self.stdin and input:
1489 selector.register(self.stdin, selectors.EVENT_WRITE)
1490 if self.stdout:
1491 selector.register(self.stdout, selectors.EVENT_READ)
1492 if self.stderr:
1493 selector.register(self.stderr, selectors.EVENT_READ)
1494
1495 while selector.get_map():
1496 timeout = self._remaining_time(endtime)
1497 if timeout is not None and timeout < 0:
1498 raise TimeoutExpired(self.args, orig_timeout)
1499
1500 ready = selector.select(timeout)
1501 self._check_timeout(endtime, orig_timeout)
1502
1503 # XXX Rewrite these to use non-blocking I/O on the file
1504 # objects; they are no longer using C stdio!
1505
1506 for key, events in ready:
1507 if key.fileobj is self.stdin:
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001508 chunk = input_view[self._input_offset :
1509 self._input_offset + _PIPE_BUF]
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001510 try:
1511 self._input_offset += os.write(key.fd, chunk)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001512 except BrokenPipeError:
1513 selector.unregister(key.fileobj)
1514 key.fileobj.close()
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001515 else:
1516 if self._input_offset >= len(self._input):
1517 selector.unregister(key.fileobj)
1518 key.fileobj.close()
1519 elif key.fileobj in (self.stdout, self.stderr):
Gregory P. Smith7b83b182013-12-08 10:58:28 -08001520 data = os.read(key.fd, 32768)
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001521 if not data:
1522 selector.unregister(key.fileobj)
1523 key.fileobj.close()
1524 self._fileobj2output[key.fileobj].append(data)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001525
1526 self.wait(timeout=self._remaining_time(endtime))
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001527
1528 # All data exchanged. Translate lists into strings.
1529 if stdout is not None:
1530 stdout = b''.join(stdout)
1531 if stderr is not None:
1532 stderr = b''.join(stderr)
1533
1534 # Translate newlines, if requested.
1535 # This also turns bytes into strings.
Steve Dower050acae2016-09-06 20:16:17 -07001536 if self.encoding or self.errors or self.universal_newlines:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001537 if stdout is not None:
1538 stdout = self._translate_newlines(stdout,
Steve Dower050acae2016-09-06 20:16:17 -07001539 self.stdout.encoding,
1540 self.stdout.errors)
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001541 if stderr is not None:
1542 stderr = self._translate_newlines(stderr,
Steve Dower050acae2016-09-06 20:16:17 -07001543 self.stderr.encoding,
1544 self.stderr.errors)
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001545
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001546 return (stdout, stderr)
1547
1548
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001549 def _save_input(self, input):
1550 # This method is called from the _communicate_with_*() methods
1551 # so that if we time out while communicating, we can continue
1552 # sending input if we retry.
1553 if self.stdin and self._input is None:
1554 self._input_offset = 0
1555 self._input = input
Steve Dower050acae2016-09-06 20:16:17 -07001556 if input is not None and (
1557 self.encoding or self.errors or self.universal_newlines):
1558 self._input = self._input.encode(self.stdin.encoding,
1559 self.stdin.errors)
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001560
1561
Christian Heimesa342c012008-04-20 21:01:16 +00001562 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001563 """Send a signal to the process."""
1564 # Skip signalling a process that we know has already died.
1565 if self.returncode is None:
1566 os.kill(self.pid, sig)
Christian Heimesa342c012008-04-20 21:01:16 +00001567
1568 def terminate(self):
1569 """Terminate the process with SIGTERM
1570 """
1571 self.send_signal(signal.SIGTERM)
1572
1573 def kill(self):
1574 """Kill the process with SIGKILL
1575 """
1576 self.send_signal(signal.SIGKILL)