blob: 696617697047d245de5f3a951834b616b926aaca [file] [log] [blame]
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001# subprocess - Subprocesses with accessible I/O streams
2#
Tim Peterse718f612004-10-12 21:51:32 +00003# For more information about this module, see PEP 324.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00004#
Peter Astrand3a708df2005-09-23 17:37:29 +00005# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00006#
Peter Astrand69bf13f2005-02-14 08:56:32 +00007# Licensed to PSF under a Contributor Agreement.
Peter Astrand3a708df2005-09-23 17:37:29 +00008# See http://www.python.org/2.4/license for licensing details.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00009
Martin Panter4afdca02016-10-25 22:20:48 +000010r"""Subprocesses with accessible I/O streams
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000011
Fredrik Lundh15aaacc2004-10-17 14:47:05 +000012This module allows you to spawn processes, connect to their
Martin Panter4afdca02016-10-25 22:20:48 +000013input/output/error pipes, and obtain their return codes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000014
Martin Panter4afdca02016-10-25 22:20:48 +000015For a complete description of this module see the Python documentation.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000016
Martin Panter4afdca02016-10-25 22:20:48 +000017Main API
18========
19run(...): Runs a command, waits for it to complete, then returns a
20 CompletedProcess instance.
21Popen(...): A class for flexibly executing a command in a new process
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000022
Martin Panter4afdca02016-10-25 22:20:48 +000023Constants
24---------
25DEVNULL: Special value that indicates that os.devnull should be used
26PIPE: Special value that indicates a pipe should be created
27STDOUT: Special value that indicates that stderr should go to stdout
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000028
29
Martin Panter4afdca02016-10-25 22:20:48 +000030Older API
31=========
32call(...): Runs a command, waits for it to complete, then returns
33 the return code.
34check_call(...): Same as call() but raises CalledProcessError()
35 if return code is not 0
36check_output(...): Same as check_call() but returns the contents of
37 stdout instead of a return code
38getoutput(...): Runs a command in the shell, waits for it to complete,
39 then returns the output
40getstatusoutput(...): Runs a command in the shell, waits for it to complete,
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -070041 then returns a (exitcode, output) tuple
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000042"""
43
Zachary Ware880d42a2018-09-10 16:16:08 -070044import builtins
45import errno
Guido van Rossumfa0054a2007-05-24 04:05:35 +000046import io
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000047import os
Reid Kleckner31aa7dd2011-03-14 12:02:10 -040048import time
Christian Heimesa342c012008-04-20 21:01:16 +000049import signal
Zachary Ware880d42a2018-09-10 16:16:08 -070050import sys
51import threading
Gregory P. Smithd23047b2010-12-04 09:10:44 +000052import warnings
Victor Stinnerae586492014-09-02 23:18:25 +020053from time import monotonic as _time
Fredrik Lundh5b3687d2004-10-12 15:26:28 +000054
Zachary Ware880d42a2018-09-10 16:16:08 -070055
56__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
57 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
58 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
59 # NOTE: We intentionally exclude list2cmdline as it is
60 # considered an internal implementation detail. issue10838.
61
62try:
63 import msvcrt
64 import _winapi
65 _mswindows = True
66except ModuleNotFoundError:
67 _mswindows = False
68 import _posixsubprocess
69 import select
70 import selectors
71else:
72 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
73 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
74 STD_ERROR_HANDLE, SW_HIDE,
75 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW,
76 ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS,
77 HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS,
78 NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS,
79 CREATE_NO_WINDOW, DETACHED_PROCESS,
80 CREATE_DEFAULT_ERROR_MODE, CREATE_BREAKAWAY_FROM_JOB)
81
82 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
83 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
84 "STD_ERROR_HANDLE", "SW_HIDE",
85 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW",
86 "STARTUPINFO",
87 "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS",
88 "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS",
89 "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS",
90 "CREATE_NO_WINDOW", "DETACHED_PROCESS",
91 "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"])
92
93
Peter Astrand454f7672005-01-01 09:36:35 +000094# Exception classes used by this module.
Gregory P. Smith54d412e2011-03-14 14:08:43 -040095class SubprocessError(Exception): pass
96
97
98class CalledProcessError(SubprocessError):
Martin Panter4afdca02016-10-25 22:20:48 +000099 """Raised when run() is called with check=True and the process
100 returns a non-zero exit status.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +0000101
Martin Panter4afdca02016-10-25 22:20:48 +0000102 Attributes:
103 cmd, returncode, stdout, stderr, output
Georg Brandlf9734072008-12-07 15:30:06 +0000104 """
Gregory P. Smith6e730002015-04-14 16:14:25 -0700105 def __init__(self, returncode, cmd, output=None, stderr=None):
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000106 self.returncode = returncode
107 self.cmd = cmd
Georg Brandlf9734072008-12-07 15:30:06 +0000108 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -0700109 self.stderr = stderr
110
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000111 def __str__(self):
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)d6da7602016-06-03 06:14:06 +0000112 if self.returncode and self.returncode < 0:
113 try:
114 return "Command '%s' died with %r." % (
115 self.cmd, signal.Signals(-self.returncode))
116 except ValueError:
117 return "Command '%s' died with unknown signal %d." % (
118 self.cmd, -self.returncode)
119 else:
120 return "Command '%s' returned non-zero exit status %d." % (
121 self.cmd, self.returncode)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000122
Gregory P. Smith6e730002015-04-14 16:14:25 -0700123 @property
124 def stdout(self):
125 """Alias for output attribute, to match stderr"""
126 return self.output
127
128 @stdout.setter
129 def stdout(self, value):
130 # There's no obvious reason to set this, but allow it anyway so
131 # .stdout is a transparent alias for .output
132 self.output = value
133
Peter Astrand454f7672005-01-01 09:36:35 +0000134
Gregory P. Smith54d412e2011-03-14 14:08:43 -0400135class TimeoutExpired(SubprocessError):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400136 """This exception is raised when the timeout expires while waiting for a
137 child process.
Martin Panter4afdca02016-10-25 22:20:48 +0000138
139 Attributes:
140 cmd, output, stdout, stderr, timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400141 """
Gregory P. Smith6e730002015-04-14 16:14:25 -0700142 def __init__(self, cmd, timeout, output=None, stderr=None):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400143 self.cmd = cmd
Reid Kleckner2b228f02011-03-16 16:57:54 -0400144 self.timeout = timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400145 self.output = output
Gregory P. Smith6e730002015-04-14 16:14:25 -0700146 self.stderr = stderr
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400147
148 def __str__(self):
149 return ("Command '%s' timed out after %s seconds" %
150 (self.cmd, self.timeout))
151
Gregory P. Smith6e730002015-04-14 16:14:25 -0700152 @property
153 def stdout(self):
154 return self.output
155
156 @stdout.setter
157 def stdout(self, value):
158 # There's no obvious reason to set this, but allow it anyway so
159 # .stdout is a transparent alias for .output
160 self.output = value
161
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400162
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700163if _mswindows:
Brian Curtin1ce6b582010-04-24 16:19:22 +0000164 class STARTUPINFO:
Subhendu Ghoshae160bb2017-02-25 20:29:05 +0530165 def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None,
Segev Finerb2a60832017-12-18 11:28:19 +0200166 hStdError=None, wShowWindow=0, lpAttributeList=None):
Subhendu Ghoshae160bb2017-02-25 20:29:05 +0530167 self.dwFlags = dwFlags
168 self.hStdInput = hStdInput
169 self.hStdOutput = hStdOutput
170 self.hStdError = hStdError
171 self.wShowWindow = wShowWindow
Segev Finerb2a60832017-12-18 11:28:19 +0200172 self.lpAttributeList = lpAttributeList or {"handle_list": []}
Victor Stinner483422f2018-07-05 22:54:17 +0200173
174 def copy(self):
175 attr_list = self.lpAttributeList.copy()
176 if 'handle_list' in attr_list:
177 attr_list['handle_list'] = list(attr_list['handle_list'])
178
179 return STARTUPINFO(dwFlags=self.dwFlags,
180 hStdInput=self.hStdInput,
181 hStdOutput=self.hStdOutput,
182 hStdError=self.hStdError,
183 wShowWindow=self.wShowWindow,
184 lpAttributeList=attr_list)
185
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200186
187 class Handle(int):
188 closed = False
189
190 def Close(self, CloseHandle=_winapi.CloseHandle):
191 if not self.closed:
192 self.closed = True
193 CloseHandle(self)
194
195 def Detach(self):
196 if not self.closed:
197 self.closed = True
198 return int(self)
199 raise ValueError("already closed")
200
201 def __repr__(self):
Serhiy Storchaka465e60e2014-07-25 23:36:00 +0300202 return "%s(%d)" % (self.__class__.__name__, int(self))
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200203
204 __del__ = Close
205 __str__ = __repr__
Zachary Ware880d42a2018-09-10 16:16:08 -0700206else:
207 # When select or poll has indicated that the file is writable,
208 # we can write up to _PIPE_BUF bytes without risk of blocking.
209 # POSIX defines PIPE_BUF as >= 512.
210 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
211
212 # poll/select have the advantage of not requiring any extra file
213 # descriptor, contrarily to epoll/kqueue (also, they require a single
214 # syscall).
215 if hasattr(selectors, 'PollSelector'):
216 _PopenSelector = selectors.PollSelector
217 else:
218 _PopenSelector = selectors.SelectSelector
Antoine Pitrou23bba4c2012-04-18 20:51:15 +0200219
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000220
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200221# This lists holds Popen instances for which the underlying process had not
222# exited at the time its __del__ method got called: those processes are wait()ed
223# for synchronously from _cleanup() when a new Popen object is created, to avoid
224# zombie processes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000225_active = []
226
227def _cleanup():
228 for inst in _active[:]:
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000229 res = inst._internal_poll(_deadstate=sys.maxsize)
Charles-François Natali134a8ba2011-08-18 18:49:39 +0200230 if res is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000231 try:
232 _active.remove(inst)
233 except ValueError:
234 # This can happen if two threads create a new Popen instance.
235 # It's harmless that it was already removed, so ignore.
236 pass
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000237
238PIPE = -1
239STDOUT = -2
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200240DEVNULL = -3
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000241
242
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200243# XXX This function is only used by multiprocessing and the test suite,
244# but it's here so that it can be imported when Python is compiled without
245# threads.
246
Victor Stinner9def2842016-01-18 12:15:08 +0100247def _optim_args_from_interpreter_flags():
248 """Return a list of command-line arguments reproducing the current
249 optimization settings in sys.flags."""
250 args = []
251 value = sys.flags.optimize
252 if value > 0:
253 args.append('-' + 'O' * value)
254 return args
255
256
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200257def _args_from_interpreter_flags():
258 """Return a list of command-line arguments reproducing the current
Victor Stinner747f48e2017-12-12 22:59:48 +0100259 settings in sys.flags, sys.warnoptions and sys._xoptions."""
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200260 flag_opt_map = {
261 'debug': 'd',
262 # 'inspect': 'i',
263 # 'interactive': 'i',
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200264 'dont_write_bytecode': 'B',
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200265 'no_site': 'S',
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200266 'verbose': 'v',
267 'bytes_warning': 'b',
268 'quiet': 'q',
Victor Stinner9def2842016-01-18 12:15:08 +0100269 # -O is handled in _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200270 }
Victor Stinner9def2842016-01-18 12:15:08 +0100271 args = _optim_args_from_interpreter_flags()
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200272 for flag, opt in flag_opt_map.items():
273 v = getattr(sys.flags, flag)
274 if v > 0:
275 args.append('-' + opt * v)
Victor Stinnerf39b6742017-11-20 15:24:56 -0800276
Victor Stinner9de36322018-11-23 17:54:20 +0100277 if sys.flags.isolated:
278 args.append('-I')
279 else:
280 if sys.flags.ignore_environment:
281 args.append('-E')
282 if sys.flags.no_user_site:
283 args.append('-s')
284
Victor Stinnerf39b6742017-11-20 15:24:56 -0800285 # -W options
Victor Stinner747f48e2017-12-12 22:59:48 +0100286 warnopts = sys.warnoptions[:]
287 bytes_warning = sys.flags.bytes_warning
288 xoptions = getattr(sys, '_xoptions', {})
289 dev_mode = ('dev' in xoptions)
290
291 if bytes_warning > 1:
292 warnopts.remove("error::BytesWarning")
293 elif bytes_warning:
294 warnopts.remove("default::BytesWarning")
295 if dev_mode:
296 warnopts.remove('default')
297 for opt in warnopts:
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200298 args.append('-W' + opt)
Victor Stinnerf39b6742017-11-20 15:24:56 -0800299
300 # -X options
Victor Stinner747f48e2017-12-12 22:59:48 +0100301 if dev_mode:
Victor Stinnerf39b6742017-11-20 15:24:56 -0800302 args.extend(('-X', 'dev'))
303 for opt in ('faulthandler', 'tracemalloc', 'importtime',
Victor Stinner91106cd2017-12-13 12:29:09 +0100304 'showalloccount', 'showrefcount', 'utf8'):
Victor Stinnerf39b6742017-11-20 15:24:56 -0800305 if opt in xoptions:
306 value = xoptions[opt]
307 if value is True:
308 arg = opt
309 else:
310 arg = '%s=%s' % (opt, value)
311 args.extend(('-X', arg))
312
Antoine Pitrouebdcd852012-05-18 18:33:07 +0200313 return args
314
315
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400316def call(*popenargs, timeout=None, **kwargs):
317 """Run command with arguments. Wait for command to complete or
318 timeout, then return the returncode attribute.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000319
320 The arguments are the same as for the Popen constructor. Example:
321
322 retcode = call(["ls", "-l"])
323 """
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200324 with Popen(*popenargs, **kwargs) as p:
325 try:
326 return p.wait(timeout=timeout)
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800327 except: # Including KeyboardInterrupt, wait handled that.
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200328 p.kill()
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800329 # We don't call p.wait() again as p.__exit__ does that for us.
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200330 raise
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000331
332
Peter Astrand454f7672005-01-01 09:36:35 +0000333def check_call(*popenargs, **kwargs):
334 """Run command with arguments. Wait for command to complete. If
335 the exit code was zero then return, otherwise raise
336 CalledProcessError. The CalledProcessError object will have the
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000337 return code in the returncode attribute.
Peter Astrand454f7672005-01-01 09:36:35 +0000338
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400339 The arguments are the same as for the call function. Example:
Peter Astrand454f7672005-01-01 09:36:35 +0000340
341 check_call(["ls", "-l"])
342 """
343 retcode = call(*popenargs, **kwargs)
Peter Astrand454f7672005-01-01 09:36:35 +0000344 if retcode:
Georg Brandlf9734072008-12-07 15:30:06 +0000345 cmd = kwargs.get("args")
346 if cmd is None:
347 cmd = popenargs[0]
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000348 raise CalledProcessError(retcode, cmd)
Georg Brandlf9734072008-12-07 15:30:06 +0000349 return 0
350
351
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400352def check_output(*popenargs, timeout=None, **kwargs):
Gregory P. Smith91110f52013-03-19 23:25:16 -0700353 r"""Run command with arguments and return its output.
Georg Brandlf9734072008-12-07 15:30:06 +0000354
355 If the exit code was non-zero it raises a CalledProcessError. The
356 CalledProcessError object will have the return code in the returncode
357 attribute and output in the output attribute.
358
359 The arguments are the same as for the Popen constructor. Example:
360
361 >>> check_output(["ls", "-l", "/dev/null"])
Georg Brandl2708f3a2009-12-20 14:38:23 +0000362 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
Georg Brandlf9734072008-12-07 15:30:06 +0000363
364 The stdout argument is not allowed as it is used internally.
Georg Brandl127d4702009-12-28 08:10:38 +0000365 To capture standard error in the result, use stderr=STDOUT.
Georg Brandlf9734072008-12-07 15:30:06 +0000366
367 >>> check_output(["/bin/sh", "-c",
Georg Brandl2708f3a2009-12-20 14:38:23 +0000368 ... "ls -l non_existent_file ; exit 0"],
Georg Brandl127d4702009-12-28 08:10:38 +0000369 ... stderr=STDOUT)
Georg Brandl2708f3a2009-12-20 14:38:23 +0000370 b'ls: non_existent_file: No such file or directory\n'
Gregory P. Smith91110f52013-03-19 23:25:16 -0700371
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300372 There is an additional optional argument, "input", allowing you to
373 pass a string to the subprocess's stdin. If you use this argument
374 you may not also use the Popen constructor's "stdin" argument, as
375 it too will be used internally. Example:
376
377 >>> check_output(["sed", "-e", "s/foo/bar/"],
378 ... input=b"when in the course of fooman events\n")
379 b'when in the course of barman events\n'
380
andyclegg7fed7bd2017-10-23 03:01:19 +0100381 By default, all communication is in bytes, and therefore any "input"
382 should be bytes, and the return value wil be bytes. If in text mode,
383 any "input" should be a string, and the return value will be a string
384 decoded according to locale encoding, or by "encoding" if set. Text mode
385 is triggered by setting any of text, encoding, errors or universal_newlines.
Georg Brandlf9734072008-12-07 15:30:06 +0000386 """
387 if 'stdout' in kwargs:
388 raise ValueError('stdout argument not allowed, it will be overridden.')
Gregory P. Smith6e730002015-04-14 16:14:25 -0700389
390 if 'input' in kwargs and kwargs['input'] is None:
391 # Explicitly passing input=None was previously equivalent to passing an
392 # empty string. That is maintained here for backwards compatibility.
393 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
394
395 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
396 **kwargs).stdout
397
398
399class CompletedProcess(object):
400 """A process that has finished running.
401
402 This is returned by run().
403
404 Attributes:
405 args: The list or str args passed to run().
406 returncode: The exit code of the process, negative for signals.
407 stdout: The standard output (None if not captured).
408 stderr: The standard error (None if not captured).
409 """
410 def __init__(self, args, returncode, stdout=None, stderr=None):
411 self.args = args
412 self.returncode = returncode
413 self.stdout = stdout
414 self.stderr = stderr
415
416 def __repr__(self):
417 args = ['args={!r}'.format(self.args),
418 'returncode={!r}'.format(self.returncode)]
419 if self.stdout is not None:
420 args.append('stdout={!r}'.format(self.stdout))
421 if self.stderr is not None:
422 args.append('stderr={!r}'.format(self.stderr))
423 return "{}({})".format(type(self).__name__, ', '.join(args))
424
425 def check_returncode(self):
426 """Raise CalledProcessError if the exit code is non-zero."""
427 if self.returncode:
428 raise CalledProcessError(self.returncode, self.args, self.stdout,
429 self.stderr)
430
431
Bo Baylesce0f33d2018-01-30 00:40:39 -0600432def run(*popenargs,
433 input=None, capture_output=False, timeout=None, check=False, **kwargs):
Gregory P. Smith6e730002015-04-14 16:14:25 -0700434 """Run command with arguments and return a CompletedProcess instance.
435
436 The returned instance will have attributes args, returncode, stdout and
437 stderr. By default, stdout and stderr are not captured, and those attributes
438 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
439
440 If check is True and the exit code was non-zero, it raises a
441 CalledProcessError. The CalledProcessError object will have the return code
442 in the returncode attribute, and output & stderr attributes if those streams
443 were captured.
444
445 If timeout is given, and the process takes too long, a TimeoutExpired
446 exception will be raised.
447
448 There is an optional argument "input", allowing you to
andyclegg7fed7bd2017-10-23 03:01:19 +0100449 pass bytes or a string to the subprocess's stdin. If you use this argument
Gregory P. Smith6e730002015-04-14 16:14:25 -0700450 you may not also use the Popen constructor's "stdin" argument, as
451 it will be used internally.
452
andyclegg7fed7bd2017-10-23 03:01:19 +0100453 By default, all communication is in bytes, and therefore any "input" should
454 be bytes, and the stdout and stderr will be bytes. If in text mode, any
455 "input" should be a string, and stdout and stderr will be strings decoded
456 according to locale encoding, or by "encoding" if set. Text mode is
457 triggered by setting any of text, encoding, errors or universal_newlines.
Gregory P. Smith6e730002015-04-14 16:14:25 -0700458
andyclegg7fed7bd2017-10-23 03:01:19 +0100459 The other arguments are the same as for the Popen constructor.
Gregory P. Smith6e730002015-04-14 16:14:25 -0700460 """
461 if input is not None:
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300462 if 'stdin' in kwargs:
463 raise ValueError('stdin and input arguments may not both be used.')
Serhiy Storchakafcd9f222013-04-22 20:20:54 +0300464 kwargs['stdin'] = PIPE
Gregory P. Smith6e730002015-04-14 16:14:25 -0700465
Bo Baylesce0f33d2018-01-30 00:40:39 -0600466 if capture_output:
467 if ('stdout' in kwargs) or ('stderr' in kwargs):
468 raise ValueError('stdout and stderr arguments may not be used '
469 'with capture_output.')
470 kwargs['stdout'] = PIPE
471 kwargs['stderr'] = PIPE
472
Gregory P. Smith6e730002015-04-14 16:14:25 -0700473 with Popen(*popenargs, **kwargs) as process:
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200474 try:
Gregory P. Smith6e730002015-04-14 16:14:25 -0700475 stdout, stderr = process.communicate(input, timeout=timeout)
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200476 except TimeoutExpired:
477 process.kill()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700478 stdout, stderr = process.communicate()
479 raise TimeoutExpired(process.args, timeout, output=stdout,
480 stderr=stderr)
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800481 except: # Including KeyboardInterrupt, communicate handled that.
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200482 process.kill()
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800483 # We don't call process.wait() as .__exit__ does that for us.
Victor Stinnerc15c88c2011-09-01 23:45:04 +0200484 raise
485 retcode = process.poll()
Gregory P. Smith6e730002015-04-14 16:14:25 -0700486 if check and retcode:
487 raise CalledProcessError(retcode, process.args,
488 output=stdout, stderr=stderr)
489 return CompletedProcess(process.args, retcode, stdout, stderr)
Peter Astrand454f7672005-01-01 09:36:35 +0000490
491
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000492def list2cmdline(seq):
493 """
494 Translate a sequence of arguments into a command line
495 string, using the same rules as the MS C runtime:
496
497 1) Arguments are delimited by white space, which is either a
498 space or a tab.
499
500 2) A string surrounded by double quotation marks is
501 interpreted as a single argument, regardless of white space
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000502 contained within. A quoted string can be embedded in an
503 argument.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000504
505 3) A double quotation mark preceded by a backslash is
506 interpreted as a literal double quotation mark.
507
508 4) Backslashes are interpreted literally, unless they
509 immediately precede a double quotation mark.
510
511 5) If backslashes immediately precede a double quotation mark,
512 every pair of backslashes is interpreted as a literal
513 backslash. If the number of backslashes is odd, the last
514 backslash escapes the next double quotation mark as
515 described in rule 3.
516 """
517
518 # See
Eric Smith3c573af2009-11-09 15:23:15 +0000519 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
520 # or search http://msdn.microsoft.com for
521 # "Parsing C++ Command-Line Arguments"
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000522 result = []
523 needquote = False
524 for arg in seq:
525 bs_buf = []
526
527 # Add a space to separate this argument from the others
528 if result:
529 result.append(' ')
530
Jean-Paul Calderone1ddd4072010-06-18 20:03:54 +0000531 needquote = (" " in arg) or ("\t" in arg) or not arg
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000532 if needquote:
533 result.append('"')
534
535 for c in arg:
536 if c == '\\':
Tim Peterse718f612004-10-12 21:51:32 +0000537 # Don't know if we need to double yet.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000538 bs_buf.append(c)
539 elif c == '"':
Christian Heimesfdab48e2008-01-20 09:06:41 +0000540 # Double backslashes.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000541 result.append('\\' * len(bs_buf)*2)
542 bs_buf = []
543 result.append('\\"')
544 else:
545 # Normal char
546 if bs_buf:
547 result.extend(bs_buf)
548 bs_buf = []
549 result.append(c)
550
Christian Heimesfdab48e2008-01-20 09:06:41 +0000551 # Add remaining backslashes, if any.
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000552 if bs_buf:
553 result.extend(bs_buf)
554
555 if needquote:
Peter Astrand7e78ade2005-03-03 21:10:23 +0000556 result.extend(bs_buf)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000557 result.append('"')
558
559 return ''.join(result)
560
561
Brett Cannona23810f2008-05-26 19:04:21 +0000562# Various tools for executing commands and looking at their output and status.
563#
Brett Cannona23810f2008-05-26 19:04:21 +0000564
565def getstatusoutput(cmd):
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700566 """Return (exitcode, output) of executing cmd in a shell.
Brett Cannona23810f2008-05-26 19:04:21 +0000567
Tim Golden60798142013-11-05 12:57:25 +0000568 Execute the string 'cmd' in a shell with 'check_output' and
Steve Dower050acae2016-09-06 20:16:17 -0700569 return a 2-tuple (status, output). The locale encoding is used
570 to decode the output and process newlines.
Tim Golden60798142013-11-05 12:57:25 +0000571
572 A trailing newline is stripped from the output.
573 The exit status for the command can be interpreted
574 according to the rules for the function 'wait'. Example:
Brett Cannona23810f2008-05-26 19:04:21 +0000575
576 >>> import subprocess
577 >>> subprocess.getstatusoutput('ls /bin/ls')
578 (0, '/bin/ls')
579 >>> subprocess.getstatusoutput('cat /bin/junk')
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700580 (1, 'cat: /bin/junk: No such file or directory')
Brett Cannona23810f2008-05-26 19:04:21 +0000581 >>> subprocess.getstatusoutput('/bin/junk')
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700582 (127, 'sh: /bin/junk: not found')
583 >>> subprocess.getstatusoutput('/bin/kill $$')
584 (-15, '')
Brett Cannona23810f2008-05-26 19:04:21 +0000585 """
Tim Goldene0041752013-11-03 12:53:17 +0000586 try:
andyclegg7fed7bd2017-10-23 03:01:19 +0100587 data = check_output(cmd, shell=True, text=True, stderr=STDOUT)
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700588 exitcode = 0
Tim Goldene0041752013-11-03 12:53:17 +0000589 except CalledProcessError as ex:
590 data = ex.output
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700591 exitcode = ex.returncode
Tim Goldene0041752013-11-03 12:53:17 +0000592 if data[-1:] == '\n':
593 data = data[:-1]
Gregory P. Smith2eb0cb42017-09-07 16:11:02 -0700594 return exitcode, data
Brett Cannona23810f2008-05-26 19:04:21 +0000595
596def getoutput(cmd):
597 """Return output (stdout or stderr) of executing cmd in a shell.
598
599 Like getstatusoutput(), except the exit status is ignored and the return
600 value is a string containing the command's output. Example:
601
602 >>> import subprocess
603 >>> subprocess.getoutput('ls /bin/ls')
604 '/bin/ls'
605 """
606 return getstatusoutput(cmd)[1]
607
608
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000609class Popen(object):
Martin Panter4afdca02016-10-25 22:20:48 +0000610 """ Execute a child program in a new process.
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200611
Martin Panter4afdca02016-10-25 22:20:48 +0000612 For a complete description of the arguments see the Python documentation.
613
614 Arguments:
615 args: A string, or a sequence of program arguments.
616
617 bufsize: supplied as the buffering argument to the open() function when
618 creating the stdin/stdout/stderr pipe file objects
619
620 executable: A replacement program to execute.
621
622 stdin, stdout and stderr: These specify the executed programs' standard
623 input, standard output and standard error file handles, respectively.
624
625 preexec_fn: (POSIX only) An object to be called in the child process
626 just before the child is executed.
627
628 close_fds: Controls closing or inheriting of file descriptors.
629
630 shell: If true, the command will be executed through the shell.
631
632 cwd: Sets the current directory before the child is executed.
633
634 env: Defines the environment variables for the new process.
635
andyclegg7fed7bd2017-10-23 03:01:19 +0100636 text: If true, decode stdin, stdout and stderr using the given encoding
637 (if set) or the system default otherwise.
638
639 universal_newlines: Alias of text, provided for backwards compatibility.
Martin Panter4afdca02016-10-25 22:20:48 +0000640
641 startupinfo and creationflags (Windows only)
642
643 restore_signals (POSIX only)
644
645 start_new_session (POSIX only)
646
647 pass_fds (POSIX only)
648
Martin Panter3dca6242016-10-25 23:41:42 +0000649 encoding and errors: Text mode encoding and error handling to use for
650 file objects stdin, stdout and stderr.
651
Martin Panter4afdca02016-10-25 22:20:48 +0000652 Attributes:
653 stdin, stdout, stderr, pid, returncode
654 """
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200655 _child_created = False # Set here since __del__ checks it
656
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700657 def __init__(self, args, bufsize=-1, executable=None,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000658 stdin=None, stdout=None, stderr=None,
Segev Finerb2a60832017-12-18 11:28:19 +0200659 preexec_fn=None, close_fds=True,
andyclegg7fed7bd2017-10-23 03:01:19 +0100660 shell=False, cwd=None, env=None, universal_newlines=None,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000661 startupinfo=None, creationflags=0,
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +0000662 restore_signals=True, start_new_session=False,
andyclegg7fed7bd2017-10-23 03:01:19 +0100663 pass_fds=(), *, encoding=None, errors=None, text=None):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000664 """Create new Popen instance."""
665 _cleanup()
Gregory P. Smithd65ba512014-04-23 00:27:17 -0700666 # Held while anything is calling waitpid before returncode has been
667 # updated to prevent clobbering returncode if wait() or poll() are
668 # called from multiple threads at once. After acquiring the lock,
669 # code must re-check self.returncode to see if another thread just
670 # finished a waitpid() call.
671 self._waitpid_lock = threading.Lock()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000672
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400673 self._input = None
674 self._communication_started = False
Guido van Rossum46a05a72007-06-07 21:56:45 +0000675 if bufsize is None:
Gregory P. Smitha1ed5392013-03-23 11:44:25 -0700676 bufsize = -1 # Restore default
Walter Dörwaldaa97f042007-05-03 21:05:51 +0000677 if not isinstance(bufsize, int):
Peter Astrand738131d2004-11-30 21:04:45 +0000678 raise TypeError("bufsize must be an integer")
679
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700680 if _mswindows:
Tim Peterse8374a52004-10-13 03:15:00 +0000681 if preexec_fn is not None:
682 raise ValueError("preexec_fn is not supported on Windows "
683 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000684 else:
685 # POSIX
Gregory P. Smith8edd99d2010-12-14 13:43:30 +0000686 if pass_fds and not close_fds:
687 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
688 close_fds = True
Tim Peterse8374a52004-10-13 03:15:00 +0000689 if startupinfo is not None:
690 raise ValueError("startupinfo is only supported on Windows "
691 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000692 if creationflags != 0:
Tim Peterse8374a52004-10-13 03:15:00 +0000693 raise ValueError("creationflags is only supported on Windows "
694 "platforms")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000695
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400696 self.args = args
Tim Peterse718f612004-10-12 21:51:32 +0000697 self.stdin = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000698 self.stdout = None
699 self.stderr = None
700 self.pid = None
701 self.returncode = None
Steve Dower050acae2016-09-06 20:16:17 -0700702 self.encoding = encoding
703 self.errors = errors
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000704
andyclegg7fed7bd2017-10-23 03:01:19 +0100705 # Validate the combinations of text and universal_newlines
706 if (text is not None and universal_newlines is not None
707 and bool(universal_newlines) != bool(text)):
708 raise SubprocessError('Cannot disambiguate when both text '
709 'and universal_newlines are supplied but '
710 'different. Pass one or the other.')
711
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000712 # Input and output objects. The general principle is like
713 # this:
714 #
715 # Parent Child
716 # ------ -----
717 # p2cwrite ---stdin---> p2cread
718 # c2pread <--stdout--- c2pwrite
719 # errread <--stderr--- errwrite
720 #
721 # On POSIX, the child objects are file descriptors. On
722 # Windows, these are Windows file handles. The parent objects
723 # are file descriptors on both platforms. The parent objects
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +0000724 # are -1 when not using PIPEs. The child objects are -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000725 # when not redirecting.
Tim Peterse718f612004-10-12 21:51:32 +0000726
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000727 (p2cread, p2cwrite,
728 c2pread, c2pwrite,
729 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
730
Antoine Pitrouc9982322011-01-04 19:07:07 +0000731 # We wrap OS handles *before* launching the child, otherwise a
732 # quickly terminating child could make our fds unwrappable
733 # (see #8458).
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000734
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -0700735 if _mswindows:
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000736 if p2cwrite != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000737 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000738 if c2pread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000739 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
Florent Xicluna3b8bfef2010-03-14 12:31:06 +0000740 if errread != -1:
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +0000741 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
Thomas Wouterscf297e42007-02-23 15:07:44 +0000742
andyclegg7fed7bd2017-10-23 03:01:19 +0100743 self.text_mode = encoding or errors or text or universal_newlines
Tim Peterse718f612004-10-12 21:51:32 +0000744
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800745 # How long to resume waiting on a child after the first ^C.
746 # There is no right value for this. The purpose is to be polite
747 # yet remain good for interactive users trying to exit a tool.
748 self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber()
749
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700750 self._closed_child_pipe_fds = False
Steve Dower050acae2016-09-06 20:16:17 -0700751
Alexey Izbysheva2670562018-10-20 03:22:31 +0300752 if self.text_mode:
753 if bufsize == 1:
754 line_buffering = True
755 # Use the default buffer size for the underlying binary streams
756 # since they don't support line buffering.
757 bufsize = -1
758 else:
759 line_buffering = False
760
Antoine Pitrouc9982322011-01-04 19:07:07 +0000761 try:
Steve Dower050acae2016-09-06 20:16:17 -0700762 if p2cwrite != -1:
763 self.stdin = io.open(p2cwrite, 'wb', bufsize)
andyclegg7fed7bd2017-10-23 03:01:19 +0100764 if self.text_mode:
Steve Dower050acae2016-09-06 20:16:17 -0700765 self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
Alexey Izbysheva2670562018-10-20 03:22:31 +0300766 line_buffering=line_buffering,
Steve Dower050acae2016-09-06 20:16:17 -0700767 encoding=encoding, errors=errors)
768 if c2pread != -1:
769 self.stdout = io.open(c2pread, 'rb', bufsize)
andyclegg7fed7bd2017-10-23 03:01:19 +0100770 if self.text_mode:
Steve Dower050acae2016-09-06 20:16:17 -0700771 self.stdout = io.TextIOWrapper(self.stdout,
772 encoding=encoding, errors=errors)
773 if errread != -1:
774 self.stderr = io.open(errread, 'rb', bufsize)
andyclegg7fed7bd2017-10-23 03:01:19 +0100775 if self.text_mode:
Steve Dower050acae2016-09-06 20:16:17 -0700776 self.stderr = io.TextIOWrapper(self.stderr,
777 encoding=encoding, errors=errors)
778
Antoine Pitrouc9982322011-01-04 19:07:07 +0000779 self._execute_child(args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +0300780 pass_fds, cwd, env,
Antoine Pitrouc9982322011-01-04 19:07:07 +0000781 startupinfo, creationflags, shell,
782 p2cread, p2cwrite,
783 c2pread, c2pwrite,
784 errread, errwrite,
785 restore_signals, start_new_session)
786 except:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800787 # Cleanup if the child failed starting.
788 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
Antoine Pitrouc9982322011-01-04 19:07:07 +0000789 try:
790 f.close()
Andrew Svetlov3438fa42012-12-17 23:35:18 +0200791 except OSError:
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800792 pass # Ignore EBADF or other errors.
793
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700794 if not self._closed_child_pipe_fds:
795 to_close = []
796 if stdin == PIPE:
797 to_close.append(p2cread)
798 if stdout == PIPE:
799 to_close.append(c2pwrite)
800 if stderr == PIPE:
801 to_close.append(errwrite)
802 if hasattr(self, '_devnull'):
803 to_close.append(self._devnull)
804 for fd in to_close:
805 try:
Segev Finer4d385172017-08-18 16:18:13 +0300806 if _mswindows and isinstance(fd, Handle):
807 fd.Close()
808 else:
809 os.close(fd)
Gregory P. Smith22ba31a2013-06-15 18:14:56 -0700810 except OSError:
Gregory P. Smithb5461b92013-06-15 18:04:26 -0700811 pass
Gregory P. Smith3d8e7762012-11-10 22:32:22 -0800812
Antoine Pitrouc9982322011-01-04 19:07:07 +0000813 raise
814
andyclegg7fed7bd2017-10-23 03:01:19 +0100815 @property
816 def universal_newlines(self):
817 # universal_newlines as retained as an alias of text_mode for API
luzpaza5293b42017-11-05 07:37:50 -0600818 # compatibility. bpo-31756
andyclegg7fed7bd2017-10-23 03:01:19 +0100819 return self.text_mode
820
821 @universal_newlines.setter
822 def universal_newlines(self, universal_newlines):
823 self.text_mode = bool(universal_newlines)
824
Steve Dower050acae2016-09-06 20:16:17 -0700825 def _translate_newlines(self, data, encoding, errors):
826 data = data.decode(encoding, errors)
Andrew Svetlov82860712012-08-19 22:13:41 +0300827 return data.replace("\r\n", "\n").replace("\r", "\n")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000828
Brian Curtin79cdb662010-12-03 02:46:02 +0000829 def __enter__(self):
830 return self
831
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800832 def __exit__(self, exc_type, value, traceback):
Brian Curtin79cdb662010-12-03 02:46:02 +0000833 if self.stdout:
834 self.stdout.close()
835 if self.stderr:
836 self.stderr.close()
Serhiy Storchakaab900c22015-02-28 12:43:08 +0200837 try: # Flushing a BufferedWriter may raise an error
838 if self.stdin:
839 self.stdin.close()
840 finally:
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800841 if exc_type == KeyboardInterrupt:
842 # https://bugs.python.org/issue25942
843 # In the case of a KeyboardInterrupt we assume the SIGINT
844 # was also already sent to our child processes. We can't
845 # block indefinitely as that is not user friendly.
846 # If we have not already waited a brief amount of time in
847 # an interrupted .wait() or .communicate() call, do so here
848 # for consistency.
849 if self._sigint_wait_secs > 0:
850 try:
851 self._wait(timeout=self._sigint_wait_secs)
852 except TimeoutExpired:
853 pass
854 self._sigint_wait_secs = 0 # Note that this has been done.
855 return # resume the KeyboardInterrupt
856
Serhiy Storchakaab900c22015-02-28 12:43:08 +0200857 # Wait for the process to terminate, to avoid zombies.
858 self.wait()
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000859
Victor Stinner9505b032017-01-06 10:44:44 +0100860 def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn):
Serhiy Storchaka72e77612014-02-10 19:20:22 +0200861 if not self._child_created:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000862 # We didn't get to successfully create a child process.
863 return
Victor Stinner5a48e212016-05-20 12:11:15 +0200864 if self.returncode is None:
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800865 # Not reading subprocess exit status creates a zombie process which
Victor Stinnerc206f1e2016-06-14 16:42:59 +0200866 # is only destroyed at the parent python process exit
Victor Stinner9505b032017-01-06 10:44:44 +0100867 _warn("subprocess %s is still running" % self.pid,
868 ResourceWarning, source=self)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000869 # In case the child hasn't been waited on, check if it's done.
Brett Cannon84df1e62010-05-14 00:33:40 +0000870 self._internal_poll(_deadstate=_maxsize)
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000871 if self.returncode is None and _active is not None:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000872 # Child is still running, keep us alive until we can wait on it.
873 _active.append(self)
874
Ross Lagerwallba102ec2011-03-16 18:40:25 +0200875 def _get_devnull(self):
876 if not hasattr(self, '_devnull'):
877 self._devnull = os.open(os.devnull, os.O_RDWR)
878 return self._devnull
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000879
Victor Stinnera5e881d2015-01-14 17:07:59 +0100880 def _stdin_write(self, input):
881 if input:
882 try:
883 self.stdin.write(input)
884 except BrokenPipeError:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000885 pass # communicate() must ignore broken pipe errors.
Victor Stinnerd52aa312017-06-08 17:30:39 +0200886 except OSError as exc:
887 if exc.errno == errno.EINVAL:
888 # bpo-19612, bpo-30418: On Windows, stdin.write() fails
889 # with EINVAL if the child process exited or if the child
890 # process is still running but closed the pipe.
Victor Stinnera5e881d2015-01-14 17:07:59 +0100891 pass
892 else:
893 raise
Victor Stinnerd52aa312017-06-08 17:30:39 +0200894
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000895 try:
896 self.stdin.close()
897 except BrokenPipeError:
898 pass # communicate() must ignore broken pipe errors.
Victor Stinnerd52aa312017-06-08 17:30:39 +0200899 except OSError as exc:
900 if exc.errno == errno.EINVAL:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +0000901 pass
902 else:
903 raise
Victor Stinnera5e881d2015-01-14 17:07:59 +0100904
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400905 def communicate(self, input=None, timeout=None):
Joel Schaerer88031a92017-09-13 21:11:20 +0200906 """Interact with process: Send data to stdin and close it.
907 Read data from stdout and stderr, until end-of-file is
908 reached. Wait for process to terminate.
Tim Peterseba28be2005-03-28 01:08:02 +0000909
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400910 The optional "input" argument should be data to be sent to the
andyclegg7fed7bd2017-10-23 03:01:19 +0100911 child process, or None, if no data should be sent to the child.
912 communicate() returns a tuple (stdout, stderr).
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400913
andyclegg7fed7bd2017-10-23 03:01:19 +0100914 By default, all communication is in bytes, and therefore any
915 "input" should be bytes, and the (stdout, stderr) will be bytes.
916 If in text mode (indicated by self.text_mode), any "input" should
917 be a string, and (stdout, stderr) will be strings decoded
918 according to locale encoding, or by "encoding" if set. Text mode
919 is triggered by setting any of text, encoding, errors or
920 universal_newlines.
Andrew Kuchling4f7b0c32014-04-14 15:08:18 -0400921 """
Peter Astrand23109f02005-03-03 20:28:59 +0000922
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400923 if self._communication_started and input:
924 raise ValueError("Cannot send input after starting communication")
925
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400926 # Optimization: If we are not worried about timeouts, we haven't
927 # started communicating, and we have one or zero pipes, using select()
928 # or threads is unnecessary.
Victor Stinner7a8d0812011-04-05 13:13:08 +0200929 if (timeout is None and not self._communication_started and
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400930 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
Tim Peterseba28be2005-03-28 01:08:02 +0000931 stdout = None
932 stderr = None
Peter Astrand23109f02005-03-03 20:28:59 +0000933 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +0100934 self._stdin_write(input)
Peter Astrand23109f02005-03-03 20:28:59 +0000935 elif self.stdout:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +0000936 stdout = self.stdout.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000937 self.stdout.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000938 elif self.stderr:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +0000939 stderr = self.stderr.read()
Georg Brandlf08a9dd2008-06-10 16:57:31 +0000940 self.stderr.close()
Peter Astrand23109f02005-03-03 20:28:59 +0000941 self.wait()
Victor Stinner7a8d0812011-04-05 13:13:08 +0200942 else:
943 if timeout is not None:
Victor Stinner949d8c92012-05-30 13:30:32 +0200944 endtime = _time() + timeout
Victor Stinner7a8d0812011-04-05 13:13:08 +0200945 else:
946 endtime = None
Tim Peterseba28be2005-03-28 01:08:02 +0000947
Victor Stinner7a8d0812011-04-05 13:13:08 +0200948 try:
949 stdout, stderr = self._communicate(input, endtime, timeout)
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800950 except KeyboardInterrupt:
951 # https://bugs.python.org/issue25942
952 # See the detailed comment in .wait().
953 if timeout is not None:
954 sigint_timeout = min(self._sigint_wait_secs,
955 self._remaining_time(endtime))
956 else:
957 sigint_timeout = self._sigint_wait_secs
958 self._sigint_wait_secs = 0 # nothing else should wait.
959 try:
960 self._wait(timeout=sigint_timeout)
961 except TimeoutExpired:
962 pass
963 raise # resume the KeyboardInterrupt
964
Victor Stinner7a8d0812011-04-05 13:13:08 +0200965 finally:
966 self._communication_started = True
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400967
Victor Stinner7a8d0812011-04-05 13:13:08 +0200968 sts = self.wait(timeout=self._remaining_time(endtime))
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400969
970 return (stdout, stderr)
Peter Astrand23109f02005-03-03 20:28:59 +0000971
Fredrik Lundh5b3687d2004-10-12 15:26:28 +0000972
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000973 def poll(self):
Martin Panter4afdca02016-10-25 22:20:48 +0000974 """Check if child process has terminated. Set and return returncode
975 attribute."""
Georg Brandl6aa2d1f2008-08-12 08:35:52 +0000976 return self._internal_poll()
977
978
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400979 def _remaining_time(self, endtime):
980 """Convenience for _communicate when computing timeouts."""
981 if endtime is None:
982 return None
983 else:
Victor Stinner949d8c92012-05-30 13:30:32 +0200984 return endtime - _time()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400985
986
Reid Kleckner2b228f02011-03-16 16:57:54 -0400987 def _check_timeout(self, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400988 """Convenience for checking if a timeout has expired."""
989 if endtime is None:
990 return
Victor Stinner949d8c92012-05-30 13:30:32 +0200991 if _time() > endtime:
Reid Kleckner2b228f02011-03-16 16:57:54 -0400992 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -0400993
994
Gregory P. Smithf4d644f2018-01-29 21:27:39 -0800995 def wait(self, timeout=None):
996 """Wait for child process to terminate; returns self.returncode."""
997 if timeout is not None:
998 endtime = _time() + timeout
999 try:
1000 return self._wait(timeout=timeout)
1001 except KeyboardInterrupt:
1002 # https://bugs.python.org/issue25942
1003 # The first keyboard interrupt waits briefly for the child to
1004 # exit under the common assumption that it also received the ^C
1005 # generated SIGINT and will exit rapidly.
1006 if timeout is not None:
1007 sigint_timeout = min(self._sigint_wait_secs,
1008 self._remaining_time(endtime))
1009 else:
1010 sigint_timeout = self._sigint_wait_secs
1011 self._sigint_wait_secs = 0 # nothing else should wait.
1012 try:
1013 self._wait(timeout=sigint_timeout)
1014 except TimeoutExpired:
1015 pass
1016 raise # resume the KeyboardInterrupt
1017
1018
Gregory P. Smithcb6fdf22015-04-07 16:11:33 -07001019 if _mswindows:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001020 #
1021 # Windows methods
1022 #
1023 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001024 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001025 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1026 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001027 if stdin is None and stdout is None and stderr is None:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001028 return (-1, -1, -1, -1, -1, -1)
Tim Peterse718f612004-10-12 21:51:32 +00001029
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001030 p2cread, p2cwrite = -1, -1
1031 c2pread, c2pwrite = -1, -1
1032 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001033
Peter Astrandd38ddf42005-02-10 08:32:50 +00001034 if stdin is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001035 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001036 if p2cread is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001037 p2cread, _ = _winapi.CreatePipe(None, 0)
1038 p2cread = Handle(p2cread)
1039 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001040 elif stdin == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001041 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1042 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001043 elif stdin == DEVNULL:
1044 p2cread = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001045 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001046 p2cread = msvcrt.get_osfhandle(stdin)
1047 else:
1048 # Assuming file-like object
1049 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1050 p2cread = self._make_inheritable(p2cread)
1051
Peter Astrandd38ddf42005-02-10 08:32:50 +00001052 if stdout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001053 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001054 if c2pwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001055 _, c2pwrite = _winapi.CreatePipe(None, 0)
1056 c2pwrite = Handle(c2pwrite)
1057 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001058 elif stdout == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001059 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1060 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001061 elif stdout == DEVNULL:
1062 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001063 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001064 c2pwrite = msvcrt.get_osfhandle(stdout)
1065 else:
1066 # Assuming file-like object
1067 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1068 c2pwrite = self._make_inheritable(c2pwrite)
1069
Peter Astrandd38ddf42005-02-10 08:32:50 +00001070 if stderr is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001071 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001072 if errwrite is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001073 _, errwrite = _winapi.CreatePipe(None, 0)
1074 errwrite = Handle(errwrite)
1075 _winapi.CloseHandle(_)
Hirokazu Yamamoto0c988172009-03-03 22:41:26 +00001076 elif stderr == PIPE:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001077 errread, errwrite = _winapi.CreatePipe(None, 0)
1078 errread, errwrite = Handle(errread), Handle(errwrite)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001079 elif stderr == STDOUT:
1080 errwrite = c2pwrite
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001081 elif stderr == DEVNULL:
1082 errwrite = msvcrt.get_osfhandle(self._get_devnull())
Peter Astrandd38ddf42005-02-10 08:32:50 +00001083 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001084 errwrite = msvcrt.get_osfhandle(stderr)
1085 else:
1086 # Assuming file-like object
1087 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1088 errwrite = self._make_inheritable(errwrite)
1089
1090 return (p2cread, p2cwrite,
1091 c2pread, c2pwrite,
1092 errread, errwrite)
1093
1094
1095 def _make_inheritable(self, handle):
1096 """Return a duplicate of handle, which is inheritable"""
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001097 h = _winapi.DuplicateHandle(
1098 _winapi.GetCurrentProcess(), handle,
1099 _winapi.GetCurrentProcess(), 0, 1,
1100 _winapi.DUPLICATE_SAME_ACCESS)
1101 return Handle(h)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001102
1103
Segev Finerb2a60832017-12-18 11:28:19 +02001104 def _filter_handle_list(self, handle_list):
1105 """Filter out console handles that can't be used
1106 in lpAttributeList["handle_list"] and make sure the list
1107 isn't empty. This also removes duplicate handles."""
1108 # An handle with it's lowest two bits set might be a special console
1109 # handle that if passed in lpAttributeList["handle_list"], will
1110 # cause it to fail.
1111 return list({handle for handle in handle_list
1112 if handle & 0x3 != 0x3
1113 or _winapi.GetFileType(handle) !=
1114 _winapi.FILE_TYPE_CHAR})
1115
1116
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001117 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001118 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001119 startupinfo, creationflags, shell,
1120 p2cread, p2cwrite,
1121 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001122 errread, errwrite,
1123 unused_restore_signals, unused_start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001124 """Execute program (MS Windows version)"""
1125
Gregory P. Smith8edd99d2010-12-14 13:43:30 +00001126 assert not pass_fds, "pass_fds not supported on Windows."
Gregory P. Smithd4cc7bf2010-12-04 11:22:11 +00001127
Guido van Rossum3172c5d2007-10-16 18:12:55 +00001128 if not isinstance(args, str):
Serhiy Storchakabe50a7b2018-02-28 01:03:46 +02001129 args = list2cmdline(args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001130
Peter Astrandc1d65362004-11-07 14:30:34 +00001131 # Process startup details
Peter Astrandd38ddf42005-02-10 08:32:50 +00001132 if startupinfo is None:
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001133 startupinfo = STARTUPINFO()
Victor Stinner483422f2018-07-05 22:54:17 +02001134 else:
1135 # bpo-34044: Copy STARTUPINFO since it is modified above,
1136 # so the caller can reuse it multiple times.
1137 startupinfo = startupinfo.copy()
Segev Finerb2a60832017-12-18 11:28:19 +02001138
1139 use_std_handles = -1 not in (p2cread, c2pwrite, errwrite)
1140 if use_std_handles:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001141 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
Peter Astrandc1d65362004-11-07 14:30:34 +00001142 startupinfo.hStdInput = p2cread
1143 startupinfo.hStdOutput = c2pwrite
1144 startupinfo.hStdError = errwrite
1145
Segev Finerb2a60832017-12-18 11:28:19 +02001146 attribute_list = startupinfo.lpAttributeList
1147 have_handle_list = bool(attribute_list and
1148 "handle_list" in attribute_list and
1149 attribute_list["handle_list"])
1150
1151 # If we were given an handle_list or need to create one
1152 if have_handle_list or (use_std_handles and close_fds):
1153 if attribute_list is None:
1154 attribute_list = startupinfo.lpAttributeList = {}
1155 handle_list = attribute_list["handle_list"] = \
1156 list(attribute_list.get("handle_list", []))
1157
1158 if use_std_handles:
1159 handle_list += [int(p2cread), int(c2pwrite), int(errwrite)]
1160
1161 handle_list[:] = self._filter_handle_list(handle_list)
1162
1163 if handle_list:
1164 if not close_fds:
1165 warnings.warn("startupinfo.lpAttributeList['handle_list'] "
1166 "overriding close_fds", RuntimeWarning)
1167
1168 # When using the handle_list we always request to inherit
1169 # handles but the only handles that will be inherited are
1170 # the ones in the handle_list
1171 close_fds = False
1172
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001173 if shell:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001174 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1175 startupinfo.wShowWindow = _winapi.SW_HIDE
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001176 comspec = os.environ.get("COMSPEC", "cmd.exe")
Tim Golden126c2962010-08-11 14:20:40 +00001177 args = '{} /c "{}"'.format (comspec, args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001178
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001179 # Start the process
1180 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001181 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
Tim Peterse8374a52004-10-13 03:15:00 +00001182 # no special security
1183 None, None,
Guido van Rossume7ba4952007-06-06 23:52:48 +00001184 int(not close_fds),
Tim Peterse8374a52004-10-13 03:15:00 +00001185 creationflags,
1186 env,
Benjamin Petersoncb90f262017-03-02 00:03:41 -08001187 os.fspath(cwd) if cwd is not None else None,
Tim Peterse8374a52004-10-13 03:15:00 +00001188 startupinfo)
Tim Goldenad537f22010-08-08 11:18:16 +00001189 finally:
1190 # Child is launched. Close the parent's copy of those pipe
1191 # handles that only the child should have open. You need
1192 # to make sure that no handles to the write end of the
1193 # output pipe are maintained in this process or else the
1194 # pipe will not close when the child process exits and the
1195 # ReadFile will hang.
1196 if p2cread != -1:
1197 p2cread.Close()
1198 if c2pwrite != -1:
1199 c2pwrite.Close()
1200 if errwrite != -1:
1201 errwrite.Close()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001202 if hasattr(self, '_devnull'):
1203 os.close(self._devnull)
Segev Finer4d385172017-08-18 16:18:13 +03001204 # Prevent a double close of these handles/fds from __init__
1205 # on error.
1206 self._closed_child_pipe_fds = True
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001207
1208 # Retain the process handle, but close the thread handle
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001209 self._child_created = True
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001210 self._handle = Handle(hp)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001211 self.pid = pid
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001212 _winapi.CloseHandle(ht)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001213
Brett Cannon84df1e62010-05-14 00:33:40 +00001214 def _internal_poll(self, _deadstate=None,
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001215 _WaitForSingleObject=_winapi.WaitForSingleObject,
1216 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1217 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001218 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001219 attribute.
1220
1221 This method is called by __del__, so it can only refer to objects
1222 in its local scope.
1223
1224 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001225 if self.returncode is None:
Brett Cannon84df1e62010-05-14 00:33:40 +00001226 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1227 self.returncode = _GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001228 return self.returncode
1229
1230
Gregory P. Smithf4d644f2018-01-29 21:27:39 -08001231 def _wait(self, timeout):
1232 """Internal implementation of wait() on Windows."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001233 if timeout is None:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001234 timeout_millis = _winapi.INFINITE
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001235 else:
Reid Kleckner91156ff2011-03-21 10:06:10 -07001236 timeout_millis = int(timeout * 1000)
Peter Astrandd38ddf42005-02-10 08:32:50 +00001237 if self.returncode is None:
Gregory P. Smithf4d644f2018-01-29 21:27:39 -08001238 # API note: Returns immediately if timeout_millis == 0.
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001239 result = _winapi.WaitForSingleObject(self._handle,
Gregory P. Smithf4d644f2018-01-29 21:27:39 -08001240 timeout_millis)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001241 if result == _winapi.WAIT_TIMEOUT:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001242 raise TimeoutExpired(self.args, timeout)
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001243 self.returncode = _winapi.GetExitCodeProcess(self._handle)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001244 return self.returncode
1245
1246
1247 def _readerthread(self, fh, buffer):
1248 buffer.append(fh.read())
Victor Stinner667d4b52010-12-25 22:40:32 +00001249 fh.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001250
1251
Reid Kleckner2b228f02011-03-16 16:57:54 -04001252 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001253 # Start reader threads feeding into a list hanging off of this
1254 # object, unless they've already been started.
1255 if self.stdout and not hasattr(self, "_stdout_buff"):
1256 self._stdout_buff = []
1257 self.stdout_thread = \
1258 threading.Thread(target=self._readerthread,
1259 args=(self.stdout, self._stdout_buff))
1260 self.stdout_thread.daemon = True
1261 self.stdout_thread.start()
1262 if self.stderr and not hasattr(self, "_stderr_buff"):
1263 self._stderr_buff = []
1264 self.stderr_thread = \
1265 threading.Thread(target=self._readerthread,
1266 args=(self.stderr, self._stderr_buff))
1267 self.stderr_thread.daemon = True
1268 self.stderr_thread.start()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001269
1270 if self.stdin:
Victor Stinnera5e881d2015-01-14 17:07:59 +01001271 self._stdin_write(input)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001272
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001273 # Wait for the reader threads, or time out. If we time out, the
1274 # threads remain reading and the fds left open in case the user
1275 # calls communicate again.
1276 if self.stdout is not None:
1277 self.stdout_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001278 if self.stdout_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001279 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001280 if self.stderr is not None:
1281 self.stderr_thread.join(self._remaining_time(endtime))
Andrew Svetlov377a1522012-08-19 20:49:39 +03001282 if self.stderr_thread.is_alive():
Reid Kleckner9a67e6c2011-03-20 08:28:07 -07001283 raise TimeoutExpired(self.args, orig_timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001284
1285 # Collect the output from and close both pipes, now that we know
1286 # both have been read successfully.
1287 stdout = None
1288 stderr = None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001289 if self.stdout:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001290 stdout = self._stdout_buff
1291 self.stdout.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001292 if self.stderr:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001293 stderr = self._stderr_buff
1294 self.stderr.close()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001295
1296 # All data exchanged. Translate lists into strings.
Peter Astrandd38ddf42005-02-10 08:32:50 +00001297 if stdout is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001298 stdout = stdout[0]
Peter Astrandd38ddf42005-02-10 08:32:50 +00001299 if stderr is not None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001300 stderr = stderr[0]
1301
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001302 return (stdout, stderr)
1303
Christian Heimesa342c012008-04-20 21:01:16 +00001304 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001305 """Send a signal to the process."""
1306 # Don't signal a process that we know has already died.
1307 if self.returncode is not None:
1308 return
Christian Heimesa342c012008-04-20 21:01:16 +00001309 if sig == signal.SIGTERM:
1310 self.terminate()
Brian Curtineb24d742010-04-12 17:16:38 +00001311 elif sig == signal.CTRL_C_EVENT:
1312 os.kill(self.pid, signal.CTRL_C_EVENT)
1313 elif sig == signal.CTRL_BREAK_EVENT:
1314 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
Christian Heimesa342c012008-04-20 21:01:16 +00001315 else:
Brian Curtin19651362010-09-07 13:24:38 +00001316 raise ValueError("Unsupported signal: {}".format(sig))
Christian Heimesa342c012008-04-20 21:01:16 +00001317
1318 def terminate(self):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001319 """Terminates the process."""
1320 # Don't terminate a process that we know has already died.
1321 if self.returncode is not None:
1322 return
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001323 try:
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001324 _winapi.TerminateProcess(self._handle, 1)
Antoine Pitroub69ef162012-03-11 19:33:29 +01001325 except PermissionError:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001326 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1327 # process already died.
Antoine Pitrou23bba4c2012-04-18 20:51:15 +02001328 rc = _winapi.GetExitCodeProcess(self._handle)
1329 if rc == _winapi.STILL_ACTIVE:
Antoine Pitrou1f9a8352012-03-11 19:29:12 +01001330 raise
1331 self.returncode = rc
Christian Heimesa342c012008-04-20 21:01:16 +00001332
1333 kill = terminate
1334
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001335 else:
1336 #
1337 # POSIX methods
1338 #
1339 def _get_handles(self, stdin, stdout, stderr):
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +00001340 """Construct and return tuple with IO objects:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001341 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1342 """
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001343 p2cread, p2cwrite = -1, -1
1344 c2pread, c2pwrite = -1, -1
1345 errread, errwrite = -1, -1
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001346
Peter Astrandd38ddf42005-02-10 08:32:50 +00001347 if stdin is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001348 pass
1349 elif stdin == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001350 p2cread, p2cwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001351 elif stdin == DEVNULL:
1352 p2cread = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001353 elif isinstance(stdin, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001354 p2cread = stdin
1355 else:
1356 # Assuming file-like object
1357 p2cread = stdin.fileno()
1358
Peter Astrandd38ddf42005-02-10 08:32:50 +00001359 if stdout is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001360 pass
1361 elif stdout == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001362 c2pread, c2pwrite = os.pipe()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001363 elif stdout == DEVNULL:
1364 c2pwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001365 elif isinstance(stdout, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001366 c2pwrite = stdout
1367 else:
1368 # Assuming file-like object
Tim Peterse718f612004-10-12 21:51:32 +00001369 c2pwrite = stdout.fileno()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001370
Peter Astrandd38ddf42005-02-10 08:32:50 +00001371 if stderr is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001372 pass
1373 elif stderr == PIPE:
Victor Stinnerdaf45552013-08-28 00:53:59 +02001374 errread, errwrite = os.pipe()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001375 elif stderr == STDOUT:
Martin Panterc7635892016-05-13 01:54:44 +00001376 if c2pwrite != -1:
1377 errwrite = c2pwrite
1378 else: # child's stdout is not set, use parent's stdout
1379 errwrite = sys.__stdout__.fileno()
Ross Lagerwallba102ec2011-03-16 18:40:25 +02001380 elif stderr == DEVNULL:
1381 errwrite = self._get_devnull()
Peter Astrandd38ddf42005-02-10 08:32:50 +00001382 elif isinstance(stderr, int):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001383 errwrite = stderr
1384 else:
1385 # Assuming file-like object
1386 errwrite = stderr.fileno()
1387
1388 return (p2cread, p2cwrite,
1389 c2pread, c2pwrite,
1390 errread, errwrite)
1391
1392
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001393 def _execute_child(self, args, executable, preexec_fn, close_fds,
Andrew Svetlov592df202012-08-15 17:36:15 +03001394 pass_fds, cwd, env,
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001395 startupinfo, creationflags, shell,
1396 p2cread, p2cwrite,
1397 c2pread, c2pwrite,
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001398 errread, errwrite,
1399 restore_signals, start_new_session):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001400 """Execute program (POSIX version)"""
1401
Victor Stinner7b3b20a2011-03-03 12:54:05 +00001402 if isinstance(args, (str, bytes)):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001403 args = [args]
Thomas Wouters89f507f2006-12-13 04:49:30 +00001404 else:
Serhiy Storchakabe50a7b2018-02-28 01:03:46 +02001405 args = list(args)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001406
1407 if shell:
Xavier de Gayeb35fc622016-12-13 16:32:21 +01001408 # On Android the default shell is at '/system/bin/sh'.
1409 unix_shell = ('/system/bin/sh' if
1410 hasattr(sys, 'getandroidapilevel') else '/bin/sh')
1411 args = [unix_shell, "-c"] + args
Stefan Krah9542cc62010-07-19 14:20:53 +00001412 if executable:
1413 args[0] = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001414
Peter Astrandd38ddf42005-02-10 08:32:50 +00001415 if executable is None:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001416 executable = args[0]
Gregory P. Smith5591b022012-10-10 03:34:47 -07001417 orig_executable = executable
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001418
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001419 # For transferring possible exec failure from child to parent.
1420 # Data format: "exception name:hex errno:description"
1421 # Pickle is not used; it is complex and involves memory allocation.
Victor Stinnerdaf45552013-08-28 00:53:59 +02001422 errpipe_read, errpipe_write = os.pipe()
Gregory P. Smith53dd8162013-12-01 16:03:24 -08001423 # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
1424 low_fds_to_close = []
1425 while errpipe_write < 3:
1426 low_fds_to_close.append(errpipe_write)
1427 errpipe_write = os.dup(errpipe_write)
1428 for low_fd in low_fds_to_close:
1429 os.close(low_fd)
Christian Heimesfdab48e2008-01-20 09:06:41 +00001430 try:
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001431 try:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001432 # We must avoid complex work that could involve
1433 # malloc or free in the child process to avoid
1434 # potential deadlocks, thus we do all this here.
1435 # and pass it to fork_exec()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001436
Victor Stinner372b8382011-06-21 17:24:21 +02001437 if env is not None:
Serhiy Storchakad174d242017-06-23 19:39:27 +03001438 env_list = []
1439 for k, v in env.items():
1440 k = os.fsencode(k)
1441 if b'=' in k:
1442 raise ValueError("illegal environment variable name")
1443 env_list.append(k + b'=' + os.fsencode(v))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001444 else:
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001445 env_list = None # Use execv instead of execve.
1446 executable = os.fsencode(executable)
1447 if os.path.dirname(executable):
1448 executable_list = (executable,)
1449 else:
1450 # This matches the behavior of os._execvpe().
1451 executable_list = tuple(
1452 os.path.join(os.fsencode(dir), executable)
1453 for dir in os.get_exec_path(env))
Gregory P. Smith361e30c2013-12-01 00:12:24 -08001454 fds_to_keep = set(pass_fds)
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001455 fds_to_keep.add(errpipe_write)
1456 self.pid = _posixsubprocess.fork_exec(
1457 args, executable_list,
Serhiy Storchaka66bffd12017-04-19 21:12:46 +03001458 close_fds, tuple(sorted(map(int, fds_to_keep))),
1459 cwd, env_list,
Gregory P. Smith59fd1bf2011-05-28 09:32:39 -07001460 p2cread, p2cwrite, c2pread, c2pwrite,
1461 errread, errwrite,
1462 errpipe_read, errpipe_write,
1463 restore_signals, start_new_session, preexec_fn)
Charles-François Natali558639f2011-08-18 19:11:29 +02001464 self._child_created = True
Facundo Batista10706e22009-06-19 20:34:30 +00001465 finally:
1466 # be sure the FD is closed no matter what
1467 os.close(errpipe_write)
1468
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001469 # self._devnull is not always defined.
1470 devnull_fd = getattr(self, '_devnull', None)
1471 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001472 os.close(p2cread)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001473 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001474 os.close(c2pwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001475 if errwrite != -1 and errread != -1 and errwrite != devnull_fd:
Facundo Batista10706e22009-06-19 20:34:30 +00001476 os.close(errwrite)
Gregory P. Smithb5461b92013-06-15 18:04:26 -07001477 if devnull_fd is not None:
1478 os.close(devnull_fd)
1479 # Prevent a double close of these fds from __init__ on error.
1480 self._closed_child_pipe_fds = True
Facundo Batista10706e22009-06-19 20:34:30 +00001481
1482 # Wait for exec to fail or succeed; possibly raising an
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001483 # exception (limited in size)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001484 errpipe_data = bytearray()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001485 while True:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001486 part = os.read(errpipe_read, 50000)
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001487 errpipe_data += part
1488 if not part or len(errpipe_data) > 50000:
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001489 break
Facundo Batista10706e22009-06-19 20:34:30 +00001490 finally:
1491 # be sure the FD is closed no matter what
1492 os.close(errpipe_read)
1493
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001494 if errpipe_data:
Gregory P. Smithe85db2b2010-12-14 14:38:00 +00001495 try:
Victor Stinnera58e2c52016-05-20 12:08:12 +02001496 pid, sts = os.waitpid(self.pid, 0)
1497 if pid == self.pid:
1498 self._handle_exitstatus(sts)
1499 else:
1500 self.returncode = sys.maxsize
Victor Stinnera5e881d2015-01-14 17:07:59 +01001501 except ChildProcessError:
1502 pass
Victor Stinnera58e2c52016-05-20 12:08:12 +02001503
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001504 try:
Gregory P. Smithf44c9da2012-11-10 23:33:17 -08001505 exception_name, hex_errno, err_msg = (
1506 errpipe_data.split(b':', 2))
Ammar Askar3fc499b2017-09-06 02:41:30 -04001507 # The encoding here should match the encoding
1508 # written in by the subprocess implementations
1509 # like _posixsubprocess
1510 err_msg = err_msg.decode()
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001511 except ValueError:
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001512 exception_name = b'SubprocessError'
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001513 hex_errno = b'0'
Ammar Askar3fc499b2017-09-06 02:41:30 -04001514 err_msg = 'Bad exception data from child: {!r}'.format(
1515 bytes(errpipe_data))
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001516 child_exception_type = getattr(
1517 builtins, exception_name.decode('ascii'),
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001518 SubprocessError)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001519 if issubclass(child_exception_type, OSError) and hex_errno:
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001520 errno_num = int(hex_errno, 16)
Gregory P. Smith5591b022012-10-10 03:34:47 -07001521 child_exec_never_called = (err_msg == "noexec")
1522 if child_exec_never_called:
1523 err_msg = ""
Gregory P. Smith8621bb52017-08-24 14:58:25 -07001524 # The error must be from chdir(cwd).
1525 err_filename = cwd
1526 else:
1527 err_filename = orig_executable
Benjamin Petersonb8bc4392010-11-20 18:24:54 +00001528 if errno_num != 0:
1529 err_msg = os.strerror(errno_num)
Gregory P. Smith8621bb52017-08-24 14:58:25 -07001530 raise child_exception_type(errno_num, err_msg, err_filename)
Gregory P. Smithfb94c5f2010-03-14 06:49:55 +00001531 raise child_exception_type(err_msg)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001532
1533
Brett Cannon84df1e62010-05-14 00:33:40 +00001534 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
1535 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
Gregory P. Smith50e16e32017-01-22 17:28:38 -08001536 _WEXITSTATUS=os.WEXITSTATUS, _WIFSTOPPED=os.WIFSTOPPED,
1537 _WSTOPSIG=os.WSTOPSIG):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001538 """All callers to this function MUST hold self._waitpid_lock."""
Brett Cannon84df1e62010-05-14 00:33:40 +00001539 # This method is called (indirectly) by __del__, so it cannot
Serhiy Storchaka72e77612014-02-10 19:20:22 +02001540 # refer to anything outside of its local scope.
Brett Cannon84df1e62010-05-14 00:33:40 +00001541 if _WIFSIGNALED(sts):
1542 self.returncode = -_WTERMSIG(sts)
1543 elif _WIFEXITED(sts):
1544 self.returncode = _WEXITSTATUS(sts)
Gregory P. Smith50e16e32017-01-22 17:28:38 -08001545 elif _WIFSTOPPED(sts):
1546 self.returncode = -_WSTOPSIG(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001547 else:
1548 # Should never happen
Gregory P. Smith8d07c262012-11-10 23:53:47 -08001549 raise SubprocessError("Unknown child exit status!")
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001550
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001551
Brett Cannon84df1e62010-05-14 00:33:40 +00001552 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
Andrew Svetlov1d960fe2012-12-24 20:08:53 +02001553 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001554 """Check if child process has terminated. Returns returncode
Brett Cannon84df1e62010-05-14 00:33:40 +00001555 attribute.
1556
1557 This method is called by __del__, so it cannot reference anything
1558 outside of the local scope (nor can any methods it calls).
1559
1560 """
Peter Astrandd38ddf42005-02-10 08:32:50 +00001561 if self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001562 if not self._waitpid_lock.acquire(False):
1563 # Something else is busy calling waitpid. Don't allow two
1564 # at once. We know nothing yet.
1565 return None
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001566 try:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001567 if self.returncode is not None:
1568 return self.returncode # Another thread waited.
Brett Cannon84df1e62010-05-14 00:33:40 +00001569 pid, sts = _waitpid(self.pid, _WNOHANG)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001570 if pid == self.pid:
1571 self._handle_exitstatus(sts)
Andrew Svetlovad28c7f2012-12-18 22:02:39 +02001572 except OSError as e:
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001573 if _deadstate is not None:
1574 self.returncode = _deadstate
Andrew Svetlov08bab072012-12-24 20:06:35 +02001575 elif e.errno == _ECHILD:
Gregory P. Smith39051712012-09-29 11:40:38 -07001576 # This happens if SIGCLD is set to be ignored or
1577 # waiting for child processes has otherwise been
1578 # disabled for our process. This child is dead, we
1579 # can't get the status.
1580 # http://bugs.python.org/issue15756
1581 self.returncode = 0
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001582 finally:
1583 self._waitpid_lock.release()
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001584 return self.returncode
1585
1586
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001587 def _try_wait(self, wait_flags):
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001588 """All callers to this function MUST hold self._waitpid_lock."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001589 try:
Charles-François Natali6e6c59b2015-02-07 13:27:50 +00001590 (pid, sts) = os.waitpid(self.pid, wait_flags)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001591 except ChildProcessError:
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001592 # This happens if SIGCLD is set to be ignored or waiting
1593 # for child processes has otherwise been disabled for our
1594 # process. This child is dead, we can't get the status.
1595 pid = self.pid
1596 sts = 0
1597 return (pid, sts)
1598
1599
Gregory P. Smithf4d644f2018-01-29 21:27:39 -08001600 def _wait(self, timeout):
1601 """Internal implementation of wait() on POSIX."""
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001602 if self.returncode is not None:
1603 return self.returncode
Reid Kleckner2b228f02011-03-16 16:57:54 -04001604
Gregory P. Smith82604e02016-11-20 16:31:07 -08001605 if timeout is not None:
1606 endtime = _time() + timeout
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001607 # Enter a busy loop if we have a timeout. This busy loop was
1608 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1609 delay = 0.0005 # 500 us -> initial delay of 1 ms
1610 while True:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001611 if self._waitpid_lock.acquire(False):
1612 try:
1613 if self.returncode is not None:
1614 break # Another thread waited.
1615 (pid, sts) = self._try_wait(os.WNOHANG)
1616 assert pid == self.pid or pid == 0
1617 if pid == self.pid:
1618 self._handle_exitstatus(sts)
1619 break
1620 finally:
1621 self._waitpid_lock.release()
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001622 remaining = self._remaining_time(endtime)
1623 if remaining <= 0:
Reid Kleckner2b228f02011-03-16 16:57:54 -04001624 raise TimeoutExpired(self.args, timeout)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001625 delay = min(delay * 2, remaining, .05)
1626 time.sleep(delay)
Gregory P. Smithf328d792012-11-10 21:06:18 -08001627 else:
1628 while self.returncode is None:
Gregory P. Smithd65ba512014-04-23 00:27:17 -07001629 with self._waitpid_lock:
1630 if self.returncode is not None:
1631 break # Another thread waited.
1632 (pid, sts) = self._try_wait(0)
1633 # Check the pid and loop as waitpid has been known to
1634 # return 0 even without WNOHANG in odd situations.
1635 # http://bugs.python.org/issue14396.
1636 if pid == self.pid:
1637 self._handle_exitstatus(sts)
Fredrik Lundh5b3687d2004-10-12 15:26:28 +00001638 return self.returncode
1639
1640
Reid Kleckner2b228f02011-03-16 16:57:54 -04001641 def _communicate(self, input, endtime, orig_timeout):
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001642 if self.stdin and not self._communication_started:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001643 # Flush stdio buffer. This might block, if the user has
1644 # been writing to .stdin in an uncontrolled fashion.
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001645 try:
1646 self.stdin.flush()
1647 except BrokenPipeError:
1648 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001649 if not input:
Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)1ef8c7e2016-06-04 00:22:17 +00001650 try:
1651 self.stdin.close()
1652 except BrokenPipeError:
1653 pass # communicate() must ignore BrokenPipeError.
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001654
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001655 stdout = None
1656 stderr = None
1657
1658 # Only create this mapping if we haven't already.
1659 if not self._communication_started:
1660 self._fileobj2output = {}
1661 if self.stdout:
1662 self._fileobj2output[self.stdout] = []
1663 if self.stderr:
1664 self._fileobj2output[self.stderr] = []
1665
1666 if self.stdout:
1667 stdout = self._fileobj2output[self.stdout]
1668 if self.stderr:
1669 stderr = self._fileobj2output[self.stderr]
1670
1671 self._save_input(input)
1672
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001673 if self._input:
1674 input_view = memoryview(self._input)
1675
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001676 with _PopenSelector() as selector:
1677 if self.stdin and input:
1678 selector.register(self.stdin, selectors.EVENT_WRITE)
1679 if self.stdout:
1680 selector.register(self.stdout, selectors.EVENT_READ)
1681 if self.stderr:
1682 selector.register(self.stderr, selectors.EVENT_READ)
1683
1684 while selector.get_map():
1685 timeout = self._remaining_time(endtime)
1686 if timeout is not None and timeout < 0:
1687 raise TimeoutExpired(self.args, orig_timeout)
1688
1689 ready = selector.select(timeout)
1690 self._check_timeout(endtime, orig_timeout)
1691
1692 # XXX Rewrite these to use non-blocking I/O on the file
1693 # objects; they are no longer using C stdio!
1694
1695 for key, events in ready:
1696 if key.fileobj is self.stdin:
Gregory P. Smith5ca129b2013-12-07 19:14:59 -08001697 chunk = input_view[self._input_offset :
1698 self._input_offset + _PIPE_BUF]
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001699 try:
1700 self._input_offset += os.write(key.fd, chunk)
Victor Stinnera5e881d2015-01-14 17:07:59 +01001701 except BrokenPipeError:
1702 selector.unregister(key.fileobj)
1703 key.fileobj.close()
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001704 else:
1705 if self._input_offset >= len(self._input):
1706 selector.unregister(key.fileobj)
1707 key.fileobj.close()
1708 elif key.fileobj in (self.stdout, self.stderr):
Gregory P. Smith7b83b182013-12-08 10:58:28 -08001709 data = os.read(key.fd, 32768)
Charles-François Natali3a4586a2013-11-08 19:56:59 +01001710 if not data:
1711 selector.unregister(key.fileobj)
1712 key.fileobj.close()
1713 self._fileobj2output[key.fileobj].append(data)
Reid Kleckner31aa7dd2011-03-14 12:02:10 -04001714
1715 self.wait(timeout=self._remaining_time(endtime))
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001716
1717 # All data exchanged. Translate lists into strings.
1718 if stdout is not None:
1719 stdout = b''.join(stdout)
1720 if stderr is not None:
1721 stderr = b''.join(stderr)
1722
1723 # Translate newlines, if requested.
1724 # This also turns bytes into strings.
andyclegg7fed7bd2017-10-23 03:01:19 +01001725 if self.text_mode:
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001726 if stdout is not None:
1727 stdout = self._translate_newlines(stdout,
Steve Dower050acae2016-09-06 20:16:17 -07001728 self.stdout.encoding,
1729 self.stdout.errors)
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001730 if stderr is not None:
1731 stderr = self._translate_newlines(stderr,
Steve Dower050acae2016-09-06 20:16:17 -07001732 self.stderr.encoding,
1733 self.stderr.errors)
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001734
Gregory P. Smithd06fa472009-07-04 02:46:54 +00001735 return (stdout, stderr)
1736
1737
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001738 def _save_input(self, input):
1739 # This method is called from the _communicate_with_*() methods
1740 # so that if we time out while communicating, we can continue
1741 # sending input if we retry.
1742 if self.stdin and self._input is None:
1743 self._input_offset = 0
1744 self._input = input
andyclegg7fed7bd2017-10-23 03:01:19 +01001745 if input is not None and self.text_mode:
Steve Dower050acae2016-09-06 20:16:17 -07001746 self._input = self._input.encode(self.stdin.encoding,
1747 self.stdin.errors)
Andrew Svetlovaa0dbdc2012-08-14 18:40:21 +03001748
1749
Christian Heimesa342c012008-04-20 21:01:16 +00001750 def send_signal(self, sig):
Gregory P. Smitha0c9caa2015-11-15 18:19:10 -08001751 """Send a signal to the process."""
1752 # Skip signalling a process that we know has already died.
1753 if self.returncode is None:
1754 os.kill(self.pid, sig)
Christian Heimesa342c012008-04-20 21:01:16 +00001755
1756 def terminate(self):
1757 """Terminate the process with SIGTERM
1758 """
1759 self.send_signal(signal.SIGTERM)
1760
1761 def kill(self):
1762 """Kill the process with SIGKILL
1763 """
1764 self.send_signal(signal.SIGKILL)