Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 1 | # |
| 2 | # Module providing the `Process` class which emulates `threading.Thread` |
| 3 | # |
| 4 | # multiprocessing/process.py |
| 5 | # |
R. David Murray | 3fc969a | 2010-12-14 01:38:16 +0000 | [diff] [blame] | 6 | # Copyright (c) 2006-2008, R Oudkerk |
Richard Oudkerk | 3e268aa | 2012-04-30 12:13:55 +0100 | [diff] [blame] | 7 | # Licensed to PSF under a Contributor Agreement. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 8 | # |
| 9 | |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 10 | __all__ = ['BaseProcess', 'current_process', 'active_children'] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 11 | |
| 12 | # |
| 13 | # Imports |
| 14 | # |
| 15 | |
| 16 | import os |
| 17 | import sys |
| 18 | import signal |
| 19 | import itertools |
Antoine Pitrou | c081c0c | 2011-07-15 22:12:24 +0200 | [diff] [blame] | 20 | from _weakrefset import WeakSet |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 21 | |
| 22 | # |
| 23 | # |
| 24 | # |
| 25 | |
| 26 | try: |
| 27 | ORIGINAL_DIR = os.path.abspath(os.getcwd()) |
| 28 | except OSError: |
| 29 | ORIGINAL_DIR = None |
| 30 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 31 | # |
| 32 | # Public functions |
| 33 | # |
| 34 | |
| 35 | def current_process(): |
| 36 | ''' |
| 37 | Return process object representing the current process |
| 38 | ''' |
| 39 | return _current_process |
| 40 | |
| 41 | def active_children(): |
| 42 | ''' |
| 43 | Return list of process objects corresponding to live child processes |
| 44 | ''' |
| 45 | _cleanup() |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 46 | return list(_children) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 47 | |
| 48 | # |
| 49 | # |
| 50 | # |
| 51 | |
| 52 | def _cleanup(): |
| 53 | # check for processes which have finished |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 54 | for p in list(_children): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 55 | if p._popen.poll() is not None: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 56 | _children.discard(p) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 57 | |
| 58 | # |
| 59 | # The `Process` class |
| 60 | # |
| 61 | |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 62 | class BaseProcess(object): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 63 | ''' |
| 64 | Process objects represent activity that is run in a separate process |
| 65 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 66 | The class is analogous to `threading.Thread` |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 67 | ''' |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 68 | def _Popen(self): |
| 69 | raise NotImplementedError |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 70 | |
Antoine Pitrou | 0bd4deb | 2011-02-25 22:07:43 +0000 | [diff] [blame] | 71 | def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, |
| 72 | *, daemon=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 73 | assert group is None, 'group argument must be None for now' |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 74 | count = next(_process_counter) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 75 | self._identity = _current_process._identity + (count,) |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 76 | self._config = _current_process._config.copy() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 77 | self._parent_pid = os.getpid() |
| 78 | self._popen = None |
| 79 | self._target = target |
| 80 | self._args = tuple(args) |
| 81 | self._kwargs = dict(kwargs) |
| 82 | self._name = name or type(self).__name__ + '-' + \ |
| 83 | ':'.join(str(i) for i in self._identity) |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 84 | if daemon is not None: |
| 85 | self.daemon = daemon |
Antoine Pitrou | c081c0c | 2011-07-15 22:12:24 +0200 | [diff] [blame] | 86 | _dangling.add(self) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 87 | |
| 88 | def run(self): |
| 89 | ''' |
| 90 | Method to be run in sub-process; can be overridden in sub-class |
| 91 | ''' |
| 92 | if self._target: |
| 93 | self._target(*self._args, **self._kwargs) |
| 94 | |
| 95 | def start(self): |
| 96 | ''' |
| 97 | Start child process |
| 98 | ''' |
| 99 | assert self._popen is None, 'cannot start a process twice' |
| 100 | assert self._parent_pid == os.getpid(), \ |
| 101 | 'can only start a process object created by current process' |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 102 | assert not _current_process._config.get('daemon'), \ |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 103 | 'daemonic processes are not allowed to have children' |
| 104 | _cleanup() |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 105 | self._popen = self._Popen(self) |
Antoine Pitrou | 176f07d | 2011-06-06 19:35:31 +0200 | [diff] [blame] | 106 | self._sentinel = self._popen.sentinel |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 107 | _children.add(self) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 108 | |
| 109 | def terminate(self): |
| 110 | ''' |
| 111 | Terminate process; sends SIGTERM signal or uses TerminateProcess() |
| 112 | ''' |
| 113 | self._popen.terminate() |
| 114 | |
| 115 | def join(self, timeout=None): |
| 116 | ''' |
| 117 | Wait until child process terminates |
| 118 | ''' |
| 119 | assert self._parent_pid == os.getpid(), 'can only join a child process' |
| 120 | assert self._popen is not None, 'can only join a started process' |
| 121 | res = self._popen.wait(timeout) |
| 122 | if res is not None: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 123 | _children.discard(self) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 124 | |
| 125 | def is_alive(self): |
| 126 | ''' |
| 127 | Return whether process is alive |
| 128 | ''' |
| 129 | if self is _current_process: |
| 130 | return True |
| 131 | assert self._parent_pid == os.getpid(), 'can only test a child process' |
| 132 | if self._popen is None: |
| 133 | return False |
| 134 | self._popen.poll() |
| 135 | return self._popen.returncode is None |
| 136 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 137 | @property |
| 138 | def name(self): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 139 | return self._name |
| 140 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 141 | @name.setter |
| 142 | def name(self, name): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 143 | assert isinstance(name, str), 'name must be a string' |
| 144 | self._name = name |
| 145 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 146 | @property |
| 147 | def daemon(self): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 148 | ''' |
| 149 | Return whether process is a daemon |
| 150 | ''' |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 151 | return self._config.get('daemon', False) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 152 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 153 | @daemon.setter |
| 154 | def daemon(self, daemonic): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 155 | ''' |
| 156 | Set whether process is a daemon |
| 157 | ''' |
| 158 | assert self._popen is None, 'process has already started' |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 159 | self._config['daemon'] = daemonic |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 160 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 161 | @property |
| 162 | def authkey(self): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 163 | return self._config['authkey'] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 164 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 165 | @authkey.setter |
| 166 | def authkey(self, authkey): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 167 | ''' |
| 168 | Set authorization key of process |
| 169 | ''' |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 170 | self._config['authkey'] = AuthenticationString(authkey) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 171 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 172 | @property |
| 173 | def exitcode(self): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 174 | ''' |
| 175 | Return exit code of process or `None` if it has yet to stop |
| 176 | ''' |
| 177 | if self._popen is None: |
| 178 | return self._popen |
| 179 | return self._popen.poll() |
| 180 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 181 | @property |
| 182 | def ident(self): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 183 | ''' |
Florent Xicluna | b519d23 | 2010-03-04 16:10:55 +0000 | [diff] [blame] | 184 | Return identifier (PID) of process or `None` if it has yet to start |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 185 | ''' |
| 186 | if self is _current_process: |
| 187 | return os.getpid() |
| 188 | else: |
| 189 | return self._popen and self._popen.pid |
| 190 | |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 191 | pid = ident |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 192 | |
Antoine Pitrou | 176f07d | 2011-06-06 19:35:31 +0200 | [diff] [blame] | 193 | @property |
| 194 | def sentinel(self): |
| 195 | ''' |
| 196 | Return a file descriptor (Unix) or handle (Windows) suitable for |
| 197 | waiting for process termination. |
| 198 | ''' |
| 199 | try: |
| 200 | return self._sentinel |
| 201 | except AttributeError: |
| 202 | raise ValueError("process not started") |
| 203 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 204 | def __repr__(self): |
| 205 | if self is _current_process: |
| 206 | status = 'started' |
| 207 | elif self._parent_pid != os.getpid(): |
| 208 | status = 'unknown' |
| 209 | elif self._popen is None: |
| 210 | status = 'initial' |
| 211 | else: |
| 212 | if self._popen.poll() is not None: |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 213 | status = self.exitcode |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 214 | else: |
| 215 | status = 'started' |
| 216 | |
| 217 | if type(status) is int: |
| 218 | if status == 0: |
| 219 | status = 'stopped' |
| 220 | else: |
| 221 | status = 'stopped[%s]' % _exitcode_to_name.get(status, status) |
| 222 | |
| 223 | return '<%s(%s, %s%s)>' % (type(self).__name__, self._name, |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 224 | status, self.daemon and ' daemon' or '') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 225 | |
| 226 | ## |
| 227 | |
| 228 | def _bootstrap(self): |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 229 | from . import util, context |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 230 | global _current_process, _process_counter, _children |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 231 | |
| 232 | try: |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 233 | if self._start_method is not None: |
| 234 | context._force_start_method(self._start_method) |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 235 | _process_counter = itertools.count(1) |
| 236 | _children = set() |
Victor Stinner | a6d865c | 2016-03-25 09:29:50 +0100 | [diff] [blame] | 237 | util._close_stdin() |
Victor Stinner | 0f83b15 | 2011-06-17 12:31:49 +0200 | [diff] [blame] | 238 | old_process = _current_process |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 239 | _current_process = self |
Victor Stinner | 0f83b15 | 2011-06-17 12:31:49 +0200 | [diff] [blame] | 240 | try: |
| 241 | util._finalizer_registry.clear() |
| 242 | util._run_after_forkers() |
| 243 | finally: |
| 244 | # delay finalization of the old process object until after |
| 245 | # _run_after_forkers() is executed |
| 246 | del old_process |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 247 | util.info('child process calling self.run()') |
| 248 | try: |
| 249 | self.run() |
| 250 | exitcode = 0 |
| 251 | finally: |
| 252 | util._exit_function() |
| 253 | except SystemExit as e: |
| 254 | if not e.args: |
| 255 | exitcode = 1 |
Richard Oudkerk | 29471de | 2012-06-06 19:04:57 +0100 | [diff] [blame] | 256 | elif isinstance(e.args[0], int): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 257 | exitcode = e.args[0] |
| 258 | else: |
Richard Oudkerk | 29471de | 2012-06-06 19:04:57 +0100 | [diff] [blame] | 259 | sys.stderr.write(str(e.args[0]) + '\n') |
Richard Oudkerk | 8731d7b | 2013-11-17 17:24:11 +0000 | [diff] [blame] | 260 | exitcode = 1 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 261 | except: |
| 262 | exitcode = 1 |
| 263 | import traceback |
Benjamin Peterson | 58ea9fe | 2008-08-19 19:17:39 +0000 | [diff] [blame] | 264 | sys.stderr.write('Process %s:\n' % self.name) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 265 | traceback.print_exc() |
Antoine Pitrou | 84a0fbf | 2012-01-27 10:52:37 +0100 | [diff] [blame] | 266 | finally: |
| 267 | util.info('process exiting with exitcode %d' % exitcode) |
| 268 | sys.stdout.flush() |
| 269 | sys.stderr.flush() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 270 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 271 | return exitcode |
| 272 | |
| 273 | # |
| 274 | # We subclass bytes to avoid accidental transmission of auth keys over network |
| 275 | # |
| 276 | |
| 277 | class AuthenticationString(bytes): |
| 278 | def __reduce__(self): |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 279 | from .context import get_spawning_popen |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 280 | if get_spawning_popen() is None: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 281 | raise TypeError( |
| 282 | 'Pickling an AuthenticationString object is ' |
| 283 | 'disallowed for security reasons' |
| 284 | ) |
| 285 | return AuthenticationString, (bytes(self),) |
| 286 | |
| 287 | # |
| 288 | # Create object representing the main process |
| 289 | # |
| 290 | |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 291 | class _MainProcess(BaseProcess): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 292 | |
| 293 | def __init__(self): |
| 294 | self._identity = () |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 295 | self._name = 'MainProcess' |
| 296 | self._parent_pid = None |
| 297 | self._popen = None |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 298 | self._config = {'authkey': AuthenticationString(os.urandom(32)), |
Richard Oudkerk | e943697 | 2013-11-02 17:05:07 +0000 | [diff] [blame] | 299 | 'semprefix': '/mp'} |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 300 | # Note that some versions of FreeBSD only allow named |
Richard Oudkerk | e943697 | 2013-11-02 17:05:07 +0000 | [diff] [blame] | 301 | # semaphores to have names of up to 14 characters. Therefore |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 302 | # we choose a short prefix. |
Richard Oudkerk | e943697 | 2013-11-02 17:05:07 +0000 | [diff] [blame] | 303 | # |
| 304 | # On MacOSX in a sandbox it may be necessary to use a |
| 305 | # different prefix -- see #19478. |
| 306 | # |
| 307 | # Everything in self._config will be inherited by descendant |
| 308 | # processes. |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 309 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 310 | |
| 311 | _current_process = _MainProcess() |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 312 | _process_counter = itertools.count(1) |
| 313 | _children = set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 314 | del _MainProcess |
| 315 | |
| 316 | # |
| 317 | # Give names to some return codes |
| 318 | # |
| 319 | |
| 320 | _exitcode_to_name = {} |
| 321 | |
| 322 | for name, signum in list(signal.__dict__.items()): |
| 323 | if name[:3]=='SIG' and '_' not in name: |
| 324 | _exitcode_to_name[-signum] = name |
Antoine Pitrou | c081c0c | 2011-07-15 22:12:24 +0200 | [diff] [blame] | 325 | |
| 326 | # For debug and leak testing |
| 327 | _dangling = WeakSet() |