Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 1 | # |
| 2 | # Package analogous to 'threading.py' but using processes |
| 3 | # |
| 4 | # multiprocessing/__init__.py |
| 5 | # |
| 6 | # This package is intended to duplicate the functionality (and much of |
| 7 | # the API) of threading.py but uses processes instead of threads. A |
| 8 | # subpackage 'multiprocessing.dummy' has the same API but is a simple |
| 9 | # wrapper for 'threading'. |
| 10 | # |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 11 | # Copyright (c) 2006-2008, R Oudkerk |
Richard Oudkerk | 3e268aa | 2012-04-30 12:13:55 +0100 | [diff] [blame] | 12 | # Licensed to PSF under a Contributor Agreement. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 13 | # |
| 14 | |
| 15 | __version__ = '0.70a1' |
| 16 | |
| 17 | __all__ = [ |
| 18 | 'Process', 'current_process', 'active_children', 'freeze_support', |
| 19 | 'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger', |
| 20 | 'allow_connection_pickling', 'BufferTooShort', 'TimeoutError', |
| 21 | 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', |
Richard Oudkerk | 3730a17 | 2012-06-15 18:26:07 +0100 | [diff] [blame] | 22 | 'Event', 'Barrier', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', |
| 23 | 'Value', 'Array', 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 24 | ] |
| 25 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 26 | # |
| 27 | # Imports |
| 28 | # |
| 29 | |
| 30 | import os |
| 31 | import sys |
| 32 | |
| 33 | from multiprocessing.process import Process, current_process, active_children |
Jesse Noller | 41faa54 | 2009-01-25 03:45:53 +0000 | [diff] [blame] | 34 | from multiprocessing.util import SUBDEBUG, SUBWARNING |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 35 | |
| 36 | # |
Richard Oudkerk | 5046e97 | 2012-10-08 13:07:00 +0100 | [diff] [blame] | 37 | # Alias for main module -- will be reset by bootstrapping child processes |
| 38 | # |
| 39 | |
| 40 | if '__main__' in sys.modules: |
| 41 | sys.modules['__mp_main__'] = sys.modules['__main__'] |
| 42 | |
| 43 | # |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 44 | # Exceptions |
| 45 | # |
| 46 | |
| 47 | class ProcessError(Exception): |
| 48 | pass |
| 49 | |
| 50 | class BufferTooShort(ProcessError): |
| 51 | pass |
| 52 | |
| 53 | class TimeoutError(ProcessError): |
| 54 | pass |
| 55 | |
| 56 | class AuthenticationError(ProcessError): |
| 57 | pass |
| 58 | |
| 59 | import _multiprocessing |
| 60 | |
| 61 | # |
| 62 | # Definitions not depending on native semaphores |
| 63 | # |
| 64 | |
| 65 | def Manager(): |
| 66 | ''' |
| 67 | Returns a manager associated with a running server process |
| 68 | |
| 69 | The managers methods such as `Lock()`, `Condition()` and `Queue()` |
| 70 | can be used to create shared objects. |
| 71 | ''' |
| 72 | from multiprocessing.managers import SyncManager |
| 73 | m = SyncManager() |
| 74 | m.start() |
| 75 | return m |
| 76 | |
| 77 | def Pipe(duplex=True): |
| 78 | ''' |
| 79 | Returns two connection object connected by a pipe |
| 80 | ''' |
| 81 | from multiprocessing.connection import Pipe |
| 82 | return Pipe(duplex) |
| 83 | |
| 84 | def cpu_count(): |
| 85 | ''' |
| 86 | Returns the number of CPUs in the system |
| 87 | ''' |
| 88 | if sys.platform == 'win32': |
| 89 | try: |
| 90 | num = int(os.environ['NUMBER_OF_PROCESSORS']) |
| 91 | except (ValueError, KeyError): |
| 92 | num = 0 |
Benjamin Peterson | 4469d0c | 2008-11-30 22:46:23 +0000 | [diff] [blame] | 93 | elif 'bsd' in sys.platform or sys.platform == 'darwin': |
Ronald Oussoren | 0b8753d | 2011-03-16 09:41:32 -0400 | [diff] [blame] | 94 | comm = '/sbin/sysctl -n hw.ncpu' |
| 95 | if sys.platform == 'darwin': |
| 96 | comm = '/usr' + comm |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 97 | try: |
Ronald Oussoren | 0b8753d | 2011-03-16 09:41:32 -0400 | [diff] [blame] | 98 | with os.popen(comm) as p: |
Brian Curtin | 50be1ca | 2010-11-01 05:10:44 +0000 | [diff] [blame] | 99 | num = int(p.read()) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 100 | except ValueError: |
| 101 | num = 0 |
| 102 | else: |
| 103 | try: |
| 104 | num = os.sysconf('SC_NPROCESSORS_ONLN') |
| 105 | except (ValueError, OSError, AttributeError): |
| 106 | num = 0 |
| 107 | |
| 108 | if num >= 1: |
| 109 | return num |
| 110 | else: |
| 111 | raise NotImplementedError('cannot determine number of cpus') |
| 112 | |
| 113 | def freeze_support(): |
| 114 | ''' |
| 115 | Check whether this is a fake forked process in a frozen executable. |
| 116 | If so then run code specified by commandline and exit. |
| 117 | ''' |
| 118 | if sys.platform == 'win32' and getattr(sys, 'frozen', False): |
| 119 | from multiprocessing.forking import freeze_support |
| 120 | freeze_support() |
| 121 | |
| 122 | def get_logger(): |
| 123 | ''' |
| 124 | Return package logger -- if it does not already exist then it is created |
| 125 | ''' |
| 126 | from multiprocessing.util import get_logger |
| 127 | return get_logger() |
| 128 | |
| 129 | def log_to_stderr(level=None): |
| 130 | ''' |
| 131 | Turn on logging and add a handler which prints to stderr |
| 132 | ''' |
| 133 | from multiprocessing.util import log_to_stderr |
| 134 | return log_to_stderr(level) |
| 135 | |
| 136 | def allow_connection_pickling(): |
| 137 | ''' |
| 138 | Install support for sending connections and sockets between processes |
| 139 | ''' |
Antoine Pitrou | 5438ed1 | 2012-04-24 22:56:57 +0200 | [diff] [blame] | 140 | # This is undocumented. In previous versions of multiprocessing |
| 141 | # its only effect was to make socket objects inheritable on Windows. |
| 142 | import multiprocessing.connection |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 143 | |
| 144 | # |
| 145 | # Definitions depending on native semaphores |
| 146 | # |
| 147 | |
| 148 | def Lock(): |
| 149 | ''' |
| 150 | Returns a non-recursive lock object |
| 151 | ''' |
| 152 | from multiprocessing.synchronize import Lock |
| 153 | return Lock() |
| 154 | |
| 155 | def RLock(): |
| 156 | ''' |
| 157 | Returns a recursive lock object |
| 158 | ''' |
| 159 | from multiprocessing.synchronize import RLock |
| 160 | return RLock() |
| 161 | |
| 162 | def Condition(lock=None): |
| 163 | ''' |
| 164 | Returns a condition object |
| 165 | ''' |
| 166 | from multiprocessing.synchronize import Condition |
| 167 | return Condition(lock) |
| 168 | |
| 169 | def Semaphore(value=1): |
| 170 | ''' |
| 171 | Returns a semaphore object |
| 172 | ''' |
| 173 | from multiprocessing.synchronize import Semaphore |
| 174 | return Semaphore(value) |
| 175 | |
| 176 | def BoundedSemaphore(value=1): |
| 177 | ''' |
| 178 | Returns a bounded semaphore object |
| 179 | ''' |
| 180 | from multiprocessing.synchronize import BoundedSemaphore |
| 181 | return BoundedSemaphore(value) |
| 182 | |
| 183 | def Event(): |
| 184 | ''' |
| 185 | Returns an event object |
| 186 | ''' |
| 187 | from multiprocessing.synchronize import Event |
| 188 | return Event() |
| 189 | |
Richard Oudkerk | 3730a17 | 2012-06-15 18:26:07 +0100 | [diff] [blame] | 190 | def Barrier(parties, action=None, timeout=None): |
| 191 | ''' |
| 192 | Returns a barrier object |
| 193 | ''' |
| 194 | from multiprocessing.synchronize import Barrier |
| 195 | return Barrier(parties, action, timeout) |
| 196 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 197 | def Queue(maxsize=0): |
| 198 | ''' |
| 199 | Returns a queue object |
| 200 | ''' |
| 201 | from multiprocessing.queues import Queue |
| 202 | return Queue(maxsize) |
| 203 | |
| 204 | def JoinableQueue(maxsize=0): |
| 205 | ''' |
| 206 | Returns a queue object |
| 207 | ''' |
| 208 | from multiprocessing.queues import JoinableQueue |
| 209 | return JoinableQueue(maxsize) |
| 210 | |
Sandro Tosi | cd77815 | 2012-02-15 23:27:00 +0100 | [diff] [blame] | 211 | def SimpleQueue(): |
| 212 | ''' |
| 213 | Returns a queue object |
| 214 | ''' |
| 215 | from multiprocessing.queues import SimpleQueue |
| 216 | return SimpleQueue() |
| 217 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 218 | def Pool(processes=None, initializer=None, initargs=(), maxtasksperchild=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 219 | ''' |
| 220 | Returns a process pool object |
| 221 | ''' |
| 222 | from multiprocessing.pool import Pool |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 223 | return Pool(processes, initializer, initargs, maxtasksperchild) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 224 | |
| 225 | def RawValue(typecode_or_type, *args): |
| 226 | ''' |
| 227 | Returns a shared object |
| 228 | ''' |
| 229 | from multiprocessing.sharedctypes import RawValue |
| 230 | return RawValue(typecode_or_type, *args) |
| 231 | |
| 232 | def RawArray(typecode_or_type, size_or_initializer): |
| 233 | ''' |
| 234 | Returns a shared array |
| 235 | ''' |
| 236 | from multiprocessing.sharedctypes import RawArray |
| 237 | return RawArray(typecode_or_type, size_or_initializer) |
| 238 | |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 239 | def Value(typecode_or_type, *args, lock=True): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 240 | ''' |
| 241 | Returns a synchronized shared object |
| 242 | ''' |
| 243 | from multiprocessing.sharedctypes import Value |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 244 | return Value(typecode_or_type, *args, lock=lock) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 245 | |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 246 | def Array(typecode_or_type, size_or_initializer, *, lock=True): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 247 | ''' |
| 248 | Returns a synchronized shared array |
| 249 | ''' |
| 250 | from multiprocessing.sharedctypes import Array |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 251 | return Array(typecode_or_type, size_or_initializer, lock=lock) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 252 | |
| 253 | # |
| 254 | # |
| 255 | # |
| 256 | |
| 257 | if sys.platform == 'win32': |
| 258 | |
| 259 | def set_executable(executable): |
| 260 | ''' |
| 261 | Sets the path to a python.exe or pythonw.exe binary used to run |
| 262 | child processes on Windows instead of sys.executable. |
| 263 | Useful for people embedding Python. |
| 264 | ''' |
| 265 | from multiprocessing.forking import set_executable |
| 266 | set_executable(executable) |
| 267 | |
| 268 | __all__ += ['set_executable'] |