Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 1 | # |
| 2 | # Package analogous to 'threading.py' but using processes |
| 3 | # |
| 4 | # multiprocessing/__init__.py |
| 5 | # |
| 6 | # This package is intended to duplicate the functionality (and much of |
| 7 | # the API) of threading.py but uses processes instead of threads. A |
| 8 | # subpackage 'multiprocessing.dummy' has the same API but is a simple |
| 9 | # wrapper for 'threading'. |
| 10 | # |
| 11 | # Try calling `multiprocessing.doc.main()` to read the html |
Ezio Melotti | e130a52 | 2011-10-19 10:58:56 +0300 | [diff] [blame] | 12 | # documentation in a webbrowser. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 13 | # |
| 14 | # |
| 15 | # Copyright (c) 2006-2008, R Oudkerk |
Richard Oudkerk | 3e268aa | 2012-04-30 12:13:55 +0100 | [diff] [blame] | 16 | # Licensed to PSF under a Contributor Agreement. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 17 | # |
| 18 | |
| 19 | __version__ = '0.70a1' |
| 20 | |
| 21 | __all__ = [ |
| 22 | 'Process', 'current_process', 'active_children', 'freeze_support', |
| 23 | 'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger', |
| 24 | 'allow_connection_pickling', 'BufferTooShort', 'TimeoutError', |
| 25 | 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', |
Richard Oudkerk | 3730a17 | 2012-06-15 18:26:07 +0100 | [diff] [blame] | 26 | 'Event', 'Barrier', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', |
| 27 | 'Value', 'Array', 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 28 | ] |
| 29 | |
| 30 | __author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)' |
| 31 | |
| 32 | # |
| 33 | # Imports |
| 34 | # |
| 35 | |
| 36 | import os |
| 37 | import sys |
| 38 | |
| 39 | from multiprocessing.process import Process, current_process, active_children |
Jesse Noller | 41faa54 | 2009-01-25 03:45:53 +0000 | [diff] [blame] | 40 | from multiprocessing.util import SUBDEBUG, SUBWARNING |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 41 | |
| 42 | # |
Richard Oudkerk | 5046e97 | 2012-10-08 13:07:00 +0100 | [diff] [blame] | 43 | # Alias for main module -- will be reset by bootstrapping child processes |
| 44 | # |
| 45 | |
| 46 | if '__main__' in sys.modules: |
| 47 | sys.modules['__mp_main__'] = sys.modules['__main__'] |
| 48 | |
| 49 | # |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 50 | # Exceptions |
| 51 | # |
| 52 | |
| 53 | class ProcessError(Exception): |
| 54 | pass |
| 55 | |
| 56 | class BufferTooShort(ProcessError): |
| 57 | pass |
| 58 | |
| 59 | class TimeoutError(ProcessError): |
| 60 | pass |
| 61 | |
| 62 | class AuthenticationError(ProcessError): |
| 63 | pass |
| 64 | |
| 65 | import _multiprocessing |
| 66 | |
| 67 | # |
| 68 | # Definitions not depending on native semaphores |
| 69 | # |
| 70 | |
| 71 | def Manager(): |
| 72 | ''' |
| 73 | Returns a manager associated with a running server process |
| 74 | |
| 75 | The managers methods such as `Lock()`, `Condition()` and `Queue()` |
| 76 | can be used to create shared objects. |
| 77 | ''' |
| 78 | from multiprocessing.managers import SyncManager |
| 79 | m = SyncManager() |
| 80 | m.start() |
| 81 | return m |
| 82 | |
| 83 | def Pipe(duplex=True): |
| 84 | ''' |
| 85 | Returns two connection object connected by a pipe |
| 86 | ''' |
| 87 | from multiprocessing.connection import Pipe |
| 88 | return Pipe(duplex) |
| 89 | |
| 90 | def cpu_count(): |
| 91 | ''' |
| 92 | Returns the number of CPUs in the system |
| 93 | ''' |
| 94 | if sys.platform == 'win32': |
| 95 | try: |
| 96 | num = int(os.environ['NUMBER_OF_PROCESSORS']) |
| 97 | except (ValueError, KeyError): |
| 98 | num = 0 |
Benjamin Peterson | 4469d0c | 2008-11-30 22:46:23 +0000 | [diff] [blame] | 99 | elif 'bsd' in sys.platform or sys.platform == 'darwin': |
Ronald Oussoren | 0b8753d | 2011-03-16 09:41:32 -0400 | [diff] [blame] | 100 | comm = '/sbin/sysctl -n hw.ncpu' |
| 101 | if sys.platform == 'darwin': |
| 102 | comm = '/usr' + comm |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 103 | try: |
Ronald Oussoren | 0b8753d | 2011-03-16 09:41:32 -0400 | [diff] [blame] | 104 | with os.popen(comm) as p: |
Brian Curtin | 50be1ca | 2010-11-01 05:10:44 +0000 | [diff] [blame] | 105 | num = int(p.read()) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 106 | except ValueError: |
| 107 | num = 0 |
| 108 | else: |
| 109 | try: |
| 110 | num = os.sysconf('SC_NPROCESSORS_ONLN') |
| 111 | except (ValueError, OSError, AttributeError): |
| 112 | num = 0 |
| 113 | |
| 114 | if num >= 1: |
| 115 | return num |
| 116 | else: |
| 117 | raise NotImplementedError('cannot determine number of cpus') |
| 118 | |
| 119 | def freeze_support(): |
| 120 | ''' |
| 121 | Check whether this is a fake forked process in a frozen executable. |
| 122 | If so then run code specified by commandline and exit. |
| 123 | ''' |
| 124 | if sys.platform == 'win32' and getattr(sys, 'frozen', False): |
| 125 | from multiprocessing.forking import freeze_support |
| 126 | freeze_support() |
| 127 | |
| 128 | def get_logger(): |
| 129 | ''' |
| 130 | Return package logger -- if it does not already exist then it is created |
| 131 | ''' |
| 132 | from multiprocessing.util import get_logger |
| 133 | return get_logger() |
| 134 | |
| 135 | def log_to_stderr(level=None): |
| 136 | ''' |
| 137 | Turn on logging and add a handler which prints to stderr |
| 138 | ''' |
| 139 | from multiprocessing.util import log_to_stderr |
| 140 | return log_to_stderr(level) |
| 141 | |
| 142 | def allow_connection_pickling(): |
| 143 | ''' |
| 144 | Install support for sending connections and sockets between processes |
| 145 | ''' |
Antoine Pitrou | 5438ed1 | 2012-04-24 22:56:57 +0200 | [diff] [blame] | 146 | # This is undocumented. In previous versions of multiprocessing |
| 147 | # its only effect was to make socket objects inheritable on Windows. |
| 148 | import multiprocessing.connection |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 149 | |
| 150 | # |
| 151 | # Definitions depending on native semaphores |
| 152 | # |
| 153 | |
| 154 | def Lock(): |
| 155 | ''' |
| 156 | Returns a non-recursive lock object |
| 157 | ''' |
| 158 | from multiprocessing.synchronize import Lock |
| 159 | return Lock() |
| 160 | |
| 161 | def RLock(): |
| 162 | ''' |
| 163 | Returns a recursive lock object |
| 164 | ''' |
| 165 | from multiprocessing.synchronize import RLock |
| 166 | return RLock() |
| 167 | |
| 168 | def Condition(lock=None): |
| 169 | ''' |
| 170 | Returns a condition object |
| 171 | ''' |
| 172 | from multiprocessing.synchronize import Condition |
| 173 | return Condition(lock) |
| 174 | |
| 175 | def Semaphore(value=1): |
| 176 | ''' |
| 177 | Returns a semaphore object |
| 178 | ''' |
| 179 | from multiprocessing.synchronize import Semaphore |
| 180 | return Semaphore(value) |
| 181 | |
| 182 | def BoundedSemaphore(value=1): |
| 183 | ''' |
| 184 | Returns a bounded semaphore object |
| 185 | ''' |
| 186 | from multiprocessing.synchronize import BoundedSemaphore |
| 187 | return BoundedSemaphore(value) |
| 188 | |
| 189 | def Event(): |
| 190 | ''' |
| 191 | Returns an event object |
| 192 | ''' |
| 193 | from multiprocessing.synchronize import Event |
| 194 | return Event() |
| 195 | |
Richard Oudkerk | 3730a17 | 2012-06-15 18:26:07 +0100 | [diff] [blame] | 196 | def Barrier(parties, action=None, timeout=None): |
| 197 | ''' |
| 198 | Returns a barrier object |
| 199 | ''' |
| 200 | from multiprocessing.synchronize import Barrier |
| 201 | return Barrier(parties, action, timeout) |
| 202 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 203 | def Queue(maxsize=0): |
| 204 | ''' |
| 205 | Returns a queue object |
| 206 | ''' |
| 207 | from multiprocessing.queues import Queue |
| 208 | return Queue(maxsize) |
| 209 | |
| 210 | def JoinableQueue(maxsize=0): |
| 211 | ''' |
| 212 | Returns a queue object |
| 213 | ''' |
| 214 | from multiprocessing.queues import JoinableQueue |
| 215 | return JoinableQueue(maxsize) |
| 216 | |
Sandro Tosi | cd77815 | 2012-02-15 23:27:00 +0100 | [diff] [blame] | 217 | def SimpleQueue(): |
| 218 | ''' |
| 219 | Returns a queue object |
| 220 | ''' |
| 221 | from multiprocessing.queues import SimpleQueue |
| 222 | return SimpleQueue() |
| 223 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 224 | def Pool(processes=None, initializer=None, initargs=(), maxtasksperchild=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 225 | ''' |
| 226 | Returns a process pool object |
| 227 | ''' |
| 228 | from multiprocessing.pool import Pool |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 229 | return Pool(processes, initializer, initargs, maxtasksperchild) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 230 | |
| 231 | def RawValue(typecode_or_type, *args): |
| 232 | ''' |
| 233 | Returns a shared object |
| 234 | ''' |
| 235 | from multiprocessing.sharedctypes import RawValue |
| 236 | return RawValue(typecode_or_type, *args) |
| 237 | |
| 238 | def RawArray(typecode_or_type, size_or_initializer): |
| 239 | ''' |
| 240 | Returns a shared array |
| 241 | ''' |
| 242 | from multiprocessing.sharedctypes import RawArray |
| 243 | return RawArray(typecode_or_type, size_or_initializer) |
| 244 | |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 245 | def Value(typecode_or_type, *args, lock=True): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 246 | ''' |
| 247 | Returns a synchronized shared object |
| 248 | ''' |
| 249 | from multiprocessing.sharedctypes import Value |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 250 | return Value(typecode_or_type, *args, lock=lock) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 251 | |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 252 | def Array(typecode_or_type, size_or_initializer, *, lock=True): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 253 | ''' |
| 254 | Returns a synchronized shared array |
| 255 | ''' |
| 256 | from multiprocessing.sharedctypes import Array |
Richard Oudkerk | 87ea780 | 2012-05-29 12:01:47 +0100 | [diff] [blame] | 257 | return Array(typecode_or_type, size_or_initializer, lock=lock) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 258 | |
| 259 | # |
| 260 | # |
| 261 | # |
| 262 | |
| 263 | if sys.platform == 'win32': |
| 264 | |
| 265 | def set_executable(executable): |
| 266 | ''' |
| 267 | Sets the path to a python.exe or pythonw.exe binary used to run |
| 268 | child processes on Windows instead of sys.executable. |
| 269 | Useful for people embedding Python. |
| 270 | ''' |
| 271 | from multiprocessing.forking import set_executable |
| 272 | set_executable(executable) |
| 273 | |
| 274 | __all__ += ['set_executable'] |