Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 1 | # |
| 2 | # Module providing the `Pool` class for managing a process pool |
| 3 | # |
| 4 | # multiprocessing/pool.py |
| 5 | # |
R. David Murray | 3fc969a | 2010-12-14 01:38:16 +0000 | [diff] [blame] | 6 | # Copyright (c) 2006-2008, R Oudkerk |
Richard Oudkerk | 3e268aa | 2012-04-30 12:13:55 +0100 | [diff] [blame] | 7 | # Licensed to PSF under a Contributor Agreement. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 8 | # |
| 9 | |
| 10 | __all__ = ['Pool'] |
| 11 | |
| 12 | # |
| 13 | # Imports |
| 14 | # |
| 15 | |
| 16 | import threading |
| 17 | import queue |
| 18 | import itertools |
| 19 | import collections |
| 20 | import time |
| 21 | |
| 22 | from multiprocessing import Process, cpu_count, TimeoutError |
| 23 | from multiprocessing.util import Finalize, debug |
| 24 | |
| 25 | # |
| 26 | # Constants representing the state of a pool |
| 27 | # |
| 28 | |
| 29 | RUN = 0 |
| 30 | CLOSE = 1 |
| 31 | TERMINATE = 2 |
| 32 | |
| 33 | # |
| 34 | # Miscellaneous |
| 35 | # |
| 36 | |
| 37 | job_counter = itertools.count() |
| 38 | |
| 39 | def mapstar(args): |
| 40 | return list(map(*args)) |
| 41 | |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 42 | def starmapstar(args): |
| 43 | return list(itertools.starmap(args[0], args[1])) |
| 44 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 45 | # |
| 46 | # Code run by worker processes |
| 47 | # |
| 48 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 49 | class MaybeEncodingError(Exception): |
| 50 | """Wraps possible unpickleable errors, so they can be |
| 51 | safely sent through the socket.""" |
| 52 | |
| 53 | def __init__(self, exc, value): |
| 54 | self.exc = repr(exc) |
| 55 | self.value = repr(value) |
| 56 | super(MaybeEncodingError, self).__init__(self.exc, self.value) |
| 57 | |
| 58 | def __str__(self): |
| 59 | return "Error sending result: '%s'. Reason: '%s'" % (self.value, |
| 60 | self.exc) |
| 61 | |
| 62 | def __repr__(self): |
| 63 | return "<MaybeEncodingError: %s>" % str(self) |
| 64 | |
| 65 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 66 | def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None): |
| 67 | assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 68 | put = outqueue.put |
| 69 | get = inqueue.get |
| 70 | if hasattr(inqueue, '_writer'): |
| 71 | inqueue._writer.close() |
| 72 | outqueue._reader.close() |
| 73 | |
| 74 | if initializer is not None: |
| 75 | initializer(*initargs) |
| 76 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 77 | completed = 0 |
| 78 | while maxtasks is None or (maxtasks and completed < maxtasks): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 79 | try: |
| 80 | task = get() |
| 81 | except (EOFError, IOError): |
| 82 | debug('worker got EOFError or IOError -- exiting') |
| 83 | break |
| 84 | |
| 85 | if task is None: |
| 86 | debug('worker got sentinel -- exiting') |
| 87 | break |
| 88 | |
| 89 | job, i, func, args, kwds = task |
| 90 | try: |
| 91 | result = (True, func(*args, **kwds)) |
| 92 | except Exception as e: |
| 93 | result = (False, e) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 94 | try: |
| 95 | put((job, i, result)) |
| 96 | except Exception as e: |
| 97 | wrapped = MaybeEncodingError(e, result[1]) |
| 98 | debug("Possible encoding error while sending result: %s" % ( |
| 99 | wrapped)) |
| 100 | put((job, i, (False, wrapped))) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 101 | completed += 1 |
| 102 | debug('worker exiting after %d tasks' % completed) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 103 | |
| 104 | # |
| 105 | # Class representing a process pool |
| 106 | # |
| 107 | |
| 108 | class Pool(object): |
| 109 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 110 | Class which supports an async version of applying functions to arguments. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 111 | ''' |
| 112 | Process = Process |
| 113 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 114 | def __init__(self, processes=None, initializer=None, initargs=(), |
| 115 | maxtasksperchild=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 116 | self._setup_queues() |
| 117 | self._taskqueue = queue.Queue() |
| 118 | self._cache = {} |
| 119 | self._state = RUN |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 120 | self._maxtasksperchild = maxtasksperchild |
| 121 | self._initializer = initializer |
| 122 | self._initargs = initargs |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 123 | |
| 124 | if processes is None: |
| 125 | try: |
| 126 | processes = cpu_count() |
| 127 | except NotImplementedError: |
| 128 | processes = 1 |
Victor Stinner | 2fae27b | 2011-06-20 17:53:35 +0200 | [diff] [blame] | 129 | if processes < 1: |
| 130 | raise ValueError("Number of processes must be at least 1") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 131 | |
Florent Xicluna | 5d1155c | 2011-10-28 14:45:05 +0200 | [diff] [blame] | 132 | if initializer is not None and not callable(initializer): |
Benjamin Peterson | f47ed4a | 2009-04-11 20:45:40 +0000 | [diff] [blame] | 133 | raise TypeError('initializer must be a callable') |
| 134 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 135 | self._processes = processes |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 136 | self._pool = [] |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 137 | self._repopulate_pool() |
| 138 | |
| 139 | self._worker_handler = threading.Thread( |
| 140 | target=Pool._handle_workers, |
| 141 | args=(self, ) |
| 142 | ) |
| 143 | self._worker_handler.daemon = True |
| 144 | self._worker_handler._state = RUN |
| 145 | self._worker_handler.start() |
| 146 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 147 | |
| 148 | self._task_handler = threading.Thread( |
| 149 | target=Pool._handle_tasks, |
| 150 | args=(self._taskqueue, self._quick_put, self._outqueue, self._pool) |
| 151 | ) |
Benjamin Peterson | fae4c62 | 2008-08-18 18:40:08 +0000 | [diff] [blame] | 152 | self._task_handler.daemon = True |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 153 | self._task_handler._state = RUN |
| 154 | self._task_handler.start() |
| 155 | |
| 156 | self._result_handler = threading.Thread( |
| 157 | target=Pool._handle_results, |
| 158 | args=(self._outqueue, self._quick_get, self._cache) |
| 159 | ) |
Benjamin Peterson | fae4c62 | 2008-08-18 18:40:08 +0000 | [diff] [blame] | 160 | self._result_handler.daemon = True |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 161 | self._result_handler._state = RUN |
| 162 | self._result_handler.start() |
| 163 | |
| 164 | self._terminate = Finalize( |
| 165 | self, self._terminate_pool, |
| 166 | args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 167 | self._worker_handler, self._task_handler, |
| 168 | self._result_handler, self._cache), |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 169 | exitpriority=15 |
| 170 | ) |
| 171 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 172 | def _join_exited_workers(self): |
| 173 | """Cleanup after any worker processes which have exited due to reaching |
| 174 | their specified lifetime. Returns True if any workers were cleaned up. |
| 175 | """ |
| 176 | cleaned = False |
| 177 | for i in reversed(range(len(self._pool))): |
| 178 | worker = self._pool[i] |
| 179 | if worker.exitcode is not None: |
| 180 | # worker exited |
| 181 | debug('cleaning up worker %d' % i) |
| 182 | worker.join() |
| 183 | cleaned = True |
| 184 | del self._pool[i] |
| 185 | return cleaned |
| 186 | |
| 187 | def _repopulate_pool(self): |
| 188 | """Bring the number of pool processes up to the specified number, |
| 189 | for use after reaping workers which have exited. |
| 190 | """ |
| 191 | for i in range(self._processes - len(self._pool)): |
| 192 | w = self.Process(target=worker, |
| 193 | args=(self._inqueue, self._outqueue, |
| 194 | self._initializer, |
| 195 | self._initargs, self._maxtasksperchild) |
| 196 | ) |
| 197 | self._pool.append(w) |
| 198 | w.name = w.name.replace('Process', 'PoolWorker') |
| 199 | w.daemon = True |
| 200 | w.start() |
| 201 | debug('added worker') |
| 202 | |
| 203 | def _maintain_pool(self): |
| 204 | """Clean up any exited workers and start replacements for them. |
| 205 | """ |
| 206 | if self._join_exited_workers(): |
| 207 | self._repopulate_pool() |
| 208 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 209 | def _setup_queues(self): |
| 210 | from .queues import SimpleQueue |
| 211 | self._inqueue = SimpleQueue() |
| 212 | self._outqueue = SimpleQueue() |
| 213 | self._quick_put = self._inqueue._writer.send |
| 214 | self._quick_get = self._outqueue._reader.recv |
| 215 | |
| 216 | def apply(self, func, args=(), kwds={}): |
| 217 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 218 | Equivalent of `func(*args, **kwds)`. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 219 | ''' |
| 220 | assert self._state == RUN |
| 221 | return self.apply_async(func, args, kwds).get() |
| 222 | |
| 223 | def map(self, func, iterable, chunksize=None): |
| 224 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 225 | Apply `func` to each element in `iterable`, collecting the results |
| 226 | in a list that is returned. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 227 | ''' |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 228 | return self._map_async(func, iterable, mapstar, chunksize).get() |
| 229 | |
| 230 | def starmap(self, func, iterable, chunksize=None): |
| 231 | ''' |
| 232 | Like `map()` method but the elements of the `iterable` are expected to |
| 233 | be iterables as well and will be unpacked as arguments. Hence |
| 234 | `func` and (a, b) becomes func(a, b). |
| 235 | ''' |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 236 | return self._map_async(func, iterable, starmapstar, chunksize).get() |
| 237 | |
| 238 | def starmap_async(self, func, iterable, chunksize=None, callback=None, |
| 239 | error_callback=None): |
| 240 | ''' |
| 241 | Asynchronous version of `starmap()` method. |
| 242 | ''' |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 243 | return self._map_async(func, iterable, starmapstar, chunksize, |
| 244 | callback, error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 245 | |
| 246 | def imap(self, func, iterable, chunksize=1): |
| 247 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 248 | Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 249 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 250 | if self._state != RUN: |
| 251 | raise ValueError("Pool not running") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 252 | if chunksize == 1: |
| 253 | result = IMapIterator(self._cache) |
| 254 | self._taskqueue.put((((result._job, i, func, (x,), {}) |
| 255 | for i, x in enumerate(iterable)), result._set_length)) |
| 256 | return result |
| 257 | else: |
| 258 | assert chunksize > 1 |
| 259 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
| 260 | result = IMapIterator(self._cache) |
| 261 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 262 | for i, x in enumerate(task_batches)), result._set_length)) |
| 263 | return (item for chunk in result for item in chunk) |
| 264 | |
| 265 | def imap_unordered(self, func, iterable, chunksize=1): |
| 266 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 267 | Like `imap()` method but ordering of results is arbitrary. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 268 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 269 | if self._state != RUN: |
| 270 | raise ValueError("Pool not running") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 271 | if chunksize == 1: |
| 272 | result = IMapUnorderedIterator(self._cache) |
| 273 | self._taskqueue.put((((result._job, i, func, (x,), {}) |
| 274 | for i, x in enumerate(iterable)), result._set_length)) |
| 275 | return result |
| 276 | else: |
| 277 | assert chunksize > 1 |
| 278 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
| 279 | result = IMapUnorderedIterator(self._cache) |
| 280 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 281 | for i, x in enumerate(task_batches)), result._set_length)) |
| 282 | return (item for chunk in result for item in chunk) |
| 283 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 284 | def apply_async(self, func, args=(), kwds={}, callback=None, |
| 285 | error_callback=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 286 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 287 | Asynchronous version of `apply()` method. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 288 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 289 | if self._state != RUN: |
| 290 | raise ValueError("Pool not running") |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 291 | result = ApplyResult(self._cache, callback, error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 292 | self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) |
| 293 | return result |
| 294 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 295 | def map_async(self, func, iterable, chunksize=None, callback=None, |
| 296 | error_callback=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 297 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 298 | Asynchronous version of `map()` method. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 299 | ''' |
Hynek Schlawack | 254af26 | 2012-10-27 12:53:02 +0200 | [diff] [blame] | 300 | return self._map_async(func, iterable, mapstar, chunksize, callback, |
| 301 | error_callback) |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 302 | |
| 303 | def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, |
| 304 | error_callback=None): |
| 305 | ''' |
| 306 | Helper function to implement map, starmap and their async counterparts. |
| 307 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 308 | if self._state != RUN: |
| 309 | raise ValueError("Pool not running") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 310 | if not hasattr(iterable, '__len__'): |
| 311 | iterable = list(iterable) |
| 312 | |
| 313 | if chunksize is None: |
| 314 | chunksize, extra = divmod(len(iterable), len(self._pool) * 4) |
| 315 | if extra: |
| 316 | chunksize += 1 |
Alexandre Vassalotti | e52e378 | 2009-07-17 09:18:18 +0000 | [diff] [blame] | 317 | if len(iterable) == 0: |
| 318 | chunksize = 0 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 319 | |
| 320 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 321 | result = MapResult(self._cache, chunksize, len(iterable), callback, |
| 322 | error_callback=error_callback) |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 323 | self._taskqueue.put((((result._job, i, mapper, (x,), {}) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 324 | for i, x in enumerate(task_batches)), None)) |
| 325 | return result |
| 326 | |
| 327 | @staticmethod |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 328 | def _handle_workers(pool): |
Charles-François Natali | f8859e1 | 2011-10-24 18:45:29 +0200 | [diff] [blame] | 329 | thread = threading.current_thread() |
| 330 | |
| 331 | # Keep maintaining workers until the cache gets drained, unless the pool |
| 332 | # is terminated. |
| 333 | while thread._state == RUN or (pool._cache and thread._state != TERMINATE): |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 334 | pool._maintain_pool() |
| 335 | time.sleep(0.1) |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 336 | # send sentinel to stop workers |
| 337 | pool._taskqueue.put(None) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 338 | debug('worker handler exiting') |
| 339 | |
| 340 | @staticmethod |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 341 | def _handle_tasks(taskqueue, put, outqueue, pool): |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 342 | thread = threading.current_thread() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 343 | |
| 344 | for taskseq, set_length in iter(taskqueue.get, None): |
| 345 | i = -1 |
| 346 | for i, task in enumerate(taskseq): |
| 347 | if thread._state: |
| 348 | debug('task handler found thread._state != RUN') |
| 349 | break |
| 350 | try: |
| 351 | put(task) |
| 352 | except IOError: |
| 353 | debug('could not put task on queue') |
| 354 | break |
| 355 | else: |
| 356 | if set_length: |
| 357 | debug('doing set_length()') |
| 358 | set_length(i+1) |
| 359 | continue |
| 360 | break |
| 361 | else: |
| 362 | debug('task handler got sentinel') |
| 363 | |
| 364 | |
| 365 | try: |
| 366 | # tell result handler to finish when cache is empty |
| 367 | debug('task handler sending sentinel to result handler') |
| 368 | outqueue.put(None) |
| 369 | |
| 370 | # tell workers there is no more work |
| 371 | debug('task handler sending sentinel to workers') |
| 372 | for p in pool: |
| 373 | put(None) |
| 374 | except IOError: |
| 375 | debug('task handler got IOError when sending sentinels') |
| 376 | |
| 377 | debug('task handler exiting') |
| 378 | |
| 379 | @staticmethod |
| 380 | def _handle_results(outqueue, get, cache): |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 381 | thread = threading.current_thread() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 382 | |
| 383 | while 1: |
| 384 | try: |
| 385 | task = get() |
| 386 | except (IOError, EOFError): |
| 387 | debug('result handler got EOFError/IOError -- exiting') |
| 388 | return |
| 389 | |
| 390 | if thread._state: |
| 391 | assert thread._state == TERMINATE |
| 392 | debug('result handler found thread._state=TERMINATE') |
| 393 | break |
| 394 | |
| 395 | if task is None: |
| 396 | debug('result handler got sentinel') |
| 397 | break |
| 398 | |
| 399 | job, i, obj = task |
| 400 | try: |
| 401 | cache[job]._set(i, obj) |
| 402 | except KeyError: |
| 403 | pass |
| 404 | |
| 405 | while cache and thread._state != TERMINATE: |
| 406 | try: |
| 407 | task = get() |
| 408 | except (IOError, EOFError): |
| 409 | debug('result handler got EOFError/IOError -- exiting') |
| 410 | return |
| 411 | |
| 412 | if task is None: |
| 413 | debug('result handler ignoring extra sentinel') |
| 414 | continue |
| 415 | job, i, obj = task |
| 416 | try: |
| 417 | cache[job]._set(i, obj) |
| 418 | except KeyError: |
| 419 | pass |
| 420 | |
| 421 | if hasattr(outqueue, '_reader'): |
| 422 | debug('ensuring that outqueue is not full') |
| 423 | # If we don't make room available in outqueue then |
| 424 | # attempts to add the sentinel (None) to outqueue may |
| 425 | # block. There is guaranteed to be no more than 2 sentinels. |
| 426 | try: |
| 427 | for i in range(10): |
| 428 | if not outqueue._reader.poll(): |
| 429 | break |
| 430 | get() |
| 431 | except (IOError, EOFError): |
| 432 | pass |
| 433 | |
| 434 | debug('result handler exiting: len(cache)=%s, thread._state=%s', |
| 435 | len(cache), thread._state) |
| 436 | |
| 437 | @staticmethod |
| 438 | def _get_tasks(func, it, size): |
| 439 | it = iter(it) |
| 440 | while 1: |
| 441 | x = tuple(itertools.islice(it, size)) |
| 442 | if not x: |
| 443 | return |
| 444 | yield (func, x) |
| 445 | |
| 446 | def __reduce__(self): |
| 447 | raise NotImplementedError( |
| 448 | 'pool objects cannot be passed between processes or pickled' |
| 449 | ) |
| 450 | |
| 451 | def close(self): |
| 452 | debug('closing pool') |
| 453 | if self._state == RUN: |
| 454 | self._state = CLOSE |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 455 | self._worker_handler._state = CLOSE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 456 | |
| 457 | def terminate(self): |
| 458 | debug('terminating pool') |
| 459 | self._state = TERMINATE |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 460 | self._worker_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 461 | self._terminate() |
| 462 | |
| 463 | def join(self): |
| 464 | debug('joining pool') |
| 465 | assert self._state in (CLOSE, TERMINATE) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 466 | self._worker_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 467 | self._task_handler.join() |
| 468 | self._result_handler.join() |
| 469 | for p in self._pool: |
| 470 | p.join() |
| 471 | |
| 472 | @staticmethod |
| 473 | def _help_stuff_finish(inqueue, task_handler, size): |
| 474 | # task_handler may be blocked trying to put items on inqueue |
| 475 | debug('removing tasks from inqueue until task handler finished') |
| 476 | inqueue._rlock.acquire() |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 477 | while task_handler.is_alive() and inqueue._reader.poll(): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 478 | inqueue._reader.recv() |
| 479 | time.sleep(0) |
| 480 | |
| 481 | @classmethod |
| 482 | def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 483 | worker_handler, task_handler, result_handler, cache): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 484 | # this is guaranteed to only be called once |
| 485 | debug('finalizing pool') |
| 486 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 487 | worker_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 488 | task_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 489 | |
| 490 | debug('helping task handler/workers to finish') |
| 491 | cls._help_stuff_finish(inqueue, task_handler, len(pool)) |
| 492 | |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 493 | assert result_handler.is_alive() or len(cache) == 0 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 494 | |
| 495 | result_handler._state = TERMINATE |
| 496 | outqueue.put(None) # sentinel |
| 497 | |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 498 | # We must wait for the worker handler to exit before terminating |
| 499 | # workers because we don't want workers to be restarted behind our back. |
| 500 | debug('joining worker handler') |
Richard Oudkerk | f29ec4b | 2012-06-18 15:54:57 +0100 | [diff] [blame] | 501 | if threading.current_thread() is not worker_handler: |
| 502 | worker_handler.join() |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 503 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 504 | # Terminate workers which haven't already finished. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 505 | if pool and hasattr(pool[0], 'terminate'): |
| 506 | debug('terminating workers') |
| 507 | for p in pool: |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 508 | if p.exitcode is None: |
| 509 | p.terminate() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 510 | |
| 511 | debug('joining task handler') |
Richard Oudkerk | f29ec4b | 2012-06-18 15:54:57 +0100 | [diff] [blame] | 512 | if threading.current_thread() is not task_handler: |
| 513 | task_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 514 | |
| 515 | debug('joining result handler') |
Richard Oudkerk | f29ec4b | 2012-06-18 15:54:57 +0100 | [diff] [blame] | 516 | if threading.current_thread() is not result_handler: |
| 517 | result_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 518 | |
| 519 | if pool and hasattr(pool[0], 'terminate'): |
| 520 | debug('joining pool workers') |
| 521 | for p in pool: |
Florent Xicluna | 998171f | 2010-03-08 13:32:17 +0000 | [diff] [blame] | 522 | if p.is_alive(): |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 523 | # worker has not yet exited |
Florent Xicluna | 998171f | 2010-03-08 13:32:17 +0000 | [diff] [blame] | 524 | debug('cleaning up worker %d' % p.pid) |
| 525 | p.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 526 | |
Richard Oudkerk | d69cfe8 | 2012-06-18 17:47:52 +0100 | [diff] [blame] | 527 | def __enter__(self): |
| 528 | return self |
| 529 | |
| 530 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 531 | self.terminate() |
| 532 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 533 | # |
| 534 | # Class whose instances are returned by `Pool.apply_async()` |
| 535 | # |
| 536 | |
| 537 | class ApplyResult(object): |
| 538 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 539 | def __init__(self, cache, callback, error_callback): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 540 | self._event = threading.Event() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 541 | self._job = next(job_counter) |
| 542 | self._cache = cache |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 543 | self._callback = callback |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 544 | self._error_callback = error_callback |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 545 | cache[self._job] = self |
| 546 | |
| 547 | def ready(self): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 548 | return self._event.is_set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 549 | |
| 550 | def successful(self): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 551 | assert self.ready() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 552 | return self._success |
| 553 | |
| 554 | def wait(self, timeout=None): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 555 | self._event.wait(timeout) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 556 | |
| 557 | def get(self, timeout=None): |
| 558 | self.wait(timeout) |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 559 | if not self.ready(): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 560 | raise TimeoutError |
| 561 | if self._success: |
| 562 | return self._value |
| 563 | else: |
| 564 | raise self._value |
| 565 | |
| 566 | def _set(self, i, obj): |
| 567 | self._success, self._value = obj |
| 568 | if self._callback and self._success: |
| 569 | self._callback(self._value) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 570 | if self._error_callback and not self._success: |
| 571 | self._error_callback(self._value) |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 572 | self._event.set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 573 | del self._cache[self._job] |
| 574 | |
| 575 | # |
| 576 | # Class whose instances are returned by `Pool.map_async()` |
| 577 | # |
| 578 | |
| 579 | class MapResult(ApplyResult): |
| 580 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 581 | def __init__(self, cache, chunksize, length, callback, error_callback): |
| 582 | ApplyResult.__init__(self, cache, callback, |
| 583 | error_callback=error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 584 | self._success = True |
| 585 | self._value = [None] * length |
| 586 | self._chunksize = chunksize |
| 587 | if chunksize <= 0: |
| 588 | self._number_left = 0 |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 589 | self._event.set() |
Richard Oudkerk | e41682b | 2012-06-06 19:04:57 +0100 | [diff] [blame] | 590 | del cache[self._job] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 591 | else: |
| 592 | self._number_left = length//chunksize + bool(length % chunksize) |
| 593 | |
| 594 | def _set(self, i, success_result): |
| 595 | success, result = success_result |
| 596 | if success: |
| 597 | self._value[i*self._chunksize:(i+1)*self._chunksize] = result |
| 598 | self._number_left -= 1 |
| 599 | if self._number_left == 0: |
| 600 | if self._callback: |
| 601 | self._callback(self._value) |
| 602 | del self._cache[self._job] |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 603 | self._event.set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 604 | else: |
| 605 | self._success = False |
| 606 | self._value = result |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 607 | if self._error_callback: |
| 608 | self._error_callback(self._value) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 609 | del self._cache[self._job] |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 610 | self._event.set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 611 | |
| 612 | # |
| 613 | # Class whose instances are returned by `Pool.imap()` |
| 614 | # |
| 615 | |
| 616 | class IMapIterator(object): |
| 617 | |
| 618 | def __init__(self, cache): |
| 619 | self._cond = threading.Condition(threading.Lock()) |
| 620 | self._job = next(job_counter) |
| 621 | self._cache = cache |
| 622 | self._items = collections.deque() |
| 623 | self._index = 0 |
| 624 | self._length = None |
| 625 | self._unsorted = {} |
| 626 | cache[self._job] = self |
| 627 | |
| 628 | def __iter__(self): |
| 629 | return self |
| 630 | |
| 631 | def next(self, timeout=None): |
| 632 | self._cond.acquire() |
| 633 | try: |
| 634 | try: |
| 635 | item = self._items.popleft() |
| 636 | except IndexError: |
| 637 | if self._index == self._length: |
| 638 | raise StopIteration |
| 639 | self._cond.wait(timeout) |
| 640 | try: |
| 641 | item = self._items.popleft() |
| 642 | except IndexError: |
| 643 | if self._index == self._length: |
| 644 | raise StopIteration |
| 645 | raise TimeoutError |
| 646 | finally: |
| 647 | self._cond.release() |
| 648 | |
| 649 | success, value = item |
| 650 | if success: |
| 651 | return value |
| 652 | raise value |
| 653 | |
| 654 | __next__ = next # XXX |
| 655 | |
| 656 | def _set(self, i, obj): |
| 657 | self._cond.acquire() |
| 658 | try: |
| 659 | if self._index == i: |
| 660 | self._items.append(obj) |
| 661 | self._index += 1 |
| 662 | while self._index in self._unsorted: |
| 663 | obj = self._unsorted.pop(self._index) |
| 664 | self._items.append(obj) |
| 665 | self._index += 1 |
| 666 | self._cond.notify() |
| 667 | else: |
| 668 | self._unsorted[i] = obj |
| 669 | |
| 670 | if self._index == self._length: |
| 671 | del self._cache[self._job] |
| 672 | finally: |
| 673 | self._cond.release() |
| 674 | |
| 675 | def _set_length(self, length): |
| 676 | self._cond.acquire() |
| 677 | try: |
| 678 | self._length = length |
| 679 | if self._index == self._length: |
| 680 | self._cond.notify() |
| 681 | del self._cache[self._job] |
| 682 | finally: |
| 683 | self._cond.release() |
| 684 | |
| 685 | # |
| 686 | # Class whose instances are returned by `Pool.imap_unordered()` |
| 687 | # |
| 688 | |
| 689 | class IMapUnorderedIterator(IMapIterator): |
| 690 | |
| 691 | def _set(self, i, obj): |
| 692 | self._cond.acquire() |
| 693 | try: |
| 694 | self._items.append(obj) |
| 695 | self._index += 1 |
| 696 | self._cond.notify() |
| 697 | if self._index == self._length: |
| 698 | del self._cache[self._job] |
| 699 | finally: |
| 700 | self._cond.release() |
| 701 | |
| 702 | # |
| 703 | # |
| 704 | # |
| 705 | |
| 706 | class ThreadPool(Pool): |
| 707 | |
| 708 | from .dummy import Process |
| 709 | |
| 710 | def __init__(self, processes=None, initializer=None, initargs=()): |
| 711 | Pool.__init__(self, processes, initializer, initargs) |
| 712 | |
| 713 | def _setup_queues(self): |
| 714 | self._inqueue = queue.Queue() |
| 715 | self._outqueue = queue.Queue() |
| 716 | self._quick_put = self._inqueue.put |
| 717 | self._quick_get = self._outqueue.get |
| 718 | |
| 719 | @staticmethod |
| 720 | def _help_stuff_finish(inqueue, task_handler, size): |
| 721 | # put sentinels at head of inqueue to make workers finish |
| 722 | inqueue.not_empty.acquire() |
| 723 | try: |
| 724 | inqueue.queue.clear() |
| 725 | inqueue.queue.extend([None] * size) |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 726 | inqueue.not_empty.notify_all() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 727 | finally: |
| 728 | inqueue.not_empty.release() |