Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 1 | # |
| 2 | # Module providing the `Pool` class for managing a process pool |
| 3 | # |
| 4 | # multiprocessing/pool.py |
| 5 | # |
R. David Murray | 3fc969a | 2010-12-14 01:38:16 +0000 | [diff] [blame] | 6 | # Copyright (c) 2006-2008, R Oudkerk |
| 7 | # All rights reserved. |
| 8 | # |
| 9 | # Redistribution and use in source and binary forms, with or without |
| 10 | # modification, are permitted provided that the following conditions |
| 11 | # are met: |
| 12 | # |
| 13 | # 1. Redistributions of source code must retain the above copyright |
| 14 | # notice, this list of conditions and the following disclaimer. |
| 15 | # 2. Redistributions in binary form must reproduce the above copyright |
| 16 | # notice, this list of conditions and the following disclaimer in the |
| 17 | # documentation and/or other materials provided with the distribution. |
| 18 | # 3. Neither the name of author nor the names of any contributors may be |
| 19 | # used to endorse or promote products derived from this software |
| 20 | # without specific prior written permission. |
| 21 | # |
| 22 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND |
| 23 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 24 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 25 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
| 26 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 27 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 28 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 29 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 30 | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 31 | # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 32 | # SUCH DAMAGE. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 33 | # |
| 34 | |
| 35 | __all__ = ['Pool'] |
| 36 | |
| 37 | # |
| 38 | # Imports |
| 39 | # |
| 40 | |
| 41 | import threading |
| 42 | import queue |
| 43 | import itertools |
| 44 | import collections |
| 45 | import time |
| 46 | |
| 47 | from multiprocessing import Process, cpu_count, TimeoutError |
| 48 | from multiprocessing.util import Finalize, debug |
| 49 | |
| 50 | # |
| 51 | # Constants representing the state of a pool |
| 52 | # |
| 53 | |
| 54 | RUN = 0 |
| 55 | CLOSE = 1 |
| 56 | TERMINATE = 2 |
| 57 | |
| 58 | # |
| 59 | # Miscellaneous |
| 60 | # |
| 61 | |
| 62 | job_counter = itertools.count() |
| 63 | |
| 64 | def mapstar(args): |
| 65 | return list(map(*args)) |
| 66 | |
| 67 | # |
| 68 | # Code run by worker processes |
| 69 | # |
| 70 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 71 | class MaybeEncodingError(Exception): |
| 72 | """Wraps possible unpickleable errors, so they can be |
| 73 | safely sent through the socket.""" |
| 74 | |
| 75 | def __init__(self, exc, value): |
| 76 | self.exc = repr(exc) |
| 77 | self.value = repr(value) |
| 78 | super(MaybeEncodingError, self).__init__(self.exc, self.value) |
| 79 | |
| 80 | def __str__(self): |
| 81 | return "Error sending result: '%s'. Reason: '%s'" % (self.value, |
| 82 | self.exc) |
| 83 | |
| 84 | def __repr__(self): |
| 85 | return "<MaybeEncodingError: %s>" % str(self) |
| 86 | |
| 87 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 88 | def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None): |
| 89 | assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 90 | put = outqueue.put |
| 91 | get = inqueue.get |
| 92 | if hasattr(inqueue, '_writer'): |
| 93 | inqueue._writer.close() |
| 94 | outqueue._reader.close() |
| 95 | |
| 96 | if initializer is not None: |
| 97 | initializer(*initargs) |
| 98 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 99 | completed = 0 |
| 100 | while maxtasks is None or (maxtasks and completed < maxtasks): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 101 | try: |
| 102 | task = get() |
| 103 | except (EOFError, IOError): |
| 104 | debug('worker got EOFError or IOError -- exiting') |
| 105 | break |
| 106 | |
| 107 | if task is None: |
| 108 | debug('worker got sentinel -- exiting') |
| 109 | break |
| 110 | |
| 111 | job, i, func, args, kwds = task |
| 112 | try: |
| 113 | result = (True, func(*args, **kwds)) |
| 114 | except Exception as e: |
| 115 | result = (False, e) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 116 | try: |
| 117 | put((job, i, result)) |
| 118 | except Exception as e: |
| 119 | wrapped = MaybeEncodingError(e, result[1]) |
| 120 | debug("Possible encoding error while sending result: %s" % ( |
| 121 | wrapped)) |
| 122 | put((job, i, (False, wrapped))) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 123 | completed += 1 |
| 124 | debug('worker exiting after %d tasks' % completed) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 125 | |
| 126 | # |
| 127 | # Class representing a process pool |
| 128 | # |
| 129 | |
| 130 | class Pool(object): |
| 131 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 132 | Class which supports an async version of applying functions to arguments. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 133 | ''' |
| 134 | Process = Process |
| 135 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 136 | def __init__(self, processes=None, initializer=None, initargs=(), |
| 137 | maxtasksperchild=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 138 | self._setup_queues() |
| 139 | self._taskqueue = queue.Queue() |
| 140 | self._cache = {} |
| 141 | self._state = RUN |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 142 | self._maxtasksperchild = maxtasksperchild |
| 143 | self._initializer = initializer |
| 144 | self._initargs = initargs |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 145 | |
| 146 | if processes is None: |
| 147 | try: |
| 148 | processes = cpu_count() |
| 149 | except NotImplementedError: |
| 150 | processes = 1 |
| 151 | |
Benjamin Peterson | f47ed4a | 2009-04-11 20:45:40 +0000 | [diff] [blame] | 152 | if initializer is not None and not hasattr(initializer, '__call__'): |
| 153 | raise TypeError('initializer must be a callable') |
| 154 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 155 | self._processes = processes |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 156 | self._pool = [] |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 157 | self._repopulate_pool() |
| 158 | |
| 159 | self._worker_handler = threading.Thread( |
| 160 | target=Pool._handle_workers, |
| 161 | args=(self, ) |
| 162 | ) |
| 163 | self._worker_handler.daemon = True |
| 164 | self._worker_handler._state = RUN |
| 165 | self._worker_handler.start() |
| 166 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 167 | |
| 168 | self._task_handler = threading.Thread( |
| 169 | target=Pool._handle_tasks, |
| 170 | args=(self._taskqueue, self._quick_put, self._outqueue, self._pool) |
| 171 | ) |
Benjamin Peterson | fae4c62 | 2008-08-18 18:40:08 +0000 | [diff] [blame] | 172 | self._task_handler.daemon = True |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 173 | self._task_handler._state = RUN |
| 174 | self._task_handler.start() |
| 175 | |
| 176 | self._result_handler = threading.Thread( |
| 177 | target=Pool._handle_results, |
| 178 | args=(self._outqueue, self._quick_get, self._cache) |
| 179 | ) |
Benjamin Peterson | fae4c62 | 2008-08-18 18:40:08 +0000 | [diff] [blame] | 180 | self._result_handler.daemon = True |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 181 | self._result_handler._state = RUN |
| 182 | self._result_handler.start() |
| 183 | |
| 184 | self._terminate = Finalize( |
| 185 | self, self._terminate_pool, |
| 186 | args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 187 | self._worker_handler, self._task_handler, |
| 188 | self._result_handler, self._cache), |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 189 | exitpriority=15 |
| 190 | ) |
| 191 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 192 | def _join_exited_workers(self): |
| 193 | """Cleanup after any worker processes which have exited due to reaching |
| 194 | their specified lifetime. Returns True if any workers were cleaned up. |
| 195 | """ |
| 196 | cleaned = False |
| 197 | for i in reversed(range(len(self._pool))): |
| 198 | worker = self._pool[i] |
| 199 | if worker.exitcode is not None: |
| 200 | # worker exited |
| 201 | debug('cleaning up worker %d' % i) |
| 202 | worker.join() |
| 203 | cleaned = True |
| 204 | del self._pool[i] |
| 205 | return cleaned |
| 206 | |
| 207 | def _repopulate_pool(self): |
| 208 | """Bring the number of pool processes up to the specified number, |
| 209 | for use after reaping workers which have exited. |
| 210 | """ |
| 211 | for i in range(self._processes - len(self._pool)): |
| 212 | w = self.Process(target=worker, |
| 213 | args=(self._inqueue, self._outqueue, |
| 214 | self._initializer, |
| 215 | self._initargs, self._maxtasksperchild) |
| 216 | ) |
| 217 | self._pool.append(w) |
| 218 | w.name = w.name.replace('Process', 'PoolWorker') |
| 219 | w.daemon = True |
| 220 | w.start() |
| 221 | debug('added worker') |
| 222 | |
| 223 | def _maintain_pool(self): |
| 224 | """Clean up any exited workers and start replacements for them. |
| 225 | """ |
| 226 | if self._join_exited_workers(): |
| 227 | self._repopulate_pool() |
| 228 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 229 | def _setup_queues(self): |
| 230 | from .queues import SimpleQueue |
| 231 | self._inqueue = SimpleQueue() |
| 232 | self._outqueue = SimpleQueue() |
| 233 | self._quick_put = self._inqueue._writer.send |
| 234 | self._quick_get = self._outqueue._reader.recv |
| 235 | |
| 236 | def apply(self, func, args=(), kwds={}): |
| 237 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 238 | Equivalent of `func(*args, **kwds)`. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 239 | ''' |
| 240 | assert self._state == RUN |
| 241 | return self.apply_async(func, args, kwds).get() |
| 242 | |
| 243 | def map(self, func, iterable, chunksize=None): |
| 244 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 245 | Apply `func` to each element in `iterable`, collecting the results |
| 246 | in a list that is returned. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 247 | ''' |
| 248 | assert self._state == RUN |
| 249 | return self.map_async(func, iterable, chunksize).get() |
| 250 | |
| 251 | def imap(self, func, iterable, chunksize=1): |
| 252 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 253 | Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 254 | ''' |
| 255 | assert self._state == RUN |
| 256 | if chunksize == 1: |
| 257 | result = IMapIterator(self._cache) |
| 258 | self._taskqueue.put((((result._job, i, func, (x,), {}) |
| 259 | for i, x in enumerate(iterable)), result._set_length)) |
| 260 | return result |
| 261 | else: |
| 262 | assert chunksize > 1 |
| 263 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
| 264 | result = IMapIterator(self._cache) |
| 265 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 266 | for i, x in enumerate(task_batches)), result._set_length)) |
| 267 | return (item for chunk in result for item in chunk) |
| 268 | |
| 269 | def imap_unordered(self, func, iterable, chunksize=1): |
| 270 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 271 | Like `imap()` method but ordering of results is arbitrary. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 272 | ''' |
| 273 | assert self._state == RUN |
| 274 | if chunksize == 1: |
| 275 | result = IMapUnorderedIterator(self._cache) |
| 276 | self._taskqueue.put((((result._job, i, func, (x,), {}) |
| 277 | for i, x in enumerate(iterable)), result._set_length)) |
| 278 | return result |
| 279 | else: |
| 280 | assert chunksize > 1 |
| 281 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
| 282 | result = IMapUnorderedIterator(self._cache) |
| 283 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 284 | for i, x in enumerate(task_batches)), result._set_length)) |
| 285 | return (item for chunk in result for item in chunk) |
| 286 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 287 | def apply_async(self, func, args=(), kwds={}, callback=None, |
| 288 | error_callback=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 289 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 290 | Asynchronous version of `apply()` method. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 291 | ''' |
| 292 | assert self._state == RUN |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 293 | result = ApplyResult(self._cache, callback, error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 294 | self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) |
| 295 | return result |
| 296 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 297 | def map_async(self, func, iterable, chunksize=None, callback=None, |
| 298 | error_callback=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 299 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 300 | Asynchronous version of `map()` method. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 301 | ''' |
| 302 | assert self._state == RUN |
| 303 | if not hasattr(iterable, '__len__'): |
| 304 | iterable = list(iterable) |
| 305 | |
| 306 | if chunksize is None: |
| 307 | chunksize, extra = divmod(len(iterable), len(self._pool) * 4) |
| 308 | if extra: |
| 309 | chunksize += 1 |
Alexandre Vassalotti | e52e378 | 2009-07-17 09:18:18 +0000 | [diff] [blame] | 310 | if len(iterable) == 0: |
| 311 | chunksize = 0 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 312 | |
| 313 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 314 | result = MapResult(self._cache, chunksize, len(iterable), callback, |
| 315 | error_callback=error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 316 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 317 | for i, x in enumerate(task_batches)), None)) |
| 318 | return result |
| 319 | |
| 320 | @staticmethod |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 321 | def _handle_workers(pool): |
| 322 | while pool._worker_handler._state == RUN and pool._state == RUN: |
| 323 | pool._maintain_pool() |
| 324 | time.sleep(0.1) |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 325 | # send sentinel to stop workers |
| 326 | pool._taskqueue.put(None) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 327 | debug('worker handler exiting') |
| 328 | |
| 329 | @staticmethod |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 330 | def _handle_tasks(taskqueue, put, outqueue, pool): |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 331 | thread = threading.current_thread() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 332 | |
| 333 | for taskseq, set_length in iter(taskqueue.get, None): |
| 334 | i = -1 |
| 335 | for i, task in enumerate(taskseq): |
| 336 | if thread._state: |
| 337 | debug('task handler found thread._state != RUN') |
| 338 | break |
| 339 | try: |
| 340 | put(task) |
| 341 | except IOError: |
| 342 | debug('could not put task on queue') |
| 343 | break |
| 344 | else: |
| 345 | if set_length: |
| 346 | debug('doing set_length()') |
| 347 | set_length(i+1) |
| 348 | continue |
| 349 | break |
| 350 | else: |
| 351 | debug('task handler got sentinel') |
| 352 | |
| 353 | |
| 354 | try: |
| 355 | # tell result handler to finish when cache is empty |
| 356 | debug('task handler sending sentinel to result handler') |
| 357 | outqueue.put(None) |
| 358 | |
| 359 | # tell workers there is no more work |
| 360 | debug('task handler sending sentinel to workers') |
| 361 | for p in pool: |
| 362 | put(None) |
| 363 | except IOError: |
| 364 | debug('task handler got IOError when sending sentinels') |
| 365 | |
| 366 | debug('task handler exiting') |
| 367 | |
| 368 | @staticmethod |
| 369 | def _handle_results(outqueue, get, cache): |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 370 | thread = threading.current_thread() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 371 | |
| 372 | while 1: |
| 373 | try: |
| 374 | task = get() |
| 375 | except (IOError, EOFError): |
| 376 | debug('result handler got EOFError/IOError -- exiting') |
| 377 | return |
| 378 | |
| 379 | if thread._state: |
| 380 | assert thread._state == TERMINATE |
| 381 | debug('result handler found thread._state=TERMINATE') |
| 382 | break |
| 383 | |
| 384 | if task is None: |
| 385 | debug('result handler got sentinel') |
| 386 | break |
| 387 | |
| 388 | job, i, obj = task |
| 389 | try: |
| 390 | cache[job]._set(i, obj) |
| 391 | except KeyError: |
| 392 | pass |
| 393 | |
| 394 | while cache and thread._state != TERMINATE: |
| 395 | try: |
| 396 | task = get() |
| 397 | except (IOError, EOFError): |
| 398 | debug('result handler got EOFError/IOError -- exiting') |
| 399 | return |
| 400 | |
| 401 | if task is None: |
| 402 | debug('result handler ignoring extra sentinel') |
| 403 | continue |
| 404 | job, i, obj = task |
| 405 | try: |
| 406 | cache[job]._set(i, obj) |
| 407 | except KeyError: |
| 408 | pass |
| 409 | |
| 410 | if hasattr(outqueue, '_reader'): |
| 411 | debug('ensuring that outqueue is not full') |
| 412 | # If we don't make room available in outqueue then |
| 413 | # attempts to add the sentinel (None) to outqueue may |
| 414 | # block. There is guaranteed to be no more than 2 sentinels. |
| 415 | try: |
| 416 | for i in range(10): |
| 417 | if not outqueue._reader.poll(): |
| 418 | break |
| 419 | get() |
| 420 | except (IOError, EOFError): |
| 421 | pass |
| 422 | |
| 423 | debug('result handler exiting: len(cache)=%s, thread._state=%s', |
| 424 | len(cache), thread._state) |
| 425 | |
| 426 | @staticmethod |
| 427 | def _get_tasks(func, it, size): |
| 428 | it = iter(it) |
| 429 | while 1: |
| 430 | x = tuple(itertools.islice(it, size)) |
| 431 | if not x: |
| 432 | return |
| 433 | yield (func, x) |
| 434 | |
| 435 | def __reduce__(self): |
| 436 | raise NotImplementedError( |
| 437 | 'pool objects cannot be passed between processes or pickled' |
| 438 | ) |
| 439 | |
| 440 | def close(self): |
| 441 | debug('closing pool') |
| 442 | if self._state == RUN: |
| 443 | self._state = CLOSE |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 444 | self._worker_handler._state = CLOSE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 445 | |
| 446 | def terminate(self): |
| 447 | debug('terminating pool') |
| 448 | self._state = TERMINATE |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 449 | self._worker_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 450 | self._terminate() |
| 451 | |
| 452 | def join(self): |
| 453 | debug('joining pool') |
| 454 | assert self._state in (CLOSE, TERMINATE) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 455 | self._worker_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 456 | self._task_handler.join() |
| 457 | self._result_handler.join() |
| 458 | for p in self._pool: |
| 459 | p.join() |
| 460 | |
| 461 | @staticmethod |
| 462 | def _help_stuff_finish(inqueue, task_handler, size): |
| 463 | # task_handler may be blocked trying to put items on inqueue |
| 464 | debug('removing tasks from inqueue until task handler finished') |
| 465 | inqueue._rlock.acquire() |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 466 | while task_handler.is_alive() and inqueue._reader.poll(): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 467 | inqueue._reader.recv() |
| 468 | time.sleep(0) |
| 469 | |
| 470 | @classmethod |
| 471 | def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 472 | worker_handler, task_handler, result_handler, cache): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 473 | # this is guaranteed to only be called once |
| 474 | debug('finalizing pool') |
| 475 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 476 | worker_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 477 | task_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 478 | |
| 479 | debug('helping task handler/workers to finish') |
| 480 | cls._help_stuff_finish(inqueue, task_handler, len(pool)) |
| 481 | |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 482 | assert result_handler.is_alive() or len(cache) == 0 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 483 | |
| 484 | result_handler._state = TERMINATE |
| 485 | outqueue.put(None) # sentinel |
| 486 | |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 487 | # We must wait for the worker handler to exit before terminating |
| 488 | # workers because we don't want workers to be restarted behind our back. |
| 489 | debug('joining worker handler') |
| 490 | worker_handler.join() |
| 491 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 492 | # Terminate workers which haven't already finished. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 493 | if pool and hasattr(pool[0], 'terminate'): |
| 494 | debug('terminating workers') |
| 495 | for p in pool: |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 496 | if p.exitcode is None: |
| 497 | p.terminate() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 498 | |
| 499 | debug('joining task handler') |
Antoine Pitrou | 7c3e577 | 2010-04-14 15:44:10 +0000 | [diff] [blame] | 500 | task_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 501 | |
| 502 | debug('joining result handler') |
Antoine Pitrou | bed9a5b | 2011-04-11 00:20:23 +0200 | [diff] [blame] | 503 | result_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 504 | |
| 505 | if pool and hasattr(pool[0], 'terminate'): |
| 506 | debug('joining pool workers') |
| 507 | for p in pool: |
Florent Xicluna | 998171f | 2010-03-08 13:32:17 +0000 | [diff] [blame] | 508 | if p.is_alive(): |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 509 | # worker has not yet exited |
Florent Xicluna | 998171f | 2010-03-08 13:32:17 +0000 | [diff] [blame] | 510 | debug('cleaning up worker %d' % p.pid) |
| 511 | p.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 512 | |
| 513 | # |
| 514 | # Class whose instances are returned by `Pool.apply_async()` |
| 515 | # |
| 516 | |
| 517 | class ApplyResult(object): |
| 518 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 519 | def __init__(self, cache, callback, error_callback): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 520 | self._cond = threading.Condition(threading.Lock()) |
| 521 | self._job = next(job_counter) |
| 522 | self._cache = cache |
| 523 | self._ready = False |
| 524 | self._callback = callback |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 525 | self._error_callback = error_callback |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 526 | cache[self._job] = self |
| 527 | |
| 528 | def ready(self): |
| 529 | return self._ready |
| 530 | |
| 531 | def successful(self): |
| 532 | assert self._ready |
| 533 | return self._success |
| 534 | |
| 535 | def wait(self, timeout=None): |
| 536 | self._cond.acquire() |
| 537 | try: |
| 538 | if not self._ready: |
| 539 | self._cond.wait(timeout) |
| 540 | finally: |
| 541 | self._cond.release() |
| 542 | |
| 543 | def get(self, timeout=None): |
| 544 | self.wait(timeout) |
| 545 | if not self._ready: |
| 546 | raise TimeoutError |
| 547 | if self._success: |
| 548 | return self._value |
| 549 | else: |
| 550 | raise self._value |
| 551 | |
| 552 | def _set(self, i, obj): |
| 553 | self._success, self._value = obj |
| 554 | if self._callback and self._success: |
| 555 | self._callback(self._value) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 556 | if self._error_callback and not self._success: |
| 557 | self._error_callback(self._value) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 558 | self._cond.acquire() |
| 559 | try: |
| 560 | self._ready = True |
| 561 | self._cond.notify() |
| 562 | finally: |
| 563 | self._cond.release() |
| 564 | del self._cache[self._job] |
| 565 | |
| 566 | # |
| 567 | # Class whose instances are returned by `Pool.map_async()` |
| 568 | # |
| 569 | |
| 570 | class MapResult(ApplyResult): |
| 571 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 572 | def __init__(self, cache, chunksize, length, callback, error_callback): |
| 573 | ApplyResult.__init__(self, cache, callback, |
| 574 | error_callback=error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 575 | self._success = True |
| 576 | self._value = [None] * length |
| 577 | self._chunksize = chunksize |
| 578 | if chunksize <= 0: |
| 579 | self._number_left = 0 |
| 580 | self._ready = True |
| 581 | else: |
| 582 | self._number_left = length//chunksize + bool(length % chunksize) |
| 583 | |
| 584 | def _set(self, i, success_result): |
| 585 | success, result = success_result |
| 586 | if success: |
| 587 | self._value[i*self._chunksize:(i+1)*self._chunksize] = result |
| 588 | self._number_left -= 1 |
| 589 | if self._number_left == 0: |
| 590 | if self._callback: |
| 591 | self._callback(self._value) |
| 592 | del self._cache[self._job] |
| 593 | self._cond.acquire() |
| 594 | try: |
| 595 | self._ready = True |
| 596 | self._cond.notify() |
| 597 | finally: |
| 598 | self._cond.release() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 599 | else: |
| 600 | self._success = False |
| 601 | self._value = result |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 602 | if self._error_callback: |
| 603 | self._error_callback(self._value) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 604 | del self._cache[self._job] |
| 605 | self._cond.acquire() |
| 606 | try: |
| 607 | self._ready = True |
| 608 | self._cond.notify() |
| 609 | finally: |
| 610 | self._cond.release() |
| 611 | |
| 612 | # |
| 613 | # Class whose instances are returned by `Pool.imap()` |
| 614 | # |
| 615 | |
| 616 | class IMapIterator(object): |
| 617 | |
| 618 | def __init__(self, cache): |
| 619 | self._cond = threading.Condition(threading.Lock()) |
| 620 | self._job = next(job_counter) |
| 621 | self._cache = cache |
| 622 | self._items = collections.deque() |
| 623 | self._index = 0 |
| 624 | self._length = None |
| 625 | self._unsorted = {} |
| 626 | cache[self._job] = self |
| 627 | |
| 628 | def __iter__(self): |
| 629 | return self |
| 630 | |
| 631 | def next(self, timeout=None): |
| 632 | self._cond.acquire() |
| 633 | try: |
| 634 | try: |
| 635 | item = self._items.popleft() |
| 636 | except IndexError: |
| 637 | if self._index == self._length: |
| 638 | raise StopIteration |
| 639 | self._cond.wait(timeout) |
| 640 | try: |
| 641 | item = self._items.popleft() |
| 642 | except IndexError: |
| 643 | if self._index == self._length: |
| 644 | raise StopIteration |
| 645 | raise TimeoutError |
| 646 | finally: |
| 647 | self._cond.release() |
| 648 | |
| 649 | success, value = item |
| 650 | if success: |
| 651 | return value |
| 652 | raise value |
| 653 | |
| 654 | __next__ = next # XXX |
| 655 | |
| 656 | def _set(self, i, obj): |
| 657 | self._cond.acquire() |
| 658 | try: |
| 659 | if self._index == i: |
| 660 | self._items.append(obj) |
| 661 | self._index += 1 |
| 662 | while self._index in self._unsorted: |
| 663 | obj = self._unsorted.pop(self._index) |
| 664 | self._items.append(obj) |
| 665 | self._index += 1 |
| 666 | self._cond.notify() |
| 667 | else: |
| 668 | self._unsorted[i] = obj |
| 669 | |
| 670 | if self._index == self._length: |
| 671 | del self._cache[self._job] |
| 672 | finally: |
| 673 | self._cond.release() |
| 674 | |
| 675 | def _set_length(self, length): |
| 676 | self._cond.acquire() |
| 677 | try: |
| 678 | self._length = length |
| 679 | if self._index == self._length: |
| 680 | self._cond.notify() |
| 681 | del self._cache[self._job] |
| 682 | finally: |
| 683 | self._cond.release() |
| 684 | |
| 685 | # |
| 686 | # Class whose instances are returned by `Pool.imap_unordered()` |
| 687 | # |
| 688 | |
| 689 | class IMapUnorderedIterator(IMapIterator): |
| 690 | |
| 691 | def _set(self, i, obj): |
| 692 | self._cond.acquire() |
| 693 | try: |
| 694 | self._items.append(obj) |
| 695 | self._index += 1 |
| 696 | self._cond.notify() |
| 697 | if self._index == self._length: |
| 698 | del self._cache[self._job] |
| 699 | finally: |
| 700 | self._cond.release() |
| 701 | |
| 702 | # |
| 703 | # |
| 704 | # |
| 705 | |
| 706 | class ThreadPool(Pool): |
| 707 | |
| 708 | from .dummy import Process |
| 709 | |
| 710 | def __init__(self, processes=None, initializer=None, initargs=()): |
| 711 | Pool.__init__(self, processes, initializer, initargs) |
| 712 | |
| 713 | def _setup_queues(self): |
| 714 | self._inqueue = queue.Queue() |
| 715 | self._outqueue = queue.Queue() |
| 716 | self._quick_put = self._inqueue.put |
| 717 | self._quick_get = self._outqueue.get |
| 718 | |
| 719 | @staticmethod |
| 720 | def _help_stuff_finish(inqueue, task_handler, size): |
| 721 | # put sentinels at head of inqueue to make workers finish |
| 722 | inqueue.not_empty.acquire() |
| 723 | try: |
| 724 | inqueue.queue.clear() |
| 725 | inqueue.queue.extend([None] * size) |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 726 | inqueue.not_empty.notify_all() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 727 | finally: |
| 728 | inqueue.not_empty.release() |