Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 1 | # |
| 2 | # Module providing the `Pool` class for managing a process pool |
| 3 | # |
| 4 | # multiprocessing/pool.py |
| 5 | # |
R. David Murray | 3fc969a | 2010-12-14 01:38:16 +0000 | [diff] [blame] | 6 | # Copyright (c) 2006-2008, R Oudkerk |
Richard Oudkerk | 3e268aa | 2012-04-30 12:13:55 +0100 | [diff] [blame] | 7 | # Licensed to PSF under a Contributor Agreement. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 8 | # |
| 9 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 10 | __all__ = ['Pool', 'ThreadPool'] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 11 | |
| 12 | # |
| 13 | # Imports |
| 14 | # |
| 15 | |
| 16 | import threading |
| 17 | import queue |
| 18 | import itertools |
| 19 | import collections |
Charles-François Natali | 37cfb0a | 2013-06-28 19:25:45 +0200 | [diff] [blame] | 20 | import os |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 21 | import time |
Richard Oudkerk | 8575783 | 2013-05-06 11:38:25 +0100 | [diff] [blame] | 22 | import traceback |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 23 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 24 | # If threading is available then ThreadPool should be provided. Therefore |
| 25 | # we avoid top-level imports which are liable to fail on some systems. |
| 26 | from . import util |
Victor Stinner | 7fa767e | 2014-03-20 09:16:38 +0100 | [diff] [blame] | 27 | from . import get_context, TimeoutError |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 28 | |
| 29 | # |
| 30 | # Constants representing the state of a pool |
| 31 | # |
| 32 | |
| 33 | RUN = 0 |
| 34 | CLOSE = 1 |
| 35 | TERMINATE = 2 |
| 36 | |
| 37 | # |
| 38 | # Miscellaneous |
| 39 | # |
| 40 | |
| 41 | job_counter = itertools.count() |
| 42 | |
| 43 | def mapstar(args): |
| 44 | return list(map(*args)) |
| 45 | |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 46 | def starmapstar(args): |
| 47 | return list(itertools.starmap(args[0], args[1])) |
| 48 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 49 | # |
Richard Oudkerk | 8575783 | 2013-05-06 11:38:25 +0100 | [diff] [blame] | 50 | # Hack to embed stringification of remote traceback in local traceback |
| 51 | # |
| 52 | |
| 53 | class RemoteTraceback(Exception): |
| 54 | def __init__(self, tb): |
| 55 | self.tb = tb |
| 56 | def __str__(self): |
| 57 | return self.tb |
| 58 | |
| 59 | class ExceptionWithTraceback: |
| 60 | def __init__(self, exc, tb): |
| 61 | tb = traceback.format_exception(type(exc), exc, tb) |
| 62 | tb = ''.join(tb) |
| 63 | self.exc = exc |
| 64 | self.tb = '\n"""\n%s"""' % tb |
| 65 | def __reduce__(self): |
| 66 | return rebuild_exc, (self.exc, self.tb) |
| 67 | |
| 68 | def rebuild_exc(exc, tb): |
| 69 | exc.__cause__ = RemoteTraceback(tb) |
| 70 | return exc |
| 71 | |
| 72 | # |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 73 | # Code run by worker processes |
| 74 | # |
| 75 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 76 | class MaybeEncodingError(Exception): |
| 77 | """Wraps possible unpickleable errors, so they can be |
| 78 | safely sent through the socket.""" |
| 79 | |
| 80 | def __init__(self, exc, value): |
| 81 | self.exc = repr(exc) |
| 82 | self.value = repr(value) |
| 83 | super(MaybeEncodingError, self).__init__(self.exc, self.value) |
| 84 | |
| 85 | def __str__(self): |
| 86 | return "Error sending result: '%s'. Reason: '%s'" % (self.value, |
| 87 | self.exc) |
| 88 | |
| 89 | def __repr__(self): |
Serhiy Storchaka | 465e60e | 2014-07-25 23:36:00 +0300 | [diff] [blame] | 90 | return "<%s: %s>" % (self.__class__.__name__, self) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 91 | |
| 92 | |
Richard Oudkerk | 80a5be1 | 2014-03-23 12:30:54 +0000 | [diff] [blame] | 93 | def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, |
| 94 | wrap_exception=False): |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 95 | assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 96 | put = outqueue.put |
| 97 | get = inqueue.get |
| 98 | if hasattr(inqueue, '_writer'): |
| 99 | inqueue._writer.close() |
| 100 | outqueue._reader.close() |
| 101 | |
| 102 | if initializer is not None: |
| 103 | initializer(*initargs) |
| 104 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 105 | completed = 0 |
| 106 | while maxtasks is None or (maxtasks and completed < maxtasks): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 107 | try: |
| 108 | task = get() |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 109 | except (EOFError, OSError): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 110 | util.debug('worker got EOFError or OSError -- exiting') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 111 | break |
| 112 | |
| 113 | if task is None: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 114 | util.debug('worker got sentinel -- exiting') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 115 | break |
| 116 | |
| 117 | job, i, func, args, kwds = task |
| 118 | try: |
| 119 | result = (True, func(*args, **kwds)) |
| 120 | except Exception as e: |
Richard Oudkerk | 80a5be1 | 2014-03-23 12:30:54 +0000 | [diff] [blame] | 121 | if wrap_exception: |
| 122 | e = ExceptionWithTraceback(e, e.__traceback__) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 123 | result = (False, e) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 124 | try: |
| 125 | put((job, i, result)) |
| 126 | except Exception as e: |
| 127 | wrapped = MaybeEncodingError(e, result[1]) |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 128 | util.debug("Possible encoding error while sending result: %s" % ( |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 129 | wrapped)) |
| 130 | put((job, i, (False, wrapped))) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 131 | completed += 1 |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 132 | util.debug('worker exiting after %d tasks' % completed) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 133 | |
| 134 | # |
| 135 | # Class representing a process pool |
| 136 | # |
| 137 | |
| 138 | class Pool(object): |
| 139 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 140 | Class which supports an async version of applying functions to arguments. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 141 | ''' |
Richard Oudkerk | 80a5be1 | 2014-03-23 12:30:54 +0000 | [diff] [blame] | 142 | _wrap_exception = True |
| 143 | |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 144 | def Process(self, *args, **kwds): |
| 145 | return self._ctx.Process(*args, **kwds) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 146 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 147 | def __init__(self, processes=None, initializer=None, initargs=(), |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 148 | maxtasksperchild=None, context=None): |
| 149 | self._ctx = context or get_context() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 150 | self._setup_queues() |
| 151 | self._taskqueue = queue.Queue() |
| 152 | self._cache = {} |
| 153 | self._state = RUN |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 154 | self._maxtasksperchild = maxtasksperchild |
| 155 | self._initializer = initializer |
| 156 | self._initargs = initargs |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 157 | |
| 158 | if processes is None: |
Charles-François Natali | 37cfb0a | 2013-06-28 19:25:45 +0200 | [diff] [blame] | 159 | processes = os.cpu_count() or 1 |
Victor Stinner | 2fae27b | 2011-06-20 17:53:35 +0200 | [diff] [blame] | 160 | if processes < 1: |
| 161 | raise ValueError("Number of processes must be at least 1") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 162 | |
Florent Xicluna | 5d1155c | 2011-10-28 14:45:05 +0200 | [diff] [blame] | 163 | if initializer is not None and not callable(initializer): |
Benjamin Peterson | f47ed4a | 2009-04-11 20:45:40 +0000 | [diff] [blame] | 164 | raise TypeError('initializer must be a callable') |
| 165 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 166 | self._processes = processes |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 167 | self._pool = [] |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 168 | self._repopulate_pool() |
| 169 | |
| 170 | self._worker_handler = threading.Thread( |
| 171 | target=Pool._handle_workers, |
| 172 | args=(self, ) |
| 173 | ) |
| 174 | self._worker_handler.daemon = True |
| 175 | self._worker_handler._state = RUN |
| 176 | self._worker_handler.start() |
| 177 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 178 | |
| 179 | self._task_handler = threading.Thread( |
| 180 | target=Pool._handle_tasks, |
Richard Oudkerk | e90cedb | 2013-10-28 23:11:58 +0000 | [diff] [blame] | 181 | args=(self._taskqueue, self._quick_put, self._outqueue, |
| 182 | self._pool, self._cache) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 183 | ) |
Benjamin Peterson | fae4c62 | 2008-08-18 18:40:08 +0000 | [diff] [blame] | 184 | self._task_handler.daemon = True |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 185 | self._task_handler._state = RUN |
| 186 | self._task_handler.start() |
| 187 | |
| 188 | self._result_handler = threading.Thread( |
| 189 | target=Pool._handle_results, |
| 190 | args=(self._outqueue, self._quick_get, self._cache) |
| 191 | ) |
Benjamin Peterson | fae4c62 | 2008-08-18 18:40:08 +0000 | [diff] [blame] | 192 | self._result_handler.daemon = True |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 193 | self._result_handler._state = RUN |
| 194 | self._result_handler.start() |
| 195 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 196 | self._terminate = util.Finalize( |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 197 | self, self._terminate_pool, |
| 198 | args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 199 | self._worker_handler, self._task_handler, |
| 200 | self._result_handler, self._cache), |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 201 | exitpriority=15 |
| 202 | ) |
| 203 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 204 | def _join_exited_workers(self): |
| 205 | """Cleanup after any worker processes which have exited due to reaching |
| 206 | their specified lifetime. Returns True if any workers were cleaned up. |
| 207 | """ |
| 208 | cleaned = False |
| 209 | for i in reversed(range(len(self._pool))): |
| 210 | worker = self._pool[i] |
| 211 | if worker.exitcode is not None: |
| 212 | # worker exited |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 213 | util.debug('cleaning up worker %d' % i) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 214 | worker.join() |
| 215 | cleaned = True |
| 216 | del self._pool[i] |
| 217 | return cleaned |
| 218 | |
| 219 | def _repopulate_pool(self): |
| 220 | """Bring the number of pool processes up to the specified number, |
| 221 | for use after reaping workers which have exited. |
| 222 | """ |
| 223 | for i in range(self._processes - len(self._pool)): |
| 224 | w = self.Process(target=worker, |
| 225 | args=(self._inqueue, self._outqueue, |
| 226 | self._initializer, |
Richard Oudkerk | 80a5be1 | 2014-03-23 12:30:54 +0000 | [diff] [blame] | 227 | self._initargs, self._maxtasksperchild, |
| 228 | self._wrap_exception) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 229 | ) |
| 230 | self._pool.append(w) |
| 231 | w.name = w.name.replace('Process', 'PoolWorker') |
| 232 | w.daemon = True |
| 233 | w.start() |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 234 | util.debug('added worker') |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 235 | |
| 236 | def _maintain_pool(self): |
| 237 | """Clean up any exited workers and start replacements for them. |
| 238 | """ |
| 239 | if self._join_exited_workers(): |
| 240 | self._repopulate_pool() |
| 241 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 242 | def _setup_queues(self): |
Richard Oudkerk | b1694cf | 2013-10-16 16:41:56 +0100 | [diff] [blame] | 243 | self._inqueue = self._ctx.SimpleQueue() |
| 244 | self._outqueue = self._ctx.SimpleQueue() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 245 | self._quick_put = self._inqueue._writer.send |
| 246 | self._quick_get = self._outqueue._reader.recv |
| 247 | |
| 248 | def apply(self, func, args=(), kwds={}): |
| 249 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 250 | Equivalent of `func(*args, **kwds)`. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 251 | ''' |
| 252 | assert self._state == RUN |
| 253 | return self.apply_async(func, args, kwds).get() |
| 254 | |
| 255 | def map(self, func, iterable, chunksize=None): |
| 256 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 257 | Apply `func` to each element in `iterable`, collecting the results |
| 258 | in a list that is returned. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 259 | ''' |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 260 | return self._map_async(func, iterable, mapstar, chunksize).get() |
| 261 | |
| 262 | def starmap(self, func, iterable, chunksize=None): |
| 263 | ''' |
| 264 | Like `map()` method but the elements of the `iterable` are expected to |
| 265 | be iterables as well and will be unpacked as arguments. Hence |
| 266 | `func` and (a, b) becomes func(a, b). |
| 267 | ''' |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 268 | return self._map_async(func, iterable, starmapstar, chunksize).get() |
| 269 | |
| 270 | def starmap_async(self, func, iterable, chunksize=None, callback=None, |
| 271 | error_callback=None): |
| 272 | ''' |
| 273 | Asynchronous version of `starmap()` method. |
| 274 | ''' |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 275 | return self._map_async(func, iterable, starmapstar, chunksize, |
| 276 | callback, error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 277 | |
| 278 | def imap(self, func, iterable, chunksize=1): |
| 279 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 280 | Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 281 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 282 | if self._state != RUN: |
| 283 | raise ValueError("Pool not running") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 284 | if chunksize == 1: |
| 285 | result = IMapIterator(self._cache) |
| 286 | self._taskqueue.put((((result._job, i, func, (x,), {}) |
| 287 | for i, x in enumerate(iterable)), result._set_length)) |
| 288 | return result |
| 289 | else: |
| 290 | assert chunksize > 1 |
| 291 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
| 292 | result = IMapIterator(self._cache) |
| 293 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 294 | for i, x in enumerate(task_batches)), result._set_length)) |
| 295 | return (item for chunk in result for item in chunk) |
| 296 | |
| 297 | def imap_unordered(self, func, iterable, chunksize=1): |
| 298 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 299 | Like `imap()` method but ordering of results is arbitrary. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 300 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 301 | if self._state != RUN: |
| 302 | raise ValueError("Pool not running") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 303 | if chunksize == 1: |
| 304 | result = IMapUnorderedIterator(self._cache) |
| 305 | self._taskqueue.put((((result._job, i, func, (x,), {}) |
| 306 | for i, x in enumerate(iterable)), result._set_length)) |
| 307 | return result |
| 308 | else: |
| 309 | assert chunksize > 1 |
| 310 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
| 311 | result = IMapUnorderedIterator(self._cache) |
| 312 | self._taskqueue.put((((result._job, i, mapstar, (x,), {}) |
| 313 | for i, x in enumerate(task_batches)), result._set_length)) |
| 314 | return (item for chunk in result for item in chunk) |
| 315 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 316 | def apply_async(self, func, args=(), kwds={}, callback=None, |
| 317 | error_callback=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 318 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 319 | Asynchronous version of `apply()` method. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 320 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 321 | if self._state != RUN: |
| 322 | raise ValueError("Pool not running") |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 323 | result = ApplyResult(self._cache, callback, error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 324 | self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) |
| 325 | return result |
| 326 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 327 | def map_async(self, func, iterable, chunksize=None, callback=None, |
| 328 | error_callback=None): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 329 | ''' |
Georg Brandl | 9290503 | 2008-11-22 08:51:39 +0000 | [diff] [blame] | 330 | Asynchronous version of `map()` method. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 331 | ''' |
Hynek Schlawack | 254af26 | 2012-10-27 12:53:02 +0200 | [diff] [blame] | 332 | return self._map_async(func, iterable, mapstar, chunksize, callback, |
| 333 | error_callback) |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 334 | |
| 335 | def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, |
| 336 | error_callback=None): |
| 337 | ''' |
| 338 | Helper function to implement map, starmap and their async counterparts. |
| 339 | ''' |
Benjamin Peterson | 3095f47 | 2012-09-25 12:45:42 -0400 | [diff] [blame] | 340 | if self._state != RUN: |
| 341 | raise ValueError("Pool not running") |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 342 | if not hasattr(iterable, '__len__'): |
| 343 | iterable = list(iterable) |
| 344 | |
| 345 | if chunksize is None: |
| 346 | chunksize, extra = divmod(len(iterable), len(self._pool) * 4) |
| 347 | if extra: |
| 348 | chunksize += 1 |
Alexandre Vassalotti | e52e378 | 2009-07-17 09:18:18 +0000 | [diff] [blame] | 349 | if len(iterable) == 0: |
| 350 | chunksize = 0 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 351 | |
| 352 | task_batches = Pool._get_tasks(func, iterable, chunksize) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 353 | result = MapResult(self._cache, chunksize, len(iterable), callback, |
| 354 | error_callback=error_callback) |
Antoine Pitrou | de911b2 | 2011-12-21 11:03:24 +0100 | [diff] [blame] | 355 | self._taskqueue.put((((result._job, i, mapper, (x,), {}) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 356 | for i, x in enumerate(task_batches)), None)) |
| 357 | return result |
| 358 | |
| 359 | @staticmethod |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 360 | def _handle_workers(pool): |
Charles-François Natali | f8859e1 | 2011-10-24 18:45:29 +0200 | [diff] [blame] | 361 | thread = threading.current_thread() |
| 362 | |
| 363 | # Keep maintaining workers until the cache gets drained, unless the pool |
| 364 | # is terminated. |
| 365 | while thread._state == RUN or (pool._cache and thread._state != TERMINATE): |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 366 | pool._maintain_pool() |
| 367 | time.sleep(0.1) |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 368 | # send sentinel to stop workers |
| 369 | pool._taskqueue.put(None) |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 370 | util.debug('worker handler exiting') |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 371 | |
| 372 | @staticmethod |
Richard Oudkerk | e90cedb | 2013-10-28 23:11:58 +0000 | [diff] [blame] | 373 | def _handle_tasks(taskqueue, put, outqueue, pool, cache): |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 374 | thread = threading.current_thread() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 375 | |
| 376 | for taskseq, set_length in iter(taskqueue.get, None): |
Serhiy Storchaka | 79fbeee | 2015-03-13 08:25:26 +0200 | [diff] [blame] | 377 | task = None |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 378 | i = -1 |
Serhiy Storchaka | 79fbeee | 2015-03-13 08:25:26 +0200 | [diff] [blame] | 379 | try: |
| 380 | for i, task in enumerate(taskseq): |
| 381 | if thread._state: |
| 382 | util.debug('task handler found thread._state != RUN') |
| 383 | break |
Richard Oudkerk | e90cedb | 2013-10-28 23:11:58 +0000 | [diff] [blame] | 384 | try: |
Serhiy Storchaka | 79fbeee | 2015-03-13 08:25:26 +0200 | [diff] [blame] | 385 | put(task) |
| 386 | except Exception as e: |
| 387 | job, ind = task[:2] |
| 388 | try: |
| 389 | cache[job]._set(ind, (False, e)) |
| 390 | except KeyError: |
| 391 | pass |
| 392 | else: |
| 393 | if set_length: |
| 394 | util.debug('doing set_length()') |
| 395 | set_length(i+1) |
| 396 | continue |
| 397 | break |
| 398 | except Exception as ex: |
| 399 | job, ind = task[:2] if task else (0, 0) |
| 400 | if job in cache: |
| 401 | cache[job]._set(ind + 1, (False, ex)) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 402 | if set_length: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 403 | util.debug('doing set_length()') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 404 | set_length(i+1) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 405 | else: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 406 | util.debug('task handler got sentinel') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 407 | |
| 408 | |
| 409 | try: |
| 410 | # tell result handler to finish when cache is empty |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 411 | util.debug('task handler sending sentinel to result handler') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 412 | outqueue.put(None) |
| 413 | |
| 414 | # tell workers there is no more work |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 415 | util.debug('task handler sending sentinel to workers') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 416 | for p in pool: |
| 417 | put(None) |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 418 | except OSError: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 419 | util.debug('task handler got OSError when sending sentinels') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 420 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 421 | util.debug('task handler exiting') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 422 | |
| 423 | @staticmethod |
| 424 | def _handle_results(outqueue, get, cache): |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 425 | thread = threading.current_thread() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 426 | |
| 427 | while 1: |
| 428 | try: |
| 429 | task = get() |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 430 | except (OSError, EOFError): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 431 | util.debug('result handler got EOFError/OSError -- exiting') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 432 | return |
| 433 | |
| 434 | if thread._state: |
| 435 | assert thread._state == TERMINATE |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 436 | util.debug('result handler found thread._state=TERMINATE') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 437 | break |
| 438 | |
| 439 | if task is None: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 440 | util.debug('result handler got sentinel') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 441 | break |
| 442 | |
| 443 | job, i, obj = task |
| 444 | try: |
| 445 | cache[job]._set(i, obj) |
| 446 | except KeyError: |
| 447 | pass |
| 448 | |
| 449 | while cache and thread._state != TERMINATE: |
| 450 | try: |
| 451 | task = get() |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 452 | except (OSError, EOFError): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 453 | util.debug('result handler got EOFError/OSError -- exiting') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 454 | return |
| 455 | |
| 456 | if task is None: |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 457 | util.debug('result handler ignoring extra sentinel') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 458 | continue |
| 459 | job, i, obj = task |
| 460 | try: |
| 461 | cache[job]._set(i, obj) |
| 462 | except KeyError: |
| 463 | pass |
| 464 | |
| 465 | if hasattr(outqueue, '_reader'): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 466 | util.debug('ensuring that outqueue is not full') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 467 | # If we don't make room available in outqueue then |
| 468 | # attempts to add the sentinel (None) to outqueue may |
| 469 | # block. There is guaranteed to be no more than 2 sentinels. |
| 470 | try: |
| 471 | for i in range(10): |
| 472 | if not outqueue._reader.poll(): |
| 473 | break |
| 474 | get() |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame] | 475 | except (OSError, EOFError): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 476 | pass |
| 477 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 478 | util.debug('result handler exiting: len(cache)=%s, thread._state=%s', |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 479 | len(cache), thread._state) |
| 480 | |
| 481 | @staticmethod |
| 482 | def _get_tasks(func, it, size): |
| 483 | it = iter(it) |
| 484 | while 1: |
| 485 | x = tuple(itertools.islice(it, size)) |
| 486 | if not x: |
| 487 | return |
| 488 | yield (func, x) |
| 489 | |
| 490 | def __reduce__(self): |
| 491 | raise NotImplementedError( |
| 492 | 'pool objects cannot be passed between processes or pickled' |
| 493 | ) |
| 494 | |
| 495 | def close(self): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 496 | util.debug('closing pool') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 497 | if self._state == RUN: |
| 498 | self._state = CLOSE |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 499 | self._worker_handler._state = CLOSE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 500 | |
| 501 | def terminate(self): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 502 | util.debug('terminating pool') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 503 | self._state = TERMINATE |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 504 | self._worker_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 505 | self._terminate() |
| 506 | |
| 507 | def join(self): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 508 | util.debug('joining pool') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 509 | assert self._state in (CLOSE, TERMINATE) |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 510 | self._worker_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 511 | self._task_handler.join() |
| 512 | self._result_handler.join() |
| 513 | for p in self._pool: |
| 514 | p.join() |
| 515 | |
| 516 | @staticmethod |
| 517 | def _help_stuff_finish(inqueue, task_handler, size): |
| 518 | # task_handler may be blocked trying to put items on inqueue |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 519 | util.debug('removing tasks from inqueue until task handler finished') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 520 | inqueue._rlock.acquire() |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 521 | while task_handler.is_alive() and inqueue._reader.poll(): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 522 | inqueue._reader.recv() |
| 523 | time.sleep(0) |
| 524 | |
| 525 | @classmethod |
| 526 | def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 527 | worker_handler, task_handler, result_handler, cache): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 528 | # this is guaranteed to only be called once |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 529 | util.debug('finalizing pool') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 530 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 531 | worker_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 532 | task_handler._state = TERMINATE |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 533 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 534 | util.debug('helping task handler/workers to finish') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 535 | cls._help_stuff_finish(inqueue, task_handler, len(pool)) |
| 536 | |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 537 | assert result_handler.is_alive() or len(cache) == 0 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 538 | |
| 539 | result_handler._state = TERMINATE |
| 540 | outqueue.put(None) # sentinel |
| 541 | |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 542 | # We must wait for the worker handler to exit before terminating |
| 543 | # workers because we don't want workers to be restarted behind our back. |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 544 | util.debug('joining worker handler') |
Richard Oudkerk | f29ec4b | 2012-06-18 15:54:57 +0100 | [diff] [blame] | 545 | if threading.current_thread() is not worker_handler: |
| 546 | worker_handler.join() |
Antoine Pitrou | 81dee6b | 2011-04-11 00:18:59 +0200 | [diff] [blame] | 547 | |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 548 | # Terminate workers which haven't already finished. |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 549 | if pool and hasattr(pool[0], 'terminate'): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 550 | util.debug('terminating workers') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 551 | for p in pool: |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 552 | if p.exitcode is None: |
| 553 | p.terminate() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 554 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 555 | util.debug('joining task handler') |
Richard Oudkerk | f29ec4b | 2012-06-18 15:54:57 +0100 | [diff] [blame] | 556 | if threading.current_thread() is not task_handler: |
| 557 | task_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 558 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 559 | util.debug('joining result handler') |
Richard Oudkerk | f29ec4b | 2012-06-18 15:54:57 +0100 | [diff] [blame] | 560 | if threading.current_thread() is not result_handler: |
| 561 | result_handler.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 562 | |
| 563 | if pool and hasattr(pool[0], 'terminate'): |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 564 | util.debug('joining pool workers') |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 565 | for p in pool: |
Florent Xicluna | 998171f | 2010-03-08 13:32:17 +0000 | [diff] [blame] | 566 | if p.is_alive(): |
Jesse Noller | 1f0b658 | 2010-01-27 03:36:01 +0000 | [diff] [blame] | 567 | # worker has not yet exited |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 568 | util.debug('cleaning up worker %d' % p.pid) |
Florent Xicluna | 998171f | 2010-03-08 13:32:17 +0000 | [diff] [blame] | 569 | p.join() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 570 | |
Richard Oudkerk | d69cfe8 | 2012-06-18 17:47:52 +0100 | [diff] [blame] | 571 | def __enter__(self): |
| 572 | return self |
| 573 | |
| 574 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 575 | self.terminate() |
| 576 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 577 | # |
| 578 | # Class whose instances are returned by `Pool.apply_async()` |
| 579 | # |
| 580 | |
| 581 | class ApplyResult(object): |
| 582 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 583 | def __init__(self, cache, callback, error_callback): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 584 | self._event = threading.Event() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 585 | self._job = next(job_counter) |
| 586 | self._cache = cache |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 587 | self._callback = callback |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 588 | self._error_callback = error_callback |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 589 | cache[self._job] = self |
| 590 | |
| 591 | def ready(self): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 592 | return self._event.is_set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 593 | |
| 594 | def successful(self): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 595 | assert self.ready() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 596 | return self._success |
| 597 | |
| 598 | def wait(self, timeout=None): |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 599 | self._event.wait(timeout) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 600 | |
| 601 | def get(self, timeout=None): |
| 602 | self.wait(timeout) |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 603 | if not self.ready(): |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 604 | raise TimeoutError |
| 605 | if self._success: |
| 606 | return self._value |
| 607 | else: |
| 608 | raise self._value |
| 609 | |
| 610 | def _set(self, i, obj): |
| 611 | self._success, self._value = obj |
| 612 | if self._callback and self._success: |
| 613 | self._callback(self._value) |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 614 | if self._error_callback and not self._success: |
| 615 | self._error_callback(self._value) |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 616 | self._event.set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 617 | del self._cache[self._job] |
| 618 | |
Richard Oudkerk | def51ca | 2013-05-06 12:10:04 +0100 | [diff] [blame] | 619 | AsyncResult = ApplyResult # create alias -- see #17805 |
| 620 | |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 621 | # |
| 622 | # Class whose instances are returned by `Pool.map_async()` |
| 623 | # |
| 624 | |
| 625 | class MapResult(ApplyResult): |
| 626 | |
Ask Solem | 2afcbf2 | 2010-11-09 20:55:52 +0000 | [diff] [blame] | 627 | def __init__(self, cache, chunksize, length, callback, error_callback): |
| 628 | ApplyResult.__init__(self, cache, callback, |
| 629 | error_callback=error_callback) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 630 | self._success = True |
| 631 | self._value = [None] * length |
| 632 | self._chunksize = chunksize |
| 633 | if chunksize <= 0: |
| 634 | self._number_left = 0 |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 635 | self._event.set() |
Richard Oudkerk | e41682b | 2012-06-06 19:04:57 +0100 | [diff] [blame] | 636 | del cache[self._job] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 637 | else: |
| 638 | self._number_left = length//chunksize + bool(length % chunksize) |
| 639 | |
| 640 | def _set(self, i, success_result): |
Charles-François Natali | 78f55ff | 2016-02-10 22:58:18 +0000 | [diff] [blame] | 641 | self._number_left -= 1 |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 642 | success, result = success_result |
Charles-François Natali | 78f55ff | 2016-02-10 22:58:18 +0000 | [diff] [blame] | 643 | if success and self._success: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 644 | self._value[i*self._chunksize:(i+1)*self._chunksize] = result |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 645 | if self._number_left == 0: |
| 646 | if self._callback: |
| 647 | self._callback(self._value) |
| 648 | del self._cache[self._job] |
Richard Oudkerk | 692130a | 2012-05-25 13:26:53 +0100 | [diff] [blame] | 649 | self._event.set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 650 | else: |
Charles-François Natali | 78f55ff | 2016-02-10 22:58:18 +0000 | [diff] [blame] | 651 | if not success and self._success: |
| 652 | # only store first exception |
| 653 | self._success = False |
| 654 | self._value = result |
| 655 | if self._number_left == 0: |
| 656 | # only consider the result ready once all jobs are done |
| 657 | if self._error_callback: |
| 658 | self._error_callback(self._value) |
| 659 | del self._cache[self._job] |
| 660 | self._event.set() |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 661 | |
| 662 | # |
| 663 | # Class whose instances are returned by `Pool.imap()` |
| 664 | # |
| 665 | |
| 666 | class IMapIterator(object): |
| 667 | |
| 668 | def __init__(self, cache): |
| 669 | self._cond = threading.Condition(threading.Lock()) |
| 670 | self._job = next(job_counter) |
| 671 | self._cache = cache |
| 672 | self._items = collections.deque() |
| 673 | self._index = 0 |
| 674 | self._length = None |
| 675 | self._unsorted = {} |
| 676 | cache[self._job] = self |
| 677 | |
| 678 | def __iter__(self): |
| 679 | return self |
| 680 | |
| 681 | def next(self, timeout=None): |
Charles-François Natali | a924fc7 | 2014-05-25 14:12:12 +0100 | [diff] [blame] | 682 | with self._cond: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 683 | try: |
| 684 | item = self._items.popleft() |
| 685 | except IndexError: |
| 686 | if self._index == self._length: |
| 687 | raise StopIteration |
| 688 | self._cond.wait(timeout) |
| 689 | try: |
| 690 | item = self._items.popleft() |
| 691 | except IndexError: |
| 692 | if self._index == self._length: |
| 693 | raise StopIteration |
| 694 | raise TimeoutError |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 695 | |
| 696 | success, value = item |
| 697 | if success: |
| 698 | return value |
| 699 | raise value |
| 700 | |
| 701 | __next__ = next # XXX |
| 702 | |
| 703 | def _set(self, i, obj): |
Charles-François Natali | a924fc7 | 2014-05-25 14:12:12 +0100 | [diff] [blame] | 704 | with self._cond: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 705 | if self._index == i: |
| 706 | self._items.append(obj) |
| 707 | self._index += 1 |
| 708 | while self._index in self._unsorted: |
| 709 | obj = self._unsorted.pop(self._index) |
| 710 | self._items.append(obj) |
| 711 | self._index += 1 |
| 712 | self._cond.notify() |
| 713 | else: |
| 714 | self._unsorted[i] = obj |
| 715 | |
| 716 | if self._index == self._length: |
| 717 | del self._cache[self._job] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 718 | |
| 719 | def _set_length(self, length): |
Charles-François Natali | a924fc7 | 2014-05-25 14:12:12 +0100 | [diff] [blame] | 720 | with self._cond: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 721 | self._length = length |
| 722 | if self._index == self._length: |
| 723 | self._cond.notify() |
| 724 | del self._cache[self._job] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 725 | |
| 726 | # |
| 727 | # Class whose instances are returned by `Pool.imap_unordered()` |
| 728 | # |
| 729 | |
| 730 | class IMapUnorderedIterator(IMapIterator): |
| 731 | |
| 732 | def _set(self, i, obj): |
Charles-François Natali | a924fc7 | 2014-05-25 14:12:12 +0100 | [diff] [blame] | 733 | with self._cond: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 734 | self._items.append(obj) |
| 735 | self._index += 1 |
| 736 | self._cond.notify() |
| 737 | if self._index == self._length: |
| 738 | del self._cache[self._job] |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 739 | |
| 740 | # |
| 741 | # |
| 742 | # |
| 743 | |
| 744 | class ThreadPool(Pool): |
Richard Oudkerk | 80a5be1 | 2014-03-23 12:30:54 +0000 | [diff] [blame] | 745 | _wrap_exception = False |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 746 | |
Richard Oudkerk | 84ed9a6 | 2013-08-14 15:35:41 +0100 | [diff] [blame] | 747 | @staticmethod |
| 748 | def Process(*args, **kwds): |
| 749 | from .dummy import Process |
| 750 | return Process(*args, **kwds) |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 751 | |
| 752 | def __init__(self, processes=None, initializer=None, initargs=()): |
| 753 | Pool.__init__(self, processes, initializer, initargs) |
| 754 | |
| 755 | def _setup_queues(self): |
| 756 | self._inqueue = queue.Queue() |
| 757 | self._outqueue = queue.Queue() |
| 758 | self._quick_put = self._inqueue.put |
| 759 | self._quick_get = self._outqueue.get |
| 760 | |
| 761 | @staticmethod |
| 762 | def _help_stuff_finish(inqueue, task_handler, size): |
| 763 | # put sentinels at head of inqueue to make workers finish |
Charles-François Natali | a924fc7 | 2014-05-25 14:12:12 +0100 | [diff] [blame] | 764 | with inqueue.not_empty: |
Benjamin Peterson | e711caf | 2008-06-11 16:44:04 +0000 | [diff] [blame] | 765 | inqueue.queue.clear() |
| 766 | inqueue.queue.extend([None] * size) |
Benjamin Peterson | 672b803 | 2008-06-11 19:14:14 +0000 | [diff] [blame] | 767 | inqueue.not_empty.notify_all() |