blob: 25075de5b40259407e8f9b5c390c1c2c3e5327aa [file] [log] [blame]
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001"""functools.py - Tools for working with functions and callable objects
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00002"""
3# Python module wrapper for _functools C module
4# to allow utilities written in Python to be added
5# to the functools module.
Łukasz Langa6f692512013-06-05 12:20:24 +02006# Written by Nick Coghlan <ncoghlan at gmail.com>,
7# Raymond Hettinger <python at rcn.com>,
8# and Łukasz Langa <lukasz at langa.pl>.
9# Copyright (C) 2006-2013 Python Software Foundation.
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000010# See C source code for _functools credits/copyright
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000011
Georg Brandl2e7346a2010-07-31 18:09:23 +000012__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
Łukasz Langa6f692512013-06-05 12:20:24 +020013 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
Nick Coghlan3daaf5f2013-11-04 23:32:16 +100014 'partialmethod', 'singledispatch']
Georg Brandl2e7346a2010-07-31 18:09:23 +000015
Antoine Pitroub5b37142012-11-13 21:35:40 +010016try:
17 from _functools import reduce
Brett Cannoncd171c82013-07-04 17:43:24 -040018except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +010019 pass
Łukasz Langa6f692512013-06-05 12:20:24 +020020from abc import get_cache_token
Raymond Hettingerec0e9102012-03-16 01:16:31 -070021from collections import namedtuple
Victor Stinner7fa767e2014-03-20 09:16:38 +010022from types import MappingProxyType
Łukasz Langa6f692512013-06-05 12:20:24 +020023from weakref import WeakKeyDictionary
Nick Coghlan457fc9a2016-09-10 20:00:02 +100024from reprlib import recursive_repr
Antoine Pitroua6a4dc82017-09-07 18:56:24 +020025from _thread import RLock
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000026
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070027
28################################################################################
29### update_wrapper() and wraps() decorator
30################################################################################
31
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000032# update_wrapper() and wraps() are tools to help write
33# wrapper functions that can handle naive introspection
34
Meador Ingeff7f64c2011-12-11 22:37:31 -060035WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
36 '__annotations__')
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000037WRAPPER_UPDATES = ('__dict__',)
38def update_wrapper(wrapper,
39 wrapped,
40 assigned = WRAPPER_ASSIGNMENTS,
41 updated = WRAPPER_UPDATES):
42 """Update a wrapper function to look like the wrapped function
43
44 wrapper is the function to be updated
45 wrapped is the original function
46 assigned is a tuple naming the attributes assigned directly
47 from the wrapped function to the wrapper function (defaults to
48 functools.WRAPPER_ASSIGNMENTS)
Thomas Wouters89f507f2006-12-13 04:49:30 +000049 updated is a tuple naming the attributes of the wrapper that
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000050 are updated with the corresponding attribute from the wrapped
51 function (defaults to functools.WRAPPER_UPDATES)
52 """
53 for attr in assigned:
Nick Coghlan98876832010-08-17 06:17:18 +000054 try:
55 value = getattr(wrapped, attr)
56 except AttributeError:
57 pass
58 else:
59 setattr(wrapper, attr, value)
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000060 for attr in updated:
Thomas Wouters89f507f2006-12-13 04:49:30 +000061 getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
Nick Coghlan24c05bc2013-07-15 21:13:08 +100062 # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
63 # from the wrapped function when updating __dict__
64 wrapper.__wrapped__ = wrapped
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000065 # Return the wrapper so this can be used as a decorator via partial()
66 return wrapper
67
68def wraps(wrapped,
69 assigned = WRAPPER_ASSIGNMENTS,
70 updated = WRAPPER_UPDATES):
71 """Decorator factory to apply update_wrapper() to a wrapper function
72
73 Returns a decorator that invokes update_wrapper() with the decorated
74 function as the wrapper argument and the arguments to wraps() as the
75 remaining arguments. Default arguments are as for update_wrapper().
76 This is a convenience function to simplify applying partial() to
77 update_wrapper().
78 """
79 return partial(update_wrapper, wrapped=wrapped,
80 assigned=assigned, updated=updated)
Raymond Hettingerc50846a2010-04-05 18:56:31 +000081
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070082
83################################################################################
84### total_ordering class decorator
85################################################################################
86
Raymond Hettinger0603d302015-01-05 21:52:10 -080087# The total ordering functions all invoke the root magic method directly
88# rather than using the corresponding operator. This avoids possible
89# infinite recursion that could occur when the operator dispatch logic
90# detects a NotImplemented result and then calls a reflected method.
Nick Coghlanf05d9812013-10-02 00:02:03 +100091
Raymond Hettingerffcd8492015-05-12 21:26:37 -070092def _gt_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -080093 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
94 op_result = self.__lt__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +100095 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -080096 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +100097 return not op_result and self != other
98
Raymond Hettingerffcd8492015-05-12 21:26:37 -070099def _le_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800100 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
101 op_result = self.__lt__(other)
102 return op_result or self == other
103
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700104def _ge_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800105 'Return a >= b. Computed by @total_ordering from (not a < b).'
106 op_result = self.__lt__(other)
107 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800108 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800109 return not op_result
110
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700111def _ge_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800112 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
113 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000114 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800115 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000116 return not op_result or self == other
117
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700118def _lt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800119 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
120 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000121 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800122 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000123 return op_result and self != other
124
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700125def _gt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800126 'Return a > b. Computed by @total_ordering from (not a <= b).'
127 op_result = self.__le__(other)
128 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800129 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800130 return not op_result
131
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700132def _lt_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800133 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
134 op_result = self.__gt__(other)
135 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800136 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800137 return not op_result and self != other
138
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700139def _ge_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800140 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
141 op_result = self.__gt__(other)
142 return op_result or self == other
143
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700144def _le_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800145 'Return a <= b. Computed by @total_ordering from (not a > b).'
146 op_result = self.__gt__(other)
147 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800148 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800149 return not op_result
150
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700151def _le_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800152 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
153 op_result = self.__ge__(other)
154 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800155 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800156 return not op_result or self == other
157
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700158def _gt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800159 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
160 op_result = self.__ge__(other)
161 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800162 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800163 return op_result and self != other
164
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700165def _lt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800166 'Return a < b. Computed by @total_ordering from (not a >= b).'
167 op_result = self.__ge__(other)
168 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800169 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800170 return not op_result
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000171
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800172_convert = {
173 '__lt__': [('__gt__', _gt_from_lt),
174 ('__le__', _le_from_lt),
175 ('__ge__', _ge_from_lt)],
176 '__le__': [('__ge__', _ge_from_le),
177 ('__lt__', _lt_from_le),
178 ('__gt__', _gt_from_le)],
179 '__gt__': [('__lt__', _lt_from_gt),
180 ('__ge__', _ge_from_gt),
181 ('__le__', _le_from_gt)],
182 '__ge__': [('__le__', _le_from_ge),
183 ('__gt__', _gt_from_ge),
184 ('__lt__', _lt_from_ge)]
185}
186
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000187def total_ordering(cls):
Georg Brandle5a26732010-05-19 21:06:36 +0000188 """Class decorator that fills in missing ordering methods"""
Raymond Hettinger3255c632010-09-16 00:31:21 +0000189 # Find user-defined comparisons (not those inherited from object).
Raymond Hettinger15ce0be2017-09-05 09:40:44 -0700190 roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)}
Raymond Hettinger56de7e22010-04-10 16:59:03 +0000191 if not roots:
192 raise ValueError('must define at least one ordering operation: < > <= >=')
193 root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800194 for opname, opfunc in _convert[root]:
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000195 if opname not in roots:
196 opfunc.__name__ = opname
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000197 setattr(cls, opname, opfunc)
198 return cls
199
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700200
201################################################################################
202### cmp_to_key() function converter
203################################################################################
204
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000205def cmp_to_key(mycmp):
Georg Brandle5a26732010-05-19 21:06:36 +0000206 """Convert a cmp= function into a key= function"""
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000207 class K(object):
Raymond Hettingera0d1d962011-03-21 17:50:28 -0700208 __slots__ = ['obj']
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700209 def __init__(self, obj):
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000210 self.obj = obj
211 def __lt__(self, other):
212 return mycmp(self.obj, other.obj) < 0
213 def __gt__(self, other):
214 return mycmp(self.obj, other.obj) > 0
215 def __eq__(self, other):
216 return mycmp(self.obj, other.obj) == 0
217 def __le__(self, other):
218 return mycmp(self.obj, other.obj) <= 0
219 def __ge__(self, other):
220 return mycmp(self.obj, other.obj) >= 0
Raymond Hettinger003be522011-05-03 11:01:32 -0700221 __hash__ = None
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000222 return K
Georg Brandl2e7346a2010-07-31 18:09:23 +0000223
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700224try:
225 from _functools import cmp_to_key
Brett Cannoncd171c82013-07-04 17:43:24 -0400226except ImportError:
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700227 pass
228
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700229
230################################################################################
Antoine Pitroub5b37142012-11-13 21:35:40 +0100231### partial() argument application
232################################################################################
233
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000234# Purely functional, no descriptor behaviour
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000235class partial:
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000236 """New function with partial application of the given arguments
Antoine Pitroub5b37142012-11-13 21:35:40 +0100237 and keywords.
238 """
Alexander Belopolskye49af342015-03-01 15:08:17 -0500239
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000240 __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
241
242 def __new__(*args, **keywords):
243 if not args:
244 raise TypeError("descriptor '__new__' of partial needs an argument")
245 if len(args) < 2:
246 raise TypeError("type 'partial' takes at least one argument")
247 cls, func, *args = args
248 if not callable(func):
249 raise TypeError("the first argument must be callable")
250 args = tuple(args)
251
252 if hasattr(func, "func"):
253 args = func.args + args
254 tmpkw = func.keywords.copy()
255 tmpkw.update(keywords)
256 keywords = tmpkw
257 del tmpkw
258 func = func.func
259
260 self = super(partial, cls).__new__(cls)
261
262 self.func = func
263 self.args = args
264 self.keywords = keywords
265 return self
266
267 def __call__(*args, **keywords):
268 if not args:
269 raise TypeError("descriptor '__call__' of partial needs an argument")
270 self, *args = args
271 newkeywords = self.keywords.copy()
272 newkeywords.update(keywords)
273 return self.func(*self.args, *args, **newkeywords)
274
275 @recursive_repr()
276 def __repr__(self):
277 qualname = type(self).__qualname__
278 args = [repr(self.func)]
279 args.extend(repr(x) for x in self.args)
280 args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
281 if type(self).__module__ == "functools":
282 return f"functools.{qualname}({', '.join(args)})"
283 return f"{qualname}({', '.join(args)})"
284
285 def __reduce__(self):
286 return type(self), (self.func,), (self.func, self.args,
287 self.keywords or None, self.__dict__ or None)
288
289 def __setstate__(self, state):
290 if not isinstance(state, tuple):
291 raise TypeError("argument to __setstate__ must be a tuple")
292 if len(state) != 4:
293 raise TypeError(f"expected 4 items in state, got {len(state)}")
294 func, args, kwds, namespace = state
295 if (not callable(func) or not isinstance(args, tuple) or
296 (kwds is not None and not isinstance(kwds, dict)) or
297 (namespace is not None and not isinstance(namespace, dict))):
298 raise TypeError("invalid partial state")
299
300 args = tuple(args) # just in case it's a subclass
301 if kwds is None:
302 kwds = {}
303 elif type(kwds) is not dict: # XXX does it need to be *exactly* dict?
304 kwds = dict(kwds)
305 if namespace is None:
306 namespace = {}
307
308 self.__dict__ = namespace
309 self.func = func
310 self.args = args
311 self.keywords = kwds
Antoine Pitroub5b37142012-11-13 21:35:40 +0100312
313try:
314 from _functools import partial
Brett Cannoncd171c82013-07-04 17:43:24 -0400315except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +0100316 pass
317
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000318# Descriptor version
319class partialmethod(object):
320 """Method descriptor with partial application of the given arguments
321 and keywords.
322
323 Supports wrapping existing descriptors and handles non-descriptor
324 callables as instance methods.
325 """
326
327 def __init__(self, func, *args, **keywords):
328 if not callable(func) and not hasattr(func, "__get__"):
329 raise TypeError("{!r} is not callable or a descriptor"
330 .format(func))
331
332 # func could be a descriptor like classmethod which isn't callable,
333 # so we can't inherit from partial (it verifies func is callable)
334 if isinstance(func, partialmethod):
335 # flattening is mandatory in order to place cls/self before all
336 # other arguments
337 # it's also more efficient since only one function will be called
338 self.func = func.func
339 self.args = func.args + args
340 self.keywords = func.keywords.copy()
341 self.keywords.update(keywords)
342 else:
343 self.func = func
344 self.args = args
345 self.keywords = keywords
346
347 def __repr__(self):
348 args = ", ".join(map(repr, self.args))
349 keywords = ", ".join("{}={!r}".format(k, v)
350 for k, v in self.keywords.items())
351 format_string = "{module}.{cls}({func}, {args}, {keywords})"
352 return format_string.format(module=self.__class__.__module__,
Serhiy Storchaka521e5862014-07-22 15:00:37 +0300353 cls=self.__class__.__qualname__,
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000354 func=self.func,
355 args=args,
356 keywords=keywords)
357
358 def _make_unbound_method(self):
359 def _method(*args, **keywords):
360 call_keywords = self.keywords.copy()
361 call_keywords.update(keywords)
362 cls_or_self, *rest = args
363 call_args = (cls_or_self,) + self.args + tuple(rest)
364 return self.func(*call_args, **call_keywords)
365 _method.__isabstractmethod__ = self.__isabstractmethod__
Yury Selivanovda5fe4f2014-01-27 17:28:37 -0500366 _method._partialmethod = self
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000367 return _method
368
369 def __get__(self, obj, cls):
370 get = getattr(self.func, "__get__", None)
371 result = None
372 if get is not None:
373 new_func = get(obj, cls)
374 if new_func is not self.func:
375 # Assume __get__ returning something new indicates the
376 # creation of an appropriate callable
377 result = partial(new_func, *self.args, **self.keywords)
378 try:
379 result.__self__ = new_func.__self__
380 except AttributeError:
381 pass
382 if result is None:
383 # If the underlying descriptor didn't do anything, treat this
384 # like an instance method
385 result = self._make_unbound_method().__get__(obj, cls)
386 return result
387
388 @property
389 def __isabstractmethod__(self):
390 return getattr(self.func, "__isabstractmethod__", False)
391
Antoine Pitroub5b37142012-11-13 21:35:40 +0100392
393################################################################################
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700394### LRU Cache function decorator
395################################################################################
396
Raymond Hettingerdce583e2012-03-16 22:12:20 -0700397_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
Nick Coghlan234515a2010-11-30 06:19:46 +0000398
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700399class _HashedSeq(list):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700400 """ This class guarantees that hash() will be called no more than once
401 per element. This is important because the lru_cache() will hash
402 the key multiple times on a cache miss.
403
404 """
405
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700406 __slots__ = 'hashvalue'
407
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700408 def __init__(self, tup, hash=hash):
409 self[:] = tup
410 self.hashvalue = hash(tup)
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700411
412 def __hash__(self):
413 return self.hashvalue
414
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700415def _make_key(args, kwds, typed,
416 kwd_mark = (object(),),
417 fasttypes = {int, str, frozenset, type(None)},
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800418 tuple=tuple, type=type, len=len):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700419 """Make a cache key from optionally typed positional and keyword arguments
420
421 The key is constructed in a way that is flat as possible rather than
422 as a nested structure that would take more memory.
423
424 If there is only a single argument and its data type is known to cache
425 its hash value, then that argument is returned without a wrapper. This
426 saves space and improves lookup speed.
427
428 """
Raymond Hettinger55037092017-09-04 17:47:53 -0700429 # All of code below relies on kwds preserving the order input by the user.
430 # Formerly, we sorted() the kwds before looping. The new way is *much*
431 # faster; however, it means that f(x=1, y=2) will now be treated as a
432 # distinct call from f(y=2, x=1) which will be cached separately.
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700433 key = args
434 if kwds:
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700435 key += kwd_mark
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800436 for item in kwds.items():
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700437 key += item
438 if typed:
439 key += tuple(type(v) for v in args)
440 if kwds:
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800441 key += tuple(type(v) for v in kwds.values())
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700442 elif len(key) == 1 and type(key[0]) in fasttypes:
443 return key[0]
444 return _HashedSeq(key)
445
Raymond Hettinger010ce322012-05-19 21:20:48 -0700446def lru_cache(maxsize=128, typed=False):
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000447 """Least-recently-used cache decorator.
Georg Brandl2e7346a2010-07-31 18:09:23 +0000448
Raymond Hettingerc79fb0e2010-12-01 03:45:41 +0000449 If *maxsize* is set to None, the LRU features are disabled and the cache
450 can grow without bound.
451
Raymond Hettingercd9fdfd2011-10-20 08:57:45 -0700452 If *typed* is True, arguments of different types will be cached separately.
453 For example, f(3.0) and f(3) will be treated as distinct calls with
454 distinct results.
455
Georg Brandl2e7346a2010-07-31 18:09:23 +0000456 Arguments to the cached function must be hashable.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000457
Raymond Hettinger7f7a5a72012-03-30 21:50:40 -0700458 View the cache statistics named tuple (hits, misses, maxsize, currsize)
459 with f.cache_info(). Clear the cache and statistics with f.cache_clear().
Raymond Hettinger00f2f972010-12-01 00:47:56 +0000460 Access the underlying function with f.__wrapped__.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000461
462 See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
Georg Brandl2e7346a2010-07-31 18:09:23 +0000463
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000464 """
Raymond Hettinger1ff50df2012-03-30 13:15:48 -0700465
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000466 # Users should only access the lru_cache through its public API:
Raymond Hettinger5e20bab2010-11-30 07:13:04 +0000467 # cache_info, cache_clear, and f.__wrapped__
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000468 # The internals of the lru_cache are encapsulated for thread safety and
469 # to allow the implementation to change (including a possible C version).
470
Raymond Hettinger4d588972014-08-12 12:44:52 -0700471 # Early detection of an erroneous call to @lru_cache without any arguments
472 # resulting in the inner function being passed to maxsize instead of an
473 # integer or None.
474 if maxsize is not None and not isinstance(maxsize, int):
475 raise TypeError('Expected maxsize to be an integer or None')
476
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300477 def decorating_function(user_function):
478 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
479 return update_wrapper(wrapper, user_function)
480
481 return decorating_function
482
483def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700484 # Constants shared by all lru cache instances:
Raymond Hettingerb6b98c02012-04-29 18:09:02 -0700485 sentinel = object() # unique object used to signal cache misses
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700486 make_key = _make_key # build a key from the function arguments
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700487 PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
488
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300489 cache = {}
490 hits = misses = 0
491 full = False
492 cache_get = cache.get # bound method to lookup a key or return None
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800493 cache_len = cache.__len__ # get cache size without calling len()
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300494 lock = RLock() # because linkedlist updates aren't threadsafe
495 root = [] # root of the circular doubly linked list
496 root[:] = [root, root, None, None] # initialize by pointing to self
Raymond Hettinger6e8c8172012-03-16 16:53:05 -0700497
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300498 if maxsize == 0:
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700499
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300500 def wrapper(*args, **kwds):
501 # No caching -- just a statistics update after a successful call
502 nonlocal misses
503 result = user_function(*args, **kwds)
504 misses += 1
505 return result
506
507 elif maxsize is None:
508
509 def wrapper(*args, **kwds):
510 # Simple caching without ordering or size limit
511 nonlocal hits, misses
512 key = make_key(args, kwds, typed)
513 result = cache_get(key, sentinel)
514 if result is not sentinel:
515 hits += 1
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700516 return result
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300517 result = user_function(*args, **kwds)
518 cache[key] = result
519 misses += 1
520 return result
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700521
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300522 else:
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700523
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300524 def wrapper(*args, **kwds):
525 # Size limited caching that tracks accesses by recency
526 nonlocal root, hits, misses, full
527 key = make_key(args, kwds, typed)
528 with lock:
529 link = cache_get(key)
530 if link is not None:
531 # Move the link to the front of the circular queue
532 link_prev, link_next, _key, result = link
533 link_prev[NEXT] = link_next
534 link_next[PREV] = link_prev
535 last = root[PREV]
536 last[NEXT] = root[PREV] = link
537 link[PREV] = last
538 link[NEXT] = root
Nick Coghlan234515a2010-11-30 06:19:46 +0000539 hits += 1
Raymond Hettinger4b779b32011-10-15 23:50:42 -0700540 return result
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300541 result = user_function(*args, **kwds)
542 with lock:
543 if key in cache:
544 # Getting here means that this same key was added to the
545 # cache while the lock was released. Since the link
546 # update is already done, we need only return the
547 # computed result and update the count of misses.
548 pass
549 elif full:
550 # Use the old root to store the new key and result.
551 oldroot = root
552 oldroot[KEY] = key
553 oldroot[RESULT] = result
554 # Empty the oldest link and make it the new root.
555 # Keep a reference to the old key and old result to
556 # prevent their ref counts from going to zero during the
557 # update. That will prevent potentially arbitrary object
558 # clean-up code (i.e. __del__) from running while we're
559 # still adjusting the links.
560 root = oldroot[NEXT]
561 oldkey = root[KEY]
562 oldresult = root[RESULT]
563 root[KEY] = root[RESULT] = None
564 # Now update the cache dictionary.
565 del cache[oldkey]
566 # Save the potentially reentrant cache[key] assignment
567 # for last, after the root and links have been put in
568 # a consistent state.
569 cache[key] = oldroot
570 else:
571 # Put result in a new link at the front of the queue.
572 last = root[PREV]
573 link = [last, root, key, result]
574 last[NEXT] = root[PREV] = cache[key] = link
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800575 # Use the cache_len bound method instead of the len() function
Raymond Hettingeraf56e0e2016-12-16 13:57:40 -0800576 # which could potentially be wrapped in an lru_cache itself.
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800577 full = (cache_len() >= maxsize)
Raymond Hettinger4b779b32011-10-15 23:50:42 -0700578 misses += 1
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300579 return result
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700580
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300581 def cache_info():
582 """Report cache statistics"""
583 with lock:
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800584 return _CacheInfo(hits, misses, maxsize, cache_len())
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700585
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300586 def cache_clear():
587 """Clear the cache and cache statistics"""
588 nonlocal hits, misses, full
589 with lock:
590 cache.clear()
591 root[:] = [root, root, None, None]
592 hits = misses = 0
593 full = False
Georg Brandl2e7346a2010-07-31 18:09:23 +0000594
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300595 wrapper.cache_info = cache_info
596 wrapper.cache_clear = cache_clear
Serhiy Storchakace2295d2015-10-24 09:51:53 +0300597 return wrapper
Nick Coghlan234515a2010-11-30 06:19:46 +0000598
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300599try:
600 from _functools import _lru_cache_wrapper
601except ImportError:
602 pass
Łukasz Langa6f692512013-06-05 12:20:24 +0200603
604
605################################################################################
606### singledispatch() - single-dispatch generic function decorator
607################################################################################
608
Łukasz Langa3720c772013-07-01 16:00:38 +0200609def _c3_merge(sequences):
610 """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
611
612 Adapted from http://www.python.org/download/releases/2.3/mro/.
613
614 """
615 result = []
616 while True:
617 sequences = [s for s in sequences if s] # purge empty sequences
618 if not sequences:
619 return result
620 for s1 in sequences: # find merge candidates among seq heads
621 candidate = s1[0]
622 for s2 in sequences:
623 if candidate in s2[1:]:
624 candidate = None
625 break # reject the current head, it appears later
626 else:
627 break
Yury Selivanov77a8cd62015-08-18 14:20:00 -0400628 if candidate is None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200629 raise RuntimeError("Inconsistent hierarchy")
630 result.append(candidate)
631 # remove the chosen candidate
632 for seq in sequences:
633 if seq[0] == candidate:
634 del seq[0]
635
636def _c3_mro(cls, abcs=None):
637 """Computes the method resolution order using extended C3 linearization.
638
639 If no *abcs* are given, the algorithm works exactly like the built-in C3
640 linearization used for method resolution.
641
642 If given, *abcs* is a list of abstract base classes that should be inserted
643 into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
644 result. The algorithm inserts ABCs where their functionality is introduced,
645 i.e. issubclass(cls, abc) returns True for the class itself but returns
646 False for all its direct base classes. Implicit ABCs for a given class
647 (either registered or inferred from the presence of a special method like
648 __len__) are inserted directly after the last ABC explicitly listed in the
649 MRO of said class. If two implicit ABCs end up next to each other in the
650 resulting MRO, their ordering depends on the order of types in *abcs*.
651
652 """
653 for i, base in enumerate(reversed(cls.__bases__)):
654 if hasattr(base, '__abstractmethods__'):
655 boundary = len(cls.__bases__) - i
656 break # Bases up to the last explicit ABC are considered first.
657 else:
658 boundary = 0
659 abcs = list(abcs) if abcs else []
660 explicit_bases = list(cls.__bases__[:boundary])
661 abstract_bases = []
662 other_bases = list(cls.__bases__[boundary:])
663 for base in abcs:
664 if issubclass(cls, base) and not any(
665 issubclass(b, base) for b in cls.__bases__
666 ):
667 # If *cls* is the class that introduces behaviour described by
668 # an ABC *base*, insert said ABC to its MRO.
669 abstract_bases.append(base)
670 for base in abstract_bases:
671 abcs.remove(base)
672 explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
673 abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
674 other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
675 return _c3_merge(
676 [[cls]] +
677 explicit_c3_mros + abstract_c3_mros + other_c3_mros +
678 [explicit_bases] + [abstract_bases] + [other_bases]
679 )
680
681def _compose_mro(cls, types):
682 """Calculates the method resolution order for a given class *cls*.
683
684 Includes relevant abstract base classes (with their respective bases) from
685 the *types* iterable. Uses a modified C3 linearization algorithm.
Łukasz Langa6f692512013-06-05 12:20:24 +0200686
687 """
688 bases = set(cls.__mro__)
Łukasz Langa3720c772013-07-01 16:00:38 +0200689 # Remove entries which are already present in the __mro__ or unrelated.
690 def is_related(typ):
691 return (typ not in bases and hasattr(typ, '__mro__')
692 and issubclass(cls, typ))
693 types = [n for n in types if is_related(n)]
694 # Remove entries which are strict bases of other entries (they will end up
695 # in the MRO anyway.
696 def is_strict_base(typ):
697 for other in types:
698 if typ != other and typ in other.__mro__:
699 return True
700 return False
701 types = [n for n in types if not is_strict_base(n)]
702 # Subclasses of the ABCs in *types* which are also implemented by
703 # *cls* can be used to stabilize ABC ordering.
704 type_set = set(types)
705 mro = []
706 for typ in types:
707 found = []
708 for sub in typ.__subclasses__():
709 if sub not in bases and issubclass(cls, sub):
710 found.append([s for s in sub.__mro__ if s in type_set])
711 if not found:
712 mro.append(typ)
713 continue
714 # Favor subclasses with the biggest number of useful bases
715 found.sort(key=len, reverse=True)
716 for sub in found:
717 for subcls in sub:
718 if subcls not in mro:
719 mro.append(subcls)
720 return _c3_mro(cls, abcs=mro)
Łukasz Langa6f692512013-06-05 12:20:24 +0200721
722def _find_impl(cls, registry):
Łukasz Langa3720c772013-07-01 16:00:38 +0200723 """Returns the best matching implementation from *registry* for type *cls*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200724
Łukasz Langa3720c772013-07-01 16:00:38 +0200725 Where there is no registered implementation for a specific type, its method
726 resolution order is used to find a more generic implementation.
727
728 Note: if *registry* does not contain an implementation for the base
729 *object* type, this function may return None.
Łukasz Langa6f692512013-06-05 12:20:24 +0200730
731 """
732 mro = _compose_mro(cls, registry.keys())
733 match = None
734 for t in mro:
735 if match is not None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200736 # If *match* is an implicit ABC but there is another unrelated,
737 # equally matching implicit ABC, refuse the temptation to guess.
738 if (t in registry and t not in cls.__mro__
739 and match not in cls.__mro__
740 and not issubclass(match, t)):
Łukasz Langa6f692512013-06-05 12:20:24 +0200741 raise RuntimeError("Ambiguous dispatch: {} or {}".format(
742 match, t))
743 break
744 if t in registry:
745 match = t
746 return registry.get(match)
747
748def singledispatch(func):
749 """Single-dispatch generic function decorator.
750
751 Transforms a function into a generic function, which can have different
752 behaviours depending upon the type of its first argument. The decorated
753 function acts as the default implementation, and additional
Łukasz Langa3720c772013-07-01 16:00:38 +0200754 implementations can be registered using the register() attribute of the
755 generic function.
Łukasz Langa6f692512013-06-05 12:20:24 +0200756
757 """
758 registry = {}
759 dispatch_cache = WeakKeyDictionary()
760 cache_token = None
761
Łukasz Langa3720c772013-07-01 16:00:38 +0200762 def dispatch(cls):
763 """generic_func.dispatch(cls) -> <function implementation>
Łukasz Langa6f692512013-06-05 12:20:24 +0200764
765 Runs the dispatch algorithm to return the best available implementation
Łukasz Langa3720c772013-07-01 16:00:38 +0200766 for the given *cls* registered on *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200767
768 """
769 nonlocal cache_token
770 if cache_token is not None:
771 current_token = get_cache_token()
772 if cache_token != current_token:
773 dispatch_cache.clear()
774 cache_token = current_token
775 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200776 impl = dispatch_cache[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200777 except KeyError:
778 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200779 impl = registry[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200780 except KeyError:
Łukasz Langa3720c772013-07-01 16:00:38 +0200781 impl = _find_impl(cls, registry)
782 dispatch_cache[cls] = impl
Łukasz Langa6f692512013-06-05 12:20:24 +0200783 return impl
784
Łukasz Langa3720c772013-07-01 16:00:38 +0200785 def register(cls, func=None):
786 """generic_func.register(cls, func) -> func
Łukasz Langa6f692512013-06-05 12:20:24 +0200787
Łukasz Langa3720c772013-07-01 16:00:38 +0200788 Registers a new implementation for the given *cls* on a *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200789
790 """
791 nonlocal cache_token
792 if func is None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200793 return lambda f: register(cls, f)
794 registry[cls] = func
795 if cache_token is None and hasattr(cls, '__abstractmethods__'):
Łukasz Langa6f692512013-06-05 12:20:24 +0200796 cache_token = get_cache_token()
797 dispatch_cache.clear()
798 return func
799
800 def wrapper(*args, **kw):
801 return dispatch(args[0].__class__)(*args, **kw)
802
803 registry[object] = func
804 wrapper.register = register
805 wrapper.dispatch = dispatch
806 wrapper.registry = MappingProxyType(registry)
807 wrapper._clear_cache = dispatch_cache.clear
808 update_wrapper(wrapper, func)
809 return wrapper