blob: 6233c30c203ea750ad799049b047cdfc30fc2b45 [file] [log] [blame]
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001"""functools.py - Tools for working with functions and callable objects
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00002"""
3# Python module wrapper for _functools C module
4# to allow utilities written in Python to be added
5# to the functools module.
Łukasz Langa6f692512013-06-05 12:20:24 +02006# Written by Nick Coghlan <ncoghlan at gmail.com>,
7# Raymond Hettinger <python at rcn.com>,
8# and Łukasz Langa <lukasz at langa.pl>.
9# Copyright (C) 2006-2013 Python Software Foundation.
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000010# See C source code for _functools credits/copyright
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000011
Georg Brandl2e7346a2010-07-31 18:09:23 +000012__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
Łukasz Langa6f692512013-06-05 12:20:24 +020013 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
Ethan Smithc6512752018-05-26 16:38:33 -040014 'partialmethod', 'singledispatch', 'singledispatchmethod']
Georg Brandl2e7346a2010-07-31 18:09:23 +000015
Łukasz Langa6f692512013-06-05 12:20:24 +020016from abc import get_cache_token
Raymond Hettingerec0e9102012-03-16 01:16:31 -070017from collections import namedtuple
INADA Naoki9811e802017-09-30 16:13:02 +090018# import types, weakref # Deferred to single_dispatch()
Nick Coghlan457fc9a2016-09-10 20:00:02 +100019from reprlib import recursive_repr
Antoine Pitroua6a4dc82017-09-07 18:56:24 +020020from _thread import RLock
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000021
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070022
23################################################################################
24### update_wrapper() and wraps() decorator
25################################################################################
26
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000027# update_wrapper() and wraps() are tools to help write
28# wrapper functions that can handle naive introspection
29
Meador Ingeff7f64c2011-12-11 22:37:31 -060030WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
31 '__annotations__')
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000032WRAPPER_UPDATES = ('__dict__',)
33def update_wrapper(wrapper,
34 wrapped,
35 assigned = WRAPPER_ASSIGNMENTS,
36 updated = WRAPPER_UPDATES):
37 """Update a wrapper function to look like the wrapped function
38
39 wrapper is the function to be updated
40 wrapped is the original function
41 assigned is a tuple naming the attributes assigned directly
42 from the wrapped function to the wrapper function (defaults to
43 functools.WRAPPER_ASSIGNMENTS)
Thomas Wouters89f507f2006-12-13 04:49:30 +000044 updated is a tuple naming the attributes of the wrapper that
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000045 are updated with the corresponding attribute from the wrapped
46 function (defaults to functools.WRAPPER_UPDATES)
47 """
48 for attr in assigned:
Nick Coghlan98876832010-08-17 06:17:18 +000049 try:
50 value = getattr(wrapped, attr)
51 except AttributeError:
52 pass
53 else:
54 setattr(wrapper, attr, value)
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000055 for attr in updated:
Thomas Wouters89f507f2006-12-13 04:49:30 +000056 getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
Nick Coghlan24c05bc2013-07-15 21:13:08 +100057 # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
58 # from the wrapped function when updating __dict__
59 wrapper.__wrapped__ = wrapped
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000060 # Return the wrapper so this can be used as a decorator via partial()
61 return wrapper
62
63def wraps(wrapped,
64 assigned = WRAPPER_ASSIGNMENTS,
65 updated = WRAPPER_UPDATES):
66 """Decorator factory to apply update_wrapper() to a wrapper function
67
68 Returns a decorator that invokes update_wrapper() with the decorated
69 function as the wrapper argument and the arguments to wraps() as the
70 remaining arguments. Default arguments are as for update_wrapper().
71 This is a convenience function to simplify applying partial() to
72 update_wrapper().
73 """
74 return partial(update_wrapper, wrapped=wrapped,
75 assigned=assigned, updated=updated)
Raymond Hettingerc50846a2010-04-05 18:56:31 +000076
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070077
78################################################################################
79### total_ordering class decorator
80################################################################################
81
Raymond Hettinger0603d302015-01-05 21:52:10 -080082# The total ordering functions all invoke the root magic method directly
83# rather than using the corresponding operator. This avoids possible
84# infinite recursion that could occur when the operator dispatch logic
85# detects a NotImplemented result and then calls a reflected method.
Nick Coghlanf05d9812013-10-02 00:02:03 +100086
Raymond Hettingerffcd8492015-05-12 21:26:37 -070087def _gt_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -080088 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
89 op_result = self.__lt__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +100090 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -080091 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +100092 return not op_result and self != other
93
Raymond Hettingerffcd8492015-05-12 21:26:37 -070094def _le_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -080095 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
96 op_result = self.__lt__(other)
97 return op_result or self == other
98
Raymond Hettingerffcd8492015-05-12 21:26:37 -070099def _ge_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800100 'Return a >= b. Computed by @total_ordering from (not a < b).'
101 op_result = self.__lt__(other)
102 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800103 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800104 return not op_result
105
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700106def _ge_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800107 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
108 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000109 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800110 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000111 return not op_result or self == other
112
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700113def _lt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800114 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
115 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000116 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800117 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000118 return op_result and self != other
119
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700120def _gt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800121 'Return a > b. Computed by @total_ordering from (not a <= b).'
122 op_result = self.__le__(other)
123 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800124 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800125 return not op_result
126
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700127def _lt_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800128 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
129 op_result = self.__gt__(other)
130 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800131 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800132 return not op_result and self != other
133
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700134def _ge_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800135 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
136 op_result = self.__gt__(other)
137 return op_result or self == other
138
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700139def _le_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800140 'Return a <= b. Computed by @total_ordering from (not a > b).'
141 op_result = self.__gt__(other)
142 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800143 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800144 return not op_result
145
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700146def _le_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800147 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
148 op_result = self.__ge__(other)
149 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800150 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800151 return not op_result or self == other
152
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700153def _gt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800154 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
155 op_result = self.__ge__(other)
156 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800157 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800158 return op_result and self != other
159
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700160def _lt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800161 'Return a < b. Computed by @total_ordering from (not a >= b).'
162 op_result = self.__ge__(other)
163 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800164 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800165 return not op_result
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000166
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800167_convert = {
168 '__lt__': [('__gt__', _gt_from_lt),
169 ('__le__', _le_from_lt),
170 ('__ge__', _ge_from_lt)],
171 '__le__': [('__ge__', _ge_from_le),
172 ('__lt__', _lt_from_le),
173 ('__gt__', _gt_from_le)],
174 '__gt__': [('__lt__', _lt_from_gt),
175 ('__ge__', _ge_from_gt),
176 ('__le__', _le_from_gt)],
177 '__ge__': [('__le__', _le_from_ge),
178 ('__gt__', _gt_from_ge),
179 ('__lt__', _lt_from_ge)]
180}
181
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000182def total_ordering(cls):
Georg Brandle5a26732010-05-19 21:06:36 +0000183 """Class decorator that fills in missing ordering methods"""
Raymond Hettinger3255c632010-09-16 00:31:21 +0000184 # Find user-defined comparisons (not those inherited from object).
Raymond Hettinger15ce0be2017-09-05 09:40:44 -0700185 roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)}
Raymond Hettinger56de7e22010-04-10 16:59:03 +0000186 if not roots:
187 raise ValueError('must define at least one ordering operation: < > <= >=')
188 root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800189 for opname, opfunc in _convert[root]:
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000190 if opname not in roots:
191 opfunc.__name__ = opname
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000192 setattr(cls, opname, opfunc)
193 return cls
194
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700195
196################################################################################
197### cmp_to_key() function converter
198################################################################################
199
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000200def cmp_to_key(mycmp):
Georg Brandle5a26732010-05-19 21:06:36 +0000201 """Convert a cmp= function into a key= function"""
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000202 class K(object):
Raymond Hettingera0d1d962011-03-21 17:50:28 -0700203 __slots__ = ['obj']
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700204 def __init__(self, obj):
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000205 self.obj = obj
206 def __lt__(self, other):
207 return mycmp(self.obj, other.obj) < 0
208 def __gt__(self, other):
209 return mycmp(self.obj, other.obj) > 0
210 def __eq__(self, other):
211 return mycmp(self.obj, other.obj) == 0
212 def __le__(self, other):
213 return mycmp(self.obj, other.obj) <= 0
214 def __ge__(self, other):
215 return mycmp(self.obj, other.obj) >= 0
Raymond Hettinger003be522011-05-03 11:01:32 -0700216 __hash__ = None
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000217 return K
Georg Brandl2e7346a2010-07-31 18:09:23 +0000218
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700219try:
220 from _functools import cmp_to_key
Brett Cannoncd171c82013-07-04 17:43:24 -0400221except ImportError:
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700222 pass
223
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700224
225################################################################################
madman-bobe25d5fc2018-10-25 15:02:10 +0100226### reduce() sequence to a single item
227################################################################################
228
229_initial_missing = object()
230
231def reduce(function, sequence, initial=_initial_missing):
232 """
233 reduce(function, sequence[, initial]) -> value
234
235 Apply a function of two arguments cumulatively to the items of a sequence,
236 from left to right, so as to reduce the sequence to a single value.
237 For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
238 ((((1+2)+3)+4)+5). If initial is present, it is placed before the items
239 of the sequence in the calculation, and serves as a default when the
240 sequence is empty.
241 """
242
243 it = iter(sequence)
244
245 if initial is _initial_missing:
246 try:
247 value = next(it)
248 except StopIteration:
249 raise TypeError("reduce() of empty sequence with no initial value") from None
250 else:
251 value = initial
252
253 for element in it:
254 value = function(value, element)
255
256 return value
257
258try:
259 from _functools import reduce
260except ImportError:
261 pass
262
263
264################################################################################
Antoine Pitroub5b37142012-11-13 21:35:40 +0100265### partial() argument application
266################################################################################
267
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000268# Purely functional, no descriptor behaviour
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000269class partial:
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000270 """New function with partial application of the given arguments
Antoine Pitroub5b37142012-11-13 21:35:40 +0100271 and keywords.
272 """
Alexander Belopolskye49af342015-03-01 15:08:17 -0500273
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000274 __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
275
276 def __new__(*args, **keywords):
277 if not args:
278 raise TypeError("descriptor '__new__' of partial needs an argument")
279 if len(args) < 2:
280 raise TypeError("type 'partial' takes at least one argument")
281 cls, func, *args = args
282 if not callable(func):
283 raise TypeError("the first argument must be callable")
284 args = tuple(args)
285
286 if hasattr(func, "func"):
287 args = func.args + args
288 tmpkw = func.keywords.copy()
289 tmpkw.update(keywords)
290 keywords = tmpkw
291 del tmpkw
292 func = func.func
293
294 self = super(partial, cls).__new__(cls)
295
296 self.func = func
297 self.args = args
298 self.keywords = keywords
299 return self
300
301 def __call__(*args, **keywords):
302 if not args:
303 raise TypeError("descriptor '__call__' of partial needs an argument")
304 self, *args = args
305 newkeywords = self.keywords.copy()
306 newkeywords.update(keywords)
307 return self.func(*self.args, *args, **newkeywords)
308
309 @recursive_repr()
310 def __repr__(self):
311 qualname = type(self).__qualname__
312 args = [repr(self.func)]
313 args.extend(repr(x) for x in self.args)
314 args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
315 if type(self).__module__ == "functools":
316 return f"functools.{qualname}({', '.join(args)})"
317 return f"{qualname}({', '.join(args)})"
318
319 def __reduce__(self):
320 return type(self), (self.func,), (self.func, self.args,
321 self.keywords or None, self.__dict__ or None)
322
323 def __setstate__(self, state):
324 if not isinstance(state, tuple):
325 raise TypeError("argument to __setstate__ must be a tuple")
326 if len(state) != 4:
327 raise TypeError(f"expected 4 items in state, got {len(state)}")
328 func, args, kwds, namespace = state
329 if (not callable(func) or not isinstance(args, tuple) or
330 (kwds is not None and not isinstance(kwds, dict)) or
331 (namespace is not None and not isinstance(namespace, dict))):
332 raise TypeError("invalid partial state")
333
334 args = tuple(args) # just in case it's a subclass
335 if kwds is None:
336 kwds = {}
337 elif type(kwds) is not dict: # XXX does it need to be *exactly* dict?
338 kwds = dict(kwds)
339 if namespace is None:
340 namespace = {}
341
342 self.__dict__ = namespace
343 self.func = func
344 self.args = args
345 self.keywords = kwds
Antoine Pitroub5b37142012-11-13 21:35:40 +0100346
347try:
348 from _functools import partial
Brett Cannoncd171c82013-07-04 17:43:24 -0400349except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +0100350 pass
351
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000352# Descriptor version
353class partialmethod(object):
354 """Method descriptor with partial application of the given arguments
355 and keywords.
356
357 Supports wrapping existing descriptors and handles non-descriptor
358 callables as instance methods.
359 """
360
361 def __init__(self, func, *args, **keywords):
362 if not callable(func) and not hasattr(func, "__get__"):
363 raise TypeError("{!r} is not callable or a descriptor"
364 .format(func))
365
366 # func could be a descriptor like classmethod which isn't callable,
367 # so we can't inherit from partial (it verifies func is callable)
368 if isinstance(func, partialmethod):
369 # flattening is mandatory in order to place cls/self before all
370 # other arguments
371 # it's also more efficient since only one function will be called
372 self.func = func.func
373 self.args = func.args + args
374 self.keywords = func.keywords.copy()
375 self.keywords.update(keywords)
376 else:
377 self.func = func
378 self.args = args
379 self.keywords = keywords
380
381 def __repr__(self):
382 args = ", ".join(map(repr, self.args))
383 keywords = ", ".join("{}={!r}".format(k, v)
384 for k, v in self.keywords.items())
385 format_string = "{module}.{cls}({func}, {args}, {keywords})"
386 return format_string.format(module=self.__class__.__module__,
Serhiy Storchaka521e5862014-07-22 15:00:37 +0300387 cls=self.__class__.__qualname__,
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000388 func=self.func,
389 args=args,
390 keywords=keywords)
391
392 def _make_unbound_method(self):
393 def _method(*args, **keywords):
394 call_keywords = self.keywords.copy()
395 call_keywords.update(keywords)
396 cls_or_self, *rest = args
397 call_args = (cls_or_self,) + self.args + tuple(rest)
398 return self.func(*call_args, **call_keywords)
399 _method.__isabstractmethod__ = self.__isabstractmethod__
Yury Selivanovda5fe4f2014-01-27 17:28:37 -0500400 _method._partialmethod = self
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000401 return _method
402
403 def __get__(self, obj, cls):
404 get = getattr(self.func, "__get__", None)
405 result = None
406 if get is not None:
407 new_func = get(obj, cls)
408 if new_func is not self.func:
409 # Assume __get__ returning something new indicates the
410 # creation of an appropriate callable
411 result = partial(new_func, *self.args, **self.keywords)
412 try:
413 result.__self__ = new_func.__self__
414 except AttributeError:
415 pass
416 if result is None:
417 # If the underlying descriptor didn't do anything, treat this
418 # like an instance method
419 result = self._make_unbound_method().__get__(obj, cls)
420 return result
421
422 @property
423 def __isabstractmethod__(self):
424 return getattr(self.func, "__isabstractmethod__", False)
425
Pablo Galindo7cd25432018-10-26 12:19:14 +0100426# Helper functions
427
428def _unwrap_partial(func):
429 while isinstance(func, partial):
430 func = func.func
431 return func
Antoine Pitroub5b37142012-11-13 21:35:40 +0100432
433################################################################################
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700434### LRU Cache function decorator
435################################################################################
436
Raymond Hettingerdce583e2012-03-16 22:12:20 -0700437_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
Nick Coghlan234515a2010-11-30 06:19:46 +0000438
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700439class _HashedSeq(list):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700440 """ This class guarantees that hash() will be called no more than once
441 per element. This is important because the lru_cache() will hash
442 the key multiple times on a cache miss.
443
444 """
445
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700446 __slots__ = 'hashvalue'
447
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700448 def __init__(self, tup, hash=hash):
449 self[:] = tup
450 self.hashvalue = hash(tup)
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700451
452 def __hash__(self):
453 return self.hashvalue
454
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700455def _make_key(args, kwds, typed,
456 kwd_mark = (object(),),
Raymond Hettingerd8080c02019-01-26 03:02:00 -0500457 fasttypes = {int, str},
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800458 tuple=tuple, type=type, len=len):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700459 """Make a cache key from optionally typed positional and keyword arguments
460
461 The key is constructed in a way that is flat as possible rather than
462 as a nested structure that would take more memory.
463
464 If there is only a single argument and its data type is known to cache
465 its hash value, then that argument is returned without a wrapper. This
466 saves space and improves lookup speed.
467
468 """
Raymond Hettinger55037092017-09-04 17:47:53 -0700469 # All of code below relies on kwds preserving the order input by the user.
470 # Formerly, we sorted() the kwds before looping. The new way is *much*
471 # faster; however, it means that f(x=1, y=2) will now be treated as a
472 # distinct call from f(y=2, x=1) which will be cached separately.
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700473 key = args
474 if kwds:
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700475 key += kwd_mark
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800476 for item in kwds.items():
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700477 key += item
478 if typed:
479 key += tuple(type(v) for v in args)
480 if kwds:
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800481 key += tuple(type(v) for v in kwds.values())
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700482 elif len(key) == 1 and type(key[0]) in fasttypes:
483 return key[0]
484 return _HashedSeq(key)
485
Raymond Hettinger010ce322012-05-19 21:20:48 -0700486def lru_cache(maxsize=128, typed=False):
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000487 """Least-recently-used cache decorator.
Georg Brandl2e7346a2010-07-31 18:09:23 +0000488
Raymond Hettingerc79fb0e2010-12-01 03:45:41 +0000489 If *maxsize* is set to None, the LRU features are disabled and the cache
490 can grow without bound.
491
Raymond Hettingercd9fdfd2011-10-20 08:57:45 -0700492 If *typed* is True, arguments of different types will be cached separately.
493 For example, f(3.0) and f(3) will be treated as distinct calls with
494 distinct results.
495
Georg Brandl2e7346a2010-07-31 18:09:23 +0000496 Arguments to the cached function must be hashable.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000497
Raymond Hettinger7f7a5a72012-03-30 21:50:40 -0700498 View the cache statistics named tuple (hits, misses, maxsize, currsize)
499 with f.cache_info(). Clear the cache and statistics with f.cache_clear().
Raymond Hettinger00f2f972010-12-01 00:47:56 +0000500 Access the underlying function with f.__wrapped__.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000501
502 See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
Georg Brandl2e7346a2010-07-31 18:09:23 +0000503
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000504 """
Raymond Hettinger1ff50df2012-03-30 13:15:48 -0700505
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000506 # Users should only access the lru_cache through its public API:
Raymond Hettinger5e20bab2010-11-30 07:13:04 +0000507 # cache_info, cache_clear, and f.__wrapped__
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000508 # The internals of the lru_cache are encapsulated for thread safety and
509 # to allow the implementation to change (including a possible C version).
510
Raymond Hettinger4d588972014-08-12 12:44:52 -0700511 # Early detection of an erroneous call to @lru_cache without any arguments
512 # resulting in the inner function being passed to maxsize instead of an
Raymond Hettingerd8080c02019-01-26 03:02:00 -0500513 # integer or None. Negative maxsize is treated as 0.
514 if isinstance(maxsize, int):
515 if maxsize < 0:
516 maxsize = 0
517 elif maxsize is not None:
Raymond Hettinger4d588972014-08-12 12:44:52 -0700518 raise TypeError('Expected maxsize to be an integer or None')
519
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300520 def decorating_function(user_function):
521 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
522 return update_wrapper(wrapper, user_function)
523
524 return decorating_function
525
526def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700527 # Constants shared by all lru cache instances:
Raymond Hettingerb6b98c02012-04-29 18:09:02 -0700528 sentinel = object() # unique object used to signal cache misses
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700529 make_key = _make_key # build a key from the function arguments
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700530 PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
531
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300532 cache = {}
533 hits = misses = 0
534 full = False
535 cache_get = cache.get # bound method to lookup a key or return None
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800536 cache_len = cache.__len__ # get cache size without calling len()
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300537 lock = RLock() # because linkedlist updates aren't threadsafe
538 root = [] # root of the circular doubly linked list
539 root[:] = [root, root, None, None] # initialize by pointing to self
Raymond Hettinger6e8c8172012-03-16 16:53:05 -0700540
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300541 if maxsize == 0:
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700542
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300543 def wrapper(*args, **kwds):
544 # No caching -- just a statistics update after a successful call
545 nonlocal misses
546 result = user_function(*args, **kwds)
547 misses += 1
548 return result
549
550 elif maxsize is None:
551
552 def wrapper(*args, **kwds):
553 # Simple caching without ordering or size limit
554 nonlocal hits, misses
555 key = make_key(args, kwds, typed)
556 result = cache_get(key, sentinel)
557 if result is not sentinel:
558 hits += 1
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700559 return result
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300560 result = user_function(*args, **kwds)
561 cache[key] = result
562 misses += 1
563 return result
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700564
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300565 else:
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700566
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300567 def wrapper(*args, **kwds):
568 # Size limited caching that tracks accesses by recency
569 nonlocal root, hits, misses, full
570 key = make_key(args, kwds, typed)
571 with lock:
572 link = cache_get(key)
573 if link is not None:
574 # Move the link to the front of the circular queue
575 link_prev, link_next, _key, result = link
576 link_prev[NEXT] = link_next
577 link_next[PREV] = link_prev
578 last = root[PREV]
579 last[NEXT] = root[PREV] = link
580 link[PREV] = last
581 link[NEXT] = root
Nick Coghlan234515a2010-11-30 06:19:46 +0000582 hits += 1
Raymond Hettinger4b779b32011-10-15 23:50:42 -0700583 return result
Raymond Hettingerd8080c02019-01-26 03:02:00 -0500584 misses += 1
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300585 result = user_function(*args, **kwds)
586 with lock:
587 if key in cache:
588 # Getting here means that this same key was added to the
589 # cache while the lock was released. Since the link
590 # update is already done, we need only return the
591 # computed result and update the count of misses.
592 pass
593 elif full:
594 # Use the old root to store the new key and result.
595 oldroot = root
596 oldroot[KEY] = key
597 oldroot[RESULT] = result
598 # Empty the oldest link and make it the new root.
599 # Keep a reference to the old key and old result to
600 # prevent their ref counts from going to zero during the
601 # update. That will prevent potentially arbitrary object
602 # clean-up code (i.e. __del__) from running while we're
603 # still adjusting the links.
604 root = oldroot[NEXT]
605 oldkey = root[KEY]
606 oldresult = root[RESULT]
607 root[KEY] = root[RESULT] = None
608 # Now update the cache dictionary.
609 del cache[oldkey]
610 # Save the potentially reentrant cache[key] assignment
611 # for last, after the root and links have been put in
612 # a consistent state.
613 cache[key] = oldroot
614 else:
615 # Put result in a new link at the front of the queue.
616 last = root[PREV]
617 link = [last, root, key, result]
618 last[NEXT] = root[PREV] = cache[key] = link
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800619 # Use the cache_len bound method instead of the len() function
Raymond Hettingeraf56e0e2016-12-16 13:57:40 -0800620 # which could potentially be wrapped in an lru_cache itself.
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800621 full = (cache_len() >= maxsize)
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300622 return result
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700623
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300624 def cache_info():
625 """Report cache statistics"""
626 with lock:
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800627 return _CacheInfo(hits, misses, maxsize, cache_len())
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700628
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300629 def cache_clear():
630 """Clear the cache and cache statistics"""
631 nonlocal hits, misses, full
632 with lock:
633 cache.clear()
634 root[:] = [root, root, None, None]
635 hits = misses = 0
636 full = False
Georg Brandl2e7346a2010-07-31 18:09:23 +0000637
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300638 wrapper.cache_info = cache_info
639 wrapper.cache_clear = cache_clear
Serhiy Storchakace2295d2015-10-24 09:51:53 +0300640 return wrapper
Nick Coghlan234515a2010-11-30 06:19:46 +0000641
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300642try:
643 from _functools import _lru_cache_wrapper
644except ImportError:
645 pass
Łukasz Langa6f692512013-06-05 12:20:24 +0200646
647
648################################################################################
649### singledispatch() - single-dispatch generic function decorator
650################################################################################
651
Łukasz Langa3720c772013-07-01 16:00:38 +0200652def _c3_merge(sequences):
653 """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
654
655 Adapted from http://www.python.org/download/releases/2.3/mro/.
656
657 """
658 result = []
659 while True:
660 sequences = [s for s in sequences if s] # purge empty sequences
661 if not sequences:
662 return result
663 for s1 in sequences: # find merge candidates among seq heads
664 candidate = s1[0]
665 for s2 in sequences:
666 if candidate in s2[1:]:
667 candidate = None
668 break # reject the current head, it appears later
669 else:
670 break
Yury Selivanov77a8cd62015-08-18 14:20:00 -0400671 if candidate is None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200672 raise RuntimeError("Inconsistent hierarchy")
673 result.append(candidate)
674 # remove the chosen candidate
675 for seq in sequences:
676 if seq[0] == candidate:
677 del seq[0]
678
679def _c3_mro(cls, abcs=None):
680 """Computes the method resolution order using extended C3 linearization.
681
682 If no *abcs* are given, the algorithm works exactly like the built-in C3
683 linearization used for method resolution.
684
685 If given, *abcs* is a list of abstract base classes that should be inserted
686 into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
687 result. The algorithm inserts ABCs where their functionality is introduced,
688 i.e. issubclass(cls, abc) returns True for the class itself but returns
689 False for all its direct base classes. Implicit ABCs for a given class
690 (either registered or inferred from the presence of a special method like
691 __len__) are inserted directly after the last ABC explicitly listed in the
692 MRO of said class. If two implicit ABCs end up next to each other in the
693 resulting MRO, their ordering depends on the order of types in *abcs*.
694
695 """
696 for i, base in enumerate(reversed(cls.__bases__)):
697 if hasattr(base, '__abstractmethods__'):
698 boundary = len(cls.__bases__) - i
699 break # Bases up to the last explicit ABC are considered first.
700 else:
701 boundary = 0
702 abcs = list(abcs) if abcs else []
703 explicit_bases = list(cls.__bases__[:boundary])
704 abstract_bases = []
705 other_bases = list(cls.__bases__[boundary:])
706 for base in abcs:
707 if issubclass(cls, base) and not any(
708 issubclass(b, base) for b in cls.__bases__
709 ):
710 # If *cls* is the class that introduces behaviour described by
711 # an ABC *base*, insert said ABC to its MRO.
712 abstract_bases.append(base)
713 for base in abstract_bases:
714 abcs.remove(base)
715 explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
716 abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
717 other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
718 return _c3_merge(
719 [[cls]] +
720 explicit_c3_mros + abstract_c3_mros + other_c3_mros +
721 [explicit_bases] + [abstract_bases] + [other_bases]
722 )
723
724def _compose_mro(cls, types):
725 """Calculates the method resolution order for a given class *cls*.
726
727 Includes relevant abstract base classes (with their respective bases) from
728 the *types* iterable. Uses a modified C3 linearization algorithm.
Łukasz Langa6f692512013-06-05 12:20:24 +0200729
730 """
731 bases = set(cls.__mro__)
Łukasz Langa3720c772013-07-01 16:00:38 +0200732 # Remove entries which are already present in the __mro__ or unrelated.
733 def is_related(typ):
734 return (typ not in bases and hasattr(typ, '__mro__')
735 and issubclass(cls, typ))
736 types = [n for n in types if is_related(n)]
737 # Remove entries which are strict bases of other entries (they will end up
738 # in the MRO anyway.
739 def is_strict_base(typ):
740 for other in types:
741 if typ != other and typ in other.__mro__:
742 return True
743 return False
744 types = [n for n in types if not is_strict_base(n)]
745 # Subclasses of the ABCs in *types* which are also implemented by
746 # *cls* can be used to stabilize ABC ordering.
747 type_set = set(types)
748 mro = []
749 for typ in types:
750 found = []
751 for sub in typ.__subclasses__():
752 if sub not in bases and issubclass(cls, sub):
753 found.append([s for s in sub.__mro__ if s in type_set])
754 if not found:
755 mro.append(typ)
756 continue
757 # Favor subclasses with the biggest number of useful bases
758 found.sort(key=len, reverse=True)
759 for sub in found:
760 for subcls in sub:
761 if subcls not in mro:
762 mro.append(subcls)
763 return _c3_mro(cls, abcs=mro)
Łukasz Langa6f692512013-06-05 12:20:24 +0200764
765def _find_impl(cls, registry):
Łukasz Langa3720c772013-07-01 16:00:38 +0200766 """Returns the best matching implementation from *registry* for type *cls*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200767
Łukasz Langa3720c772013-07-01 16:00:38 +0200768 Where there is no registered implementation for a specific type, its method
769 resolution order is used to find a more generic implementation.
770
771 Note: if *registry* does not contain an implementation for the base
772 *object* type, this function may return None.
Łukasz Langa6f692512013-06-05 12:20:24 +0200773
774 """
775 mro = _compose_mro(cls, registry.keys())
776 match = None
777 for t in mro:
778 if match is not None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200779 # If *match* is an implicit ABC but there is another unrelated,
780 # equally matching implicit ABC, refuse the temptation to guess.
781 if (t in registry and t not in cls.__mro__
782 and match not in cls.__mro__
783 and not issubclass(match, t)):
Łukasz Langa6f692512013-06-05 12:20:24 +0200784 raise RuntimeError("Ambiguous dispatch: {} or {}".format(
785 match, t))
786 break
787 if t in registry:
788 match = t
789 return registry.get(match)
790
791def singledispatch(func):
792 """Single-dispatch generic function decorator.
793
794 Transforms a function into a generic function, which can have different
795 behaviours depending upon the type of its first argument. The decorated
796 function acts as the default implementation, and additional
Łukasz Langa3720c772013-07-01 16:00:38 +0200797 implementations can be registered using the register() attribute of the
798 generic function.
Łukasz Langa6f692512013-06-05 12:20:24 +0200799 """
INADA Naoki9811e802017-09-30 16:13:02 +0900800 # There are many programs that use functools without singledispatch, so we
801 # trade-off making singledispatch marginally slower for the benefit of
802 # making start-up of such applications slightly faster.
803 import types, weakref
804
Łukasz Langa6f692512013-06-05 12:20:24 +0200805 registry = {}
INADA Naoki9811e802017-09-30 16:13:02 +0900806 dispatch_cache = weakref.WeakKeyDictionary()
Łukasz Langa6f692512013-06-05 12:20:24 +0200807 cache_token = None
808
Łukasz Langa3720c772013-07-01 16:00:38 +0200809 def dispatch(cls):
810 """generic_func.dispatch(cls) -> <function implementation>
Łukasz Langa6f692512013-06-05 12:20:24 +0200811
812 Runs the dispatch algorithm to return the best available implementation
Łukasz Langa3720c772013-07-01 16:00:38 +0200813 for the given *cls* registered on *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200814
815 """
816 nonlocal cache_token
817 if cache_token is not None:
818 current_token = get_cache_token()
819 if cache_token != current_token:
820 dispatch_cache.clear()
821 cache_token = current_token
822 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200823 impl = dispatch_cache[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200824 except KeyError:
825 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200826 impl = registry[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200827 except KeyError:
Łukasz Langa3720c772013-07-01 16:00:38 +0200828 impl = _find_impl(cls, registry)
829 dispatch_cache[cls] = impl
Łukasz Langa6f692512013-06-05 12:20:24 +0200830 return impl
831
Łukasz Langa3720c772013-07-01 16:00:38 +0200832 def register(cls, func=None):
833 """generic_func.register(cls, func) -> func
Łukasz Langa6f692512013-06-05 12:20:24 +0200834
Łukasz Langa3720c772013-07-01 16:00:38 +0200835 Registers a new implementation for the given *cls* on a *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200836
837 """
838 nonlocal cache_token
839 if func is None:
Łukasz Langae5697532017-12-11 13:56:31 -0800840 if isinstance(cls, type):
841 return lambda f: register(cls, f)
842 ann = getattr(cls, '__annotations__', {})
843 if not ann:
844 raise TypeError(
845 f"Invalid first argument to `register()`: {cls!r}. "
846 f"Use either `@register(some_class)` or plain `@register` "
847 f"on an annotated function."
848 )
849 func = cls
850
851 # only import typing if annotation parsing is necessary
852 from typing import get_type_hints
853 argname, cls = next(iter(get_type_hints(func).items()))
854 assert isinstance(cls, type), (
855 f"Invalid annotation for {argname!r}. {cls!r} is not a class."
856 )
Łukasz Langa3720c772013-07-01 16:00:38 +0200857 registry[cls] = func
858 if cache_token is None and hasattr(cls, '__abstractmethods__'):
Łukasz Langa6f692512013-06-05 12:20:24 +0200859 cache_token = get_cache_token()
860 dispatch_cache.clear()
861 return func
862
863 def wrapper(*args, **kw):
Dong-hee Na445f1b32018-07-10 16:26:36 +0900864 if not args:
865 raise TypeError(f'{funcname} requires at least '
866 '1 positional argument')
867
Łukasz Langa6f692512013-06-05 12:20:24 +0200868 return dispatch(args[0].__class__)(*args, **kw)
869
Dong-hee Na445f1b32018-07-10 16:26:36 +0900870 funcname = getattr(func, '__name__', 'singledispatch function')
Łukasz Langa6f692512013-06-05 12:20:24 +0200871 registry[object] = func
872 wrapper.register = register
873 wrapper.dispatch = dispatch
INADA Naoki9811e802017-09-30 16:13:02 +0900874 wrapper.registry = types.MappingProxyType(registry)
Łukasz Langa6f692512013-06-05 12:20:24 +0200875 wrapper._clear_cache = dispatch_cache.clear
876 update_wrapper(wrapper, func)
877 return wrapper
Ethan Smithc6512752018-05-26 16:38:33 -0400878
879
880# Descriptor version
881class singledispatchmethod:
882 """Single-dispatch generic method descriptor.
883
884 Supports wrapping existing descriptors and handles non-descriptor
885 callables as instance methods.
886 """
887
888 def __init__(self, func):
889 if not callable(func) and not hasattr(func, "__get__"):
890 raise TypeError(f"{func!r} is not callable or a descriptor")
891
892 self.dispatcher = singledispatch(func)
893 self.func = func
894
895 def register(self, cls, method=None):
896 """generic_method.register(cls, func) -> func
897
898 Registers a new implementation for the given *cls* on a *generic_method*.
899 """
900 return self.dispatcher.register(cls, func=method)
901
902 def __get__(self, obj, cls):
903 def _method(*args, **kwargs):
904 method = self.dispatcher.dispatch(args[0].__class__)
905 return method.__get__(obj, cls)(*args, **kwargs)
906
907 _method.__isabstractmethod__ = self.__isabstractmethod__
908 _method.register = self.register
909 update_wrapper(_method, self.func)
910 return _method
911
912 @property
913 def __isabstractmethod__(self):
914 return getattr(self.func, '__isabstractmethod__', False)
Carl Meyerd658dea2018-08-28 01:11:56 -0600915
916
917################################################################################
918### cached_property() - computed once per instance, cached as attribute
919################################################################################
920
921_NOT_FOUND = object()
922
923
924class cached_property:
925 def __init__(self, func):
926 self.func = func
927 self.attrname = None
928 self.__doc__ = func.__doc__
929 self.lock = RLock()
930
931 def __set_name__(self, owner, name):
932 if self.attrname is None:
933 self.attrname = name
934 elif name != self.attrname:
935 raise TypeError(
936 "Cannot assign the same cached_property to two different names "
937 f"({self.attrname!r} and {name!r})."
938 )
939
940 def __get__(self, instance, owner):
941 if instance is None:
942 return self
943 if self.attrname is None:
944 raise TypeError(
945 "Cannot use cached_property instance without calling __set_name__ on it.")
946 try:
947 cache = instance.__dict__
948 except AttributeError: # not all objects have __dict__ (e.g. class defines slots)
949 msg = (
950 f"No '__dict__' attribute on {type(instance).__name__!r} "
951 f"instance to cache {self.attrname!r} property."
952 )
953 raise TypeError(msg) from None
954 val = cache.get(self.attrname, _NOT_FOUND)
955 if val is _NOT_FOUND:
956 with self.lock:
957 # check if another thread filled cache while we awaited lock
958 val = cache.get(self.attrname, _NOT_FOUND)
959 if val is _NOT_FOUND:
960 val = self.func(instance)
961 try:
962 cache[self.attrname] = val
963 except TypeError:
964 msg = (
965 f"The '__dict__' attribute on {type(instance).__name__!r} instance "
966 f"does not support item assignment for caching {self.attrname!r} property."
967 )
968 raise TypeError(msg) from None
969 return val