blob: 1daa1d1775911afd4267330e989758e4456bc134 [file] [log] [blame]
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001"""functools.py - Tools for working with functions and callable objects
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00002"""
3# Python module wrapper for _functools C module
4# to allow utilities written in Python to be added
5# to the functools module.
Łukasz Langa6f692512013-06-05 12:20:24 +02006# Written by Nick Coghlan <ncoghlan at gmail.com>,
7# Raymond Hettinger <python at rcn.com>,
8# and Łukasz Langa <lukasz at langa.pl>.
9# Copyright (C) 2006-2013 Python Software Foundation.
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000010# See C source code for _functools credits/copyright
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000011
Georg Brandl2e7346a2010-07-31 18:09:23 +000012__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
Łukasz Langa6f692512013-06-05 12:20:24 +020013 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
Nick Coghlan3daaf5f2013-11-04 23:32:16 +100014 'partialmethod', 'singledispatch']
Georg Brandl2e7346a2010-07-31 18:09:23 +000015
Antoine Pitroub5b37142012-11-13 21:35:40 +010016try:
17 from _functools import reduce
Brett Cannoncd171c82013-07-04 17:43:24 -040018except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +010019 pass
Łukasz Langa6f692512013-06-05 12:20:24 +020020from abc import get_cache_token
Raymond Hettingerec0e9102012-03-16 01:16:31 -070021from collections import namedtuple
INADA Naoki9811e802017-09-30 16:13:02 +090022# import types, weakref # Deferred to single_dispatch()
Nick Coghlan457fc9a2016-09-10 20:00:02 +100023from reprlib import recursive_repr
Antoine Pitroua6a4dc82017-09-07 18:56:24 +020024from _thread import RLock
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000025
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070026
27################################################################################
28### update_wrapper() and wraps() decorator
29################################################################################
30
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000031# update_wrapper() and wraps() are tools to help write
32# wrapper functions that can handle naive introspection
33
Meador Ingeff7f64c2011-12-11 22:37:31 -060034WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
35 '__annotations__')
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000036WRAPPER_UPDATES = ('__dict__',)
37def update_wrapper(wrapper,
38 wrapped,
39 assigned = WRAPPER_ASSIGNMENTS,
40 updated = WRAPPER_UPDATES):
41 """Update a wrapper function to look like the wrapped function
42
43 wrapper is the function to be updated
44 wrapped is the original function
45 assigned is a tuple naming the attributes assigned directly
46 from the wrapped function to the wrapper function (defaults to
47 functools.WRAPPER_ASSIGNMENTS)
Thomas Wouters89f507f2006-12-13 04:49:30 +000048 updated is a tuple naming the attributes of the wrapper that
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000049 are updated with the corresponding attribute from the wrapped
50 function (defaults to functools.WRAPPER_UPDATES)
51 """
52 for attr in assigned:
Nick Coghlan98876832010-08-17 06:17:18 +000053 try:
54 value = getattr(wrapped, attr)
55 except AttributeError:
56 pass
57 else:
58 setattr(wrapper, attr, value)
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000059 for attr in updated:
Thomas Wouters89f507f2006-12-13 04:49:30 +000060 getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
Nick Coghlan24c05bc2013-07-15 21:13:08 +100061 # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
62 # from the wrapped function when updating __dict__
63 wrapper.__wrapped__ = wrapped
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000064 # Return the wrapper so this can be used as a decorator via partial()
65 return wrapper
66
67def wraps(wrapped,
68 assigned = WRAPPER_ASSIGNMENTS,
69 updated = WRAPPER_UPDATES):
70 """Decorator factory to apply update_wrapper() to a wrapper function
71
72 Returns a decorator that invokes update_wrapper() with the decorated
73 function as the wrapper argument and the arguments to wraps() as the
74 remaining arguments. Default arguments are as for update_wrapper().
75 This is a convenience function to simplify applying partial() to
76 update_wrapper().
77 """
78 return partial(update_wrapper, wrapped=wrapped,
79 assigned=assigned, updated=updated)
Raymond Hettingerc50846a2010-04-05 18:56:31 +000080
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070081
82################################################################################
83### total_ordering class decorator
84################################################################################
85
Raymond Hettinger0603d302015-01-05 21:52:10 -080086# The total ordering functions all invoke the root magic method directly
87# rather than using the corresponding operator. This avoids possible
88# infinite recursion that could occur when the operator dispatch logic
89# detects a NotImplemented result and then calls a reflected method.
Nick Coghlanf05d9812013-10-02 00:02:03 +100090
Raymond Hettingerffcd8492015-05-12 21:26:37 -070091def _gt_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -080092 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
93 op_result = self.__lt__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +100094 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -080095 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +100096 return not op_result and self != other
97
Raymond Hettingerffcd8492015-05-12 21:26:37 -070098def _le_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -080099 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
100 op_result = self.__lt__(other)
101 return op_result or self == other
102
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700103def _ge_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800104 'Return a >= b. Computed by @total_ordering from (not a < b).'
105 op_result = self.__lt__(other)
106 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800107 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800108 return not op_result
109
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700110def _ge_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800111 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
112 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000113 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800114 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000115 return not op_result or self == other
116
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700117def _lt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800118 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
119 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000120 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800121 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000122 return op_result and self != other
123
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700124def _gt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800125 'Return a > b. Computed by @total_ordering from (not a <= b).'
126 op_result = self.__le__(other)
127 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800128 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800129 return not op_result
130
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700131def _lt_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800132 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
133 op_result = self.__gt__(other)
134 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800135 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800136 return not op_result and self != other
137
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700138def _ge_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800139 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
140 op_result = self.__gt__(other)
141 return op_result or self == other
142
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700143def _le_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800144 'Return a <= b. Computed by @total_ordering from (not a > b).'
145 op_result = self.__gt__(other)
146 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800147 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800148 return not op_result
149
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700150def _le_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800151 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
152 op_result = self.__ge__(other)
153 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800154 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800155 return not op_result or self == other
156
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700157def _gt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800158 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
159 op_result = self.__ge__(other)
160 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800161 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800162 return op_result and self != other
163
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700164def _lt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800165 'Return a < b. Computed by @total_ordering from (not a >= b).'
166 op_result = self.__ge__(other)
167 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800168 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800169 return not op_result
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000170
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800171_convert = {
172 '__lt__': [('__gt__', _gt_from_lt),
173 ('__le__', _le_from_lt),
174 ('__ge__', _ge_from_lt)],
175 '__le__': [('__ge__', _ge_from_le),
176 ('__lt__', _lt_from_le),
177 ('__gt__', _gt_from_le)],
178 '__gt__': [('__lt__', _lt_from_gt),
179 ('__ge__', _ge_from_gt),
180 ('__le__', _le_from_gt)],
181 '__ge__': [('__le__', _le_from_ge),
182 ('__gt__', _gt_from_ge),
183 ('__lt__', _lt_from_ge)]
184}
185
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000186def total_ordering(cls):
Georg Brandle5a26732010-05-19 21:06:36 +0000187 """Class decorator that fills in missing ordering methods"""
Raymond Hettinger3255c632010-09-16 00:31:21 +0000188 # Find user-defined comparisons (not those inherited from object).
Raymond Hettinger15ce0be2017-09-05 09:40:44 -0700189 roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)}
Raymond Hettinger56de7e22010-04-10 16:59:03 +0000190 if not roots:
191 raise ValueError('must define at least one ordering operation: < > <= >=')
192 root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800193 for opname, opfunc in _convert[root]:
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000194 if opname not in roots:
195 opfunc.__name__ = opname
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000196 setattr(cls, opname, opfunc)
197 return cls
198
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700199
200################################################################################
201### cmp_to_key() function converter
202################################################################################
203
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000204def cmp_to_key(mycmp):
Georg Brandle5a26732010-05-19 21:06:36 +0000205 """Convert a cmp= function into a key= function"""
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000206 class K(object):
Raymond Hettingera0d1d962011-03-21 17:50:28 -0700207 __slots__ = ['obj']
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700208 def __init__(self, obj):
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000209 self.obj = obj
210 def __lt__(self, other):
211 return mycmp(self.obj, other.obj) < 0
212 def __gt__(self, other):
213 return mycmp(self.obj, other.obj) > 0
214 def __eq__(self, other):
215 return mycmp(self.obj, other.obj) == 0
216 def __le__(self, other):
217 return mycmp(self.obj, other.obj) <= 0
218 def __ge__(self, other):
219 return mycmp(self.obj, other.obj) >= 0
Raymond Hettinger003be522011-05-03 11:01:32 -0700220 __hash__ = None
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000221 return K
Georg Brandl2e7346a2010-07-31 18:09:23 +0000222
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700223try:
224 from _functools import cmp_to_key
Brett Cannoncd171c82013-07-04 17:43:24 -0400225except ImportError:
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700226 pass
227
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700228
229################################################################################
Antoine Pitroub5b37142012-11-13 21:35:40 +0100230### partial() argument application
231################################################################################
232
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000233# Purely functional, no descriptor behaviour
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000234class partial:
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000235 """New function with partial application of the given arguments
Antoine Pitroub5b37142012-11-13 21:35:40 +0100236 and keywords.
237 """
Alexander Belopolskye49af342015-03-01 15:08:17 -0500238
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000239 __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
240
241 def __new__(*args, **keywords):
242 if not args:
243 raise TypeError("descriptor '__new__' of partial needs an argument")
244 if len(args) < 2:
245 raise TypeError("type 'partial' takes at least one argument")
246 cls, func, *args = args
247 if not callable(func):
248 raise TypeError("the first argument must be callable")
249 args = tuple(args)
250
251 if hasattr(func, "func"):
252 args = func.args + args
253 tmpkw = func.keywords.copy()
254 tmpkw.update(keywords)
255 keywords = tmpkw
256 del tmpkw
257 func = func.func
258
259 self = super(partial, cls).__new__(cls)
260
261 self.func = func
262 self.args = args
263 self.keywords = keywords
264 return self
265
266 def __call__(*args, **keywords):
267 if not args:
268 raise TypeError("descriptor '__call__' of partial needs an argument")
269 self, *args = args
270 newkeywords = self.keywords.copy()
271 newkeywords.update(keywords)
272 return self.func(*self.args, *args, **newkeywords)
273
274 @recursive_repr()
275 def __repr__(self):
276 qualname = type(self).__qualname__
277 args = [repr(self.func)]
278 args.extend(repr(x) for x in self.args)
279 args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
280 if type(self).__module__ == "functools":
281 return f"functools.{qualname}({', '.join(args)})"
282 return f"{qualname}({', '.join(args)})"
283
284 def __reduce__(self):
285 return type(self), (self.func,), (self.func, self.args,
286 self.keywords or None, self.__dict__ or None)
287
288 def __setstate__(self, state):
289 if not isinstance(state, tuple):
290 raise TypeError("argument to __setstate__ must be a tuple")
291 if len(state) != 4:
292 raise TypeError(f"expected 4 items in state, got {len(state)}")
293 func, args, kwds, namespace = state
294 if (not callable(func) or not isinstance(args, tuple) or
295 (kwds is not None and not isinstance(kwds, dict)) or
296 (namespace is not None and not isinstance(namespace, dict))):
297 raise TypeError("invalid partial state")
298
299 args = tuple(args) # just in case it's a subclass
300 if kwds is None:
301 kwds = {}
302 elif type(kwds) is not dict: # XXX does it need to be *exactly* dict?
303 kwds = dict(kwds)
304 if namespace is None:
305 namespace = {}
306
307 self.__dict__ = namespace
308 self.func = func
309 self.args = args
310 self.keywords = kwds
Antoine Pitroub5b37142012-11-13 21:35:40 +0100311
312try:
313 from _functools import partial
Brett Cannoncd171c82013-07-04 17:43:24 -0400314except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +0100315 pass
316
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000317# Descriptor version
318class partialmethod(object):
319 """Method descriptor with partial application of the given arguments
320 and keywords.
321
322 Supports wrapping existing descriptors and handles non-descriptor
323 callables as instance methods.
324 """
325
Serhiy Storchakaa37f3562019-04-01 10:59:24 +0300326 def __init__(*args, **keywords):
327 if len(args) >= 2:
328 self, func, *args = args
329 elif not args:
330 raise TypeError("descriptor '__init__' of partialmethod "
331 "needs an argument")
332 elif 'func' in keywords:
333 func = keywords.pop('func')
334 self, *args = args
335 else:
336 raise TypeError("type 'partialmethod' takes at least one argument, "
337 "got %d" % (len(args)-1))
338 args = tuple(args)
339
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000340 if not callable(func) and not hasattr(func, "__get__"):
341 raise TypeError("{!r} is not callable or a descriptor"
342 .format(func))
343
344 # func could be a descriptor like classmethod which isn't callable,
345 # so we can't inherit from partial (it verifies func is callable)
346 if isinstance(func, partialmethod):
347 # flattening is mandatory in order to place cls/self before all
348 # other arguments
349 # it's also more efficient since only one function will be called
350 self.func = func.func
351 self.args = func.args + args
352 self.keywords = func.keywords.copy()
353 self.keywords.update(keywords)
354 else:
355 self.func = func
356 self.args = args
357 self.keywords = keywords
358
359 def __repr__(self):
360 args = ", ".join(map(repr, self.args))
361 keywords = ", ".join("{}={!r}".format(k, v)
362 for k, v in self.keywords.items())
363 format_string = "{module}.{cls}({func}, {args}, {keywords})"
364 return format_string.format(module=self.__class__.__module__,
Serhiy Storchaka521e5862014-07-22 15:00:37 +0300365 cls=self.__class__.__qualname__,
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000366 func=self.func,
367 args=args,
368 keywords=keywords)
369
370 def _make_unbound_method(self):
371 def _method(*args, **keywords):
372 call_keywords = self.keywords.copy()
373 call_keywords.update(keywords)
374 cls_or_self, *rest = args
375 call_args = (cls_or_self,) + self.args + tuple(rest)
376 return self.func(*call_args, **call_keywords)
377 _method.__isabstractmethod__ = self.__isabstractmethod__
Yury Selivanovda5fe4f2014-01-27 17:28:37 -0500378 _method._partialmethod = self
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000379 return _method
380
381 def __get__(self, obj, cls):
382 get = getattr(self.func, "__get__", None)
383 result = None
384 if get is not None:
385 new_func = get(obj, cls)
386 if new_func is not self.func:
387 # Assume __get__ returning something new indicates the
388 # creation of an appropriate callable
389 result = partial(new_func, *self.args, **self.keywords)
390 try:
391 result.__self__ = new_func.__self__
392 except AttributeError:
393 pass
394 if result is None:
395 # If the underlying descriptor didn't do anything, treat this
396 # like an instance method
397 result = self._make_unbound_method().__get__(obj, cls)
398 return result
399
400 @property
401 def __isabstractmethod__(self):
402 return getattr(self.func, "__isabstractmethod__", False)
403
Antoine Pitroub5b37142012-11-13 21:35:40 +0100404
405################################################################################
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700406### LRU Cache function decorator
407################################################################################
408
Raymond Hettingerdce583e2012-03-16 22:12:20 -0700409_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
Nick Coghlan234515a2010-11-30 06:19:46 +0000410
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700411class _HashedSeq(list):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700412 """ This class guarantees that hash() will be called no more than once
413 per element. This is important because the lru_cache() will hash
414 the key multiple times on a cache miss.
415
416 """
417
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700418 __slots__ = 'hashvalue'
419
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700420 def __init__(self, tup, hash=hash):
421 self[:] = tup
422 self.hashvalue = hash(tup)
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700423
424 def __hash__(self):
425 return self.hashvalue
426
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700427def _make_key(args, kwds, typed,
428 kwd_mark = (object(),),
Miss Islington (bot)b2b023c2019-01-26 00:23:40 -0800429 fasttypes = {int, str},
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800430 tuple=tuple, type=type, len=len):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700431 """Make a cache key from optionally typed positional and keyword arguments
432
433 The key is constructed in a way that is flat as possible rather than
434 as a nested structure that would take more memory.
435
436 If there is only a single argument and its data type is known to cache
437 its hash value, then that argument is returned without a wrapper. This
438 saves space and improves lookup speed.
439
440 """
Raymond Hettinger55037092017-09-04 17:47:53 -0700441 # All of code below relies on kwds preserving the order input by the user.
442 # Formerly, we sorted() the kwds before looping. The new way is *much*
443 # faster; however, it means that f(x=1, y=2) will now be treated as a
444 # distinct call from f(y=2, x=1) which will be cached separately.
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700445 key = args
446 if kwds:
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700447 key += kwd_mark
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800448 for item in kwds.items():
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700449 key += item
450 if typed:
451 key += tuple(type(v) for v in args)
452 if kwds:
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800453 key += tuple(type(v) for v in kwds.values())
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700454 elif len(key) == 1 and type(key[0]) in fasttypes:
455 return key[0]
456 return _HashedSeq(key)
457
Raymond Hettinger010ce322012-05-19 21:20:48 -0700458def lru_cache(maxsize=128, typed=False):
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000459 """Least-recently-used cache decorator.
Georg Brandl2e7346a2010-07-31 18:09:23 +0000460
Raymond Hettingerc79fb0e2010-12-01 03:45:41 +0000461 If *maxsize* is set to None, the LRU features are disabled and the cache
462 can grow without bound.
463
Raymond Hettingercd9fdfd2011-10-20 08:57:45 -0700464 If *typed* is True, arguments of different types will be cached separately.
465 For example, f(3.0) and f(3) will be treated as distinct calls with
466 distinct results.
467
Georg Brandl2e7346a2010-07-31 18:09:23 +0000468 Arguments to the cached function must be hashable.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000469
Raymond Hettinger7f7a5a72012-03-30 21:50:40 -0700470 View the cache statistics named tuple (hits, misses, maxsize, currsize)
471 with f.cache_info(). Clear the cache and statistics with f.cache_clear().
Raymond Hettinger00f2f972010-12-01 00:47:56 +0000472 Access the underlying function with f.__wrapped__.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000473
474 See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
Georg Brandl2e7346a2010-07-31 18:09:23 +0000475
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000476 """
Raymond Hettinger1ff50df2012-03-30 13:15:48 -0700477
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000478 # Users should only access the lru_cache through its public API:
Raymond Hettinger5e20bab2010-11-30 07:13:04 +0000479 # cache_info, cache_clear, and f.__wrapped__
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000480 # The internals of the lru_cache are encapsulated for thread safety and
481 # to allow the implementation to change (including a possible C version).
482
Raymond Hettinger4d588972014-08-12 12:44:52 -0700483 # Early detection of an erroneous call to @lru_cache without any arguments
484 # resulting in the inner function being passed to maxsize instead of an
Miss Islington (bot)b2b023c2019-01-26 00:23:40 -0800485 # integer or None. Negative maxsize is treated as 0.
486 if isinstance(maxsize, int):
487 if maxsize < 0:
488 maxsize = 0
489 elif maxsize is not None:
Raymond Hettinger4d588972014-08-12 12:44:52 -0700490 raise TypeError('Expected maxsize to be an integer or None')
491
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300492 def decorating_function(user_function):
493 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
494 return update_wrapper(wrapper, user_function)
495
496 return decorating_function
497
498def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700499 # Constants shared by all lru cache instances:
Raymond Hettingerb6b98c02012-04-29 18:09:02 -0700500 sentinel = object() # unique object used to signal cache misses
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700501 make_key = _make_key # build a key from the function arguments
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700502 PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
503
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300504 cache = {}
505 hits = misses = 0
506 full = False
507 cache_get = cache.get # bound method to lookup a key or return None
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800508 cache_len = cache.__len__ # get cache size without calling len()
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300509 lock = RLock() # because linkedlist updates aren't threadsafe
510 root = [] # root of the circular doubly linked list
511 root[:] = [root, root, None, None] # initialize by pointing to self
Raymond Hettinger6e8c8172012-03-16 16:53:05 -0700512
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300513 if maxsize == 0:
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700514
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300515 def wrapper(*args, **kwds):
Miss Islington (bot)533a9b42019-01-31 15:35:00 -0800516 # No caching -- just a statistics update
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300517 nonlocal misses
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300518 misses += 1
Miss Islington (bot)533a9b42019-01-31 15:35:00 -0800519 result = user_function(*args, **kwds)
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300520 return result
521
522 elif maxsize is None:
523
524 def wrapper(*args, **kwds):
525 # Simple caching without ordering or size limit
526 nonlocal hits, misses
527 key = make_key(args, kwds, typed)
528 result = cache_get(key, sentinel)
529 if result is not sentinel:
530 hits += 1
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700531 return result
Miss Islington (bot)533a9b42019-01-31 15:35:00 -0800532 misses += 1
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300533 result = user_function(*args, **kwds)
534 cache[key] = result
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300535 return result
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700536
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300537 else:
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700538
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300539 def wrapper(*args, **kwds):
540 # Size limited caching that tracks accesses by recency
541 nonlocal root, hits, misses, full
542 key = make_key(args, kwds, typed)
543 with lock:
544 link = cache_get(key)
545 if link is not None:
546 # Move the link to the front of the circular queue
547 link_prev, link_next, _key, result = link
548 link_prev[NEXT] = link_next
549 link_next[PREV] = link_prev
550 last = root[PREV]
551 last[NEXT] = root[PREV] = link
552 link[PREV] = last
553 link[NEXT] = root
Nick Coghlan234515a2010-11-30 06:19:46 +0000554 hits += 1
Raymond Hettinger4b779b32011-10-15 23:50:42 -0700555 return result
Miss Islington (bot)b2b023c2019-01-26 00:23:40 -0800556 misses += 1
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300557 result = user_function(*args, **kwds)
558 with lock:
559 if key in cache:
560 # Getting here means that this same key was added to the
561 # cache while the lock was released. Since the link
562 # update is already done, we need only return the
563 # computed result and update the count of misses.
564 pass
565 elif full:
566 # Use the old root to store the new key and result.
567 oldroot = root
568 oldroot[KEY] = key
569 oldroot[RESULT] = result
570 # Empty the oldest link and make it the new root.
571 # Keep a reference to the old key and old result to
572 # prevent their ref counts from going to zero during the
573 # update. That will prevent potentially arbitrary object
574 # clean-up code (i.e. __del__) from running while we're
575 # still adjusting the links.
576 root = oldroot[NEXT]
577 oldkey = root[KEY]
578 oldresult = root[RESULT]
579 root[KEY] = root[RESULT] = None
580 # Now update the cache dictionary.
581 del cache[oldkey]
582 # Save the potentially reentrant cache[key] assignment
583 # for last, after the root and links have been put in
584 # a consistent state.
585 cache[key] = oldroot
586 else:
587 # Put result in a new link at the front of the queue.
588 last = root[PREV]
589 link = [last, root, key, result]
590 last[NEXT] = root[PREV] = cache[key] = link
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800591 # Use the cache_len bound method instead of the len() function
Raymond Hettingeraf56e0e2016-12-16 13:57:40 -0800592 # which could potentially be wrapped in an lru_cache itself.
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800593 full = (cache_len() >= maxsize)
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300594 return result
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700595
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300596 def cache_info():
597 """Report cache statistics"""
598 with lock:
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800599 return _CacheInfo(hits, misses, maxsize, cache_len())
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700600
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300601 def cache_clear():
602 """Clear the cache and cache statistics"""
603 nonlocal hits, misses, full
604 with lock:
605 cache.clear()
606 root[:] = [root, root, None, None]
607 hits = misses = 0
608 full = False
Georg Brandl2e7346a2010-07-31 18:09:23 +0000609
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300610 wrapper.cache_info = cache_info
611 wrapper.cache_clear = cache_clear
Serhiy Storchakace2295d2015-10-24 09:51:53 +0300612 return wrapper
Nick Coghlan234515a2010-11-30 06:19:46 +0000613
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300614try:
615 from _functools import _lru_cache_wrapper
616except ImportError:
617 pass
Łukasz Langa6f692512013-06-05 12:20:24 +0200618
619
620################################################################################
621### singledispatch() - single-dispatch generic function decorator
622################################################################################
623
Łukasz Langa3720c772013-07-01 16:00:38 +0200624def _c3_merge(sequences):
625 """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
626
627 Adapted from http://www.python.org/download/releases/2.3/mro/.
628
629 """
630 result = []
631 while True:
632 sequences = [s for s in sequences if s] # purge empty sequences
633 if not sequences:
634 return result
635 for s1 in sequences: # find merge candidates among seq heads
636 candidate = s1[0]
637 for s2 in sequences:
638 if candidate in s2[1:]:
639 candidate = None
640 break # reject the current head, it appears later
641 else:
642 break
Yury Selivanov77a8cd62015-08-18 14:20:00 -0400643 if candidate is None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200644 raise RuntimeError("Inconsistent hierarchy")
645 result.append(candidate)
646 # remove the chosen candidate
647 for seq in sequences:
648 if seq[0] == candidate:
649 del seq[0]
650
651def _c3_mro(cls, abcs=None):
652 """Computes the method resolution order using extended C3 linearization.
653
654 If no *abcs* are given, the algorithm works exactly like the built-in C3
655 linearization used for method resolution.
656
657 If given, *abcs* is a list of abstract base classes that should be inserted
658 into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
659 result. The algorithm inserts ABCs where their functionality is introduced,
660 i.e. issubclass(cls, abc) returns True for the class itself but returns
661 False for all its direct base classes. Implicit ABCs for a given class
662 (either registered or inferred from the presence of a special method like
663 __len__) are inserted directly after the last ABC explicitly listed in the
664 MRO of said class. If two implicit ABCs end up next to each other in the
665 resulting MRO, their ordering depends on the order of types in *abcs*.
666
667 """
668 for i, base in enumerate(reversed(cls.__bases__)):
669 if hasattr(base, '__abstractmethods__'):
670 boundary = len(cls.__bases__) - i
671 break # Bases up to the last explicit ABC are considered first.
672 else:
673 boundary = 0
674 abcs = list(abcs) if abcs else []
675 explicit_bases = list(cls.__bases__[:boundary])
676 abstract_bases = []
677 other_bases = list(cls.__bases__[boundary:])
678 for base in abcs:
679 if issubclass(cls, base) and not any(
680 issubclass(b, base) for b in cls.__bases__
681 ):
682 # If *cls* is the class that introduces behaviour described by
683 # an ABC *base*, insert said ABC to its MRO.
684 abstract_bases.append(base)
685 for base in abstract_bases:
686 abcs.remove(base)
687 explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
688 abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
689 other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
690 return _c3_merge(
691 [[cls]] +
692 explicit_c3_mros + abstract_c3_mros + other_c3_mros +
693 [explicit_bases] + [abstract_bases] + [other_bases]
694 )
695
696def _compose_mro(cls, types):
697 """Calculates the method resolution order for a given class *cls*.
698
699 Includes relevant abstract base classes (with their respective bases) from
700 the *types* iterable. Uses a modified C3 linearization algorithm.
Łukasz Langa6f692512013-06-05 12:20:24 +0200701
702 """
703 bases = set(cls.__mro__)
Łukasz Langa3720c772013-07-01 16:00:38 +0200704 # Remove entries which are already present in the __mro__ or unrelated.
705 def is_related(typ):
706 return (typ not in bases and hasattr(typ, '__mro__')
707 and issubclass(cls, typ))
708 types = [n for n in types if is_related(n)]
709 # Remove entries which are strict bases of other entries (they will end up
710 # in the MRO anyway.
711 def is_strict_base(typ):
712 for other in types:
713 if typ != other and typ in other.__mro__:
714 return True
715 return False
716 types = [n for n in types if not is_strict_base(n)]
717 # Subclasses of the ABCs in *types* which are also implemented by
718 # *cls* can be used to stabilize ABC ordering.
719 type_set = set(types)
720 mro = []
721 for typ in types:
722 found = []
723 for sub in typ.__subclasses__():
724 if sub not in bases and issubclass(cls, sub):
725 found.append([s for s in sub.__mro__ if s in type_set])
726 if not found:
727 mro.append(typ)
728 continue
729 # Favor subclasses with the biggest number of useful bases
730 found.sort(key=len, reverse=True)
731 for sub in found:
732 for subcls in sub:
733 if subcls not in mro:
734 mro.append(subcls)
735 return _c3_mro(cls, abcs=mro)
Łukasz Langa6f692512013-06-05 12:20:24 +0200736
737def _find_impl(cls, registry):
Łukasz Langa3720c772013-07-01 16:00:38 +0200738 """Returns the best matching implementation from *registry* for type *cls*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200739
Łukasz Langa3720c772013-07-01 16:00:38 +0200740 Where there is no registered implementation for a specific type, its method
741 resolution order is used to find a more generic implementation.
742
743 Note: if *registry* does not contain an implementation for the base
744 *object* type, this function may return None.
Łukasz Langa6f692512013-06-05 12:20:24 +0200745
746 """
747 mro = _compose_mro(cls, registry.keys())
748 match = None
749 for t in mro:
750 if match is not None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200751 # If *match* is an implicit ABC but there is another unrelated,
752 # equally matching implicit ABC, refuse the temptation to guess.
753 if (t in registry and t not in cls.__mro__
754 and match not in cls.__mro__
755 and not issubclass(match, t)):
Łukasz Langa6f692512013-06-05 12:20:24 +0200756 raise RuntimeError("Ambiguous dispatch: {} or {}".format(
757 match, t))
758 break
759 if t in registry:
760 match = t
761 return registry.get(match)
762
763def singledispatch(func):
764 """Single-dispatch generic function decorator.
765
766 Transforms a function into a generic function, which can have different
767 behaviours depending upon the type of its first argument. The decorated
768 function acts as the default implementation, and additional
Łukasz Langa3720c772013-07-01 16:00:38 +0200769 implementations can be registered using the register() attribute of the
770 generic function.
Łukasz Langa6f692512013-06-05 12:20:24 +0200771 """
INADA Naoki9811e802017-09-30 16:13:02 +0900772 # There are many programs that use functools without singledispatch, so we
773 # trade-off making singledispatch marginally slower for the benefit of
774 # making start-up of such applications slightly faster.
775 import types, weakref
776
Łukasz Langa6f692512013-06-05 12:20:24 +0200777 registry = {}
INADA Naoki9811e802017-09-30 16:13:02 +0900778 dispatch_cache = weakref.WeakKeyDictionary()
Łukasz Langa6f692512013-06-05 12:20:24 +0200779 cache_token = None
780
Łukasz Langa3720c772013-07-01 16:00:38 +0200781 def dispatch(cls):
782 """generic_func.dispatch(cls) -> <function implementation>
Łukasz Langa6f692512013-06-05 12:20:24 +0200783
784 Runs the dispatch algorithm to return the best available implementation
Łukasz Langa3720c772013-07-01 16:00:38 +0200785 for the given *cls* registered on *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200786
787 """
788 nonlocal cache_token
789 if cache_token is not None:
790 current_token = get_cache_token()
791 if cache_token != current_token:
792 dispatch_cache.clear()
793 cache_token = current_token
794 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200795 impl = dispatch_cache[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200796 except KeyError:
797 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200798 impl = registry[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200799 except KeyError:
Łukasz Langa3720c772013-07-01 16:00:38 +0200800 impl = _find_impl(cls, registry)
801 dispatch_cache[cls] = impl
Łukasz Langa6f692512013-06-05 12:20:24 +0200802 return impl
803
Łukasz Langa3720c772013-07-01 16:00:38 +0200804 def register(cls, func=None):
805 """generic_func.register(cls, func) -> func
Łukasz Langa6f692512013-06-05 12:20:24 +0200806
Łukasz Langa3720c772013-07-01 16:00:38 +0200807 Registers a new implementation for the given *cls* on a *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200808
809 """
810 nonlocal cache_token
811 if func is None:
Łukasz Langae5697532017-12-11 13:56:31 -0800812 if isinstance(cls, type):
813 return lambda f: register(cls, f)
814 ann = getattr(cls, '__annotations__', {})
815 if not ann:
816 raise TypeError(
817 f"Invalid first argument to `register()`: {cls!r}. "
818 f"Use either `@register(some_class)` or plain `@register` "
819 f"on an annotated function."
820 )
821 func = cls
822
823 # only import typing if annotation parsing is necessary
824 from typing import get_type_hints
825 argname, cls = next(iter(get_type_hints(func).items()))
826 assert isinstance(cls, type), (
827 f"Invalid annotation for {argname!r}. {cls!r} is not a class."
828 )
Łukasz Langa3720c772013-07-01 16:00:38 +0200829 registry[cls] = func
830 if cache_token is None and hasattr(cls, '__abstractmethods__'):
Łukasz Langa6f692512013-06-05 12:20:24 +0200831 cache_token = get_cache_token()
832 dispatch_cache.clear()
833 return func
834
835 def wrapper(*args, **kw):
Miss Islington (bot)df9f6332018-07-10 00:48:57 -0700836 if not args:
837 raise TypeError(f'{funcname} requires at least '
838 '1 positional argument')
839
Łukasz Langa6f692512013-06-05 12:20:24 +0200840 return dispatch(args[0].__class__)(*args, **kw)
841
Miss Islington (bot)df9f6332018-07-10 00:48:57 -0700842 funcname = getattr(func, '__name__', 'singledispatch function')
Łukasz Langa6f692512013-06-05 12:20:24 +0200843 registry[object] = func
844 wrapper.register = register
845 wrapper.dispatch = dispatch
INADA Naoki9811e802017-09-30 16:13:02 +0900846 wrapper.registry = types.MappingProxyType(registry)
Łukasz Langa6f692512013-06-05 12:20:24 +0200847 wrapper._clear_cache = dispatch_cache.clear
848 update_wrapper(wrapper, func)
849 return wrapper