blob: 0873f207154b7daab8a39c8fadc9923977e509f2 [file] [log] [blame]
Thomas Wouters73e5a5b2006-06-08 15:35:45 +00001"""functools.py - Tools for working with functions and callable objects
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00002"""
3# Python module wrapper for _functools C module
4# to allow utilities written in Python to be added
5# to the functools module.
Łukasz Langa6f692512013-06-05 12:20:24 +02006# Written by Nick Coghlan <ncoghlan at gmail.com>,
7# Raymond Hettinger <python at rcn.com>,
8# and Łukasz Langa <lukasz at langa.pl>.
9# Copyright (C) 2006-2013 Python Software Foundation.
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000010# See C source code for _functools credits/copyright
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000011
Georg Brandl2e7346a2010-07-31 18:09:23 +000012__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
Łukasz Langa6f692512013-06-05 12:20:24 +020013 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
Nick Coghlan3daaf5f2013-11-04 23:32:16 +100014 'partialmethod', 'singledispatch']
Georg Brandl2e7346a2010-07-31 18:09:23 +000015
Antoine Pitroub5b37142012-11-13 21:35:40 +010016try:
17 from _functools import reduce
Brett Cannoncd171c82013-07-04 17:43:24 -040018except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +010019 pass
Łukasz Langa6f692512013-06-05 12:20:24 +020020from abc import get_cache_token
Raymond Hettingerec0e9102012-03-16 01:16:31 -070021from collections import namedtuple
Victor Stinner7fa767e2014-03-20 09:16:38 +010022from types import MappingProxyType
Łukasz Langa6f692512013-06-05 12:20:24 +020023from weakref import WeakKeyDictionary
Nick Coghlan457fc9a2016-09-10 20:00:02 +100024from reprlib import recursive_repr
Raymond Hettingercbe88132010-08-14 22:22:10 +000025try:
Raymond Hettingerfd541172013-03-01 03:47:57 -080026 from _thread import RLock
Serhiy Storchakaba9ac5b2015-05-20 10:33:40 +030027except ImportError:
Raymond Hettinger409f6632013-03-01 23:20:13 -080028 class RLock:
Raymond Hettingerf96b2b02013-03-08 21:11:55 -070029 'Dummy reentrant lock for builds without threads'
Raymond Hettinger409f6632013-03-01 23:20:13 -080030 def __enter__(self): pass
31 def __exit__(self, exctype, excinst, exctb): pass
Thomas Wouters4d70c3d2006-06-08 14:42:34 +000032
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070033
34################################################################################
35### update_wrapper() and wraps() decorator
36################################################################################
37
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000038# update_wrapper() and wraps() are tools to help write
39# wrapper functions that can handle naive introspection
40
Meador Ingeff7f64c2011-12-11 22:37:31 -060041WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__',
42 '__annotations__')
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000043WRAPPER_UPDATES = ('__dict__',)
44def update_wrapper(wrapper,
45 wrapped,
46 assigned = WRAPPER_ASSIGNMENTS,
47 updated = WRAPPER_UPDATES):
48 """Update a wrapper function to look like the wrapped function
49
50 wrapper is the function to be updated
51 wrapped is the original function
52 assigned is a tuple naming the attributes assigned directly
53 from the wrapped function to the wrapper function (defaults to
54 functools.WRAPPER_ASSIGNMENTS)
Thomas Wouters89f507f2006-12-13 04:49:30 +000055 updated is a tuple naming the attributes of the wrapper that
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000056 are updated with the corresponding attribute from the wrapped
57 function (defaults to functools.WRAPPER_UPDATES)
58 """
59 for attr in assigned:
Nick Coghlan98876832010-08-17 06:17:18 +000060 try:
61 value = getattr(wrapped, attr)
62 except AttributeError:
63 pass
64 else:
65 setattr(wrapper, attr, value)
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000066 for attr in updated:
Thomas Wouters89f507f2006-12-13 04:49:30 +000067 getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
Nick Coghlan24c05bc2013-07-15 21:13:08 +100068 # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
69 # from the wrapped function when updating __dict__
70 wrapper.__wrapped__ = wrapped
Thomas Wouters73e5a5b2006-06-08 15:35:45 +000071 # Return the wrapper so this can be used as a decorator via partial()
72 return wrapper
73
74def wraps(wrapped,
75 assigned = WRAPPER_ASSIGNMENTS,
76 updated = WRAPPER_UPDATES):
77 """Decorator factory to apply update_wrapper() to a wrapper function
78
79 Returns a decorator that invokes update_wrapper() with the decorated
80 function as the wrapper argument and the arguments to wraps() as the
81 remaining arguments. Default arguments are as for update_wrapper().
82 This is a convenience function to simplify applying partial() to
83 update_wrapper().
84 """
85 return partial(update_wrapper, wrapped=wrapped,
86 assigned=assigned, updated=updated)
Raymond Hettingerc50846a2010-04-05 18:56:31 +000087
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -070088
89################################################################################
90### total_ordering class decorator
91################################################################################
92
Raymond Hettinger0603d302015-01-05 21:52:10 -080093# The total ordering functions all invoke the root magic method directly
94# rather than using the corresponding operator. This avoids possible
95# infinite recursion that could occur when the operator dispatch logic
96# detects a NotImplemented result and then calls a reflected method.
Nick Coghlanf05d9812013-10-02 00:02:03 +100097
Raymond Hettingerffcd8492015-05-12 21:26:37 -070098def _gt_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -080099 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).'
100 op_result = self.__lt__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000101 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800102 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000103 return not op_result and self != other
104
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700105def _le_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800106 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).'
107 op_result = self.__lt__(other)
108 return op_result or self == other
109
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700110def _ge_from_lt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800111 'Return a >= b. Computed by @total_ordering from (not a < b).'
112 op_result = self.__lt__(other)
113 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800114 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800115 return not op_result
116
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700117def _ge_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800118 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).'
119 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000120 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800121 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000122 return not op_result or self == other
123
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700124def _lt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800125 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).'
126 op_result = self.__le__(other)
Nick Coghlanf05d9812013-10-02 00:02:03 +1000127 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800128 return op_result
Nick Coghlanf05d9812013-10-02 00:02:03 +1000129 return op_result and self != other
130
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700131def _gt_from_le(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800132 'Return a > b. Computed by @total_ordering from (not a <= b).'
133 op_result = self.__le__(other)
134 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800135 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800136 return not op_result
137
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700138def _lt_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800139 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).'
140 op_result = self.__gt__(other)
141 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800142 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800143 return not op_result and self != other
144
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700145def _ge_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800146 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).'
147 op_result = self.__gt__(other)
148 return op_result or self == other
149
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700150def _le_from_gt(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800151 'Return a <= b. Computed by @total_ordering from (not a > b).'
152 op_result = self.__gt__(other)
153 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800154 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800155 return not op_result
156
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700157def _le_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800158 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).'
159 op_result = self.__ge__(other)
160 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800161 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800162 return not op_result or self == other
163
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700164def _gt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800165 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).'
166 op_result = self.__ge__(other)
167 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800168 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800169 return op_result and self != other
170
Raymond Hettingerffcd8492015-05-12 21:26:37 -0700171def _lt_from_ge(self, other, NotImplemented=NotImplemented):
Raymond Hettinger0603d302015-01-05 21:52:10 -0800172 'Return a < b. Computed by @total_ordering from (not a >= b).'
173 op_result = self.__ge__(other)
174 if op_result is NotImplemented:
Raymond Hettingere5db8632015-01-06 22:16:10 -0800175 return op_result
Raymond Hettinger0603d302015-01-05 21:52:10 -0800176 return not op_result
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000177
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800178_convert = {
179 '__lt__': [('__gt__', _gt_from_lt),
180 ('__le__', _le_from_lt),
181 ('__ge__', _ge_from_lt)],
182 '__le__': [('__ge__', _ge_from_le),
183 ('__lt__', _lt_from_le),
184 ('__gt__', _gt_from_le)],
185 '__gt__': [('__lt__', _lt_from_gt),
186 ('__ge__', _ge_from_gt),
187 ('__le__', _le_from_gt)],
188 '__ge__': [('__le__', _le_from_ge),
189 ('__gt__', _gt_from_ge),
190 ('__lt__', _lt_from_ge)]
191}
192
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000193def total_ordering(cls):
Georg Brandle5a26732010-05-19 21:06:36 +0000194 """Class decorator that fills in missing ordering methods"""
Raymond Hettinger3255c632010-09-16 00:31:21 +0000195 # Find user-defined comparisons (not those inherited from object).
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800196 roots = [op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)]
Raymond Hettinger56de7e22010-04-10 16:59:03 +0000197 if not roots:
198 raise ValueError('must define at least one ordering operation: < > <= >=')
199 root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
Raymond Hettinger1a8ada82015-01-13 22:57:35 -0800200 for opname, opfunc in _convert[root]:
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000201 if opname not in roots:
202 opfunc.__name__ = opname
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000203 setattr(cls, opname, opfunc)
204 return cls
205
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700206
207################################################################################
208### cmp_to_key() function converter
209################################################################################
210
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000211def cmp_to_key(mycmp):
Georg Brandle5a26732010-05-19 21:06:36 +0000212 """Convert a cmp= function into a key= function"""
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000213 class K(object):
Raymond Hettingera0d1d962011-03-21 17:50:28 -0700214 __slots__ = ['obj']
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700215 def __init__(self, obj):
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000216 self.obj = obj
217 def __lt__(self, other):
218 return mycmp(self.obj, other.obj) < 0
219 def __gt__(self, other):
220 return mycmp(self.obj, other.obj) > 0
221 def __eq__(self, other):
222 return mycmp(self.obj, other.obj) == 0
223 def __le__(self, other):
224 return mycmp(self.obj, other.obj) <= 0
225 def __ge__(self, other):
226 return mycmp(self.obj, other.obj) >= 0
Raymond Hettinger003be522011-05-03 11:01:32 -0700227 __hash__ = None
Raymond Hettingerc50846a2010-04-05 18:56:31 +0000228 return K
Georg Brandl2e7346a2010-07-31 18:09:23 +0000229
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700230try:
231 from _functools import cmp_to_key
Brett Cannoncd171c82013-07-04 17:43:24 -0400232except ImportError:
Raymond Hettinger7ab9e222011-04-05 02:33:54 -0700233 pass
234
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700235
236################################################################################
Antoine Pitroub5b37142012-11-13 21:35:40 +0100237### partial() argument application
238################################################################################
239
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000240# Purely functional, no descriptor behaviour
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000241class partial:
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000242 """New function with partial application of the given arguments
Antoine Pitroub5b37142012-11-13 21:35:40 +0100243 and keywords.
244 """
Alexander Belopolskye49af342015-03-01 15:08:17 -0500245
Nick Coghlan457fc9a2016-09-10 20:00:02 +1000246 __slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
247
248 def __new__(*args, **keywords):
249 if not args:
250 raise TypeError("descriptor '__new__' of partial needs an argument")
251 if len(args) < 2:
252 raise TypeError("type 'partial' takes at least one argument")
253 cls, func, *args = args
254 if not callable(func):
255 raise TypeError("the first argument must be callable")
256 args = tuple(args)
257
258 if hasattr(func, "func"):
259 args = func.args + args
260 tmpkw = func.keywords.copy()
261 tmpkw.update(keywords)
262 keywords = tmpkw
263 del tmpkw
264 func = func.func
265
266 self = super(partial, cls).__new__(cls)
267
268 self.func = func
269 self.args = args
270 self.keywords = keywords
271 return self
272
273 def __call__(*args, **keywords):
274 if not args:
275 raise TypeError("descriptor '__call__' of partial needs an argument")
276 self, *args = args
277 newkeywords = self.keywords.copy()
278 newkeywords.update(keywords)
279 return self.func(*self.args, *args, **newkeywords)
280
281 @recursive_repr()
282 def __repr__(self):
283 qualname = type(self).__qualname__
284 args = [repr(self.func)]
285 args.extend(repr(x) for x in self.args)
286 args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
287 if type(self).__module__ == "functools":
288 return f"functools.{qualname}({', '.join(args)})"
289 return f"{qualname}({', '.join(args)})"
290
291 def __reduce__(self):
292 return type(self), (self.func,), (self.func, self.args,
293 self.keywords or None, self.__dict__ or None)
294
295 def __setstate__(self, state):
296 if not isinstance(state, tuple):
297 raise TypeError("argument to __setstate__ must be a tuple")
298 if len(state) != 4:
299 raise TypeError(f"expected 4 items in state, got {len(state)}")
300 func, args, kwds, namespace = state
301 if (not callable(func) or not isinstance(args, tuple) or
302 (kwds is not None and not isinstance(kwds, dict)) or
303 (namespace is not None and not isinstance(namespace, dict))):
304 raise TypeError("invalid partial state")
305
306 args = tuple(args) # just in case it's a subclass
307 if kwds is None:
308 kwds = {}
309 elif type(kwds) is not dict: # XXX does it need to be *exactly* dict?
310 kwds = dict(kwds)
311 if namespace is None:
312 namespace = {}
313
314 self.__dict__ = namespace
315 self.func = func
316 self.args = args
317 self.keywords = kwds
Antoine Pitroub5b37142012-11-13 21:35:40 +0100318
319try:
320 from _functools import partial
Brett Cannoncd171c82013-07-04 17:43:24 -0400321except ImportError:
Antoine Pitroub5b37142012-11-13 21:35:40 +0100322 pass
323
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000324# Descriptor version
325class partialmethod(object):
326 """Method descriptor with partial application of the given arguments
327 and keywords.
328
329 Supports wrapping existing descriptors and handles non-descriptor
330 callables as instance methods.
331 """
332
333 def __init__(self, func, *args, **keywords):
334 if not callable(func) and not hasattr(func, "__get__"):
335 raise TypeError("{!r} is not callable or a descriptor"
336 .format(func))
337
338 # func could be a descriptor like classmethod which isn't callable,
339 # so we can't inherit from partial (it verifies func is callable)
340 if isinstance(func, partialmethod):
341 # flattening is mandatory in order to place cls/self before all
342 # other arguments
343 # it's also more efficient since only one function will be called
344 self.func = func.func
345 self.args = func.args + args
346 self.keywords = func.keywords.copy()
347 self.keywords.update(keywords)
348 else:
349 self.func = func
350 self.args = args
351 self.keywords = keywords
352
353 def __repr__(self):
354 args = ", ".join(map(repr, self.args))
355 keywords = ", ".join("{}={!r}".format(k, v)
356 for k, v in self.keywords.items())
357 format_string = "{module}.{cls}({func}, {args}, {keywords})"
358 return format_string.format(module=self.__class__.__module__,
Serhiy Storchaka521e5862014-07-22 15:00:37 +0300359 cls=self.__class__.__qualname__,
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000360 func=self.func,
361 args=args,
362 keywords=keywords)
363
364 def _make_unbound_method(self):
365 def _method(*args, **keywords):
366 call_keywords = self.keywords.copy()
367 call_keywords.update(keywords)
368 cls_or_self, *rest = args
369 call_args = (cls_or_self,) + self.args + tuple(rest)
370 return self.func(*call_args, **call_keywords)
371 _method.__isabstractmethod__ = self.__isabstractmethod__
Yury Selivanovda5fe4f2014-01-27 17:28:37 -0500372 _method._partialmethod = self
Nick Coghlanf4cb48a2013-11-03 16:41:46 +1000373 return _method
374
375 def __get__(self, obj, cls):
376 get = getattr(self.func, "__get__", None)
377 result = None
378 if get is not None:
379 new_func = get(obj, cls)
380 if new_func is not self.func:
381 # Assume __get__ returning something new indicates the
382 # creation of an appropriate callable
383 result = partial(new_func, *self.args, **self.keywords)
384 try:
385 result.__self__ = new_func.__self__
386 except AttributeError:
387 pass
388 if result is None:
389 # If the underlying descriptor didn't do anything, treat this
390 # like an instance method
391 result = self._make_unbound_method().__get__(obj, cls)
392 return result
393
394 @property
395 def __isabstractmethod__(self):
396 return getattr(self.func, "__isabstractmethod__", False)
397
Antoine Pitroub5b37142012-11-13 21:35:40 +0100398
399################################################################################
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700400### LRU Cache function decorator
401################################################################################
402
Raymond Hettingerdce583e2012-03-16 22:12:20 -0700403_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])
Nick Coghlan234515a2010-11-30 06:19:46 +0000404
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700405class _HashedSeq(list):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700406 """ This class guarantees that hash() will be called no more than once
407 per element. This is important because the lru_cache() will hash
408 the key multiple times on a cache miss.
409
410 """
411
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700412 __slots__ = 'hashvalue'
413
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700414 def __init__(self, tup, hash=hash):
415 self[:] = tup
416 self.hashvalue = hash(tup)
Raymond Hettinger9acbb602012-04-30 22:32:16 -0700417
418 def __hash__(self):
419 return self.hashvalue
420
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700421def _make_key(args, kwds, typed,
422 kwd_mark = (object(),),
423 fasttypes = {int, str, frozenset, type(None)},
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800424 tuple=tuple, type=type, len=len):
Raymond Hettingerf96b2b02013-03-08 21:11:55 -0700425 """Make a cache key from optionally typed positional and keyword arguments
426
427 The key is constructed in a way that is flat as possible rather than
428 as a nested structure that would take more memory.
429
430 If there is only a single argument and its data type is known to cache
431 its hash value, then that argument is returned without a wrapper. This
432 saves space and improves lookup speed.
433
434 """
Raymond Hettinger55037092017-09-04 17:47:53 -0700435 # All of code below relies on kwds preserving the order input by the user.
436 # Formerly, we sorted() the kwds before looping. The new way is *much*
437 # faster; however, it means that f(x=1, y=2) will now be treated as a
438 # distinct call from f(y=2, x=1) which will be cached separately.
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700439 key = args
440 if kwds:
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700441 key += kwd_mark
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800442 for item in kwds.items():
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700443 key += item
444 if typed:
445 key += tuple(type(v) for v in args)
446 if kwds:
Raymond Hettinger4ee39142017-01-08 17:28:20 -0800447 key += tuple(type(v) for v in kwds.values())
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700448 elif len(key) == 1 and type(key[0]) in fasttypes:
449 return key[0]
450 return _HashedSeq(key)
451
Raymond Hettinger010ce322012-05-19 21:20:48 -0700452def lru_cache(maxsize=128, typed=False):
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000453 """Least-recently-used cache decorator.
Georg Brandl2e7346a2010-07-31 18:09:23 +0000454
Raymond Hettingerc79fb0e2010-12-01 03:45:41 +0000455 If *maxsize* is set to None, the LRU features are disabled and the cache
456 can grow without bound.
457
Raymond Hettingercd9fdfd2011-10-20 08:57:45 -0700458 If *typed* is True, arguments of different types will be cached separately.
459 For example, f(3.0) and f(3) will be treated as distinct calls with
460 distinct results.
461
Georg Brandl2e7346a2010-07-31 18:09:23 +0000462 Arguments to the cached function must be hashable.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000463
Raymond Hettinger7f7a5a72012-03-30 21:50:40 -0700464 View the cache statistics named tuple (hits, misses, maxsize, currsize)
465 with f.cache_info(). Clear the cache and statistics with f.cache_clear().
Raymond Hettinger00f2f972010-12-01 00:47:56 +0000466 Access the underlying function with f.__wrapped__.
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000467
468 See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used
Georg Brandl2e7346a2010-07-31 18:09:23 +0000469
Benjamin Peterson1f594ad2010-08-08 13:17:07 +0000470 """
Raymond Hettinger1ff50df2012-03-30 13:15:48 -0700471
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000472 # Users should only access the lru_cache through its public API:
Raymond Hettinger5e20bab2010-11-30 07:13:04 +0000473 # cache_info, cache_clear, and f.__wrapped__
Raymond Hettinger5fa40c02010-11-25 08:11:57 +0000474 # The internals of the lru_cache are encapsulated for thread safety and
475 # to allow the implementation to change (including a possible C version).
476
Raymond Hettinger4d588972014-08-12 12:44:52 -0700477 # Early detection of an erroneous call to @lru_cache without any arguments
478 # resulting in the inner function being passed to maxsize instead of an
479 # integer or None.
480 if maxsize is not None and not isinstance(maxsize, int):
481 raise TypeError('Expected maxsize to be an integer or None')
482
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300483 def decorating_function(user_function):
484 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo)
485 return update_wrapper(wrapper, user_function)
486
487 return decorating_function
488
489def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo):
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700490 # Constants shared by all lru cache instances:
Raymond Hettingerb6b98c02012-04-29 18:09:02 -0700491 sentinel = object() # unique object used to signal cache misses
Raymond Hettinger0c9050c2012-06-04 00:21:14 -0700492 make_key = _make_key # build a key from the function arguments
Raymond Hettinger9f0ab9f2012-04-29 14:55:27 -0700493 PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
494
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300495 cache = {}
496 hits = misses = 0
497 full = False
498 cache_get = cache.get # bound method to lookup a key or return None
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800499 cache_len = cache.__len__ # get cache size without calling len()
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300500 lock = RLock() # because linkedlist updates aren't threadsafe
501 root = [] # root of the circular doubly linked list
502 root[:] = [root, root, None, None] # initialize by pointing to self
Raymond Hettinger6e8c8172012-03-16 16:53:05 -0700503
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300504 if maxsize == 0:
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700505
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300506 def wrapper(*args, **kwds):
507 # No caching -- just a statistics update after a successful call
508 nonlocal misses
509 result = user_function(*args, **kwds)
510 misses += 1
511 return result
512
513 elif maxsize is None:
514
515 def wrapper(*args, **kwds):
516 # Simple caching without ordering or size limit
517 nonlocal hits, misses
518 key = make_key(args, kwds, typed)
519 result = cache_get(key, sentinel)
520 if result is not sentinel:
521 hits += 1
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700522 return result
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300523 result = user_function(*args, **kwds)
524 cache[key] = result
525 misses += 1
526 return result
Raymond Hettinger7e0c5812012-03-17 15:10:24 -0700527
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300528 else:
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700529
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300530 def wrapper(*args, **kwds):
531 # Size limited caching that tracks accesses by recency
532 nonlocal root, hits, misses, full
533 key = make_key(args, kwds, typed)
534 with lock:
535 link = cache_get(key)
536 if link is not None:
537 # Move the link to the front of the circular queue
538 link_prev, link_next, _key, result = link
539 link_prev[NEXT] = link_next
540 link_next[PREV] = link_prev
541 last = root[PREV]
542 last[NEXT] = root[PREV] = link
543 link[PREV] = last
544 link[NEXT] = root
Nick Coghlan234515a2010-11-30 06:19:46 +0000545 hits += 1
Raymond Hettinger4b779b32011-10-15 23:50:42 -0700546 return result
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300547 result = user_function(*args, **kwds)
548 with lock:
549 if key in cache:
550 # Getting here means that this same key was added to the
551 # cache while the lock was released. Since the link
552 # update is already done, we need only return the
553 # computed result and update the count of misses.
554 pass
555 elif full:
556 # Use the old root to store the new key and result.
557 oldroot = root
558 oldroot[KEY] = key
559 oldroot[RESULT] = result
560 # Empty the oldest link and make it the new root.
561 # Keep a reference to the old key and old result to
562 # prevent their ref counts from going to zero during the
563 # update. That will prevent potentially arbitrary object
564 # clean-up code (i.e. __del__) from running while we're
565 # still adjusting the links.
566 root = oldroot[NEXT]
567 oldkey = root[KEY]
568 oldresult = root[RESULT]
569 root[KEY] = root[RESULT] = None
570 # Now update the cache dictionary.
571 del cache[oldkey]
572 # Save the potentially reentrant cache[key] assignment
573 # for last, after the root and links have been put in
574 # a consistent state.
575 cache[key] = oldroot
576 else:
577 # Put result in a new link at the front of the queue.
578 last = root[PREV]
579 link = [last, root, key, result]
580 last[NEXT] = root[PREV] = cache[key] = link
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800581 # Use the cache_len bound method instead of the len() function
Raymond Hettingeraf56e0e2016-12-16 13:57:40 -0800582 # which could potentially be wrapped in an lru_cache itself.
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800583 full = (cache_len() >= maxsize)
Raymond Hettinger4b779b32011-10-15 23:50:42 -0700584 misses += 1
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300585 return result
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700586
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300587 def cache_info():
588 """Report cache statistics"""
589 with lock:
Raymond Hettingerb2d4b3d2016-12-16 14:59:37 -0800590 return _CacheInfo(hits, misses, maxsize, cache_len())
Raymond Hettingerbc8e81d2012-03-17 00:24:09 -0700591
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300592 def cache_clear():
593 """Clear the cache and cache statistics"""
594 nonlocal hits, misses, full
595 with lock:
596 cache.clear()
597 root[:] = [root, root, None, None]
598 hits = misses = 0
599 full = False
Georg Brandl2e7346a2010-07-31 18:09:23 +0000600
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300601 wrapper.cache_info = cache_info
602 wrapper.cache_clear = cache_clear
Serhiy Storchakace2295d2015-10-24 09:51:53 +0300603 return wrapper
Nick Coghlan234515a2010-11-30 06:19:46 +0000604
Serhiy Storchaka46c56112015-05-24 21:53:49 +0300605try:
606 from _functools import _lru_cache_wrapper
607except ImportError:
608 pass
Łukasz Langa6f692512013-06-05 12:20:24 +0200609
610
611################################################################################
612### singledispatch() - single-dispatch generic function decorator
613################################################################################
614
Łukasz Langa3720c772013-07-01 16:00:38 +0200615def _c3_merge(sequences):
616 """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
617
618 Adapted from http://www.python.org/download/releases/2.3/mro/.
619
620 """
621 result = []
622 while True:
623 sequences = [s for s in sequences if s] # purge empty sequences
624 if not sequences:
625 return result
626 for s1 in sequences: # find merge candidates among seq heads
627 candidate = s1[0]
628 for s2 in sequences:
629 if candidate in s2[1:]:
630 candidate = None
631 break # reject the current head, it appears later
632 else:
633 break
Yury Selivanov77a8cd62015-08-18 14:20:00 -0400634 if candidate is None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200635 raise RuntimeError("Inconsistent hierarchy")
636 result.append(candidate)
637 # remove the chosen candidate
638 for seq in sequences:
639 if seq[0] == candidate:
640 del seq[0]
641
642def _c3_mro(cls, abcs=None):
643 """Computes the method resolution order using extended C3 linearization.
644
645 If no *abcs* are given, the algorithm works exactly like the built-in C3
646 linearization used for method resolution.
647
648 If given, *abcs* is a list of abstract base classes that should be inserted
649 into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
650 result. The algorithm inserts ABCs where their functionality is introduced,
651 i.e. issubclass(cls, abc) returns True for the class itself but returns
652 False for all its direct base classes. Implicit ABCs for a given class
653 (either registered or inferred from the presence of a special method like
654 __len__) are inserted directly after the last ABC explicitly listed in the
655 MRO of said class. If two implicit ABCs end up next to each other in the
656 resulting MRO, their ordering depends on the order of types in *abcs*.
657
658 """
659 for i, base in enumerate(reversed(cls.__bases__)):
660 if hasattr(base, '__abstractmethods__'):
661 boundary = len(cls.__bases__) - i
662 break # Bases up to the last explicit ABC are considered first.
663 else:
664 boundary = 0
665 abcs = list(abcs) if abcs else []
666 explicit_bases = list(cls.__bases__[:boundary])
667 abstract_bases = []
668 other_bases = list(cls.__bases__[boundary:])
669 for base in abcs:
670 if issubclass(cls, base) and not any(
671 issubclass(b, base) for b in cls.__bases__
672 ):
673 # If *cls* is the class that introduces behaviour described by
674 # an ABC *base*, insert said ABC to its MRO.
675 abstract_bases.append(base)
676 for base in abstract_bases:
677 abcs.remove(base)
678 explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
679 abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
680 other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
681 return _c3_merge(
682 [[cls]] +
683 explicit_c3_mros + abstract_c3_mros + other_c3_mros +
684 [explicit_bases] + [abstract_bases] + [other_bases]
685 )
686
687def _compose_mro(cls, types):
688 """Calculates the method resolution order for a given class *cls*.
689
690 Includes relevant abstract base classes (with their respective bases) from
691 the *types* iterable. Uses a modified C3 linearization algorithm.
Łukasz Langa6f692512013-06-05 12:20:24 +0200692
693 """
694 bases = set(cls.__mro__)
Łukasz Langa3720c772013-07-01 16:00:38 +0200695 # Remove entries which are already present in the __mro__ or unrelated.
696 def is_related(typ):
697 return (typ not in bases and hasattr(typ, '__mro__')
698 and issubclass(cls, typ))
699 types = [n for n in types if is_related(n)]
700 # Remove entries which are strict bases of other entries (they will end up
701 # in the MRO anyway.
702 def is_strict_base(typ):
703 for other in types:
704 if typ != other and typ in other.__mro__:
705 return True
706 return False
707 types = [n for n in types if not is_strict_base(n)]
708 # Subclasses of the ABCs in *types* which are also implemented by
709 # *cls* can be used to stabilize ABC ordering.
710 type_set = set(types)
711 mro = []
712 for typ in types:
713 found = []
714 for sub in typ.__subclasses__():
715 if sub not in bases and issubclass(cls, sub):
716 found.append([s for s in sub.__mro__ if s in type_set])
717 if not found:
718 mro.append(typ)
719 continue
720 # Favor subclasses with the biggest number of useful bases
721 found.sort(key=len, reverse=True)
722 for sub in found:
723 for subcls in sub:
724 if subcls not in mro:
725 mro.append(subcls)
726 return _c3_mro(cls, abcs=mro)
Łukasz Langa6f692512013-06-05 12:20:24 +0200727
728def _find_impl(cls, registry):
Łukasz Langa3720c772013-07-01 16:00:38 +0200729 """Returns the best matching implementation from *registry* for type *cls*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200730
Łukasz Langa3720c772013-07-01 16:00:38 +0200731 Where there is no registered implementation for a specific type, its method
732 resolution order is used to find a more generic implementation.
733
734 Note: if *registry* does not contain an implementation for the base
735 *object* type, this function may return None.
Łukasz Langa6f692512013-06-05 12:20:24 +0200736
737 """
738 mro = _compose_mro(cls, registry.keys())
739 match = None
740 for t in mro:
741 if match is not None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200742 # If *match* is an implicit ABC but there is another unrelated,
743 # equally matching implicit ABC, refuse the temptation to guess.
744 if (t in registry and t not in cls.__mro__
745 and match not in cls.__mro__
746 and not issubclass(match, t)):
Łukasz Langa6f692512013-06-05 12:20:24 +0200747 raise RuntimeError("Ambiguous dispatch: {} or {}".format(
748 match, t))
749 break
750 if t in registry:
751 match = t
752 return registry.get(match)
753
754def singledispatch(func):
755 """Single-dispatch generic function decorator.
756
757 Transforms a function into a generic function, which can have different
758 behaviours depending upon the type of its first argument. The decorated
759 function acts as the default implementation, and additional
Łukasz Langa3720c772013-07-01 16:00:38 +0200760 implementations can be registered using the register() attribute of the
761 generic function.
Łukasz Langa6f692512013-06-05 12:20:24 +0200762
763 """
764 registry = {}
765 dispatch_cache = WeakKeyDictionary()
766 cache_token = None
767
Łukasz Langa3720c772013-07-01 16:00:38 +0200768 def dispatch(cls):
769 """generic_func.dispatch(cls) -> <function implementation>
Łukasz Langa6f692512013-06-05 12:20:24 +0200770
771 Runs the dispatch algorithm to return the best available implementation
Łukasz Langa3720c772013-07-01 16:00:38 +0200772 for the given *cls* registered on *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200773
774 """
775 nonlocal cache_token
776 if cache_token is not None:
777 current_token = get_cache_token()
778 if cache_token != current_token:
779 dispatch_cache.clear()
780 cache_token = current_token
781 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200782 impl = dispatch_cache[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200783 except KeyError:
784 try:
Łukasz Langa3720c772013-07-01 16:00:38 +0200785 impl = registry[cls]
Łukasz Langa6f692512013-06-05 12:20:24 +0200786 except KeyError:
Łukasz Langa3720c772013-07-01 16:00:38 +0200787 impl = _find_impl(cls, registry)
788 dispatch_cache[cls] = impl
Łukasz Langa6f692512013-06-05 12:20:24 +0200789 return impl
790
Łukasz Langa3720c772013-07-01 16:00:38 +0200791 def register(cls, func=None):
792 """generic_func.register(cls, func) -> func
Łukasz Langa6f692512013-06-05 12:20:24 +0200793
Łukasz Langa3720c772013-07-01 16:00:38 +0200794 Registers a new implementation for the given *cls* on a *generic_func*.
Łukasz Langa6f692512013-06-05 12:20:24 +0200795
796 """
797 nonlocal cache_token
798 if func is None:
Łukasz Langa3720c772013-07-01 16:00:38 +0200799 return lambda f: register(cls, f)
800 registry[cls] = func
801 if cache_token is None and hasattr(cls, '__abstractmethods__'):
Łukasz Langa6f692512013-06-05 12:20:24 +0200802 cache_token = get_cache_token()
803 dispatch_cache.clear()
804 return func
805
806 def wrapper(*args, **kw):
807 return dispatch(args[0].__class__)(*args, **kw)
808
809 registry[object] = func
810 wrapper.register = register
811 wrapper.dispatch = dispatch
812 wrapper.registry = MappingProxyType(registry)
813 wrapper._clear_cache = dispatch_cache.clear
814 update_wrapper(wrapper, func)
815 return wrapper