Issue #7105: weak dict iterators are fragile because of unpredictable GC runs
Backport the fix from pyton 3.x for this issue.
diff --git a/Lib/weakref.py b/Lib/weakref.py
index 88c60e7..787c885 100644
--- a/Lib/weakref.py
+++ b/Lib/weakref.py
@@ -20,7 +20,7 @@
ProxyType,
ReferenceType)
-from _weakrefset import WeakSet
+from _weakrefset import WeakSet, _IterationGuard
from exceptions import ReferenceError
@@ -48,10 +48,24 @@
def remove(wr, selfref=ref(self)):
self = selfref()
if self is not None:
- del self.data[wr.key]
+ if self._iterating:
+ self._pending_removals.append(wr.key)
+ else:
+ del self.data[wr.key]
self._remove = remove
+ # A list of keys to be removed
+ self._pending_removals = []
+ self._iterating = set()
UserDict.UserDict.__init__(self, *args, **kw)
+ def _commit_removals(self):
+ l = self._pending_removals
+ d = self.data
+ # We shouldn't encounter any KeyError, because this method should
+ # always be called *before* mutating the dict.
+ while l:
+ del d[l.pop()]
+
def __getitem__(self, key):
o = self.data[key]()
if o is None:
@@ -59,6 +73,11 @@
else:
return o
+ def __delitem__(self, key):
+ if self._pending_removals:
+ self._commit_removals()
+ del self.data[key]
+
def __contains__(self, key):
try:
o = self.data[key]()
@@ -77,8 +96,15 @@
return "<WeakValueDictionary at %s>" % id(self)
def __setitem__(self, key, value):
+ if self._pending_removals:
+ self._commit_removals()
self.data[key] = KeyedRef(value, self._remove, key)
+ def clear(self):
+ if self._pending_removals:
+ self._commit_removals()
+ self.data.clear()
+
def copy(self):
new = WeakValueDictionary()
for key, wr in self.data.items():
@@ -120,16 +146,18 @@
return L
def iteritems(self):
- for wr in self.data.itervalues():
- value = wr()
- if value is not None:
- yield wr.key, value
+ with _IterationGuard(self):
+ for wr in self.data.itervalues():
+ value = wr()
+ if value is not None:
+ yield wr.key, value
def iterkeys(self):
- return self.data.iterkeys()
+ with _IterationGuard(self):
+ for k in self.data.iterkeys():
+ yield k
- def __iter__(self):
- return self.data.iterkeys()
+ __iter__ = iterkeys
def itervaluerefs(self):
"""Return an iterator that yields the weak references to the values.
@@ -141,15 +169,20 @@
keep the values around longer than needed.
"""
- return self.data.itervalues()
+ with _IterationGuard(self):
+ for wr in self.data.itervalues():
+ yield wr
def itervalues(self):
- for wr in self.data.itervalues():
- obj = wr()
- if obj is not None:
- yield obj
+ with _IterationGuard(self):
+ for wr in self.data.itervalues():
+ obj = wr()
+ if obj is not None:
+ yield obj
def popitem(self):
+ if self._pending_removals:
+ self._commit_removals()
while 1:
key, wr = self.data.popitem()
o = wr()
@@ -157,6 +190,8 @@
return key, o
def pop(self, key, *args):
+ if self._pending_removals:
+ self._commit_removals()
try:
o = self.data.pop(key)()
except KeyError:
@@ -172,12 +207,16 @@
try:
wr = self.data[key]
except KeyError:
+ if self._pending_removals:
+ self._commit_removals()
self.data[key] = KeyedRef(default, self._remove, key)
return default
else:
return wr()
def update(self, dict=None, **kwargs):
+ if self._pending_removals:
+ self._commit_removals()
d = self.data
if dict is not None:
if not hasattr(dict, "items"):
@@ -245,9 +284,29 @@
def remove(k, selfref=ref(self)):
self = selfref()
if self is not None:
- del self.data[k]
+ if self._iterating:
+ self._pending_removals.append(k)
+ else:
+ del self.data[k]
self._remove = remove
- if dict is not None: self.update(dict)
+ # A list of dead weakrefs (keys to be removed)
+ self._pending_removals = []
+ self._iterating = set()
+ if dict is not None:
+ self.update(dict)
+
+ def _commit_removals(self):
+ # NOTE: We don't need to call this method before mutating the dict,
+ # because a dead weakref never compares equal to a live weakref,
+ # even if they happened to refer to equal objects.
+ # However, it means keys may already have been removed.
+ l = self._pending_removals
+ d = self.data
+ while l:
+ try:
+ del d[l.pop()]
+ except KeyError:
+ pass
def __delitem__(self, key):
del self.data[ref(key)]
@@ -306,10 +365,11 @@
return L
def iteritems(self):
- for wr, value in self.data.iteritems():
- key = wr()
- if key is not None:
- yield key, value
+ with _IterationGuard(self):
+ for wr, value in self.data.iteritems():
+ key = wr()
+ if key is not None:
+ yield key, value
def iterkeyrefs(self):
"""Return an iterator that yields the weak references to the keys.
@@ -321,19 +381,23 @@
keep the keys around longer than needed.
"""
- return self.data.iterkeys()
+ with _IterationGuard(self):
+ for wr in self.data.iterkeys():
+ yield wr
def iterkeys(self):
- for wr in self.data.iterkeys():
- obj = wr()
- if obj is not None:
- yield obj
+ with _IterationGuard(self):
+ for wr in self.data.iterkeys():
+ obj = wr()
+ if obj is not None:
+ yield obj
- def __iter__(self):
- return self.iterkeys()
+ __iter__ = iterkeys
def itervalues(self):
- return self.data.itervalues()
+ with _IterationGuard(self):
+ for value in self.data.itervalues():
+ yield value
def keyrefs(self):
"""Return a list of weak references to the keys.