Issue #14288: Serialization support for builtin iterators.
diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py
index f655c29..f185a82 100644
--- a/Lib/test/seq_tests.py
+++ b/Lib/test/seq_tests.py
@@ -4,6 +4,7 @@
import unittest
import sys
+import pickle
# Various iterables
# This is used for checking the constructor (here and in test_deque.py)
@@ -388,3 +389,9 @@
self.assertEqual(a.index(0, -4*sys.maxsize, 4*sys.maxsize), 2)
self.assertRaises(ValueError, a.index, 0, 4*sys.maxsize,-4*sys.maxsize)
self.assertRaises(ValueError, a.index, 2, 0, -10)
+
+ def test_pickle(self):
+ lst = self.type2test([4, 5, 6, 7])
+ lst2 = pickle.loads(pickle.dumps(lst))
+ self.assertEqual(lst2, lst)
+ self.assertNotEqual(id(lst2), id(lst))
diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
index 434e495..9385339 100755
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -285,6 +285,20 @@
self.assertEqual(a.x, b.x)
self.assertEqual(type(a), type(b))
+ def test_iterator_pickle(self):
+ data = array.array(self.typecode, self.example)
+ orgit = iter(data)
+ d = pickle.dumps(orgit)
+ it = pickle.loads(d)
+ self.assertEqual(type(orgit), type(it))
+ self.assertEqual(list(it), list(data))
+
+ if len(data):
+ it = pickle.loads(d)
+ next(it)
+ d = pickle.dumps(it)
+ self.assertEqual(list(it), list(data)[1:])
+
def test_insert(self):
a = array.array(self.typecode, self.example)
a.insert(0, self.example[0])
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index bfa5ee7..88bccd9 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -14,6 +14,7 @@
import traceback
from test.support import TESTFN, unlink, run_unittest, check_warnings
from operator import neg
+import pickle
try:
import pty, signal
except ImportError:
@@ -110,7 +111,30 @@
def __iter__(self):
raise RuntimeError
+def filter_char(arg):
+ return ord(arg) > ord("d")
+
+def map_char(arg):
+ return chr(ord(arg)+1)
+
class BuiltinTest(unittest.TestCase):
+ # Helper to check picklability
+ def check_iter_pickle(self, it, seq):
+ itorg = it
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(list(it), seq)
+
+ #test the iterator after dropping one from it
+ it = pickle.loads(d)
+ try:
+ next(it)
+ except StopIteration:
+ return
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(list(it), seq[1:])
def test_import(self):
__import__('sys')
@@ -566,6 +590,11 @@
self.assertEqual(list(filter(lambda x: x>=3, (1, 2, 3, 4))), [3, 4])
self.assertRaises(TypeError, list, filter(42, (1, 2)))
+ def test_filter_pickle(self):
+ f1 = filter(filter_char, "abcdeabcde")
+ f2 = filter(filter_char, "abcdeabcde")
+ self.check_iter_pickle(f1, list(f2))
+
def test_getattr(self):
self.assertTrue(getattr(sys, 'stdout') is sys.stdout)
self.assertRaises(TypeError, getattr, sys, 1)
@@ -759,6 +788,11 @@
raise RuntimeError
self.assertRaises(RuntimeError, list, map(badfunc, range(5)))
+ def test_map_pickle(self):
+ m1 = map(map_char, "Is this the real life?")
+ m2 = map(map_char, "Is this the real life?")
+ self.check_iter_pickle(m1, list(m2))
+
def test_max(self):
self.assertEqual(max('123123'), '3')
self.assertEqual(max(1, 2, 3), 3)
@@ -1300,6 +1334,13 @@
return i
self.assertRaises(ValueError, list, zip(BadSeq(), BadSeq()))
+ def test_zip_pickle(self):
+ a = (1, 2, 3)
+ b = (4, 5, 6)
+ t = [(1, 4), (2, 5), (3, 6)]
+ z1 = zip(a, b)
+ self.check_iter_pickle(z1, t)
+
def test_format(self):
# Test the basic machinery of the format() builtin. Don't test
# the specifics of the various formatters
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index 203fc5c..8ce6c22 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -518,6 +518,24 @@
q = pickle.loads(ps)
self.assertEqual(b, q)
+ def test_iterator_pickling(self):
+ for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0":
+ it = itorg = iter(self.type2test(b))
+ data = list(self.type2test(b))
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(list(it), data)
+
+ it = pickle.loads(d)
+ try:
+ next(it)
+ except StopIteration:
+ continue
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(list(it), data[1:])
+
def test_strip(self):
b = self.type2test(b'mississippi')
self.assertEqual(b.strip(b'i'), b'mississipp')
diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
index 0dcadeb..24125cd 100644
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -471,6 +471,19 @@
## self.assertNotEqual(id(d), id(e))
## self.assertEqual(id(e), id(e[-1]))
+ def test_iterator_pickle(self):
+ data = deque(range(200))
+ it = itorg = iter(data)
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(list(it), list(data))
+
+ it = pickle.loads(d)
+ next(it)
+ d = pickle.dumps(it)
+ self.assertEqual(list(it), list(data)[1:])
+
def test_deepcopy(self):
mut = [10]
d = deque([mut])
diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py
index 387dd32..b43e20d 100644
--- a/Lib/test/test_dict.py
+++ b/Lib/test/test_dict.py
@@ -2,7 +2,9 @@
from test import support
import collections, random, string
+import collections.abc
import gc, weakref
+import pickle
class DictTest(unittest.TestCase):
@@ -803,6 +805,58 @@
pass
self._tracked(MyDict())
+ def test_iterator_pickling(self):
+ data = {1:"a", 2:"b", 3:"c"}
+ it = iter(data)
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(sorted(it), sorted(data))
+
+ it = pickle.loads(d)
+ try:
+ drop = next(it)
+ except StopIteration:
+ return
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ del data[drop]
+ self.assertEqual(sorted(it), sorted(data))
+
+ def test_itemiterator_pickling(self):
+ data = {1:"a", 2:"b", 3:"c"}
+ # dictviews aren't picklable, only their iterators
+ itorg = iter(data.items())
+ d = pickle.dumps(itorg)
+ it = pickle.loads(d)
+ # note that the type of type of the unpickled iterator
+ # is not necessarily the same as the original. It is
+ # merely an object supporting the iterator protocol, yielding
+ # the same objects as the original one.
+ # self.assertEqual(type(itorg), type(it))
+ self.assertTrue(isinstance(it, collections.abc.Iterator))
+ self.assertEqual(dict(it), data)
+
+ it = pickle.loads(d)
+ drop = next(it)
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ del data[drop[0]]
+ self.assertEqual(dict(it), data)
+
+ def test_valuesiterator_pickling(self):
+ data = {1:"a", 2:"b", 3:"c"}
+ # data.values() isn't picklable, only its iterator
+ it = iter(data.values())
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(sorted(list(it)), sorted(list(data.values())))
+
+ it = pickle.loads(d)
+ drop = next(it)
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ values = list(it) + [drop]
+ self.assertEqual(sorted(values), sorted(list(data.values())))
from test import mapping_tests
diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py
index 095820b..2e904cf 100644
--- a/Lib/test/test_enumerate.py
+++ b/Lib/test/test_enumerate.py
@@ -1,5 +1,6 @@
import unittest
import sys
+import pickle
from test import support
@@ -61,7 +62,25 @@
def __iter__(self):
return self
-class EnumerateTestCase(unittest.TestCase):
+class PickleTest:
+ # Helper to check picklability
+ def check_pickle(self, itorg, seq):
+ d = pickle.dumps(itorg)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(list(it), seq)
+
+ it = pickle.loads(d)
+ try:
+ next(it)
+ except StopIteration:
+ self.assertFalse(seq[1:])
+ return
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(list(it), seq[1:])
+
+class EnumerateTestCase(unittest.TestCase, PickleTest):
enum = enumerate
seq, res = 'abc', [(0,'a'), (1,'b'), (2,'c')]
@@ -73,6 +92,9 @@
self.assertEqual(list(self.enum(self.seq)), self.res)
self.enum.__doc__
+ def test_pickle(self):
+ self.check_pickle(self.enum(self.seq), self.res)
+
def test_getitemseqn(self):
self.assertEqual(list(self.enum(G(self.seq))), self.res)
e = self.enum(G(''))
@@ -126,7 +148,7 @@
seq = range(10,20000,2)
res = list(zip(range(20000), seq))
-class TestReversed(unittest.TestCase):
+class TestReversed(unittest.TestCase, PickleTest):
def test_simple(self):
class A:
@@ -212,6 +234,10 @@
ngi = NoGetItem()
self.assertRaises(TypeError, reversed, ngi)
+ def test_pickle(self):
+ for data in 'abc', range(5), tuple(enumerate('abc')), range(1,17,5):
+ self.check_pickle(reversed(data), list(data)[::-1])
+
class EnumerateStartTestCase(EnumerateTestCase):
diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py
index e424303..b2f3e65 100644
--- a/Lib/test/test_iter.py
+++ b/Lib/test/test_iter.py
@@ -2,6 +2,8 @@
import unittest
from test.support import run_unittest, TESTFN, unlink, cpython_only
+import pickle
+import collections.abc
# Test result of triple loop (too big to inline)
TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2),
@@ -28,6 +30,8 @@
raise StopIteration
self.i = res + 1
return res
+ def __iter__(self):
+ return self
class IteratingSequenceClass:
def __init__(self, n):
@@ -49,7 +53,9 @@
class TestCase(unittest.TestCase):
# Helper to check that an iterator returns a given sequence
- def check_iterator(self, it, seq):
+ def check_iterator(self, it, seq, pickle=True):
+ if pickle:
+ self.check_pickle(it, seq)
res = []
while 1:
try:
@@ -60,12 +66,33 @@
self.assertEqual(res, seq)
# Helper to check that a for loop generates a given sequence
- def check_for_loop(self, expr, seq):
+ def check_for_loop(self, expr, seq, pickle=True):
+ if pickle:
+ self.check_pickle(iter(expr), seq)
res = []
for val in expr:
res.append(val)
self.assertEqual(res, seq)
+ # Helper to check picklability
+ def check_pickle(self, itorg, seq):
+ d = pickle.dumps(itorg)
+ it = pickle.loads(d)
+ # Cannot assert type equality because dict iterators unpickle as list
+ # iterators.
+ # self.assertEqual(type(itorg), type(it))
+ self.assertTrue(isinstance(it, collections.abc.Iterator))
+ self.assertEqual(list(it), seq)
+
+ it = pickle.loads(d)
+ try:
+ next(it)
+ except StopIteration:
+ return
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(list(it), seq[1:])
+
# Test basic use of iter() function
def test_iter_basic(self):
self.check_iterator(iter(range(10)), list(range(10)))
@@ -138,7 +165,7 @@
if i > 100:
raise IndexError # Emergency stop
return i
- self.check_iterator(iter(C(), 10), list(range(10)))
+ self.check_iterator(iter(C(), 10), list(range(10)), pickle=False)
# Test two-argument iter() with function
def test_iter_function(self):
@@ -146,7 +173,7 @@
i = state[0]
state[0] = i+1
return i
- self.check_iterator(iter(spam, 10), list(range(10)))
+ self.check_iterator(iter(spam, 10), list(range(10)), pickle=False)
# Test two-argument iter() with function that raises StopIteration
def test_iter_function_stop(self):
@@ -156,7 +183,7 @@
raise StopIteration
state[0] = i+1
return i
- self.check_iterator(iter(spam, 20), list(range(10)))
+ self.check_iterator(iter(spam, 20), list(range(10)), pickle=False)
# Test exception propagation through function iterator
def test_exception_function(self):
@@ -198,7 +225,7 @@
if i == 10:
raise StopIteration
return SequenceClass.__getitem__(self, i)
- self.check_for_loop(MySequenceClass(20), list(range(10)))
+ self.check_for_loop(MySequenceClass(20), list(range(10)), pickle=False)
# Test a big range
def test_iter_big_range(self):
@@ -237,8 +264,8 @@
f.close()
f = open(TESTFN, "r")
try:
- self.check_for_loop(f, ["0\n", "1\n", "2\n", "3\n", "4\n"])
- self.check_for_loop(f, [])
+ self.check_for_loop(f, ["0\n", "1\n", "2\n", "3\n", "4\n"], pickle=False)
+ self.check_for_loop(f, [], pickle=False)
finally:
f.close()
try:
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 957991c..90e85a9 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -37,6 +37,13 @@
'Test predicate'
return x%2==1
+def tupleize(*args):
+ return args
+
+def irange(n):
+ for i in range(n):
+ yield i
+
class StopNow:
'Class emulating an empty iterable.'
def __iter__(self):
@@ -55,8 +62,59 @@
'Factorial'
return prod(range(1, n+1))
+# root level methods for pickling ability
+def testR(r):
+ return r[0]
+
+def testR2(r):
+ return r[2]
+
+def underten(x):
+ return x<10
+
class TestBasicOps(unittest.TestCase):
+ def pickletest(self, it, stop=4, take=1, compare=None):
+ """Test that an iterator is the same after pickling, also when part-consumed"""
+ def expand(it, i=0):
+ # Recursively expand iterables, within sensible bounds
+ if i > 10:
+ raise RuntimeError("infinite recursion encountered")
+ if isinstance(it, str):
+ return it
+ try:
+ l = list(islice(it, stop))
+ except TypeError:
+ return it # can't expand it
+ return [expand(e, i+1) for e in l]
+
+ # Test the initial copy against the original
+ dump = pickle.dumps(it)
+ i2 = pickle.loads(dump)
+ self.assertEqual(type(it), type(i2))
+ a, b = expand(it), expand(i2)
+ self.assertEqual(a, b)
+ if compare:
+ c = expand(compare)
+ self.assertEqual(a, c)
+
+ # Take from the copy, and create another copy and compare them.
+ i3 = pickle.loads(dump)
+ took = 0
+ try:
+ for i in range(take):
+ next(i3)
+ took += 1
+ except StopIteration:
+ pass #in case there is less data than 'take'
+ dump = pickle.dumps(i3)
+ i4 = pickle.loads(dump)
+ a, b = expand(i3), expand(i4)
+ self.assertEqual(a, b)
+ if compare:
+ c = expand(compare[took:])
+ self.assertEqual(a, c);
+
def test_accumulate(self):
self.assertEqual(list(accumulate(range(10))), # one positional arg
[0, 1, 3, 6, 10, 15, 21, 28, 36, 45])
@@ -83,6 +141,7 @@
[2, 16, 144, 720, 5040, 0, 0, 0, 0, 0])
with self.assertRaises(TypeError):
list(accumulate(s, chr)) # unary-operation
+ self.pickletest(accumulate(range(10))) # test pickling
def test_chain(self):
@@ -106,14 +165,43 @@
self.assertEqual(take(4, chain.from_iterable(['abc', 'def'])), list('abcd'))
self.assertRaises(TypeError, list, chain.from_iterable([2, 3]))
+ def test_chain_reducible(self):
+ operators = [copy.deepcopy,
+ lambda s: pickle.loads(pickle.dumps(s))]
+ for oper in operators:
+ it = chain('abc', 'def')
+ self.assertEqual(list(oper(it)), list('abcdef'))
+ self.assertEqual(next(it), 'a')
+ self.assertEqual(list(oper(it)), list('bcdef'))
+
+ self.assertEqual(list(oper(chain(''))), [])
+ self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd'))
+ self.assertRaises(TypeError, list, oper(chain(2, 3)))
+ self.pickletest(chain('abc', 'def'), compare=list('abcdef'))
+
def test_combinations(self):
self.assertRaises(TypeError, combinations, 'abc') # missing r argument
self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments
self.assertRaises(TypeError, combinations, None) # pool is not iterable
self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative
- self.assertEqual(list(combinations('abc', 32)), []) # r > n
- self.assertEqual(list(combinations(range(4), 3)),
- [(0,1,2), (0,1,3), (0,2,3), (1,2,3)])
+
+ for op in (lambda a:a, lambda a:pickle.loads(pickle.dumps(a))):
+ self.assertEqual(list(op(combinations('abc', 32))), []) # r > n
+
+ self.assertEqual(list(op(combinations('ABCD', 2))),
+ [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+ testIntermediate = combinations('ABCD', 2)
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)),
+ [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')])
+
+ self.assertEqual(list(op(combinations(range(4), 3))),
+ [(0,1,2), (0,1,3), (0,2,3), (1,2,3)])
+ testIntermediate = combinations(range(4), 3)
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)),
+ [(0,1,3), (0,2,3), (1,2,3)])
+
def combinations1(iterable, r):
'Pure python version shown in the docs'
@@ -168,6 +256,9 @@
self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
+ self.pickletest(combinations(values, r)) # test pickling
+
+ # Test implementation detail: tuple re-use
@support.impl_detail("tuple reuse is specific to CPython")
def test_combinations_tuple_reuse(self):
self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)
@@ -179,8 +270,15 @@
self.assertRaises(TypeError, cwr, 'abc', 2, 1) # too many arguments
self.assertRaises(TypeError, cwr, None) # pool is not iterable
self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative
- self.assertEqual(list(cwr('ABC', 2)),
- [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+
+ for op in (lambda a:a, lambda a:pickle.loads(pickle.dumps(a))):
+ self.assertEqual(list(op(cwr('ABC', 2))),
+ [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+ testIntermediate = cwr('ABC', 2)
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)),
+ [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')])
+
def cwr1(iterable, r):
'Pure python version shown in the docs'
@@ -239,6 +337,10 @@
self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version
self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version
+ self.pickletest(cwr(values,r)) # test pickling
+
+ # Test implementation detail: tuple re-use
+
@support.impl_detail("tuple reuse is specific to CPython")
def test_combinations_with_replacement_tuple_reuse(self):
cwr = combinations_with_replacement
@@ -305,6 +407,8 @@
self.assertEqual(result, list(permutations(values, None))) # test r as None
self.assertEqual(result, list(permutations(values))) # test default r
+ self.pickletest(permutations(values, r)) # test pickling
+
@support.impl_detail("tuple resuse is CPython specific")
def test_permutations_tuple_reuse(self):
self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1)
@@ -359,6 +463,24 @@
self.assertRaises(TypeError, compress, range(6)) # too few args
self.assertRaises(TypeError, compress, range(6), None) # too many args
+ # check copy, deepcopy, pickle
+ for op in (lambda a:copy.copy(a), lambda a:copy.deepcopy(a), lambda a:pickle.loads(pickle.dumps(a))):
+ for data, selectors, result1, result2 in [
+ ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'),
+ ('ABCDEF', [0,0,0,0,0,0], '', ''),
+ ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'),
+ ('ABCDEF', [1,0,1], 'AC', 'C'),
+ ('ABC', [0,1,1,1,1,1], 'BC', 'C'),
+ ]:
+
+ self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1))
+ self.assertEqual(list(op(compress(data, selectors))), list(result1))
+ testIntermediate = compress(data, selectors)
+ if result1:
+ next(testIntermediate)
+ self.assertEqual(list(op(testIntermediate)), list(result2))
+
+
def test_count(self):
self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)])
self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)])
@@ -393,7 +515,7 @@
c = count(value)
self.assertEqual(next(copy.copy(c)), value)
self.assertEqual(next(copy.deepcopy(c)), value)
- self.assertEqual(next(pickle.loads(pickle.dumps(c))), value)
+ self.pickletest(count(value))
#check proper internal error handling for large "step' sizes
count(1, maxsize+5); sys.exc_info()
@@ -440,6 +562,7 @@
else:
r2 = ('count(%r, %r)' % (i, j)).replace('L', '')
self.assertEqual(r1, r2)
+ self.pickletest(count(i, j))
def test_cycle(self):
self.assertEqual(take(10, cycle('abc')), list('abcabcabca'))
@@ -448,6 +571,18 @@
self.assertRaises(TypeError, cycle, 5)
self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0])
+ # check copy, deepcopy, pickle
+ c = cycle('abc')
+ self.assertEqual(next(c), 'a')
+ #simple copy currently not supported, because __reduce__ returns
+ #an internal iterator
+ #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab'))
+ self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab'))
+ self.assertEqual(take(10, pickle.loads(pickle.dumps(c))), list('bcabcabcab'))
+ next(c)
+ self.assertEqual(take(10, pickle.loads(pickle.dumps(c))), list('cabcabcabc'))
+ self.pickletest(cycle('abc'))
+
def test_groupby(self):
# Check whether it accepts arguments correctly
self.assertEqual([], list(groupby([])))
@@ -466,18 +601,37 @@
dup.append(elem)
self.assertEqual(s, dup)
+ # Check normal pickled
+ dup = []
+ for k, g in pickle.loads(pickle.dumps(groupby(s, testR))):
+ for elem in g:
+ self.assertEqual(k, elem[0])
+ dup.append(elem)
+ self.assertEqual(s, dup)
+
# Check nested case
dup = []
- for k, g in groupby(s, lambda r:r[0]):
- for ik, ig in groupby(g, lambda r:r[2]):
+ for k, g in groupby(s, testR):
+ for ik, ig in groupby(g, testR2):
for elem in ig:
self.assertEqual(k, elem[0])
self.assertEqual(ik, elem[2])
dup.append(elem)
self.assertEqual(s, dup)
+ # Check nested and pickled
+ dup = []
+ for k, g in pickle.loads(pickle.dumps(groupby(s, testR))):
+ for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2))):
+ for elem in ig:
+ self.assertEqual(k, elem[0])
+ self.assertEqual(ik, elem[2])
+ dup.append(elem)
+ self.assertEqual(s, dup)
+
+
# Check case where inner iterator is not used
- keys = [k for k, g in groupby(s, lambda r:r[0])]
+ keys = [k for k, g in groupby(s, testR)]
expectedkeys = set([r[0] for r in s])
self.assertEqual(set(keys), expectedkeys)
self.assertEqual(len(keys), len(expectedkeys))
@@ -548,6 +702,20 @@
self.assertRaises(TypeError, filter, isEven, 3)
self.assertRaises(TypeError, next, filter(range(6), range(6)))
+ # check copy, deepcopy, pickle
+ ans = [0,2,4]
+
+ c = filter(isEven, range(6))
+ self.assertEqual(list(copy.copy(c)), ans)
+ c = filter(isEven, range(6))
+ self.assertEqual(list(copy.deepcopy(c)), ans)
+ c = filter(isEven, range(6))
+ self.assertEqual(list(pickle.loads(pickle.dumps(c))), ans)
+ next(c)
+ self.assertEqual(list(pickle.loads(pickle.dumps(c))), ans[1:])
+ c = filter(isEven, range(6))
+ self.pickletest(c)
+
def test_filterfalse(self):
self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5])
self.assertEqual(list(filterfalse(None, [0,1,0,2,0])), [0,0,0])
@@ -558,6 +726,7 @@
self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7)
self.assertRaises(TypeError, filterfalse, isEven, 3)
self.assertRaises(TypeError, next, filterfalse(range(6), range(6)))
+ self.pickletest(filterfalse(isEven, range(6)))
def test_zip(self):
# XXX This is rather silly now that builtin zip() calls zip()...
@@ -582,6 +751,23 @@
ids = list(map(id, list(zip('abc', 'def'))))
self.assertEqual(len(dict.fromkeys(ids)), len(ids))
+ # check copy, deepcopy, pickle
+ ans = [(x,y) for x, y in copy.copy(zip('abc',count()))]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+ ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+ ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count())))]
+ self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)])
+
+ testIntermediate = zip('abc',count())
+ next(testIntermediate)
+ ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate))]
+ self.assertEqual(ans, [('b', 1), ('c', 2)])
+
+ self.pickletest(zip('abc', count()))
+
def test_ziplongest(self):
for args in [
['abc', range(6)],
@@ -631,6 +817,12 @@
ids = list(map(id, list(zip_longest('abc', 'def'))))
self.assertEqual(len(dict.fromkeys(ids)), len(ids))
+ def test_zip_longest_pickling(self):
+ self.pickletest(zip_longest("abc", "def"))
+ self.pickletest(zip_longest("abc", "defgh"))
+ self.pickletest(zip_longest("abc", "defgh", fillvalue=1))
+ self.pickletest(zip_longest("", "defgh"))
+
def test_bug_7244(self):
class Repeater:
@@ -734,6 +926,20 @@
self.assertEqual(len(set(map(id, product('abc', 'def')))), 1)
self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1)
+ def test_product_pickling(self):
+ # check copy, deepcopy, pickle
+ for args, result in [
+ ([], [()]), # zero iterables
+ (['ab'], [('a',), ('b',)]), # one iterable
+ ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables
+ ([range(0), range(2), range(3)], []), # first iterable with zero length
+ ([range(2), range(0), range(3)], []), # middle iterable with zero length
+ ([range(2), range(3), range(0)], []), # last iterable with zero length
+ ]:
+ self.assertEqual(list(copy.copy(product(*args))), result)
+ self.assertEqual(list(copy.deepcopy(product(*args))), result)
+ self.pickletest(product(*args))
+
def test_repeat(self):
self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a'])
self.assertEqual(lzip(range(3),repeat('a')),
@@ -752,11 +958,16 @@
list(r)
self.assertEqual(repr(r), 'repeat((1+0j), 0)')
+ # check copy, deepcopy, pickle
+ c = repeat(object='a', times=10)
+ self.assertEqual(next(c), 'a')
+ self.assertEqual(take(2, copy.copy(c)), list('a' * 2))
+ self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2))
+ self.pickletest(repeat(object='a', times=10))
+
def test_map(self):
self.assertEqual(list(map(operator.pow, range(3), range(1,7))),
[0**1, 1**2, 2**3])
- def tupleize(*args):
- return args
self.assertEqual(list(map(tupleize, 'abc', range(5))),
[('a',0),('b',1),('c',2)])
self.assertEqual(list(map(tupleize, 'abc', count())),
@@ -771,6 +982,18 @@
self.assertRaises(ValueError, next, map(errfunc, [4], [5]))
self.assertRaises(TypeError, next, map(onearg, [4], [5]))
+ # check copy, deepcopy, pickle
+ ans = [('a',0),('b',1),('c',2)]
+
+ c = map(tupleize, 'abc', count())
+ self.assertEqual(list(copy.copy(c)), ans)
+
+ c = map(tupleize, 'abc', count())
+ self.assertEqual(list(copy.deepcopy(c)), ans)
+
+ c = map(tupleize, 'abc', count())
+ self.pickletest(c)
+
def test_starmap(self):
self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))),
[0**1, 1**2, 2**3])
@@ -785,6 +1008,18 @@
self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)]))
self.assertRaises(TypeError, next, starmap(onearg, [(4,5)]))
+ # check copy, deepcopy, pickle
+ ans = [0**1, 1**2, 2**3]
+
+ c = starmap(operator.pow, zip(range(3), range(1,7)))
+ self.assertEqual(list(copy.copy(c)), ans)
+
+ c = starmap(operator.pow, zip(range(3), range(1,7)))
+ self.assertEqual(list(copy.deepcopy(c)), ans)
+
+ c = starmap(operator.pow, zip(range(3), range(1,7)))
+ self.pickletest(c)
+
def test_islice(self):
for args in [ # islice(args) should agree with range(args)
(10, 20, 3),
@@ -817,17 +1052,18 @@
self.assertEqual(list(it), list(range(3, 10)))
# Test invalid arguments
- self.assertRaises(TypeError, islice, range(10))
- self.assertRaises(TypeError, islice, range(10), 1, 2, 3, 4)
- self.assertRaises(ValueError, islice, range(10), -5, 10, 1)
- self.assertRaises(ValueError, islice, range(10), 1, -5, -1)
- self.assertRaises(ValueError, islice, range(10), 1, 10, -1)
- self.assertRaises(ValueError, islice, range(10), 1, 10, 0)
- self.assertRaises(ValueError, islice, range(10), 'a')
- self.assertRaises(ValueError, islice, range(10), 'a', 1)
- self.assertRaises(ValueError, islice, range(10), 1, 'a')
- self.assertRaises(ValueError, islice, range(10), 'a', 1, 1)
- self.assertRaises(ValueError, islice, range(10), 1, 'a', 1)
+ ra = range(10)
+ self.assertRaises(TypeError, islice, ra)
+ self.assertRaises(TypeError, islice, ra, 1, 2, 3, 4)
+ self.assertRaises(ValueError, islice, ra, -5, 10, 1)
+ self.assertRaises(ValueError, islice, ra, 1, -5, -1)
+ self.assertRaises(ValueError, islice, ra, 1, 10, -1)
+ self.assertRaises(ValueError, islice, ra, 1, 10, 0)
+ self.assertRaises(ValueError, islice, ra, 'a')
+ self.assertRaises(ValueError, islice, ra, 'a', 1)
+ self.assertRaises(ValueError, islice, ra, 1, 'a')
+ self.assertRaises(ValueError, islice, ra, 'a', 1, 1)
+ self.assertRaises(ValueError, islice, ra, 1, 'a', 1)
self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
# Issue #10323: Less islice in a predictable state
@@ -835,9 +1071,22 @@
self.assertEqual(list(islice(c, 1, 3, 50)), [1])
self.assertEqual(next(c), 3)
+ # check copy, deepcopy, pickle
+ for args in [ # islice(args) should agree with range(args)
+ (10, 20, 3),
+ (10, 3, 20),
+ (10, 20),
+ (10, 3),
+ (20,)
+ ]:
+ self.assertEqual(list(copy.copy(islice(range(100), *args))),
+ list(range(*args)))
+ self.assertEqual(list(copy.deepcopy(islice(range(100), *args))),
+ list(range(*args)))
+ self.pickletest(islice(range(100), *args))
+
def test_takewhile(self):
data = [1, 3, 5, 20, 2, 4, 6, 8]
- underten = lambda x: x<10
self.assertEqual(list(takewhile(underten, data)), [1, 3, 5])
self.assertEqual(list(takewhile(underten, [])), [])
self.assertRaises(TypeError, takewhile)
@@ -849,9 +1098,14 @@
self.assertEqual(list(t), [1, 1, 1])
self.assertRaises(StopIteration, next, t)
+ # check copy, deepcopy, pickle
+ self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5])
+ self.assertEqual(list(copy.deepcopy(takewhile(underten, data))),
+ [1, 3, 5])
+ self.pickletest(takewhile(underten, data))
+
def test_dropwhile(self):
data = [1, 3, 5, 20, 2, 4, 6, 8]
- underten = lambda x: x<10
self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8])
self.assertEqual(list(dropwhile(underten, [])), [])
self.assertRaises(TypeError, dropwhile)
@@ -860,11 +1114,14 @@
self.assertRaises(TypeError, next, dropwhile(10, [(4,5)]))
self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)]))
+ # check copy, deepcopy, pickle
+ self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8])
+ self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))),
+ [20, 2, 4, 6, 8])
+ self.pickletest(dropwhile(underten, data))
+
def test_tee(self):
n = 200
- def irange(n):
- for i in range(n):
- yield i
a, b = tee([]) # test empty iterator
self.assertEqual(list(a), [])
@@ -949,6 +1206,67 @@
del a
self.assertRaises(ReferenceError, getattr, p, '__class__')
+ ans = list('abc')
+ long_ans = list(range(10000))
+
+ # check copy
+ a, b = tee('abc')
+ self.assertEqual(list(copy.copy(a)), ans)
+ self.assertEqual(list(copy.copy(b)), ans)
+ a, b = tee(list(range(10000)))
+ self.assertEqual(list(copy.copy(a)), long_ans)
+ self.assertEqual(list(copy.copy(b)), long_ans)
+
+ # check partially consumed copy
+ a, b = tee('abc')
+ take(2, a)
+ take(1, b)
+ self.assertEqual(list(copy.copy(a)), ans[2:])
+ self.assertEqual(list(copy.copy(b)), ans[1:])
+ self.assertEqual(list(a), ans[2:])
+ self.assertEqual(list(b), ans[1:])
+ a, b = tee(range(10000))
+ take(100, a)
+ take(60, b)
+ self.assertEqual(list(copy.copy(a)), long_ans[100:])
+ self.assertEqual(list(copy.copy(b)), long_ans[60:])
+ self.assertEqual(list(a), long_ans[100:])
+ self.assertEqual(list(b), long_ans[60:])
+
+ # check deepcopy
+ a, b = tee('abc')
+ self.assertEqual(list(copy.deepcopy(a)), ans)
+ self.assertEqual(list(copy.deepcopy(b)), ans)
+ self.assertEqual(list(a), ans)
+ self.assertEqual(list(b), ans)
+ a, b = tee(range(10000))
+ self.assertEqual(list(copy.deepcopy(a)), long_ans)
+ self.assertEqual(list(copy.deepcopy(b)), long_ans)
+ self.assertEqual(list(a), long_ans)
+ self.assertEqual(list(b), long_ans)
+
+ # check partially consumed deepcopy
+ a, b = tee('abc')
+ take(2, a)
+ take(1, b)
+ self.assertEqual(list(copy.deepcopy(a)), ans[2:])
+ self.assertEqual(list(copy.deepcopy(b)), ans[1:])
+ self.assertEqual(list(a), ans[2:])
+ self.assertEqual(list(b), ans[1:])
+ a, b = tee(range(10000))
+ take(100, a)
+ take(60, b)
+ self.assertEqual(list(copy.deepcopy(a)), long_ans[100:])
+ self.assertEqual(list(copy.deepcopy(b)), long_ans[60:])
+ self.assertEqual(list(a), long_ans[100:])
+ self.assertEqual(list(b), long_ans[60:])
+
+ # check pickle
+ self.pickletest(iter(tee('abc')))
+ a, b = tee('abc')
+ self.pickletest(a, compare=ans)
+ self.pickletest(b, compare=ans)
+
def test_StopIteration(self):
self.assertRaises(StopIteration, next, zip())
@@ -974,9 +1292,21 @@
class TestExamples(unittest.TestCase):
- def test_accumlate(self):
+ def test_accumulate(self):
self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15])
+ def test_accumulate_reducible(self):
+ # check copy, deepcopy, pickle
+ data = [1, 2, 3, 4, 5]
+ accumulated = [1, 3, 6, 10, 15]
+ it = accumulate(data)
+
+ self.assertEqual(list(pickle.loads(pickle.dumps(it))), accumulated[:])
+ self.assertEqual(next(it), 1)
+ self.assertEqual(list(pickle.loads(pickle.dumps(it))), accumulated[1:])
+ self.assertEqual(list(copy.deepcopy(it)), accumulated[1:])
+ self.assertEqual(list(copy.copy(it)), accumulated[1:])
+
def test_chain(self):
self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF')
diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py
index d7e6629..302c1d8 100644
--- a/Lib/test/test_list.py
+++ b/Lib/test/test_list.py
@@ -1,5 +1,6 @@
import sys
from test import support, list_tests
+import pickle
class ListTest(list_tests.CommonTest):
type2test = list
@@ -69,6 +70,33 @@
check(10) # check our checking code
check(1000000)
+ def test_iterator_pickle(self):
+ # Userlist iterators don't support pickling yet since
+ # they are based on generators.
+ data = self.type2test([4, 5, 6, 7])
+ it = itorg = iter(data)
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(self.type2test(it), self.type2test(data))
+
+ it = pickle.loads(d)
+ next(it)
+ d = pickle.dumps(it)
+ self.assertEqual(self.type2test(it), self.type2test(data)[1:])
+
+ def test_reversed_pickle(self):
+ data = self.type2test([4, 5, 6, 7])
+ it = itorg = reversed(data)
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(self.type2test(it), self.type2test(reversed(data)))
+
+ it = pickle.loads(d)
+ next(it)
+ d = pickle.dumps(it)
+ self.assertEqual(self.type2test(it), self.type2test(reversed(data))[1:])
def test_main(verbose=None):
support.run_unittest(ListTest)
diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py
index 2e335cc..f18046d 100644
--- a/Lib/test/test_range.py
+++ b/Lib/test/test_range.py
@@ -341,13 +341,35 @@
def test_pickling(self):
testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
- (13, 21, 3), (-2, 2, 2)]
+ (13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)]
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
for t in testcases:
r = range(*t)
self.assertEqual(list(pickle.loads(pickle.dumps(r, proto))),
list(r))
+ def test_iterator_pickling(self):
+ testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
+ (13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)]
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ for t in testcases:
+ it = itorg = iter(range(*t))
+ data = list(range(*t))
+
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(list(it), data)
+
+ it = pickle.loads(d)
+ try:
+ next(it)
+ except StopIteration:
+ continue
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(list(it), data[1:])
+
def test_odd_bug(self):
# This used to raise a "SystemError: NULL result without error"
# because the range validation step was eating the exception
diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py
index 6642440..8e9e587 100644
--- a/Lib/test/test_set.py
+++ b/Lib/test/test_set.py
@@ -9,6 +9,7 @@
import sys
import warnings
import collections
+import collections.abc
class PassThru(Exception):
pass
@@ -234,6 +235,26 @@
dup = pickle.loads(p)
self.assertEqual(self.s.x, dup.x)
+ def test_iterator_pickling(self):
+ itorg = iter(self.s)
+ data = self.thetype(self.s)
+ d = pickle.dumps(itorg)
+ it = pickle.loads(d)
+ # Set iterators unpickle as list iterators due to the
+ # undefined order of set items.
+ # self.assertEqual(type(itorg), type(it))
+ self.assertTrue(isinstance(it, collections.abc.Iterator))
+ self.assertEqual(self.thetype(it), data)
+
+ it = pickle.loads(d)
+ try:
+ drop = next(it)
+ except StopIteration:
+ return
+ d = pickle.dumps(it)
+ it = pickle.loads(d)
+ self.assertEqual(self.thetype(it), data - self.thetype((drop,)))
+
def test_deepcopy(self):
class Tracer:
def __init__(self, value):
diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py
index 1464a0e..d928db2 100644
--- a/Lib/test/test_tuple.py
+++ b/Lib/test/test_tuple.py
@@ -1,6 +1,7 @@
from test import support, seq_tests
import gc
+import pickle
class TupleTest(seq_tests.CommonTest):
type2test = tuple
@@ -164,6 +165,34 @@
check(10) # check our checking code
check(1000000)
+ def test_iterator_pickle(self):
+ # Userlist iterators don't support pickling yet since
+ # they are based on generators.
+ data = self.type2test([4, 5, 6, 7])
+ itorg = iter(data)
+ d = pickle.dumps(itorg)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(self.type2test(it), self.type2test(data))
+
+ it = pickle.loads(d)
+ next(it)
+ d = pickle.dumps(it)
+ self.assertEqual(self.type2test(it), self.type2test(data)[1:])
+
+ def test_reversed_pickle(self):
+ data = self.type2test([4, 5, 6, 7])
+ itorg = reversed(data)
+ d = pickle.dumps(itorg)
+ it = pickle.loads(d)
+ self.assertEqual(type(itorg), type(it))
+ self.assertEqual(self.type2test(it), self.type2test(reversed(data)))
+
+ it = pickle.loads(d)
+ next(it)
+ d = pickle.dumps(it)
+ self.assertEqual(self.type2test(it), self.type2test(reversed(data))[1:])
+
def test_main():
support.run_unittest(TupleTest)