blob: fe5a50bb42fa000d80daa6c9fe606dc72fbd64bf [file] [log] [blame]
Serhiy Storchakabfe18242015-03-31 13:12:37 +03001import collections
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002import copyreg
Serhiy Storchakabfe18242015-03-31 13:12:37 +03003import dbm
Collin Winter771d8342009-04-16 03:18:06 +00004import io
Serhiy Storchakabfe18242015-03-31 13:12:37 +03005import functools
tjb9004371c0a2019-02-18 23:30:51 +08006import os
Pierre Glaser289f1f82019-05-08 23:08:25 +02007import math
Tim Peters4190fb82003-02-02 16:09:05 +00008import pickle
Tim Peters31f119e2003-02-03 16:20:13 +00009import pickletools
tjb9004371c0a2019-02-18 23:30:51 +080010import shutil
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -080011import struct
Antoine Pitrou82be19f2011-08-29 23:09:33 +020012import sys
tjb9004371c0a2019-02-18 23:30:51 +080013import threading
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010014import unittest
Antoine Pitrou16c4ce12011-03-11 21:30:43 +010015import weakref
tjb9004371c0a2019-02-18 23:30:51 +080016from textwrap import dedent
Antoine Pitroud9dfaa92009-06-04 20:32:06 +000017from http.cookies import SimpleCookie
Tim Peters4190fb82003-02-02 16:09:05 +000018
Antoine Pitrou91f43802019-05-26 17:10:09 +020019try:
20 import _testbuffer
21except ImportError:
22 _testbuffer = None
23
Serhiy Storchaka7279bef2015-11-29 13:12:10 +020024from test import support
Antoine Pitrou82be19f2011-08-29 23:09:33 +020025from test.support import (
Serhiy Storchaka7279bef2015-11-29 13:12:10 +020026 TestFailed, TESTFN, run_with_locale, no_tracing,
tjb9004371c0a2019-02-18 23:30:51 +080027 _2G, _4G, bigmemtest, reap_threads, forget,
Victor Stinnerb39d41b2020-06-17 18:40:14 +020028 save_restore_warnings_filters
Antoine Pitrou82be19f2011-08-29 23:09:33 +020029 )
Tim Peterse089c682001-04-10 03:41:41 +000030
Guido van Rossum98297ee2007-11-06 21:34:58 +000031from pickle import bytes_types
32
Victor Stinnerb39d41b2020-06-17 18:40:14 +020033
34# bpo-41003: Save/restore warnings filters to leave them unchanged.
35# Ignore filters installed by numpy.
36try:
37 with save_restore_warnings_filters():
38 import numpy as np
39except ImportError:
40 np = None
41
42
Serhiy Storchakac6b54b42015-09-29 15:33:24 +030043requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
44 "test is only meaningful on 32-bit builds")
45
Tim Petersee1a53c2003-02-02 02:57:53 +000046# Tests that try a number of pickle protocols should have a
47# for proto in protocols:
Tim Peters8587b3c2003-02-13 15:44:41 +000048# kind of outer loop.
Tim Peters8587b3c2003-02-13 15:44:41 +000049protocols = range(pickle.HIGHEST_PROTOCOL + 1)
Tim Petersee1a53c2003-02-02 02:57:53 +000050
Tim Peters22e71712003-02-03 22:27:38 +000051
52# Return True if opcode code appears in the pickle, else False.
53def opcode_in_pickle(code, pickle):
54 for op, dummy, dummy in pickletools.genops(pickle):
Guido van Rossumcfe5f202007-05-08 21:26:54 +000055 if op.code == code.decode("latin-1"):
Tim Peters22e71712003-02-03 22:27:38 +000056 return True
57 return False
58
Tim Peters8d2613a2003-02-11 16:40:16 +000059# Return the number of times opcode code appears in pickle.
60def count_opcode(code, pickle):
61 n = 0
62 for op, dummy, dummy in pickletools.genops(pickle):
Guido van Rossumcfe5f202007-05-08 21:26:54 +000063 if op.code == code.decode("latin-1"):
Tim Peters8d2613a2003-02-11 16:40:16 +000064 n += 1
65 return n
66
Antoine Pitrou04248a82010-10-12 20:51:21 +000067
68class UnseekableIO(io.BytesIO):
69 def peek(self, *args):
70 raise NotImplementedError
71
72 def seekable(self):
73 return False
74
75 def seek(self, *args):
76 raise io.UnsupportedOperation
77
78 def tell(self):
79 raise io.UnsupportedOperation
80
81
Antoine Pitrou9f378722020-02-23 23:33:53 +010082class MinimalIO(object):
83 """
84 A file-like object that doesn't support readinto().
85 """
86 def __init__(self, *args):
87 self._bio = io.BytesIO(*args)
88 self.getvalue = self._bio.getvalue
89 self.read = self._bio.read
90 self.readline = self._bio.readline
91 self.write = self._bio.write
92
93
Tim Peters3e667d52003-02-04 21:47:44 +000094# We can't very well test the extension registry without putting known stuff
95# in it, but we have to be careful to restore its original state. Code
96# should do this:
97#
98# e = ExtensionSaver(extension_code)
99# try:
100# fiddle w/ the extension registry's stuff for extension_code
101# finally:
102# e.restore()
103
104class ExtensionSaver:
105 # Remember current registration for code (if any), and remove it (if
106 # there is one).
107 def __init__(self, code):
108 self.code = code
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +0000109 if code in copyreg._inverted_registry:
110 self.pair = copyreg._inverted_registry[code]
111 copyreg.remove_extension(self.pair[0], self.pair[1], code)
Tim Peters3e667d52003-02-04 21:47:44 +0000112 else:
113 self.pair = None
114
115 # Restore previous registration for code.
116 def restore(self):
117 code = self.code
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +0000118 curpair = copyreg._inverted_registry.get(code)
Tim Peters3e667d52003-02-04 21:47:44 +0000119 if curpair is not None:
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +0000120 copyreg.remove_extension(curpair[0], curpair[1], code)
Tim Peters3e667d52003-02-04 21:47:44 +0000121 pair = self.pair
122 if pair is not None:
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +0000123 copyreg.add_extension(pair[0], pair[1], code)
Tim Peters3e667d52003-02-04 21:47:44 +0000124
Jeremy Hylton66426532001-10-15 21:38:56 +0000125class C:
Guido van Rossum47b9ff62006-08-24 00:41:19 +0000126 def __eq__(self, other):
127 return self.__dict__ == other.__dict__
Jeremy Hylton66426532001-10-15 21:38:56 +0000128
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000129class D(C):
130 def __init__(self, arg):
131 pass
132
133class E(C):
134 def __getinitargs__(self):
135 return ()
136
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100137class H(object):
138 pass
139
Serhiy Storchaka608c2132015-11-07 11:16:10 +0200140# Hashable mutable key
141class K(object):
142 def __init__(self, value):
143 self.value = value
144
145 def __reduce__(self):
146 # Shouldn't support the recursion itself
147 return K, (self.value,)
148
Jeremy Hylton66426532001-10-15 21:38:56 +0000149import __main__
150__main__.C = C
151C.__module__ = "__main__"
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000152__main__.D = D
153D.__module__ = "__main__"
154__main__.E = E
155E.__module__ = "__main__"
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100156__main__.H = H
157H.__module__ = "__main__"
Serhiy Storchaka608c2132015-11-07 11:16:10 +0200158__main__.K = K
159K.__module__ = "__main__"
Jeremy Hylton66426532001-10-15 21:38:56 +0000160
161class myint(int):
162 def __init__(self, x):
163 self.str = str(x)
164
165class initarg(C):
Guido van Rossum1444f672001-12-19 16:38:29 +0000166
Jeremy Hylton66426532001-10-15 21:38:56 +0000167 def __init__(self, a, b):
168 self.a = a
169 self.b = b
170
171 def __getinitargs__(self):
172 return self.a, self.b
173
Guido van Rossum04a86612001-12-19 16:58:54 +0000174class metaclass(type):
175 pass
176
Guido van Rossum52cc1d82007-03-18 15:41:51 +0000177class use_metaclass(object, metaclass=metaclass):
178 pass
Guido van Rossum04a86612001-12-19 16:58:54 +0000179
Antoine Pitrouffd41d92011-10-04 09:23:04 +0200180class pickling_metaclass(type):
181 def __eq__(self, other):
182 return (type(self) == type(other) and
183 self.reduce_args == other.reduce_args)
184
185 def __reduce__(self):
186 return (create_dynamic_class, self.reduce_args)
187
188def create_dynamic_class(name, bases):
189 result = pickling_metaclass(name, bases, dict())
190 result.reduce_args = (name, bases)
191 return result
192
Antoine Pitrou91f43802019-05-26 17:10:09 +0200193
194class ZeroCopyBytes(bytes):
195 readonly = True
196 c_contiguous = True
197 f_contiguous = True
198 zero_copy_reconstruct = True
199
200 def __reduce_ex__(self, protocol):
201 if protocol >= 5:
202 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None
203 else:
204 return type(self)._reconstruct, (bytes(self),)
205
206 def __repr__(self):
207 return "{}({!r})".format(self.__class__.__name__, bytes(self))
208
209 __str__ = __repr__
210
211 @classmethod
212 def _reconstruct(cls, obj):
213 with memoryview(obj) as m:
214 obj = m.obj
215 if type(obj) is cls:
216 # Zero-copy
217 return obj
218 else:
219 return cls(obj)
220
221
222class ZeroCopyBytearray(bytearray):
223 readonly = False
224 c_contiguous = True
225 f_contiguous = True
226 zero_copy_reconstruct = True
227
228 def __reduce_ex__(self, protocol):
229 if protocol >= 5:
230 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None
231 else:
232 return type(self)._reconstruct, (bytes(self),)
233
234 def __repr__(self):
235 return "{}({!r})".format(self.__class__.__name__, bytes(self))
236
237 __str__ = __repr__
238
239 @classmethod
240 def _reconstruct(cls, obj):
241 with memoryview(obj) as m:
242 obj = m.obj
243 if type(obj) is cls:
244 # Zero-copy
245 return obj
246 else:
247 return cls(obj)
248
249
250if _testbuffer is not None:
251
252 class PicklableNDArray:
253 # A not-really-zero-copy picklable ndarray, as the ndarray()
254 # constructor doesn't allow for it
255
256 zero_copy_reconstruct = False
257
258 def __init__(self, *args, **kwargs):
259 self.array = _testbuffer.ndarray(*args, **kwargs)
260
261 def __getitem__(self, idx):
262 cls = type(self)
263 new = cls.__new__(cls)
264 new.array = self.array[idx]
265 return new
266
267 @property
268 def readonly(self):
269 return self.array.readonly
270
271 @property
272 def c_contiguous(self):
273 return self.array.c_contiguous
274
275 @property
276 def f_contiguous(self):
277 return self.array.f_contiguous
278
279 def __eq__(self, other):
280 if not isinstance(other, PicklableNDArray):
281 return NotImplemented
282 return (other.array.format == self.array.format and
283 other.array.shape == self.array.shape and
284 other.array.strides == self.array.strides and
285 other.array.readonly == self.array.readonly and
286 other.array.tobytes() == self.array.tobytes())
287
288 def __ne__(self, other):
289 if not isinstance(other, PicklableNDArray):
290 return NotImplemented
291 return not (self == other)
292
293 def __repr__(self):
294 return (f"{type(self)}(shape={self.array.shape},"
295 f"strides={self.array.strides}, "
296 f"bytes={self.array.tobytes()})")
297
298 def __reduce_ex__(self, protocol):
299 if not self.array.contiguous:
300 raise NotImplementedError("Reconstructing a non-contiguous "
301 "ndarray does not seem possible")
302 ndarray_kwargs = {"shape": self.array.shape,
303 "strides": self.array.strides,
304 "format": self.array.format,
305 "flags": (0 if self.readonly
306 else _testbuffer.ND_WRITABLE)}
307 pb = pickle.PickleBuffer(self.array)
308 if protocol >= 5:
309 return (type(self)._reconstruct,
310 (pb, ndarray_kwargs))
311 else:
312 # Need to serialize the bytes in physical order
313 with pb.raw() as m:
314 return (type(self)._reconstruct,
315 (m.tobytes(), ndarray_kwargs))
316
317 @classmethod
318 def _reconstruct(cls, obj, kwargs):
319 with memoryview(obj) as m:
320 # For some reason, ndarray() wants a list of integers...
321 # XXX This only works if format == 'B'
322 items = list(m.tobytes())
323 return cls(items, **kwargs)
324
325
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300326# DATA0 .. DATA4 are the pickles we expect under the various protocols, for
Tim Peters70b02d72003-02-02 17:26:40 +0000327# the object returned by create_data().
Tim Petersee1a53c2003-02-02 02:57:53 +0000328
Guido van Rossum98297ee2007-11-06 21:34:58 +0000329DATA0 = (
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200330 b'(lp0\nL0L\naL1L\naF2.0\n'
331 b'ac__builtin__\ncomple'
332 b'x\np1\n(F3.0\nF0.0\ntp2\n'
333 b'Rp3\naL1L\naL-1L\naL255'
334 b'L\naL-255L\naL-256L\naL'
335 b'65535L\naL-65535L\naL-'
336 b'65536L\naL2147483647L'
337 b'\naL-2147483647L\naL-2'
338 b'147483648L\na(Vabc\np4'
339 b'\ng4\nccopy_reg\n_recon'
340 b'structor\np5\n(c__main'
341 b'__\nC\np6\nc__builtin__'
342 b'\nobject\np7\nNtp8\nRp9\n'
343 b'(dp10\nVfoo\np11\nL1L\ns'
344 b'Vbar\np12\nL2L\nsbg9\ntp'
345 b'13\nag13\naL5L\na.'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000346)
Tim Peterse9358162001-01-22 22:05:20 +0000347
Guido van Rossum98297ee2007-11-06 21:34:58 +0000348# Disassembly of DATA0
Tim Peters70b02d72003-02-02 17:26:40 +0000349DATA0_DIS = """\
350 0: ( MARK
351 1: l LIST (MARK at 0)
Guido van Rossum98297ee2007-11-06 21:34:58 +0000352 2: p PUT 0
353 5: L LONG 0
Mark Dickinson8dd05142009-01-20 20:43:58 +0000354 9: a APPEND
355 10: L LONG 1
356 14: a APPEND
357 15: F FLOAT 2.0
358 20: a APPEND
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200359 21: c GLOBAL '__builtin__ complex'
360 42: p PUT 1
361 45: ( MARK
362 46: F FLOAT 3.0
363 51: F FLOAT 0.0
364 56: t TUPLE (MARK at 45)
365 57: p PUT 2
366 60: R REDUCE
367 61: p PUT 3
368 64: a APPEND
369 65: L LONG 1
370 69: a APPEND
371 70: L LONG -1
372 75: a APPEND
373 76: L LONG 255
374 82: a APPEND
375 83: L LONG -255
376 90: a APPEND
377 91: L LONG -256
378 98: a APPEND
379 99: L LONG 65535
380 107: a APPEND
381 108: L LONG -65535
382 117: a APPEND
383 118: L LONG -65536
384 127: a APPEND
385 128: L LONG 2147483647
386 141: a APPEND
387 142: L LONG -2147483647
388 156: a APPEND
389 157: L LONG -2147483648
390 171: a APPEND
391 172: ( MARK
392 173: V UNICODE 'abc'
393 178: p PUT 4
394 181: g GET 4
395 184: c GLOBAL 'copy_reg _reconstructor'
396 209: p PUT 5
397 212: ( MARK
398 213: c GLOBAL '__main__ C'
399 225: p PUT 6
400 228: c GLOBAL '__builtin__ object'
401 248: p PUT 7
402 251: N NONE
403 252: t TUPLE (MARK at 212)
404 253: p PUT 8
405 256: R REDUCE
406 257: p PUT 9
407 260: ( MARK
408 261: d DICT (MARK at 260)
409 262: p PUT 10
410 266: V UNICODE 'foo'
411 271: p PUT 11
412 275: L LONG 1
413 279: s SETITEM
414 280: V UNICODE 'bar'
415 285: p PUT 12
416 289: L LONG 2
417 293: s SETITEM
418 294: b BUILD
419 295: g GET 9
420 298: t TUPLE (MARK at 172)
421 299: p PUT 13
422 303: a APPEND
423 304: g GET 13
424 308: a APPEND
425 309: L LONG 5
426 313: a APPEND
427 314: . STOP
Tim Peters70b02d72003-02-02 17:26:40 +0000428highest protocol among opcodes = 0
429"""
430
Guido van Rossum98297ee2007-11-06 21:34:58 +0000431DATA1 = (
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200432 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__'
433 b'builtin__\ncomplex\nq\x01'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000434 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t'
435 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ'
436 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff'
437 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab'
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200438 b'cq\x04h\x04ccopy_reg\n_reco'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000439 b'nstructor\nq\x05(c__main'
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200440 b'__\nC\nq\x06c__builtin__\n'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000441 b'object\nq\x07Ntq\x08Rq\t}q\n('
442 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar'
443 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.'
444)
Tim Peters70b02d72003-02-02 17:26:40 +0000445
Guido van Rossum98297ee2007-11-06 21:34:58 +0000446# Disassembly of DATA1
Tim Peters70b02d72003-02-02 17:26:40 +0000447DATA1_DIS = """\
448 0: ] EMPTY_LIST
Guido van Rossum98297ee2007-11-06 21:34:58 +0000449 1: q BINPUT 0
Tim Peters70b02d72003-02-02 17:26:40 +0000450 3: ( MARK
451 4: K BININT1 0
Guido van Rossum98297ee2007-11-06 21:34:58 +0000452 6: K BININT1 1
453 8: G BINFLOAT 2.0
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200454 17: c GLOBAL '__builtin__ complex'
455 38: q BINPUT 1
456 40: ( MARK
457 41: G BINFLOAT 3.0
458 50: G BINFLOAT 0.0
459 59: t TUPLE (MARK at 40)
460 60: q BINPUT 2
461 62: R REDUCE
462 63: q BINPUT 3
463 65: K BININT1 1
464 67: J BININT -1
465 72: K BININT1 255
466 74: J BININT -255
467 79: J BININT -256
468 84: M BININT2 65535
469 87: J BININT -65535
470 92: J BININT -65536
471 97: J BININT 2147483647
472 102: J BININT -2147483647
473 107: J BININT -2147483648
474 112: ( MARK
475 113: X BINUNICODE 'abc'
476 121: q BINPUT 4
477 123: h BINGET 4
478 125: c GLOBAL 'copy_reg _reconstructor'
479 150: q BINPUT 5
480 152: ( MARK
481 153: c GLOBAL '__main__ C'
482 165: q BINPUT 6
483 167: c GLOBAL '__builtin__ object'
484 187: q BINPUT 7
485 189: N NONE
486 190: t TUPLE (MARK at 152)
487 191: q BINPUT 8
488 193: R REDUCE
489 194: q BINPUT 9
490 196: } EMPTY_DICT
491 197: q BINPUT 10
492 199: ( MARK
493 200: X BINUNICODE 'foo'
494 208: q BINPUT 11
495 210: K BININT1 1
496 212: X BINUNICODE 'bar'
497 220: q BINPUT 12
498 222: K BININT1 2
499 224: u SETITEMS (MARK at 199)
500 225: b BUILD
501 226: h BINGET 9
502 228: t TUPLE (MARK at 112)
503 229: q BINPUT 13
504 231: h BINGET 13
505 233: K BININT1 5
506 235: e APPENDS (MARK at 3)
507 236: . STOP
Tim Peters70b02d72003-02-02 17:26:40 +0000508highest protocol among opcodes = 1
509"""
Tim Peterse0c446b2001-10-18 21:57:37 +0000510
Guido van Rossum98297ee2007-11-06 21:34:58 +0000511DATA2 = (
512 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200513 b'__builtin__\ncomplex\n'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000514 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00'
515 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff'
516 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff'
517 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a'
518 b'bcq\x04h\x04c__main__\nC\nq\x05'
519 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01'
520 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh'
521 b'\nK\x05e.'
522)
Tim Petersfc273752003-03-02 04:54:24 +0000523
Guido van Rossum98297ee2007-11-06 21:34:58 +0000524# Disassembly of DATA2
Tim Petersfc273752003-03-02 04:54:24 +0000525DATA2_DIS = """\
526 0: \x80 PROTO 2
527 2: ] EMPTY_LIST
Guido van Rossum98297ee2007-11-06 21:34:58 +0000528 3: q BINPUT 0
Tim Petersfc273752003-03-02 04:54:24 +0000529 5: ( MARK
530 6: K BININT1 0
Guido van Rossum98297ee2007-11-06 21:34:58 +0000531 8: K BININT1 1
532 10: G BINFLOAT 2.0
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200533 19: c GLOBAL '__builtin__ complex'
534 40: q BINPUT 1
535 42: G BINFLOAT 3.0
536 51: G BINFLOAT 0.0
537 60: \x86 TUPLE2
538 61: q BINPUT 2
539 63: R REDUCE
540 64: q BINPUT 3
541 66: K BININT1 1
542 68: J BININT -1
543 73: K BININT1 255
544 75: J BININT -255
545 80: J BININT -256
546 85: M BININT2 65535
547 88: J BININT -65535
548 93: J BININT -65536
549 98: J BININT 2147483647
550 103: J BININT -2147483647
551 108: J BININT -2147483648
552 113: ( MARK
553 114: X BINUNICODE 'abc'
554 122: q BINPUT 4
555 124: h BINGET 4
556 126: c GLOBAL '__main__ C'
557 138: q BINPUT 5
558 140: ) EMPTY_TUPLE
559 141: \x81 NEWOBJ
560 142: q BINPUT 6
561 144: } EMPTY_DICT
562 145: q BINPUT 7
563 147: ( MARK
564 148: X BINUNICODE 'foo'
565 156: q BINPUT 8
566 158: K BININT1 1
567 160: X BINUNICODE 'bar'
568 168: q BINPUT 9
569 170: K BININT1 2
570 172: u SETITEMS (MARK at 147)
571 173: b BUILD
572 174: h BINGET 6
573 176: t TUPLE (MARK at 113)
574 177: q BINPUT 10
575 179: h BINGET 10
576 181: K BININT1 5
577 183: e APPENDS (MARK at 5)
578 184: . STOP
Tim Petersfc273752003-03-02 04:54:24 +0000579highest protocol among opcodes = 2
580"""
581
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300582DATA3 = (
583 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
584 b'builtins\ncomplex\nq\x01G'
585 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
586 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
587 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
588 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
589 b'\x04h\x04c__main__\nC\nq\x05)\x81q'
590 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
591 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
592 b'e.'
593)
594
595# Disassembly of DATA3
596DATA3_DIS = """\
597 0: \x80 PROTO 3
598 2: ] EMPTY_LIST
599 3: q BINPUT 0
600 5: ( MARK
601 6: K BININT1 0
602 8: K BININT1 1
603 10: G BINFLOAT 2.0
604 19: c GLOBAL 'builtins complex'
605 37: q BINPUT 1
606 39: G BINFLOAT 3.0
607 48: G BINFLOAT 0.0
608 57: \x86 TUPLE2
609 58: q BINPUT 2
610 60: R REDUCE
611 61: q BINPUT 3
612 63: K BININT1 1
613 65: J BININT -1
614 70: K BININT1 255
615 72: J BININT -255
616 77: J BININT -256
617 82: M BININT2 65535
618 85: J BININT -65535
619 90: J BININT -65536
620 95: J BININT 2147483647
621 100: J BININT -2147483647
622 105: J BININT -2147483648
623 110: ( MARK
624 111: X BINUNICODE 'abc'
625 119: q BINPUT 4
626 121: h BINGET 4
627 123: c GLOBAL '__main__ C'
628 135: q BINPUT 5
629 137: ) EMPTY_TUPLE
630 138: \x81 NEWOBJ
631 139: q BINPUT 6
632 141: } EMPTY_DICT
633 142: q BINPUT 7
634 144: ( MARK
635 145: X BINUNICODE 'bar'
636 153: q BINPUT 8
637 155: K BININT1 2
638 157: X BINUNICODE 'foo'
639 165: q BINPUT 9
640 167: K BININT1 1
641 169: u SETITEMS (MARK at 144)
642 170: b BUILD
643 171: h BINGET 6
644 173: t TUPLE (MARK at 110)
645 174: q BINPUT 10
646 176: h BINGET 10
647 178: K BININT1 5
648 180: e APPENDS (MARK at 5)
649 181: . STOP
650highest protocol among opcodes = 2
651"""
652
653DATA4 = (
654 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
655 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
656 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
657 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
658 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
659 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
660 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
661 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
662 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
663)
664
665# Disassembly of DATA4
666DATA4_DIS = """\
667 0: \x80 PROTO 4
668 2: \x95 FRAME 168
669 11: ] EMPTY_LIST
670 12: \x94 MEMOIZE
671 13: ( MARK
672 14: K BININT1 0
673 16: K BININT1 1
674 18: G BINFLOAT 2.0
675 27: \x8c SHORT_BINUNICODE 'builtins'
676 37: \x94 MEMOIZE
677 38: \x8c SHORT_BINUNICODE 'complex'
678 47: \x94 MEMOIZE
679 48: \x93 STACK_GLOBAL
680 49: \x94 MEMOIZE
681 50: G BINFLOAT 3.0
682 59: G BINFLOAT 0.0
683 68: \x86 TUPLE2
684 69: \x94 MEMOIZE
685 70: R REDUCE
686 71: \x94 MEMOIZE
687 72: K BININT1 1
688 74: J BININT -1
689 79: K BININT1 255
690 81: J BININT -255
691 86: J BININT -256
692 91: M BININT2 65535
693 94: J BININT -65535
694 99: J BININT -65536
695 104: J BININT 2147483647
696 109: J BININT -2147483647
697 114: J BININT -2147483648
698 119: ( MARK
699 120: \x8c SHORT_BINUNICODE 'abc'
700 125: \x94 MEMOIZE
701 126: h BINGET 6
702 128: \x8c SHORT_BINUNICODE '__main__'
703 138: \x94 MEMOIZE
704 139: \x8c SHORT_BINUNICODE 'C'
705 142: \x94 MEMOIZE
706 143: \x93 STACK_GLOBAL
707 144: \x94 MEMOIZE
708 145: ) EMPTY_TUPLE
709 146: \x81 NEWOBJ
710 147: \x94 MEMOIZE
711 148: } EMPTY_DICT
712 149: \x94 MEMOIZE
713 150: ( MARK
714 151: \x8c SHORT_BINUNICODE 'bar'
715 156: \x94 MEMOIZE
716 157: K BININT1 2
717 159: \x8c SHORT_BINUNICODE 'foo'
718 164: \x94 MEMOIZE
719 165: K BININT1 1
720 167: u SETITEMS (MARK at 150)
721 168: b BUILD
722 169: h BINGET 10
723 171: t TUPLE (MARK at 119)
724 172: \x94 MEMOIZE
725 173: h BINGET 14
726 175: K BININT1 5
727 177: e APPENDS (MARK at 13)
728 178: . STOP
729highest protocol among opcodes = 4
730"""
731
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000732# set([1,2]) pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300733DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000734
735# xrange(5) pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300736DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000737
738# a SimpleCookie() object pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300739DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
740 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
741 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
742 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
743 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
744 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000745
746# set([3]) pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300747DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000748
Walter Doerwald9d1dbca2013-12-02 11:41:01 +0100749python2_exceptions_without_args = (
750 ArithmeticError,
751 AssertionError,
752 AttributeError,
753 BaseException,
754 BufferError,
755 BytesWarning,
756 DeprecationWarning,
757 EOFError,
758 EnvironmentError,
759 Exception,
760 FloatingPointError,
761 FutureWarning,
762 GeneratorExit,
763 IOError,
764 ImportError,
765 ImportWarning,
766 IndentationError,
767 IndexError,
768 KeyError,
769 KeyboardInterrupt,
770 LookupError,
771 MemoryError,
772 NameError,
773 NotImplementedError,
774 OSError,
775 OverflowError,
776 PendingDeprecationWarning,
777 ReferenceError,
778 RuntimeError,
779 RuntimeWarning,
780 # StandardError is gone in Python 3, we map it to Exception
781 StopIteration,
782 SyntaxError,
783 SyntaxWarning,
784 SystemError,
785 SystemExit,
786 TabError,
787 TypeError,
788 UnboundLocalError,
789 UnicodeError,
790 UnicodeWarning,
791 UserWarning,
792 ValueError,
793 Warning,
794 ZeroDivisionError,
795)
796
797exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
798
Walter Doerwald9d1dbca2013-12-02 11:41:01 +0100799# UnicodeEncodeError object pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300800DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
801 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
802 b'U\x03badq\x03tq\x04Rq\x05.')
Walter Doerwald9d1dbca2013-12-02 11:41:01 +0100803
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000804
Jeremy Hylton66426532001-10-15 21:38:56 +0000805def create_data():
Tim Peterse9358162001-01-22 22:05:20 +0000806 c = C()
807 c.foo = 1
808 c.bar = 2
Guido van Rossume2a383d2007-01-15 16:59:06 +0000809 x = [0, 1, 2.0, 3.0+0j]
Tim Peters461922a2001-04-09 20:07:05 +0000810 # Append some integer test cases at cPickle.c's internal size
811 # cutoffs.
812 uint1max = 0xff
813 uint2max = 0xffff
814 int4max = 0x7fffffff
815 x.extend([1, -1,
816 uint1max, -uint1max, -uint1max-1,
817 uint2max, -uint2max, -uint2max-1,
818 int4max, -int4max, -int4max-1])
Tim Peterse9358162001-01-22 22:05:20 +0000819 y = ('abc', 'abc', c, c)
820 x.append(y)
821 x.append(y)
822 x.append(5)
Jeremy Hylton66426532001-10-15 21:38:56 +0000823 return x
Tim Petersc58440f2001-04-09 17:16:31 +0000824
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100825
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300826class AbstractUnpickleTests(unittest.TestCase):
827 # Subclass must define self.loads.
Antoine Pitrou3ab9cfc2013-11-24 14:33:37 +0100828
Jeremy Hylton66426532001-10-15 21:38:56 +0000829 _testdata = create_data()
Tim Petersc58440f2001-04-09 17:16:31 +0000830
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100831 def assert_is_copy(self, obj, objcopy, msg=None):
832 """Utility method to verify if two objects are copies of each others.
833 """
834 if msg is None:
835 msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
836 self.assertEqual(obj, objcopy, msg=msg)
837 self.assertIs(type(obj), type(objcopy), msg=msg)
838 if hasattr(obj, '__dict__'):
839 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
840 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
841 if hasattr(obj, '__slots__'):
842 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
843 for slot in obj.__slots__:
844 self.assertEqual(
845 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
846 self.assertEqual(getattr(obj, slot, None),
847 getattr(objcopy, slot, None), msg=msg)
848
Serhiy Storchaka7279bef2015-11-29 13:12:10 +0200849 def check_unpickling_error(self, errors, data):
850 with self.subTest(data=data), \
851 self.assertRaises(errors):
852 try:
853 self.loads(data)
854 except BaseException as exc:
855 if support.verbose > 1:
856 print('%-32r - %s: %s' %
857 (data, exc.__class__.__name__, exc))
858 raise
859
Guido van Rossum98297ee2007-11-06 21:34:58 +0000860 def test_load_from_data0(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100861 self.assert_is_copy(self._testdata, self.loads(DATA0))
Guido van Rossum98297ee2007-11-06 21:34:58 +0000862
863 def test_load_from_data1(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100864 self.assert_is_copy(self._testdata, self.loads(DATA1))
Guido van Rossum98297ee2007-11-06 21:34:58 +0000865
866 def test_load_from_data2(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100867 self.assert_is_copy(self._testdata, self.loads(DATA2))
Jeremy Hylton66426532001-10-15 21:38:56 +0000868
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300869 def test_load_from_data3(self):
870 self.assert_is_copy(self._testdata, self.loads(DATA3))
871
872 def test_load_from_data4(self):
873 self.assert_is_copy(self._testdata, self.loads(DATA4))
874
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000875 def test_load_classic_instance(self):
876 # See issue5180. Test loading 2.x pickles that
877 # contain an instance of old style class.
878 for X, args in [(C, ()), (D, ('x',)), (E, ())]:
879 xname = X.__name__.encode('ascii')
880 # Protocol 0 (text mode pickle):
881 """
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200882 0: ( MARK
883 1: i INST '__main__ X' (MARK at 0)
884 13: p PUT 0
885 16: ( MARK
886 17: d DICT (MARK at 16)
887 18: p PUT 1
888 21: b BUILD
889 22: . STOP
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000890 """
891 pickle0 = (b"(i__main__\n"
892 b"X\n"
893 b"p0\n"
894 b"(dp1\nb.").replace(b'X', xname)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100895 self.assert_is_copy(X(*args), self.loads(pickle0))
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000896
897 # Protocol 1 (binary mode pickle)
898 """
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200899 0: ( MARK
900 1: c GLOBAL '__main__ X'
901 13: q BINPUT 0
902 15: o OBJ (MARK at 0)
903 16: q BINPUT 1
904 18: } EMPTY_DICT
905 19: q BINPUT 2
906 21: b BUILD
907 22: . STOP
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000908 """
909 pickle1 = (b'(c__main__\n'
910 b'X\n'
911 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100912 self.assert_is_copy(X(*args), self.loads(pickle1))
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000913
914 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
915 """
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200916 0: \x80 PROTO 2
917 2: ( MARK
918 3: c GLOBAL '__main__ X'
919 15: q BINPUT 0
920 17: o OBJ (MARK at 2)
921 18: q BINPUT 1
922 20: } EMPTY_DICT
923 21: q BINPUT 2
924 23: b BUILD
925 24: . STOP
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000926 """
927 pickle2 = (b'\x80\x02(c__main__\n'
928 b'X\n'
929 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100930 self.assert_is_copy(X(*args), self.loads(pickle2))
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000931
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300932 def test_maxint64(self):
933 maxint64 = (1 << 63) - 1
934 data = b'I' + str(maxint64).encode("ascii") + b'\n.'
935 got = self.loads(data)
936 self.assert_is_copy(maxint64, got)
937
938 # Try too with a bogus literal.
939 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +0200940 self.check_unpickling_error(ValueError, data)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300941
942 def test_unpickle_from_2x(self):
943 # Unpickle non-trivial data from Python 2.x.
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300944 loaded = self.loads(DATA_SET)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300945 self.assertEqual(loaded, set([1, 2]))
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300946 loaded = self.loads(DATA_XRANGE)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300947 self.assertEqual(type(loaded), type(range(0)))
948 self.assertEqual(list(loaded), list(range(5)))
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300949 loaded = self.loads(DATA_COOKIE)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300950 self.assertEqual(type(loaded), SimpleCookie)
951 self.assertEqual(list(loaded.keys()), ["key"])
952 self.assertEqual(loaded["key"].value, "value")
953
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300954 # Exception objects without arguments pickled from 2.x with protocol 2
955 for exc in python2_exceptions_without_args:
956 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300957 loaded = self.loads(data)
958 self.assertIs(type(loaded), exc)
959
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300960 # StandardError is mapped to Exception, test that separately
961 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300962 self.assertIs(type(loaded), Exception)
963
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300964 loaded = self.loads(DATA_UEERR)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300965 self.assertIs(type(loaded), UnicodeEncodeError)
966 self.assertEqual(loaded.object, "foo")
967 self.assertEqual(loaded.encoding, "ascii")
968 self.assertEqual(loaded.start, 0)
969 self.assertEqual(loaded.end, 1)
970 self.assertEqual(loaded.reason, "bad")
971
972 def test_load_python2_str_as_bytes(self):
973 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
974 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
975 encoding="bytes"), b'a\x00\xa0')
976 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
977 self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
978 encoding="bytes"), b'a\x00\xa0')
979 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
980 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
981 encoding="bytes"), b'a\x00\xa0')
982
983 def test_load_python2_unicode_as_str(self):
984 # From Python 2: pickle.dumps(u'π', protocol=0)
985 self.assertEqual(self.loads(b'V\\u03c0\n.',
986 encoding='bytes'), 'π')
987 # From Python 2: pickle.dumps(u'π', protocol=1)
988 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
989 encoding="bytes"), 'π')
990 # From Python 2: pickle.dumps(u'π', protocol=2)
991 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
992 encoding="bytes"), 'π')
993
994 def test_load_long_python2_str_as_bytes(self):
995 # From Python 2: pickle.dumps('x' * 300, protocol=1)
996 self.assertEqual(self.loads(pickle.BINSTRING +
997 struct.pack("<I", 300) +
998 b'x' * 300 + pickle.STOP,
999 encoding='bytes'), b'x' * 300)
1000
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001001 def test_constants(self):
1002 self.assertIsNone(self.loads(b'N.'))
1003 self.assertIs(self.loads(b'\x88.'), True)
1004 self.assertIs(self.loads(b'\x89.'), False)
1005 self.assertIs(self.loads(b'I01\n.'), True)
1006 self.assertIs(self.loads(b'I00\n.'), False)
1007
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001008 def test_empty_bytestring(self):
1009 # issue 11286
1010 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
1011 self.assertEqual(empty, '')
1012
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001013 def test_short_binbytes(self):
1014 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
1015 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1016
1017 def test_binbytes(self):
1018 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
1019 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1020
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001021 @requires_32b
1022 def test_negative_32b_binbytes(self):
1023 # On 32-bit builds, a BINBYTES of 2**31 or more is refused
1024 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001025 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1026 dumped)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001027
1028 @requires_32b
1029 def test_negative_32b_binunicode(self):
1030 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused
1031 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001032 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1033 dumped)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001034
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001035 def test_short_binunicode(self):
1036 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
1037 self.assertEqual(self.loads(dumped), '\u20ac\x00')
1038
1039 def test_misc_get(self):
Claudiu Popa6f03b232019-11-24 20:15:08 +01001040 self.check_unpickling_error(pickle.UnpicklingError, b'g0\np0')
1041 self.check_unpickling_error(pickle.UnpicklingError, b'jens:')
1042 self.check_unpickling_error(pickle.UnpicklingError, b'hens:')
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001043 self.assert_is_copy([(100,), (100,)],
1044 self.loads(b'((Kdtp0\nh\x00l.))'))
1045
Serhiy Storchakae0606192015-09-29 22:10:07 +03001046 def test_binbytes8(self):
1047 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
1048 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1049
1050 def test_binunicode8(self):
1051 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
1052 self.assertEqual(self.loads(dumped), '\u20ac\x00')
1053
Antoine Pitrou91f43802019-05-26 17:10:09 +02001054 def test_bytearray8(self):
1055 dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.'
1056 self.assertEqual(self.loads(dumped), bytearray(b'xxx'))
1057
Serhiy Storchakae0606192015-09-29 22:10:07 +03001058 @requires_32b
1059 def test_large_32b_binbytes8(self):
1060 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001061 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1062 dumped)
Serhiy Storchakae0606192015-09-29 22:10:07 +03001063
1064 @requires_32b
Antoine Pitrou91f43802019-05-26 17:10:09 +02001065 def test_large_32b_bytearray8(self):
1066 dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1067 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1068 dumped)
1069
1070 @requires_32b
Serhiy Storchakae0606192015-09-29 22:10:07 +03001071 def test_large_32b_binunicode8(self):
1072 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001073 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1074 dumped)
Serhiy Storchakae0606192015-09-29 22:10:07 +03001075
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001076 def test_get(self):
1077 pickled = b'((lp100000\ng100000\nt.'
1078 unpickled = self.loads(pickled)
1079 self.assertEqual(unpickled, ([],)*2)
1080 self.assertIs(unpickled[0], unpickled[1])
1081
1082 def test_binget(self):
1083 pickled = b'(]q\xffh\xfft.'
1084 unpickled = self.loads(pickled)
1085 self.assertEqual(unpickled, ([],)*2)
1086 self.assertIs(unpickled[0], unpickled[1])
1087
1088 def test_long_binget(self):
1089 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
1090 unpickled = self.loads(pickled)
1091 self.assertEqual(unpickled, ([],)*2)
1092 self.assertIs(unpickled[0], unpickled[1])
1093
1094 def test_dup(self):
1095 pickled = b'((l2t.'
1096 unpickled = self.loads(pickled)
1097 self.assertEqual(unpickled, ([],)*2)
1098 self.assertIs(unpickled[0], unpickled[1])
1099
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001100 def test_negative_put(self):
1101 # Issue #12847
1102 dumped = b'Va\np-1\n.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001103 self.check_unpickling_error(ValueError, dumped)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001104
1105 @requires_32b
1106 def test_negative_32b_binput(self):
1107 # Issue #12847
1108 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001109 self.check_unpickling_error(ValueError, dumped)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001110
1111 def test_badly_escaped_string(self):
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001112 self.check_unpickling_error(ValueError, b"S'\\'\n.")
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001113
1114 def test_badly_quoted_string(self):
1115 # Issue #17710
1116 badpickles = [b"S'\n.",
1117 b'S"\n.',
1118 b'S\' \n.',
1119 b'S" \n.',
1120 b'S\'"\n.',
1121 b'S"\'\n.',
1122 b"S' ' \n.",
1123 b'S" " \n.',
1124 b"S ''\n.",
1125 b'S ""\n.',
1126 b'S \n.',
1127 b'S\n.',
1128 b'S.']
1129 for p in badpickles:
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001130 self.check_unpickling_error(pickle.UnpicklingError, p)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001131
1132 def test_correctly_quoted_string(self):
1133 goodpickles = [(b"S''\n.", ''),
1134 (b'S""\n.', ''),
1135 (b'S"\\n"\n.', '\n'),
1136 (b"S'\\n'\n.", '\n')]
1137 for p, expected in goodpickles:
1138 self.assertEqual(self.loads(p), expected)
1139
1140 def test_frame_readline(self):
1141 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.'
1142 # 0: \x80 PROTO 4
1143 # 2: \x95 FRAME 5
1144 # 11: I INT 42
1145 # 15: . STOP
1146 self.assertEqual(self.loads(pickled), 42)
1147
1148 def test_compat_unpickle(self):
1149 # xrange(1, 7)
1150 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.'
1151 unpickled = self.loads(pickled)
1152 self.assertIs(type(unpickled), range)
1153 self.assertEqual(unpickled, range(1, 7))
1154 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6])
1155 # reduce
1156 pickled = b'\x80\x02c__builtin__\nreduce\n.'
1157 self.assertIs(self.loads(pickled), functools.reduce)
1158 # whichdb.whichdb
1159 pickled = b'\x80\x02cwhichdb\nwhichdb\n.'
1160 self.assertIs(self.loads(pickled), dbm.whichdb)
1161 # Exception(), StandardError()
1162 for name in (b'Exception', b'StandardError'):
1163 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.')
1164 unpickled = self.loads(pickled)
1165 self.assertIs(type(unpickled), Exception)
1166 self.assertEqual(str(unpickled), 'ugh')
1167 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2})
1168 for name in (b'UserDict', b'IterableUserDict'):
1169 pickled = (b'\x80\x02(cUserDict\n' + name +
1170 b'\no}U\x04data}K\x01K\x02ssb.')
1171 unpickled = self.loads(pickled)
1172 self.assertIs(type(unpickled), collections.UserDict)
1173 self.assertEqual(unpickled, collections.UserDict({1: 2}))
1174
Serhiy Storchakae9b30742015-11-23 15:17:43 +02001175 def test_bad_stack(self):
1176 badpickles = [
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001177 b'.', # STOP
1178 b'0', # POP
1179 b'1', # POP_MARK
1180 b'2', # DUP
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02001181 b'(2',
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001182 b'R', # REDUCE
1183 b')R',
1184 b'a', # APPEND
1185 b'Na',
1186 b'b', # BUILD
1187 b'Nb',
1188 b'd', # DICT
1189 b'e', # APPENDS
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02001190 b'(e',
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001191 b'ibuiltins\nlist\n', # INST
1192 b'l', # LIST
1193 b'o', # OBJ
1194 b'(o',
1195 b'p1\n', # PUT
1196 b'q\x00', # BINPUT
1197 b'r\x00\x00\x00\x00', # LONG_BINPUT
1198 b's', # SETITEM
1199 b'Ns',
1200 b'NNs',
1201 b't', # TUPLE
1202 b'u', # SETITEMS
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02001203 b'(u',
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001204 b'}(Nu',
1205 b'\x81', # NEWOBJ
1206 b')\x81',
1207 b'\x85', # TUPLE1
1208 b'\x86', # TUPLE2
1209 b'N\x86',
1210 b'\x87', # TUPLE3
1211 b'N\x87',
1212 b'NN\x87',
1213 b'\x90', # ADDITEMS
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02001214 b'(\x90',
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001215 b'\x91', # FROZENSET
1216 b'\x92', # NEWOBJ_EX
1217 b')}\x92',
1218 b'\x93', # STACK_GLOBAL
1219 b'Vlist\n\x93',
1220 b'\x94', # MEMOIZE
Serhiy Storchakae9b30742015-11-23 15:17:43 +02001221 ]
1222 for p in badpickles:
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001223 self.check_unpickling_error(self.bad_stack_errors, p)
Serhiy Storchakae9b30742015-11-23 15:17:43 +02001224
1225 def test_bad_mark(self):
1226 badpickles = [
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02001227 b'N(.', # STOP
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001228 b'N(2', # DUP
1229 b'cbuiltins\nlist\n)(R', # REDUCE
1230 b'cbuiltins\nlist\n()R',
1231 b']N(a', # APPEND
1232 # BUILD
1233 b'cbuiltins\nValueError\n)R}(b',
1234 b'cbuiltins\nValueError\n)R(}b',
1235 b'(Nd', # DICT
1236 b'N(p1\n', # PUT
1237 b'N(q\x00', # BINPUT
1238 b'N(r\x00\x00\x00\x00', # LONG_BINPUT
1239 b'}NN(s', # SETITEM
1240 b'}N(Ns',
1241 b'}(NNs',
1242 b'}((u', # SETITEMS
1243 b'cbuiltins\nlist\n)(\x81', # NEWOBJ
1244 b'cbuiltins\nlist\n()\x81',
1245 b'N(\x85', # TUPLE1
1246 b'NN(\x86', # TUPLE2
1247 b'N(N\x86',
1248 b'NNN(\x87', # TUPLE3
1249 b'NN(N\x87',
1250 b'N(NN\x87',
1251 b']((\x90', # ADDITEMS
1252 # NEWOBJ_EX
1253 b'cbuiltins\nlist\n)}(\x92',
1254 b'cbuiltins\nlist\n)(}\x92',
1255 b'cbuiltins\nlist\n()}\x92',
1256 # STACK_GLOBAL
1257 b'Vbuiltins\n(Vlist\n\x93',
1258 b'Vbuiltins\nVlist\n(\x93',
1259 b'N(\x94', # MEMOIZE
Serhiy Storchakae9b30742015-11-23 15:17:43 +02001260 ]
1261 for p in badpickles:
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02001262 self.check_unpickling_error(self.bad_stack_errors, p)
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001263
1264 def test_truncated_data(self):
1265 self.check_unpickling_error(EOFError, b'')
1266 self.check_unpickling_error(EOFError, b'N')
1267 badpickles = [
1268 b'B', # BINBYTES
1269 b'B\x03\x00\x00',
1270 b'B\x03\x00\x00\x00',
1271 b'B\x03\x00\x00\x00ab',
1272 b'C', # SHORT_BINBYTES
1273 b'C\x03',
1274 b'C\x03ab',
1275 b'F', # FLOAT
1276 b'F0.0',
1277 b'F0.00',
1278 b'G', # BINFLOAT
1279 b'G\x00\x00\x00\x00\x00\x00\x00',
1280 b'I', # INT
1281 b'I0',
1282 b'J', # BININT
1283 b'J\x00\x00\x00',
1284 b'K', # BININT1
1285 b'L', # LONG
1286 b'L0',
1287 b'L10',
1288 b'L0L',
1289 b'L10L',
1290 b'M', # BININT2
1291 b'M\x00',
1292 # b'P', # PERSID
1293 # b'Pabc',
1294 b'S', # STRING
1295 b"S'abc'",
1296 b'T', # BINSTRING
1297 b'T\x03\x00\x00',
1298 b'T\x03\x00\x00\x00',
1299 b'T\x03\x00\x00\x00ab',
1300 b'U', # SHORT_BINSTRING
1301 b'U\x03',
1302 b'U\x03ab',
1303 b'V', # UNICODE
1304 b'Vabc',
1305 b'X', # BINUNICODE
1306 b'X\x03\x00\x00',
1307 b'X\x03\x00\x00\x00',
1308 b'X\x03\x00\x00\x00ab',
1309 b'(c', # GLOBAL
1310 b'(cbuiltins',
1311 b'(cbuiltins\n',
1312 b'(cbuiltins\nlist',
1313 b'Ng', # GET
1314 b'Ng0',
1315 b'(i', # INST
1316 b'(ibuiltins',
1317 b'(ibuiltins\n',
1318 b'(ibuiltins\nlist',
1319 b'Nh', # BINGET
1320 b'Nj', # LONG_BINGET
1321 b'Nj\x00\x00\x00',
1322 b'Np', # PUT
1323 b'Np0',
1324 b'Nq', # BINPUT
1325 b'Nr', # LONG_BINPUT
1326 b'Nr\x00\x00\x00',
1327 b'\x80', # PROTO
1328 b'\x82', # EXT1
1329 b'\x83', # EXT2
1330 b'\x84\x01',
1331 b'\x84', # EXT4
1332 b'\x84\x01\x00\x00',
1333 b'\x8a', # LONG1
1334 b'\x8b', # LONG4
1335 b'\x8b\x00\x00\x00',
1336 b'\x8c', # SHORT_BINUNICODE
1337 b'\x8c\x03',
1338 b'\x8c\x03ab',
1339 b'\x8d', # BINUNICODE8
1340 b'\x8d\x03\x00\x00\x00\x00\x00\x00',
1341 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00',
1342 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab',
1343 b'\x8e', # BINBYTES8
1344 b'\x8e\x03\x00\x00\x00\x00\x00\x00',
1345 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00',
1346 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab',
Antoine Pitrou91f43802019-05-26 17:10:09 +02001347 b'\x96', # BYTEARRAY8
1348 b'\x96\x03\x00\x00\x00\x00\x00\x00',
1349 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00',
1350 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab',
Serhiy Storchaka7279bef2015-11-29 13:12:10 +02001351 b'\x95', # FRAME
1352 b'\x95\x02\x00\x00\x00\x00\x00\x00',
1353 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00',
1354 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N',
1355 ]
1356 for p in badpickles:
1357 self.check_unpickling_error(self.truncated_errors, p)
Serhiy Storchakae9b30742015-11-23 15:17:43 +02001358
tjb9004371c0a2019-02-18 23:30:51 +08001359 @reap_threads
1360 def test_unpickle_module_race(self):
1361 # https://bugs.python.org/issue34572
1362 locker_module = dedent("""
1363 import threading
1364 barrier = threading.Barrier(2)
1365 """)
1366 locking_import_module = dedent("""
1367 import locker
1368 locker.barrier.wait()
1369 class ToBeUnpickled(object):
1370 pass
1371 """)
1372
1373 os.mkdir(TESTFN)
1374 self.addCleanup(shutil.rmtree, TESTFN)
1375 sys.path.insert(0, TESTFN)
1376 self.addCleanup(sys.path.remove, TESTFN)
1377 with open(os.path.join(TESTFN, "locker.py"), "wb") as f:
1378 f.write(locker_module.encode('utf-8'))
1379 with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f:
1380 f.write(locking_import_module.encode('utf-8'))
1381 self.addCleanup(forget, "locker")
1382 self.addCleanup(forget, "locking_import")
1383
1384 import locker
1385
1386 pickle_bytes = (
1387 b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.')
1388
1389 # Then try to unpickle two of these simultaneously
1390 # One of them will cause the module import, and we want it to block
1391 # until the other one either:
1392 # - fails (before the patch for this issue)
1393 # - blocks on the import lock for the module, as it should
1394 results = []
1395 barrier = threading.Barrier(3)
1396 def t():
1397 # This ensures the threads have all started
1398 # presumably barrier release is faster than thread startup
1399 barrier.wait()
1400 results.append(pickle.loads(pickle_bytes))
1401
1402 t1 = threading.Thread(target=t)
1403 t2 = threading.Thread(target=t)
1404 t1.start()
1405 t2.start()
1406
1407 barrier.wait()
1408 # could have delay here
1409 locker.barrier.wait()
1410
1411 t1.join()
1412 t2.join()
1413
1414 from locking_import import ToBeUnpickled
1415 self.assertEqual(
1416 [type(x) for x in results],
1417 [ToBeUnpickled] * 2)
1418
1419
Serhiy Storchakac6b54b42015-09-29 15:33:24 +03001420
1421class AbstractPickleTests(unittest.TestCase):
1422 # Subclass must define self.dumps, self.loads.
1423
1424 optimized = False
1425
1426 _testdata = AbstractUnpickleTests._testdata
1427
1428 def setUp(self):
1429 pass
1430
1431 assert_is_copy = AbstractUnpickleTests.assert_is_copy
1432
1433 def test_misc(self):
1434 # test various datatypes not tested by testdata
1435 for proto in protocols:
1436 x = myint(4)
1437 s = self.dumps(x, proto)
1438 y = self.loads(s)
1439 self.assert_is_copy(x, y)
1440
1441 x = (1, ())
1442 s = self.dumps(x, proto)
1443 y = self.loads(s)
1444 self.assert_is_copy(x, y)
1445
1446 x = initarg(1, x)
1447 s = self.dumps(x, proto)
1448 y = self.loads(s)
1449 self.assert_is_copy(x, y)
1450
1451 # XXX test __reduce__ protocol?
1452
1453 def test_roundtrip_equality(self):
1454 expected = self._testdata
1455 for proto in protocols:
1456 s = self.dumps(expected, proto)
1457 got = self.loads(s)
1458 self.assert_is_copy(expected, got)
1459
Tim Peters70b02d72003-02-02 17:26:40 +00001460 # There are gratuitous differences between pickles produced by
1461 # pickle and cPickle, largely because cPickle starts PUT indices at
1462 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() --
1463 # there's a comment with an exclamation point there whose meaning
1464 # is a mystery. cPickle also suppresses PUT for objects with a refcount
1465 # of 1.
1466 def dont_test_disassembly(self):
Guido van Rossum34d19282007-08-09 01:03:29 +00001467 from io import StringIO
Tim Peters70b02d72003-02-02 17:26:40 +00001468 from pickletools import dis
1469
1470 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
1471 s = self.dumps(self._testdata, proto)
1472 filelike = StringIO()
1473 dis(s, out=filelike)
1474 got = filelike.getvalue()
1475 self.assertEqual(expected, got)
Jeremy Hylton66426532001-10-15 21:38:56 +00001476
1477 def test_recursive_list(self):
1478 l = []
1479 l.append(l)
Tim Peters70b02d72003-02-02 17:26:40 +00001480 for proto in protocols:
1481 s = self.dumps(l, proto)
1482 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001483 self.assertIsInstance(x, list)
Armin Rigo2b3eb402003-10-28 12:05:48 +00001484 self.assertEqual(len(x), 1)
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001485 self.assertIs(x[0], x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001486
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001487 def test_recursive_tuple_and_list(self):
Collin Winter8ca69de2009-05-26 16:53:41 +00001488 t = ([],)
1489 t[0].append(t)
1490 for proto in protocols:
1491 s = self.dumps(t, proto)
1492 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001493 self.assertIsInstance(x, tuple)
Collin Winter8ca69de2009-05-26 16:53:41 +00001494 self.assertEqual(len(x), 1)
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001495 self.assertIsInstance(x[0], list)
Collin Winter8ca69de2009-05-26 16:53:41 +00001496 self.assertEqual(len(x[0]), 1)
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001497 self.assertIs(x[0][0], x)
Collin Winter8ca69de2009-05-26 16:53:41 +00001498
Jeremy Hylton66426532001-10-15 21:38:56 +00001499 def test_recursive_dict(self):
1500 d = {}
1501 d[1] = d
Tim Peters70b02d72003-02-02 17:26:40 +00001502 for proto in protocols:
1503 s = self.dumps(d, proto)
1504 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001505 self.assertIsInstance(x, dict)
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001506 self.assertEqual(list(x.keys()), [1])
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001507 self.assertIs(x[1], x)
1508
1509 def test_recursive_dict_key(self):
1510 d = {}
1511 k = K(d)
1512 d[k] = 1
1513 for proto in protocols:
1514 s = self.dumps(d, proto)
1515 x = self.loads(s)
1516 self.assertIsInstance(x, dict)
1517 self.assertEqual(len(x.keys()), 1)
1518 self.assertIsInstance(list(x.keys())[0], K)
1519 self.assertIs(list(x.keys())[0].value, x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001520
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001521 def test_recursive_set(self):
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001522 y = set()
1523 k = K(y)
1524 y.add(k)
1525 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001526 s = self.dumps(y, proto)
1527 x = self.loads(s)
1528 self.assertIsInstance(x, set)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001529 self.assertEqual(len(x), 1)
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001530 self.assertIsInstance(list(x)[0], K)
1531 self.assertIs(list(x)[0].value, x)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001532
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001533 def test_recursive_list_subclass(self):
1534 y = MyList()
1535 y.append(y)
1536 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001537 s = self.dumps(y, proto)
1538 x = self.loads(s)
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001539 self.assertIsInstance(x, MyList)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001540 self.assertEqual(len(x), 1)
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001541 self.assertIs(x[0], x)
1542
1543 def test_recursive_dict_subclass(self):
1544 d = MyDict()
1545 d[1] = d
1546 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
1547 s = self.dumps(d, proto)
1548 x = self.loads(s)
1549 self.assertIsInstance(x, MyDict)
1550 self.assertEqual(list(x.keys()), [1])
1551 self.assertIs(x[1], x)
1552
1553 def test_recursive_dict_subclass_key(self):
1554 d = MyDict()
1555 k = K(d)
1556 d[k] = 1
1557 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
1558 s = self.dumps(d, proto)
1559 x = self.loads(s)
1560 self.assertIsInstance(x, MyDict)
1561 self.assertEqual(len(list(x.keys())), 1)
1562 self.assertIsInstance(list(x.keys())[0], K)
1563 self.assertIs(list(x.keys())[0].value, x)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001564
Jeremy Hylton66426532001-10-15 21:38:56 +00001565 def test_recursive_inst(self):
1566 i = C()
1567 i.attr = i
Tim Peters70b02d72003-02-02 17:26:40 +00001568 for proto in protocols:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001569 s = self.dumps(i, proto)
Tim Peters70b02d72003-02-02 17:26:40 +00001570 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001571 self.assertIsInstance(x, C)
Armin Rigo2b3eb402003-10-28 12:05:48 +00001572 self.assertEqual(dir(x), dir(i))
Ezio Melottiaaef3442013-03-04 15:17:56 +02001573 self.assertIs(x.attr, x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001574
1575 def test_recursive_multi(self):
1576 l = []
1577 d = {1:l}
1578 i = C()
1579 i.attr = d
1580 l.append(i)
Tim Peters70b02d72003-02-02 17:26:40 +00001581 for proto in protocols:
1582 s = self.dumps(l, proto)
1583 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001584 self.assertIsInstance(x, list)
Armin Rigo2b3eb402003-10-28 12:05:48 +00001585 self.assertEqual(len(x), 1)
1586 self.assertEqual(dir(x[0]), dir(i))
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001587 self.assertEqual(list(x[0].attr.keys()), [1])
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001588 self.assertTrue(x[0].attr[1] is x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001589
Serhiy Storchaka608c2132015-11-07 11:16:10 +02001590 def check_recursive_collection_and_inst(self, factory):
1591 h = H()
1592 y = factory([h])
1593 h.attr = y
1594 for proto in protocols:
1595 s = self.dumps(y, proto)
1596 x = self.loads(s)
1597 self.assertIsInstance(x, type(y))
1598 self.assertEqual(len(x), 1)
1599 self.assertIsInstance(list(x)[0], H)
1600 self.assertIs(list(x)[0].attr, x)
1601
1602 def test_recursive_list_and_inst(self):
1603 self.check_recursive_collection_and_inst(list)
1604
1605 def test_recursive_tuple_and_inst(self):
1606 self.check_recursive_collection_and_inst(tuple)
1607
1608 def test_recursive_dict_and_inst(self):
1609 self.check_recursive_collection_and_inst(dict.fromkeys)
1610
1611 def test_recursive_set_and_inst(self):
1612 self.check_recursive_collection_and_inst(set)
1613
1614 def test_recursive_frozenset_and_inst(self):
1615 self.check_recursive_collection_and_inst(frozenset)
1616
1617 def test_recursive_list_subclass_and_inst(self):
1618 self.check_recursive_collection_and_inst(MyList)
1619
1620 def test_recursive_tuple_subclass_and_inst(self):
1621 self.check_recursive_collection_and_inst(MyTuple)
1622
1623 def test_recursive_dict_subclass_and_inst(self):
1624 self.check_recursive_collection_and_inst(MyDict.fromkeys)
1625
1626 def test_recursive_set_subclass_and_inst(self):
1627 self.check_recursive_collection_and_inst(MySet)
1628
1629 def test_recursive_frozenset_subclass_and_inst(self):
1630 self.check_recursive_collection_and_inst(MyFrozenSet)
1631
Walter Dörwald9b775532007-06-08 14:30:53 +00001632 def test_unicode(self):
Benjamin Petersond1486302008-12-27 16:58:50 +00001633 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
Victor Stinner485fb562010-04-13 11:07:24 +00001634 '<\\>', '<\\\U00012345>',
1635 # surrogates
1636 '<\udc80>']
Walter Dörwald9b775532007-06-08 14:30:53 +00001637 for proto in protocols:
1638 for u in endcases:
1639 p = self.dumps(u, proto)
1640 u2 = self.loads(p)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001641 self.assert_is_copy(u, u2)
Tim Peterse089c682001-04-10 03:41:41 +00001642
Alexandre Vassalotti554d8782008-12-27 07:32:41 +00001643 def test_unicode_high_plane(self):
1644 t = '\U00012345'
1645 for proto in protocols:
1646 p = self.dumps(t, proto)
1647 t2 = self.loads(p)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001648 self.assert_is_copy(t, t2)
Alexandre Vassalotti554d8782008-12-27 07:32:41 +00001649
Guido van Rossumf4169812008-03-17 22:56:06 +00001650 def test_bytes(self):
1651 for proto in protocols:
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05001652 for s in b'', b'xyz', b'xyz'*100:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001653 p = self.dumps(s, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001654 self.assert_is_copy(s, self.loads(p))
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05001655 for s in [bytes([i]) for i in range(256)]:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001656 p = self.dumps(s, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001657 self.assert_is_copy(s, self.loads(p))
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05001658 for s in [bytes([i, i]) for i in range(256)]:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001659 p = self.dumps(s, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001660 self.assert_is_copy(s, self.loads(p))
Guido van Rossumf4169812008-03-17 22:56:06 +00001661
Antoine Pitrou91f43802019-05-26 17:10:09 +02001662 def test_bytearray(self):
1663 for proto in protocols:
1664 for s in b'', b'xyz', b'xyz'*100:
1665 b = bytearray(s)
1666 p = self.dumps(b, proto)
1667 bb = self.loads(p)
1668 self.assertIsNot(bb, b)
1669 self.assert_is_copy(b, bb)
1670 if proto <= 3:
1671 # bytearray is serialized using a global reference
1672 self.assertIn(b'bytearray', p)
1673 self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p))
1674 elif proto == 4:
1675 self.assertIn(b'bytearray', p)
1676 self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p))
1677 elif proto == 5:
1678 self.assertNotIn(b'bytearray', p)
1679 self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p))
1680
Jeremy Hylton66426532001-10-15 21:38:56 +00001681 def test_ints(self):
Tim Petersee1a53c2003-02-02 02:57:53 +00001682 for proto in protocols:
Christian Heimesa37d4c62007-12-04 23:02:19 +00001683 n = sys.maxsize
Tim Petersee1a53c2003-02-02 02:57:53 +00001684 while n:
1685 for expected in (-n, n):
1686 s = self.dumps(expected, proto)
1687 n2 = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001688 self.assert_is_copy(expected, n2)
Tim Petersee1a53c2003-02-02 02:57:53 +00001689 n = n >> 1
Tim Peters19ef62d2001-08-28 22:21:18 +00001690
Tim Petersee1a53c2003-02-02 02:57:53 +00001691 def test_long(self):
1692 for proto in protocols:
Tim Petersbf2674b2003-02-02 07:51:32 +00001693 # 256 bytes is where LONG4 begins.
Tim Petersee1a53c2003-02-02 02:57:53 +00001694 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257:
Guido van Rossume2a383d2007-01-15 16:59:06 +00001695 nbase = 1 << nbits
Tim Petersee1a53c2003-02-02 02:57:53 +00001696 for npos in nbase-1, nbase, nbase+1:
1697 for n in npos, -npos:
1698 pickle = self.dumps(n, proto)
1699 got = self.loads(pickle)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001700 self.assert_is_copy(n, got)
Tim Petersee1a53c2003-02-02 02:57:53 +00001701 # Try a monster. This is quadratic-time in protos 0 & 1, so don't
1702 # bother with those.
Guido van Rossume2a383d2007-01-15 16:59:06 +00001703 nbase = int("deadbeeffeedface", 16)
Tim Petersee1a53c2003-02-02 02:57:53 +00001704 nbase += nbase << 1000000
1705 for n in nbase, -nbase:
Tim Petersee1a53c2003-02-02 02:57:53 +00001706 p = self.dumps(n, 2)
Tim Petersee1a53c2003-02-02 02:57:53 +00001707 got = self.loads(p)
Antoine Pitroud5df1942013-11-23 21:20:49 +01001708 # assert_is_copy is very expensive here as it precomputes
1709 # a failure message by computing the repr() of n and got,
1710 # we just do the check ourselves.
1711 self.assertIs(type(got), int)
1712 self.assertEqual(n, got)
Tim Petersee1a53c2003-02-02 02:57:53 +00001713
Mark Dickinsoncddcf442009-01-24 21:46:33 +00001714 def test_float(self):
1715 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
1716 3.14, 263.44582062374053, 6.022e23, 1e30]
1717 test_values = test_values + [-x for x in test_values]
1718 for proto in protocols:
1719 for value in test_values:
1720 pickle = self.dumps(value, proto)
1721 got = self.loads(pickle)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001722 self.assert_is_copy(value, got)
Mark Dickinsoncddcf442009-01-24 21:46:33 +00001723
Thomas Wouters477c8d52006-05-27 19:21:47 +00001724 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
1725 def test_float_format(self):
Guido van Rossumf4169812008-03-17 22:56:06 +00001726 # make sure that floats are formatted locale independent with proto 0
1727 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001728
Jeremy Hylton66426532001-10-15 21:38:56 +00001729 def test_reduce(self):
Antoine Pitrouc1764dd2013-12-28 16:57:37 +01001730 for proto in protocols:
1731 inst = AAA()
1732 dumped = self.dumps(inst, proto)
1733 loaded = self.loads(dumped)
1734 self.assertEqual(loaded, REDUCE_A)
Jeremy Hylton66426532001-10-15 21:38:56 +00001735
1736 def test_getinitargs(self):
Antoine Pitrouc1764dd2013-12-28 16:57:37 +01001737 for proto in protocols:
1738 inst = initarg(1, 2)
1739 dumped = self.dumps(inst, proto)
1740 loaded = self.loads(dumped)
1741 self.assert_is_copy(inst, loaded)
Jeremy Hylton66426532001-10-15 21:38:56 +00001742
Guido van Rossum04a86612001-12-19 16:58:54 +00001743 def test_metaclass(self):
1744 a = use_metaclass()
Tim Peters70b02d72003-02-02 17:26:40 +00001745 for proto in protocols:
1746 s = self.dumps(a, proto)
1747 b = self.loads(s)
1748 self.assertEqual(a.__class__, b.__class__)
Guido van Rossum04a86612001-12-19 16:58:54 +00001749
Antoine Pitrouffd41d92011-10-04 09:23:04 +02001750 def test_dynamic_class(self):
1751 a = create_dynamic_class("my_dynamic_class", (object,))
1752 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
1753 for proto in protocols:
1754 s = self.dumps(a, proto)
1755 b = self.loads(s)
1756 self.assertEqual(a, b)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001757 self.assertIs(type(a), type(b))
Antoine Pitrouffd41d92011-10-04 09:23:04 +02001758
Michael W. Hudson7bb466a2002-03-05 13:27:58 +00001759 def test_structseq(self):
1760 import time
Michael W. Hudson0e025302002-03-06 17:11:18 +00001761 import os
Tim Peters70b02d72003-02-02 17:26:40 +00001762
1763 t = time.localtime()
1764 for proto in protocols:
1765 s = self.dumps(t, proto)
Michael W. Hudson0e025302002-03-06 17:11:18 +00001766 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001767 self.assert_is_copy(t, u)
Anthony Sottile8377cd42019-02-25 14:32:27 -08001768 t = os.stat(os.curdir)
1769 s = self.dumps(t, proto)
1770 u = self.loads(s)
1771 self.assert_is_copy(t, u)
Tim Peters70b02d72003-02-02 17:26:40 +00001772 if hasattr(os, "statvfs"):
1773 t = os.statvfs(os.curdir)
1774 s = self.dumps(t, proto)
1775 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001776 self.assert_is_copy(t, u)
Michael W. Hudson7bb466a2002-03-05 13:27:58 +00001777
Łukasz Langaf3078fb2012-03-12 19:46:12 +01001778 def test_ellipsis(self):
1779 for proto in protocols:
1780 s = self.dumps(..., proto)
1781 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001782 self.assertIs(..., u)
Łukasz Langaf3078fb2012-03-12 19:46:12 +01001783
1784 def test_notimplemented(self):
1785 for proto in protocols:
1786 s = self.dumps(NotImplemented, proto)
1787 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001788 self.assertIs(NotImplemented, u)
Łukasz Langaf3078fb2012-03-12 19:46:12 +01001789
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -08001790 def test_singleton_types(self):
1791 # Issue #6477: Test that types of built-in singletons can be pickled.
1792 singletons = [None, ..., NotImplemented]
1793 for singleton in singletons:
1794 for proto in protocols:
1795 s = self.dumps(type(singleton), proto)
1796 u = self.loads(s)
1797 self.assertIs(type(singleton), u)
1798
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001799 # Tests for protocol 2
1800
Tim Peters4190fb82003-02-02 16:09:05 +00001801 def test_proto(self):
Tim Peters4190fb82003-02-02 16:09:05 +00001802 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001803 pickled = self.dumps(None, proto)
Tim Peters4190fb82003-02-02 16:09:05 +00001804 if proto >= 2:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001805 proto_header = pickle.PROTO + bytes([proto])
1806 self.assertTrue(pickled.startswith(proto_header))
1807 else:
1808 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
Tim Peters4190fb82003-02-02 16:09:05 +00001809
1810 oob = protocols[-1] + 1 # a future protocol
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001811 build_none = pickle.NONE + pickle.STOP
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001812 badpickle = pickle.PROTO + bytes([oob]) + build_none
Tim Peters4190fb82003-02-02 16:09:05 +00001813 try:
1814 self.loads(badpickle)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001815 except ValueError as err:
1816 self.assertIn("unsupported pickle protocol", str(err))
Tim Peters4190fb82003-02-02 16:09:05 +00001817 else:
1818 self.fail("expected bad protocol number to raise ValueError")
1819
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001820 def test_long1(self):
Guido van Rossume2a383d2007-01-15 16:59:06 +00001821 x = 12345678910111213141516178920
Tim Peters61bf2572003-02-03 21:31:22 +00001822 for proto in protocols:
1823 s = self.dumps(x, proto)
1824 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001825 self.assert_is_copy(x, y)
Tim Peters22e71712003-02-03 22:27:38 +00001826 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001827
1828 def test_long4(self):
Guido van Rossume2a383d2007-01-15 16:59:06 +00001829 x = 12345678910111213141516178920 << (256*8)
Tim Peters61bf2572003-02-03 21:31:22 +00001830 for proto in protocols:
1831 s = self.dumps(x, proto)
1832 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001833 self.assert_is_copy(x, y)
Tim Peters22e71712003-02-03 22:27:38 +00001834 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001835
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001836 def test_short_tuples(self):
Tim Peters1d63c9f2003-02-02 20:29:39 +00001837 # Map (proto, len(tuple)) to expected opcode.
1838 expected_opcode = {(0, 0): pickle.TUPLE,
1839 (0, 1): pickle.TUPLE,
1840 (0, 2): pickle.TUPLE,
1841 (0, 3): pickle.TUPLE,
1842 (0, 4): pickle.TUPLE,
1843
1844 (1, 0): pickle.EMPTY_TUPLE,
1845 (1, 1): pickle.TUPLE,
1846 (1, 2): pickle.TUPLE,
1847 (1, 3): pickle.TUPLE,
1848 (1, 4): pickle.TUPLE,
1849
1850 (2, 0): pickle.EMPTY_TUPLE,
1851 (2, 1): pickle.TUPLE1,
1852 (2, 2): pickle.TUPLE2,
1853 (2, 3): pickle.TUPLE3,
1854 (2, 4): pickle.TUPLE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001855
1856 (3, 0): pickle.EMPTY_TUPLE,
1857 (3, 1): pickle.TUPLE1,
1858 (3, 2): pickle.TUPLE2,
1859 (3, 3): pickle.TUPLE3,
1860 (3, 4): pickle.TUPLE,
Tim Peters1d63c9f2003-02-02 20:29:39 +00001861 }
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001862 a = ()
Guido van Rossum025bc2f2003-01-28 04:20:02 +00001863 b = (1,)
1864 c = (1, 2)
1865 d = (1, 2, 3)
1866 e = (1, 2, 3, 4)
Tim Peters4190fb82003-02-02 16:09:05 +00001867 for proto in protocols:
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001868 for x in a, b, c, d, e:
1869 s = self.dumps(x, proto)
1870 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001871 self.assert_is_copy(x, y)
1872 expected = expected_opcode[min(proto, 3), len(x)]
1873 self.assertTrue(opcode_in_pickle(expected, s))
Tim Peters1d63c9f2003-02-02 20:29:39 +00001874
Guido van Rossum7d97d312003-01-28 04:25:27 +00001875 def test_singletons(self):
Tim Peters61bf2572003-02-03 21:31:22 +00001876 # Map (proto, singleton) to expected opcode.
1877 expected_opcode = {(0, None): pickle.NONE,
1878 (1, None): pickle.NONE,
1879 (2, None): pickle.NONE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001880 (3, None): pickle.NONE,
Tim Peters61bf2572003-02-03 21:31:22 +00001881
1882 (0, True): pickle.INT,
1883 (1, True): pickle.INT,
1884 (2, True): pickle.NEWTRUE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001885 (3, True): pickle.NEWTRUE,
Tim Peters61bf2572003-02-03 21:31:22 +00001886
1887 (0, False): pickle.INT,
1888 (1, False): pickle.INT,
1889 (2, False): pickle.NEWFALSE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001890 (3, False): pickle.NEWFALSE,
Tim Peters61bf2572003-02-03 21:31:22 +00001891 }
Tim Peters4190fb82003-02-02 16:09:05 +00001892 for proto in protocols:
Guido van Rossum7d97d312003-01-28 04:25:27 +00001893 for x in None, False, True:
1894 s = self.dumps(x, proto)
1895 y = self.loads(s)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001896 self.assertTrue(x is y, (proto, x, s, y))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001897 expected = expected_opcode[min(proto, 3), x]
1898 self.assertTrue(opcode_in_pickle(expected, s))
Tim Peters3c67d792003-02-02 17:59:11 +00001899
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001900 def test_newobj_tuple(self):
Guido van Rossum3d8c01b2003-01-28 19:48:18 +00001901 x = MyTuple([1, 2, 3])
1902 x.foo = 42
1903 x.bar = "hello"
Tim Peters894453a2003-02-03 22:32:18 +00001904 for proto in protocols:
1905 s = self.dumps(x, proto)
1906 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001907 self.assert_is_copy(x, y)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001908
1909 def test_newobj_list(self):
Guido van Rossum3d8c01b2003-01-28 19:48:18 +00001910 x = MyList([1, 2, 3])
1911 x.foo = 42
1912 x.bar = "hello"
Tim Peters894453a2003-02-03 22:32:18 +00001913 for proto in protocols:
1914 s = self.dumps(x, proto)
1915 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001916 self.assert_is_copy(x, y)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001917
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001918 def test_newobj_generic(self):
Tim Peters5013bd92003-02-03 22:28:41 +00001919 for proto in protocols:
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001920 for C in myclasses:
1921 B = C.__base__
1922 x = C(C.sample)
1923 x.foo = 42
1924 s = self.dumps(x, proto)
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001925 y = self.loads(s)
1926 detail = (proto, C, B, x, y, type(y))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001927 self.assert_is_copy(x, y) # XXX revisit
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001928 self.assertEqual(B(x), B(y), detail)
1929 self.assertEqual(x.__dict__, y.__dict__, detail)
1930
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01001931 def test_newobj_proxies(self):
1932 # NEWOBJ should use the __class__ rather than the raw type
1933 classes = myclasses[:]
1934 # Cannot create weakproxies to these classes
1935 for c in (MyInt, MyTuple):
1936 classes.remove(c)
1937 for proto in protocols:
1938 for C in classes:
1939 B = C.__base__
1940 x = C(C.sample)
1941 x.foo = 42
1942 p = weakref.proxy(x)
1943 s = self.dumps(p, proto)
1944 y = self.loads(s)
1945 self.assertEqual(type(y), type(x)) # rather than type(p)
1946 detail = (proto, C, B, x, y, type(y))
1947 self.assertEqual(B(x), B(y), detail)
1948 self.assertEqual(x.__dict__, y.__dict__, detail)
1949
Benjamin Peterson80f78a32015-07-02 16:18:38 -05001950 def test_newobj_not_class(self):
1951 # Issue 24552
1952 global SimpleNewObj
1953 save = SimpleNewObj
Benjamin Petersond3a2a952015-07-02 16:58:22 -05001954 o = SimpleNewObj.__new__(SimpleNewObj)
Benjamin Peterson80f78a32015-07-02 16:18:38 -05001955 b = self.dumps(o, 4)
1956 try:
1957 SimpleNewObj = 42
1958 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b)
1959 finally:
1960 SimpleNewObj = save
1961
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00001962 # Register a type with copyreg, with extension code extcode. Pickle
Tim Peters22e71712003-02-03 22:27:38 +00001963 # an object of that type. Check that the resulting pickle uses opcode
1964 # (EXT[124]) under proto 2, and not in proto 1.
Tim Peters3e667d52003-02-04 21:47:44 +00001965
Tim Peters22e71712003-02-03 22:27:38 +00001966 def produce_global_ext(self, extcode, opcode):
Tim Peters3e667d52003-02-04 21:47:44 +00001967 e = ExtensionSaver(extcode)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001968 try:
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00001969 copyreg.add_extension(__name__, "MyList", extcode)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001970 x = MyList([1, 2, 3])
1971 x.foo = 42
1972 x.bar = "hello"
1973
Tim Peters22e71712003-02-03 22:27:38 +00001974 # Dump using protocol 1 for comparison.
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001975 s1 = self.dumps(x, 1)
Ezio Melottib58e0bd2010-01-23 15:40:09 +00001976 self.assertIn(__name__.encode("utf-8"), s1)
1977 self.assertIn(b"MyList", s1)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001978 self.assertFalse(opcode_in_pickle(opcode, s1))
Tim Peters3e667d52003-02-04 21:47:44 +00001979
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001980 y = self.loads(s1)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001981 self.assert_is_copy(x, y)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001982
Tim Peters22e71712003-02-03 22:27:38 +00001983 # Dump using protocol 2 for test.
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001984 s2 = self.dumps(x, 2)
Ezio Melottib58e0bd2010-01-23 15:40:09 +00001985 self.assertNotIn(__name__.encode("utf-8"), s2)
1986 self.assertNotIn(b"MyList", s2)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001987 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
Tim Peters3e667d52003-02-04 21:47:44 +00001988
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001989 y = self.loads(s2)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001990 self.assert_is_copy(x, y)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001991 finally:
Tim Peters3e667d52003-02-04 21:47:44 +00001992 e.restore()
Tim Peters22e71712003-02-03 22:27:38 +00001993
1994 def test_global_ext1(self):
Tim Peters3e667d52003-02-04 21:47:44 +00001995 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code
1996 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001997
1998 def test_global_ext2(self):
Tim Peters3e667d52003-02-04 21:47:44 +00001999 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code
2000 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code
2001 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness
Guido van Rossum0322d0f2003-01-29 06:12:46 +00002002
2003 def test_global_ext4(self):
Tim Peters3e667d52003-02-04 21:47:44 +00002004 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code
2005 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code
2006 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness
2007
Tim Peters8d2613a2003-02-11 16:40:16 +00002008 def test_list_chunking(self):
2009 n = 10 # too small to chunk
Guido van Rossum805365e2007-05-07 22:24:25 +00002010 x = list(range(n))
Tim Peters8d2613a2003-02-11 16:40:16 +00002011 for proto in protocols:
2012 s = self.dumps(x, proto)
2013 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002014 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00002015 num_appends = count_opcode(pickle.APPENDS, s)
2016 self.assertEqual(num_appends, proto > 0)
2017
2018 n = 2500 # expect at least two chunks when proto > 0
Guido van Rossum805365e2007-05-07 22:24:25 +00002019 x = list(range(n))
Tim Peters8d2613a2003-02-11 16:40:16 +00002020 for proto in protocols:
2021 s = self.dumps(x, proto)
2022 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002023 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00002024 num_appends = count_opcode(pickle.APPENDS, s)
2025 if proto == 0:
2026 self.assertEqual(num_appends, 0)
2027 else:
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00002028 self.assertTrue(num_appends >= 2)
Tim Peters8d2613a2003-02-11 16:40:16 +00002029
2030 def test_dict_chunking(self):
2031 n = 10 # too small to chunk
2032 x = dict.fromkeys(range(n))
2033 for proto in protocols:
2034 s = self.dumps(x, proto)
Ezio Melottie9615932010-01-24 19:26:24 +00002035 self.assertIsInstance(s, bytes_types)
Tim Peters8d2613a2003-02-11 16:40:16 +00002036 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002037 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00002038 num_setitems = count_opcode(pickle.SETITEMS, s)
2039 self.assertEqual(num_setitems, proto > 0)
2040
2041 n = 2500 # expect at least two chunks when proto > 0
2042 x = dict.fromkeys(range(n))
2043 for proto in protocols:
2044 s = self.dumps(x, proto)
2045 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002046 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00002047 num_setitems = count_opcode(pickle.SETITEMS, s)
2048 if proto == 0:
2049 self.assertEqual(num_setitems, 0)
2050 else:
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00002051 self.assertTrue(num_setitems >= 2)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00002052
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002053 def test_set_chunking(self):
2054 n = 10 # too small to chunk
2055 x = set(range(n))
2056 for proto in protocols:
2057 s = self.dumps(x, proto)
2058 y = self.loads(s)
2059 self.assert_is_copy(x, y)
2060 num_additems = count_opcode(pickle.ADDITEMS, s)
2061 if proto < 4:
2062 self.assertEqual(num_additems, 0)
2063 else:
2064 self.assertEqual(num_additems, 1)
2065
2066 n = 2500 # expect at least two chunks when proto >= 4
2067 x = set(range(n))
2068 for proto in protocols:
2069 s = self.dumps(x, proto)
2070 y = self.loads(s)
2071 self.assert_is_copy(x, y)
2072 num_additems = count_opcode(pickle.ADDITEMS, s)
2073 if proto < 4:
2074 self.assertEqual(num_additems, 0)
2075 else:
2076 self.assertGreaterEqual(num_additems, 2)
2077
Tim Peterse9ef2032003-02-13 18:42:00 +00002078 def test_simple_newobj(self):
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002079 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__
Tim Peterse9ef2032003-02-13 18:42:00 +00002080 x.abc = 666
2081 for proto in protocols:
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002082 with self.subTest(proto=proto):
2083 s = self.dumps(x, proto)
2084 if proto < 1:
Serhiy Storchaka3daaafb2017-11-16 09:44:43 +02002085 self.assertIn(b'\nI64206', s) # INT
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002086 else:
2087 self.assertIn(b'M\xce\xfa', s) # BININT2
2088 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
2089 2 <= proto)
2090 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
2091 y = self.loads(s) # will raise TypeError if __init__ called
2092 self.assert_is_copy(x, y)
2093
2094 def test_complex_newobj(self):
2095 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__
2096 x.abc = 666
2097 for proto in protocols:
2098 with self.subTest(proto=proto):
2099 s = self.dumps(x, proto)
2100 if proto < 1:
Serhiy Storchaka3daaafb2017-11-16 09:44:43 +02002101 self.assertIn(b'\nI64206', s) # INT
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002102 elif proto < 2:
2103 self.assertIn(b'M\xce\xfa', s) # BININT2
2104 elif proto < 4:
2105 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE
2106 else:
2107 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE
2108 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
2109 2 <= proto)
2110 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
2111 y = self.loads(s) # will raise TypeError if __init__ called
2112 self.assert_is_copy(x, y)
2113
2114 def test_complex_newobj_ex(self):
2115 x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__
2116 x.abc = 666
2117 for proto in protocols:
2118 with self.subTest(proto=proto):
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002119 s = self.dumps(x, proto)
2120 if proto < 1:
Serhiy Storchaka3daaafb2017-11-16 09:44:43 +02002121 self.assertIn(b'\nI64206', s) # INT
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002122 elif proto < 2:
2123 self.assertIn(b'M\xce\xfa', s) # BININT2
Serhiy Storchaka0d554d72015-10-10 22:42:18 +03002124 elif proto < 4:
2125 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002126 else:
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02002127 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE
2128 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s))
2129 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
2130 4 <= proto)
2131 y = self.loads(s) # will raise TypeError if __init__ called
2132 self.assert_is_copy(x, y)
Tim Peterse9ef2032003-02-13 18:42:00 +00002133
Tim Peters42f08ac2003-02-11 22:43:24 +00002134 def test_newobj_list_slots(self):
2135 x = SlotList([1, 2, 3])
2136 x.foo = 42
2137 x.bar = "hello"
2138 s = self.dumps(x, 2)
2139 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002140 self.assert_is_copy(x, y)
Tim Peters42f08ac2003-02-11 22:43:24 +00002141
Guido van Rossum2a30b212003-02-18 22:41:24 +00002142 def test_reduce_overrides_default_reduce_ex(self):
Collin Winter771d8342009-04-16 03:18:06 +00002143 for proto in protocols:
Guido van Rossum2a30b212003-02-18 22:41:24 +00002144 x = REX_one()
2145 self.assertEqual(x._reduce_called, 0)
2146 s = self.dumps(x, proto)
2147 self.assertEqual(x._reduce_called, 1)
2148 y = self.loads(s)
2149 self.assertEqual(y._reduce_called, 0)
2150
2151 def test_reduce_ex_called(self):
Collin Winter771d8342009-04-16 03:18:06 +00002152 for proto in protocols:
Guido van Rossum2a30b212003-02-18 22:41:24 +00002153 x = REX_two()
2154 self.assertEqual(x._proto, None)
2155 s = self.dumps(x, proto)
2156 self.assertEqual(x._proto, proto)
2157 y = self.loads(s)
2158 self.assertEqual(y._proto, None)
2159
2160 def test_reduce_ex_overrides_reduce(self):
Collin Winter771d8342009-04-16 03:18:06 +00002161 for proto in protocols:
Guido van Rossum2a30b212003-02-18 22:41:24 +00002162 x = REX_three()
2163 self.assertEqual(x._proto, None)
2164 s = self.dumps(x, proto)
2165 self.assertEqual(x._proto, proto)
2166 y = self.loads(s)
2167 self.assertEqual(y._proto, None)
2168
Guido van Rossumd8faa362007-04-27 19:54:29 +00002169 def test_reduce_ex_calls_base(self):
Collin Winter771d8342009-04-16 03:18:06 +00002170 for proto in protocols:
Guido van Rossumd8faa362007-04-27 19:54:29 +00002171 x = REX_four()
2172 self.assertEqual(x._proto, None)
2173 s = self.dumps(x, proto)
2174 self.assertEqual(x._proto, proto)
2175 y = self.loads(s)
2176 self.assertEqual(y._proto, proto)
2177
2178 def test_reduce_calls_base(self):
Collin Winter771d8342009-04-16 03:18:06 +00002179 for proto in protocols:
Guido van Rossumd8faa362007-04-27 19:54:29 +00002180 x = REX_five()
2181 self.assertEqual(x._reduce_called, 0)
2182 s = self.dumps(x, proto)
2183 self.assertEqual(x._reduce_called, 1)
2184 y = self.loads(s)
2185 self.assertEqual(y._reduce_called, 1)
2186
Brett Cannon31f59292011-02-21 19:29:56 +00002187 @no_tracing
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00002188 def test_bad_getattr(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002189 # Issue #3514: crash when there is an infinite loop in __getattr__
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00002190 x = BadGetattr()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002191 for proto in protocols:
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00002192 self.assertRaises(RuntimeError, self.dumps, x, proto)
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00002193
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00002194 def test_reduce_bad_iterator(self):
2195 # Issue4176: crash when 4th and 5th items of __reduce__()
2196 # are not iterators
2197 class C(object):
2198 def __reduce__(self):
2199 # 4th item is not an iterator
2200 return list, (), None, [], None
2201 class D(object):
2202 def __reduce__(self):
2203 # 5th item is not an iterator
2204 return dict, (), None, None, []
2205
Serhiy Storchakabeb652c2015-12-30 21:00:08 +02002206 # Python implementation is less strict and also accepts iterables.
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00002207 for proto in protocols:
Amaury Forgeot d'Arc6285ffd2008-10-31 17:52:47 +00002208 try:
2209 self.dumps(C(), proto)
Serhiy Storchakabeb652c2015-12-30 21:00:08 +02002210 except pickle.PicklingError:
Amaury Forgeot d'Arc6285ffd2008-10-31 17:52:47 +00002211 pass
2212 try:
2213 self.dumps(D(), proto)
Serhiy Storchakabeb652c2015-12-30 21:00:08 +02002214 except pickle.PicklingError:
Amaury Forgeot d'Arc6285ffd2008-10-31 17:52:47 +00002215 pass
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00002216
Collin Winter771d8342009-04-16 03:18:06 +00002217 def test_many_puts_and_gets(self):
2218 # Test that internal data structures correctly deal with lots of
2219 # puts/gets.
2220 keys = ("aaa" + str(i) for i in range(100))
2221 large_dict = dict((k, [4, 5, 6]) for k in keys)
2222 obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
2223
2224 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002225 with self.subTest(proto=proto):
2226 dumped = self.dumps(obj, proto)
2227 loaded = self.loads(dumped)
2228 self.assert_is_copy(obj, loaded)
Collin Winter771d8342009-04-16 03:18:06 +00002229
Antoine Pitroua9f48a02009-05-02 21:41:14 +00002230 def test_attribute_name_interning(self):
2231 # Test that attribute names of pickled objects are interned when
2232 # unpickling.
2233 for proto in protocols:
2234 x = C()
2235 x.foo = 42
2236 x.bar = "hello"
2237 s = self.dumps(x, proto)
2238 y = self.loads(s)
2239 x_keys = sorted(x.__dict__)
2240 y_keys = sorted(y.__dict__)
2241 for x_key, y_key in zip(x_keys, y_keys):
2242 self.assertIs(x_key, y_key)
2243
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00002244 def test_pickle_to_2x(self):
2245 # Pickle non-trivial data with protocol 2, expecting that it yields
2246 # the same result as Python 2.x did.
2247 # NOTE: this test is a bit too strong since we can produce different
2248 # bytecode that 2.x will still understand.
2249 dumped = self.dumps(range(5), 2)
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03002250 self.assertEqual(dumped, DATA_XRANGE)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00002251 dumped = self.dumps(set([3]), 2)
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03002252 self.assertEqual(dumped, DATA_SET2)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00002253
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002254 def test_large_pickles(self):
2255 # Test the correctness of internal buffering routines when handling
2256 # large data.
2257 for proto in protocols:
Antoine Pitrou04248a82010-10-12 20:51:21 +00002258 data = (1, min, b'xy' * (30 * 1024), len)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002259 dumped = self.dumps(data, proto)
2260 loaded = self.loads(dumped)
Antoine Pitrou04248a82010-10-12 20:51:21 +00002261 self.assertEqual(len(loaded), len(data))
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002262 self.assertEqual(loaded, data)
2263
Antoine Pitrou3c7e9282011-08-13 20:15:19 +02002264 def test_int_pickling_efficiency(self):
2265 # Test compacity of int representation (see issue #12744)
2266 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002267 with self.subTest(proto=proto):
2268 pickles = [self.dumps(2**n, proto) for n in range(70)]
2269 sizes = list(map(len, pickles))
2270 # the size function is monotonic
2271 self.assertEqual(sorted(sizes), sizes)
2272 if proto >= 2:
2273 for p in pickles:
2274 self.assertFalse(opcode_in_pickle(pickle.LONG, p))
Antoine Pitrou3c7e9282011-08-13 20:15:19 +02002275
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002276 def _check_pickling_with_opcode(self, obj, opcode, proto):
2277 pickled = self.dumps(obj, proto)
2278 self.assertTrue(opcode_in_pickle(opcode, pickled))
2279 unpickled = self.loads(pickled)
2280 self.assertEqual(obj, unpickled)
2281
2282 def test_appends_on_non_lists(self):
2283 # Issue #17720
2284 obj = REX_six([1, 2, 3])
2285 for proto in protocols:
2286 if proto == 0:
2287 self._check_pickling_with_opcode(obj, pickle.APPEND, proto)
2288 else:
2289 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto)
2290
2291 def test_setitems_on_non_dicts(self):
2292 obj = REX_seven({1: -1, 2: -2, 3: -3})
2293 for proto in protocols:
2294 if proto == 0:
2295 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto)
2296 else:
2297 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
2298
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002299 # Exercise framing (proto >= 4) for significant workloads
2300
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002301 FRAME_SIZE_MIN = 4
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002302 FRAME_SIZE_TARGET = 64 * 1024
2303
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01002304 def check_frame_opcodes(self, pickled):
2305 """
2306 Check the arguments of FRAME opcodes in a protocol 4+ pickle.
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002307
2308 Note that binary objects that are larger than FRAME_SIZE_TARGET are not
2309 framed by default and are therefore considered a frame by themselves in
2310 the following consistency check.
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01002311 """
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002312 frame_end = frameless_start = None
Antoine Pitrou91f43802019-05-26 17:10:09 +02002313 frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8',
2314 'BINUNICODE8', 'BYTEARRAY8'}
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01002315 for op, arg, pos in pickletools.genops(pickled):
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002316 if frame_end is not None:
2317 self.assertLessEqual(pos, frame_end)
2318 if pos == frame_end:
2319 frame_end = None
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002320
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002321 if frame_end is not None: # framed
2322 self.assertNotEqual(op.name, 'FRAME')
2323 if op.name in frameless_opcodes:
2324 # Only short bytes and str objects should be written
2325 # in a frame
2326 self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET)
2327
2328 else: # not framed
2329 if (op.name == 'FRAME' or
2330 (op.name in frameless_opcodes and
2331 len(arg) > self.FRAME_SIZE_TARGET)):
2332 # Frame or large bytes or str object
2333 if frameless_start is not None:
2334 # Only short data should be written outside of a frame
2335 self.assertLess(pos - frameless_start,
2336 self.FRAME_SIZE_MIN)
2337 frameless_start = None
2338 elif frameless_start is None and op.name != 'PROTO':
2339 frameless_start = pos
2340
2341 if op.name == 'FRAME':
2342 self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN)
2343 frame_end = pos + 9 + arg
2344
2345 pos = len(pickled)
2346 if frame_end is not None:
2347 self.assertEqual(frame_end, pos)
2348 elif frameless_start is not None:
2349 self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN)
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01002350
Neil Schemenauer52a48e62019-07-30 11:08:18 -07002351 @support.skip_if_pgo_task
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002352 def test_framing_many_objects(self):
2353 obj = list(range(10**5))
2354 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2355 with self.subTest(proto=proto):
2356 pickled = self.dumps(obj, proto)
2357 unpickled = self.loads(pickled)
2358 self.assertEqual(obj, unpickled)
Antoine Pitrou3ab9cfc2013-11-24 14:33:37 +01002359 bytes_per_frame = (len(pickled) /
2360 count_opcode(pickle.FRAME, pickled))
2361 self.assertGreater(bytes_per_frame,
2362 self.FRAME_SIZE_TARGET / 2)
2363 self.assertLessEqual(bytes_per_frame,
2364 self.FRAME_SIZE_TARGET * 1)
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01002365 self.check_frame_opcodes(pickled)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002366
2367 def test_framing_large_objects(self):
2368 N = 1024 * 1024
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002369 small_items = [[i] for i in range(10)]
2370 obj = [b'x' * N, *small_items, b'y' * N, 'z' * N]
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002371 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
Serhiy Storchaka0a2da502018-01-11 13:03:20 +02002372 for fast in [False, True]:
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002373 with self.subTest(proto=proto, fast=fast):
Serhiy Storchaka0a2da502018-01-11 13:03:20 +02002374 if not fast:
2375 # fast=False by default.
2376 # This covers in-memory pickling with pickle.dumps().
2377 pickled = self.dumps(obj, proto)
2378 else:
2379 # Pickler is required when fast=True.
2380 if not hasattr(self, 'pickler'):
2381 continue
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002382 buf = io.BytesIO()
2383 pickler = self.pickler(buf, protocol=proto)
2384 pickler.fast = fast
2385 pickler.dump(obj)
2386 pickled = buf.getvalue()
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002387 unpickled = self.loads(pickled)
2388 # More informative error message in case of failure.
2389 self.assertEqual([len(x) for x in obj],
2390 [len(x) for x in unpickled])
2391 # Perform full equality check if the lengths match.
2392 self.assertEqual(obj, unpickled)
2393 n_frames = count_opcode(pickle.FRAME, pickled)
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002394 # A single frame for small objects between
2395 # first two large objects.
2396 self.assertEqual(n_frames, 1)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002397 self.check_frame_opcodes(pickled)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002398
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08002399 def test_optional_frames(self):
2400 if pickle.HIGHEST_PROTOCOL < 4:
2401 return
2402
2403 def remove_frames(pickled, keep_frame=None):
2404 """Remove frame opcodes from the given pickle."""
2405 frame_starts = []
2406 # 1 byte for the opcode and 8 for the argument
2407 frame_opcode_size = 9
2408 for opcode, _, pos in pickletools.genops(pickled):
2409 if opcode.name == 'FRAME':
2410 frame_starts.append(pos)
2411
2412 newpickle = bytearray()
2413 last_frame_end = 0
2414 for i, pos in enumerate(frame_starts):
2415 if keep_frame and keep_frame(i):
2416 continue
2417 newpickle += pickled[last_frame_end:pos]
2418 last_frame_end = pos + frame_opcode_size
2419 newpickle += pickled[last_frame_end:]
2420 return newpickle
2421
Alexandre Vassalotti5e411b72013-11-23 20:58:24 -08002422 frame_size = self.FRAME_SIZE_TARGET
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08002423 num_frames = 20
penguindustin96466302019-05-06 14:57:17 -04002424 # Large byte objects (dict values) intermittent with small objects
Serhiy Storchaka1211c9a2018-01-20 16:42:44 +02002425 # (dict keys)
Antoine Pitrou91f43802019-05-26 17:10:09 +02002426 for bytes_type in (bytes, bytearray):
2427 obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)}
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08002428
Antoine Pitrou91f43802019-05-26 17:10:09 +02002429 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2430 pickled = self.dumps(obj, proto)
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08002431
Antoine Pitrou91f43802019-05-26 17:10:09 +02002432 frameless_pickle = remove_frames(pickled)
2433 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
2434 self.assertEqual(obj, self.loads(frameless_pickle))
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08002435
Antoine Pitrou91f43802019-05-26 17:10:09 +02002436 some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
2437 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
2438 count_opcode(pickle.FRAME, pickled))
2439 self.assertEqual(obj, self.loads(some_frames_pickle))
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08002440
Neil Schemenauer52a48e62019-07-30 11:08:18 -07002441 @support.skip_if_pgo_task
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002442 def test_framed_write_sizes_with_delayed_writer(self):
2443 class ChunkAccumulator:
2444 """Accumulate pickler output in a list of raw chunks."""
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002445 def __init__(self):
2446 self.chunks = []
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002447 def write(self, chunk):
2448 self.chunks.append(chunk)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002449 def concatenate_chunks(self):
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002450 return b"".join(self.chunks)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002451
2452 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002453 objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)})
2454 for i in range(int(1e4))]
2455 # Add a large unique ASCII string
2456 objects.append('0123456789abcdef' *
2457 (self.FRAME_SIZE_TARGET // 16 + 1))
2458
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002459 # Protocol 4 packs groups of small objects into frames and issues
2460 # calls to write only once or twice per frame:
2461 # The C pickler issues one call to write per-frame (header and
2462 # contents) while Python pickler issues two calls to write: one for
2463 # the frame header and one for the frame binary contents.
2464 writer = ChunkAccumulator()
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002465 self.pickler(writer, proto).dump(objects)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002466
2467 # Actually read the binary content of the chunks after the end
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002468 # of the call to dump: any memoryview passed to write should not
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002469 # be released otherwise this delayed access would not be possible.
2470 pickled = writer.concatenate_chunks()
2471 reconstructed = self.loads(pickled)
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002472 self.assertEqual(reconstructed, objects)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002473 self.assertGreater(len(writer.chunks), 1)
2474
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002475 # memoryviews should own the memory.
2476 del objects
2477 support.gc_collect()
2478 self.assertEqual(writer.concatenate_chunks(), pickled)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002479
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002480 n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002481 # There should be at least one call to write per frame
2482 self.assertGreaterEqual(len(writer.chunks), n_frames)
2483
2484 # but not too many either: there can be one for the proto,
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002485 # one per-frame header, one per frame for the actual contents,
2486 # and two for the header.
2487 self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002488
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002489 chunk_sizes = [len(c) for c in writer.chunks]
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002490 large_sizes = [s for s in chunk_sizes
2491 if s >= self.FRAME_SIZE_TARGET]
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002492 medium_sizes = [s for s in chunk_sizes
2493 if 9 < s < self.FRAME_SIZE_TARGET]
2494 small_sizes = [s for s in chunk_sizes if s <= 9]
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002495
2496 # Large chunks should not be too large:
2497 for chunk_size in large_sizes:
Serhiy Storchaka5b76bdb2018-01-13 00:28:31 +02002498 self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET,
2499 chunk_sizes)
2500 # There shouldn't bee too many small chunks: the protocol header,
2501 # the frame headers and the large string headers are written
2502 # in small chunks.
2503 self.assertLessEqual(len(small_sizes),
2504 len(large_sizes) + len(medium_sizes) + 3,
2505 chunk_sizes)
Olivier Grisel3cd7c6e2018-01-06 16:18:54 +01002506
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002507 def test_nested_names(self):
2508 global Nested
2509 class Nested:
2510 class A:
2511 class B:
2512 class C:
2513 pass
Serhiy Storchaka58e41342015-03-31 14:07:24 +03002514 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002515 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
2516 with self.subTest(proto=proto, obj=obj):
2517 unpickled = self.loads(self.dumps(obj, proto))
2518 self.assertIs(obj, unpickled)
2519
Serhiy Storchaka58e41342015-03-31 14:07:24 +03002520 def test_recursive_nested_names(self):
2521 global Recursive
2522 class Recursive:
2523 pass
2524 Recursive.mod = sys.modules[Recursive.__module__]
2525 Recursive.__qualname__ = 'Recursive.mod.Recursive'
2526 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2527 with self.subTest(proto=proto):
2528 unpickled = self.loads(self.dumps(Recursive, proto))
2529 self.assertIs(unpickled, Recursive)
2530 del Recursive.mod # break reference loop
2531
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002532 def test_py_methods(self):
2533 global PyMethodsTest
2534 class PyMethodsTest:
2535 @staticmethod
2536 def cheese():
2537 return "cheese"
2538 @classmethod
2539 def wine(cls):
2540 assert cls is PyMethodsTest
2541 return "wine"
2542 def biscuits(self):
2543 assert isinstance(self, PyMethodsTest)
2544 return "biscuits"
2545 class Nested:
2546 "Nested class"
2547 @staticmethod
2548 def ketchup():
2549 return "ketchup"
2550 @classmethod
2551 def maple(cls):
2552 assert cls is PyMethodsTest.Nested
2553 return "maple"
2554 def pie(self):
2555 assert isinstance(self, PyMethodsTest.Nested)
2556 return "pie"
2557
2558 py_methods = (
2559 PyMethodsTest.cheese,
2560 PyMethodsTest.wine,
2561 PyMethodsTest().biscuits,
2562 PyMethodsTest.Nested.ketchup,
2563 PyMethodsTest.Nested.maple,
2564 PyMethodsTest.Nested().pie
2565 )
2566 py_unbound_methods = (
2567 (PyMethodsTest.biscuits, PyMethodsTest),
2568 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
2569 )
Serhiy Storchaka58e41342015-03-31 14:07:24 +03002570 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002571 for method in py_methods:
2572 with self.subTest(proto=proto, method=method):
2573 unpickled = self.loads(self.dumps(method, proto))
2574 self.assertEqual(method(), unpickled())
2575 for method, cls in py_unbound_methods:
2576 obj = cls()
2577 with self.subTest(proto=proto, method=method):
2578 unpickled = self.loads(self.dumps(method, proto))
2579 self.assertEqual(method(obj), unpickled(obj))
2580
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002581 def test_c_methods(self):
2582 global Subclass
2583 class Subclass(tuple):
2584 class Nested(str):
2585 pass
2586
2587 c_methods = (
2588 # bound built-in method
2589 ("abcd".index, ("c",)),
2590 # unbound built-in method
2591 (str.index, ("abcd", "c")),
2592 # bound "slot" method
2593 ([1, 2, 3].__len__, ()),
2594 # unbound "slot" method
2595 (list.__len__, ([1, 2, 3],)),
2596 # bound "coexist" method
2597 ({1, 2}.__contains__, (2,)),
2598 # unbound "coexist" method
2599 (set.__contains__, ({1, 2}, 2)),
2600 # built-in class method
2601 (dict.fromkeys, (("a", 1), ("b", 2))),
2602 # built-in static method
2603 (bytearray.maketrans, (b"abc", b"xyz")),
2604 # subclass methods
2605 (Subclass([1,2,2]).count, (2,)),
2606 (Subclass.count, (Subclass([1,2,2]), 2)),
2607 (Subclass.Nested("sweet").count, ("e",)),
2608 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
2609 )
Serhiy Storchaka58e41342015-03-31 14:07:24 +03002610 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002611 for method, args in c_methods:
2612 with self.subTest(proto=proto, method=method):
2613 unpickled = self.loads(self.dumps(method, proto))
2614 self.assertEqual(method(*args), unpickled(*args))
2615
Serhiy Storchakabfe18242015-03-31 13:12:37 +03002616 def test_compat_pickle(self):
2617 tests = [
2618 (range(1, 7), '__builtin__', 'xrange'),
2619 (map(int, '123'), 'itertools', 'imap'),
2620 (functools.reduce, '__builtin__', 'reduce'),
2621 (dbm.whichdb, 'whichdb', 'whichdb'),
2622 (Exception(), 'exceptions', 'Exception'),
2623 (collections.UserDict(), 'UserDict', 'IterableUserDict'),
2624 (collections.UserList(), 'UserList', 'UserList'),
2625 (collections.defaultdict(), 'collections', 'defaultdict'),
2626 ]
2627 for val, mod, name in tests:
2628 for proto in range(3):
2629 with self.subTest(type=type(val), proto=proto):
2630 pickled = self.dumps(val, proto)
2631 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled)
2632 self.assertIs(type(self.loads(pickled)), type(val))
2633
Antoine Pitrou6cd5eda2014-12-02 00:20:03 +01002634 def test_local_lookup_error(self):
2635 # Test that whichmodule() errors out cleanly when looking up
2636 # an assumed globally-reachable object fails.
2637 def f():
2638 pass
2639 # Since the function is local, lookup will fail
2640 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2641 with self.assertRaises((AttributeError, pickle.PicklingError)):
2642 pickletools.dis(self.dumps(f, proto))
2643 # Same without a __module__ attribute (exercises a different path
2644 # in _pickle.c).
2645 del f.__module__
2646 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2647 with self.assertRaises((AttributeError, pickle.PicklingError)):
2648 pickletools.dis(self.dumps(f, proto))
2649 # Yet a different path.
2650 f.__name__ = f.__qualname__
2651 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2652 with self.assertRaises((AttributeError, pickle.PicklingError)):
2653 pickletools.dis(self.dumps(f, proto))
2654
Antoine Pitrou91f43802019-05-26 17:10:09 +02002655 #
2656 # PEP 574 tests below
2657 #
2658
2659 def buffer_like_objects(self):
2660 # Yield buffer-like objects with the bytestring "abcdef" in them
2661 bytestring = b"abcdefgh"
2662 yield ZeroCopyBytes(bytestring)
2663 yield ZeroCopyBytearray(bytestring)
2664 if _testbuffer is not None:
2665 items = list(bytestring)
2666 value = int.from_bytes(bytestring, byteorder='little')
2667 for flags in (0, _testbuffer.ND_WRITABLE):
2668 # 1-D, contiguous
2669 yield PicklableNDArray(items, format='B', shape=(8,),
2670 flags=flags)
2671 # 2-D, C-contiguous
2672 yield PicklableNDArray(items, format='B', shape=(4, 2),
2673 strides=(2, 1), flags=flags)
2674 # 2-D, Fortran-contiguous
2675 yield PicklableNDArray(items, format='B',
2676 shape=(4, 2), strides=(1, 4),
2677 flags=flags)
2678
2679 def test_in_band_buffers(self):
2680 # Test in-band buffers (PEP 574)
2681 for obj in self.buffer_like_objects():
2682 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2683 data = self.dumps(obj, proto)
2684 if obj.c_contiguous and proto >= 5:
2685 # The raw memory bytes are serialized in physical order
2686 self.assertIn(b"abcdefgh", data)
2687 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0)
2688 if proto >= 5:
2689 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data),
2690 1 if obj.readonly else 0)
2691 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data),
2692 0 if obj.readonly else 1)
2693 # Return a true value from buffer_callback should have
2694 # the same effect
2695 def buffer_callback(obj):
2696 return True
2697 data2 = self.dumps(obj, proto,
2698 buffer_callback=buffer_callback)
2699 self.assertEqual(data2, data)
2700
2701 new = self.loads(data)
2702 # It's a copy
2703 self.assertIsNot(new, obj)
2704 self.assertIs(type(new), type(obj))
2705 self.assertEqual(new, obj)
2706
2707 # XXX Unfortunately cannot test non-contiguous array
2708 # (see comment in PicklableNDArray.__reduce_ex__)
2709
2710 def test_oob_buffers(self):
2711 # Test out-of-band buffers (PEP 574)
2712 for obj in self.buffer_like_objects():
2713 for proto in range(0, 5):
2714 # Need protocol >= 5 for buffer_callback
2715 with self.assertRaises(ValueError):
2716 self.dumps(obj, proto,
2717 buffer_callback=[].append)
2718 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2719 buffers = []
2720 buffer_callback = lambda pb: buffers.append(pb.raw())
2721 data = self.dumps(obj, proto,
2722 buffer_callback=buffer_callback)
2723 self.assertNotIn(b"abcdefgh", data)
2724 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0)
2725 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0)
2726 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1)
2727 self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data),
2728 1 if obj.readonly else 0)
2729
2730 if obj.c_contiguous:
2731 self.assertEqual(bytes(buffers[0]), b"abcdefgh")
2732 # Need buffers argument to unpickle properly
2733 with self.assertRaises(pickle.UnpicklingError):
2734 self.loads(data)
2735
2736 new = self.loads(data, buffers=buffers)
2737 if obj.zero_copy_reconstruct:
2738 # Zero-copy achieved
2739 self.assertIs(new, obj)
2740 else:
2741 self.assertIs(type(new), type(obj))
2742 self.assertEqual(new, obj)
2743 # Non-sequence buffers accepted too
2744 new = self.loads(data, buffers=iter(buffers))
2745 if obj.zero_copy_reconstruct:
2746 # Zero-copy achieved
2747 self.assertIs(new, obj)
2748 else:
2749 self.assertIs(type(new), type(obj))
2750 self.assertEqual(new, obj)
2751
2752 def test_oob_buffers_writable_to_readonly(self):
2753 # Test reconstructing readonly object from writable buffer
2754 obj = ZeroCopyBytes(b"foobar")
2755 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2756 buffers = []
2757 buffer_callback = buffers.append
2758 data = self.dumps(obj, proto, buffer_callback=buffer_callback)
2759
2760 buffers = map(bytearray, buffers)
2761 new = self.loads(data, buffers=buffers)
2762 self.assertIs(type(new), type(obj))
2763 self.assertEqual(new, obj)
2764
2765 def test_picklebuffer_error(self):
2766 # PickleBuffer forbidden with protocol < 5
2767 pb = pickle.PickleBuffer(b"foobar")
2768 for proto in range(0, 5):
2769 with self.assertRaises(pickle.PickleError):
2770 self.dumps(pb, proto)
2771
2772 def test_buffer_callback_error(self):
2773 def buffer_callback(buffers):
2774 1/0
2775 pb = pickle.PickleBuffer(b"foobar")
2776 with self.assertRaises(ZeroDivisionError):
2777 self.dumps(pb, 5, buffer_callback=buffer_callback)
2778
2779 def test_buffers_error(self):
2780 pb = pickle.PickleBuffer(b"foobar")
2781 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2782 data = self.dumps(pb, proto, buffer_callback=[].append)
2783 # Non iterable buffers
2784 with self.assertRaises(TypeError):
2785 self.loads(data, buffers=object())
2786 # Buffer iterable exhausts too early
2787 with self.assertRaises(pickle.UnpicklingError):
2788 self.loads(data, buffers=[])
2789
Markus Mohrhard898318b2019-07-26 00:00:34 +08002790 def test_inband_accept_default_buffers_argument(self):
2791 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2792 data_pickled = self.dumps(1, proto, buffer_callback=None)
2793 data = self.loads(data_pickled, buffers=None)
2794
Antoine Pitrou91f43802019-05-26 17:10:09 +02002795 @unittest.skipIf(np is None, "Test needs Numpy")
2796 def test_buffers_numpy(self):
2797 def check_no_copy(x, y):
2798 np.testing.assert_equal(x, y)
2799 self.assertEqual(x.ctypes.data, y.ctypes.data)
2800
2801 def check_copy(x, y):
2802 np.testing.assert_equal(x, y)
2803 self.assertNotEqual(x.ctypes.data, y.ctypes.data)
2804
2805 def check_array(arr):
2806 # In-band
2807 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2808 data = self.dumps(arr, proto)
2809 new = self.loads(data)
2810 check_copy(arr, new)
2811 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2812 buffer_callback = lambda _: True
2813 data = self.dumps(arr, proto, buffer_callback=buffer_callback)
2814 new = self.loads(data)
2815 check_copy(arr, new)
2816 # Out-of-band
2817 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2818 buffers = []
2819 buffer_callback = buffers.append
2820 data = self.dumps(arr, proto, buffer_callback=buffer_callback)
2821 new = self.loads(data, buffers=buffers)
2822 if arr.flags.c_contiguous or arr.flags.f_contiguous:
2823 check_no_copy(arr, new)
2824 else:
2825 check_copy(arr, new)
2826
2827 # 1-D
2828 arr = np.arange(6)
2829 check_array(arr)
2830 # 1-D, non-contiguous
2831 check_array(arr[::2])
2832 # 2-D, C-contiguous
2833 arr = np.arange(12).reshape((3, 4))
2834 check_array(arr)
2835 # 2-D, F-contiguous
2836 check_array(arr.T)
2837 # 2-D, non-contiguous
2838 check_array(arr[::2])
2839
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002840
2841class BigmemPickleTests(unittest.TestCase):
2842
Victor Stinner8c663fd2017-11-08 14:44:44 -08002843 # Binary protocols can serialize longs of up to 2 GiB-1
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002844
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02002845 @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002846 def test_huge_long_32b(self, size):
2847 data = 1 << (8 * size)
2848 try:
2849 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002850 if proto < 2:
2851 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002852 with self.subTest(proto=proto):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002853 with self.assertRaises((ValueError, OverflowError)):
2854 self.dumps(data, protocol=proto)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002855 finally:
2856 data = None
2857
Victor Stinner8c663fd2017-11-08 14:44:44 -08002858 # Protocol 3 can serialize up to 4 GiB-1 as a bytes object
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002859 # (older protocols don't have a dedicated opcode for bytes and are
2860 # too inefficient)
2861
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02002862 @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002863 def test_huge_bytes_32b(self, size):
2864 data = b"abcd" * (size // 4)
2865 try:
2866 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002867 if proto < 3:
2868 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002869 with self.subTest(proto=proto):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002870 try:
2871 pickled = self.dumps(data, protocol=proto)
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002872 header = (pickle.BINBYTES +
2873 struct.pack("<I", len(data)))
2874 data_start = pickled.index(data)
2875 self.assertEqual(
2876 header,
2877 pickled[data_start-len(header):data_start])
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002878 finally:
2879 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002880 finally:
2881 data = None
2882
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02002883 @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002884 def test_huge_bytes_64b(self, size):
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002885 data = b"acbd" * (size // 4)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002886 try:
2887 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002888 if proto < 3:
2889 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002890 with self.subTest(proto=proto):
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002891 if proto == 3:
2892 # Protocol 3 does not support large bytes objects.
2893 # Verify that we do not crash when processing one.
2894 with self.assertRaises((ValueError, OverflowError)):
2895 self.dumps(data, protocol=proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002896 continue
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002897 try:
2898 pickled = self.dumps(data, protocol=proto)
2899 header = (pickle.BINBYTES8 +
2900 struct.pack("<Q", len(data)))
2901 data_start = pickled.index(data)
2902 self.assertEqual(
2903 header,
2904 pickled[data_start-len(header):data_start])
2905 finally:
2906 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002907 finally:
2908 data = None
2909
2910 # All protocols use 1-byte per printable ASCII character; we add another
2911 # byte because the encoded form has to be copied into the internal buffer.
2912
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02002913 @bigmemtest(size=_2G, memuse=8, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002914 def test_huge_str_32b(self, size):
2915 data = "abcd" * (size // 4)
2916 try:
2917 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002918 if proto == 0:
2919 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002920 with self.subTest(proto=proto):
2921 try:
2922 pickled = self.dumps(data, protocol=proto)
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002923 header = (pickle.BINUNICODE +
2924 struct.pack("<I", len(data)))
2925 data_start = pickled.index(b'abcd')
2926 self.assertEqual(
2927 header,
2928 pickled[data_start-len(header):data_start])
2929 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
2930 pickled.index(b"abcd")), len(data))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002931 finally:
2932 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002933 finally:
2934 data = None
2935
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002936 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
2937 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
2938 # unicode strings however.
Antoine Pitroue897e952011-08-30 23:39:34 +02002939
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02002940 @bigmemtest(size=_4G, memuse=8, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002941 def test_huge_str_64b(self, size):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002942 data = "abcd" * (size // 4)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002943 try:
2944 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002945 if proto == 0:
2946 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002947 with self.subTest(proto=proto):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002948 if proto < 4:
2949 with self.assertRaises((ValueError, OverflowError)):
2950 self.dumps(data, protocol=proto)
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002951 continue
2952 try:
2953 pickled = self.dumps(data, protocol=proto)
2954 header = (pickle.BINUNICODE8 +
2955 struct.pack("<Q", len(data)))
2956 data_start = pickled.index(b'abcd')
2957 self.assertEqual(
2958 header,
2959 pickled[data_start-len(header):data_start])
2960 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
2961 pickled.index(b"abcd")), len(data))
2962 finally:
2963 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002964 finally:
2965 data = None
2966
Antoine Pitrou3c7e9282011-08-13 20:15:19 +02002967
Guido van Rossum2a30b212003-02-18 22:41:24 +00002968# Test classes for reduce_ex
2969
2970class REX_one(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002971 """No __reduce_ex__ here, but inheriting it from object"""
Guido van Rossum2a30b212003-02-18 22:41:24 +00002972 _reduce_called = 0
2973 def __reduce__(self):
2974 self._reduce_called = 1
2975 return REX_one, ()
Guido van Rossum2a30b212003-02-18 22:41:24 +00002976
2977class REX_two(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002978 """No __reduce__ here, but inheriting it from object"""
Guido van Rossum2a30b212003-02-18 22:41:24 +00002979 _proto = None
2980 def __reduce_ex__(self, proto):
2981 self._proto = proto
2982 return REX_two, ()
Guido van Rossum2a30b212003-02-18 22:41:24 +00002983
2984class REX_three(object):
2985 _proto = None
2986 def __reduce_ex__(self, proto):
2987 self._proto = proto
2988 return REX_two, ()
2989 def __reduce__(self):
Collin Winter3add4d72007-08-29 23:37:32 +00002990 raise TestFailed("This __reduce__ shouldn't be called")
Guido van Rossum2a30b212003-02-18 22:41:24 +00002991
Guido van Rossumd8faa362007-04-27 19:54:29 +00002992class REX_four(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002993 """Calling base class method should succeed"""
Guido van Rossumd8faa362007-04-27 19:54:29 +00002994 _proto = None
2995 def __reduce_ex__(self, proto):
2996 self._proto = proto
2997 return object.__reduce_ex__(self, proto)
Guido van Rossumd8faa362007-04-27 19:54:29 +00002998
2999class REX_five(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07003000 """This one used to fail with infinite recursion"""
Guido van Rossumd8faa362007-04-27 19:54:29 +00003001 _reduce_called = 0
3002 def __reduce__(self):
3003 self._reduce_called = 1
3004 return object.__reduce__(self)
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07003005
3006class REX_six(object):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003007 """This class is used to check the 4th argument (list iterator) of
3008 the reduce protocol.
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07003009 """
3010 def __init__(self, items=None):
3011 self.items = items if items is not None else []
3012 def __eq__(self, other):
Serhiy Storchakabe700022016-03-04 09:39:47 +02003013 return type(self) is type(other) and self.items == other.items
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07003014 def append(self, item):
3015 self.items.append(item)
3016 def __reduce__(self):
3017 return type(self), (), None, iter(self.items), None
3018
3019class REX_seven(object):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003020 """This class is used to check the 5th argument (dict iterator) of
3021 the reduce protocol.
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07003022 """
3023 def __init__(self, table=None):
3024 self.table = table if table is not None else {}
3025 def __eq__(self, other):
Serhiy Storchakabe700022016-03-04 09:39:47 +02003026 return type(self) is type(other) and self.table == other.table
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07003027 def __setitem__(self, key, value):
3028 self.table[key] = value
3029 def __reduce__(self):
3030 return type(self), (), None, None, iter(self.table.items())
3031
Guido van Rossumd8faa362007-04-27 19:54:29 +00003032
Guido van Rossum2a30b212003-02-18 22:41:24 +00003033# Test classes for newobj
Tim Peters080c88b2003-02-15 03:01:11 +00003034
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003035class MyInt(int):
3036 sample = 1
3037
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003038class MyFloat(float):
3039 sample = 1.0
3040
3041class MyComplex(complex):
3042 sample = 1.0 + 0.0j
3043
3044class MyStr(str):
3045 sample = "hello"
3046
Guido van Rossumef87d6e2007-05-02 19:09:54 +00003047class MyUnicode(str):
3048 sample = "hello \u1234"
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003049
Guido van Rossum533dbcf2003-01-28 17:55:05 +00003050class MyTuple(tuple):
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003051 sample = (1, 2, 3)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00003052
3053class MyList(list):
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003054 sample = [1, 2, 3]
3055
3056class MyDict(dict):
3057 sample = {"a": 1, "b": 2}
3058
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003059class MySet(set):
3060 sample = {"a", "b"}
3061
3062class MyFrozenSet(frozenset):
3063 sample = frozenset({"a", "b"})
3064
Mark Dickinson5c2db372009-12-05 20:28:34 +00003065myclasses = [MyInt, MyFloat,
Guido van Rossum206b9a72003-03-02 13:53:18 +00003066 MyComplex,
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003067 MyStr, MyUnicode,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003068 MyTuple, MyList, MyDict, MySet, MyFrozenSet]
Guido van Rossum5d9113d2003-01-29 17:58:45 +00003069
Guido van Rossum533dbcf2003-01-28 17:55:05 +00003070
Guido van Rossumc8d6ef52003-01-28 22:02:31 +00003071class SlotList(MyList):
3072 __slots__ = ["foo"]
3073
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02003074class SimpleNewObj(int):
3075 def __init__(self, *args, **kwargs):
Tim Peterse9ef2032003-02-13 18:42:00 +00003076 # raise an error, to make sure this isn't called
3077 raise TypeError("SimpleNewObj.__init__() didn't expect to get called")
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003078 def __eq__(self, other):
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02003079 return int(self) == int(other) and self.__dict__ == other.__dict__
3080
3081class ComplexNewObj(SimpleNewObj):
3082 def __getnewargs__(self):
3083 return ('%X' % self, 16)
3084
3085class ComplexNewObjEx(SimpleNewObj):
3086 def __getnewargs_ex__(self):
3087 return ('%X' % self,), {'base': 16}
Tim Peterse9ef2032003-02-13 18:42:00 +00003088
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00003089class BadGetattr:
3090 def __getattr__(self, key):
3091 self.foo
3092
Collin Winter771d8342009-04-16 03:18:06 +00003093
Jeremy Hylton66426532001-10-15 21:38:56 +00003094class AbstractPickleModuleTests(unittest.TestCase):
3095
3096 def test_dump_closed_file(self):
Walter Dörwald11b41f62007-06-20 12:46:31 +00003097 f = open(TESTFN, "wb")
Guido van Rossum3b0a3292002-08-09 16:38:32 +00003098 try:
3099 f.close()
Serhiy Storchaka65452562017-11-15 14:01:08 +02003100 self.assertRaises(ValueError, self.dump, 123, f)
Guido van Rossum3b0a3292002-08-09 16:38:32 +00003101 finally:
Serhiy Storchaka38ab7d42019-05-31 11:29:39 +03003102 support.unlink(TESTFN)
Jeremy Hylton66426532001-10-15 21:38:56 +00003103
3104 def test_load_closed_file(self):
Walter Dörwald11b41f62007-06-20 12:46:31 +00003105 f = open(TESTFN, "wb")
Guido van Rossum3b0a3292002-08-09 16:38:32 +00003106 try:
3107 f.close()
Serhiy Storchaka65452562017-11-15 14:01:08 +02003108 self.assertRaises(ValueError, self.dump, 123, f)
Guido van Rossum3b0a3292002-08-09 16:38:32 +00003109 finally:
Serhiy Storchaka38ab7d42019-05-31 11:29:39 +03003110 support.unlink(TESTFN)
Jeremy Hylton4c8be852002-11-13 22:10:47 +00003111
Collin Winter771d8342009-04-16 03:18:06 +00003112 def test_load_from_and_dump_to_file(self):
3113 stream = io.BytesIO()
3114 data = [123, {}, 124]
Serhiy Storchaka65452562017-11-15 14:01:08 +02003115 self.dump(data, stream)
Collin Winter771d8342009-04-16 03:18:06 +00003116 stream.seek(0)
Serhiy Storchaka65452562017-11-15 14:01:08 +02003117 unpickled = self.load(stream)
Collin Winter771d8342009-04-16 03:18:06 +00003118 self.assertEqual(unpickled, data)
3119
Tim Petersc0c93702003-02-13 19:30:57 +00003120 def test_highest_protocol(self):
3121 # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
Antoine Pitrou91f43802019-05-26 17:10:09 +02003122 self.assertEqual(pickle.HIGHEST_PROTOCOL, 5)
Tim Petersc0c93702003-02-13 19:30:57 +00003123
Martin v. Löwis544f1192004-07-27 05:22:33 +00003124 def test_callapi(self):
Collin Winter771d8342009-04-16 03:18:06 +00003125 f = io.BytesIO()
Martin v. Löwis544f1192004-07-27 05:22:33 +00003126 # With and without keyword arguments
Serhiy Storchaka65452562017-11-15 14:01:08 +02003127 self.dump(123, f, -1)
3128 self.dump(123, file=f, protocol=-1)
3129 self.dumps(123, -1)
3130 self.dumps(123, protocol=-1)
3131 self.Pickler(f, -1)
3132 self.Pickler(f, protocol=-1)
Tim Petersc0c93702003-02-13 19:30:57 +00003133
Serhiy Storchaka38ab7d42019-05-31 11:29:39 +03003134 def test_dump_text_file(self):
3135 f = open(TESTFN, "w")
3136 try:
3137 for proto in protocols:
3138 self.assertRaises(TypeError, self.dump, 123, f, proto)
3139 finally:
3140 f.close()
3141 support.unlink(TESTFN)
3142
3143 def test_incomplete_input(self):
3144 s = io.BytesIO(b"X''.")
3145 self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s)
3146
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00003147 def test_bad_init(self):
3148 # Test issue3664 (pickle can segfault from a badly initialized Pickler).
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00003149 # Override initialization without calling __init__() of the superclass.
Serhiy Storchaka65452562017-11-15 14:01:08 +02003150 class BadPickler(self.Pickler):
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00003151 def __init__(self): pass
3152
Serhiy Storchaka65452562017-11-15 14:01:08 +02003153 class BadUnpickler(self.Unpickler):
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00003154 def __init__(self): pass
3155
3156 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
3157 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load)
3158
Antoine Pitrou91f43802019-05-26 17:10:09 +02003159 def check_dumps_loads_oob_buffers(self, dumps, loads):
3160 # No need to do the full gamut of tests here, just enough to
3161 # check that dumps() and loads() redirect their arguments
3162 # to the underlying Pickler and Unpickler, respectively.
3163 obj = ZeroCopyBytes(b"foo")
3164
3165 for proto in range(0, 5):
3166 # Need protocol >= 5 for buffer_callback
3167 with self.assertRaises(ValueError):
3168 dumps(obj, protocol=proto,
3169 buffer_callback=[].append)
3170 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3171 buffers = []
3172 buffer_callback = buffers.append
3173 data = dumps(obj, protocol=proto,
3174 buffer_callback=buffer_callback)
3175 self.assertNotIn(b"foo", data)
3176 self.assertEqual(bytes(buffers[0]), b"foo")
3177 # Need buffers argument to unpickle properly
3178 with self.assertRaises(pickle.UnpicklingError):
3179 loads(data)
3180 new = loads(data, buffers=buffers)
3181 self.assertIs(new, obj)
3182
3183 def test_dumps_loads_oob_buffers(self):
3184 # Test out-of-band buffers (PEP 574) with top-level dumps() and loads()
3185 self.check_dumps_loads_oob_buffers(self.dumps, self.loads)
3186
3187 def test_dump_load_oob_buffers(self):
3188 # Test out-of-band buffers (PEP 574) with top-level dump() and load()
3189 def dumps(obj, **kwargs):
3190 f = io.BytesIO()
3191 self.dump(obj, f, **kwargs)
3192 return f.getvalue()
3193
3194 def loads(data, **kwargs):
3195 f = io.BytesIO(data)
3196 return self.load(f, **kwargs)
3197
3198 self.check_dumps_loads_oob_buffers(dumps, loads)
3199
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00003200
Jeremy Hylton4c8be852002-11-13 22:10:47 +00003201class AbstractPersistentPicklerTests(unittest.TestCase):
3202
3203 # This class defines persistent_id() and persistent_load()
3204 # functions that should be used by the pickler. All even integers
3205 # are pickled using persistent ids.
3206
3207 def persistent_id(self, object):
3208 if isinstance(object, int) and object % 2 == 0:
3209 self.id_count += 1
3210 return str(object)
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08003211 elif object == "test_false_value":
3212 self.false_count += 1
3213 return ""
Jeremy Hylton4c8be852002-11-13 22:10:47 +00003214 else:
3215 return None
3216
3217 def persistent_load(self, oid):
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08003218 if not oid:
3219 self.load_false_count += 1
3220 return "test_false_value"
3221 else:
3222 self.load_count += 1
3223 object = int(oid)
3224 assert object % 2 == 0
3225 return object
Jeremy Hylton4c8be852002-11-13 22:10:47 +00003226
3227 def test_persistence(self):
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08003228 L = list(range(10)) + ["test_false_value"]
3229 for proto in protocols:
3230 self.id_count = 0
3231 self.false_count = 0
3232 self.load_false_count = 0
3233 self.load_count = 0
3234 self.assertEqual(self.loads(self.dumps(L, proto)), L)
3235 self.assertEqual(self.id_count, 5)
3236 self.assertEqual(self.false_count, 1)
3237 self.assertEqual(self.load_count, 5)
3238 self.assertEqual(self.load_false_count, 1)
Guido van Rossum98297ee2007-11-06 21:34:58 +00003239
Collin Winter771d8342009-04-16 03:18:06 +00003240
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003241class AbstractIdentityPersistentPicklerTests(unittest.TestCase):
3242
3243 def persistent_id(self, obj):
3244 return obj
3245
3246 def persistent_load(self, pid):
3247 return pid
3248
3249 def _check_return_correct_type(self, obj, proto):
3250 unpickled = self.loads(self.dumps(obj, proto))
3251 self.assertIsInstance(unpickled, type(obj))
3252 self.assertEqual(unpickled, obj)
3253
3254 def test_return_correct_type(self):
3255 for proto in protocols:
3256 # Protocol 0 supports only ASCII strings.
3257 if proto == 0:
3258 self._check_return_correct_type("abc", 0)
3259 else:
3260 for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]:
3261 self._check_return_correct_type(obj, proto)
3262
3263 def test_protocol0_is_ascii_only(self):
3264 non_ascii_str = "\N{EMPTY SET}"
3265 self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0)
3266 pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.'
3267 self.assertRaises(pickle.UnpicklingError, self.loads, pickled)
3268
3269
Collin Winter771d8342009-04-16 03:18:06 +00003270class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
3271
3272 pickler_class = None
3273 unpickler_class = None
3274
3275 def setUp(self):
3276 assert self.pickler_class
3277 assert self.unpickler_class
3278
3279 def test_clear_pickler_memo(self):
3280 # To test whether clear_memo() has any effect, we pickle an object,
3281 # then pickle it again without clearing the memo; the two serialized
3282 # forms should be different. If we clear_memo() and then pickle the
3283 # object again, the third serialized form should be identical to the
3284 # first one we obtained.
3285 data = ["abcdefg", "abcdefg", 44]
Serhiy Storchakac8695292018-04-04 00:11:27 +03003286 for proto in protocols:
3287 f = io.BytesIO()
3288 pickler = self.pickler_class(f, proto)
Collin Winter771d8342009-04-16 03:18:06 +00003289
Serhiy Storchakac8695292018-04-04 00:11:27 +03003290 pickler.dump(data)
3291 first_pickled = f.getvalue()
Collin Winter771d8342009-04-16 03:18:06 +00003292
Serhiy Storchakac8695292018-04-04 00:11:27 +03003293 # Reset BytesIO object.
3294 f.seek(0)
3295 f.truncate()
Collin Winter771d8342009-04-16 03:18:06 +00003296
Serhiy Storchakac8695292018-04-04 00:11:27 +03003297 pickler.dump(data)
3298 second_pickled = f.getvalue()
Collin Winter771d8342009-04-16 03:18:06 +00003299
Serhiy Storchakac8695292018-04-04 00:11:27 +03003300 # Reset the Pickler and BytesIO objects.
3301 pickler.clear_memo()
3302 f.seek(0)
3303 f.truncate()
Collin Winter771d8342009-04-16 03:18:06 +00003304
Serhiy Storchakac8695292018-04-04 00:11:27 +03003305 pickler.dump(data)
3306 third_pickled = f.getvalue()
Collin Winter771d8342009-04-16 03:18:06 +00003307
Serhiy Storchakac8695292018-04-04 00:11:27 +03003308 self.assertNotEqual(first_pickled, second_pickled)
3309 self.assertEqual(first_pickled, third_pickled)
Collin Winter771d8342009-04-16 03:18:06 +00003310
3311 def test_priming_pickler_memo(self):
3312 # Verify that we can set the Pickler's memo attribute.
3313 data = ["abcdefg", "abcdefg", 44]
3314 f = io.BytesIO()
3315 pickler = self.pickler_class(f)
3316
3317 pickler.dump(data)
3318 first_pickled = f.getvalue()
3319
3320 f = io.BytesIO()
3321 primed = self.pickler_class(f)
3322 primed.memo = pickler.memo
3323
3324 primed.dump(data)
3325 primed_pickled = f.getvalue()
3326
3327 self.assertNotEqual(first_pickled, primed_pickled)
3328
3329 def test_priming_unpickler_memo(self):
3330 # Verify that we can set the Unpickler's memo attribute.
3331 data = ["abcdefg", "abcdefg", 44]
3332 f = io.BytesIO()
3333 pickler = self.pickler_class(f)
3334
3335 pickler.dump(data)
3336 first_pickled = f.getvalue()
3337
3338 f = io.BytesIO()
3339 primed = self.pickler_class(f)
3340 primed.memo = pickler.memo
3341
3342 primed.dump(data)
3343 primed_pickled = f.getvalue()
3344
3345 unpickler = self.unpickler_class(io.BytesIO(first_pickled))
3346 unpickled_data1 = unpickler.load()
3347
3348 self.assertEqual(unpickled_data1, data)
3349
3350 primed = self.unpickler_class(io.BytesIO(primed_pickled))
3351 primed.memo = unpickler.memo
3352 unpickled_data2 = primed.load()
3353
3354 primed.memo.clear()
3355
3356 self.assertEqual(unpickled_data2, data)
3357 self.assertTrue(unpickled_data2 is unpickled_data1)
3358
3359 def test_reusing_unpickler_objects(self):
3360 data1 = ["abcdefg", "abcdefg", 44]
3361 f = io.BytesIO()
3362 pickler = self.pickler_class(f)
3363 pickler.dump(data1)
3364 pickled1 = f.getvalue()
3365
3366 data2 = ["abcdefg", 44, 44]
3367 f = io.BytesIO()
3368 pickler = self.pickler_class(f)
3369 pickler.dump(data2)
3370 pickled2 = f.getvalue()
3371
3372 f = io.BytesIO()
3373 f.write(pickled1)
3374 f.seek(0)
3375 unpickler = self.unpickler_class(f)
3376 self.assertEqual(unpickler.load(), data1)
3377
3378 f.seek(0)
3379 f.truncate()
3380 f.write(pickled2)
3381 f.seek(0)
3382 self.assertEqual(unpickler.load(), data2)
3383
Antoine Pitrou9f378722020-02-23 23:33:53 +01003384 def _check_multiple_unpicklings(self, ioclass, *, seekable=True):
Antoine Pitrou04248a82010-10-12 20:51:21 +00003385 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003386 with self.subTest(proto=proto):
3387 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
3388 f = ioclass()
3389 pickler = self.pickler_class(f, protocol=proto)
3390 pickler.dump(data1)
3391 pickled = f.getvalue()
Antoine Pitrou04248a82010-10-12 20:51:21 +00003392
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003393 N = 5
3394 f = ioclass(pickled * N)
3395 unpickler = self.unpickler_class(f)
3396 for i in range(N):
Antoine Pitrou9f378722020-02-23 23:33:53 +01003397 if seekable:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003398 pos = f.tell()
3399 self.assertEqual(unpickler.load(), data1)
Antoine Pitrou9f378722020-02-23 23:33:53 +01003400 if seekable:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003401 self.assertEqual(f.tell(), pos + len(pickled))
3402 self.assertRaises(EOFError, unpickler.load)
Antoine Pitrou04248a82010-10-12 20:51:21 +00003403
3404 def test_multiple_unpicklings_seekable(self):
3405 self._check_multiple_unpicklings(io.BytesIO)
3406
3407 def test_multiple_unpicklings_unseekable(self):
Antoine Pitrou9f378722020-02-23 23:33:53 +01003408 self._check_multiple_unpicklings(UnseekableIO, seekable=False)
3409
3410 def test_multiple_unpicklings_minimal(self):
3411 # File-like object that doesn't support peek() and readinto()
3412 # (bpo-39681)
3413 self._check_multiple_unpicklings(MinimalIO, seekable=False)
Antoine Pitrou04248a82010-10-12 20:51:21 +00003414
Antoine Pitrouf6c7a852011-08-11 21:04:02 +02003415 def test_unpickling_buffering_readline(self):
3416 # Issue #12687: the unpickler's buffering logic could fail with
3417 # text mode opcodes.
3418 data = list(range(10))
3419 for proto in protocols:
3420 for buf_size in range(1, 11):
3421 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size)
3422 pickler = self.pickler_class(f, protocol=proto)
3423 pickler.dump(data)
3424 f.seek(0)
3425 unpickler = self.unpickler_class(f)
3426 self.assertEqual(unpickler.load(), data)
3427
Collin Winter771d8342009-04-16 03:18:06 +00003428
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003429# Tests for dispatch_table attribute
3430
3431REDUCE_A = 'reduce_A'
3432
3433class AAA(object):
3434 def __reduce__(self):
3435 return str, (REDUCE_A,)
3436
3437class BBB(object):
Pierre Glaser65d98d02019-05-08 21:40:25 +02003438 def __init__(self):
3439 # Add an instance attribute to enable state-saving routines at pickling
3440 # time.
3441 self.a = "some attribute"
3442
3443 def __setstate__(self, state):
3444 self.a = "BBB.__setstate__"
3445
3446
3447def setstate_bbb(obj, state):
3448 """Custom state setter for BBB objects
3449
3450 Such callable may be created by other persons than the ones who created the
3451 BBB class. If passed as the state_setter item of a custom reducer, this
3452 allows for custom state setting behavior of BBB objects. One can think of
3453 it as the analogous of list_setitems or dict_setitems but for foreign
3454 classes/functions.
3455 """
3456 obj.a = "custom state_setter"
3457
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003458
Pierre Glaser289f1f82019-05-08 23:08:25 +02003459
3460class AbstractCustomPicklerClass:
3461 """Pickler implementing a reducing hook using reducer_override."""
3462 def reducer_override(self, obj):
3463 obj_name = getattr(obj, "__name__", None)
3464
3465 if obj_name == 'f':
3466 # asking the pickler to save f as 5
3467 return int, (5, )
3468
3469 if obj_name == 'MyClass':
3470 return str, ('some str',)
3471
3472 elif obj_name == 'g':
3473 # in this case, the callback returns an invalid result (not a 2-5
3474 # tuple or a string), the pickler should raise a proper error.
3475 return False
3476
3477 elif obj_name == 'h':
3478 # Simulate a case when the reducer fails. The error should
3479 # be propagated to the original ``dump`` call.
3480 raise ValueError('The reducer just failed')
3481
3482 return NotImplemented
3483
3484class AbstractHookTests(unittest.TestCase):
3485 def test_pickler_hook(self):
3486 # test the ability of a custom, user-defined CPickler subclass to
3487 # override the default reducing routines of any type using the method
3488 # reducer_override
3489
3490 def f():
3491 pass
3492
3493 def g():
3494 pass
3495
3496 def h():
3497 pass
3498
3499 class MyClass:
3500 pass
3501
3502 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3503 with self.subTest(proto=proto):
3504 bio = io.BytesIO()
3505 p = self.pickler_class(bio, proto)
3506
3507 p.dump([f, MyClass, math.log])
3508 new_f, some_str, math_log = pickle.loads(bio.getvalue())
3509
3510 self.assertEqual(new_f, 5)
3511 self.assertEqual(some_str, 'some str')
3512 # math.log does not have its usual reducer overriden, so the
3513 # custom reduction callback should silently direct the pickler
3514 # to the default pickling by attribute, by returning
3515 # NotImplemented
3516 self.assertIs(math_log, math.log)
3517
3518 with self.assertRaises(pickle.PicklingError):
3519 p.dump(g)
3520
3521 with self.assertRaisesRegex(
3522 ValueError, 'The reducer just failed'):
3523 p.dump(h)
3524
Pierre Glaser0f2f35e2020-02-02 19:55:21 +01003525 @support.cpython_only
3526 def test_reducer_override_no_reference_cycle(self):
3527 # bpo-39492: reducer_override used to induce a spurious reference cycle
3528 # inside the Pickler object, that could prevent all serialized objects
3529 # from being garbage-collected without explicity invoking gc.collect.
3530
3531 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3532 with self.subTest(proto=proto):
3533 def f():
3534 pass
3535
3536 wr = weakref.ref(f)
3537
3538 bio = io.BytesIO()
3539 p = self.pickler_class(bio, proto)
3540 p.dump(f)
3541 new_f = pickle.loads(bio.getvalue())
3542 assert new_f == 5
3543
3544 del p
3545 del f
3546
3547 self.assertIsNone(wr())
3548
Pierre Glaser289f1f82019-05-08 23:08:25 +02003549
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003550class AbstractDispatchTableTests(unittest.TestCase):
3551
3552 def test_default_dispatch_table(self):
3553 # No dispatch_table attribute by default
3554 f = io.BytesIO()
3555 p = self.pickler_class(f, 0)
3556 with self.assertRaises(AttributeError):
3557 p.dispatch_table
3558 self.assertFalse(hasattr(p, 'dispatch_table'))
3559
3560 def test_class_dispatch_table(self):
3561 # A dispatch_table attribute can be specified class-wide
3562 dt = self.get_dispatch_table()
3563
3564 class MyPickler(self.pickler_class):
3565 dispatch_table = dt
3566
3567 def dumps(obj, protocol=None):
3568 f = io.BytesIO()
3569 p = MyPickler(f, protocol)
3570 self.assertEqual(p.dispatch_table, dt)
3571 p.dump(obj)
3572 return f.getvalue()
3573
3574 self._test_dispatch_table(dumps, dt)
3575
3576 def test_instance_dispatch_table(self):
3577 # A dispatch_table attribute can also be specified instance-wide
3578 dt = self.get_dispatch_table()
3579
3580 def dumps(obj, protocol=None):
3581 f = io.BytesIO()
3582 p = self.pickler_class(f, protocol)
3583 p.dispatch_table = dt
3584 self.assertEqual(p.dispatch_table, dt)
3585 p.dump(obj)
3586 return f.getvalue()
3587
3588 self._test_dispatch_table(dumps, dt)
3589
3590 def _test_dispatch_table(self, dumps, dispatch_table):
3591 def custom_load_dump(obj):
3592 return pickle.loads(dumps(obj, 0))
3593
3594 def default_load_dump(obj):
3595 return pickle.loads(pickle.dumps(obj, 0))
3596
3597 # pickling complex numbers using protocol 0 relies on copyreg
3598 # so check pickling a complex number still works
3599 z = 1 + 2j
3600 self.assertEqual(custom_load_dump(z), z)
3601 self.assertEqual(default_load_dump(z), z)
3602
3603 # modify pickling of complex
3604 REDUCE_1 = 'reduce_1'
3605 def reduce_1(obj):
3606 return str, (REDUCE_1,)
3607 dispatch_table[complex] = reduce_1
3608 self.assertEqual(custom_load_dump(z), REDUCE_1)
3609 self.assertEqual(default_load_dump(z), z)
3610
3611 # check picklability of AAA and BBB
3612 a = AAA()
3613 b = BBB()
3614 self.assertEqual(custom_load_dump(a), REDUCE_A)
3615 self.assertIsInstance(custom_load_dump(b), BBB)
3616 self.assertEqual(default_load_dump(a), REDUCE_A)
3617 self.assertIsInstance(default_load_dump(b), BBB)
3618
3619 # modify pickling of BBB
3620 dispatch_table[BBB] = reduce_1
3621 self.assertEqual(custom_load_dump(a), REDUCE_A)
3622 self.assertEqual(custom_load_dump(b), REDUCE_1)
3623 self.assertEqual(default_load_dump(a), REDUCE_A)
3624 self.assertIsInstance(default_load_dump(b), BBB)
3625
3626 # revert pickling of BBB and modify pickling of AAA
3627 REDUCE_2 = 'reduce_2'
3628 def reduce_2(obj):
3629 return str, (REDUCE_2,)
3630 dispatch_table[AAA] = reduce_2
3631 del dispatch_table[BBB]
3632 self.assertEqual(custom_load_dump(a), REDUCE_2)
3633 self.assertIsInstance(custom_load_dump(b), BBB)
3634 self.assertEqual(default_load_dump(a), REDUCE_A)
3635 self.assertIsInstance(default_load_dump(b), BBB)
3636
Pierre Glaser65d98d02019-05-08 21:40:25 +02003637 # End-to-end testing of save_reduce with the state_setter keyword
3638 # argument. This is a dispatch_table test as the primary goal of
3639 # state_setter is to tweak objects reduction behavior.
3640 # In particular, state_setter is useful when the default __setstate__
3641 # behavior is not flexible enough.
3642
3643 # No custom reducer for b has been registered for now, so
3644 # BBB.__setstate__ should be used at unpickling time
3645 self.assertEqual(default_load_dump(b).a, "BBB.__setstate__")
3646
3647 def reduce_bbb(obj):
3648 return BBB, (), obj.__dict__, None, None, setstate_bbb
3649
3650 dispatch_table[BBB] = reduce_bbb
3651
3652 # The custom reducer reduce_bbb includes a state setter, that should
3653 # have priority over BBB.__setstate__
3654 self.assertEqual(custom_load_dump(b).a, "custom state_setter")
3655
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003656
Guido van Rossum98297ee2007-11-06 21:34:58 +00003657if __name__ == "__main__":
3658 # Print some stuff that can be used to rewrite DATA{0,1,2}
3659 from pickletools import dis
3660 x = create_data()
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03003661 for i in range(pickle.HIGHEST_PROTOCOL+1):
Guido van Rossum98297ee2007-11-06 21:34:58 +00003662 p = pickle.dumps(x, i)
3663 print("DATA{0} = (".format(i))
3664 for j in range(0, len(p), 20):
3665 b = bytes(p[j:j+20])
3666 print(" {0!r}".format(b))
3667 print(")")
3668 print()
3669 print("# Disassembly of DATA{0}".format(i))
3670 print("DATA{0}_DIS = \"\"\"\\".format(i))
3671 dis(p)
3672 print("\"\"\"")
3673 print()