blob: c6f4f6cb43c56e3aead2e039da2ee7b797ef80a7 [file] [log] [blame]
Serhiy Storchakabfe18242015-03-31 13:12:37 +03001import collections
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002import copyreg
Serhiy Storchakabfe18242015-03-31 13:12:37 +03003import dbm
Collin Winter771d8342009-04-16 03:18:06 +00004import io
Serhiy Storchakabfe18242015-03-31 13:12:37 +03005import functools
Tim Peters4190fb82003-02-02 16:09:05 +00006import pickle
Tim Peters31f119e2003-02-03 16:20:13 +00007import pickletools
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08008import struct
Antoine Pitrou82be19f2011-08-29 23:09:33 +02009import sys
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010010import unittest
Antoine Pitrou16c4ce12011-03-11 21:30:43 +010011import weakref
Antoine Pitroud9dfaa92009-06-04 20:32:06 +000012from http.cookies import SimpleCookie
Tim Peters4190fb82003-02-02 16:09:05 +000013
Antoine Pitrou82be19f2011-08-29 23:09:33 +020014from test.support import (
Antoine Pitrouee763e22011-08-29 23:14:53 +020015 TestFailed, TESTFN, run_with_locale, no_tracing,
Antoine Pitrou94190bb2011-10-04 10:22:36 +020016 _2G, _4G, bigmemtest,
Antoine Pitrou82be19f2011-08-29 23:09:33 +020017 )
Tim Peterse089c682001-04-10 03:41:41 +000018
Guido van Rossum98297ee2007-11-06 21:34:58 +000019from pickle import bytes_types
20
Serhiy Storchakac6b54b42015-09-29 15:33:24 +030021requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
22 "test is only meaningful on 32-bit builds")
23
Tim Petersee1a53c2003-02-02 02:57:53 +000024# Tests that try a number of pickle protocols should have a
25# for proto in protocols:
Tim Peters8587b3c2003-02-13 15:44:41 +000026# kind of outer loop.
Tim Peters8587b3c2003-02-13 15:44:41 +000027protocols = range(pickle.HIGHEST_PROTOCOL + 1)
Tim Petersee1a53c2003-02-02 02:57:53 +000028
Tim Peters22e71712003-02-03 22:27:38 +000029
30# Return True if opcode code appears in the pickle, else False.
31def opcode_in_pickle(code, pickle):
32 for op, dummy, dummy in pickletools.genops(pickle):
Guido van Rossumcfe5f202007-05-08 21:26:54 +000033 if op.code == code.decode("latin-1"):
Tim Peters22e71712003-02-03 22:27:38 +000034 return True
35 return False
36
Tim Peters8d2613a2003-02-11 16:40:16 +000037# Return the number of times opcode code appears in pickle.
38def count_opcode(code, pickle):
39 n = 0
40 for op, dummy, dummy in pickletools.genops(pickle):
Guido van Rossumcfe5f202007-05-08 21:26:54 +000041 if op.code == code.decode("latin-1"):
Tim Peters8d2613a2003-02-11 16:40:16 +000042 n += 1
43 return n
44
Antoine Pitrou04248a82010-10-12 20:51:21 +000045
46class UnseekableIO(io.BytesIO):
47 def peek(self, *args):
48 raise NotImplementedError
49
50 def seekable(self):
51 return False
52
53 def seek(self, *args):
54 raise io.UnsupportedOperation
55
56 def tell(self):
57 raise io.UnsupportedOperation
58
59
Tim Peters3e667d52003-02-04 21:47:44 +000060# We can't very well test the extension registry without putting known stuff
61# in it, but we have to be careful to restore its original state. Code
62# should do this:
63#
64# e = ExtensionSaver(extension_code)
65# try:
66# fiddle w/ the extension registry's stuff for extension_code
67# finally:
68# e.restore()
69
70class ExtensionSaver:
71 # Remember current registration for code (if any), and remove it (if
72 # there is one).
73 def __init__(self, code):
74 self.code = code
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000075 if code in copyreg._inverted_registry:
76 self.pair = copyreg._inverted_registry[code]
77 copyreg.remove_extension(self.pair[0], self.pair[1], code)
Tim Peters3e667d52003-02-04 21:47:44 +000078 else:
79 self.pair = None
80
81 # Restore previous registration for code.
82 def restore(self):
83 code = self.code
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000084 curpair = copyreg._inverted_registry.get(code)
Tim Peters3e667d52003-02-04 21:47:44 +000085 if curpair is not None:
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000086 copyreg.remove_extension(curpair[0], curpair[1], code)
Tim Peters3e667d52003-02-04 21:47:44 +000087 pair = self.pair
88 if pair is not None:
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000089 copyreg.add_extension(pair[0], pair[1], code)
Tim Peters3e667d52003-02-04 21:47:44 +000090
Jeremy Hylton66426532001-10-15 21:38:56 +000091class C:
Guido van Rossum47b9ff62006-08-24 00:41:19 +000092 def __eq__(self, other):
93 return self.__dict__ == other.__dict__
Jeremy Hylton66426532001-10-15 21:38:56 +000094
Alexander Belopolskyd92f0402010-07-17 22:50:45 +000095class D(C):
96 def __init__(self, arg):
97 pass
98
99class E(C):
100 def __getinitargs__(self):
101 return ()
102
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100103class H(object):
104 pass
105
Jeremy Hylton66426532001-10-15 21:38:56 +0000106import __main__
107__main__.C = C
108C.__module__ = "__main__"
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000109__main__.D = D
110D.__module__ = "__main__"
111__main__.E = E
112E.__module__ = "__main__"
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100113__main__.H = H
114H.__module__ = "__main__"
Jeremy Hylton66426532001-10-15 21:38:56 +0000115
116class myint(int):
117 def __init__(self, x):
118 self.str = str(x)
119
120class initarg(C):
Guido van Rossum1444f672001-12-19 16:38:29 +0000121
Jeremy Hylton66426532001-10-15 21:38:56 +0000122 def __init__(self, a, b):
123 self.a = a
124 self.b = b
125
126 def __getinitargs__(self):
127 return self.a, self.b
128
Guido van Rossum04a86612001-12-19 16:58:54 +0000129class metaclass(type):
130 pass
131
Guido van Rossum52cc1d82007-03-18 15:41:51 +0000132class use_metaclass(object, metaclass=metaclass):
133 pass
Guido van Rossum04a86612001-12-19 16:58:54 +0000134
Antoine Pitrouffd41d92011-10-04 09:23:04 +0200135class pickling_metaclass(type):
136 def __eq__(self, other):
137 return (type(self) == type(other) and
138 self.reduce_args == other.reduce_args)
139
140 def __reduce__(self):
141 return (create_dynamic_class, self.reduce_args)
142
143def create_dynamic_class(name, bases):
144 result = pickling_metaclass(name, bases, dict())
145 result.reduce_args = (name, bases)
146 return result
147
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300148# DATA0 .. DATA4 are the pickles we expect under the various protocols, for
Tim Peters70b02d72003-02-02 17:26:40 +0000149# the object returned by create_data().
Tim Petersee1a53c2003-02-02 02:57:53 +0000150
Guido van Rossum98297ee2007-11-06 21:34:58 +0000151DATA0 = (
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200152 b'(lp0\nL0L\naL1L\naF2.0\n'
153 b'ac__builtin__\ncomple'
154 b'x\np1\n(F3.0\nF0.0\ntp2\n'
155 b'Rp3\naL1L\naL-1L\naL255'
156 b'L\naL-255L\naL-256L\naL'
157 b'65535L\naL-65535L\naL-'
158 b'65536L\naL2147483647L'
159 b'\naL-2147483647L\naL-2'
160 b'147483648L\na(Vabc\np4'
161 b'\ng4\nccopy_reg\n_recon'
162 b'structor\np5\n(c__main'
163 b'__\nC\np6\nc__builtin__'
164 b'\nobject\np7\nNtp8\nRp9\n'
165 b'(dp10\nVfoo\np11\nL1L\ns'
166 b'Vbar\np12\nL2L\nsbg9\ntp'
167 b'13\nag13\naL5L\na.'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000168)
Tim Peterse9358162001-01-22 22:05:20 +0000169
Guido van Rossum98297ee2007-11-06 21:34:58 +0000170# Disassembly of DATA0
Tim Peters70b02d72003-02-02 17:26:40 +0000171DATA0_DIS = """\
172 0: ( MARK
173 1: l LIST (MARK at 0)
Guido van Rossum98297ee2007-11-06 21:34:58 +0000174 2: p PUT 0
175 5: L LONG 0
Mark Dickinson8dd05142009-01-20 20:43:58 +0000176 9: a APPEND
177 10: L LONG 1
178 14: a APPEND
179 15: F FLOAT 2.0
180 20: a APPEND
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200181 21: c GLOBAL '__builtin__ complex'
182 42: p PUT 1
183 45: ( MARK
184 46: F FLOAT 3.0
185 51: F FLOAT 0.0
186 56: t TUPLE (MARK at 45)
187 57: p PUT 2
188 60: R REDUCE
189 61: p PUT 3
190 64: a APPEND
191 65: L LONG 1
192 69: a APPEND
193 70: L LONG -1
194 75: a APPEND
195 76: L LONG 255
196 82: a APPEND
197 83: L LONG -255
198 90: a APPEND
199 91: L LONG -256
200 98: a APPEND
201 99: L LONG 65535
202 107: a APPEND
203 108: L LONG -65535
204 117: a APPEND
205 118: L LONG -65536
206 127: a APPEND
207 128: L LONG 2147483647
208 141: a APPEND
209 142: L LONG -2147483647
210 156: a APPEND
211 157: L LONG -2147483648
212 171: a APPEND
213 172: ( MARK
214 173: V UNICODE 'abc'
215 178: p PUT 4
216 181: g GET 4
217 184: c GLOBAL 'copy_reg _reconstructor'
218 209: p PUT 5
219 212: ( MARK
220 213: c GLOBAL '__main__ C'
221 225: p PUT 6
222 228: c GLOBAL '__builtin__ object'
223 248: p PUT 7
224 251: N NONE
225 252: t TUPLE (MARK at 212)
226 253: p PUT 8
227 256: R REDUCE
228 257: p PUT 9
229 260: ( MARK
230 261: d DICT (MARK at 260)
231 262: p PUT 10
232 266: V UNICODE 'foo'
233 271: p PUT 11
234 275: L LONG 1
235 279: s SETITEM
236 280: V UNICODE 'bar'
237 285: p PUT 12
238 289: L LONG 2
239 293: s SETITEM
240 294: b BUILD
241 295: g GET 9
242 298: t TUPLE (MARK at 172)
243 299: p PUT 13
244 303: a APPEND
245 304: g GET 13
246 308: a APPEND
247 309: L LONG 5
248 313: a APPEND
249 314: . STOP
Tim Peters70b02d72003-02-02 17:26:40 +0000250highest protocol among opcodes = 0
251"""
252
Guido van Rossum98297ee2007-11-06 21:34:58 +0000253DATA1 = (
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200254 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__'
255 b'builtin__\ncomplex\nq\x01'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000256 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t'
257 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ'
258 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff'
259 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab'
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200260 b'cq\x04h\x04ccopy_reg\n_reco'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000261 b'nstructor\nq\x05(c__main'
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200262 b'__\nC\nq\x06c__builtin__\n'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000263 b'object\nq\x07Ntq\x08Rq\t}q\n('
264 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar'
265 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.'
266)
Tim Peters70b02d72003-02-02 17:26:40 +0000267
Guido van Rossum98297ee2007-11-06 21:34:58 +0000268# Disassembly of DATA1
Tim Peters70b02d72003-02-02 17:26:40 +0000269DATA1_DIS = """\
270 0: ] EMPTY_LIST
Guido van Rossum98297ee2007-11-06 21:34:58 +0000271 1: q BINPUT 0
Tim Peters70b02d72003-02-02 17:26:40 +0000272 3: ( MARK
273 4: K BININT1 0
Guido van Rossum98297ee2007-11-06 21:34:58 +0000274 6: K BININT1 1
275 8: G BINFLOAT 2.0
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200276 17: c GLOBAL '__builtin__ complex'
277 38: q BINPUT 1
278 40: ( MARK
279 41: G BINFLOAT 3.0
280 50: G BINFLOAT 0.0
281 59: t TUPLE (MARK at 40)
282 60: q BINPUT 2
283 62: R REDUCE
284 63: q BINPUT 3
285 65: K BININT1 1
286 67: J BININT -1
287 72: K BININT1 255
288 74: J BININT -255
289 79: J BININT -256
290 84: M BININT2 65535
291 87: J BININT -65535
292 92: J BININT -65536
293 97: J BININT 2147483647
294 102: J BININT -2147483647
295 107: J BININT -2147483648
296 112: ( MARK
297 113: X BINUNICODE 'abc'
298 121: q BINPUT 4
299 123: h BINGET 4
300 125: c GLOBAL 'copy_reg _reconstructor'
301 150: q BINPUT 5
302 152: ( MARK
303 153: c GLOBAL '__main__ C'
304 165: q BINPUT 6
305 167: c GLOBAL '__builtin__ object'
306 187: q BINPUT 7
307 189: N NONE
308 190: t TUPLE (MARK at 152)
309 191: q BINPUT 8
310 193: R REDUCE
311 194: q BINPUT 9
312 196: } EMPTY_DICT
313 197: q BINPUT 10
314 199: ( MARK
315 200: X BINUNICODE 'foo'
316 208: q BINPUT 11
317 210: K BININT1 1
318 212: X BINUNICODE 'bar'
319 220: q BINPUT 12
320 222: K BININT1 2
321 224: u SETITEMS (MARK at 199)
322 225: b BUILD
323 226: h BINGET 9
324 228: t TUPLE (MARK at 112)
325 229: q BINPUT 13
326 231: h BINGET 13
327 233: K BININT1 5
328 235: e APPENDS (MARK at 3)
329 236: . STOP
Tim Peters70b02d72003-02-02 17:26:40 +0000330highest protocol among opcodes = 1
331"""
Tim Peterse0c446b2001-10-18 21:57:37 +0000332
Guido van Rossum98297ee2007-11-06 21:34:58 +0000333DATA2 = (
334 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200335 b'__builtin__\ncomplex\n'
Guido van Rossum98297ee2007-11-06 21:34:58 +0000336 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00'
337 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff'
338 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff'
339 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a'
340 b'bcq\x04h\x04c__main__\nC\nq\x05'
341 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01'
342 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh'
343 b'\nK\x05e.'
344)
Tim Petersfc273752003-03-02 04:54:24 +0000345
Guido van Rossum98297ee2007-11-06 21:34:58 +0000346# Disassembly of DATA2
Tim Petersfc273752003-03-02 04:54:24 +0000347DATA2_DIS = """\
348 0: \x80 PROTO 2
349 2: ] EMPTY_LIST
Guido van Rossum98297ee2007-11-06 21:34:58 +0000350 3: q BINPUT 0
Tim Petersfc273752003-03-02 04:54:24 +0000351 5: ( MARK
352 6: K BININT1 0
Guido van Rossum98297ee2007-11-06 21:34:58 +0000353 8: K BININT1 1
354 10: G BINFLOAT 2.0
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200355 19: c GLOBAL '__builtin__ complex'
356 40: q BINPUT 1
357 42: G BINFLOAT 3.0
358 51: G BINFLOAT 0.0
359 60: \x86 TUPLE2
360 61: q BINPUT 2
361 63: R REDUCE
362 64: q BINPUT 3
363 66: K BININT1 1
364 68: J BININT -1
365 73: K BININT1 255
366 75: J BININT -255
367 80: J BININT -256
368 85: M BININT2 65535
369 88: J BININT -65535
370 93: J BININT -65536
371 98: J BININT 2147483647
372 103: J BININT -2147483647
373 108: J BININT -2147483648
374 113: ( MARK
375 114: X BINUNICODE 'abc'
376 122: q BINPUT 4
377 124: h BINGET 4
378 126: c GLOBAL '__main__ C'
379 138: q BINPUT 5
380 140: ) EMPTY_TUPLE
381 141: \x81 NEWOBJ
382 142: q BINPUT 6
383 144: } EMPTY_DICT
384 145: q BINPUT 7
385 147: ( MARK
386 148: X BINUNICODE 'foo'
387 156: q BINPUT 8
388 158: K BININT1 1
389 160: X BINUNICODE 'bar'
390 168: q BINPUT 9
391 170: K BININT1 2
392 172: u SETITEMS (MARK at 147)
393 173: b BUILD
394 174: h BINGET 6
395 176: t TUPLE (MARK at 113)
396 177: q BINPUT 10
397 179: h BINGET 10
398 181: K BININT1 5
399 183: e APPENDS (MARK at 5)
400 184: . STOP
Tim Petersfc273752003-03-02 04:54:24 +0000401highest protocol among opcodes = 2
402"""
403
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300404DATA3 = (
405 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
406 b'builtins\ncomplex\nq\x01G'
407 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
408 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
409 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
410 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
411 b'\x04h\x04c__main__\nC\nq\x05)\x81q'
412 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
413 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
414 b'e.'
415)
416
417# Disassembly of DATA3
418DATA3_DIS = """\
419 0: \x80 PROTO 3
420 2: ] EMPTY_LIST
421 3: q BINPUT 0
422 5: ( MARK
423 6: K BININT1 0
424 8: K BININT1 1
425 10: G BINFLOAT 2.0
426 19: c GLOBAL 'builtins complex'
427 37: q BINPUT 1
428 39: G BINFLOAT 3.0
429 48: G BINFLOAT 0.0
430 57: \x86 TUPLE2
431 58: q BINPUT 2
432 60: R REDUCE
433 61: q BINPUT 3
434 63: K BININT1 1
435 65: J BININT -1
436 70: K BININT1 255
437 72: J BININT -255
438 77: J BININT -256
439 82: M BININT2 65535
440 85: J BININT -65535
441 90: J BININT -65536
442 95: J BININT 2147483647
443 100: J BININT -2147483647
444 105: J BININT -2147483648
445 110: ( MARK
446 111: X BINUNICODE 'abc'
447 119: q BINPUT 4
448 121: h BINGET 4
449 123: c GLOBAL '__main__ C'
450 135: q BINPUT 5
451 137: ) EMPTY_TUPLE
452 138: \x81 NEWOBJ
453 139: q BINPUT 6
454 141: } EMPTY_DICT
455 142: q BINPUT 7
456 144: ( MARK
457 145: X BINUNICODE 'bar'
458 153: q BINPUT 8
459 155: K BININT1 2
460 157: X BINUNICODE 'foo'
461 165: q BINPUT 9
462 167: K BININT1 1
463 169: u SETITEMS (MARK at 144)
464 170: b BUILD
465 171: h BINGET 6
466 173: t TUPLE (MARK at 110)
467 174: q BINPUT 10
468 176: h BINGET 10
469 178: K BININT1 5
470 180: e APPENDS (MARK at 5)
471 181: . STOP
472highest protocol among opcodes = 2
473"""
474
475DATA4 = (
476 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
477 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
478 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
479 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
480 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
481 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
482 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
483 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
484 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
485)
486
487# Disassembly of DATA4
488DATA4_DIS = """\
489 0: \x80 PROTO 4
490 2: \x95 FRAME 168
491 11: ] EMPTY_LIST
492 12: \x94 MEMOIZE
493 13: ( MARK
494 14: K BININT1 0
495 16: K BININT1 1
496 18: G BINFLOAT 2.0
497 27: \x8c SHORT_BINUNICODE 'builtins'
498 37: \x94 MEMOIZE
499 38: \x8c SHORT_BINUNICODE 'complex'
500 47: \x94 MEMOIZE
501 48: \x93 STACK_GLOBAL
502 49: \x94 MEMOIZE
503 50: G BINFLOAT 3.0
504 59: G BINFLOAT 0.0
505 68: \x86 TUPLE2
506 69: \x94 MEMOIZE
507 70: R REDUCE
508 71: \x94 MEMOIZE
509 72: K BININT1 1
510 74: J BININT -1
511 79: K BININT1 255
512 81: J BININT -255
513 86: J BININT -256
514 91: M BININT2 65535
515 94: J BININT -65535
516 99: J BININT -65536
517 104: J BININT 2147483647
518 109: J BININT -2147483647
519 114: J BININT -2147483648
520 119: ( MARK
521 120: \x8c SHORT_BINUNICODE 'abc'
522 125: \x94 MEMOIZE
523 126: h BINGET 6
524 128: \x8c SHORT_BINUNICODE '__main__'
525 138: \x94 MEMOIZE
526 139: \x8c SHORT_BINUNICODE 'C'
527 142: \x94 MEMOIZE
528 143: \x93 STACK_GLOBAL
529 144: \x94 MEMOIZE
530 145: ) EMPTY_TUPLE
531 146: \x81 NEWOBJ
532 147: \x94 MEMOIZE
533 148: } EMPTY_DICT
534 149: \x94 MEMOIZE
535 150: ( MARK
536 151: \x8c SHORT_BINUNICODE 'bar'
537 156: \x94 MEMOIZE
538 157: K BININT1 2
539 159: \x8c SHORT_BINUNICODE 'foo'
540 164: \x94 MEMOIZE
541 165: K BININT1 1
542 167: u SETITEMS (MARK at 150)
543 168: b BUILD
544 169: h BINGET 10
545 171: t TUPLE (MARK at 119)
546 172: \x94 MEMOIZE
547 173: h BINGET 14
548 175: K BININT1 5
549 177: e APPENDS (MARK at 13)
550 178: . STOP
551highest protocol among opcodes = 4
552"""
553
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000554# set([1,2]) pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300555DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000556
557# xrange(5) pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300558DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000559
560# a SimpleCookie() object pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300561DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
562 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
563 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
564 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
565 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
566 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000567
568# set([3]) pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300569DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000570
Walter Doerwald9d1dbca2013-12-02 11:41:01 +0100571python2_exceptions_without_args = (
572 ArithmeticError,
573 AssertionError,
574 AttributeError,
575 BaseException,
576 BufferError,
577 BytesWarning,
578 DeprecationWarning,
579 EOFError,
580 EnvironmentError,
581 Exception,
582 FloatingPointError,
583 FutureWarning,
584 GeneratorExit,
585 IOError,
586 ImportError,
587 ImportWarning,
588 IndentationError,
589 IndexError,
590 KeyError,
591 KeyboardInterrupt,
592 LookupError,
593 MemoryError,
594 NameError,
595 NotImplementedError,
596 OSError,
597 OverflowError,
598 PendingDeprecationWarning,
599 ReferenceError,
600 RuntimeError,
601 RuntimeWarning,
602 # StandardError is gone in Python 3, we map it to Exception
603 StopIteration,
604 SyntaxError,
605 SyntaxWarning,
606 SystemError,
607 SystemExit,
608 TabError,
609 TypeError,
610 UnboundLocalError,
611 UnicodeError,
612 UnicodeWarning,
613 UserWarning,
614 ValueError,
615 Warning,
616 ZeroDivisionError,
617)
618
619exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
620
Walter Doerwald9d1dbca2013-12-02 11:41:01 +0100621# UnicodeEncodeError object pickled from 2.x with protocol 2
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300622DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
623 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
624 b'U\x03badq\x03tq\x04Rq\x05.')
Walter Doerwald9d1dbca2013-12-02 11:41:01 +0100625
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000626
Jeremy Hylton66426532001-10-15 21:38:56 +0000627def create_data():
Tim Peterse9358162001-01-22 22:05:20 +0000628 c = C()
629 c.foo = 1
630 c.bar = 2
Guido van Rossume2a383d2007-01-15 16:59:06 +0000631 x = [0, 1, 2.0, 3.0+0j]
Tim Peters461922a2001-04-09 20:07:05 +0000632 # Append some integer test cases at cPickle.c's internal size
633 # cutoffs.
634 uint1max = 0xff
635 uint2max = 0xffff
636 int4max = 0x7fffffff
637 x.extend([1, -1,
638 uint1max, -uint1max, -uint1max-1,
639 uint2max, -uint2max, -uint2max-1,
640 int4max, -int4max, -int4max-1])
Tim Peterse9358162001-01-22 22:05:20 +0000641 y = ('abc', 'abc', c, c)
642 x.append(y)
643 x.append(y)
644 x.append(5)
Jeremy Hylton66426532001-10-15 21:38:56 +0000645 return x
Tim Petersc58440f2001-04-09 17:16:31 +0000646
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100647
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300648class AbstractUnpickleTests(unittest.TestCase):
649 # Subclass must define self.loads.
Antoine Pitrou3ab9cfc2013-11-24 14:33:37 +0100650
Jeremy Hylton66426532001-10-15 21:38:56 +0000651 _testdata = create_data()
Tim Petersc58440f2001-04-09 17:16:31 +0000652
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100653 def assert_is_copy(self, obj, objcopy, msg=None):
654 """Utility method to verify if two objects are copies of each others.
655 """
656 if msg is None:
657 msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
658 self.assertEqual(obj, objcopy, msg=msg)
659 self.assertIs(type(obj), type(objcopy), msg=msg)
660 if hasattr(obj, '__dict__'):
661 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
662 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
663 if hasattr(obj, '__slots__'):
664 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
665 for slot in obj.__slots__:
666 self.assertEqual(
667 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
668 self.assertEqual(getattr(obj, slot, None),
669 getattr(objcopy, slot, None), msg=msg)
670
Guido van Rossum98297ee2007-11-06 21:34:58 +0000671 def test_load_from_data0(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100672 self.assert_is_copy(self._testdata, self.loads(DATA0))
Guido van Rossum98297ee2007-11-06 21:34:58 +0000673
674 def test_load_from_data1(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100675 self.assert_is_copy(self._testdata, self.loads(DATA1))
Guido van Rossum98297ee2007-11-06 21:34:58 +0000676
677 def test_load_from_data2(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100678 self.assert_is_copy(self._testdata, self.loads(DATA2))
Jeremy Hylton66426532001-10-15 21:38:56 +0000679
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300680 def test_load_from_data3(self):
681 self.assert_is_copy(self._testdata, self.loads(DATA3))
682
683 def test_load_from_data4(self):
684 self.assert_is_copy(self._testdata, self.loads(DATA4))
685
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000686 def test_load_classic_instance(self):
687 # See issue5180. Test loading 2.x pickles that
688 # contain an instance of old style class.
689 for X, args in [(C, ()), (D, ('x',)), (E, ())]:
690 xname = X.__name__.encode('ascii')
691 # Protocol 0 (text mode pickle):
692 """
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200693 0: ( MARK
694 1: i INST '__main__ X' (MARK at 0)
695 13: p PUT 0
696 16: ( MARK
697 17: d DICT (MARK at 16)
698 18: p PUT 1
699 21: b BUILD
700 22: . STOP
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000701 """
702 pickle0 = (b"(i__main__\n"
703 b"X\n"
704 b"p0\n"
705 b"(dp1\nb.").replace(b'X', xname)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100706 self.assert_is_copy(X(*args), self.loads(pickle0))
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000707
708 # Protocol 1 (binary mode pickle)
709 """
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200710 0: ( MARK
711 1: c GLOBAL '__main__ X'
712 13: q BINPUT 0
713 15: o OBJ (MARK at 0)
714 16: q BINPUT 1
715 18: } EMPTY_DICT
716 19: q BINPUT 2
717 21: b BUILD
718 22: . STOP
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000719 """
720 pickle1 = (b'(c__main__\n'
721 b'X\n'
722 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100723 self.assert_is_copy(X(*args), self.loads(pickle1))
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000724
725 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
726 """
Serhiy Storchakafa310ee2015-02-15 14:10:03 +0200727 0: \x80 PROTO 2
728 2: ( MARK
729 3: c GLOBAL '__main__ X'
730 15: q BINPUT 0
731 17: o OBJ (MARK at 2)
732 18: q BINPUT 1
733 20: } EMPTY_DICT
734 21: q BINPUT 2
735 23: b BUILD
736 24: . STOP
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000737 """
738 pickle2 = (b'\x80\x02(c__main__\n'
739 b'X\n'
740 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100741 self.assert_is_copy(X(*args), self.loads(pickle2))
Alexander Belopolskyd92f0402010-07-17 22:50:45 +0000742
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300743 def test_maxint64(self):
744 maxint64 = (1 << 63) - 1
745 data = b'I' + str(maxint64).encode("ascii") + b'\n.'
746 got = self.loads(data)
747 self.assert_is_copy(maxint64, got)
748
749 # Try too with a bogus literal.
750 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
751 self.assertRaises(ValueError, self.loads, data)
752
753 def test_pop_empty_stack(self):
754 # Test issue7455
755 s = b'0'
756 self.assertRaises((pickle.UnpicklingError, IndexError), self.loads, s)
757
758 def test_unpickle_from_2x(self):
759 # Unpickle non-trivial data from Python 2.x.
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300760 loaded = self.loads(DATA_SET)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300761 self.assertEqual(loaded, set([1, 2]))
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300762 loaded = self.loads(DATA_XRANGE)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300763 self.assertEqual(type(loaded), type(range(0)))
764 self.assertEqual(list(loaded), list(range(5)))
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300765 loaded = self.loads(DATA_COOKIE)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300766 self.assertEqual(type(loaded), SimpleCookie)
767 self.assertEqual(list(loaded.keys()), ["key"])
768 self.assertEqual(loaded["key"].value, "value")
769
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300770 # Exception objects without arguments pickled from 2.x with protocol 2
771 for exc in python2_exceptions_without_args:
772 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300773 loaded = self.loads(data)
774 self.assertIs(type(loaded), exc)
775
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300776 # StandardError is mapped to Exception, test that separately
777 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300778 self.assertIs(type(loaded), Exception)
779
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300780 loaded = self.loads(DATA_UEERR)
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300781 self.assertIs(type(loaded), UnicodeEncodeError)
782 self.assertEqual(loaded.object, "foo")
783 self.assertEqual(loaded.encoding, "ascii")
784 self.assertEqual(loaded.start, 0)
785 self.assertEqual(loaded.end, 1)
786 self.assertEqual(loaded.reason, "bad")
787
788 def test_load_python2_str_as_bytes(self):
789 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
790 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
791 encoding="bytes"), b'a\x00\xa0')
792 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
793 self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
794 encoding="bytes"), b'a\x00\xa0')
795 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
796 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
797 encoding="bytes"), b'a\x00\xa0')
798
799 def test_load_python2_unicode_as_str(self):
800 # From Python 2: pickle.dumps(u'π', protocol=0)
801 self.assertEqual(self.loads(b'V\\u03c0\n.',
802 encoding='bytes'), 'π')
803 # From Python 2: pickle.dumps(u'π', protocol=1)
804 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
805 encoding="bytes"), 'π')
806 # From Python 2: pickle.dumps(u'π', protocol=2)
807 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
808 encoding="bytes"), 'π')
809
810 def test_load_long_python2_str_as_bytes(self):
811 # From Python 2: pickle.dumps('x' * 300, protocol=1)
812 self.assertEqual(self.loads(pickle.BINSTRING +
813 struct.pack("<I", 300) +
814 b'x' * 300 + pickle.STOP,
815 encoding='bytes'), b'x' * 300)
816
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300817 def test_constants(self):
818 self.assertIsNone(self.loads(b'N.'))
819 self.assertIs(self.loads(b'\x88.'), True)
820 self.assertIs(self.loads(b'\x89.'), False)
821 self.assertIs(self.loads(b'I01\n.'), True)
822 self.assertIs(self.loads(b'I00\n.'), False)
823
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300824 def test_empty_bytestring(self):
825 # issue 11286
826 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
827 self.assertEqual(empty, '')
828
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300829 def test_short_binbytes(self):
830 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
831 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
832
833 def test_binbytes(self):
834 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
835 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
836
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300837 @requires_32b
838 def test_negative_32b_binbytes(self):
839 # On 32-bit builds, a BINBYTES of 2**31 or more is refused
840 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.'
841 with self.assertRaises((pickle.UnpicklingError, OverflowError)):
842 self.loads(dumped)
843
844 @requires_32b
845 def test_negative_32b_binunicode(self):
846 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused
847 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.'
848 with self.assertRaises((pickle.UnpicklingError, OverflowError)):
849 self.loads(dumped)
850
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300851 def test_short_binunicode(self):
852 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
853 self.assertEqual(self.loads(dumped), '\u20ac\x00')
854
855 def test_misc_get(self):
856 self.assertRaises(KeyError, self.loads, b'g0\np0')
857 self.assert_is_copy([(100,), (100,)],
858 self.loads(b'((Kdtp0\nh\x00l.))'))
859
Serhiy Storchakae0606192015-09-29 22:10:07 +0300860 def test_binbytes8(self):
861 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
862 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
863
864 def test_binunicode8(self):
865 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
866 self.assertEqual(self.loads(dumped), '\u20ac\x00')
867
868 @requires_32b
869 def test_large_32b_binbytes8(self):
870 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
871 with self.assertRaises((pickle.UnpicklingError, OverflowError)):
872 self.loads(dumped)
873
874 @requires_32b
875 def test_large_32b_binunicode8(self):
876 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
877 with self.assertRaises((pickle.UnpicklingError, OverflowError)):
878 self.loads(dumped)
879
Serhiy Storchakab8b951f2015-09-29 15:49:58 +0300880 def test_get(self):
881 pickled = b'((lp100000\ng100000\nt.'
882 unpickled = self.loads(pickled)
883 self.assertEqual(unpickled, ([],)*2)
884 self.assertIs(unpickled[0], unpickled[1])
885
886 def test_binget(self):
887 pickled = b'(]q\xffh\xfft.'
888 unpickled = self.loads(pickled)
889 self.assertEqual(unpickled, ([],)*2)
890 self.assertIs(unpickled[0], unpickled[1])
891
892 def test_long_binget(self):
893 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
894 unpickled = self.loads(pickled)
895 self.assertEqual(unpickled, ([],)*2)
896 self.assertIs(unpickled[0], unpickled[1])
897
898 def test_dup(self):
899 pickled = b'((l2t.'
900 unpickled = self.loads(pickled)
901 self.assertEqual(unpickled, ([],)*2)
902 self.assertIs(unpickled[0], unpickled[1])
903
Serhiy Storchakac6b54b42015-09-29 15:33:24 +0300904 def test_negative_put(self):
905 # Issue #12847
906 dumped = b'Va\np-1\n.'
907 self.assertRaises(ValueError, self.loads, dumped)
908
909 @requires_32b
910 def test_negative_32b_binput(self):
911 # Issue #12847
912 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
913 self.assertRaises(ValueError, self.loads, dumped)
914
915 def test_badly_escaped_string(self):
916 self.assertRaises(ValueError, self.loads, b"S'\\'\n.")
917
918 def test_badly_quoted_string(self):
919 # Issue #17710
920 badpickles = [b"S'\n.",
921 b'S"\n.',
922 b'S\' \n.',
923 b'S" \n.',
924 b'S\'"\n.',
925 b'S"\'\n.',
926 b"S' ' \n.",
927 b'S" " \n.',
928 b"S ''\n.",
929 b'S ""\n.',
930 b'S \n.',
931 b'S\n.',
932 b'S.']
933 for p in badpickles:
934 self.assertRaises(pickle.UnpicklingError, self.loads, p)
935
936 def test_correctly_quoted_string(self):
937 goodpickles = [(b"S''\n.", ''),
938 (b'S""\n.', ''),
939 (b'S"\\n"\n.', '\n'),
940 (b"S'\\n'\n.", '\n')]
941 for p, expected in goodpickles:
942 self.assertEqual(self.loads(p), expected)
943
944 def test_frame_readline(self):
945 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.'
946 # 0: \x80 PROTO 4
947 # 2: \x95 FRAME 5
948 # 11: I INT 42
949 # 15: . STOP
950 self.assertEqual(self.loads(pickled), 42)
951
952 def test_compat_unpickle(self):
953 # xrange(1, 7)
954 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.'
955 unpickled = self.loads(pickled)
956 self.assertIs(type(unpickled), range)
957 self.assertEqual(unpickled, range(1, 7))
958 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6])
959 # reduce
960 pickled = b'\x80\x02c__builtin__\nreduce\n.'
961 self.assertIs(self.loads(pickled), functools.reduce)
962 # whichdb.whichdb
963 pickled = b'\x80\x02cwhichdb\nwhichdb\n.'
964 self.assertIs(self.loads(pickled), dbm.whichdb)
965 # Exception(), StandardError()
966 for name in (b'Exception', b'StandardError'):
967 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.')
968 unpickled = self.loads(pickled)
969 self.assertIs(type(unpickled), Exception)
970 self.assertEqual(str(unpickled), 'ugh')
971 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2})
972 for name in (b'UserDict', b'IterableUserDict'):
973 pickled = (b'\x80\x02(cUserDict\n' + name +
974 b'\no}U\x04data}K\x01K\x02ssb.')
975 unpickled = self.loads(pickled)
976 self.assertIs(type(unpickled), collections.UserDict)
977 self.assertEqual(unpickled, collections.UserDict({1: 2}))
978
979
980class AbstractPickleTests(unittest.TestCase):
981 # Subclass must define self.dumps, self.loads.
982
983 optimized = False
984
985 _testdata = AbstractUnpickleTests._testdata
986
987 def setUp(self):
988 pass
989
990 assert_is_copy = AbstractUnpickleTests.assert_is_copy
991
992 def test_misc(self):
993 # test various datatypes not tested by testdata
994 for proto in protocols:
995 x = myint(4)
996 s = self.dumps(x, proto)
997 y = self.loads(s)
998 self.assert_is_copy(x, y)
999
1000 x = (1, ())
1001 s = self.dumps(x, proto)
1002 y = self.loads(s)
1003 self.assert_is_copy(x, y)
1004
1005 x = initarg(1, x)
1006 s = self.dumps(x, proto)
1007 y = self.loads(s)
1008 self.assert_is_copy(x, y)
1009
1010 # XXX test __reduce__ protocol?
1011
1012 def test_roundtrip_equality(self):
1013 expected = self._testdata
1014 for proto in protocols:
1015 s = self.dumps(expected, proto)
1016 got = self.loads(s)
1017 self.assert_is_copy(expected, got)
1018
Tim Peters70b02d72003-02-02 17:26:40 +00001019 # There are gratuitous differences between pickles produced by
1020 # pickle and cPickle, largely because cPickle starts PUT indices at
1021 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() --
1022 # there's a comment with an exclamation point there whose meaning
1023 # is a mystery. cPickle also suppresses PUT for objects with a refcount
1024 # of 1.
1025 def dont_test_disassembly(self):
Guido van Rossum34d19282007-08-09 01:03:29 +00001026 from io import StringIO
Tim Peters70b02d72003-02-02 17:26:40 +00001027 from pickletools import dis
1028
1029 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
1030 s = self.dumps(self._testdata, proto)
1031 filelike = StringIO()
1032 dis(s, out=filelike)
1033 got = filelike.getvalue()
1034 self.assertEqual(expected, got)
Jeremy Hylton66426532001-10-15 21:38:56 +00001035
1036 def test_recursive_list(self):
1037 l = []
1038 l.append(l)
Tim Peters70b02d72003-02-02 17:26:40 +00001039 for proto in protocols:
1040 s = self.dumps(l, proto)
1041 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001042 self.assertIsInstance(x, list)
Armin Rigo2b3eb402003-10-28 12:05:48 +00001043 self.assertEqual(len(x), 1)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001044 self.assertTrue(x is x[0])
Jeremy Hylton66426532001-10-15 21:38:56 +00001045
Collin Winter8ca69de2009-05-26 16:53:41 +00001046 def test_recursive_tuple(self):
1047 t = ([],)
1048 t[0].append(t)
1049 for proto in protocols:
1050 s = self.dumps(t, proto)
1051 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001052 self.assertIsInstance(x, tuple)
Collin Winter8ca69de2009-05-26 16:53:41 +00001053 self.assertEqual(len(x), 1)
1054 self.assertEqual(len(x[0]), 1)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001055 self.assertTrue(x is x[0][0])
Collin Winter8ca69de2009-05-26 16:53:41 +00001056
Jeremy Hylton66426532001-10-15 21:38:56 +00001057 def test_recursive_dict(self):
1058 d = {}
1059 d[1] = d
Tim Peters70b02d72003-02-02 17:26:40 +00001060 for proto in protocols:
1061 s = self.dumps(d, proto)
1062 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001063 self.assertIsInstance(x, dict)
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001064 self.assertEqual(list(x.keys()), [1])
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001065 self.assertTrue(x[1] is x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001066
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001067 def test_recursive_set(self):
1068 h = H()
1069 y = set({h})
1070 h.attr = y
1071 for proto in protocols:
1072 s = self.dumps(y, proto)
1073 x = self.loads(s)
1074 self.assertIsInstance(x, set)
1075 self.assertIs(list(x)[0].attr, x)
1076 self.assertEqual(len(x), 1)
1077
1078 def test_recursive_frozenset(self):
1079 h = H()
1080 y = frozenset({h})
1081 h.attr = y
1082 for proto in protocols:
1083 s = self.dumps(y, proto)
1084 x = self.loads(s)
1085 self.assertIsInstance(x, frozenset)
1086 self.assertIs(list(x)[0].attr, x)
1087 self.assertEqual(len(x), 1)
1088
Jeremy Hylton66426532001-10-15 21:38:56 +00001089 def test_recursive_inst(self):
1090 i = C()
1091 i.attr = i
Tim Peters70b02d72003-02-02 17:26:40 +00001092 for proto in protocols:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001093 s = self.dumps(i, proto)
Tim Peters70b02d72003-02-02 17:26:40 +00001094 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001095 self.assertIsInstance(x, C)
Armin Rigo2b3eb402003-10-28 12:05:48 +00001096 self.assertEqual(dir(x), dir(i))
Ezio Melottiaaef3442013-03-04 15:17:56 +02001097 self.assertIs(x.attr, x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001098
1099 def test_recursive_multi(self):
1100 l = []
1101 d = {1:l}
1102 i = C()
1103 i.attr = d
1104 l.append(i)
Tim Peters70b02d72003-02-02 17:26:40 +00001105 for proto in protocols:
1106 s = self.dumps(l, proto)
1107 x = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001108 self.assertIsInstance(x, list)
Armin Rigo2b3eb402003-10-28 12:05:48 +00001109 self.assertEqual(len(x), 1)
1110 self.assertEqual(dir(x[0]), dir(i))
Guido van Rossumcc2b0162007-02-11 06:12:03 +00001111 self.assertEqual(list(x[0].attr.keys()), [1])
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001112 self.assertTrue(x[0].attr[1] is x)
Jeremy Hylton66426532001-10-15 21:38:56 +00001113
Walter Dörwald9b775532007-06-08 14:30:53 +00001114 def test_unicode(self):
Benjamin Petersond1486302008-12-27 16:58:50 +00001115 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
Victor Stinner485fb562010-04-13 11:07:24 +00001116 '<\\>', '<\\\U00012345>',
1117 # surrogates
1118 '<\udc80>']
Walter Dörwald9b775532007-06-08 14:30:53 +00001119 for proto in protocols:
1120 for u in endcases:
1121 p = self.dumps(u, proto)
1122 u2 = self.loads(p)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001123 self.assert_is_copy(u, u2)
Tim Peterse089c682001-04-10 03:41:41 +00001124
Alexandre Vassalotti554d8782008-12-27 07:32:41 +00001125 def test_unicode_high_plane(self):
1126 t = '\U00012345'
1127 for proto in protocols:
1128 p = self.dumps(t, proto)
1129 t2 = self.loads(p)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001130 self.assert_is_copy(t, t2)
Alexandre Vassalotti554d8782008-12-27 07:32:41 +00001131
Guido van Rossumf4169812008-03-17 22:56:06 +00001132 def test_bytes(self):
1133 for proto in protocols:
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05001134 for s in b'', b'xyz', b'xyz'*100:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001135 p = self.dumps(s, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001136 self.assert_is_copy(s, self.loads(p))
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05001137 for s in [bytes([i]) for i in range(256)]:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001138 p = self.dumps(s, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001139 self.assert_is_copy(s, self.loads(p))
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05001140 for s in [bytes([i, i]) for i in range(256)]:
Ezio Melottiaaef3442013-03-04 15:17:56 +02001141 p = self.dumps(s, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001142 self.assert_is_copy(s, self.loads(p))
Guido van Rossumf4169812008-03-17 22:56:06 +00001143
Jeremy Hylton66426532001-10-15 21:38:56 +00001144 def test_ints(self):
Tim Petersee1a53c2003-02-02 02:57:53 +00001145 for proto in protocols:
Christian Heimesa37d4c62007-12-04 23:02:19 +00001146 n = sys.maxsize
Tim Petersee1a53c2003-02-02 02:57:53 +00001147 while n:
1148 for expected in (-n, n):
1149 s = self.dumps(expected, proto)
1150 n2 = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001151 self.assert_is_copy(expected, n2)
Tim Petersee1a53c2003-02-02 02:57:53 +00001152 n = n >> 1
Tim Peters19ef62d2001-08-28 22:21:18 +00001153
Tim Petersee1a53c2003-02-02 02:57:53 +00001154 def test_long(self):
1155 for proto in protocols:
Tim Petersbf2674b2003-02-02 07:51:32 +00001156 # 256 bytes is where LONG4 begins.
Tim Petersee1a53c2003-02-02 02:57:53 +00001157 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257:
Guido van Rossume2a383d2007-01-15 16:59:06 +00001158 nbase = 1 << nbits
Tim Petersee1a53c2003-02-02 02:57:53 +00001159 for npos in nbase-1, nbase, nbase+1:
1160 for n in npos, -npos:
1161 pickle = self.dumps(n, proto)
1162 got = self.loads(pickle)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001163 self.assert_is_copy(n, got)
Tim Petersee1a53c2003-02-02 02:57:53 +00001164 # Try a monster. This is quadratic-time in protos 0 & 1, so don't
1165 # bother with those.
Guido van Rossume2a383d2007-01-15 16:59:06 +00001166 nbase = int("deadbeeffeedface", 16)
Tim Petersee1a53c2003-02-02 02:57:53 +00001167 nbase += nbase << 1000000
1168 for n in nbase, -nbase:
Tim Petersee1a53c2003-02-02 02:57:53 +00001169 p = self.dumps(n, 2)
Tim Petersee1a53c2003-02-02 02:57:53 +00001170 got = self.loads(p)
Antoine Pitroud5df1942013-11-23 21:20:49 +01001171 # assert_is_copy is very expensive here as it precomputes
1172 # a failure message by computing the repr() of n and got,
1173 # we just do the check ourselves.
1174 self.assertIs(type(got), int)
1175 self.assertEqual(n, got)
Tim Petersee1a53c2003-02-02 02:57:53 +00001176
Mark Dickinsoncddcf442009-01-24 21:46:33 +00001177 def test_float(self):
1178 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
1179 3.14, 263.44582062374053, 6.022e23, 1e30]
1180 test_values = test_values + [-x for x in test_values]
1181 for proto in protocols:
1182 for value in test_values:
1183 pickle = self.dumps(value, proto)
1184 got = self.loads(pickle)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001185 self.assert_is_copy(value, got)
Mark Dickinsoncddcf442009-01-24 21:46:33 +00001186
Thomas Wouters477c8d52006-05-27 19:21:47 +00001187 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
1188 def test_float_format(self):
Guido van Rossumf4169812008-03-17 22:56:06 +00001189 # make sure that floats are formatted locale independent with proto 0
1190 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
Thomas Wouters477c8d52006-05-27 19:21:47 +00001191
Jeremy Hylton66426532001-10-15 21:38:56 +00001192 def test_reduce(self):
Antoine Pitrouc1764dd2013-12-28 16:57:37 +01001193 for proto in protocols:
1194 inst = AAA()
1195 dumped = self.dumps(inst, proto)
1196 loaded = self.loads(dumped)
1197 self.assertEqual(loaded, REDUCE_A)
Jeremy Hylton66426532001-10-15 21:38:56 +00001198
1199 def test_getinitargs(self):
Antoine Pitrouc1764dd2013-12-28 16:57:37 +01001200 for proto in protocols:
1201 inst = initarg(1, 2)
1202 dumped = self.dumps(inst, proto)
1203 loaded = self.loads(dumped)
1204 self.assert_is_copy(inst, loaded)
Jeremy Hylton66426532001-10-15 21:38:56 +00001205
Guido van Rossum04a86612001-12-19 16:58:54 +00001206 def test_metaclass(self):
1207 a = use_metaclass()
Tim Peters70b02d72003-02-02 17:26:40 +00001208 for proto in protocols:
1209 s = self.dumps(a, proto)
1210 b = self.loads(s)
1211 self.assertEqual(a.__class__, b.__class__)
Guido van Rossum04a86612001-12-19 16:58:54 +00001212
Antoine Pitrouffd41d92011-10-04 09:23:04 +02001213 def test_dynamic_class(self):
1214 a = create_dynamic_class("my_dynamic_class", (object,))
1215 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
1216 for proto in protocols:
1217 s = self.dumps(a, proto)
1218 b = self.loads(s)
1219 self.assertEqual(a, b)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001220 self.assertIs(type(a), type(b))
Antoine Pitrouffd41d92011-10-04 09:23:04 +02001221
Michael W. Hudson7bb466a2002-03-05 13:27:58 +00001222 def test_structseq(self):
1223 import time
Michael W. Hudson0e025302002-03-06 17:11:18 +00001224 import os
Tim Peters70b02d72003-02-02 17:26:40 +00001225
1226 t = time.localtime()
1227 for proto in protocols:
1228 s = self.dumps(t, proto)
Michael W. Hudson0e025302002-03-06 17:11:18 +00001229 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001230 self.assert_is_copy(t, u)
Tim Peters70b02d72003-02-02 17:26:40 +00001231 if hasattr(os, "stat"):
1232 t = os.stat(os.curdir)
1233 s = self.dumps(t, proto)
1234 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001235 self.assert_is_copy(t, u)
Tim Peters70b02d72003-02-02 17:26:40 +00001236 if hasattr(os, "statvfs"):
1237 t = os.statvfs(os.curdir)
1238 s = self.dumps(t, proto)
1239 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001240 self.assert_is_copy(t, u)
Michael W. Hudson7bb466a2002-03-05 13:27:58 +00001241
Łukasz Langaf3078fb2012-03-12 19:46:12 +01001242 def test_ellipsis(self):
1243 for proto in protocols:
1244 s = self.dumps(..., proto)
1245 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001246 self.assertIs(..., u)
Łukasz Langaf3078fb2012-03-12 19:46:12 +01001247
1248 def test_notimplemented(self):
1249 for proto in protocols:
1250 s = self.dumps(NotImplemented, proto)
1251 u = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001252 self.assertIs(NotImplemented, u)
Łukasz Langaf3078fb2012-03-12 19:46:12 +01001253
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -08001254 def test_singleton_types(self):
1255 # Issue #6477: Test that types of built-in singletons can be pickled.
1256 singletons = [None, ..., NotImplemented]
1257 for singleton in singletons:
1258 for proto in protocols:
1259 s = self.dumps(type(singleton), proto)
1260 u = self.loads(s)
1261 self.assertIs(type(singleton), u)
1262
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001263 # Tests for protocol 2
1264
Tim Peters4190fb82003-02-02 16:09:05 +00001265 def test_proto(self):
Tim Peters4190fb82003-02-02 16:09:05 +00001266 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001267 pickled = self.dumps(None, proto)
Tim Peters4190fb82003-02-02 16:09:05 +00001268 if proto >= 2:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001269 proto_header = pickle.PROTO + bytes([proto])
1270 self.assertTrue(pickled.startswith(proto_header))
1271 else:
1272 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
Tim Peters4190fb82003-02-02 16:09:05 +00001273
1274 oob = protocols[-1] + 1 # a future protocol
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001275 build_none = pickle.NONE + pickle.STOP
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001276 badpickle = pickle.PROTO + bytes([oob]) + build_none
Tim Peters4190fb82003-02-02 16:09:05 +00001277 try:
1278 self.loads(badpickle)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001279 except ValueError as err:
1280 self.assertIn("unsupported pickle protocol", str(err))
Tim Peters4190fb82003-02-02 16:09:05 +00001281 else:
1282 self.fail("expected bad protocol number to raise ValueError")
1283
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001284 def test_long1(self):
Guido van Rossume2a383d2007-01-15 16:59:06 +00001285 x = 12345678910111213141516178920
Tim Peters61bf2572003-02-03 21:31:22 +00001286 for proto in protocols:
1287 s = self.dumps(x, proto)
1288 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001289 self.assert_is_copy(x, y)
Tim Peters22e71712003-02-03 22:27:38 +00001290 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001291
1292 def test_long4(self):
Guido van Rossume2a383d2007-01-15 16:59:06 +00001293 x = 12345678910111213141516178920 << (256*8)
Tim Peters61bf2572003-02-03 21:31:22 +00001294 for proto in protocols:
1295 s = self.dumps(x, proto)
1296 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001297 self.assert_is_copy(x, y)
Tim Peters22e71712003-02-03 22:27:38 +00001298 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001299
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001300 def test_short_tuples(self):
Tim Peters1d63c9f2003-02-02 20:29:39 +00001301 # Map (proto, len(tuple)) to expected opcode.
1302 expected_opcode = {(0, 0): pickle.TUPLE,
1303 (0, 1): pickle.TUPLE,
1304 (0, 2): pickle.TUPLE,
1305 (0, 3): pickle.TUPLE,
1306 (0, 4): pickle.TUPLE,
1307
1308 (1, 0): pickle.EMPTY_TUPLE,
1309 (1, 1): pickle.TUPLE,
1310 (1, 2): pickle.TUPLE,
1311 (1, 3): pickle.TUPLE,
1312 (1, 4): pickle.TUPLE,
1313
1314 (2, 0): pickle.EMPTY_TUPLE,
1315 (2, 1): pickle.TUPLE1,
1316 (2, 2): pickle.TUPLE2,
1317 (2, 3): pickle.TUPLE3,
1318 (2, 4): pickle.TUPLE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001319
1320 (3, 0): pickle.EMPTY_TUPLE,
1321 (3, 1): pickle.TUPLE1,
1322 (3, 2): pickle.TUPLE2,
1323 (3, 3): pickle.TUPLE3,
1324 (3, 4): pickle.TUPLE,
Tim Peters1d63c9f2003-02-02 20:29:39 +00001325 }
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001326 a = ()
Guido van Rossum025bc2f2003-01-28 04:20:02 +00001327 b = (1,)
1328 c = (1, 2)
1329 d = (1, 2, 3)
1330 e = (1, 2, 3, 4)
Tim Peters4190fb82003-02-02 16:09:05 +00001331 for proto in protocols:
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001332 for x in a, b, c, d, e:
1333 s = self.dumps(x, proto)
1334 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001335 self.assert_is_copy(x, y)
1336 expected = expected_opcode[min(proto, 3), len(x)]
1337 self.assertTrue(opcode_in_pickle(expected, s))
Tim Peters1d63c9f2003-02-02 20:29:39 +00001338
Guido van Rossum7d97d312003-01-28 04:25:27 +00001339 def test_singletons(self):
Tim Peters61bf2572003-02-03 21:31:22 +00001340 # Map (proto, singleton) to expected opcode.
1341 expected_opcode = {(0, None): pickle.NONE,
1342 (1, None): pickle.NONE,
1343 (2, None): pickle.NONE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001344 (3, None): pickle.NONE,
Tim Peters61bf2572003-02-03 21:31:22 +00001345
1346 (0, True): pickle.INT,
1347 (1, True): pickle.INT,
1348 (2, True): pickle.NEWTRUE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001349 (3, True): pickle.NEWTRUE,
Tim Peters61bf2572003-02-03 21:31:22 +00001350
1351 (0, False): pickle.INT,
1352 (1, False): pickle.INT,
1353 (2, False): pickle.NEWFALSE,
Guido van Rossumf4169812008-03-17 22:56:06 +00001354 (3, False): pickle.NEWFALSE,
Tim Peters61bf2572003-02-03 21:31:22 +00001355 }
Tim Peters4190fb82003-02-02 16:09:05 +00001356 for proto in protocols:
Guido van Rossum7d97d312003-01-28 04:25:27 +00001357 for x in None, False, True:
1358 s = self.dumps(x, proto)
1359 y = self.loads(s)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001360 self.assertTrue(x is y, (proto, x, s, y))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001361 expected = expected_opcode[min(proto, 3), x]
1362 self.assertTrue(opcode_in_pickle(expected, s))
Tim Peters3c67d792003-02-02 17:59:11 +00001363
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001364 def test_newobj_tuple(self):
Guido van Rossum3d8c01b2003-01-28 19:48:18 +00001365 x = MyTuple([1, 2, 3])
1366 x.foo = 42
1367 x.bar = "hello"
Tim Peters894453a2003-02-03 22:32:18 +00001368 for proto in protocols:
1369 s = self.dumps(x, proto)
1370 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001371 self.assert_is_copy(x, y)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001372
1373 def test_newobj_list(self):
Guido van Rossum3d8c01b2003-01-28 19:48:18 +00001374 x = MyList([1, 2, 3])
1375 x.foo = 42
1376 x.bar = "hello"
Tim Peters894453a2003-02-03 22:32:18 +00001377 for proto in protocols:
1378 s = self.dumps(x, proto)
1379 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001380 self.assert_is_copy(x, y)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001381
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001382 def test_newobj_generic(self):
Tim Peters5013bd92003-02-03 22:28:41 +00001383 for proto in protocols:
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001384 for C in myclasses:
1385 B = C.__base__
1386 x = C(C.sample)
1387 x.foo = 42
1388 s = self.dumps(x, proto)
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001389 y = self.loads(s)
1390 detail = (proto, C, B, x, y, type(y))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001391 self.assert_is_copy(x, y) # XXX revisit
Guido van Rossum5d9113d2003-01-29 17:58:45 +00001392 self.assertEqual(B(x), B(y), detail)
1393 self.assertEqual(x.__dict__, y.__dict__, detail)
1394
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01001395 def test_newobj_proxies(self):
1396 # NEWOBJ should use the __class__ rather than the raw type
1397 classes = myclasses[:]
1398 # Cannot create weakproxies to these classes
1399 for c in (MyInt, MyTuple):
1400 classes.remove(c)
1401 for proto in protocols:
1402 for C in classes:
1403 B = C.__base__
1404 x = C(C.sample)
1405 x.foo = 42
1406 p = weakref.proxy(x)
1407 s = self.dumps(p, proto)
1408 y = self.loads(s)
1409 self.assertEqual(type(y), type(x)) # rather than type(p)
1410 detail = (proto, C, B, x, y, type(y))
1411 self.assertEqual(B(x), B(y), detail)
1412 self.assertEqual(x.__dict__, y.__dict__, detail)
1413
Benjamin Peterson80f78a32015-07-02 16:18:38 -05001414 def test_newobj_not_class(self):
1415 # Issue 24552
1416 global SimpleNewObj
1417 save = SimpleNewObj
Benjamin Petersond3a2a952015-07-02 16:58:22 -05001418 o = SimpleNewObj.__new__(SimpleNewObj)
Benjamin Peterson80f78a32015-07-02 16:18:38 -05001419 b = self.dumps(o, 4)
1420 try:
1421 SimpleNewObj = 42
1422 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b)
1423 finally:
1424 SimpleNewObj = save
1425
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00001426 # Register a type with copyreg, with extension code extcode. Pickle
Tim Peters22e71712003-02-03 22:27:38 +00001427 # an object of that type. Check that the resulting pickle uses opcode
1428 # (EXT[124]) under proto 2, and not in proto 1.
Tim Peters3e667d52003-02-04 21:47:44 +00001429
Tim Peters22e71712003-02-03 22:27:38 +00001430 def produce_global_ext(self, extcode, opcode):
Tim Peters3e667d52003-02-04 21:47:44 +00001431 e = ExtensionSaver(extcode)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001432 try:
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00001433 copyreg.add_extension(__name__, "MyList", extcode)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001434 x = MyList([1, 2, 3])
1435 x.foo = 42
1436 x.bar = "hello"
1437
Tim Peters22e71712003-02-03 22:27:38 +00001438 # Dump using protocol 1 for comparison.
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001439 s1 = self.dumps(x, 1)
Ezio Melottib58e0bd2010-01-23 15:40:09 +00001440 self.assertIn(__name__.encode("utf-8"), s1)
1441 self.assertIn(b"MyList", s1)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001442 self.assertFalse(opcode_in_pickle(opcode, s1))
Tim Peters3e667d52003-02-04 21:47:44 +00001443
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001444 y = self.loads(s1)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001445 self.assert_is_copy(x, y)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001446
Tim Peters22e71712003-02-03 22:27:38 +00001447 # Dump using protocol 2 for test.
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001448 s2 = self.dumps(x, 2)
Ezio Melottib58e0bd2010-01-23 15:40:09 +00001449 self.assertNotIn(__name__.encode("utf-8"), s2)
1450 self.assertNotIn(b"MyList", s2)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001451 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
Tim Peters3e667d52003-02-04 21:47:44 +00001452
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001453 y = self.loads(s2)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001454 self.assert_is_copy(x, y)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001455 finally:
Tim Peters3e667d52003-02-04 21:47:44 +00001456 e.restore()
Tim Peters22e71712003-02-03 22:27:38 +00001457
1458 def test_global_ext1(self):
Tim Peters3e667d52003-02-04 21:47:44 +00001459 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code
1460 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001461
1462 def test_global_ext2(self):
Tim Peters3e667d52003-02-04 21:47:44 +00001463 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code
1464 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code
1465 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001466
1467 def test_global_ext4(self):
Tim Peters3e667d52003-02-04 21:47:44 +00001468 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code
1469 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code
1470 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness
1471
Tim Peters8d2613a2003-02-11 16:40:16 +00001472 def test_list_chunking(self):
1473 n = 10 # too small to chunk
Guido van Rossum805365e2007-05-07 22:24:25 +00001474 x = list(range(n))
Tim Peters8d2613a2003-02-11 16:40:16 +00001475 for proto in protocols:
1476 s = self.dumps(x, proto)
1477 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001478 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00001479 num_appends = count_opcode(pickle.APPENDS, s)
1480 self.assertEqual(num_appends, proto > 0)
1481
1482 n = 2500 # expect at least two chunks when proto > 0
Guido van Rossum805365e2007-05-07 22:24:25 +00001483 x = list(range(n))
Tim Peters8d2613a2003-02-11 16:40:16 +00001484 for proto in protocols:
1485 s = self.dumps(x, proto)
1486 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001487 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00001488 num_appends = count_opcode(pickle.APPENDS, s)
1489 if proto == 0:
1490 self.assertEqual(num_appends, 0)
1491 else:
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001492 self.assertTrue(num_appends >= 2)
Tim Peters8d2613a2003-02-11 16:40:16 +00001493
1494 def test_dict_chunking(self):
1495 n = 10 # too small to chunk
1496 x = dict.fromkeys(range(n))
1497 for proto in protocols:
1498 s = self.dumps(x, proto)
Ezio Melottie9615932010-01-24 19:26:24 +00001499 self.assertIsInstance(s, bytes_types)
Tim Peters8d2613a2003-02-11 16:40:16 +00001500 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001501 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00001502 num_setitems = count_opcode(pickle.SETITEMS, s)
1503 self.assertEqual(num_setitems, proto > 0)
1504
1505 n = 2500 # expect at least two chunks when proto > 0
1506 x = dict.fromkeys(range(n))
1507 for proto in protocols:
1508 s = self.dumps(x, proto)
1509 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001510 self.assert_is_copy(x, y)
Tim Peters8d2613a2003-02-11 16:40:16 +00001511 num_setitems = count_opcode(pickle.SETITEMS, s)
1512 if proto == 0:
1513 self.assertEqual(num_setitems, 0)
1514 else:
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001515 self.assertTrue(num_setitems >= 2)
Guido van Rossum0322d0f2003-01-29 06:12:46 +00001516
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001517 def test_set_chunking(self):
1518 n = 10 # too small to chunk
1519 x = set(range(n))
1520 for proto in protocols:
1521 s = self.dumps(x, proto)
1522 y = self.loads(s)
1523 self.assert_is_copy(x, y)
1524 num_additems = count_opcode(pickle.ADDITEMS, s)
1525 if proto < 4:
1526 self.assertEqual(num_additems, 0)
1527 else:
1528 self.assertEqual(num_additems, 1)
1529
1530 n = 2500 # expect at least two chunks when proto >= 4
1531 x = set(range(n))
1532 for proto in protocols:
1533 s = self.dumps(x, proto)
1534 y = self.loads(s)
1535 self.assert_is_copy(x, y)
1536 num_additems = count_opcode(pickle.ADDITEMS, s)
1537 if proto < 4:
1538 self.assertEqual(num_additems, 0)
1539 else:
1540 self.assertGreaterEqual(num_additems, 2)
1541
Tim Peterse9ef2032003-02-13 18:42:00 +00001542 def test_simple_newobj(self):
1543 x = object.__new__(SimpleNewObj) # avoid __init__
1544 x.abc = 666
1545 for proto in protocols:
1546 s = self.dumps(x, proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001547 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
1548 2 <= proto < 4)
1549 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
1550 proto >= 4)
Tim Peterse9ef2032003-02-13 18:42:00 +00001551 y = self.loads(s) # will raise TypeError if __init__ called
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001552 self.assert_is_copy(x, y)
Tim Peterse9ef2032003-02-13 18:42:00 +00001553
Tim Peters42f08ac2003-02-11 22:43:24 +00001554 def test_newobj_list_slots(self):
1555 x = SlotList([1, 2, 3])
1556 x.foo = 42
1557 x.bar = "hello"
1558 s = self.dumps(x, 2)
1559 y = self.loads(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001560 self.assert_is_copy(x, y)
Tim Peters42f08ac2003-02-11 22:43:24 +00001561
Guido van Rossum2a30b212003-02-18 22:41:24 +00001562 def test_reduce_overrides_default_reduce_ex(self):
Collin Winter771d8342009-04-16 03:18:06 +00001563 for proto in protocols:
Guido van Rossum2a30b212003-02-18 22:41:24 +00001564 x = REX_one()
1565 self.assertEqual(x._reduce_called, 0)
1566 s = self.dumps(x, proto)
1567 self.assertEqual(x._reduce_called, 1)
1568 y = self.loads(s)
1569 self.assertEqual(y._reduce_called, 0)
1570
1571 def test_reduce_ex_called(self):
Collin Winter771d8342009-04-16 03:18:06 +00001572 for proto in protocols:
Guido van Rossum2a30b212003-02-18 22:41:24 +00001573 x = REX_two()
1574 self.assertEqual(x._proto, None)
1575 s = self.dumps(x, proto)
1576 self.assertEqual(x._proto, proto)
1577 y = self.loads(s)
1578 self.assertEqual(y._proto, None)
1579
1580 def test_reduce_ex_overrides_reduce(self):
Collin Winter771d8342009-04-16 03:18:06 +00001581 for proto in protocols:
Guido van Rossum2a30b212003-02-18 22:41:24 +00001582 x = REX_three()
1583 self.assertEqual(x._proto, None)
1584 s = self.dumps(x, proto)
1585 self.assertEqual(x._proto, proto)
1586 y = self.loads(s)
1587 self.assertEqual(y._proto, None)
1588
Guido van Rossumd8faa362007-04-27 19:54:29 +00001589 def test_reduce_ex_calls_base(self):
Collin Winter771d8342009-04-16 03:18:06 +00001590 for proto in protocols:
Guido van Rossumd8faa362007-04-27 19:54:29 +00001591 x = REX_four()
1592 self.assertEqual(x._proto, None)
1593 s = self.dumps(x, proto)
1594 self.assertEqual(x._proto, proto)
1595 y = self.loads(s)
1596 self.assertEqual(y._proto, proto)
1597
1598 def test_reduce_calls_base(self):
Collin Winter771d8342009-04-16 03:18:06 +00001599 for proto in protocols:
Guido van Rossumd8faa362007-04-27 19:54:29 +00001600 x = REX_five()
1601 self.assertEqual(x._reduce_called, 0)
1602 s = self.dumps(x, proto)
1603 self.assertEqual(x._reduce_called, 1)
1604 y = self.loads(s)
1605 self.assertEqual(y._reduce_called, 1)
1606
Brett Cannon31f59292011-02-21 19:29:56 +00001607 @no_tracing
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00001608 def test_bad_getattr(self):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001609 # Issue #3514: crash when there is an infinite loop in __getattr__
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00001610 x = BadGetattr()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001611 for proto in protocols:
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00001612 self.assertRaises(RuntimeError, self.dumps, x, proto)
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00001613
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00001614 def test_reduce_bad_iterator(self):
1615 # Issue4176: crash when 4th and 5th items of __reduce__()
1616 # are not iterators
1617 class C(object):
1618 def __reduce__(self):
1619 # 4th item is not an iterator
1620 return list, (), None, [], None
1621 class D(object):
1622 def __reduce__(self):
1623 # 5th item is not an iterator
1624 return dict, (), None, None, []
1625
Amaury Forgeot d'Arc6285ffd2008-10-31 17:52:47 +00001626 # Protocol 0 is less strict and also accept iterables.
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00001627 for proto in protocols:
Amaury Forgeot d'Arc6285ffd2008-10-31 17:52:47 +00001628 try:
1629 self.dumps(C(), proto)
1630 except (pickle.PickleError):
1631 pass
1632 try:
1633 self.dumps(D(), proto)
1634 except (pickle.PickleError):
1635 pass
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00001636
Collin Winter771d8342009-04-16 03:18:06 +00001637 def test_many_puts_and_gets(self):
1638 # Test that internal data structures correctly deal with lots of
1639 # puts/gets.
1640 keys = ("aaa" + str(i) for i in range(100))
1641 large_dict = dict((k, [4, 5, 6]) for k in keys)
1642 obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
1643
1644 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001645 with self.subTest(proto=proto):
1646 dumped = self.dumps(obj, proto)
1647 loaded = self.loads(dumped)
1648 self.assert_is_copy(obj, loaded)
Collin Winter771d8342009-04-16 03:18:06 +00001649
Antoine Pitroua9f48a02009-05-02 21:41:14 +00001650 def test_attribute_name_interning(self):
1651 # Test that attribute names of pickled objects are interned when
1652 # unpickling.
1653 for proto in protocols:
1654 x = C()
1655 x.foo = 42
1656 x.bar = "hello"
1657 s = self.dumps(x, proto)
1658 y = self.loads(s)
1659 x_keys = sorted(x.__dict__)
1660 y_keys = sorted(y.__dict__)
1661 for x_key, y_key in zip(x_keys, y_keys):
1662 self.assertIs(x_key, y_key)
1663
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001664 def test_pickle_to_2x(self):
1665 # Pickle non-trivial data with protocol 2, expecting that it yields
1666 # the same result as Python 2.x did.
1667 # NOTE: this test is a bit too strong since we can produce different
1668 # bytecode that 2.x will still understand.
1669 dumped = self.dumps(range(5), 2)
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001670 self.assertEqual(dumped, DATA_XRANGE)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001671 dumped = self.dumps(set([3]), 2)
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03001672 self.assertEqual(dumped, DATA_SET2)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001673
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001674 def test_large_pickles(self):
1675 # Test the correctness of internal buffering routines when handling
1676 # large data.
1677 for proto in protocols:
Antoine Pitrou04248a82010-10-12 20:51:21 +00001678 data = (1, min, b'xy' * (30 * 1024), len)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001679 dumped = self.dumps(data, proto)
1680 loaded = self.loads(dumped)
Antoine Pitrou04248a82010-10-12 20:51:21 +00001681 self.assertEqual(len(loaded), len(data))
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001682 self.assertEqual(loaded, data)
1683
Antoine Pitrou3c7e9282011-08-13 20:15:19 +02001684 def test_int_pickling_efficiency(self):
1685 # Test compacity of int representation (see issue #12744)
1686 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001687 with self.subTest(proto=proto):
1688 pickles = [self.dumps(2**n, proto) for n in range(70)]
1689 sizes = list(map(len, pickles))
1690 # the size function is monotonic
1691 self.assertEqual(sorted(sizes), sizes)
1692 if proto >= 2:
1693 for p in pickles:
1694 self.assertFalse(opcode_in_pickle(pickle.LONG, p))
Antoine Pitrou3c7e9282011-08-13 20:15:19 +02001695
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07001696 def _check_pickling_with_opcode(self, obj, opcode, proto):
1697 pickled = self.dumps(obj, proto)
1698 self.assertTrue(opcode_in_pickle(opcode, pickled))
1699 unpickled = self.loads(pickled)
1700 self.assertEqual(obj, unpickled)
1701
1702 def test_appends_on_non_lists(self):
1703 # Issue #17720
1704 obj = REX_six([1, 2, 3])
1705 for proto in protocols:
1706 if proto == 0:
1707 self._check_pickling_with_opcode(obj, pickle.APPEND, proto)
1708 else:
1709 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto)
1710
1711 def test_setitems_on_non_dicts(self):
1712 obj = REX_seven({1: -1, 2: -2, 3: -3})
1713 for proto in protocols:
1714 if proto == 0:
1715 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto)
1716 else:
1717 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
1718
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001719 # Exercise framing (proto >= 4) for significant workloads
1720
1721 FRAME_SIZE_TARGET = 64 * 1024
1722
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01001723 def check_frame_opcodes(self, pickled):
1724 """
1725 Check the arguments of FRAME opcodes in a protocol 4+ pickle.
1726 """
1727 frame_opcode_size = 9
1728 last_arg = last_pos = None
1729 for op, arg, pos in pickletools.genops(pickled):
1730 if op.name != 'FRAME':
1731 continue
1732 if last_pos is not None:
1733 # The previous frame's size should be equal to the number
1734 # of bytes up to the current frame.
1735 frame_size = pos - last_pos - frame_opcode_size
1736 self.assertEqual(frame_size, last_arg)
1737 last_arg, last_pos = arg, pos
1738 # The last frame's size should be equal to the number of bytes up
1739 # to the pickle's end.
1740 frame_size = len(pickled) - last_pos - frame_opcode_size
1741 self.assertEqual(frame_size, last_arg)
1742
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001743 def test_framing_many_objects(self):
1744 obj = list(range(10**5))
1745 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1746 with self.subTest(proto=proto):
1747 pickled = self.dumps(obj, proto)
1748 unpickled = self.loads(pickled)
1749 self.assertEqual(obj, unpickled)
Antoine Pitrou3ab9cfc2013-11-24 14:33:37 +01001750 bytes_per_frame = (len(pickled) /
1751 count_opcode(pickle.FRAME, pickled))
1752 self.assertGreater(bytes_per_frame,
1753 self.FRAME_SIZE_TARGET / 2)
1754 self.assertLessEqual(bytes_per_frame,
1755 self.FRAME_SIZE_TARGET * 1)
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01001756 self.check_frame_opcodes(pickled)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001757
1758 def test_framing_large_objects(self):
1759 N = 1024 * 1024
1760 obj = [b'x' * N, b'y' * N, b'z' * N]
1761 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1762 with self.subTest(proto=proto):
1763 pickled = self.dumps(obj, proto)
1764 unpickled = self.loads(pickled)
1765 self.assertEqual(obj, unpickled)
Antoine Pitrou3ab9cfc2013-11-24 14:33:37 +01001766 n_frames = count_opcode(pickle.FRAME, pickled)
Alexandre Vassalotti28d271e2013-12-01 16:27:46 -08001767 self.assertGreaterEqual(n_frames, len(obj))
Antoine Pitrou6e8bc502013-12-03 09:51:40 +01001768 self.check_frame_opcodes(pickled)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001769
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08001770 def test_optional_frames(self):
1771 if pickle.HIGHEST_PROTOCOL < 4:
1772 return
1773
1774 def remove_frames(pickled, keep_frame=None):
1775 """Remove frame opcodes from the given pickle."""
1776 frame_starts = []
1777 # 1 byte for the opcode and 8 for the argument
1778 frame_opcode_size = 9
1779 for opcode, _, pos in pickletools.genops(pickled):
1780 if opcode.name == 'FRAME':
1781 frame_starts.append(pos)
1782
1783 newpickle = bytearray()
1784 last_frame_end = 0
1785 for i, pos in enumerate(frame_starts):
1786 if keep_frame and keep_frame(i):
1787 continue
1788 newpickle += pickled[last_frame_end:pos]
1789 last_frame_end = pos + frame_opcode_size
1790 newpickle += pickled[last_frame_end:]
1791 return newpickle
1792
Alexandre Vassalotti5e411b72013-11-23 20:58:24 -08001793 frame_size = self.FRAME_SIZE_TARGET
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08001794 num_frames = 20
Alexandre Vassalotti5e411b72013-11-23 20:58:24 -08001795 obj = [bytes([i]) * frame_size for i in range(num_frames)]
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08001796
1797 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1798 pickled = self.dumps(obj, proto)
1799
1800 frameless_pickle = remove_frames(pickled)
1801 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
1802 self.assertEqual(obj, self.loads(frameless_pickle))
1803
Alexandre Vassalotti5e411b72013-11-23 20:58:24 -08001804 some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08001805 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
1806 count_opcode(pickle.FRAME, pickled))
1807 self.assertEqual(obj, self.loads(some_frames_pickle))
1808
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001809 def test_nested_names(self):
1810 global Nested
1811 class Nested:
1812 class A:
1813 class B:
1814 class C:
1815 pass
1816
1817 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1818 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
1819 with self.subTest(proto=proto, obj=obj):
1820 unpickled = self.loads(self.dumps(obj, proto))
1821 self.assertIs(obj, unpickled)
1822
1823 def test_py_methods(self):
1824 global PyMethodsTest
1825 class PyMethodsTest:
1826 @staticmethod
1827 def cheese():
1828 return "cheese"
1829 @classmethod
1830 def wine(cls):
1831 assert cls is PyMethodsTest
1832 return "wine"
1833 def biscuits(self):
1834 assert isinstance(self, PyMethodsTest)
1835 return "biscuits"
1836 class Nested:
1837 "Nested class"
1838 @staticmethod
1839 def ketchup():
1840 return "ketchup"
1841 @classmethod
1842 def maple(cls):
1843 assert cls is PyMethodsTest.Nested
1844 return "maple"
1845 def pie(self):
1846 assert isinstance(self, PyMethodsTest.Nested)
1847 return "pie"
1848
1849 py_methods = (
1850 PyMethodsTest.cheese,
1851 PyMethodsTest.wine,
1852 PyMethodsTest().biscuits,
1853 PyMethodsTest.Nested.ketchup,
1854 PyMethodsTest.Nested.maple,
1855 PyMethodsTest.Nested().pie
1856 )
1857 py_unbound_methods = (
1858 (PyMethodsTest.biscuits, PyMethodsTest),
1859 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
1860 )
1861 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1862 for method in py_methods:
1863 with self.subTest(proto=proto, method=method):
1864 unpickled = self.loads(self.dumps(method, proto))
1865 self.assertEqual(method(), unpickled())
1866 for method, cls in py_unbound_methods:
1867 obj = cls()
1868 with self.subTest(proto=proto, method=method):
1869 unpickled = self.loads(self.dumps(method, proto))
1870 self.assertEqual(method(obj), unpickled(obj))
1871
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001872 def test_c_methods(self):
1873 global Subclass
1874 class Subclass(tuple):
1875 class Nested(str):
1876 pass
1877
1878 c_methods = (
1879 # bound built-in method
1880 ("abcd".index, ("c",)),
1881 # unbound built-in method
1882 (str.index, ("abcd", "c")),
1883 # bound "slot" method
1884 ([1, 2, 3].__len__, ()),
1885 # unbound "slot" method
1886 (list.__len__, ([1, 2, 3],)),
1887 # bound "coexist" method
1888 ({1, 2}.__contains__, (2,)),
1889 # unbound "coexist" method
1890 (set.__contains__, ({1, 2}, 2)),
1891 # built-in class method
1892 (dict.fromkeys, (("a", 1), ("b", 2))),
1893 # built-in static method
1894 (bytearray.maketrans, (b"abc", b"xyz")),
1895 # subclass methods
1896 (Subclass([1,2,2]).count, (2,)),
1897 (Subclass.count, (Subclass([1,2,2]), 2)),
1898 (Subclass.Nested("sweet").count, ("e",)),
1899 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
1900 )
1901 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1902 for method, args in c_methods:
1903 with self.subTest(proto=proto, method=method):
1904 unpickled = self.loads(self.dumps(method, proto))
1905 self.assertEqual(method(*args), unpickled(*args))
1906
Serhiy Storchakabfe18242015-03-31 13:12:37 +03001907 def test_compat_pickle(self):
1908 tests = [
1909 (range(1, 7), '__builtin__', 'xrange'),
1910 (map(int, '123'), 'itertools', 'imap'),
1911 (functools.reduce, '__builtin__', 'reduce'),
1912 (dbm.whichdb, 'whichdb', 'whichdb'),
1913 (Exception(), 'exceptions', 'Exception'),
1914 (collections.UserDict(), 'UserDict', 'IterableUserDict'),
1915 (collections.UserList(), 'UserList', 'UserList'),
1916 (collections.defaultdict(), 'collections', 'defaultdict'),
1917 ]
1918 for val, mod, name in tests:
1919 for proto in range(3):
1920 with self.subTest(type=type(val), proto=proto):
1921 pickled = self.dumps(val, proto)
1922 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled)
1923 self.assertIs(type(self.loads(pickled)), type(val))
1924
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001925
1926class BigmemPickleTests(unittest.TestCase):
1927
1928 # Binary protocols can serialize longs of up to 2GB-1
1929
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02001930 @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001931 def test_huge_long_32b(self, size):
1932 data = 1 << (8 * size)
1933 try:
1934 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001935 if proto < 2:
1936 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001937 with self.subTest(proto=proto):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001938 with self.assertRaises((ValueError, OverflowError)):
1939 self.dumps(data, protocol=proto)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001940 finally:
1941 data = None
1942
1943 # Protocol 3 can serialize up to 4GB-1 as a bytes object
1944 # (older protocols don't have a dedicated opcode for bytes and are
1945 # too inefficient)
1946
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02001947 @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001948 def test_huge_bytes_32b(self, size):
1949 data = b"abcd" * (size // 4)
1950 try:
1951 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001952 if proto < 3:
1953 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001954 with self.subTest(proto=proto):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001955 try:
1956 pickled = self.dumps(data, protocol=proto)
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001957 header = (pickle.BINBYTES +
1958 struct.pack("<I", len(data)))
1959 data_start = pickled.index(data)
1960 self.assertEqual(
1961 header,
1962 pickled[data_start-len(header):data_start])
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001963 finally:
1964 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001965 finally:
1966 data = None
1967
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02001968 @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001969 def test_huge_bytes_64b(self, size):
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001970 data = b"acbd" * (size // 4)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001971 try:
1972 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001973 if proto < 3:
1974 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001975 with self.subTest(proto=proto):
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001976 if proto == 3:
1977 # Protocol 3 does not support large bytes objects.
1978 # Verify that we do not crash when processing one.
1979 with self.assertRaises((ValueError, OverflowError)):
1980 self.dumps(data, protocol=proto)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001981 continue
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08001982 try:
1983 pickled = self.dumps(data, protocol=proto)
1984 header = (pickle.BINBYTES8 +
1985 struct.pack("<Q", len(data)))
1986 data_start = pickled.index(data)
1987 self.assertEqual(
1988 header,
1989 pickled[data_start-len(header):data_start])
1990 finally:
1991 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001992 finally:
1993 data = None
1994
1995 # All protocols use 1-byte per printable ASCII character; we add another
1996 # byte because the encoded form has to be copied into the internal buffer.
1997
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02001998 @bigmemtest(size=_2G, memuse=8, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001999 def test_huge_str_32b(self, size):
2000 data = "abcd" * (size // 4)
2001 try:
2002 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002003 if proto == 0:
2004 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002005 with self.subTest(proto=proto):
2006 try:
2007 pickled = self.dumps(data, protocol=proto)
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002008 header = (pickle.BINUNICODE +
2009 struct.pack("<I", len(data)))
2010 data_start = pickled.index(b'abcd')
2011 self.assertEqual(
2012 header,
2013 pickled[data_start-len(header):data_start])
2014 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
2015 pickled.index(b"abcd")), len(data))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002016 finally:
2017 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002018 finally:
2019 data = None
2020
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002021 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
2022 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
2023 # unicode strings however.
Antoine Pitroue897e952011-08-30 23:39:34 +02002024
Serhiy Storchaka4847e4e2014-01-10 13:37:54 +02002025 @bigmemtest(size=_4G, memuse=8, dry_run=False)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002026 def test_huge_str_64b(self, size):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002027 data = "abcd" * (size // 4)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002028 try:
2029 for proto in protocols:
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002030 if proto == 0:
2031 continue
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002032 with self.subTest(proto=proto):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002033 if proto < 4:
2034 with self.assertRaises((ValueError, OverflowError)):
2035 self.dumps(data, protocol=proto)
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002036 continue
2037 try:
2038 pickled = self.dumps(data, protocol=proto)
2039 header = (pickle.BINUNICODE8 +
2040 struct.pack("<Q", len(data)))
2041 data_start = pickled.index(b'abcd')
2042 self.assertEqual(
2043 header,
2044 pickled[data_start-len(header):data_start])
2045 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
2046 pickled.index(b"abcd")), len(data))
2047 finally:
2048 pickled = None
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002049 finally:
2050 data = None
2051
Antoine Pitrou3c7e9282011-08-13 20:15:19 +02002052
Guido van Rossum2a30b212003-02-18 22:41:24 +00002053# Test classes for reduce_ex
2054
2055class REX_one(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002056 """No __reduce_ex__ here, but inheriting it from object"""
Guido van Rossum2a30b212003-02-18 22:41:24 +00002057 _reduce_called = 0
2058 def __reduce__(self):
2059 self._reduce_called = 1
2060 return REX_one, ()
Guido van Rossum2a30b212003-02-18 22:41:24 +00002061
2062class REX_two(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002063 """No __reduce__ here, but inheriting it from object"""
Guido van Rossum2a30b212003-02-18 22:41:24 +00002064 _proto = None
2065 def __reduce_ex__(self, proto):
2066 self._proto = proto
2067 return REX_two, ()
Guido van Rossum2a30b212003-02-18 22:41:24 +00002068
2069class REX_three(object):
2070 _proto = None
2071 def __reduce_ex__(self, proto):
2072 self._proto = proto
2073 return REX_two, ()
2074 def __reduce__(self):
Collin Winter3add4d72007-08-29 23:37:32 +00002075 raise TestFailed("This __reduce__ shouldn't be called")
Guido van Rossum2a30b212003-02-18 22:41:24 +00002076
Guido van Rossumd8faa362007-04-27 19:54:29 +00002077class REX_four(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002078 """Calling base class method should succeed"""
Guido van Rossumd8faa362007-04-27 19:54:29 +00002079 _proto = None
2080 def __reduce_ex__(self, proto):
2081 self._proto = proto
2082 return object.__reduce_ex__(self, proto)
Guido van Rossumd8faa362007-04-27 19:54:29 +00002083
2084class REX_five(object):
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002085 """This one used to fail with infinite recursion"""
Guido van Rossumd8faa362007-04-27 19:54:29 +00002086 _reduce_called = 0
2087 def __reduce__(self):
2088 self._reduce_called = 1
2089 return object.__reduce__(self)
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002090
2091class REX_six(object):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002092 """This class is used to check the 4th argument (list iterator) of
2093 the reduce protocol.
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002094 """
2095 def __init__(self, items=None):
2096 self.items = items if items is not None else []
2097 def __eq__(self, other):
2098 return type(self) is type(other) and self.items == self.items
2099 def append(self, item):
2100 self.items.append(item)
2101 def __reduce__(self):
2102 return type(self), (), None, iter(self.items), None
2103
2104class REX_seven(object):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002105 """This class is used to check the 5th argument (dict iterator) of
2106 the reduce protocol.
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07002107 """
2108 def __init__(self, table=None):
2109 self.table = table if table is not None else {}
2110 def __eq__(self, other):
2111 return type(self) is type(other) and self.table == self.table
2112 def __setitem__(self, key, value):
2113 self.table[key] = value
2114 def __reduce__(self):
2115 return type(self), (), None, None, iter(self.table.items())
2116
Guido van Rossumd8faa362007-04-27 19:54:29 +00002117
Guido van Rossum2a30b212003-02-18 22:41:24 +00002118# Test classes for newobj
Tim Peters080c88b2003-02-15 03:01:11 +00002119
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002120class MyInt(int):
2121 sample = 1
2122
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002123class MyFloat(float):
2124 sample = 1.0
2125
2126class MyComplex(complex):
2127 sample = 1.0 + 0.0j
2128
2129class MyStr(str):
2130 sample = "hello"
2131
Guido van Rossumef87d6e2007-05-02 19:09:54 +00002132class MyUnicode(str):
2133 sample = "hello \u1234"
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002134
Guido van Rossum533dbcf2003-01-28 17:55:05 +00002135class MyTuple(tuple):
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002136 sample = (1, 2, 3)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00002137
2138class MyList(list):
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002139 sample = [1, 2, 3]
2140
2141class MyDict(dict):
2142 sample = {"a": 1, "b": 2}
2143
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002144class MySet(set):
2145 sample = {"a", "b"}
2146
2147class MyFrozenSet(frozenset):
2148 sample = frozenset({"a", "b"})
2149
Mark Dickinson5c2db372009-12-05 20:28:34 +00002150myclasses = [MyInt, MyFloat,
Guido van Rossum206b9a72003-03-02 13:53:18 +00002151 MyComplex,
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002152 MyStr, MyUnicode,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002153 MyTuple, MyList, MyDict, MySet, MyFrozenSet]
Guido van Rossum5d9113d2003-01-29 17:58:45 +00002154
Guido van Rossum533dbcf2003-01-28 17:55:05 +00002155
Guido van Rossumc8d6ef52003-01-28 22:02:31 +00002156class SlotList(MyList):
2157 __slots__ = ["foo"]
2158
Tim Peterse9ef2032003-02-13 18:42:00 +00002159class SimpleNewObj(object):
2160 def __init__(self, a, b, c):
2161 # raise an error, to make sure this isn't called
2162 raise TypeError("SimpleNewObj.__init__() didn't expect to get called")
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002163 def __eq__(self, other):
2164 return self.__dict__ == other.__dict__
Tim Peterse9ef2032003-02-13 18:42:00 +00002165
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00002166class BadGetattr:
2167 def __getattr__(self, key):
2168 self.foo
2169
Collin Winter771d8342009-04-16 03:18:06 +00002170
Jeremy Hylton66426532001-10-15 21:38:56 +00002171class AbstractPickleModuleTests(unittest.TestCase):
2172
2173 def test_dump_closed_file(self):
Guido van Rossum3b0a3292002-08-09 16:38:32 +00002174 import os
Walter Dörwald11b41f62007-06-20 12:46:31 +00002175 f = open(TESTFN, "wb")
Guido van Rossum3b0a3292002-08-09 16:38:32 +00002176 try:
2177 f.close()
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002178 self.assertRaises(ValueError, pickle.dump, 123, f)
Guido van Rossum3b0a3292002-08-09 16:38:32 +00002179 finally:
2180 os.remove(TESTFN)
Jeremy Hylton66426532001-10-15 21:38:56 +00002181
2182 def test_load_closed_file(self):
Guido van Rossum3b0a3292002-08-09 16:38:32 +00002183 import os
Walter Dörwald11b41f62007-06-20 12:46:31 +00002184 f = open(TESTFN, "wb")
Guido van Rossum3b0a3292002-08-09 16:38:32 +00002185 try:
2186 f.close()
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002187 self.assertRaises(ValueError, pickle.dump, 123, f)
Guido van Rossum3b0a3292002-08-09 16:38:32 +00002188 finally:
2189 os.remove(TESTFN)
Jeremy Hylton4c8be852002-11-13 22:10:47 +00002190
Collin Winter771d8342009-04-16 03:18:06 +00002191 def test_load_from_and_dump_to_file(self):
2192 stream = io.BytesIO()
2193 data = [123, {}, 124]
2194 pickle.dump(data, stream)
2195 stream.seek(0)
2196 unpickled = pickle.load(stream)
2197 self.assertEqual(unpickled, data)
2198
Tim Petersc0c93702003-02-13 19:30:57 +00002199 def test_highest_protocol(self):
2200 # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002201 self.assertEqual(pickle.HIGHEST_PROTOCOL, 4)
Tim Petersc0c93702003-02-13 19:30:57 +00002202
Martin v. Löwis544f1192004-07-27 05:22:33 +00002203 def test_callapi(self):
Collin Winter771d8342009-04-16 03:18:06 +00002204 f = io.BytesIO()
Martin v. Löwis544f1192004-07-27 05:22:33 +00002205 # With and without keyword arguments
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002206 pickle.dump(123, f, -1)
2207 pickle.dump(123, file=f, protocol=-1)
2208 pickle.dumps(123, -1)
2209 pickle.dumps(123, protocol=-1)
2210 pickle.Pickler(f, -1)
2211 pickle.Pickler(f, protocol=-1)
Tim Petersc0c93702003-02-13 19:30:57 +00002212
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00002213 def test_bad_init(self):
2214 # Test issue3664 (pickle can segfault from a badly initialized Pickler).
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00002215 # Override initialization without calling __init__() of the superclass.
2216 class BadPickler(pickle.Pickler):
2217 def __init__(self): pass
2218
2219 class BadUnpickler(pickle.Unpickler):
2220 def __init__(self): pass
2221
2222 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
2223 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load)
2224
Amaury Forgeot d'Arc3e4e72f2008-11-11 20:05:06 +00002225 def test_bad_input(self):
2226 # Test issue4298
2227 s = bytes([0x58, 0, 0, 0, 0x54])
2228 self.assertRaises(EOFError, pickle.loads, s)
2229
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00002230
Jeremy Hylton4c8be852002-11-13 22:10:47 +00002231class AbstractPersistentPicklerTests(unittest.TestCase):
2232
2233 # This class defines persistent_id() and persistent_load()
2234 # functions that should be used by the pickler. All even integers
2235 # are pickled using persistent ids.
2236
2237 def persistent_id(self, object):
2238 if isinstance(object, int) and object % 2 == 0:
2239 self.id_count += 1
2240 return str(object)
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08002241 elif object == "test_false_value":
2242 self.false_count += 1
2243 return ""
Jeremy Hylton4c8be852002-11-13 22:10:47 +00002244 else:
2245 return None
2246
2247 def persistent_load(self, oid):
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08002248 if not oid:
2249 self.load_false_count += 1
2250 return "test_false_value"
2251 else:
2252 self.load_count += 1
2253 object = int(oid)
2254 assert object % 2 == 0
2255 return object
Jeremy Hylton4c8be852002-11-13 22:10:47 +00002256
2257 def test_persistence(self):
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08002258 L = list(range(10)) + ["test_false_value"]
2259 for proto in protocols:
2260 self.id_count = 0
2261 self.false_count = 0
2262 self.load_false_count = 0
2263 self.load_count = 0
2264 self.assertEqual(self.loads(self.dumps(L, proto)), L)
2265 self.assertEqual(self.id_count, 5)
2266 self.assertEqual(self.false_count, 1)
2267 self.assertEqual(self.load_count, 5)
2268 self.assertEqual(self.load_false_count, 1)
Guido van Rossum98297ee2007-11-06 21:34:58 +00002269
Collin Winter771d8342009-04-16 03:18:06 +00002270
2271class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
2272
2273 pickler_class = None
2274 unpickler_class = None
2275
2276 def setUp(self):
2277 assert self.pickler_class
2278 assert self.unpickler_class
2279
2280 def test_clear_pickler_memo(self):
2281 # To test whether clear_memo() has any effect, we pickle an object,
2282 # then pickle it again without clearing the memo; the two serialized
2283 # forms should be different. If we clear_memo() and then pickle the
2284 # object again, the third serialized form should be identical to the
2285 # first one we obtained.
2286 data = ["abcdefg", "abcdefg", 44]
2287 f = io.BytesIO()
2288 pickler = self.pickler_class(f)
2289
2290 pickler.dump(data)
2291 first_pickled = f.getvalue()
2292
Serhiy Storchaka50254c52013-08-29 11:35:43 +03002293 # Reset BytesIO object.
Collin Winter771d8342009-04-16 03:18:06 +00002294 f.seek(0)
2295 f.truncate()
2296
2297 pickler.dump(data)
2298 second_pickled = f.getvalue()
2299
Serhiy Storchaka50254c52013-08-29 11:35:43 +03002300 # Reset the Pickler and BytesIO objects.
Collin Winter771d8342009-04-16 03:18:06 +00002301 pickler.clear_memo()
2302 f.seek(0)
2303 f.truncate()
2304
2305 pickler.dump(data)
2306 third_pickled = f.getvalue()
2307
2308 self.assertNotEqual(first_pickled, second_pickled)
2309 self.assertEqual(first_pickled, third_pickled)
2310
2311 def test_priming_pickler_memo(self):
2312 # Verify that we can set the Pickler's memo attribute.
2313 data = ["abcdefg", "abcdefg", 44]
2314 f = io.BytesIO()
2315 pickler = self.pickler_class(f)
2316
2317 pickler.dump(data)
2318 first_pickled = f.getvalue()
2319
2320 f = io.BytesIO()
2321 primed = self.pickler_class(f)
2322 primed.memo = pickler.memo
2323
2324 primed.dump(data)
2325 primed_pickled = f.getvalue()
2326
2327 self.assertNotEqual(first_pickled, primed_pickled)
2328
2329 def test_priming_unpickler_memo(self):
2330 # Verify that we can set the Unpickler's memo attribute.
2331 data = ["abcdefg", "abcdefg", 44]
2332 f = io.BytesIO()
2333 pickler = self.pickler_class(f)
2334
2335 pickler.dump(data)
2336 first_pickled = f.getvalue()
2337
2338 f = io.BytesIO()
2339 primed = self.pickler_class(f)
2340 primed.memo = pickler.memo
2341
2342 primed.dump(data)
2343 primed_pickled = f.getvalue()
2344
2345 unpickler = self.unpickler_class(io.BytesIO(first_pickled))
2346 unpickled_data1 = unpickler.load()
2347
2348 self.assertEqual(unpickled_data1, data)
2349
2350 primed = self.unpickler_class(io.BytesIO(primed_pickled))
2351 primed.memo = unpickler.memo
2352 unpickled_data2 = primed.load()
2353
2354 primed.memo.clear()
2355
2356 self.assertEqual(unpickled_data2, data)
2357 self.assertTrue(unpickled_data2 is unpickled_data1)
2358
2359 def test_reusing_unpickler_objects(self):
2360 data1 = ["abcdefg", "abcdefg", 44]
2361 f = io.BytesIO()
2362 pickler = self.pickler_class(f)
2363 pickler.dump(data1)
2364 pickled1 = f.getvalue()
2365
2366 data2 = ["abcdefg", 44, 44]
2367 f = io.BytesIO()
2368 pickler = self.pickler_class(f)
2369 pickler.dump(data2)
2370 pickled2 = f.getvalue()
2371
2372 f = io.BytesIO()
2373 f.write(pickled1)
2374 f.seek(0)
2375 unpickler = self.unpickler_class(f)
2376 self.assertEqual(unpickler.load(), data1)
2377
2378 f.seek(0)
2379 f.truncate()
2380 f.write(pickled2)
2381 f.seek(0)
2382 self.assertEqual(unpickler.load(), data2)
2383
Antoine Pitrou04248a82010-10-12 20:51:21 +00002384 def _check_multiple_unpicklings(self, ioclass):
2385 for proto in protocols:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002386 with self.subTest(proto=proto):
2387 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
2388 f = ioclass()
2389 pickler = self.pickler_class(f, protocol=proto)
2390 pickler.dump(data1)
2391 pickled = f.getvalue()
Antoine Pitrou04248a82010-10-12 20:51:21 +00002392
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002393 N = 5
2394 f = ioclass(pickled * N)
2395 unpickler = self.unpickler_class(f)
2396 for i in range(N):
2397 if f.seekable():
2398 pos = f.tell()
2399 self.assertEqual(unpickler.load(), data1)
2400 if f.seekable():
2401 self.assertEqual(f.tell(), pos + len(pickled))
2402 self.assertRaises(EOFError, unpickler.load)
Antoine Pitrou04248a82010-10-12 20:51:21 +00002403
2404 def test_multiple_unpicklings_seekable(self):
2405 self._check_multiple_unpicklings(io.BytesIO)
2406
2407 def test_multiple_unpicklings_unseekable(self):
2408 self._check_multiple_unpicklings(UnseekableIO)
2409
Antoine Pitrouf6c7a852011-08-11 21:04:02 +02002410 def test_unpickling_buffering_readline(self):
2411 # Issue #12687: the unpickler's buffering logic could fail with
2412 # text mode opcodes.
2413 data = list(range(10))
2414 for proto in protocols:
2415 for buf_size in range(1, 11):
2416 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size)
2417 pickler = self.pickler_class(f, protocol=proto)
2418 pickler.dump(data)
2419 f.seek(0)
2420 unpickler = self.unpickler_class(f)
2421 self.assertEqual(unpickler.load(), data)
2422
Collin Winter771d8342009-04-16 03:18:06 +00002423
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01002424# Tests for dispatch_table attribute
2425
2426REDUCE_A = 'reduce_A'
2427
2428class AAA(object):
2429 def __reduce__(self):
2430 return str, (REDUCE_A,)
2431
2432class BBB(object):
2433 pass
2434
2435class AbstractDispatchTableTests(unittest.TestCase):
2436
2437 def test_default_dispatch_table(self):
2438 # No dispatch_table attribute by default
2439 f = io.BytesIO()
2440 p = self.pickler_class(f, 0)
2441 with self.assertRaises(AttributeError):
2442 p.dispatch_table
2443 self.assertFalse(hasattr(p, 'dispatch_table'))
2444
2445 def test_class_dispatch_table(self):
2446 # A dispatch_table attribute can be specified class-wide
2447 dt = self.get_dispatch_table()
2448
2449 class MyPickler(self.pickler_class):
2450 dispatch_table = dt
2451
2452 def dumps(obj, protocol=None):
2453 f = io.BytesIO()
2454 p = MyPickler(f, protocol)
2455 self.assertEqual(p.dispatch_table, dt)
2456 p.dump(obj)
2457 return f.getvalue()
2458
2459 self._test_dispatch_table(dumps, dt)
2460
2461 def test_instance_dispatch_table(self):
2462 # A dispatch_table attribute can also be specified instance-wide
2463 dt = self.get_dispatch_table()
2464
2465 def dumps(obj, protocol=None):
2466 f = io.BytesIO()
2467 p = self.pickler_class(f, protocol)
2468 p.dispatch_table = dt
2469 self.assertEqual(p.dispatch_table, dt)
2470 p.dump(obj)
2471 return f.getvalue()
2472
2473 self._test_dispatch_table(dumps, dt)
2474
2475 def _test_dispatch_table(self, dumps, dispatch_table):
2476 def custom_load_dump(obj):
2477 return pickle.loads(dumps(obj, 0))
2478
2479 def default_load_dump(obj):
2480 return pickle.loads(pickle.dumps(obj, 0))
2481
2482 # pickling complex numbers using protocol 0 relies on copyreg
2483 # so check pickling a complex number still works
2484 z = 1 + 2j
2485 self.assertEqual(custom_load_dump(z), z)
2486 self.assertEqual(default_load_dump(z), z)
2487
2488 # modify pickling of complex
2489 REDUCE_1 = 'reduce_1'
2490 def reduce_1(obj):
2491 return str, (REDUCE_1,)
2492 dispatch_table[complex] = reduce_1
2493 self.assertEqual(custom_load_dump(z), REDUCE_1)
2494 self.assertEqual(default_load_dump(z), z)
2495
2496 # check picklability of AAA and BBB
2497 a = AAA()
2498 b = BBB()
2499 self.assertEqual(custom_load_dump(a), REDUCE_A)
2500 self.assertIsInstance(custom_load_dump(b), BBB)
2501 self.assertEqual(default_load_dump(a), REDUCE_A)
2502 self.assertIsInstance(default_load_dump(b), BBB)
2503
2504 # modify pickling of BBB
2505 dispatch_table[BBB] = reduce_1
2506 self.assertEqual(custom_load_dump(a), REDUCE_A)
2507 self.assertEqual(custom_load_dump(b), REDUCE_1)
2508 self.assertEqual(default_load_dump(a), REDUCE_A)
2509 self.assertIsInstance(default_load_dump(b), BBB)
2510
2511 # revert pickling of BBB and modify pickling of AAA
2512 REDUCE_2 = 'reduce_2'
2513 def reduce_2(obj):
2514 return str, (REDUCE_2,)
2515 dispatch_table[AAA] = reduce_2
2516 del dispatch_table[BBB]
2517 self.assertEqual(custom_load_dump(a), REDUCE_2)
2518 self.assertIsInstance(custom_load_dump(b), BBB)
2519 self.assertEqual(default_load_dump(a), REDUCE_A)
2520 self.assertIsInstance(default_load_dump(b), BBB)
2521
2522
Guido van Rossum98297ee2007-11-06 21:34:58 +00002523if __name__ == "__main__":
2524 # Print some stuff that can be used to rewrite DATA{0,1,2}
2525 from pickletools import dis
2526 x = create_data()
Serhiy Storchakab8b951f2015-09-29 15:49:58 +03002527 for i in range(pickle.HIGHEST_PROTOCOL+1):
Guido van Rossum98297ee2007-11-06 21:34:58 +00002528 p = pickle.dumps(x, i)
2529 print("DATA{0} = (".format(i))
2530 for j in range(0, len(p), 20):
2531 b = bytes(p[j:j+20])
2532 print(" {0!r}".format(b))
2533 print(")")
2534 print()
2535 print("# Disassembly of DATA{0}".format(i))
2536 print("DATA{0}_DIS = \"\"\"\\".format(i))
2537 dis(p)
2538 print("\"\"\"")
2539 print()