blob: 8449340e7f49e4de3291ebbb52c7f0b207a643bc [file] [log] [blame]
Guido van Rossum54f22ed2000-02-04 15:10:34 +00001"""Create portable serialized representations of Python objects.
Guido van Rossuma48061a1995-01-10 00:31:14 +00002
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00003See module copyreg for a mechanism for registering custom picklers.
Tim Peters22a449a2003-01-27 20:16:36 +00004See module pickletools source for extensive comments.
Guido van Rossuma48061a1995-01-10 00:31:14 +00005
Guido van Rossume467be61997-12-05 19:42:42 +00006Classes:
Guido van Rossuma48061a1995-01-10 00:31:14 +00007
Guido van Rossume467be61997-12-05 19:42:42 +00008 Pickler
9 Unpickler
Guido van Rossuma48061a1995-01-10 00:31:14 +000010
Guido van Rossume467be61997-12-05 19:42:42 +000011Functions:
Guido van Rossuma48061a1995-01-10 00:31:14 +000012
Guido van Rossume467be61997-12-05 19:42:42 +000013 dump(object, file)
14 dumps(object) -> string
15 load(file) -> object
16 loads(string) -> object
Guido van Rossuma48061a1995-01-10 00:31:14 +000017
Guido van Rossume467be61997-12-05 19:42:42 +000018Misc variables:
Guido van Rossuma48061a1995-01-10 00:31:14 +000019
Fred Drakefe82acc1998-02-13 03:24:48 +000020 __version__
Guido van Rossume467be61997-12-05 19:42:42 +000021 format_version
22 compatible_formats
Guido van Rossuma48061a1995-01-10 00:31:14 +000023
Guido van Rossuma48061a1995-01-10 00:31:14 +000024"""
25
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010026from types import FunctionType, BuiltinFunctionType, ModuleType
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000027from copyreg import dispatch_table
28from copyreg import _extension_registry, _inverted_registry, _extension_cache
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +030029from itertools import islice
Guido van Rossumd3703791998-10-22 20:15:36 +000030import sys
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +030031from sys import maxsize
32from struct import pack, unpack
Skip Montanaro23bafc62001-02-18 03:10:09 +000033import re
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000034import io
Walter Dörwald42748a82007-06-12 16:40:17 +000035import codecs
Antoine Pitroud9dfaa92009-06-04 20:32:06 +000036import _compat_pickle
Guido van Rossuma48061a1995-01-10 00:31:14 +000037
Skip Montanaro352674d2001-02-07 23:14:30 +000038__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
39 "Unpickler", "dump", "dumps", "load", "loads"]
40
Guido van Rossum98297ee2007-11-06 21:34:58 +000041# Shortcut for use in isinstance testing
Alexandre Vassalotti8cb02b62008-05-03 01:42:49 +000042bytes_types = (bytes, bytearray)
Guido van Rossum98297ee2007-11-06 21:34:58 +000043
Tim Petersc0c12b52003-01-29 00:56:17 +000044# These are purely informational; no code uses these.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010045format_version = "4.0" # File format version we write
Guido van Rossumf29d3d62003-01-27 22:47:53 +000046compatible_formats = ["1.0", # Original protocol 0
Guido van Rossumbc64e222003-01-28 16:34:19 +000047 "1.1", # Protocol 0 with INST added
Guido van Rossumf29d3d62003-01-27 22:47:53 +000048 "1.2", # Original protocol 1
49 "1.3", # Protocol 1 with BINFLOAT added
50 "2.0", # Protocol 2
Guido van Rossumf4169812008-03-17 22:56:06 +000051 "3.0", # Protocol 3
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010052 "4.0", # Protocol 4
Guido van Rossumf29d3d62003-01-27 22:47:53 +000053 ] # Old format versions we can read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000054
Guido van Rossum99603b02007-07-20 00:22:32 +000055# This is the highest protocol number we know how to read.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010056HIGHEST_PROTOCOL = 4
Tim Peters8587b3c2003-02-13 15:44:41 +000057
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000058# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
Guido van Rossumf4169812008-03-17 22:56:06 +000059# We intentionally write a protocol that Python 2.x cannot read;
60# there are too many issues with that.
61DEFAULT_PROTOCOL = 3
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000062
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000063class PickleError(Exception):
Neal Norwitzefbb67b2002-05-30 12:12:04 +000064 """A common base class for the other pickling exceptions."""
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000065 pass
66
67class PicklingError(PickleError):
68 """This exception is raised when an unpicklable object is passed to the
69 dump() method.
70
71 """
72 pass
73
74class UnpicklingError(PickleError):
75 """This exception is raised when there is a problem unpickling an object,
76 such as a security violation.
77
78 Note that other exceptions may also be raised during unpickling, including
79 (but not necessarily limited to) AttributeError, EOFError, ImportError,
80 and IndexError.
81
82 """
83 pass
Guido van Rossum7849da81995-03-09 14:08:35 +000084
Tim Petersc0c12b52003-01-29 00:56:17 +000085# An instance of _Stop is raised by Unpickler.load_stop() in response to
86# the STOP opcode, passing the object that is the result of unpickling.
Guido van Rossumff871742000-12-13 18:11:56 +000087class _Stop(Exception):
88 def __init__(self, value):
89 self.value = value
90
Guido van Rossum533dbcf2003-01-28 17:55:05 +000091# Jython has PyStringMap; it's a dict subclass with string keys
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000092try:
93 from org.python.core import PyStringMap
Brett Cannoncd171c82013-07-04 17:43:24 -040094except ImportError:
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000095 PyStringMap = None
96
Tim Peters22a449a2003-01-27 20:16:36 +000097# Pickle opcodes. See pickletools.py for extensive docs. The listing
98# here is in kind-of alphabetical order of 1-character pickle code.
99# pickletools groups them by purpose.
Guido van Rossumdbb718f2001-09-21 19:22:34 +0000100
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000101MARK = b'(' # push special markobject on stack
102STOP = b'.' # every pickle ends with STOP
103POP = b'0' # discard topmost stack item
104POP_MARK = b'1' # discard stack top through topmost markobject
105DUP = b'2' # duplicate top stack item
106FLOAT = b'F' # push float object; decimal string argument
107INT = b'I' # push integer or bool; decimal string argument
108BININT = b'J' # push four-byte signed int
109BININT1 = b'K' # push 1-byte unsigned int
110LONG = b'L' # push long; decimal string argument
111BININT2 = b'M' # push 2-byte unsigned int
112NONE = b'N' # push None
113PERSID = b'P' # push persistent object; id is taken from string arg
114BINPERSID = b'Q' # " " " ; " " " " stack
115REDUCE = b'R' # apply callable to argtuple, both on stack
116STRING = b'S' # push string; NL-terminated string argument
117BINSTRING = b'T' # push string; counted binary string argument
118SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
119UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
120BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
121APPEND = b'a' # append stack top to list below it
122BUILD = b'b' # call __setstate__ or __dict__.update()
123GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
124DICT = b'd' # build a dict from stack items
125EMPTY_DICT = b'}' # push empty dict
126APPENDS = b'e' # extend list on stack by topmost stack slice
127GET = b'g' # push item from memo on stack; index is string arg
128BINGET = b'h' # " " " " " " ; " " 1-byte arg
129INST = b'i' # build & push class instance
130LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
131LIST = b'l' # build list from topmost stack items
132EMPTY_LIST = b']' # push empty list
133OBJ = b'o' # build & push class instance
134PUT = b'p' # store stack top in memo; index is string arg
135BINPUT = b'q' # " " " " " ; " " 1-byte arg
136LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
137SETITEM = b's' # add key+value pair to dict
138TUPLE = b't' # build tuple from topmost stack items
139EMPTY_TUPLE = b')' # push empty tuple
140SETITEMS = b'u' # modify dict by adding topmost key+value pairs
141BINFLOAT = b'G' # push float; arg is 8-byte float encoding
Tim Peters22a449a2003-01-27 20:16:36 +0000142
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000143TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
144FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
Guido van Rossum77f6a652002-04-03 22:41:51 +0000145
Guido van Rossum586c9e82003-01-29 06:16:12 +0000146# Protocol 2
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000147
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000148PROTO = b'\x80' # identify pickle protocol
149NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
150EXT1 = b'\x82' # push object from extension registry; 1-byte index
151EXT2 = b'\x83' # ditto, but 2-byte index
152EXT4 = b'\x84' # ditto, but 4-byte index
153TUPLE1 = b'\x85' # build 1-tuple from stack top
154TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
155TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
156NEWTRUE = b'\x88' # push True
157NEWFALSE = b'\x89' # push False
158LONG1 = b'\x8a' # push long from < 256 bytes
159LONG4 = b'\x8b' # push really big long
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000160
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000161_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
162
Guido van Rossumf4169812008-03-17 22:56:06 +0000163# Protocol 3 (Python 3.x)
164
165BINBYTES = b'B' # push bytes; counted binary string argument
166SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
Guido van Rossuma48061a1995-01-10 00:31:14 +0000167
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100168# Protocol 4
169SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
170BINUNICODE8 = b'\x8d' # push very long string
171BINBYTES8 = b'\x8e' # push very long bytes string
172EMPTY_SET = b'\x8f' # push empty set on the stack
173ADDITEMS = b'\x90' # modify set by adding topmost stack items
174FROZENSET = b'\x91' # build frozenset from topmost stack items
175NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
176STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
177MEMOIZE = b'\x94' # store top of the stack in memo
178FRAME = b'\x95' # indicate the beginning of a new frame
179
180__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
181
182
183class _Framer:
184
185 _FRAME_SIZE_TARGET = 64 * 1024
186
187 def __init__(self, file_write):
188 self.file_write = file_write
189 self.current_frame = None
190
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100191 def start_framing(self):
192 self.current_frame = io.BytesIO()
193
194 def end_framing(self):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800195 if self.current_frame and self.current_frame.tell() > 0:
196 self.commit_frame(force=True)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100197 self.current_frame = None
198
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800199 def commit_frame(self, force=False):
200 if self.current_frame:
201 f = self.current_frame
202 if f.tell() >= self._FRAME_SIZE_TARGET or force:
203 with f.getbuffer() as data:
204 n = len(data)
205 write = self.file_write
206 write(FRAME)
207 write(pack("<Q", n))
208 write(data)
209 f.seek(0)
210 f.truncate()
211
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100212 def write(self, data):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800213 if self.current_frame:
214 return self.current_frame.write(data)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100215 else:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800216 return self.file_write(data)
217
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100218
219class _Unframer:
220
221 def __init__(self, file_read, file_readline, file_tell=None):
222 self.file_read = file_read
223 self.file_readline = file_readline
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100224 self.current_frame = None
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100225
226 def read(self, n):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800227 if self.current_frame:
228 data = self.current_frame.read(n)
229 if not data and n != 0:
230 self.current_frame = None
231 return self.file_read(n)
232 if len(data) < n:
233 raise UnpicklingError(
234 "pickle exhausted before end of frame")
235 return data
236 else:
237 return self.file_read(n)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100238
239 def readline(self):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800240 if self.current_frame:
241 data = self.current_frame.readline()
242 if not data:
243 self.current_frame = None
244 return self.file_readline()
245 if data[-1] != b'\n':
246 raise UnpicklingError(
247 "pickle exhausted before end of frame")
248 return data
249 else:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100250 return self.file_readline()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100251
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800252 def load_frame(self, frame_size):
253 if self.current_frame and self.current_frame.read() != b'':
254 raise UnpicklingError(
255 "beginning of a new frame before end of current frame")
256 self.current_frame = io.BytesIO(self.file_read(frame_size))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100257
258
259# Tools used for pickling.
260
261def _getattribute(obj, name, allow_qualname=False):
262 dotted_path = name.split(".")
263 if not allow_qualname and len(dotted_path) > 1:
264 raise AttributeError("Can't get qualified attribute {!r} on {!r}; " +
265 "use protocols >= 4 to enable support"
266 .format(name, obj))
267 for subpath in dotted_path:
268 if subpath == '<locals>':
269 raise AttributeError("Can't get local attribute {!r} on {!r}"
270 .format(name, obj))
271 try:
272 obj = getattr(obj, subpath)
273 except AttributeError:
274 raise AttributeError("Can't get attribute {!r} on {!r}"
275 .format(name, obj))
276 return obj
277
278def whichmodule(obj, name, allow_qualname=False):
279 """Find the module an object belong to."""
280 module_name = getattr(obj, '__module__', None)
281 if module_name is not None:
282 return module_name
283 for module_name, module in sys.modules.items():
284 if module_name == '__main__' or module is None:
285 continue
286 try:
287 if _getattribute(module, name, allow_qualname) is obj:
288 return module_name
289 except AttributeError:
290 pass
291 return '__main__'
292
293def encode_long(x):
294 r"""Encode a long to a two's complement little-endian binary string.
295 Note that 0 is a special case, returning an empty string, to save a
296 byte in the LONG1 pickling context.
297
298 >>> encode_long(0)
299 b''
300 >>> encode_long(255)
301 b'\xff\x00'
302 >>> encode_long(32767)
303 b'\xff\x7f'
304 >>> encode_long(-256)
305 b'\x00\xff'
306 >>> encode_long(-32768)
307 b'\x00\x80'
308 >>> encode_long(-128)
309 b'\x80'
310 >>> encode_long(127)
311 b'\x7f'
312 >>>
313 """
314 if x == 0:
315 return b''
316 nbytes = (x.bit_length() >> 3) + 1
317 result = x.to_bytes(nbytes, byteorder='little', signed=True)
318 if x < 0 and nbytes > 1:
319 if result[-1] == 0xff and (result[-2] & 0x80) != 0:
320 result = result[:-1]
321 return result
322
323def decode_long(data):
324 r"""Decode a long from a two's complement little-endian binary string.
325
326 >>> decode_long(b'')
327 0
328 >>> decode_long(b"\xff\x00")
329 255
330 >>> decode_long(b"\xff\x7f")
331 32767
332 >>> decode_long(b"\x00\xff")
333 -256
334 >>> decode_long(b"\x00\x80")
335 -32768
336 >>> decode_long(b"\x80")
337 -128
338 >>> decode_long(b"\x7f")
339 127
340 """
341 return int.from_bytes(data, byteorder='little', signed=True)
342
Skip Montanaro23bafc62001-02-18 03:10:09 +0000343
Guido van Rossum1be31752003-01-28 15:19:53 +0000344# Pickling machinery
345
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000346class _Pickler:
Guido van Rossuma48061a1995-01-10 00:31:14 +0000347
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000348 def __init__(self, file, protocol=None, *, fix_imports=True):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000349 """This takes a binary file for writing a pickle data stream.
350
Guido van Rossumcf117b02003-02-09 17:19:41 +0000351 The optional protocol argument tells the pickler to use the
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100352 given protocol; supported protocols are 0, 1, 2, 3 and 4. The
353 default protocol is 3; a backward-incompatible protocol designed for
354 Python 3.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000355
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000356 Specifying a negative protocol version selects the highest
Tim Peters5bd2a792003-02-01 16:45:06 +0000357 protocol version supported. The higher the protocol used, the
358 more recent the version of Python needed to read the pickle
359 produced.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000360
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000361 The file argument must have a write() method that accepts a single
362 bytes argument. It can thus be a file object opened for binary
363 writing, a io.BytesIO instance, or any other custom object that
364 meets this interface.
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000365
366 If fix_imports is True and protocol is less than 3, pickle will try to
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100367 map the new Python 3 names to the old module names used in Python 2,
368 so that the pickle data stream is readable with Python 2.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000369 """
Guido van Rossumcf117b02003-02-09 17:19:41 +0000370 if protocol is None:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000371 protocol = DEFAULT_PROTOCOL
Guido van Rossumcf117b02003-02-09 17:19:41 +0000372 if protocol < 0:
Tim Peters8587b3c2003-02-13 15:44:41 +0000373 protocol = HIGHEST_PROTOCOL
374 elif not 0 <= protocol <= HIGHEST_PROTOCOL:
375 raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000376 try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100377 self._file_write = file.write
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000378 except AttributeError:
379 raise TypeError("file must have a 'write' attribute")
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800380 self.framer = _Framer(self._file_write)
381 self.write = self.framer.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000382 self.memo = {}
Guido van Rossumcf117b02003-02-09 17:19:41 +0000383 self.proto = int(protocol)
384 self.bin = protocol >= 1
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000385 self.fast = 0
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000386 self.fix_imports = fix_imports and protocol < 3
Guido van Rossuma48061a1995-01-10 00:31:14 +0000387
Fred Drake7f781c92002-05-01 20:33:53 +0000388 def clear_memo(self):
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000389 """Clears the pickler's "memo".
390
391 The memo is the data structure that remembers which objects the
Tim Petersb377f8a2003-01-28 00:23:36 +0000392 pickler has already seen, so that shared or recursive objects are
393 pickled by reference and not by value. This method is useful when
394 re-using picklers.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000395
396 """
Fred Drake7f781c92002-05-01 20:33:53 +0000397 self.memo.clear()
398
Guido van Rossum3a41c612003-01-28 15:10:22 +0000399 def dump(self, obj):
Tim Peters5bd2a792003-02-01 16:45:06 +0000400 """Write a pickled representation of obj to the open file."""
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +0000401 # Check whether Pickler was initialized correctly. This is
402 # only needed to mimic the behavior of _pickle.Pickler.dump().
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100403 if not hasattr(self, "_file_write"):
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +0000404 raise PicklingError("Pickler.__init__() was not called by "
405 "%s.__init__()" % (self.__class__.__name__,))
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000406 if self.proto >= 2:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800407 self.write(PROTO + pack("<B", self.proto))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100408 if self.proto >= 4:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800409 self.framer.start_framing()
Guido van Rossum3a41c612003-01-28 15:10:22 +0000410 self.save(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000411 self.write(STOP)
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800412 self.framer.end_framing()
Guido van Rossuma48061a1995-01-10 00:31:14 +0000413
Jeremy Hylton3422c992003-01-24 19:29:52 +0000414 def memoize(self, obj):
415 """Store an object in the memo."""
416
Tim Peterse46b73f2003-01-27 21:22:10 +0000417 # The Pickler memo is a dictionary mapping object ids to 2-tuples
418 # that contain the Unpickler memo key and the object being memoized.
419 # The memo key is written to the pickle and will become
Jeremy Hylton3422c992003-01-24 19:29:52 +0000420 # the key in the Unpickler's memo. The object is stored in the
Tim Peterse46b73f2003-01-27 21:22:10 +0000421 # Pickler memo so that transient objects are kept alive during
422 # pickling.
Jeremy Hylton3422c992003-01-24 19:29:52 +0000423
Tim Peterse46b73f2003-01-27 21:22:10 +0000424 # The use of the Unpickler memo length as the memo key is just a
425 # convention. The only requirement is that the memo values be unique.
426 # But there appears no advantage to any other scheme, and this
Tim Peterscbd0a322003-01-28 00:24:43 +0000427 # scheme allows the Unpickler memo to be implemented as a plain (but
Tim Peterse46b73f2003-01-27 21:22:10 +0000428 # growable) array, indexed by memo key.
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000429 if self.fast:
430 return
Guido van Rossum9b40e802003-01-30 06:37:41 +0000431 assert id(obj) not in self.memo
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100432 idx = len(self.memo)
433 self.write(self.put(idx))
434 self.memo[id(obj)] = idx, obj
Jeremy Hylton3422c992003-01-24 19:29:52 +0000435
Tim Petersbb38e302003-01-27 21:25:41 +0000436 # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100437 def put(self, idx):
438 if self.proto >= 4:
439 return MEMOIZE
440 elif self.bin:
441 if idx < 256:
442 return BINPUT + pack("<B", idx)
Guido van Rossum5c938d02003-01-28 03:03:08 +0000443 else:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100444 return LONG_BINPUT + pack("<I", idx)
445 else:
446 return PUT + repr(idx).encode("ascii") + b'\n'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000447
Tim Petersbb38e302003-01-27 21:25:41 +0000448 # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300449 def get(self, i):
Tim Petersc32d8242001-04-10 02:48:53 +0000450 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000451 if i < 256:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300452 return BINGET + pack("<B", i)
Guido van Rossum5c938d02003-01-28 03:03:08 +0000453 else:
Antoine Pitroubf6ecf92012-11-24 20:40:21 +0100454 return LONG_BINGET + pack("<I", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000455
Guido van Rossum39478e82007-08-27 17:23:59 +0000456 return GET + repr(i).encode("ascii") + b'\n'
Tim Peters2344fae2001-01-15 00:50:52 +0000457
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000458 def save(self, obj, save_persistent_id=True):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800459 self.framer.commit_frame()
460
Guido van Rossumbc64e222003-01-28 16:34:19 +0000461 # Check for persistent id (defined by a subclass)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000462 pid = self.persistent_id(obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000463 if pid is not None and save_persistent_id:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000464 self.save_pers(pid)
465 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000466
Guido van Rossumbc64e222003-01-28 16:34:19 +0000467 # Check the memo
468 x = self.memo.get(id(obj))
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300469 if x is not None:
Guido van Rossumbc64e222003-01-28 16:34:19 +0000470 self.write(self.get(x[0]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000471 return
472
Guido van Rossumbc64e222003-01-28 16:34:19 +0000473 # Check the type dispatch table
Guido van Rossum3a41c612003-01-28 15:10:22 +0000474 t = type(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000475 f = self.dispatch.get(t)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300476 if f is not None:
Guido van Rossumbc64e222003-01-28 16:34:19 +0000477 f(self, obj) # Call unbound method with explicit self
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000478 return
479
Antoine Pitrou8d3c2902012-03-04 18:31:48 +0100480 # Check private dispatch table if any, or else copyreg.dispatch_table
481 reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300482 if reduce is not None:
Guido van Rossumc53f0092003-02-18 22:05:12 +0000483 rv = reduce(obj)
484 else:
Antoine Pitrouffd41d92011-10-04 09:23:04 +0200485 # Check for a class with a custom metaclass; treat as regular class
486 try:
487 issc = issubclass(t, type)
488 except TypeError: # t is not a class (old Boost; see SF #502085)
489 issc = False
490 if issc:
491 self.save_global(obj)
492 return
493
Guido van Rossumc53f0092003-02-18 22:05:12 +0000494 # Check for a __reduce_ex__ method, fall back to __reduce__
495 reduce = getattr(obj, "__reduce_ex__", None)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300496 if reduce is not None:
Guido van Rossumc53f0092003-02-18 22:05:12 +0000497 rv = reduce(self.proto)
498 else:
499 reduce = getattr(obj, "__reduce__", None)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300500 if reduce is not None:
Guido van Rossumc53f0092003-02-18 22:05:12 +0000501 rv = reduce()
502 else:
503 raise PicklingError("Can't pickle %r object: %r" %
504 (t.__name__, obj))
Tim Petersb32a8312003-01-28 00:48:09 +0000505
Guido van Rossumbc64e222003-01-28 16:34:19 +0000506 # Check for string returned by reduce(), meaning "save as global"
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000507 if isinstance(rv, str):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000508 self.save_global(obj, rv)
Tim Petersb32a8312003-01-28 00:48:09 +0000509 return
510
Guido van Rossumbc64e222003-01-28 16:34:19 +0000511 # Assert that reduce() returned a tuple
Guido van Rossum13257902007-06-07 23:15:56 +0000512 if not isinstance(rv, tuple):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000513 raise PicklingError("%s must return string or tuple" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000514
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000515 # Assert that it returned an appropriately sized tuple
Guido van Rossumbc64e222003-01-28 16:34:19 +0000516 l = len(rv)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000517 if not (2 <= l <= 5):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000518 raise PicklingError("Tuple returned by %s must have "
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000519 "two to five elements" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000520
Guido van Rossumbc64e222003-01-28 16:34:19 +0000521 # Save the reduce() output and finally memoize the object
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000522 self.save_reduce(obj=obj, *rv)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000523
Guido van Rossum3a41c612003-01-28 15:10:22 +0000524 def persistent_id(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000525 # This exists so a subclass can override it
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000526 return None
527
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000528 def save_pers(self, pid):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000529 # Save a persistent id reference
Tim Petersbd1cdb92003-01-28 01:03:10 +0000530 if self.bin:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000531 self.save(pid, save_persistent_id=False)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000532 self.write(BINPERSID)
Tim Petersbd1cdb92003-01-28 01:03:10 +0000533 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000534 self.write(PERSID + str(pid).encode("ascii") + b'\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000535
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100536 def save_reduce(self, func, args, state=None, listitems=None,
537 dictitems=None, obj=None):
Jeremy Hyltone3a565e2003-06-29 16:59:59 +0000538 # This API is called by some subclasses
Guido van Rossumbc64e222003-01-28 16:34:19 +0000539
Guido van Rossum13257902007-06-07 23:15:56 +0000540 if not isinstance(args, tuple):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100541 raise PicklingError("args from save_reduce() must be a tuple")
Florent Xicluna5d1155c2011-10-28 14:45:05 +0200542 if not callable(func):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100543 raise PicklingError("func from save_reduce() must be callable")
Guido van Rossumbc64e222003-01-28 16:34:19 +0000544
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000545 save = self.save
Guido van Rossumbc64e222003-01-28 16:34:19 +0000546 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000547
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100548 func_name = getattr(func, "__name__", "")
549 if self.proto >= 4 and func_name == "__newobj_ex__":
550 cls, args, kwargs = args
551 if not hasattr(cls, "__new__"):
552 raise PicklingError("args[0] from {} args has no __new__"
553 .format(func_name))
554 if obj is not None and cls is not obj.__class__:
555 raise PicklingError("args[0] from {} args has the wrong class"
556 .format(func_name))
557 save(cls)
558 save(args)
559 save(kwargs)
560 write(NEWOBJ_EX)
561 elif self.proto >= 2 and func_name == "__newobj__":
562 # A __reduce__ implementation can direct protocol 2 or newer to
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000563 # use the more efficient NEWOBJ opcode, while still
564 # allowing protocol 0 and 1 to work normally. For this to
565 # work, the function returned by __reduce__ should be
566 # called __newobj__, and its first argument should be a
Florent Xiclunaaa6c1d22011-12-12 18:54:29 +0100567 # class. The implementation for __newobj__
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000568 # should be as follows, although pickle has no way to
569 # verify this:
570 #
571 # def __newobj__(cls, *args):
572 # return cls.__new__(cls, *args)
573 #
574 # Protocols 0 and 1 will pickle a reference to __newobj__,
575 # while protocol 2 (and above) will pickle a reference to
576 # cls, the remaining args tuple, and the NEWOBJ code,
577 # which calls cls.__new__(cls, *args) at unpickling time
578 # (see load_newobj below). If __reduce__ returns a
579 # three-tuple, the state from the third tuple item will be
580 # pickled regardless of the protocol, calling __setstate__
581 # at unpickling time (see load_build below).
582 #
583 # Note that no standard __newobj__ implementation exists;
584 # you have to provide your own. This is to enforce
585 # compatibility with Python 2.2 (pickles written using
586 # protocol 0 or 1 in Python 2.3 should be unpicklable by
587 # Python 2.2).
588 cls = args[0]
589 if not hasattr(cls, "__new__"):
590 raise PicklingError(
591 "args[0] from __newobj__ args has no __new__")
Guido van Rossumf7f45172003-01-31 17:17:49 +0000592 if obj is not None and cls is not obj.__class__:
593 raise PicklingError(
594 "args[0] from __newobj__ args has the wrong class")
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000595 args = args[1:]
596 save(cls)
597 save(args)
598 write(NEWOBJ)
599 else:
600 save(func)
601 save(args)
602 write(REDUCE)
Tim Peters2344fae2001-01-15 00:50:52 +0000603
Guido van Rossumf7f45172003-01-31 17:17:49 +0000604 if obj is not None:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100605 # If the object is already in the memo, this means it is
606 # recursive. In this case, throw away everything we put on the
607 # stack, and fetch the object back from the memo.
608 if id(obj) in self.memo:
609 write(POP + self.get(self.memo[id(obj)][0]))
610 else:
611 self.memoize(obj)
Guido van Rossumf7f45172003-01-31 17:17:49 +0000612
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000613 # More new special cases (that work with older protocols as
614 # well): when __reduce__ returns a tuple with 4 or 5 items,
615 # the 4th and 5th item should be iterators that provide list
616 # items and dict items (as (key, value) tuples), or None.
617
618 if listitems is not None:
619 self._batch_appends(listitems)
620
621 if dictitems is not None:
622 self._batch_setitems(dictitems)
623
Tim Petersc32d8242001-04-10 02:48:53 +0000624 if state is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000625 save(state)
626 write(BUILD)
627
Guido van Rossumbc64e222003-01-28 16:34:19 +0000628 # Methods below this point are dispatched through the dispatch table
629
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000630 dispatch = {}
631
Guido van Rossum3a41c612003-01-28 15:10:22 +0000632 def save_none(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000633 self.write(NONE)
Guido van Rossum13257902007-06-07 23:15:56 +0000634 dispatch[type(None)] = save_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000635
Łukasz Langaf3078fb2012-03-12 19:46:12 +0100636 def save_ellipsis(self, obj):
637 self.save_global(Ellipsis, 'Ellipsis')
638 dispatch[type(Ellipsis)] = save_ellipsis
639
640 def save_notimplemented(self, obj):
641 self.save_global(NotImplemented, 'NotImplemented')
642 dispatch[type(NotImplemented)] = save_notimplemented
643
Guido van Rossum3a41c612003-01-28 15:10:22 +0000644 def save_bool(self, obj):
Guido van Rossum7d97d312003-01-28 04:25:27 +0000645 if self.proto >= 2:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300646 self.write(NEWTRUE if obj else NEWFALSE)
Guido van Rossum7d97d312003-01-28 04:25:27 +0000647 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300648 self.write(TRUE if obj else FALSE)
Guido van Rossum77f6a652002-04-03 22:41:51 +0000649 dispatch[bool] = save_bool
650
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300651 def save_long(self, obj):
Guido van Rossumddefaf32007-01-14 03:31:43 +0000652 if self.bin:
653 # If the int is small enough to fit in a signed 4-byte 2's-comp
654 # format, we can store it more efficiently than the general
655 # case.
656 # First one- and two-byte unsigned ints:
657 if obj >= 0:
658 if obj <= 0xff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300659 self.write(BININT1 + pack("<B", obj))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000660 return
661 if obj <= 0xffff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300662 self.write(BININT2 + pack("<H", obj))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000663 return
664 # Next check for 4-byte signed ints:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300665 if -0x80000000 <= obj <= 0x7fffffff:
Guido van Rossumddefaf32007-01-14 03:31:43 +0000666 self.write(BININT + pack("<i", obj))
667 return
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000668 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000669 encoded = encode_long(obj)
670 n = len(encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000671 if n < 256:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300672 self.write(LONG1 + pack("<B", n) + encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000673 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000674 self.write(LONG4 + pack("<i", n) + encoded)
Tim Petersee1a53c2003-02-02 02:57:53 +0000675 return
Mark Dickinson8dd05142009-01-20 20:43:58 +0000676 self.write(LONG + repr(obj).encode("ascii") + b'L\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000677 dispatch[int] = save_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000678
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300679 def save_float(self, obj):
Guido van Rossumd3703791998-10-22 20:15:36 +0000680 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000681 self.write(BINFLOAT + pack('>d', obj))
Guido van Rossumd3703791998-10-22 20:15:36 +0000682 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000683 self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000684 dispatch[float] = save_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000685
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300686 def save_bytes(self, obj):
Guido van Rossumf4169812008-03-17 22:56:06 +0000687 if self.proto < 3:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300688 if not obj: # bytes object is empty
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -0500689 self.save_reduce(bytes, (), obj=obj)
690 else:
691 self.save_reduce(codecs.encode,
692 (str(obj, 'latin1'), 'latin1'), obj=obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000693 return
694 n = len(obj)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100695 if n <= 0xff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300696 self.write(SHORT_BINBYTES + pack("<B", n) + obj)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100697 elif n > 0xffffffff and self.proto >= 4:
698 self.write(BINBYTES8 + pack("<Q", n) + obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000699 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300700 self.write(BINBYTES + pack("<I", n) + obj)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000701 self.memoize(obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000702 dispatch[bytes] = save_bytes
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000703
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300704 def save_str(self, obj):
Tim Petersc32d8242001-04-10 02:48:53 +0000705 if self.bin:
Victor Stinner485fb562010-04-13 11:07:24 +0000706 encoded = obj.encode('utf-8', 'surrogatepass')
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000707 n = len(encoded)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100708 if n <= 0xff and self.proto >= 4:
709 self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
710 elif n > 0xffffffff and self.proto >= 4:
711 self.write(BINUNICODE8 + pack("<Q", n) + encoded)
712 else:
713 self.write(BINUNICODE + pack("<I", n) + encoded)
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000714 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000715 obj = obj.replace("\\", "\\u005c")
716 obj = obj.replace("\n", "\\u000a")
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100717 self.write(UNICODE + obj.encode('raw-unicode-escape') +
718 b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000719 self.memoize(obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000720 dispatch[str] = save_str
Tim Peters658cba62001-02-09 20:06:00 +0000721
Guido van Rossum3a41c612003-01-28 15:10:22 +0000722 def save_tuple(self, obj):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300723 if not obj: # tuple is empty
724 if self.bin:
725 self.write(EMPTY_TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000726 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300727 self.write(MARK + TUPLE)
Tim Petersd97da802003-01-28 05:48:29 +0000728 return
729
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300730 n = len(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000731 save = self.save
732 memo = self.memo
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300733 if n <= 3 and self.proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000734 for element in obj:
Tim Petersd97da802003-01-28 05:48:29 +0000735 save(element)
736 # Subtle. Same as in the big comment below.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000737 if id(obj) in memo:
738 get = self.get(memo[id(obj)][0])
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300739 self.write(POP * n + get)
Tim Petersd97da802003-01-28 05:48:29 +0000740 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300741 self.write(_tuplesize2code[n])
Guido van Rossum3a41c612003-01-28 15:10:22 +0000742 self.memoize(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000743 return
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000744
Tim Peters1d63c9f2003-02-02 20:29:39 +0000745 # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
Tim Petersff57bff2003-01-28 05:34:53 +0000746 # has more than 3 elements.
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300747 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000748 write(MARK)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000749 for element in obj:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000750 save(element)
751
Tim Peters1d63c9f2003-02-02 20:29:39 +0000752 if id(obj) in memo:
Tim Petersf558da02003-01-28 02:09:55 +0000753 # Subtle. d was not in memo when we entered save_tuple(), so
754 # the process of saving the tuple's elements must have saved
755 # the tuple itself: the tuple is recursive. The proper action
756 # now is to throw away everything we put on the stack, and
757 # simply GET the tuple (it's already constructed). This check
758 # could have been done in the "for element" loop instead, but
759 # recursive tuples are a rare thing.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000760 get = self.get(memo[id(obj)][0])
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300761 if self.bin:
Tim Petersf558da02003-01-28 02:09:55 +0000762 write(POP_MARK + get)
763 else: # proto 0 -- POP_MARK not available
Tim Petersd97da802003-01-28 05:48:29 +0000764 write(POP * (n+1) + get)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000765 return
766
Tim Peters1d63c9f2003-02-02 20:29:39 +0000767 # No recursion.
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300768 write(TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000769 self.memoize(obj)
Jeremy Hylton3422c992003-01-24 19:29:52 +0000770
Guido van Rossum13257902007-06-07 23:15:56 +0000771 dispatch[tuple] = save_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000772
Guido van Rossum3a41c612003-01-28 15:10:22 +0000773 def save_list(self, obj):
Tim Petersc32d8242001-04-10 02:48:53 +0000774 if self.bin:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300775 self.write(EMPTY_LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000776 else: # proto 0 -- can't use EMPTY_LIST
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300777 self.write(MARK + LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000778
779 self.memoize(obj)
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000780 self._batch_appends(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000781
Guido van Rossum13257902007-06-07 23:15:56 +0000782 dispatch[list] = save_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000783
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000784 _BATCHSIZE = 1000
785
786 def _batch_appends(self, items):
787 # Helper to batch up APPENDS sequences
788 save = self.save
789 write = self.write
790
791 if not self.bin:
792 for x in items:
793 save(x)
794 write(APPEND)
795 return
796
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300797 it = iter(items)
798 while True:
799 tmp = list(islice(it, self._BATCHSIZE))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000800 n = len(tmp)
801 if n > 1:
802 write(MARK)
803 for x in tmp:
804 save(x)
805 write(APPENDS)
806 elif n:
807 save(tmp[0])
808 write(APPEND)
809 # else tmp is empty, and we're done
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300810 if n < self._BATCHSIZE:
811 return
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000812
Guido van Rossum3a41c612003-01-28 15:10:22 +0000813 def save_dict(self, obj):
Tim Petersc32d8242001-04-10 02:48:53 +0000814 if self.bin:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300815 self.write(EMPTY_DICT)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000816 else: # proto 0 -- can't use EMPTY_DICT
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300817 self.write(MARK + DICT)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000818
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000819 self.memoize(obj)
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000820 self._batch_setitems(obj.items())
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000821
Guido van Rossum13257902007-06-07 23:15:56 +0000822 dispatch[dict] = save_dict
823 if PyStringMap is not None:
Jeremy Hylton2b9d0291998-05-27 22:38:22 +0000824 dispatch[PyStringMap] = save_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000825
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000826 def _batch_setitems(self, items):
827 # Helper to batch up SETITEMS sequences; proto >= 1 only
828 save = self.save
829 write = self.write
830
831 if not self.bin:
832 for k, v in items:
833 save(k)
834 save(v)
835 write(SETITEM)
836 return
837
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300838 it = iter(items)
839 while True:
840 tmp = list(islice(it, self._BATCHSIZE))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000841 n = len(tmp)
842 if n > 1:
843 write(MARK)
844 for k, v in tmp:
845 save(k)
846 save(v)
847 write(SETITEMS)
848 elif n:
849 k, v = tmp[0]
850 save(k)
851 save(v)
852 write(SETITEM)
853 # else tmp is empty, and we're done
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300854 if n < self._BATCHSIZE:
855 return
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000856
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100857 def save_set(self, obj):
858 save = self.save
859 write = self.write
860
861 if self.proto < 4:
862 self.save_reduce(set, (list(obj),), obj=obj)
863 return
864
865 write(EMPTY_SET)
866 self.memoize(obj)
867
868 it = iter(obj)
869 while True:
870 batch = list(islice(it, self._BATCHSIZE))
871 n = len(batch)
872 if n > 0:
873 write(MARK)
874 for item in batch:
875 save(item)
876 write(ADDITEMS)
877 if n < self._BATCHSIZE:
878 return
879 dispatch[set] = save_set
880
881 def save_frozenset(self, obj):
882 save = self.save
883 write = self.write
884
885 if self.proto < 4:
886 self.save_reduce(frozenset, (list(obj),), obj=obj)
887 return
888
889 write(MARK)
890 for item in obj:
891 save(item)
892
893 if id(obj) in self.memo:
894 # If the object is already in the memo, this means it is
895 # recursive. In this case, throw away everything we put on the
896 # stack, and fetch the object back from the memo.
897 write(POP_MARK + self.get(self.memo[id(obj)][0]))
898 return
899
900 write(FROZENSET)
901 self.memoize(obj)
902 dispatch[frozenset] = save_frozenset
903
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300904 def save_global(self, obj, name=None):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000905 write = self.write
906 memo = self.memo
907
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100908 if name is None and self.proto >= 4:
909 name = getattr(obj, '__qualname__', None)
Tim Petersc32d8242001-04-10 02:48:53 +0000910 if name is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000911 name = obj.__name__
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000912
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100913 module_name = whichmodule(obj, name, allow_qualname=self.proto >= 4)
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000914 try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100915 __import__(module_name, level=0)
916 module = sys.modules[module_name]
917 obj2 = _getattribute(module, name, allow_qualname=self.proto >= 4)
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000918 except (ImportError, KeyError, AttributeError):
919 raise PicklingError(
920 "Can't pickle %r: it's not found as %s.%s" %
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100921 (obj, module_name, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000922 else:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100923 if obj2 is not obj:
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000924 raise PicklingError(
925 "Can't pickle %r: it's not the same object as %s.%s" %
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100926 (obj, module_name, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000927
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000928 if self.proto >= 2:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100929 code = _extension_registry.get((module_name, name))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000930 if code:
931 assert code > 0
932 if code <= 0xff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300933 write(EXT1 + pack("<B", code))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000934 elif code <= 0xffff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300935 write(EXT2 + pack("<H", code))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000936 else:
937 write(EXT4 + pack("<i", code))
938 return
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000939 # Non-ASCII identifiers are supported only with protocols >= 3.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100940 if self.proto >= 4:
941 self.save(module_name)
942 self.save(name)
943 write(STACK_GLOBAL)
944 elif self.proto >= 3:
945 write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000946 bytes(name, "utf-8") + b'\n')
947 else:
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000948 if self.fix_imports:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100949 r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
950 r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
951 if (module_name, name) in r_name_mapping:
952 module_name, name = r_name_mapping[(module_name, name)]
953 if module_name in r_import_mapping:
954 module_name = r_import_mapping[module_name]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000955 try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100956 write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000957 bytes(name, "ascii") + b'\n')
958 except UnicodeEncodeError:
959 raise PicklingError(
960 "can't pickle global identifier '%s.%s' using "
961 "pickle protocol %i" % (module, name, self.proto))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000962
Guido van Rossum3a41c612003-01-28 15:10:22 +0000963 self.memoize(obj)
Tim Peters3b769832003-01-28 03:51:36 +0000964
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100965 def save_method(self, obj):
966 if obj.__self__ is None or type(obj.__self__) is ModuleType:
967 self.save_global(obj)
968 else:
969 self.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
970
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000971 dispatch[FunctionType] = save_global
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100972 dispatch[BuiltinFunctionType] = save_method
Guido van Rossum13257902007-06-07 23:15:56 +0000973 dispatch[type] = save_global
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000974
Guido van Rossuma48061a1995-01-10 00:31:14 +0000975
Guido van Rossum1be31752003-01-28 15:19:53 +0000976# Unpickling machinery
977
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000978class _Unpickler:
Guido van Rossuma48061a1995-01-10 00:31:14 +0000979
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000980 def __init__(self, file, *, fix_imports=True,
981 encoding="ASCII", errors="strict"):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000982 """This takes a binary file for reading a pickle data stream.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000983
Tim Peters5bd2a792003-02-01 16:45:06 +0000984 The protocol version of the pickle is detected automatically, so no
985 proto argument is needed.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000986
Guido van Rossumfeea0782007-10-10 18:00:50 +0000987 The file-like object must have two methods, a read() method
988 that takes an integer argument, and a readline() method that
989 requires no arguments. Both methods should return bytes.
990 Thus file-like object can be a binary file object opened for
991 reading, a BytesIO object, or any other custom object that
992 meets this interface.
Guido van Rossumf4169812008-03-17 22:56:06 +0000993
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000994 Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
995 which are used to control compatiblity support for pickle stream
996 generated by Python 2.x. If *fix_imports* is True, pickle will try to
997 map the old Python 2.x names to the new names used in Python 3.x. The
998 *encoding* and *errors* tell pickle how to decode 8-bit string
999 instances pickled by Python 2.x; these default to 'ASCII' and
1000 'strict', respectively.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +00001001 """
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001002 self._file_readline = file.readline
1003 self._file_read = file.read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001004 self.memo = {}
Guido van Rossumf4169812008-03-17 22:56:06 +00001005 self.encoding = encoding
1006 self.errors = errors
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001007 self.proto = 0
1008 self.fix_imports = fix_imports
Guido van Rossuma48061a1995-01-10 00:31:14 +00001009
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001010 def load(self):
Guido van Rossum3a41c612003-01-28 15:10:22 +00001011 """Read a pickled object representation from the open file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +00001012
Guido van Rossum3a41c612003-01-28 15:10:22 +00001013 Return the reconstituted object hierarchy specified in the file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +00001014 """
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +00001015 # Check whether Unpickler was initialized correctly. This is
1016 # only needed to mimic the behavior of _pickle.Unpickler.dump().
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001017 if not hasattr(self, "_file_read"):
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +00001018 raise UnpicklingError("Unpickler.__init__() was not called by "
1019 "%s.__init__()" % (self.__class__.__name__,))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001020 self._unframer = _Unframer(self._file_read, self._file_readline)
1021 self.read = self._unframer.read
1022 self.readline = self._unframer.readline
Jeremy Hylton20747fa2001-11-09 16:15:04 +00001023 self.mark = object() # any new unique object
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001024 self.stack = []
1025 self.append = self.stack.append
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001026 self.proto = 0
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001027 read = self.read
1028 dispatch = self.dispatch
1029 try:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001030 while True:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001031 key = read(1)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001032 if not key:
1033 raise EOFError
Guido van Rossum98297ee2007-11-06 21:34:58 +00001034 assert isinstance(key, bytes_types)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001035 dispatch[key[0]](self)
Guido van Rossumb940e112007-01-10 16:19:56 +00001036 except _Stop as stopinst:
Guido van Rossumff871742000-12-13 18:11:56 +00001037 return stopinst.value
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001038
Tim Petersc23d18a2003-01-28 01:41:51 +00001039 # Return largest index k such that self.stack[k] is self.mark.
1040 # If the stack doesn't contain a mark, eventually raises IndexError.
1041 # This could be sped by maintaining another stack, of indices at which
1042 # the mark appears. For that matter, the latter stack would suffice,
1043 # and we wouldn't need to push mark objects on self.stack at all.
1044 # Doing so is probably a good thing, though, since if the pickle is
1045 # corrupt (or hostile) we may get a clue from finding self.mark embedded
1046 # in unpickled objects.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001047 def marker(self):
1048 stack = self.stack
1049 mark = self.mark
1050 k = len(stack)-1
1051 while stack[k] is not mark: k = k-1
1052 return k
1053
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001054 def persistent_load(self, pid):
Benjamin Peterson49956b22009-01-10 17:05:44 +00001055 raise UnpicklingError("unsupported persistent id encountered")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001056
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001057 dispatch = {}
1058
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001059 def load_proto(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001060 proto = self.read(1)[0]
Guido van Rossumf4169812008-03-17 22:56:06 +00001061 if not 0 <= proto <= HIGHEST_PROTOCOL:
Guido van Rossum26d95c32007-08-27 23:18:54 +00001062 raise ValueError("unsupported pickle protocol: %d" % proto)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001063 self.proto = proto
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001064 dispatch[PROTO[0]] = load_proto
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001065
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08001066 def load_frame(self):
1067 frame_size, = unpack('<Q', self.read(8))
1068 if frame_size > sys.maxsize:
1069 raise ValueError("frame size > sys.maxsize: %d" % frame_size)
1070 self._unframer.load_frame(frame_size)
1071 dispatch[FRAME[0]] = load_frame
1072
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001073 def load_persid(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001074 pid = self.readline()[:-1].decode("ascii")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001075 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001076 dispatch[PERSID[0]] = load_persid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001077
1078 def load_binpersid(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001079 pid = self.stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001080 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001081 dispatch[BINPERSID[0]] = load_binpersid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001082
1083 def load_none(self):
1084 self.append(None)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001085 dispatch[NONE[0]] = load_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001086
Guido van Rossum7d97d312003-01-28 04:25:27 +00001087 def load_false(self):
1088 self.append(False)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001089 dispatch[NEWFALSE[0]] = load_false
Guido van Rossum7d97d312003-01-28 04:25:27 +00001090
1091 def load_true(self):
1092 self.append(True)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001093 dispatch[NEWTRUE[0]] = load_true
Guido van Rossum7d97d312003-01-28 04:25:27 +00001094
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001095 def load_int(self):
Tim Peters19ef62d2001-08-28 22:21:18 +00001096 data = self.readline()
Guido van Rossume2763392002-04-05 19:30:08 +00001097 if data == FALSE[1:]:
1098 val = False
1099 elif data == TRUE[1:]:
1100 val = True
1101 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001102 val = int(data, 0)
Guido van Rossume2763392002-04-05 19:30:08 +00001103 self.append(val)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001104 dispatch[INT[0]] = load_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001105
1106 def load_binint(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001107 self.append(unpack('<i', self.read(4))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001108 dispatch[BININT[0]] = load_binint
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001109
1110 def load_binint1(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001111 self.append(self.read(1)[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001112 dispatch[BININT1[0]] = load_binint1
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001113
1114 def load_binint2(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001115 self.append(unpack('<H', self.read(2))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001116 dispatch[BININT2[0]] = load_binint2
Tim Peters2344fae2001-01-15 00:50:52 +00001117
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001118 def load_long(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001119 val = self.readline()[:-1]
1120 if val and val[-1] == b'L'[0]:
Mark Dickinson8dd05142009-01-20 20:43:58 +00001121 val = val[:-1]
Guido van Rossumfeea0782007-10-10 18:00:50 +00001122 self.append(int(val, 0))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001123 dispatch[LONG[0]] = load_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001124
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001125 def load_long1(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001126 n = self.read(1)[0]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001127 data = self.read(n)
1128 self.append(decode_long(data))
1129 dispatch[LONG1[0]] = load_long1
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001130
1131 def load_long4(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001132 n, = unpack('<i', self.read(4))
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001133 if n < 0:
1134 # Corrupt or hostile pickle -- we never write one like this
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001135 raise UnpicklingError("LONG pickle has negative byte count")
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001136 data = self.read(n)
1137 self.append(decode_long(data))
1138 dispatch[LONG4[0]] = load_long4
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001139
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001140 def load_float(self):
Guido van Rossumff871742000-12-13 18:11:56 +00001141 self.append(float(self.readline()[:-1]))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001142 dispatch[FLOAT[0]] = load_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001143
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001144 def load_binfloat(self):
Guido van Rossumd3703791998-10-22 20:15:36 +00001145 self.append(unpack('>d', self.read(8))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001146 dispatch[BINFLOAT[0]] = load_binfloat
Guido van Rossumd3703791998-10-22 20:15:36 +00001147
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001148 def load_string(self):
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07001149 data = self.readline()[:-1]
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001150 # Strip outermost quotes
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07001151 if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
1152 data = data[1:-1]
Martin v. Löwis8a8da792002-08-14 07:46:28 +00001153 else:
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07001154 raise UnpicklingError("the STRING opcode argument must be quoted")
1155 self.append(codecs.escape_decode(data)[0]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001156 .decode(self.encoding, self.errors))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001157 dispatch[STRING[0]] = load_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001158
1159 def load_binstring(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001160 # Deprecated BINSTRING uses signed 32-bit length
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001161 len, = unpack('<i', self.read(4))
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001162 if len < 0:
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001163 raise UnpicklingError("BINSTRING pickle has negative byte count")
Guido van Rossumf4169812008-03-17 22:56:06 +00001164 data = self.read(len)
1165 value = str(data, self.encoding, self.errors)
1166 self.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001167 dispatch[BINSTRING[0]] = load_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001168
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001169 def load_binbytes(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001170 len, = unpack('<I', self.read(4))
1171 if len > maxsize:
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001172 raise UnpicklingError("BINBYTES exceeds system's maximum size "
1173 "of %d bytes" % maxsize)
Guido van Rossumf4169812008-03-17 22:56:06 +00001174 self.append(self.read(len))
1175 dispatch[BINBYTES[0]] = load_binbytes
1176
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +00001177 def load_unicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001178 self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
1179 dispatch[UNICODE[0]] = load_unicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +00001180
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001181 def load_binunicode(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001182 len, = unpack('<I', self.read(4))
1183 if len > maxsize:
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001184 raise UnpicklingError("BINUNICODE exceeds system's maximum size "
1185 "of %d bytes" % maxsize)
Victor Stinner485fb562010-04-13 11:07:24 +00001186 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001187 dispatch[BINUNICODE[0]] = load_binunicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +00001188
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001189 def load_binunicode8(self):
1190 len, = unpack('<Q', self.read(8))
1191 if len > maxsize:
1192 raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
1193 "of %d bytes" % maxsize)
1194 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1195 dispatch[BINUNICODE8[0]] = load_binunicode8
1196
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001197 def load_short_binstring(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001198 len = self.read(1)[0]
1199 data = self.read(len)
Guido van Rossumf4169812008-03-17 22:56:06 +00001200 value = str(data, self.encoding, self.errors)
1201 self.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001202 dispatch[SHORT_BINSTRING[0]] = load_short_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001203
Guido van Rossumf4169812008-03-17 22:56:06 +00001204 def load_short_binbytes(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001205 len = self.read(1)[0]
1206 self.append(self.read(len))
Guido van Rossumf4169812008-03-17 22:56:06 +00001207 dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
1208
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001209 def load_short_binunicode(self):
1210 len = self.read(1)[0]
1211 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1212 dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
1213
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001214 def load_tuple(self):
1215 k = self.marker()
1216 self.stack[k:] = [tuple(self.stack[k+1:])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001217 dispatch[TUPLE[0]] = load_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001218
1219 def load_empty_tuple(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001220 self.append(())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001221 dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001222
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001223 def load_tuple1(self):
1224 self.stack[-1] = (self.stack[-1],)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001225 dispatch[TUPLE1[0]] = load_tuple1
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001226
1227 def load_tuple2(self):
1228 self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001229 dispatch[TUPLE2[0]] = load_tuple2
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001230
1231 def load_tuple3(self):
1232 self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001233 dispatch[TUPLE3[0]] = load_tuple3
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001234
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001235 def load_empty_list(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001236 self.append([])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001237 dispatch[EMPTY_LIST[0]] = load_empty_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001238
1239 def load_empty_dictionary(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001240 self.append({})
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001241 dispatch[EMPTY_DICT[0]] = load_empty_dictionary
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001242
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001243 def load_empty_set(self):
1244 self.append(set())
1245 dispatch[EMPTY_SET[0]] = load_empty_set
1246
1247 def load_frozenset(self):
1248 k = self.marker()
1249 self.stack[k:] = [frozenset(self.stack[k+1:])]
1250 dispatch[FROZENSET[0]] = load_frozenset
1251
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001252 def load_list(self):
1253 k = self.marker()
1254 self.stack[k:] = [self.stack[k+1:]]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001255 dispatch[LIST[0]] = load_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001256
1257 def load_dict(self):
1258 k = self.marker()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001259 items = self.stack[k+1:]
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001260 d = {items[i]: items[i+1]
1261 for i in range(0, len(items), 2)}
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001262 self.stack[k:] = [d]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001263 dispatch[DICT[0]] = load_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001264
Tim Petersd01c1e92003-01-30 15:41:46 +00001265 # INST and OBJ differ only in how they get a class object. It's not
1266 # only sensible to do the rest in a common routine, the two routines
1267 # previously diverged and grew different bugs.
1268 # klass is the class to instantiate, and k points to the topmost mark
1269 # object, following which are the arguments for klass.__init__.
1270 def _instantiate(self, klass, k):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001271 args = tuple(self.stack[k+1:])
1272 del self.stack[k:]
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001273 if (args or not isinstance(klass, type) or
1274 hasattr(klass, "__getinitargs__")):
Guido van Rossum743d17e1998-09-15 20:25:57 +00001275 try:
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001276 value = klass(*args)
Guido van Rossumb940e112007-01-10 16:19:56 +00001277 except TypeError as err:
Guido van Rossum26d95c32007-08-27 23:18:54 +00001278 raise TypeError("in constructor for %s: %s" %
1279 (klass.__name__, str(err)), sys.exc_info()[2])
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001280 else:
1281 value = klass.__new__(klass)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001282 self.append(value)
Tim Petersd01c1e92003-01-30 15:41:46 +00001283
1284 def load_inst(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001285 module = self.readline()[:-1].decode("ascii")
1286 name = self.readline()[:-1].decode("ascii")
Tim Petersd01c1e92003-01-30 15:41:46 +00001287 klass = self.find_class(module, name)
1288 self._instantiate(klass, self.marker())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001289 dispatch[INST[0]] = load_inst
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001290
1291 def load_obj(self):
Tim Petersd01c1e92003-01-30 15:41:46 +00001292 # Stack is ... markobject classobject arg1 arg2 ...
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001293 k = self.marker()
Tim Petersd01c1e92003-01-30 15:41:46 +00001294 klass = self.stack.pop(k+1)
1295 self._instantiate(klass, k)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001296 dispatch[OBJ[0]] = load_obj
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001297
Guido van Rossum3a41c612003-01-28 15:10:22 +00001298 def load_newobj(self):
1299 args = self.stack.pop()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001300 cls = self.stack.pop()
Guido van Rossum3a41c612003-01-28 15:10:22 +00001301 obj = cls.__new__(cls, *args)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001302 self.append(obj)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001303 dispatch[NEWOBJ[0]] = load_newobj
Guido van Rossum3a41c612003-01-28 15:10:22 +00001304
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001305 def load_newobj_ex(self):
1306 kwargs = self.stack.pop()
1307 args = self.stack.pop()
1308 cls = self.stack.pop()
1309 obj = cls.__new__(cls, *args, **kwargs)
1310 self.append(obj)
1311 dispatch[NEWOBJ_EX[0]] = load_newobj_ex
1312
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001313 def load_global(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001314 module = self.readline()[:-1].decode("utf-8")
1315 name = self.readline()[:-1].decode("utf-8")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001316 klass = self.find_class(module, name)
1317 self.append(klass)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001318 dispatch[GLOBAL[0]] = load_global
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001319
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001320 def load_stack_global(self):
1321 name = self.stack.pop()
1322 module = self.stack.pop()
1323 if type(name) is not str or type(module) is not str:
1324 raise UnpicklingError("STACK_GLOBAL requires str")
1325 self.append(self.find_class(module, name))
1326 dispatch[STACK_GLOBAL[0]] = load_stack_global
1327
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001328 def load_ext1(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001329 code = self.read(1)[0]
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001330 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001331 dispatch[EXT1[0]] = load_ext1
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001332
1333 def load_ext2(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001334 code, = unpack('<H', self.read(2))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001335 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001336 dispatch[EXT2[0]] = load_ext2
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001337
1338 def load_ext4(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001339 code, = unpack('<i', self.read(4))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001340 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001341 dispatch[EXT4[0]] = load_ext4
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001342
1343 def get_extension(self, code):
1344 nil = []
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001345 obj = _extension_cache.get(code, nil)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001346 if obj is not nil:
1347 self.append(obj)
1348 return
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001349 key = _inverted_registry.get(code)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001350 if not key:
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001351 if code <= 0: # note that 0 is forbidden
1352 # Corrupt or hostile pickle.
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001353 raise UnpicklingError("EXT specifies code <= 0")
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001354 raise ValueError("unregistered extension code %d" % code)
1355 obj = self.find_class(*key)
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001356 _extension_cache[code] = obj
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001357 self.append(obj)
1358
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001359 def find_class(self, module, name):
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001360 # Subclasses may override this.
1361 if self.proto < 3 and self.fix_imports:
1362 if (module, name) in _compat_pickle.NAME_MAPPING:
1363 module, name = _compat_pickle.NAME_MAPPING[(module, name)]
1364 if module in _compat_pickle.IMPORT_MAPPING:
1365 module = _compat_pickle.IMPORT_MAPPING[module]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001366 __import__(module, level=0)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001367 return _getattribute(sys.modules[module], name,
1368 allow_qualname=self.proto >= 4)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001369
1370 def load_reduce(self):
1371 stack = self.stack
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001372 args = stack.pop()
1373 func = stack[-1]
Guido van Rossum99603b02007-07-20 00:22:32 +00001374 try:
1375 value = func(*args)
1376 except:
1377 print(sys.exc_info())
1378 print(func, args)
1379 raise
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001380 stack[-1] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001381 dispatch[REDUCE[0]] = load_reduce
Guido van Rossuma48061a1995-01-10 00:31:14 +00001382
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001383 def load_pop(self):
1384 del self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001385 dispatch[POP[0]] = load_pop
Guido van Rossum7b5430f1995-03-04 22:25:21 +00001386
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001387 def load_pop_mark(self):
1388 k = self.marker()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001389 del self.stack[k:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001390 dispatch[POP_MARK[0]] = load_pop_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001391
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001392 def load_dup(self):
Guido van Rossumb1062fc1998-03-31 17:00:46 +00001393 self.append(self.stack[-1])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001394 dispatch[DUP[0]] = load_dup
Guido van Rossuma48061a1995-01-10 00:31:14 +00001395
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001396 def load_get(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001397 i = int(self.readline()[:-1])
1398 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001399 dispatch[GET[0]] = load_get
Guido van Rossum78536471996-04-12 13:36:27 +00001400
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001401 def load_binget(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001402 i = self.read(1)[0]
1403 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001404 dispatch[BINGET[0]] = load_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001405
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001406 def load_long_binget(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001407 i, = unpack('<I', self.read(4))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001408 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001409 dispatch[LONG_BINGET[0]] = load_long_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001410
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001411 def load_put(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001412 i = int(self.readline()[:-1])
Antoine Pitrou55549ec2011-08-30 00:27:10 +02001413 if i < 0:
1414 raise ValueError("negative PUT argument")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001415 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001416 dispatch[PUT[0]] = load_put
Guido van Rossuma48061a1995-01-10 00:31:14 +00001417
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001418 def load_binput(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001419 i = self.read(1)[0]
Antoine Pitrou55549ec2011-08-30 00:27:10 +02001420 if i < 0:
1421 raise ValueError("negative BINPUT argument")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001422 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001423 dispatch[BINPUT[0]] = load_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001424
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001425 def load_long_binput(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001426 i, = unpack('<I', self.read(4))
1427 if i > maxsize:
Antoine Pitrou55549ec2011-08-30 00:27:10 +02001428 raise ValueError("negative LONG_BINPUT argument")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001429 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001430 dispatch[LONG_BINPUT[0]] = load_long_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001431
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001432 def load_memoize(self):
1433 memo = self.memo
1434 memo[len(memo)] = self.stack[-1]
1435 dispatch[MEMOIZE[0]] = load_memoize
1436
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001437 def load_append(self):
1438 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001439 value = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001440 list = stack[-1]
1441 list.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001442 dispatch[APPEND[0]] = load_append
Guido van Rossuma48061a1995-01-10 00:31:14 +00001443
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001444 def load_appends(self):
1445 stack = self.stack
1446 mark = self.marker()
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07001447 list_obj = stack[mark - 1]
1448 items = stack[mark + 1:]
1449 if isinstance(list_obj, list):
1450 list_obj.extend(items)
1451 else:
1452 append = list_obj.append
1453 for item in items:
1454 append(item)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001455 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001456 dispatch[APPENDS[0]] = load_appends
Tim Peters2344fae2001-01-15 00:50:52 +00001457
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001458 def load_setitem(self):
1459 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001460 value = stack.pop()
1461 key = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001462 dict = stack[-1]
1463 dict[key] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001464 dispatch[SETITEM[0]] = load_setitem
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001465
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001466 def load_setitems(self):
1467 stack = self.stack
1468 mark = self.marker()
1469 dict = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001470 for i in range(mark + 1, len(stack), 2):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001471 dict[stack[i]] = stack[i + 1]
Guido van Rossuma48061a1995-01-10 00:31:14 +00001472
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001473 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001474 dispatch[SETITEMS[0]] = load_setitems
Guido van Rossuma48061a1995-01-10 00:31:14 +00001475
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001476 def load_additems(self):
1477 stack = self.stack
1478 mark = self.marker()
1479 set_obj = stack[mark - 1]
1480 items = stack[mark + 1:]
1481 if isinstance(set_obj, set):
1482 set_obj.update(items)
1483 else:
1484 add = set_obj.add
1485 for item in items:
1486 add(item)
1487 del stack[mark:]
1488 dispatch[ADDITEMS[0]] = load_additems
1489
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001490 def load_build(self):
1491 stack = self.stack
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001492 state = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001493 inst = stack[-1]
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001494 setstate = getattr(inst, "__setstate__", None)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001495 if setstate is not None:
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001496 setstate(state)
1497 return
1498 slotstate = None
1499 if isinstance(state, tuple) and len(state) == 2:
1500 state, slotstate = state
1501 if state:
Alexandre Vassalottiebfecfd2009-05-25 18:50:33 +00001502 inst_dict = inst.__dict__
Antoine Pitroua9f48a02009-05-02 21:41:14 +00001503 intern = sys.intern
Alexandre Vassalottiebfecfd2009-05-25 18:50:33 +00001504 for k, v in state.items():
1505 if type(k) is str:
1506 inst_dict[intern(k)] = v
1507 else:
1508 inst_dict[k] = v
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001509 if slotstate:
1510 for k, v in slotstate.items():
1511 setattr(inst, k, v)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001512 dispatch[BUILD[0]] = load_build
Guido van Rossuma48061a1995-01-10 00:31:14 +00001513
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001514 def load_mark(self):
1515 self.append(self.mark)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001516 dispatch[MARK[0]] = load_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001517
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001518 def load_stop(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001519 value = self.stack.pop()
Guido van Rossumff871742000-12-13 18:11:56 +00001520 raise _Stop(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001521 dispatch[STOP[0]] = load_stop
Guido van Rossuma48061a1995-01-10 00:31:14 +00001522
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001523
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001524# Shorthands
1525
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001526def _dump(obj, file, protocol=None, *, fix_imports=True):
1527 _Pickler(file, protocol, fix_imports=fix_imports).dump(obj)
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001528
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001529def _dumps(obj, protocol=None, *, fix_imports=True):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001530 f = io.BytesIO()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001531 _Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001532 res = f.getvalue()
Guido van Rossum98297ee2007-11-06 21:34:58 +00001533 assert isinstance(res, bytes_types)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001534 return res
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001535
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001536def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict"):
1537 return _Unpickler(file, fix_imports=fix_imports,
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001538 encoding=encoding, errors=errors).load()
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001539
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001540def _loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001541 if isinstance(s, str):
1542 raise TypeError("Can't load pickle from unicode string")
1543 file = io.BytesIO(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001544 return _Unpickler(file, fix_imports=fix_imports,
1545 encoding=encoding, errors=errors).load()
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001546
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001547# Use the faster _pickle if possible
1548try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001549 from _pickle import (
1550 PickleError,
1551 PicklingError,
1552 UnpicklingError,
1553 Pickler,
1554 Unpickler,
1555 dump,
1556 dumps,
1557 load,
1558 loads
1559 )
Brett Cannoncd171c82013-07-04 17:43:24 -04001560except ImportError:
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001561 Pickler, Unpickler = _Pickler, _Unpickler
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001562 dump, dumps, load, loads = _dump, _dumps, _load, _loads
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001563
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001564# Doctest
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001565def _test():
1566 import doctest
1567 return doctest.testmod()
1568
1569if __name__ == "__main__":
Florent Xicluna54540ec2011-11-04 08:29:17 +01001570 import argparse
Alexander Belopolsky455f7bd2010-07-27 23:02:38 +00001571 parser = argparse.ArgumentParser(
1572 description='display contents of the pickle files')
1573 parser.add_argument(
1574 'pickle_file', type=argparse.FileType('br'),
1575 nargs='*', help='the pickle file')
1576 parser.add_argument(
1577 '-t', '--test', action='store_true',
1578 help='run self-test suite')
1579 parser.add_argument(
1580 '-v', action='store_true',
1581 help='run verbosely; only affects self-test run')
1582 args = parser.parse_args()
1583 if args.test:
1584 _test()
1585 else:
1586 if not args.pickle_file:
1587 parser.print_help()
1588 else:
1589 import pprint
1590 for f in args.pickle_file:
1591 obj = load(f)
1592 pprint.pprint(obj)