blob: 7760425e76d631af828de543c0a2d8dac2646669 [file] [log] [blame]
Guido van Rossum54f22ed2000-02-04 15:10:34 +00001"""Create portable serialized representations of Python objects.
Guido van Rossuma48061a1995-01-10 00:31:14 +00002
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00003See module copyreg for a mechanism for registering custom picklers.
Tim Peters22a449a2003-01-27 20:16:36 +00004See module pickletools source for extensive comments.
Guido van Rossuma48061a1995-01-10 00:31:14 +00005
Guido van Rossume467be61997-12-05 19:42:42 +00006Classes:
Guido van Rossuma48061a1995-01-10 00:31:14 +00007
Guido van Rossume467be61997-12-05 19:42:42 +00008 Pickler
9 Unpickler
Guido van Rossuma48061a1995-01-10 00:31:14 +000010
Guido van Rossume467be61997-12-05 19:42:42 +000011Functions:
Guido van Rossuma48061a1995-01-10 00:31:14 +000012
Guido van Rossume467be61997-12-05 19:42:42 +000013 dump(object, file)
14 dumps(object) -> string
15 load(file) -> object
16 loads(string) -> object
Guido van Rossuma48061a1995-01-10 00:31:14 +000017
Guido van Rossume467be61997-12-05 19:42:42 +000018Misc variables:
Guido van Rossuma48061a1995-01-10 00:31:14 +000019
Fred Drakefe82acc1998-02-13 03:24:48 +000020 __version__
Guido van Rossume467be61997-12-05 19:42:42 +000021 format_version
22 compatible_formats
Guido van Rossuma48061a1995-01-10 00:31:14 +000023
Guido van Rossuma48061a1995-01-10 00:31:14 +000024"""
25
Victor Stinner7fa767e2014-03-20 09:16:38 +010026from types import FunctionType
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000027from copyreg import dispatch_table
28from copyreg import _extension_registry, _inverted_registry, _extension_cache
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +030029from itertools import islice
Guido van Rossumd3703791998-10-22 20:15:36 +000030import sys
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +030031from sys import maxsize
32from struct import pack, unpack
Skip Montanaro23bafc62001-02-18 03:10:09 +000033import re
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000034import io
Walter Dörwald42748a82007-06-12 16:40:17 +000035import codecs
Antoine Pitroud9dfaa92009-06-04 20:32:06 +000036import _compat_pickle
Guido van Rossuma48061a1995-01-10 00:31:14 +000037
Skip Montanaro352674d2001-02-07 23:14:30 +000038__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
39 "Unpickler", "dump", "dumps", "load", "loads"]
40
Guido van Rossum98297ee2007-11-06 21:34:58 +000041# Shortcut for use in isinstance testing
Alexandre Vassalotti8cb02b62008-05-03 01:42:49 +000042bytes_types = (bytes, bytearray)
Guido van Rossum98297ee2007-11-06 21:34:58 +000043
Tim Petersc0c12b52003-01-29 00:56:17 +000044# These are purely informational; no code uses these.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010045format_version = "4.0" # File format version we write
Guido van Rossumf29d3d62003-01-27 22:47:53 +000046compatible_formats = ["1.0", # Original protocol 0
Guido van Rossumbc64e222003-01-28 16:34:19 +000047 "1.1", # Protocol 0 with INST added
Guido van Rossumf29d3d62003-01-27 22:47:53 +000048 "1.2", # Original protocol 1
49 "1.3", # Protocol 1 with BINFLOAT added
50 "2.0", # Protocol 2
Guido van Rossumf4169812008-03-17 22:56:06 +000051 "3.0", # Protocol 3
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010052 "4.0", # Protocol 4
Guido van Rossumf29d3d62003-01-27 22:47:53 +000053 ] # Old format versions we can read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000054
Guido van Rossum99603b02007-07-20 00:22:32 +000055# This is the highest protocol number we know how to read.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010056HIGHEST_PROTOCOL = 4
Tim Peters8587b3c2003-02-13 15:44:41 +000057
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000058# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
Guido van Rossumf4169812008-03-17 22:56:06 +000059# We intentionally write a protocol that Python 2.x cannot read;
60# there are too many issues with that.
61DEFAULT_PROTOCOL = 3
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000062
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000063class PickleError(Exception):
Neal Norwitzefbb67b2002-05-30 12:12:04 +000064 """A common base class for the other pickling exceptions."""
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000065 pass
66
67class PicklingError(PickleError):
68 """This exception is raised when an unpicklable object is passed to the
69 dump() method.
70
71 """
72 pass
73
74class UnpicklingError(PickleError):
75 """This exception is raised when there is a problem unpickling an object,
76 such as a security violation.
77
78 Note that other exceptions may also be raised during unpickling, including
79 (but not necessarily limited to) AttributeError, EOFError, ImportError,
80 and IndexError.
81
82 """
83 pass
Guido van Rossum7849da81995-03-09 14:08:35 +000084
Tim Petersc0c12b52003-01-29 00:56:17 +000085# An instance of _Stop is raised by Unpickler.load_stop() in response to
86# the STOP opcode, passing the object that is the result of unpickling.
Guido van Rossumff871742000-12-13 18:11:56 +000087class _Stop(Exception):
88 def __init__(self, value):
89 self.value = value
90
Guido van Rossum533dbcf2003-01-28 17:55:05 +000091# Jython has PyStringMap; it's a dict subclass with string keys
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000092try:
93 from org.python.core import PyStringMap
Brett Cannoncd171c82013-07-04 17:43:24 -040094except ImportError:
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000095 PyStringMap = None
96
Tim Peters22a449a2003-01-27 20:16:36 +000097# Pickle opcodes. See pickletools.py for extensive docs. The listing
98# here is in kind-of alphabetical order of 1-character pickle code.
99# pickletools groups them by purpose.
Guido van Rossumdbb718f2001-09-21 19:22:34 +0000100
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000101MARK = b'(' # push special markobject on stack
102STOP = b'.' # every pickle ends with STOP
103POP = b'0' # discard topmost stack item
104POP_MARK = b'1' # discard stack top through topmost markobject
105DUP = b'2' # duplicate top stack item
106FLOAT = b'F' # push float object; decimal string argument
107INT = b'I' # push integer or bool; decimal string argument
108BININT = b'J' # push four-byte signed int
109BININT1 = b'K' # push 1-byte unsigned int
110LONG = b'L' # push long; decimal string argument
111BININT2 = b'M' # push 2-byte unsigned int
112NONE = b'N' # push None
113PERSID = b'P' # push persistent object; id is taken from string arg
114BINPERSID = b'Q' # " " " ; " " " " stack
115REDUCE = b'R' # apply callable to argtuple, both on stack
116STRING = b'S' # push string; NL-terminated string argument
117BINSTRING = b'T' # push string; counted binary string argument
118SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
119UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
120BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
121APPEND = b'a' # append stack top to list below it
122BUILD = b'b' # call __setstate__ or __dict__.update()
123GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
124DICT = b'd' # build a dict from stack items
125EMPTY_DICT = b'}' # push empty dict
126APPENDS = b'e' # extend list on stack by topmost stack slice
127GET = b'g' # push item from memo on stack; index is string arg
128BINGET = b'h' # " " " " " " ; " " 1-byte arg
129INST = b'i' # build & push class instance
130LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
131LIST = b'l' # build list from topmost stack items
132EMPTY_LIST = b']' # push empty list
133OBJ = b'o' # build & push class instance
134PUT = b'p' # store stack top in memo; index is string arg
135BINPUT = b'q' # " " " " " ; " " 1-byte arg
136LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
137SETITEM = b's' # add key+value pair to dict
138TUPLE = b't' # build tuple from topmost stack items
139EMPTY_TUPLE = b')' # push empty tuple
140SETITEMS = b'u' # modify dict by adding topmost key+value pairs
141BINFLOAT = b'G' # push float; arg is 8-byte float encoding
Tim Peters22a449a2003-01-27 20:16:36 +0000142
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000143TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
144FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
Guido van Rossum77f6a652002-04-03 22:41:51 +0000145
Guido van Rossum586c9e82003-01-29 06:16:12 +0000146# Protocol 2
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000147
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000148PROTO = b'\x80' # identify pickle protocol
149NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
150EXT1 = b'\x82' # push object from extension registry; 1-byte index
151EXT2 = b'\x83' # ditto, but 2-byte index
152EXT4 = b'\x84' # ditto, but 4-byte index
153TUPLE1 = b'\x85' # build 1-tuple from stack top
154TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
155TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
156NEWTRUE = b'\x88' # push True
157NEWFALSE = b'\x89' # push False
158LONG1 = b'\x8a' # push long from < 256 bytes
159LONG4 = b'\x8b' # push really big long
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000160
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000161_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
162
Guido van Rossumf4169812008-03-17 22:56:06 +0000163# Protocol 3 (Python 3.x)
164
165BINBYTES = b'B' # push bytes; counted binary string argument
166SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
Guido van Rossuma48061a1995-01-10 00:31:14 +0000167
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100168# Protocol 4
169SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
170BINUNICODE8 = b'\x8d' # push very long string
171BINBYTES8 = b'\x8e' # push very long bytes string
172EMPTY_SET = b'\x8f' # push empty set on the stack
173ADDITEMS = b'\x90' # modify set by adding topmost stack items
174FROZENSET = b'\x91' # build frozenset from topmost stack items
175NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
176STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
177MEMOIZE = b'\x94' # store top of the stack in memo
178FRAME = b'\x95' # indicate the beginning of a new frame
179
180__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
181
182
183class _Framer:
184
185 _FRAME_SIZE_TARGET = 64 * 1024
186
187 def __init__(self, file_write):
188 self.file_write = file_write
189 self.current_frame = None
190
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100191 def start_framing(self):
192 self.current_frame = io.BytesIO()
193
194 def end_framing(self):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800195 if self.current_frame and self.current_frame.tell() > 0:
196 self.commit_frame(force=True)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100197 self.current_frame = None
198
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800199 def commit_frame(self, force=False):
200 if self.current_frame:
201 f = self.current_frame
202 if f.tell() >= self._FRAME_SIZE_TARGET or force:
203 with f.getbuffer() as data:
204 n = len(data)
205 write = self.file_write
206 write(FRAME)
207 write(pack("<Q", n))
208 write(data)
209 f.seek(0)
210 f.truncate()
211
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100212 def write(self, data):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800213 if self.current_frame:
214 return self.current_frame.write(data)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100215 else:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800216 return self.file_write(data)
217
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100218
219class _Unframer:
220
221 def __init__(self, file_read, file_readline, file_tell=None):
222 self.file_read = file_read
223 self.file_readline = file_readline
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100224 self.current_frame = None
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100225
226 def read(self, n):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800227 if self.current_frame:
228 data = self.current_frame.read(n)
229 if not data and n != 0:
230 self.current_frame = None
231 return self.file_read(n)
232 if len(data) < n:
233 raise UnpicklingError(
234 "pickle exhausted before end of frame")
235 return data
236 else:
237 return self.file_read(n)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100238
239 def readline(self):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800240 if self.current_frame:
241 data = self.current_frame.readline()
242 if not data:
243 self.current_frame = None
244 return self.file_readline()
Serhiy Storchaka21d75332015-01-26 10:37:01 +0200245 if data[-1] != b'\n'[0]:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800246 raise UnpicklingError(
247 "pickle exhausted before end of frame")
248 return data
249 else:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100250 return self.file_readline()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100251
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800252 def load_frame(self, frame_size):
253 if self.current_frame and self.current_frame.read() != b'':
254 raise UnpicklingError(
255 "beginning of a new frame before end of current frame")
256 self.current_frame = io.BytesIO(self.file_read(frame_size))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100257
258
259# Tools used for pickling.
260
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300261def _getattribute(obj, name):
262 for subpath in name.split('.'):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100263 if subpath == '<locals>':
264 raise AttributeError("Can't get local attribute {!r} on {!r}"
265 .format(name, obj))
266 try:
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300267 parent = obj
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100268 obj = getattr(obj, subpath)
269 except AttributeError:
270 raise AttributeError("Can't get attribute {!r} on {!r}"
271 .format(name, obj))
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300272 return obj, parent
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100273
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300274def whichmodule(obj, name):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100275 """Find the module an object belong to."""
276 module_name = getattr(obj, '__module__', None)
277 if module_name is not None:
278 return module_name
Antoine Pitroue1618492014-10-04 22:15:27 +0200279 # Protect the iteration by using a list copy of sys.modules against dynamic
280 # modules that trigger imports of other modules upon calls to getattr.
281 for module_name, module in list(sys.modules.items()):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100282 if module_name == '__main__' or module is None:
283 continue
284 try:
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300285 if _getattribute(module, name)[0] is obj:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100286 return module_name
287 except AttributeError:
288 pass
289 return '__main__'
290
291def encode_long(x):
292 r"""Encode a long to a two's complement little-endian binary string.
293 Note that 0 is a special case, returning an empty string, to save a
294 byte in the LONG1 pickling context.
295
296 >>> encode_long(0)
297 b''
298 >>> encode_long(255)
299 b'\xff\x00'
300 >>> encode_long(32767)
301 b'\xff\x7f'
302 >>> encode_long(-256)
303 b'\x00\xff'
304 >>> encode_long(-32768)
305 b'\x00\x80'
306 >>> encode_long(-128)
307 b'\x80'
308 >>> encode_long(127)
309 b'\x7f'
310 >>>
311 """
312 if x == 0:
313 return b''
314 nbytes = (x.bit_length() >> 3) + 1
315 result = x.to_bytes(nbytes, byteorder='little', signed=True)
316 if x < 0 and nbytes > 1:
317 if result[-1] == 0xff and (result[-2] & 0x80) != 0:
318 result = result[:-1]
319 return result
320
321def decode_long(data):
322 r"""Decode a long from a two's complement little-endian binary string.
323
324 >>> decode_long(b'')
325 0
326 >>> decode_long(b"\xff\x00")
327 255
328 >>> decode_long(b"\xff\x7f")
329 32767
330 >>> decode_long(b"\x00\xff")
331 -256
332 >>> decode_long(b"\x00\x80")
333 -32768
334 >>> decode_long(b"\x80")
335 -128
336 >>> decode_long(b"\x7f")
337 127
338 """
339 return int.from_bytes(data, byteorder='little', signed=True)
340
Skip Montanaro23bafc62001-02-18 03:10:09 +0000341
Guido van Rossum1be31752003-01-28 15:19:53 +0000342# Pickling machinery
343
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000344class _Pickler:
Guido van Rossuma48061a1995-01-10 00:31:14 +0000345
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000346 def __init__(self, file, protocol=None, *, fix_imports=True):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000347 """This takes a binary file for writing a pickle data stream.
348
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800349 The optional *protocol* argument tells the pickler to use the
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100350 given protocol; supported protocols are 0, 1, 2, 3 and 4. The
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800351 default protocol is 3; a backward-incompatible protocol designed
352 for Python 3.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000353
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000354 Specifying a negative protocol version selects the highest
Tim Peters5bd2a792003-02-01 16:45:06 +0000355 protocol version supported. The higher the protocol used, the
356 more recent the version of Python needed to read the pickle
357 produced.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000358
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800359 The *file* argument must have a write() method that accepts a
360 single bytes argument. It can thus be a file object opened for
Martin Panter7462b6492015-11-02 03:37:02 +0000361 binary writing, an io.BytesIO instance, or any other custom
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800362 object that meets this interface.
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000363
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800364 If *fix_imports* is True and *protocol* is less than 3, pickle
365 will try to map the new Python 3 names to the old module names
366 used in Python 2, so that the pickle data stream is readable
367 with Python 2.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000368 """
Guido van Rossumcf117b02003-02-09 17:19:41 +0000369 if protocol is None:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000370 protocol = DEFAULT_PROTOCOL
Guido van Rossumcf117b02003-02-09 17:19:41 +0000371 if protocol < 0:
Tim Peters8587b3c2003-02-13 15:44:41 +0000372 protocol = HIGHEST_PROTOCOL
373 elif not 0 <= protocol <= HIGHEST_PROTOCOL:
374 raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000375 try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100376 self._file_write = file.write
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000377 except AttributeError:
378 raise TypeError("file must have a 'write' attribute")
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800379 self.framer = _Framer(self._file_write)
380 self.write = self.framer.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000381 self.memo = {}
Guido van Rossumcf117b02003-02-09 17:19:41 +0000382 self.proto = int(protocol)
383 self.bin = protocol >= 1
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000384 self.fast = 0
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000385 self.fix_imports = fix_imports and protocol < 3
Guido van Rossuma48061a1995-01-10 00:31:14 +0000386
Fred Drake7f781c92002-05-01 20:33:53 +0000387 def clear_memo(self):
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000388 """Clears the pickler's "memo".
389
390 The memo is the data structure that remembers which objects the
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800391 pickler has already seen, so that shared or recursive objects
392 are pickled by reference and not by value. This method is
393 useful when re-using picklers.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000394 """
Fred Drake7f781c92002-05-01 20:33:53 +0000395 self.memo.clear()
396
Guido van Rossum3a41c612003-01-28 15:10:22 +0000397 def dump(self, obj):
Tim Peters5bd2a792003-02-01 16:45:06 +0000398 """Write a pickled representation of obj to the open file."""
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +0000399 # Check whether Pickler was initialized correctly. This is
400 # only needed to mimic the behavior of _pickle.Pickler.dump().
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100401 if not hasattr(self, "_file_write"):
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +0000402 raise PicklingError("Pickler.__init__() was not called by "
403 "%s.__init__()" % (self.__class__.__name__,))
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000404 if self.proto >= 2:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800405 self.write(PROTO + pack("<B", self.proto))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100406 if self.proto >= 4:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800407 self.framer.start_framing()
Guido van Rossum3a41c612003-01-28 15:10:22 +0000408 self.save(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000409 self.write(STOP)
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800410 self.framer.end_framing()
Guido van Rossuma48061a1995-01-10 00:31:14 +0000411
Jeremy Hylton3422c992003-01-24 19:29:52 +0000412 def memoize(self, obj):
413 """Store an object in the memo."""
414
Tim Peterse46b73f2003-01-27 21:22:10 +0000415 # The Pickler memo is a dictionary mapping object ids to 2-tuples
416 # that contain the Unpickler memo key and the object being memoized.
417 # The memo key is written to the pickle and will become
Jeremy Hylton3422c992003-01-24 19:29:52 +0000418 # the key in the Unpickler's memo. The object is stored in the
Tim Peterse46b73f2003-01-27 21:22:10 +0000419 # Pickler memo so that transient objects are kept alive during
420 # pickling.
Jeremy Hylton3422c992003-01-24 19:29:52 +0000421
Tim Peterse46b73f2003-01-27 21:22:10 +0000422 # The use of the Unpickler memo length as the memo key is just a
423 # convention. The only requirement is that the memo values be unique.
424 # But there appears no advantage to any other scheme, and this
Tim Peterscbd0a322003-01-28 00:24:43 +0000425 # scheme allows the Unpickler memo to be implemented as a plain (but
Tim Peterse46b73f2003-01-27 21:22:10 +0000426 # growable) array, indexed by memo key.
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000427 if self.fast:
428 return
Guido van Rossum9b40e802003-01-30 06:37:41 +0000429 assert id(obj) not in self.memo
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100430 idx = len(self.memo)
431 self.write(self.put(idx))
432 self.memo[id(obj)] = idx, obj
Jeremy Hylton3422c992003-01-24 19:29:52 +0000433
Tim Petersbb38e302003-01-27 21:25:41 +0000434 # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100435 def put(self, idx):
436 if self.proto >= 4:
437 return MEMOIZE
438 elif self.bin:
439 if idx < 256:
440 return BINPUT + pack("<B", idx)
Guido van Rossum5c938d02003-01-28 03:03:08 +0000441 else:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100442 return LONG_BINPUT + pack("<I", idx)
443 else:
444 return PUT + repr(idx).encode("ascii") + b'\n'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000445
Tim Petersbb38e302003-01-27 21:25:41 +0000446 # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300447 def get(self, i):
Tim Petersc32d8242001-04-10 02:48:53 +0000448 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000449 if i < 256:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300450 return BINGET + pack("<B", i)
Guido van Rossum5c938d02003-01-28 03:03:08 +0000451 else:
Antoine Pitroubf6ecf92012-11-24 20:40:21 +0100452 return LONG_BINGET + pack("<I", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000453
Guido van Rossum39478e82007-08-27 17:23:59 +0000454 return GET + repr(i).encode("ascii") + b'\n'
Tim Peters2344fae2001-01-15 00:50:52 +0000455
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000456 def save(self, obj, save_persistent_id=True):
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -0800457 self.framer.commit_frame()
458
Guido van Rossumbc64e222003-01-28 16:34:19 +0000459 # Check for persistent id (defined by a subclass)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000460 pid = self.persistent_id(obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000461 if pid is not None and save_persistent_id:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000462 self.save_pers(pid)
463 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000464
Guido van Rossumbc64e222003-01-28 16:34:19 +0000465 # Check the memo
466 x = self.memo.get(id(obj))
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300467 if x is not None:
Guido van Rossumbc64e222003-01-28 16:34:19 +0000468 self.write(self.get(x[0]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000469 return
470
Guido van Rossumbc64e222003-01-28 16:34:19 +0000471 # Check the type dispatch table
Guido van Rossum3a41c612003-01-28 15:10:22 +0000472 t = type(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000473 f = self.dispatch.get(t)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300474 if f is not None:
Guido van Rossumbc64e222003-01-28 16:34:19 +0000475 f(self, obj) # Call unbound method with explicit self
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000476 return
477
Antoine Pitrou8d3c2902012-03-04 18:31:48 +0100478 # Check private dispatch table if any, or else copyreg.dispatch_table
479 reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300480 if reduce is not None:
Guido van Rossumc53f0092003-02-18 22:05:12 +0000481 rv = reduce(obj)
482 else:
Antoine Pitrouffd41d92011-10-04 09:23:04 +0200483 # Check for a class with a custom metaclass; treat as regular class
484 try:
485 issc = issubclass(t, type)
486 except TypeError: # t is not a class (old Boost; see SF #502085)
487 issc = False
488 if issc:
489 self.save_global(obj)
490 return
491
Guido van Rossumc53f0092003-02-18 22:05:12 +0000492 # Check for a __reduce_ex__ method, fall back to __reduce__
493 reduce = getattr(obj, "__reduce_ex__", None)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300494 if reduce is not None:
Guido van Rossumc53f0092003-02-18 22:05:12 +0000495 rv = reduce(self.proto)
496 else:
497 reduce = getattr(obj, "__reduce__", None)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300498 if reduce is not None:
Guido van Rossumc53f0092003-02-18 22:05:12 +0000499 rv = reduce()
500 else:
501 raise PicklingError("Can't pickle %r object: %r" %
502 (t.__name__, obj))
Tim Petersb32a8312003-01-28 00:48:09 +0000503
Guido van Rossumbc64e222003-01-28 16:34:19 +0000504 # Check for string returned by reduce(), meaning "save as global"
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000505 if isinstance(rv, str):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000506 self.save_global(obj, rv)
Tim Petersb32a8312003-01-28 00:48:09 +0000507 return
508
Guido van Rossumbc64e222003-01-28 16:34:19 +0000509 # Assert that reduce() returned a tuple
Guido van Rossum13257902007-06-07 23:15:56 +0000510 if not isinstance(rv, tuple):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000511 raise PicklingError("%s must return string or tuple" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000512
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000513 # Assert that it returned an appropriately sized tuple
Guido van Rossumbc64e222003-01-28 16:34:19 +0000514 l = len(rv)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000515 if not (2 <= l <= 5):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000516 raise PicklingError("Tuple returned by %s must have "
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000517 "two to five elements" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000518
Guido van Rossumbc64e222003-01-28 16:34:19 +0000519 # Save the reduce() output and finally memoize the object
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000520 self.save_reduce(obj=obj, *rv)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000521
Guido van Rossum3a41c612003-01-28 15:10:22 +0000522 def persistent_id(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000523 # This exists so a subclass can override it
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000524 return None
525
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000526 def save_pers(self, pid):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000527 # Save a persistent id reference
Tim Petersbd1cdb92003-01-28 01:03:10 +0000528 if self.bin:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000529 self.save(pid, save_persistent_id=False)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000530 self.write(BINPERSID)
Tim Petersbd1cdb92003-01-28 01:03:10 +0000531 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000532 self.write(PERSID + str(pid).encode("ascii") + b'\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000533
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100534 def save_reduce(self, func, args, state=None, listitems=None,
535 dictitems=None, obj=None):
Jeremy Hyltone3a565e2003-06-29 16:59:59 +0000536 # This API is called by some subclasses
Guido van Rossumbc64e222003-01-28 16:34:19 +0000537
Guido van Rossum13257902007-06-07 23:15:56 +0000538 if not isinstance(args, tuple):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100539 raise PicklingError("args from save_reduce() must be a tuple")
Florent Xicluna5d1155c2011-10-28 14:45:05 +0200540 if not callable(func):
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100541 raise PicklingError("func from save_reduce() must be callable")
Guido van Rossumbc64e222003-01-28 16:34:19 +0000542
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000543 save = self.save
Guido van Rossumbc64e222003-01-28 16:34:19 +0000544 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000545
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100546 func_name = getattr(func, "__name__", "")
547 if self.proto >= 4 and func_name == "__newobj_ex__":
548 cls, args, kwargs = args
549 if not hasattr(cls, "__new__"):
550 raise PicklingError("args[0] from {} args has no __new__"
551 .format(func_name))
552 if obj is not None and cls is not obj.__class__:
553 raise PicklingError("args[0] from {} args has the wrong class"
554 .format(func_name))
555 save(cls)
556 save(args)
557 save(kwargs)
558 write(NEWOBJ_EX)
559 elif self.proto >= 2 and func_name == "__newobj__":
560 # A __reduce__ implementation can direct protocol 2 or newer to
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000561 # use the more efficient NEWOBJ opcode, while still
562 # allowing protocol 0 and 1 to work normally. For this to
563 # work, the function returned by __reduce__ should be
564 # called __newobj__, and its first argument should be a
Florent Xiclunaaa6c1d22011-12-12 18:54:29 +0100565 # class. The implementation for __newobj__
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000566 # should be as follows, although pickle has no way to
567 # verify this:
568 #
569 # def __newobj__(cls, *args):
570 # return cls.__new__(cls, *args)
571 #
572 # Protocols 0 and 1 will pickle a reference to __newobj__,
573 # while protocol 2 (and above) will pickle a reference to
574 # cls, the remaining args tuple, and the NEWOBJ code,
575 # which calls cls.__new__(cls, *args) at unpickling time
576 # (see load_newobj below). If __reduce__ returns a
577 # three-tuple, the state from the third tuple item will be
578 # pickled regardless of the protocol, calling __setstate__
579 # at unpickling time (see load_build below).
580 #
581 # Note that no standard __newobj__ implementation exists;
582 # you have to provide your own. This is to enforce
583 # compatibility with Python 2.2 (pickles written using
584 # protocol 0 or 1 in Python 2.3 should be unpicklable by
585 # Python 2.2).
586 cls = args[0]
587 if not hasattr(cls, "__new__"):
588 raise PicklingError(
589 "args[0] from __newobj__ args has no __new__")
Guido van Rossumf7f45172003-01-31 17:17:49 +0000590 if obj is not None and cls is not obj.__class__:
591 raise PicklingError(
592 "args[0] from __newobj__ args has the wrong class")
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000593 args = args[1:]
594 save(cls)
595 save(args)
596 write(NEWOBJ)
597 else:
598 save(func)
599 save(args)
600 write(REDUCE)
Tim Peters2344fae2001-01-15 00:50:52 +0000601
Guido van Rossumf7f45172003-01-31 17:17:49 +0000602 if obj is not None:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100603 # If the object is already in the memo, this means it is
604 # recursive. In this case, throw away everything we put on the
605 # stack, and fetch the object back from the memo.
606 if id(obj) in self.memo:
607 write(POP + self.get(self.memo[id(obj)][0]))
608 else:
609 self.memoize(obj)
Guido van Rossumf7f45172003-01-31 17:17:49 +0000610
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000611 # More new special cases (that work with older protocols as
612 # well): when __reduce__ returns a tuple with 4 or 5 items,
613 # the 4th and 5th item should be iterators that provide list
614 # items and dict items (as (key, value) tuples), or None.
615
616 if listitems is not None:
617 self._batch_appends(listitems)
618
619 if dictitems is not None:
620 self._batch_setitems(dictitems)
621
Tim Petersc32d8242001-04-10 02:48:53 +0000622 if state is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000623 save(state)
624 write(BUILD)
625
Guido van Rossumbc64e222003-01-28 16:34:19 +0000626 # Methods below this point are dispatched through the dispatch table
627
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000628 dispatch = {}
629
Guido van Rossum3a41c612003-01-28 15:10:22 +0000630 def save_none(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000631 self.write(NONE)
Guido van Rossum13257902007-06-07 23:15:56 +0000632 dispatch[type(None)] = save_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000633
Guido van Rossum3a41c612003-01-28 15:10:22 +0000634 def save_bool(self, obj):
Guido van Rossum7d97d312003-01-28 04:25:27 +0000635 if self.proto >= 2:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300636 self.write(NEWTRUE if obj else NEWFALSE)
Guido van Rossum7d97d312003-01-28 04:25:27 +0000637 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300638 self.write(TRUE if obj else FALSE)
Guido van Rossum77f6a652002-04-03 22:41:51 +0000639 dispatch[bool] = save_bool
640
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300641 def save_long(self, obj):
Guido van Rossumddefaf32007-01-14 03:31:43 +0000642 if self.bin:
643 # If the int is small enough to fit in a signed 4-byte 2's-comp
644 # format, we can store it more efficiently than the general
645 # case.
646 # First one- and two-byte unsigned ints:
647 if obj >= 0:
648 if obj <= 0xff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300649 self.write(BININT1 + pack("<B", obj))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000650 return
651 if obj <= 0xffff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300652 self.write(BININT2 + pack("<H", obj))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000653 return
654 # Next check for 4-byte signed ints:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300655 if -0x80000000 <= obj <= 0x7fffffff:
Guido van Rossumddefaf32007-01-14 03:31:43 +0000656 self.write(BININT + pack("<i", obj))
657 return
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000658 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000659 encoded = encode_long(obj)
660 n = len(encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000661 if n < 256:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300662 self.write(LONG1 + pack("<B", n) + encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000663 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000664 self.write(LONG4 + pack("<i", n) + encoded)
Tim Petersee1a53c2003-02-02 02:57:53 +0000665 return
Mark Dickinson8dd05142009-01-20 20:43:58 +0000666 self.write(LONG + repr(obj).encode("ascii") + b'L\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000667 dispatch[int] = save_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000668
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300669 def save_float(self, obj):
Guido van Rossumd3703791998-10-22 20:15:36 +0000670 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000671 self.write(BINFLOAT + pack('>d', obj))
Guido van Rossumd3703791998-10-22 20:15:36 +0000672 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000673 self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000674 dispatch[float] = save_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000675
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300676 def save_bytes(self, obj):
Guido van Rossumf4169812008-03-17 22:56:06 +0000677 if self.proto < 3:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300678 if not obj: # bytes object is empty
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -0500679 self.save_reduce(bytes, (), obj=obj)
680 else:
681 self.save_reduce(codecs.encode,
682 (str(obj, 'latin1'), 'latin1'), obj=obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000683 return
684 n = len(obj)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100685 if n <= 0xff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300686 self.write(SHORT_BINBYTES + pack("<B", n) + obj)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100687 elif n > 0xffffffff and self.proto >= 4:
688 self.write(BINBYTES8 + pack("<Q", n) + obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000689 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300690 self.write(BINBYTES + pack("<I", n) + obj)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000691 self.memoize(obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000692 dispatch[bytes] = save_bytes
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000693
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300694 def save_str(self, obj):
Tim Petersc32d8242001-04-10 02:48:53 +0000695 if self.bin:
Victor Stinner485fb562010-04-13 11:07:24 +0000696 encoded = obj.encode('utf-8', 'surrogatepass')
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000697 n = len(encoded)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100698 if n <= 0xff and self.proto >= 4:
699 self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
700 elif n > 0xffffffff and self.proto >= 4:
701 self.write(BINUNICODE8 + pack("<Q", n) + encoded)
702 else:
703 self.write(BINUNICODE + pack("<I", n) + encoded)
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000704 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000705 obj = obj.replace("\\", "\\u005c")
706 obj = obj.replace("\n", "\\u000a")
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100707 self.write(UNICODE + obj.encode('raw-unicode-escape') +
708 b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000709 self.memoize(obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000710 dispatch[str] = save_str
Tim Peters658cba62001-02-09 20:06:00 +0000711
Guido van Rossum3a41c612003-01-28 15:10:22 +0000712 def save_tuple(self, obj):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300713 if not obj: # tuple is empty
714 if self.bin:
715 self.write(EMPTY_TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000716 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300717 self.write(MARK + TUPLE)
Tim Petersd97da802003-01-28 05:48:29 +0000718 return
719
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300720 n = len(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000721 save = self.save
722 memo = self.memo
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300723 if n <= 3 and self.proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000724 for element in obj:
Tim Petersd97da802003-01-28 05:48:29 +0000725 save(element)
726 # Subtle. Same as in the big comment below.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000727 if id(obj) in memo:
728 get = self.get(memo[id(obj)][0])
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300729 self.write(POP * n + get)
Tim Petersd97da802003-01-28 05:48:29 +0000730 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300731 self.write(_tuplesize2code[n])
Guido van Rossum3a41c612003-01-28 15:10:22 +0000732 self.memoize(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000733 return
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000734
Tim Peters1d63c9f2003-02-02 20:29:39 +0000735 # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
Tim Petersff57bff2003-01-28 05:34:53 +0000736 # has more than 3 elements.
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300737 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000738 write(MARK)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000739 for element in obj:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000740 save(element)
741
Tim Peters1d63c9f2003-02-02 20:29:39 +0000742 if id(obj) in memo:
Tim Petersf558da02003-01-28 02:09:55 +0000743 # Subtle. d was not in memo when we entered save_tuple(), so
744 # the process of saving the tuple's elements must have saved
745 # the tuple itself: the tuple is recursive. The proper action
746 # now is to throw away everything we put on the stack, and
747 # simply GET the tuple (it's already constructed). This check
748 # could have been done in the "for element" loop instead, but
749 # recursive tuples are a rare thing.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000750 get = self.get(memo[id(obj)][0])
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300751 if self.bin:
Tim Petersf558da02003-01-28 02:09:55 +0000752 write(POP_MARK + get)
753 else: # proto 0 -- POP_MARK not available
Tim Petersd97da802003-01-28 05:48:29 +0000754 write(POP * (n+1) + get)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000755 return
756
Tim Peters1d63c9f2003-02-02 20:29:39 +0000757 # No recursion.
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300758 write(TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000759 self.memoize(obj)
Jeremy Hylton3422c992003-01-24 19:29:52 +0000760
Guido van Rossum13257902007-06-07 23:15:56 +0000761 dispatch[tuple] = save_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000762
Guido van Rossum3a41c612003-01-28 15:10:22 +0000763 def save_list(self, obj):
Tim Petersc32d8242001-04-10 02:48:53 +0000764 if self.bin:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300765 self.write(EMPTY_LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000766 else: # proto 0 -- can't use EMPTY_LIST
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300767 self.write(MARK + LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000768
769 self.memoize(obj)
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000770 self._batch_appends(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000771
Guido van Rossum13257902007-06-07 23:15:56 +0000772 dispatch[list] = save_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000773
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000774 _BATCHSIZE = 1000
775
776 def _batch_appends(self, items):
777 # Helper to batch up APPENDS sequences
778 save = self.save
779 write = self.write
780
781 if not self.bin:
782 for x in items:
783 save(x)
784 write(APPEND)
785 return
786
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300787 it = iter(items)
788 while True:
789 tmp = list(islice(it, self._BATCHSIZE))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000790 n = len(tmp)
791 if n > 1:
792 write(MARK)
793 for x in tmp:
794 save(x)
795 write(APPENDS)
796 elif n:
797 save(tmp[0])
798 write(APPEND)
799 # else tmp is empty, and we're done
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300800 if n < self._BATCHSIZE:
801 return
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000802
Guido van Rossum3a41c612003-01-28 15:10:22 +0000803 def save_dict(self, obj):
Tim Petersc32d8242001-04-10 02:48:53 +0000804 if self.bin:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300805 self.write(EMPTY_DICT)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000806 else: # proto 0 -- can't use EMPTY_DICT
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300807 self.write(MARK + DICT)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000808
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000809 self.memoize(obj)
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000810 self._batch_setitems(obj.items())
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000811
Guido van Rossum13257902007-06-07 23:15:56 +0000812 dispatch[dict] = save_dict
813 if PyStringMap is not None:
Jeremy Hylton2b9d0291998-05-27 22:38:22 +0000814 dispatch[PyStringMap] = save_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000815
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000816 def _batch_setitems(self, items):
817 # Helper to batch up SETITEMS sequences; proto >= 1 only
818 save = self.save
819 write = self.write
820
821 if not self.bin:
822 for k, v in items:
823 save(k)
824 save(v)
825 write(SETITEM)
826 return
827
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300828 it = iter(items)
829 while True:
830 tmp = list(islice(it, self._BATCHSIZE))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000831 n = len(tmp)
832 if n > 1:
833 write(MARK)
834 for k, v in tmp:
835 save(k)
836 save(v)
837 write(SETITEMS)
838 elif n:
839 k, v = tmp[0]
840 save(k)
841 save(v)
842 write(SETITEM)
843 # else tmp is empty, and we're done
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300844 if n < self._BATCHSIZE:
845 return
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000846
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100847 def save_set(self, obj):
848 save = self.save
849 write = self.write
850
851 if self.proto < 4:
852 self.save_reduce(set, (list(obj),), obj=obj)
853 return
854
855 write(EMPTY_SET)
856 self.memoize(obj)
857
858 it = iter(obj)
859 while True:
860 batch = list(islice(it, self._BATCHSIZE))
861 n = len(batch)
862 if n > 0:
863 write(MARK)
864 for item in batch:
865 save(item)
866 write(ADDITEMS)
867 if n < self._BATCHSIZE:
868 return
869 dispatch[set] = save_set
870
871 def save_frozenset(self, obj):
872 save = self.save
873 write = self.write
874
875 if self.proto < 4:
876 self.save_reduce(frozenset, (list(obj),), obj=obj)
877 return
878
879 write(MARK)
880 for item in obj:
881 save(item)
882
883 if id(obj) in self.memo:
884 # If the object is already in the memo, this means it is
885 # recursive. In this case, throw away everything we put on the
886 # stack, and fetch the object back from the memo.
887 write(POP_MARK + self.get(self.memo[id(obj)][0]))
888 return
889
890 write(FROZENSET)
891 self.memoize(obj)
892 dispatch[frozenset] = save_frozenset
893
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300894 def save_global(self, obj, name=None):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000895 write = self.write
896 memo = self.memo
897
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300898 if name is None:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100899 name = getattr(obj, '__qualname__', None)
Tim Petersc32d8242001-04-10 02:48:53 +0000900 if name is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000901 name = obj.__name__
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000902
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300903 module_name = whichmodule(obj, name)
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000904 try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100905 __import__(module_name, level=0)
906 module = sys.modules[module_name]
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300907 obj2, parent = _getattribute(module, name)
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000908 except (ImportError, KeyError, AttributeError):
909 raise PicklingError(
910 "Can't pickle %r: it's not found as %s.%s" %
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100911 (obj, module_name, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000912 else:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100913 if obj2 is not obj:
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000914 raise PicklingError(
915 "Can't pickle %r: it's not the same object as %s.%s" %
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100916 (obj, module_name, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000917
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000918 if self.proto >= 2:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100919 code = _extension_registry.get((module_name, name))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000920 if code:
921 assert code > 0
922 if code <= 0xff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300923 write(EXT1 + pack("<B", code))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000924 elif code <= 0xffff:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +0300925 write(EXT2 + pack("<H", code))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000926 else:
927 write(EXT4 + pack("<i", code))
928 return
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300929 lastname = name.rpartition('.')[2]
930 if parent is module:
931 name = lastname
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000932 # Non-ASCII identifiers are supported only with protocols >= 3.
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100933 if self.proto >= 4:
934 self.save(module_name)
935 self.save(name)
936 write(STACK_GLOBAL)
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300937 elif parent is not module:
938 self.save_reduce(getattr, (parent, lastname))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100939 elif self.proto >= 3:
940 write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000941 bytes(name, "utf-8") + b'\n')
942 else:
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000943 if self.fix_imports:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100944 r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
945 r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
946 if (module_name, name) in r_name_mapping:
947 module_name, name = r_name_mapping[(module_name, name)]
Serhiy Storchakabfe18242015-03-31 13:12:37 +0300948 elif module_name in r_import_mapping:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100949 module_name = r_import_mapping[module_name]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000950 try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100951 write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000952 bytes(name, "ascii") + b'\n')
953 except UnicodeEncodeError:
954 raise PicklingError(
955 "can't pickle global identifier '%s.%s' using "
956 "pickle protocol %i" % (module, name, self.proto))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000957
Guido van Rossum3a41c612003-01-28 15:10:22 +0000958 self.memoize(obj)
Tim Peters3b769832003-01-28 03:51:36 +0000959
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -0800960 def save_type(self, obj):
961 if obj is type(None):
962 return self.save_reduce(type, (None,), obj=obj)
963 elif obj is type(NotImplemented):
964 return self.save_reduce(type, (NotImplemented,), obj=obj)
965 elif obj is type(...):
966 return self.save_reduce(type, (...,), obj=obj)
967 return self.save_global(obj)
968
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000969 dispatch[FunctionType] = save_global
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -0800970 dispatch[type] = save_type
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000971
Guido van Rossuma48061a1995-01-10 00:31:14 +0000972
Guido van Rossum1be31752003-01-28 15:19:53 +0000973# Unpickling machinery
974
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000975class _Unpickler:
Guido van Rossuma48061a1995-01-10 00:31:14 +0000976
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000977 def __init__(self, file, *, fix_imports=True,
978 encoding="ASCII", errors="strict"):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000979 """This takes a binary file for reading a pickle data stream.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000980
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800981 The protocol version of the pickle is detected automatically, so
982 no proto argument is needed.
983
984 The argument *file* must have two methods, a read() method that
985 takes an integer argument, and a readline() method that requires
986 no arguments. Both methods should return bytes. Thus *file*
Martin Panter7462b6492015-11-02 03:37:02 +0000987 can be a binary file object opened for reading, an io.BytesIO
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800988 object, or any other custom object that meets this interface.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000989
Guido van Rossumfeea0782007-10-10 18:00:50 +0000990 The file-like object must have two methods, a read() method
991 that takes an integer argument, and a readline() method that
992 requires no arguments. Both methods should return bytes.
993 Thus file-like object can be a binary file object opened for
994 reading, a BytesIO object, or any other custom object that
995 meets this interface.
Guido van Rossumf4169812008-03-17 22:56:06 +0000996
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800997 Optional keyword arguments are *fix_imports*, *encoding* and
Martin Panter46f50722016-05-26 05:35:26 +0000998 *errors*, which are used to control compatibility support for
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -0800999 pickle stream generated by Python 2. If *fix_imports* is True,
1000 pickle will try to map the old Python 2 names to the new names
1001 used in Python 3. The *encoding* and *errors* tell pickle how
1002 to decode 8-bit string instances pickled by Python 2; these
1003 default to 'ASCII' and 'strict', respectively. *encoding* can be
1004 'bytes' to read theses 8-bit string instances as bytes objects.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +00001005 """
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001006 self._file_readline = file.readline
1007 self._file_read = file.read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001008 self.memo = {}
Guido van Rossumf4169812008-03-17 22:56:06 +00001009 self.encoding = encoding
1010 self.errors = errors
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001011 self.proto = 0
1012 self.fix_imports = fix_imports
Guido van Rossuma48061a1995-01-10 00:31:14 +00001013
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001014 def load(self):
Guido van Rossum3a41c612003-01-28 15:10:22 +00001015 """Read a pickled object representation from the open file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +00001016
Guido van Rossum3a41c612003-01-28 15:10:22 +00001017 Return the reconstituted object hierarchy specified in the file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +00001018 """
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +00001019 # Check whether Unpickler was initialized correctly. This is
1020 # only needed to mimic the behavior of _pickle.Unpickler.dump().
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001021 if not hasattr(self, "_file_read"):
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +00001022 raise UnpicklingError("Unpickler.__init__() was not called by "
1023 "%s.__init__()" % (self.__class__.__name__,))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001024 self._unframer = _Unframer(self._file_read, self._file_readline)
1025 self.read = self._unframer.read
1026 self.readline = self._unframer.readline
Jeremy Hylton20747fa2001-11-09 16:15:04 +00001027 self.mark = object() # any new unique object
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001028 self.stack = []
1029 self.append = self.stack.append
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001030 self.proto = 0
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001031 read = self.read
1032 dispatch = self.dispatch
1033 try:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001034 while True:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001035 key = read(1)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001036 if not key:
1037 raise EOFError
Guido van Rossum98297ee2007-11-06 21:34:58 +00001038 assert isinstance(key, bytes_types)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001039 dispatch[key[0]](self)
Guido van Rossumb940e112007-01-10 16:19:56 +00001040 except _Stop as stopinst:
Guido van Rossumff871742000-12-13 18:11:56 +00001041 return stopinst.value
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001042
Tim Petersc23d18a2003-01-28 01:41:51 +00001043 # Return largest index k such that self.stack[k] is self.mark.
1044 # If the stack doesn't contain a mark, eventually raises IndexError.
1045 # This could be sped by maintaining another stack, of indices at which
1046 # the mark appears. For that matter, the latter stack would suffice,
1047 # and we wouldn't need to push mark objects on self.stack at all.
1048 # Doing so is probably a good thing, though, since if the pickle is
1049 # corrupt (or hostile) we may get a clue from finding self.mark embedded
1050 # in unpickled objects.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001051 def marker(self):
1052 stack = self.stack
1053 mark = self.mark
1054 k = len(stack)-1
1055 while stack[k] is not mark: k = k-1
1056 return k
1057
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001058 def persistent_load(self, pid):
Benjamin Peterson49956b22009-01-10 17:05:44 +00001059 raise UnpicklingError("unsupported persistent id encountered")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001060
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001061 dispatch = {}
1062
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001063 def load_proto(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001064 proto = self.read(1)[0]
Guido van Rossumf4169812008-03-17 22:56:06 +00001065 if not 0 <= proto <= HIGHEST_PROTOCOL:
Guido van Rossum26d95c32007-08-27 23:18:54 +00001066 raise ValueError("unsupported pickle protocol: %d" % proto)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001067 self.proto = proto
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001068 dispatch[PROTO[0]] = load_proto
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001069
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08001070 def load_frame(self):
1071 frame_size, = unpack('<Q', self.read(8))
1072 if frame_size > sys.maxsize:
1073 raise ValueError("frame size > sys.maxsize: %d" % frame_size)
1074 self._unframer.load_frame(frame_size)
1075 dispatch[FRAME[0]] = load_frame
1076
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001077 def load_persid(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001078 pid = self.readline()[:-1].decode("ascii")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001079 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001080 dispatch[PERSID[0]] = load_persid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001081
1082 def load_binpersid(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001083 pid = self.stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001084 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001085 dispatch[BINPERSID[0]] = load_binpersid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001086
1087 def load_none(self):
1088 self.append(None)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001089 dispatch[NONE[0]] = load_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001090
Guido van Rossum7d97d312003-01-28 04:25:27 +00001091 def load_false(self):
1092 self.append(False)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001093 dispatch[NEWFALSE[0]] = load_false
Guido van Rossum7d97d312003-01-28 04:25:27 +00001094
1095 def load_true(self):
1096 self.append(True)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001097 dispatch[NEWTRUE[0]] = load_true
Guido van Rossum7d97d312003-01-28 04:25:27 +00001098
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001099 def load_int(self):
Tim Peters19ef62d2001-08-28 22:21:18 +00001100 data = self.readline()
Guido van Rossume2763392002-04-05 19:30:08 +00001101 if data == FALSE[1:]:
1102 val = False
1103 elif data == TRUE[1:]:
1104 val = True
1105 else:
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001106 val = int(data, 0)
Guido van Rossume2763392002-04-05 19:30:08 +00001107 self.append(val)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001108 dispatch[INT[0]] = load_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001109
1110 def load_binint(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001111 self.append(unpack('<i', self.read(4))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001112 dispatch[BININT[0]] = load_binint
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001113
1114 def load_binint1(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001115 self.append(self.read(1)[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001116 dispatch[BININT1[0]] = load_binint1
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001117
1118 def load_binint2(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001119 self.append(unpack('<H', self.read(2))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001120 dispatch[BININT2[0]] = load_binint2
Tim Peters2344fae2001-01-15 00:50:52 +00001121
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001122 def load_long(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001123 val = self.readline()[:-1]
1124 if val and val[-1] == b'L'[0]:
Mark Dickinson8dd05142009-01-20 20:43:58 +00001125 val = val[:-1]
Guido van Rossumfeea0782007-10-10 18:00:50 +00001126 self.append(int(val, 0))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001127 dispatch[LONG[0]] = load_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001128
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001129 def load_long1(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001130 n = self.read(1)[0]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001131 data = self.read(n)
1132 self.append(decode_long(data))
1133 dispatch[LONG1[0]] = load_long1
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001134
1135 def load_long4(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001136 n, = unpack('<i', self.read(4))
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001137 if n < 0:
1138 # Corrupt or hostile pickle -- we never write one like this
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001139 raise UnpicklingError("LONG pickle has negative byte count")
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001140 data = self.read(n)
1141 self.append(decode_long(data))
1142 dispatch[LONG4[0]] = load_long4
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001143
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001144 def load_float(self):
Guido van Rossumff871742000-12-13 18:11:56 +00001145 self.append(float(self.readline()[:-1]))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001146 dispatch[FLOAT[0]] = load_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001147
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001148 def load_binfloat(self):
Guido van Rossumd3703791998-10-22 20:15:36 +00001149 self.append(unpack('>d', self.read(8))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001150 dispatch[BINFLOAT[0]] = load_binfloat
Guido van Rossumd3703791998-10-22 20:15:36 +00001151
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08001152 def _decode_string(self, value):
1153 # Used to allow strings from Python 2 to be decoded either as
1154 # bytes or Unicode strings. This should be used only with the
1155 # STRING, BINSTRING and SHORT_BINSTRING opcodes.
1156 if self.encoding == "bytes":
1157 return value
1158 else:
1159 return value.decode(self.encoding, self.errors)
1160
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001161 def load_string(self):
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07001162 data = self.readline()[:-1]
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001163 # Strip outermost quotes
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07001164 if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
1165 data = data[1:-1]
Martin v. Löwis8a8da792002-08-14 07:46:28 +00001166 else:
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07001167 raise UnpicklingError("the STRING opcode argument must be quoted")
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08001168 self.append(self._decode_string(codecs.escape_decode(data)[0]))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001169 dispatch[STRING[0]] = load_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001170
1171 def load_binstring(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001172 # Deprecated BINSTRING uses signed 32-bit length
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001173 len, = unpack('<i', self.read(4))
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001174 if len < 0:
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001175 raise UnpicklingError("BINSTRING pickle has negative byte count")
Guido van Rossumf4169812008-03-17 22:56:06 +00001176 data = self.read(len)
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08001177 self.append(self._decode_string(data))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001178 dispatch[BINSTRING[0]] = load_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001179
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001180 def load_binbytes(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001181 len, = unpack('<I', self.read(4))
1182 if len > maxsize:
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001183 raise UnpicklingError("BINBYTES exceeds system's maximum size "
1184 "of %d bytes" % maxsize)
Guido van Rossumf4169812008-03-17 22:56:06 +00001185 self.append(self.read(len))
1186 dispatch[BINBYTES[0]] = load_binbytes
1187
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +00001188 def load_unicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001189 self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
1190 dispatch[UNICODE[0]] = load_unicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +00001191
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001192 def load_binunicode(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001193 len, = unpack('<I', self.read(4))
1194 if len > maxsize:
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001195 raise UnpicklingError("BINUNICODE exceeds system's maximum size "
1196 "of %d bytes" % maxsize)
Victor Stinner485fb562010-04-13 11:07:24 +00001197 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001198 dispatch[BINUNICODE[0]] = load_binunicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +00001199
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001200 def load_binunicode8(self):
1201 len, = unpack('<Q', self.read(8))
1202 if len > maxsize:
1203 raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
1204 "of %d bytes" % maxsize)
1205 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1206 dispatch[BINUNICODE8[0]] = load_binunicode8
1207
Serhiy Storchakae0606192015-09-29 22:10:07 +03001208 def load_binbytes8(self):
1209 len, = unpack('<Q', self.read(8))
1210 if len > maxsize:
1211 raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
1212 "of %d bytes" % maxsize)
1213 self.append(self.read(len))
1214 dispatch[BINBYTES8[0]] = load_binbytes8
1215
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001216 def load_short_binstring(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001217 len = self.read(1)[0]
1218 data = self.read(len)
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08001219 self.append(self._decode_string(data))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001220 dispatch[SHORT_BINSTRING[0]] = load_short_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001221
Guido van Rossumf4169812008-03-17 22:56:06 +00001222 def load_short_binbytes(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001223 len = self.read(1)[0]
1224 self.append(self.read(len))
Guido van Rossumf4169812008-03-17 22:56:06 +00001225 dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
1226
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001227 def load_short_binunicode(self):
1228 len = self.read(1)[0]
1229 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
1230 dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
1231
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001232 def load_tuple(self):
1233 k = self.marker()
1234 self.stack[k:] = [tuple(self.stack[k+1:])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001235 dispatch[TUPLE[0]] = load_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001236
1237 def load_empty_tuple(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001238 self.append(())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001239 dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001240
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001241 def load_tuple1(self):
1242 self.stack[-1] = (self.stack[-1],)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001243 dispatch[TUPLE1[0]] = load_tuple1
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001244
1245 def load_tuple2(self):
1246 self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001247 dispatch[TUPLE2[0]] = load_tuple2
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001248
1249 def load_tuple3(self):
1250 self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001251 dispatch[TUPLE3[0]] = load_tuple3
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001252
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001253 def load_empty_list(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001254 self.append([])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001255 dispatch[EMPTY_LIST[0]] = load_empty_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001256
1257 def load_empty_dictionary(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001258 self.append({})
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001259 dispatch[EMPTY_DICT[0]] = load_empty_dictionary
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001260
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001261 def load_empty_set(self):
1262 self.append(set())
1263 dispatch[EMPTY_SET[0]] = load_empty_set
1264
1265 def load_frozenset(self):
1266 k = self.marker()
1267 self.stack[k:] = [frozenset(self.stack[k+1:])]
1268 dispatch[FROZENSET[0]] = load_frozenset
1269
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001270 def load_list(self):
1271 k = self.marker()
1272 self.stack[k:] = [self.stack[k+1:]]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001273 dispatch[LIST[0]] = load_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001274
1275 def load_dict(self):
1276 k = self.marker()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001277 items = self.stack[k+1:]
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001278 d = {items[i]: items[i+1]
1279 for i in range(0, len(items), 2)}
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001280 self.stack[k:] = [d]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001281 dispatch[DICT[0]] = load_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001282
Tim Petersd01c1e92003-01-30 15:41:46 +00001283 # INST and OBJ differ only in how they get a class object. It's not
1284 # only sensible to do the rest in a common routine, the two routines
1285 # previously diverged and grew different bugs.
1286 # klass is the class to instantiate, and k points to the topmost mark
1287 # object, following which are the arguments for klass.__init__.
1288 def _instantiate(self, klass, k):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001289 args = tuple(self.stack[k+1:])
1290 del self.stack[k:]
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001291 if (args or not isinstance(klass, type) or
1292 hasattr(klass, "__getinitargs__")):
Guido van Rossum743d17e1998-09-15 20:25:57 +00001293 try:
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001294 value = klass(*args)
Guido van Rossumb940e112007-01-10 16:19:56 +00001295 except TypeError as err:
Guido van Rossum26d95c32007-08-27 23:18:54 +00001296 raise TypeError("in constructor for %s: %s" %
1297 (klass.__name__, str(err)), sys.exc_info()[2])
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001298 else:
1299 value = klass.__new__(klass)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001300 self.append(value)
Tim Petersd01c1e92003-01-30 15:41:46 +00001301
1302 def load_inst(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001303 module = self.readline()[:-1].decode("ascii")
1304 name = self.readline()[:-1].decode("ascii")
Tim Petersd01c1e92003-01-30 15:41:46 +00001305 klass = self.find_class(module, name)
1306 self._instantiate(klass, self.marker())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001307 dispatch[INST[0]] = load_inst
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001308
1309 def load_obj(self):
Tim Petersd01c1e92003-01-30 15:41:46 +00001310 # Stack is ... markobject classobject arg1 arg2 ...
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001311 k = self.marker()
Tim Petersd01c1e92003-01-30 15:41:46 +00001312 klass = self.stack.pop(k+1)
1313 self._instantiate(klass, k)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001314 dispatch[OBJ[0]] = load_obj
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001315
Guido van Rossum3a41c612003-01-28 15:10:22 +00001316 def load_newobj(self):
1317 args = self.stack.pop()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001318 cls = self.stack.pop()
Guido van Rossum3a41c612003-01-28 15:10:22 +00001319 obj = cls.__new__(cls, *args)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001320 self.append(obj)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001321 dispatch[NEWOBJ[0]] = load_newobj
Guido van Rossum3a41c612003-01-28 15:10:22 +00001322
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001323 def load_newobj_ex(self):
1324 kwargs = self.stack.pop()
1325 args = self.stack.pop()
1326 cls = self.stack.pop()
1327 obj = cls.__new__(cls, *args, **kwargs)
1328 self.append(obj)
1329 dispatch[NEWOBJ_EX[0]] = load_newobj_ex
1330
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001331 def load_global(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001332 module = self.readline()[:-1].decode("utf-8")
1333 name = self.readline()[:-1].decode("utf-8")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001334 klass = self.find_class(module, name)
1335 self.append(klass)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001336 dispatch[GLOBAL[0]] = load_global
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001337
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001338 def load_stack_global(self):
1339 name = self.stack.pop()
1340 module = self.stack.pop()
1341 if type(name) is not str or type(module) is not str:
1342 raise UnpicklingError("STACK_GLOBAL requires str")
1343 self.append(self.find_class(module, name))
1344 dispatch[STACK_GLOBAL[0]] = load_stack_global
1345
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001346 def load_ext1(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001347 code = self.read(1)[0]
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001348 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001349 dispatch[EXT1[0]] = load_ext1
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001350
1351 def load_ext2(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001352 code, = unpack('<H', self.read(2))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001353 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001354 dispatch[EXT2[0]] = load_ext2
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001355
1356 def load_ext4(self):
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001357 code, = unpack('<i', self.read(4))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001358 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001359 dispatch[EXT4[0]] = load_ext4
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001360
1361 def get_extension(self, code):
1362 nil = []
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001363 obj = _extension_cache.get(code, nil)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001364 if obj is not nil:
1365 self.append(obj)
1366 return
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001367 key = _inverted_registry.get(code)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001368 if not key:
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001369 if code <= 0: # note that 0 is forbidden
1370 # Corrupt or hostile pickle.
Alexandre Vassalotticc757172013-04-14 02:25:10 -07001371 raise UnpicklingError("EXT specifies code <= 0")
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001372 raise ValueError("unregistered extension code %d" % code)
1373 obj = self.find_class(*key)
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001374 _extension_cache[code] = obj
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001375 self.append(obj)
1376
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001377 def find_class(self, module, name):
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001378 # Subclasses may override this.
1379 if self.proto < 3 and self.fix_imports:
1380 if (module, name) in _compat_pickle.NAME_MAPPING:
1381 module, name = _compat_pickle.NAME_MAPPING[(module, name)]
Serhiy Storchakabfe18242015-03-31 13:12:37 +03001382 elif module in _compat_pickle.IMPORT_MAPPING:
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001383 module = _compat_pickle.IMPORT_MAPPING[module]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001384 __import__(module, level=0)
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001385 if self.proto >= 4:
1386 return _getattribute(sys.modules[module], name)[0]
1387 else:
1388 return getattr(sys.modules[module], name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001389
1390 def load_reduce(self):
1391 stack = self.stack
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001392 args = stack.pop()
1393 func = stack[-1]
Serhiy Storchakaa8d83f52015-12-01 00:39:25 +02001394 stack[-1] = func(*args)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001395 dispatch[REDUCE[0]] = load_reduce
Guido van Rossuma48061a1995-01-10 00:31:14 +00001396
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001397 def load_pop(self):
1398 del self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001399 dispatch[POP[0]] = load_pop
Guido van Rossum7b5430f1995-03-04 22:25:21 +00001400
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001401 def load_pop_mark(self):
1402 k = self.marker()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001403 del self.stack[k:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001404 dispatch[POP_MARK[0]] = load_pop_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001405
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001406 def load_dup(self):
Guido van Rossumb1062fc1998-03-31 17:00:46 +00001407 self.append(self.stack[-1])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001408 dispatch[DUP[0]] = load_dup
Guido van Rossuma48061a1995-01-10 00:31:14 +00001409
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001410 def load_get(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001411 i = int(self.readline()[:-1])
1412 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001413 dispatch[GET[0]] = load_get
Guido van Rossum78536471996-04-12 13:36:27 +00001414
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001415 def load_binget(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001416 i = self.read(1)[0]
1417 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001418 dispatch[BINGET[0]] = load_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001419
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001420 def load_long_binget(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001421 i, = unpack('<I', self.read(4))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001422 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001423 dispatch[LONG_BINGET[0]] = load_long_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001424
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001425 def load_put(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001426 i = int(self.readline()[:-1])
Antoine Pitrou55549ec2011-08-30 00:27:10 +02001427 if i < 0:
1428 raise ValueError("negative PUT argument")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001429 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001430 dispatch[PUT[0]] = load_put
Guido van Rossuma48061a1995-01-10 00:31:14 +00001431
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001432 def load_binput(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001433 i = self.read(1)[0]
Antoine Pitrou55549ec2011-08-30 00:27:10 +02001434 if i < 0:
1435 raise ValueError("negative BINPUT argument")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001436 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001437 dispatch[BINPUT[0]] = load_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001438
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001439 def load_long_binput(self):
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001440 i, = unpack('<I', self.read(4))
1441 if i > maxsize:
Antoine Pitrou55549ec2011-08-30 00:27:10 +02001442 raise ValueError("negative LONG_BINPUT argument")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001443 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001444 dispatch[LONG_BINPUT[0]] = load_long_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001445
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001446 def load_memoize(self):
1447 memo = self.memo
1448 memo[len(memo)] = self.stack[-1]
1449 dispatch[MEMOIZE[0]] = load_memoize
1450
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001451 def load_append(self):
1452 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001453 value = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001454 list = stack[-1]
1455 list.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001456 dispatch[APPEND[0]] = load_append
Guido van Rossuma48061a1995-01-10 00:31:14 +00001457
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001458 def load_appends(self):
1459 stack = self.stack
1460 mark = self.marker()
Alexandre Vassalotti1f7492c2013-04-20 13:19:46 -07001461 list_obj = stack[mark - 1]
1462 items = stack[mark + 1:]
1463 if isinstance(list_obj, list):
1464 list_obj.extend(items)
1465 else:
1466 append = list_obj.append
1467 for item in items:
1468 append(item)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001469 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001470 dispatch[APPENDS[0]] = load_appends
Tim Peters2344fae2001-01-15 00:50:52 +00001471
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001472 def load_setitem(self):
1473 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001474 value = stack.pop()
1475 key = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001476 dict = stack[-1]
1477 dict[key] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001478 dispatch[SETITEM[0]] = load_setitem
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001479
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001480 def load_setitems(self):
1481 stack = self.stack
1482 mark = self.marker()
1483 dict = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001484 for i in range(mark + 1, len(stack), 2):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001485 dict[stack[i]] = stack[i + 1]
Guido van Rossuma48061a1995-01-10 00:31:14 +00001486
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001487 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001488 dispatch[SETITEMS[0]] = load_setitems
Guido van Rossuma48061a1995-01-10 00:31:14 +00001489
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001490 def load_additems(self):
1491 stack = self.stack
1492 mark = self.marker()
1493 set_obj = stack[mark - 1]
1494 items = stack[mark + 1:]
1495 if isinstance(set_obj, set):
1496 set_obj.update(items)
1497 else:
1498 add = set_obj.add
1499 for item in items:
1500 add(item)
1501 del stack[mark:]
1502 dispatch[ADDITEMS[0]] = load_additems
1503
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001504 def load_build(self):
1505 stack = self.stack
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001506 state = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001507 inst = stack[-1]
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001508 setstate = getattr(inst, "__setstate__", None)
Serhiy Storchakaa3e32c92013-04-14 13:37:02 +03001509 if setstate is not None:
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001510 setstate(state)
1511 return
1512 slotstate = None
1513 if isinstance(state, tuple) and len(state) == 2:
1514 state, slotstate = state
1515 if state:
Alexandre Vassalottiebfecfd2009-05-25 18:50:33 +00001516 inst_dict = inst.__dict__
Antoine Pitroua9f48a02009-05-02 21:41:14 +00001517 intern = sys.intern
Alexandre Vassalottiebfecfd2009-05-25 18:50:33 +00001518 for k, v in state.items():
1519 if type(k) is str:
1520 inst_dict[intern(k)] = v
1521 else:
1522 inst_dict[k] = v
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001523 if slotstate:
1524 for k, v in slotstate.items():
1525 setattr(inst, k, v)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001526 dispatch[BUILD[0]] = load_build
Guido van Rossuma48061a1995-01-10 00:31:14 +00001527
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001528 def load_mark(self):
1529 self.append(self.mark)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001530 dispatch[MARK[0]] = load_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001531
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001532 def load_stop(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001533 value = self.stack.pop()
Guido van Rossumff871742000-12-13 18:11:56 +00001534 raise _Stop(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001535 dispatch[STOP[0]] = load_stop
Guido van Rossuma48061a1995-01-10 00:31:14 +00001536
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001537
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001538# Shorthands
1539
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001540def _dump(obj, file, protocol=None, *, fix_imports=True):
1541 _Pickler(file, protocol, fix_imports=fix_imports).dump(obj)
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001542
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001543def _dumps(obj, protocol=None, *, fix_imports=True):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001544 f = io.BytesIO()
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001545 _Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001546 res = f.getvalue()
Guido van Rossum98297ee2007-11-06 21:34:58 +00001547 assert isinstance(res, bytes_types)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001548 return res
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001549
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001550def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict"):
1551 return _Unpickler(file, fix_imports=fix_imports,
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001552 encoding=encoding, errors=errors).load()
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001553
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001554def _loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001555 if isinstance(s, str):
1556 raise TypeError("Can't load pickle from unicode string")
1557 file = io.BytesIO(s)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001558 return _Unpickler(file, fix_imports=fix_imports,
1559 encoding=encoding, errors=errors).load()
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001560
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001561# Use the faster _pickle if possible
1562try:
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001563 from _pickle import (
1564 PickleError,
1565 PicklingError,
1566 UnpicklingError,
1567 Pickler,
1568 Unpickler,
1569 dump,
1570 dumps,
1571 load,
1572 loads
1573 )
Brett Cannoncd171c82013-07-04 17:43:24 -04001574except ImportError:
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001575 Pickler, Unpickler = _Pickler, _Unpickler
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001576 dump, dumps, load, loads = _dump, _dumps, _load, _loads
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001577
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001578# Doctest
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001579def _test():
1580 import doctest
1581 return doctest.testmod()
1582
1583if __name__ == "__main__":
Florent Xicluna54540ec2011-11-04 08:29:17 +01001584 import argparse
Alexander Belopolsky455f7bd2010-07-27 23:02:38 +00001585 parser = argparse.ArgumentParser(
1586 description='display contents of the pickle files')
1587 parser.add_argument(
1588 'pickle_file', type=argparse.FileType('br'),
1589 nargs='*', help='the pickle file')
1590 parser.add_argument(
1591 '-t', '--test', action='store_true',
1592 help='run self-test suite')
1593 parser.add_argument(
1594 '-v', action='store_true',
1595 help='run verbosely; only affects self-test run')
1596 args = parser.parse_args()
1597 if args.test:
1598 _test()
1599 else:
1600 if not args.pickle_file:
1601 parser.print_help()
1602 else:
1603 import pprint
1604 for f in args.pickle_file:
1605 obj = load(f)
1606 pprint.pprint(obj)