blob: aca8fd183c23acdd98d840f2328317c4aec36c79 [file] [log] [blame]
Guido van Rossum54f22ed2000-02-04 15:10:34 +00001"""Create portable serialized representations of Python objects.
Guido van Rossuma48061a1995-01-10 00:31:14 +00002
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +00003See module copyreg for a mechanism for registering custom picklers.
Tim Peters22a449a2003-01-27 20:16:36 +00004See module pickletools source for extensive comments.
Guido van Rossuma48061a1995-01-10 00:31:14 +00005
Guido van Rossume467be61997-12-05 19:42:42 +00006Classes:
Guido van Rossuma48061a1995-01-10 00:31:14 +00007
Guido van Rossume467be61997-12-05 19:42:42 +00008 Pickler
9 Unpickler
Guido van Rossuma48061a1995-01-10 00:31:14 +000010
Guido van Rossume467be61997-12-05 19:42:42 +000011Functions:
Guido van Rossuma48061a1995-01-10 00:31:14 +000012
Guido van Rossume467be61997-12-05 19:42:42 +000013 dump(object, file)
14 dumps(object) -> string
15 load(file) -> object
16 loads(string) -> object
Guido van Rossuma48061a1995-01-10 00:31:14 +000017
Guido van Rossume467be61997-12-05 19:42:42 +000018Misc variables:
Guido van Rossuma48061a1995-01-10 00:31:14 +000019
Fred Drakefe82acc1998-02-13 03:24:48 +000020 __version__
Guido van Rossume467be61997-12-05 19:42:42 +000021 format_version
22 compatible_formats
Guido van Rossuma48061a1995-01-10 00:31:14 +000023
Guido van Rossuma48061a1995-01-10 00:31:14 +000024"""
25
Guido van Rossum743d17e1998-09-15 20:25:57 +000026__version__ = "$Revision$" # Code version
Guido van Rossuma48061a1995-01-10 00:31:14 +000027
Guido van Rossum13257902007-06-07 23:15:56 +000028from types import FunctionType, BuiltinFunctionType
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +000029from copyreg import dispatch_table
30from copyreg import _extension_registry, _inverted_registry, _extension_cache
Guido van Rossumd3703791998-10-22 20:15:36 +000031import marshal
32import sys
33import struct
Skip Montanaro23bafc62001-02-18 03:10:09 +000034import re
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000035import io
Walter Dörwald42748a82007-06-12 16:40:17 +000036import codecs
Antoine Pitroud9dfaa92009-06-04 20:32:06 +000037import _compat_pickle
Guido van Rossuma48061a1995-01-10 00:31:14 +000038
Skip Montanaro352674d2001-02-07 23:14:30 +000039__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
40 "Unpickler", "dump", "dumps", "load", "loads"]
41
Guido van Rossum98297ee2007-11-06 21:34:58 +000042# Shortcut for use in isinstance testing
Alexandre Vassalotti8cb02b62008-05-03 01:42:49 +000043bytes_types = (bytes, bytearray)
Guido van Rossum98297ee2007-11-06 21:34:58 +000044
Tim Petersc0c12b52003-01-29 00:56:17 +000045# These are purely informational; no code uses these.
Guido van Rossumf4169812008-03-17 22:56:06 +000046format_version = "3.0" # File format version we write
Guido van Rossumf29d3d62003-01-27 22:47:53 +000047compatible_formats = ["1.0", # Original protocol 0
Guido van Rossumbc64e222003-01-28 16:34:19 +000048 "1.1", # Protocol 0 with INST added
Guido van Rossumf29d3d62003-01-27 22:47:53 +000049 "1.2", # Original protocol 1
50 "1.3", # Protocol 1 with BINFLOAT added
51 "2.0", # Protocol 2
Guido van Rossumf4169812008-03-17 22:56:06 +000052 "3.0", # Protocol 3
Guido van Rossumf29d3d62003-01-27 22:47:53 +000053 ] # Old format versions we can read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000054
Guido van Rossum99603b02007-07-20 00:22:32 +000055# This is the highest protocol number we know how to read.
Guido van Rossumf4169812008-03-17 22:56:06 +000056HIGHEST_PROTOCOL = 3
Tim Peters8587b3c2003-02-13 15:44:41 +000057
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000058# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
Guido van Rossumf4169812008-03-17 22:56:06 +000059# We intentionally write a protocol that Python 2.x cannot read;
60# there are too many issues with that.
61DEFAULT_PROTOCOL = 3
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000062
Guido van Rossume0b90422003-01-28 03:17:21 +000063# Why use struct.pack() for pickling but marshal.loads() for
Tim Petersc0c12b52003-01-29 00:56:17 +000064# unpickling? struct.pack() is 40% faster than marshal.dumps(), but
Guido van Rossume0b90422003-01-28 03:17:21 +000065# marshal.loads() is twice as fast as struct.unpack()!
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000066mloads = marshal.loads
Guido van Rossum0c891ce1995-03-14 15:09:05 +000067
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000068class PickleError(Exception):
Neal Norwitzefbb67b2002-05-30 12:12:04 +000069 """A common base class for the other pickling exceptions."""
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000070 pass
71
72class PicklingError(PickleError):
73 """This exception is raised when an unpicklable object is passed to the
74 dump() method.
75
76 """
77 pass
78
79class UnpicklingError(PickleError):
80 """This exception is raised when there is a problem unpickling an object,
81 such as a security violation.
82
83 Note that other exceptions may also be raised during unpickling, including
84 (but not necessarily limited to) AttributeError, EOFError, ImportError,
85 and IndexError.
86
87 """
88 pass
Guido van Rossum7849da81995-03-09 14:08:35 +000089
Tim Petersc0c12b52003-01-29 00:56:17 +000090# An instance of _Stop is raised by Unpickler.load_stop() in response to
91# the STOP opcode, passing the object that is the result of unpickling.
Guido van Rossumff871742000-12-13 18:11:56 +000092class _Stop(Exception):
93 def __init__(self, value):
94 self.value = value
95
Guido van Rossum533dbcf2003-01-28 17:55:05 +000096# Jython has PyStringMap; it's a dict subclass with string keys
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000097try:
98 from org.python.core import PyStringMap
99except ImportError:
100 PyStringMap = None
101
Tim Peters22a449a2003-01-27 20:16:36 +0000102# Pickle opcodes. See pickletools.py for extensive docs. The listing
103# here is in kind-of alphabetical order of 1-character pickle code.
104# pickletools groups them by purpose.
Guido van Rossumdbb718f2001-09-21 19:22:34 +0000105
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000106MARK = b'(' # push special markobject on stack
107STOP = b'.' # every pickle ends with STOP
108POP = b'0' # discard topmost stack item
109POP_MARK = b'1' # discard stack top through topmost markobject
110DUP = b'2' # duplicate top stack item
111FLOAT = b'F' # push float object; decimal string argument
112INT = b'I' # push integer or bool; decimal string argument
113BININT = b'J' # push four-byte signed int
114BININT1 = b'K' # push 1-byte unsigned int
115LONG = b'L' # push long; decimal string argument
116BININT2 = b'M' # push 2-byte unsigned int
117NONE = b'N' # push None
118PERSID = b'P' # push persistent object; id is taken from string arg
119BINPERSID = b'Q' # " " " ; " " " " stack
120REDUCE = b'R' # apply callable to argtuple, both on stack
121STRING = b'S' # push string; NL-terminated string argument
122BINSTRING = b'T' # push string; counted binary string argument
123SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
124UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
125BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
126APPEND = b'a' # append stack top to list below it
127BUILD = b'b' # call __setstate__ or __dict__.update()
128GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
129DICT = b'd' # build a dict from stack items
130EMPTY_DICT = b'}' # push empty dict
131APPENDS = b'e' # extend list on stack by topmost stack slice
132GET = b'g' # push item from memo on stack; index is string arg
133BINGET = b'h' # " " " " " " ; " " 1-byte arg
134INST = b'i' # build & push class instance
135LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
136LIST = b'l' # build list from topmost stack items
137EMPTY_LIST = b']' # push empty list
138OBJ = b'o' # build & push class instance
139PUT = b'p' # store stack top in memo; index is string arg
140BINPUT = b'q' # " " " " " ; " " 1-byte arg
141LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
142SETITEM = b's' # add key+value pair to dict
143TUPLE = b't' # build tuple from topmost stack items
144EMPTY_TUPLE = b')' # push empty tuple
145SETITEMS = b'u' # modify dict by adding topmost key+value pairs
146BINFLOAT = b'G' # push float; arg is 8-byte float encoding
Tim Peters22a449a2003-01-27 20:16:36 +0000147
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000148TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
149FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
Guido van Rossum77f6a652002-04-03 22:41:51 +0000150
Guido van Rossum586c9e82003-01-29 06:16:12 +0000151# Protocol 2
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000152
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000153PROTO = b'\x80' # identify pickle protocol
154NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
155EXT1 = b'\x82' # push object from extension registry; 1-byte index
156EXT2 = b'\x83' # ditto, but 2-byte index
157EXT4 = b'\x84' # ditto, but 4-byte index
158TUPLE1 = b'\x85' # build 1-tuple from stack top
159TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
160TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
161NEWTRUE = b'\x88' # push True
162NEWFALSE = b'\x89' # push False
163LONG1 = b'\x8a' # push long from < 256 bytes
164LONG4 = b'\x8b' # push really big long
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000165
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000166_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
167
Guido van Rossumf4169812008-03-17 22:56:06 +0000168# Protocol 3 (Python 3.x)
169
170BINBYTES = b'B' # push bytes; counted binary string argument
171SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
Guido van Rossuma48061a1995-01-10 00:31:14 +0000172
Skip Montanaro23bafc62001-02-18 03:10:09 +0000173__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
174
Guido van Rossum1be31752003-01-28 15:19:53 +0000175# Pickling machinery
176
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000177class _Pickler:
Guido van Rossuma48061a1995-01-10 00:31:14 +0000178
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000179 def __init__(self, file, protocol=None, *, fix_imports=True):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000180 """This takes a binary file for writing a pickle data stream.
181
Guido van Rossumcf117b02003-02-09 17:19:41 +0000182 The optional protocol argument tells the pickler to use the
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000183 given protocol; supported protocols are 0, 1, 2, 3. The default
184 protocol is 3; a backward-incompatible protocol designed for
185 Python 3.0.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000186
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000187 Specifying a negative protocol version selects the highest
Tim Peters5bd2a792003-02-01 16:45:06 +0000188 protocol version supported. The higher the protocol used, the
189 more recent the version of Python needed to read the pickle
190 produced.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000191
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000192 The file argument must have a write() method that accepts a single
193 bytes argument. It can thus be a file object opened for binary
194 writing, a io.BytesIO instance, or any other custom object that
195 meets this interface.
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000196
197 If fix_imports is True and protocol is less than 3, pickle will try to
198 map the new Python 3.x names to the old module names used in Python
199 2.x, so that the pickle data stream is readable with Python 2.x.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000200 """
Guido van Rossumcf117b02003-02-09 17:19:41 +0000201 if protocol is None:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000202 protocol = DEFAULT_PROTOCOL
Guido van Rossumcf117b02003-02-09 17:19:41 +0000203 if protocol < 0:
Tim Peters8587b3c2003-02-13 15:44:41 +0000204 protocol = HIGHEST_PROTOCOL
205 elif not 0 <= protocol <= HIGHEST_PROTOCOL:
206 raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000207 try:
208 self.write = file.write
209 except AttributeError:
210 raise TypeError("file must have a 'write' attribute")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000211 self.memo = {}
Guido van Rossumcf117b02003-02-09 17:19:41 +0000212 self.proto = int(protocol)
213 self.bin = protocol >= 1
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000214 self.fast = 0
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000215 self.fix_imports = fix_imports and protocol < 3
Guido van Rossuma48061a1995-01-10 00:31:14 +0000216
Fred Drake7f781c92002-05-01 20:33:53 +0000217 def clear_memo(self):
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000218 """Clears the pickler's "memo".
219
220 The memo is the data structure that remembers which objects the
Tim Petersb377f8a2003-01-28 00:23:36 +0000221 pickler has already seen, so that shared or recursive objects are
222 pickled by reference and not by value. This method is useful when
223 re-using picklers.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000224
225 """
Fred Drake7f781c92002-05-01 20:33:53 +0000226 self.memo.clear()
227
Guido van Rossum3a41c612003-01-28 15:10:22 +0000228 def dump(self, obj):
Tim Peters5bd2a792003-02-01 16:45:06 +0000229 """Write a pickled representation of obj to the open file."""
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +0000230 # Check whether Pickler was initialized correctly. This is
231 # only needed to mimic the behavior of _pickle.Pickler.dump().
232 if not hasattr(self, "write"):
233 raise PicklingError("Pickler.__init__() was not called by "
234 "%s.__init__()" % (self.__class__.__name__,))
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000235 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000236 self.write(PROTO + bytes([self.proto]))
Guido van Rossum3a41c612003-01-28 15:10:22 +0000237 self.save(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000238 self.write(STOP)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000239
Jeremy Hylton3422c992003-01-24 19:29:52 +0000240 def memoize(self, obj):
241 """Store an object in the memo."""
242
Tim Peterse46b73f2003-01-27 21:22:10 +0000243 # The Pickler memo is a dictionary mapping object ids to 2-tuples
244 # that contain the Unpickler memo key and the object being memoized.
245 # The memo key is written to the pickle and will become
Jeremy Hylton3422c992003-01-24 19:29:52 +0000246 # the key in the Unpickler's memo. The object is stored in the
Tim Peterse46b73f2003-01-27 21:22:10 +0000247 # Pickler memo so that transient objects are kept alive during
248 # pickling.
Jeremy Hylton3422c992003-01-24 19:29:52 +0000249
Tim Peterse46b73f2003-01-27 21:22:10 +0000250 # The use of the Unpickler memo length as the memo key is just a
251 # convention. The only requirement is that the memo values be unique.
252 # But there appears no advantage to any other scheme, and this
Tim Peterscbd0a322003-01-28 00:24:43 +0000253 # scheme allows the Unpickler memo to be implemented as a plain (but
Tim Peterse46b73f2003-01-27 21:22:10 +0000254 # growable) array, indexed by memo key.
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000255 if self.fast:
256 return
Guido van Rossum9b40e802003-01-30 06:37:41 +0000257 assert id(obj) not in self.memo
Jeremy Hylton3422c992003-01-24 19:29:52 +0000258 memo_len = len(self.memo)
259 self.write(self.put(memo_len))
Tim Peters518df0d2003-01-28 01:00:38 +0000260 self.memo[id(obj)] = memo_len, obj
Jeremy Hylton3422c992003-01-24 19:29:52 +0000261
Tim Petersbb38e302003-01-27 21:25:41 +0000262 # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000263 def put(self, i, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000264 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000265 if i < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000266 return BINPUT + bytes([i])
Guido van Rossum5c938d02003-01-28 03:03:08 +0000267 else:
268 return LONG_BINPUT + pack("<i", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000269
Guido van Rossum39478e82007-08-27 17:23:59 +0000270 return PUT + repr(i).encode("ascii") + b'\n'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000271
Tim Petersbb38e302003-01-27 21:25:41 +0000272 # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000273 def get(self, i, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000274 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000275 if i < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000276 return BINGET + bytes([i])
Guido van Rossum5c938d02003-01-28 03:03:08 +0000277 else:
278 return LONG_BINGET + pack("<i", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000279
Guido van Rossum39478e82007-08-27 17:23:59 +0000280 return GET + repr(i).encode("ascii") + b'\n'
Tim Peters2344fae2001-01-15 00:50:52 +0000281
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000282 def save(self, obj, save_persistent_id=True):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000283 # Check for persistent id (defined by a subclass)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000284 pid = self.persistent_id(obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000285 if pid is not None and save_persistent_id:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000286 self.save_pers(pid)
287 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000288
Guido van Rossumbc64e222003-01-28 16:34:19 +0000289 # Check the memo
290 x = self.memo.get(id(obj))
291 if x:
292 self.write(self.get(x[0]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000293 return
294
Guido van Rossumbc64e222003-01-28 16:34:19 +0000295 # Check the type dispatch table
Guido van Rossum3a41c612003-01-28 15:10:22 +0000296 t = type(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000297 f = self.dispatch.get(t)
298 if f:
299 f(self, obj) # Call unbound method with explicit self
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000300 return
301
Guido van Rossumbc64e222003-01-28 16:34:19 +0000302 # Check for a class with a custom metaclass; treat as regular class
Tim Petersb32a8312003-01-28 00:48:09 +0000303 try:
Guido van Rossum13257902007-06-07 23:15:56 +0000304 issc = issubclass(t, type)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000305 except TypeError: # t is not a class (old Boost; see SF #502085)
Tim Petersb32a8312003-01-28 00:48:09 +0000306 issc = 0
307 if issc:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000308 self.save_global(obj)
Tim Petersb32a8312003-01-28 00:48:09 +0000309 return
310
Alexandre Vassalottif7fa63d2008-05-11 08:55:36 +0000311 # Check copyreg.dispatch_table
Guido van Rossumbc64e222003-01-28 16:34:19 +0000312 reduce = dispatch_table.get(t)
Guido van Rossumc53f0092003-02-18 22:05:12 +0000313 if reduce:
314 rv = reduce(obj)
315 else:
316 # Check for a __reduce_ex__ method, fall back to __reduce__
317 reduce = getattr(obj, "__reduce_ex__", None)
318 if reduce:
319 rv = reduce(self.proto)
320 else:
321 reduce = getattr(obj, "__reduce__", None)
322 if reduce:
323 rv = reduce()
324 else:
325 raise PicklingError("Can't pickle %r object: %r" %
326 (t.__name__, obj))
Tim Petersb32a8312003-01-28 00:48:09 +0000327
Guido van Rossumbc64e222003-01-28 16:34:19 +0000328 # Check for string returned by reduce(), meaning "save as global"
Guido van Rossum3172c5d2007-10-16 18:12:55 +0000329 if isinstance(rv, str):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000330 self.save_global(obj, rv)
Tim Petersb32a8312003-01-28 00:48:09 +0000331 return
332
Guido van Rossumbc64e222003-01-28 16:34:19 +0000333 # Assert that reduce() returned a tuple
Guido van Rossum13257902007-06-07 23:15:56 +0000334 if not isinstance(rv, tuple):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000335 raise PicklingError("%s must return string or tuple" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000336
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000337 # Assert that it returned an appropriately sized tuple
Guido van Rossumbc64e222003-01-28 16:34:19 +0000338 l = len(rv)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000339 if not (2 <= l <= 5):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000340 raise PicklingError("Tuple returned by %s must have "
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000341 "two to five elements" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000342
Guido van Rossumbc64e222003-01-28 16:34:19 +0000343 # Save the reduce() output and finally memoize the object
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000344 self.save_reduce(obj=obj, *rv)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000345
Guido van Rossum3a41c612003-01-28 15:10:22 +0000346 def persistent_id(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000347 # This exists so a subclass can override it
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000348 return None
349
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000350 def save_pers(self, pid):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000351 # Save a persistent id reference
Tim Petersbd1cdb92003-01-28 01:03:10 +0000352 if self.bin:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000353 self.save(pid, save_persistent_id=False)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000354 self.write(BINPERSID)
Tim Petersbd1cdb92003-01-28 01:03:10 +0000355 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000356 self.write(PERSID + str(pid).encode("ascii") + b'\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000357
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000358 def save_reduce(self, func, args, state=None,
359 listitems=None, dictitems=None, obj=None):
Jeremy Hyltone3a565e2003-06-29 16:59:59 +0000360 # This API is called by some subclasses
Guido van Rossumbc64e222003-01-28 16:34:19 +0000361
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000362 # Assert that args is a tuple
Guido van Rossum13257902007-06-07 23:15:56 +0000363 if not isinstance(args, tuple):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000364 raise PicklingError("args from save_reduce() should be a tuple")
Guido van Rossumbc64e222003-01-28 16:34:19 +0000365
366 # Assert that func is callable
Guido van Rossumd59da4b2007-05-22 18:11:13 +0000367 if not hasattr(func, '__call__'):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000368 raise PicklingError("func from save_reduce() should be callable")
Guido van Rossumbc64e222003-01-28 16:34:19 +0000369
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000370 save = self.save
Guido van Rossumbc64e222003-01-28 16:34:19 +0000371 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000372
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000373 # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ
374 if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__":
375 # A __reduce__ implementation can direct protocol 2 to
376 # use the more efficient NEWOBJ opcode, while still
377 # allowing protocol 0 and 1 to work normally. For this to
378 # work, the function returned by __reduce__ should be
379 # called __newobj__, and its first argument should be a
380 # new-style class. The implementation for __newobj__
381 # should be as follows, although pickle has no way to
382 # verify this:
383 #
384 # def __newobj__(cls, *args):
385 # return cls.__new__(cls, *args)
386 #
387 # Protocols 0 and 1 will pickle a reference to __newobj__,
388 # while protocol 2 (and above) will pickle a reference to
389 # cls, the remaining args tuple, and the NEWOBJ code,
390 # which calls cls.__new__(cls, *args) at unpickling time
391 # (see load_newobj below). If __reduce__ returns a
392 # three-tuple, the state from the third tuple item will be
393 # pickled regardless of the protocol, calling __setstate__
394 # at unpickling time (see load_build below).
395 #
396 # Note that no standard __newobj__ implementation exists;
397 # you have to provide your own. This is to enforce
398 # compatibility with Python 2.2 (pickles written using
399 # protocol 0 or 1 in Python 2.3 should be unpicklable by
400 # Python 2.2).
401 cls = args[0]
402 if not hasattr(cls, "__new__"):
403 raise PicklingError(
404 "args[0] from __newobj__ args has no __new__")
Guido van Rossumf7f45172003-01-31 17:17:49 +0000405 if obj is not None and cls is not obj.__class__:
406 raise PicklingError(
407 "args[0] from __newobj__ args has the wrong class")
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000408 args = args[1:]
409 save(cls)
410 save(args)
411 write(NEWOBJ)
412 else:
413 save(func)
414 save(args)
415 write(REDUCE)
Tim Peters2344fae2001-01-15 00:50:52 +0000416
Guido van Rossumf7f45172003-01-31 17:17:49 +0000417 if obj is not None:
418 self.memoize(obj)
419
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000420 # More new special cases (that work with older protocols as
421 # well): when __reduce__ returns a tuple with 4 or 5 items,
422 # the 4th and 5th item should be iterators that provide list
423 # items and dict items (as (key, value) tuples), or None.
424
425 if listitems is not None:
426 self._batch_appends(listitems)
427
428 if dictitems is not None:
429 self._batch_setitems(dictitems)
430
Tim Petersc32d8242001-04-10 02:48:53 +0000431 if state is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000432 save(state)
433 write(BUILD)
434
Guido van Rossumbc64e222003-01-28 16:34:19 +0000435 # Methods below this point are dispatched through the dispatch table
436
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000437 dispatch = {}
438
Guido van Rossum3a41c612003-01-28 15:10:22 +0000439 def save_none(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000440 self.write(NONE)
Guido van Rossum13257902007-06-07 23:15:56 +0000441 dispatch[type(None)] = save_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000442
Guido van Rossum3a41c612003-01-28 15:10:22 +0000443 def save_bool(self, obj):
Guido van Rossum7d97d312003-01-28 04:25:27 +0000444 if self.proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000445 self.write(obj and NEWTRUE or NEWFALSE)
Guido van Rossum7d97d312003-01-28 04:25:27 +0000446 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000447 self.write(obj and TRUE or FALSE)
Guido van Rossum77f6a652002-04-03 22:41:51 +0000448 dispatch[bool] = save_bool
449
Guido van Rossum3a41c612003-01-28 15:10:22 +0000450 def save_long(self, obj, pack=struct.pack):
Guido van Rossumddefaf32007-01-14 03:31:43 +0000451 if self.bin:
452 # If the int is small enough to fit in a signed 4-byte 2's-comp
453 # format, we can store it more efficiently than the general
454 # case.
455 # First one- and two-byte unsigned ints:
456 if obj >= 0:
457 if obj <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000458 self.write(BININT1 + bytes([obj]))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000459 return
460 if obj <= 0xffff:
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000461 self.write(BININT2 + bytes([obj&0xff, obj>>8]))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000462 return
463 # Next check for 4-byte signed ints:
464 high_bits = obj >> 31 # note that Python shift sign-extends
465 if high_bits == 0 or high_bits == -1:
466 # All high bits are copies of bit 2**31, so the value
467 # fits in a 4-byte signed int.
468 self.write(BININT + pack("<i", obj))
469 return
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000470 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000471 encoded = encode_long(obj)
472 n = len(encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000473 if n < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000474 self.write(LONG1 + bytes([n]) + encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000475 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000476 self.write(LONG4 + pack("<i", n) + encoded)
Tim Petersee1a53c2003-02-02 02:57:53 +0000477 return
Mark Dickinson8dd05142009-01-20 20:43:58 +0000478 self.write(LONG + repr(obj).encode("ascii") + b'L\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000479 dispatch[int] = save_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000480
Guido van Rossum3a41c612003-01-28 15:10:22 +0000481 def save_float(self, obj, pack=struct.pack):
Guido van Rossumd3703791998-10-22 20:15:36 +0000482 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000483 self.write(BINFLOAT + pack('>d', obj))
Guido van Rossumd3703791998-10-22 20:15:36 +0000484 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000485 self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000486 dispatch[float] = save_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000487
Guido van Rossumf4169812008-03-17 22:56:06 +0000488 def save_bytes(self, obj, pack=struct.pack):
489 if self.proto < 3:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000490 self.save_reduce(bytes, (list(obj),), obj=obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000491 return
492 n = len(obj)
493 if n < 256:
494 self.write(SHORT_BINBYTES + bytes([n]) + bytes(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000495 else:
Guido van Rossumf4169812008-03-17 22:56:06 +0000496 self.write(BINBYTES + pack("<i", n) + bytes(obj))
Guido van Rossum3a41c612003-01-28 15:10:22 +0000497 self.memoize(obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000498 dispatch[bytes] = save_bytes
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000499
Guido van Rossumf4169812008-03-17 22:56:06 +0000500 def save_str(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000501 if self.bin:
Victor Stinner485fb562010-04-13 11:07:24 +0000502 encoded = obj.encode('utf-8', 'surrogatepass')
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000503 n = len(encoded)
504 self.write(BINUNICODE + pack("<i", n) + encoded)
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000505 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000506 obj = obj.replace("\\", "\\u005c")
507 obj = obj.replace("\n", "\\u000a")
Guido van Rossum1255ed62007-05-04 20:30:19 +0000508 self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) +
509 b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000510 self.memoize(obj)
Guido van Rossumf4169812008-03-17 22:56:06 +0000511 dispatch[str] = save_str
Tim Peters658cba62001-02-09 20:06:00 +0000512
Guido van Rossum3a41c612003-01-28 15:10:22 +0000513 def save_tuple(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000514 write = self.write
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000515 proto = self.proto
516
Guido van Rossum3a41c612003-01-28 15:10:22 +0000517 n = len(obj)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000518 if n == 0:
519 if proto:
520 write(EMPTY_TUPLE)
521 else:
522 write(MARK + TUPLE)
Tim Petersd97da802003-01-28 05:48:29 +0000523 return
524
525 save = self.save
526 memo = self.memo
527 if n <= 3 and proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000528 for element in obj:
Tim Petersd97da802003-01-28 05:48:29 +0000529 save(element)
530 # Subtle. Same as in the big comment below.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000531 if id(obj) in memo:
532 get = self.get(memo[id(obj)][0])
Tim Petersd97da802003-01-28 05:48:29 +0000533 write(POP * n + get)
534 else:
535 write(_tuplesize2code[n])
Guido van Rossum3a41c612003-01-28 15:10:22 +0000536 self.memoize(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000537 return
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000538
Tim Peters1d63c9f2003-02-02 20:29:39 +0000539 # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
Tim Petersff57bff2003-01-28 05:34:53 +0000540 # has more than 3 elements.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000541 write(MARK)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000542 for element in obj:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000543 save(element)
544
Tim Peters1d63c9f2003-02-02 20:29:39 +0000545 if id(obj) in memo:
Tim Petersf558da02003-01-28 02:09:55 +0000546 # Subtle. d was not in memo when we entered save_tuple(), so
547 # the process of saving the tuple's elements must have saved
548 # the tuple itself: the tuple is recursive. The proper action
549 # now is to throw away everything we put on the stack, and
550 # simply GET the tuple (it's already constructed). This check
551 # could have been done in the "for element" loop instead, but
552 # recursive tuples are a rare thing.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000553 get = self.get(memo[id(obj)][0])
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000554 if proto:
Tim Petersf558da02003-01-28 02:09:55 +0000555 write(POP_MARK + get)
556 else: # proto 0 -- POP_MARK not available
Tim Petersd97da802003-01-28 05:48:29 +0000557 write(POP * (n+1) + get)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000558 return
559
Tim Peters1d63c9f2003-02-02 20:29:39 +0000560 # No recursion.
Tim Peters518df0d2003-01-28 01:00:38 +0000561 self.write(TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000562 self.memoize(obj)
Jeremy Hylton3422c992003-01-24 19:29:52 +0000563
Guido van Rossum13257902007-06-07 23:15:56 +0000564 dispatch[tuple] = save_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000565
Guido van Rossum3a41c612003-01-28 15:10:22 +0000566 def save_list(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000567 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000568
Tim Petersc32d8242001-04-10 02:48:53 +0000569 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000570 write(EMPTY_LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000571 else: # proto 0 -- can't use EMPTY_LIST
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000572 write(MARK + LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000573
574 self.memoize(obj)
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000575 self._batch_appends(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000576
Guido van Rossum13257902007-06-07 23:15:56 +0000577 dispatch[list] = save_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000578
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000579 _BATCHSIZE = 1000
580
581 def _batch_appends(self, items):
582 # Helper to batch up APPENDS sequences
583 save = self.save
584 write = self.write
585
586 if not self.bin:
587 for x in items:
588 save(x)
589 write(APPEND)
590 return
591
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000592 items = iter(items)
Guido van Rossum805365e2007-05-07 22:24:25 +0000593 r = range(self._BATCHSIZE)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000594 while items is not None:
595 tmp = []
596 for i in r:
597 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000598 x = next(items)
Guido van Rossum5aac4e62003-02-06 22:57:00 +0000599 tmp.append(x)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000600 except StopIteration:
601 items = None
602 break
603 n = len(tmp)
604 if n > 1:
605 write(MARK)
606 for x in tmp:
607 save(x)
608 write(APPENDS)
609 elif n:
610 save(tmp[0])
611 write(APPEND)
612 # else tmp is empty, and we're done
613
Guido van Rossum3a41c612003-01-28 15:10:22 +0000614 def save_dict(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000615 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000616
Tim Petersc32d8242001-04-10 02:48:53 +0000617 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000618 write(EMPTY_DICT)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000619 else: # proto 0 -- can't use EMPTY_DICT
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000620 write(MARK + DICT)
621
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000622 self.memoize(obj)
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000623 self._batch_setitems(obj.items())
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000624
Guido van Rossum13257902007-06-07 23:15:56 +0000625 dispatch[dict] = save_dict
626 if PyStringMap is not None:
Jeremy Hylton2b9d0291998-05-27 22:38:22 +0000627 dispatch[PyStringMap] = save_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000628
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000629 def _batch_setitems(self, items):
630 # Helper to batch up SETITEMS sequences; proto >= 1 only
631 save = self.save
632 write = self.write
633
634 if not self.bin:
635 for k, v in items:
636 save(k)
637 save(v)
638 write(SETITEM)
639 return
640
Alexandre Vassalottic7db1d62008-05-14 21:57:18 +0000641 items = iter(items)
Guido van Rossum805365e2007-05-07 22:24:25 +0000642 r = range(self._BATCHSIZE)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000643 while items is not None:
644 tmp = []
645 for i in r:
646 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000647 tmp.append(next(items))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000648 except StopIteration:
649 items = None
650 break
651 n = len(tmp)
652 if n > 1:
653 write(MARK)
654 for k, v in tmp:
655 save(k)
656 save(v)
657 write(SETITEMS)
658 elif n:
659 k, v = tmp[0]
660 save(k)
661 save(v)
662 write(SETITEM)
663 # else tmp is empty, and we're done
664
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000665 def save_global(self, obj, name=None, pack=struct.pack):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000666 write = self.write
667 memo = self.memo
668
Tim Petersc32d8242001-04-10 02:48:53 +0000669 if name is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000670 name = obj.__name__
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000671
Jeremy Hylton4f0dcc92003-01-31 18:33:18 +0000672 module = getattr(obj, "__module__", None)
673 if module is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000674 module = whichmodule(obj, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000675
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000676 try:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000677 __import__(module, level=0)
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000678 mod = sys.modules[module]
679 klass = getattr(mod, name)
680 except (ImportError, KeyError, AttributeError):
681 raise PicklingError(
682 "Can't pickle %r: it's not found as %s.%s" %
Guido van Rossum3a41c612003-01-28 15:10:22 +0000683 (obj, module, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000684 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000685 if klass is not obj:
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000686 raise PicklingError(
687 "Can't pickle %r: it's not the same object as %s.%s" %
Guido van Rossum3a41c612003-01-28 15:10:22 +0000688 (obj, module, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000689
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000690 if self.proto >= 2:
Guido van Rossumd4b920c2003-02-04 01:54:49 +0000691 code = _extension_registry.get((module, name))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000692 if code:
693 assert code > 0
694 if code <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000695 write(EXT1 + bytes([code]))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000696 elif code <= 0xffff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000697 write(EXT2 + bytes([code&0xff, code>>8]))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000698 else:
699 write(EXT4 + pack("<i", code))
700 return
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000701 # Non-ASCII identifiers are supported only with protocols >= 3.
702 if self.proto >= 3:
703 write(GLOBAL + bytes(module, "utf-8") + b'\n' +
704 bytes(name, "utf-8") + b'\n')
705 else:
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000706 if self.fix_imports:
707 if (module, name) in _compat_pickle.REVERSE_NAME_MAPPING:
708 module, name = _compat_pickle.REVERSE_NAME_MAPPING[(module, name)]
709 if module in _compat_pickle.REVERSE_IMPORT_MAPPING:
710 module = _compat_pickle.REVERSE_IMPORT_MAPPING[module]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000711 try:
712 write(GLOBAL + bytes(module, "ascii") + b'\n' +
713 bytes(name, "ascii") + b'\n')
714 except UnicodeEncodeError:
715 raise PicklingError(
716 "can't pickle global identifier '%s.%s' using "
717 "pickle protocol %i" % (module, name, self.proto))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000718
Guido van Rossum3a41c612003-01-28 15:10:22 +0000719 self.memoize(obj)
Tim Peters3b769832003-01-28 03:51:36 +0000720
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000721 dispatch[FunctionType] = save_global
722 dispatch[BuiltinFunctionType] = save_global
Guido van Rossum13257902007-06-07 23:15:56 +0000723 dispatch[type] = save_global
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000724
Guido van Rossum1be31752003-01-28 15:19:53 +0000725# Pickling helpers
Guido van Rossuma48061a1995-01-10 00:31:14 +0000726
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000727def _keep_alive(x, memo):
728 """Keeps a reference to the object x in the memo.
729
730 Because we remember objects by their id, we have
731 to assure that possibly temporary objects are kept
732 alive by referencing them.
733 We store a reference at the id of the memo, which should
734 normally not be used unless someone tries to deepcopy
735 the memo itself...
736 """
737 try:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000738 memo[id(memo)].append(x)
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000739 except KeyError:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000740 # aha, this is the first one :-)
741 memo[id(memo)]=[x]
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000742
743
Tim Petersc0c12b52003-01-29 00:56:17 +0000744# A cache for whichmodule(), mapping a function object to the name of
745# the module in which the function was found.
746
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000747classmap = {} # called classmap for backwards compatibility
Guido van Rossuma48061a1995-01-10 00:31:14 +0000748
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000749def whichmodule(func, funcname):
750 """Figure out the module in which a function occurs.
Guido van Rossuma48061a1995-01-10 00:31:14 +0000751
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000752 Search sys.modules for the module.
753 Cache in classmap.
754 Return a module name.
Tim Petersc0c12b52003-01-29 00:56:17 +0000755 If the function cannot be found, return "__main__".
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000756 """
Jeremy Hylton4f0dcc92003-01-31 18:33:18 +0000757 # Python functions should always get an __module__ from their globals.
758 mod = getattr(func, "__module__", None)
759 if mod is not None:
760 return mod
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000761 if func in classmap:
762 return classmap[func]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000763
Guido van Rossum634e53f2007-02-26 07:07:02 +0000764 for name, module in list(sys.modules.items()):
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000765 if module is None:
Jeremy Hylton065a5ab2002-09-19 22:57:26 +0000766 continue # skip dummy package entries
Jeremy Hyltoncc1fccb2003-02-06 16:23:01 +0000767 if name != '__main__' and getattr(module, funcname, None) is func:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000768 break
769 else:
770 name = '__main__'
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000771 classmap[func] = name
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000772 return name
Guido van Rossuma48061a1995-01-10 00:31:14 +0000773
774
Guido van Rossum1be31752003-01-28 15:19:53 +0000775# Unpickling machinery
776
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000777class _Unpickler:
Guido van Rossuma48061a1995-01-10 00:31:14 +0000778
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000779 def __init__(self, file, *, fix_imports=True,
780 encoding="ASCII", errors="strict"):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000781 """This takes a binary file for reading a pickle data stream.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000782
Tim Peters5bd2a792003-02-01 16:45:06 +0000783 The protocol version of the pickle is detected automatically, so no
784 proto argument is needed.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000785
Guido van Rossumfeea0782007-10-10 18:00:50 +0000786 The file-like object must have two methods, a read() method
787 that takes an integer argument, and a readline() method that
788 requires no arguments. Both methods should return bytes.
789 Thus file-like object can be a binary file object opened for
790 reading, a BytesIO object, or any other custom object that
791 meets this interface.
Guido van Rossumf4169812008-03-17 22:56:06 +0000792
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000793 Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
794 which are used to control compatiblity support for pickle stream
795 generated by Python 2.x. If *fix_imports* is True, pickle will try to
796 map the old Python 2.x names to the new names used in Python 3.x. The
797 *encoding* and *errors* tell pickle how to decode 8-bit string
798 instances pickled by Python 2.x; these default to 'ASCII' and
799 'strict', respectively.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000800 """
Guido van Rossumfeea0782007-10-10 18:00:50 +0000801 self.readline = file.readline
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000802 self.read = file.read
803 self.memo = {}
Guido van Rossumf4169812008-03-17 22:56:06 +0000804 self.encoding = encoding
805 self.errors = errors
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000806 self.proto = 0
807 self.fix_imports = fix_imports
Guido van Rossuma48061a1995-01-10 00:31:14 +0000808
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000809 def load(self):
Guido van Rossum3a41c612003-01-28 15:10:22 +0000810 """Read a pickled object representation from the open file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000811
Guido van Rossum3a41c612003-01-28 15:10:22 +0000812 Return the reconstituted object hierarchy specified in the file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000813 """
Alexandre Vassalotti3cfcab92008-12-27 09:30:39 +0000814 # Check whether Unpickler was initialized correctly. This is
815 # only needed to mimic the behavior of _pickle.Unpickler.dump().
816 if not hasattr(self, "read"):
817 raise UnpicklingError("Unpickler.__init__() was not called by "
818 "%s.__init__()" % (self.__class__.__name__,))
Jeremy Hylton20747fa2001-11-09 16:15:04 +0000819 self.mark = object() # any new unique object
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000820 self.stack = []
821 self.append = self.stack.append
822 read = self.read
823 dispatch = self.dispatch
824 try:
825 while 1:
826 key = read(1)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000827 if not key:
828 raise EOFError
Guido van Rossum98297ee2007-11-06 21:34:58 +0000829 assert isinstance(key, bytes_types)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000830 dispatch[key[0]](self)
Guido van Rossumb940e112007-01-10 16:19:56 +0000831 except _Stop as stopinst:
Guido van Rossumff871742000-12-13 18:11:56 +0000832 return stopinst.value
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000833
Tim Petersc23d18a2003-01-28 01:41:51 +0000834 # Return largest index k such that self.stack[k] is self.mark.
835 # If the stack doesn't contain a mark, eventually raises IndexError.
836 # This could be sped by maintaining another stack, of indices at which
837 # the mark appears. For that matter, the latter stack would suffice,
838 # and we wouldn't need to push mark objects on self.stack at all.
839 # Doing so is probably a good thing, though, since if the pickle is
840 # corrupt (or hostile) we may get a clue from finding self.mark embedded
841 # in unpickled objects.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000842 def marker(self):
843 stack = self.stack
844 mark = self.mark
845 k = len(stack)-1
846 while stack[k] is not mark: k = k-1
847 return k
848
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000849 def persistent_load(self, pid):
Benjamin Peterson49956b22009-01-10 17:05:44 +0000850 raise UnpicklingError("unsupported persistent id encountered")
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000851
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000852 dispatch = {}
853
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000854 def load_proto(self):
855 proto = ord(self.read(1))
Guido van Rossumf4169812008-03-17 22:56:06 +0000856 if not 0 <= proto <= HIGHEST_PROTOCOL:
Guido van Rossum26d95c32007-08-27 23:18:54 +0000857 raise ValueError("unsupported pickle protocol: %d" % proto)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000858 self.proto = proto
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000859 dispatch[PROTO[0]] = load_proto
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000860
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000861 def load_persid(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000862 pid = self.readline()[:-1].decode("ascii")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000863 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000864 dispatch[PERSID[0]] = load_persid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000865
866 def load_binpersid(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +0000867 pid = self.stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000868 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000869 dispatch[BINPERSID[0]] = load_binpersid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000870
871 def load_none(self):
872 self.append(None)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000873 dispatch[NONE[0]] = load_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000874
Guido van Rossum7d97d312003-01-28 04:25:27 +0000875 def load_false(self):
876 self.append(False)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000877 dispatch[NEWFALSE[0]] = load_false
Guido van Rossum7d97d312003-01-28 04:25:27 +0000878
879 def load_true(self):
880 self.append(True)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000881 dispatch[NEWTRUE[0]] = load_true
Guido van Rossum7d97d312003-01-28 04:25:27 +0000882
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000883 def load_int(self):
Tim Peters19ef62d2001-08-28 22:21:18 +0000884 data = self.readline()
Guido van Rossume2763392002-04-05 19:30:08 +0000885 if data == FALSE[1:]:
886 val = False
887 elif data == TRUE[1:]:
888 val = True
889 else:
890 try:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000891 val = int(data, 0)
Guido van Rossume2763392002-04-05 19:30:08 +0000892 except ValueError:
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000893 val = int(data, 0)
Guido van Rossume2763392002-04-05 19:30:08 +0000894 self.append(val)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000895 dispatch[INT[0]] = load_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000896
897 def load_binint(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000898 self.append(mloads(b'i' + self.read(4)))
899 dispatch[BININT[0]] = load_binint
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000900
901 def load_binint1(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +0000902 self.append(ord(self.read(1)))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000903 dispatch[BININT1[0]] = load_binint1
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000904
905 def load_binint2(self):
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000906 self.append(mloads(b'i' + self.read(2) + b'\000\000'))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000907 dispatch[BININT2[0]] = load_binint2
Tim Peters2344fae2001-01-15 00:50:52 +0000908
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000909 def load_long(self):
Guido van Rossumfeea0782007-10-10 18:00:50 +0000910 val = self.readline()[:-1].decode("ascii")
Mark Dickinson8dd05142009-01-20 20:43:58 +0000911 if val and val[-1] == 'L':
912 val = val[:-1]
Guido van Rossumfeea0782007-10-10 18:00:50 +0000913 self.append(int(val, 0))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000914 dispatch[LONG[0]] = load_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000915
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000916 def load_long1(self):
917 n = ord(self.read(1))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000918 data = self.read(n)
919 self.append(decode_long(data))
920 dispatch[LONG1[0]] = load_long1
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000921
922 def load_long4(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000923 n = mloads(b'i' + self.read(4))
924 data = self.read(n)
925 self.append(decode_long(data))
926 dispatch[LONG4[0]] = load_long4
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000927
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000928 def load_float(self):
Guido van Rossumff871742000-12-13 18:11:56 +0000929 self.append(float(self.readline()[:-1]))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000930 dispatch[FLOAT[0]] = load_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000931
Guido van Rossumd3703791998-10-22 20:15:36 +0000932 def load_binfloat(self, unpack=struct.unpack):
933 self.append(unpack('>d', self.read(8))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000934 dispatch[BINFLOAT[0]] = load_binfloat
Guido van Rossumd3703791998-10-22 20:15:36 +0000935
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000936 def load_string(self):
Guido van Rossum98297ee2007-11-06 21:34:58 +0000937 orig = self.readline()
938 rep = orig[:-1]
Guido van Rossum26d95c32007-08-27 23:18:54 +0000939 for q in (b'"', b"'"): # double or single quote
Martin v. Löwis8a8da792002-08-14 07:46:28 +0000940 if rep.startswith(q):
941 if not rep.endswith(q):
Guido van Rossum26d95c32007-08-27 23:18:54 +0000942 raise ValueError("insecure string pickle")
Martin v. Löwis8a8da792002-08-14 07:46:28 +0000943 rep = rep[len(q):-len(q)]
944 break
945 else:
Guido van Rossum98297ee2007-11-06 21:34:58 +0000946 raise ValueError("insecure string pickle: %r" % orig)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000947 self.append(codecs.escape_decode(rep)[0]
948 .decode(self.encoding, self.errors))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000949 dispatch[STRING[0]] = load_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000950
951 def load_binstring(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000952 len = mloads(b'i' + self.read(4))
Guido van Rossumf4169812008-03-17 22:56:06 +0000953 data = self.read(len)
954 value = str(data, self.encoding, self.errors)
955 self.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000956 dispatch[BINSTRING[0]] = load_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000957
Guido van Rossumf4169812008-03-17 22:56:06 +0000958 def load_binbytes(self):
959 len = mloads(b'i' + self.read(4))
960 self.append(self.read(len))
961 dispatch[BINBYTES[0]] = load_binbytes
962
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000963 def load_unicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000964 self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
965 dispatch[UNICODE[0]] = load_unicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000966
967 def load_binunicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000968 len = mloads(b'i' + self.read(4))
Victor Stinner485fb562010-04-13 11:07:24 +0000969 self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000970 dispatch[BINUNICODE[0]] = load_binunicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000971
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000972 def load_short_binstring(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +0000973 len = ord(self.read(1))
Guido van Rossumf4169812008-03-17 22:56:06 +0000974 data = bytes(self.read(len))
975 value = str(data, self.encoding, self.errors)
976 self.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000977 dispatch[SHORT_BINSTRING[0]] = load_short_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000978
Guido van Rossumf4169812008-03-17 22:56:06 +0000979 def load_short_binbytes(self):
980 len = ord(self.read(1))
981 self.append(bytes(self.read(len)))
982 dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
983
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000984 def load_tuple(self):
985 k = self.marker()
986 self.stack[k:] = [tuple(self.stack[k+1:])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000987 dispatch[TUPLE[0]] = load_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000988
989 def load_empty_tuple(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000990 self.append(())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000991 dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000992
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000993 def load_tuple1(self):
994 self.stack[-1] = (self.stack[-1],)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000995 dispatch[TUPLE1[0]] = load_tuple1
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000996
997 def load_tuple2(self):
998 self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000999 dispatch[TUPLE2[0]] = load_tuple2
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001000
1001 def load_tuple3(self):
1002 self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001003 dispatch[TUPLE3[0]] = load_tuple3
Guido van Rossum44f0ea52003-01-28 04:14:51 +00001004
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001005 def load_empty_list(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001006 self.append([])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001007 dispatch[EMPTY_LIST[0]] = load_empty_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001008
1009 def load_empty_dictionary(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001010 self.append({})
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001011 dispatch[EMPTY_DICT[0]] = load_empty_dictionary
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001012
1013 def load_list(self):
1014 k = self.marker()
1015 self.stack[k:] = [self.stack[k+1:]]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001016 dispatch[LIST[0]] = load_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001017
1018 def load_dict(self):
1019 k = self.marker()
1020 d = {}
1021 items = self.stack[k+1:]
1022 for i in range(0, len(items), 2):
1023 key = items[i]
1024 value = items[i+1]
1025 d[key] = value
1026 self.stack[k:] = [d]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001027 dispatch[DICT[0]] = load_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001028
Tim Petersd01c1e92003-01-30 15:41:46 +00001029 # INST and OBJ differ only in how they get a class object. It's not
1030 # only sensible to do the rest in a common routine, the two routines
1031 # previously diverged and grew different bugs.
1032 # klass is the class to instantiate, and k points to the topmost mark
1033 # object, following which are the arguments for klass.__init__.
1034 def _instantiate(self, klass, k):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001035 args = tuple(self.stack[k+1:])
1036 del self.stack[k:]
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001037 if (args or not isinstance(klass, type) or
1038 hasattr(klass, "__getinitargs__")):
Guido van Rossum743d17e1998-09-15 20:25:57 +00001039 try:
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001040 value = klass(*args)
Guido van Rossumb940e112007-01-10 16:19:56 +00001041 except TypeError as err:
Guido van Rossum26d95c32007-08-27 23:18:54 +00001042 raise TypeError("in constructor for %s: %s" %
1043 (klass.__name__, str(err)), sys.exc_info()[2])
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001044 else:
1045 value = klass.__new__(klass)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001046 self.append(value)
Tim Petersd01c1e92003-01-30 15:41:46 +00001047
1048 def load_inst(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001049 module = self.readline()[:-1].decode("ascii")
1050 name = self.readline()[:-1].decode("ascii")
Tim Petersd01c1e92003-01-30 15:41:46 +00001051 klass = self.find_class(module, name)
1052 self._instantiate(klass, self.marker())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001053 dispatch[INST[0]] = load_inst
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001054
1055 def load_obj(self):
Tim Petersd01c1e92003-01-30 15:41:46 +00001056 # Stack is ... markobject classobject arg1 arg2 ...
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001057 k = self.marker()
Tim Petersd01c1e92003-01-30 15:41:46 +00001058 klass = self.stack.pop(k+1)
1059 self._instantiate(klass, k)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001060 dispatch[OBJ[0]] = load_obj
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001061
Guido van Rossum3a41c612003-01-28 15:10:22 +00001062 def load_newobj(self):
1063 args = self.stack.pop()
1064 cls = self.stack[-1]
1065 obj = cls.__new__(cls, *args)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001066 self.stack[-1] = obj
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001067 dispatch[NEWOBJ[0]] = load_newobj
Guido van Rossum3a41c612003-01-28 15:10:22 +00001068
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001069 def load_global(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001070 module = self.readline()[:-1].decode("utf-8")
1071 name = self.readline()[:-1].decode("utf-8")
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001072 klass = self.find_class(module, name)
1073 self.append(klass)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001074 dispatch[GLOBAL[0]] = load_global
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001075
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001076 def load_ext1(self):
1077 code = ord(self.read(1))
1078 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001079 dispatch[EXT1[0]] = load_ext1
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001080
1081 def load_ext2(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001082 code = mloads(b'i' + self.read(2) + b'\000\000')
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001083 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001084 dispatch[EXT2[0]] = load_ext2
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001085
1086 def load_ext4(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001087 code = mloads(b'i' + self.read(4))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001088 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001089 dispatch[EXT4[0]] = load_ext4
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001090
1091 def get_extension(self, code):
1092 nil = []
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001093 obj = _extension_cache.get(code, nil)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001094 if obj is not nil:
1095 self.append(obj)
1096 return
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001097 key = _inverted_registry.get(code)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001098 if not key:
1099 raise ValueError("unregistered extension code %d" % code)
1100 obj = self.find_class(*key)
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001101 _extension_cache[code] = obj
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001102 self.append(obj)
1103
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001104 def find_class(self, module, name):
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001105 # Subclasses may override this.
1106 if self.proto < 3 and self.fix_imports:
1107 if (module, name) in _compat_pickle.NAME_MAPPING:
1108 module, name = _compat_pickle.NAME_MAPPING[(module, name)]
1109 if module in _compat_pickle.IMPORT_MAPPING:
1110 module = _compat_pickle.IMPORT_MAPPING[module]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001111 __import__(module, level=0)
Barry Warsawbf4d9592001-11-15 23:42:58 +00001112 mod = sys.modules[module]
1113 klass = getattr(mod, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001114 return klass
1115
1116 def load_reduce(self):
1117 stack = self.stack
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001118 args = stack.pop()
1119 func = stack[-1]
Guido van Rossum99603b02007-07-20 00:22:32 +00001120 try:
1121 value = func(*args)
1122 except:
1123 print(sys.exc_info())
1124 print(func, args)
1125 raise
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001126 stack[-1] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001127 dispatch[REDUCE[0]] = load_reduce
Guido van Rossuma48061a1995-01-10 00:31:14 +00001128
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001129 def load_pop(self):
1130 del self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001131 dispatch[POP[0]] = load_pop
Guido van Rossum7b5430f1995-03-04 22:25:21 +00001132
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001133 def load_pop_mark(self):
1134 k = self.marker()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001135 del self.stack[k:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001136 dispatch[POP_MARK[0]] = load_pop_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001137
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001138 def load_dup(self):
Guido van Rossumb1062fc1998-03-31 17:00:46 +00001139 self.append(self.stack[-1])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001140 dispatch[DUP[0]] = load_dup
Guido van Rossuma48061a1995-01-10 00:31:14 +00001141
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001142 def load_get(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001143 i = int(self.readline()[:-1])
1144 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001145 dispatch[GET[0]] = load_get
Guido van Rossum78536471996-04-12 13:36:27 +00001146
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001147 def load_binget(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001148 i = self.read(1)[0]
1149 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001150 dispatch[BINGET[0]] = load_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001151
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001152 def load_long_binget(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001153 i = mloads(b'i' + self.read(4))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001154 self.append(self.memo[i])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001155 dispatch[LONG_BINGET[0]] = load_long_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001156
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001157 def load_put(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001158 i = int(self.readline()[:-1])
1159 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001160 dispatch[PUT[0]] = load_put
Guido van Rossuma48061a1995-01-10 00:31:14 +00001161
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001162 def load_binput(self):
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001163 i = self.read(1)[0]
1164 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001165 dispatch[BINPUT[0]] = load_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001166
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001167 def load_long_binput(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001168 i = mloads(b'i' + self.read(4))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001169 self.memo[i] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001170 dispatch[LONG_BINPUT[0]] = load_long_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001171
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001172 def load_append(self):
1173 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001174 value = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001175 list = stack[-1]
1176 list.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001177 dispatch[APPEND[0]] = load_append
Guido van Rossuma48061a1995-01-10 00:31:14 +00001178
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001179 def load_appends(self):
1180 stack = self.stack
1181 mark = self.marker()
1182 list = stack[mark - 1]
Tim Peters209ad952003-01-28 01:44:45 +00001183 list.extend(stack[mark + 1:])
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001184 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001185 dispatch[APPENDS[0]] = load_appends
Tim Peters2344fae2001-01-15 00:50:52 +00001186
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001187 def load_setitem(self):
1188 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001189 value = stack.pop()
1190 key = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001191 dict = stack[-1]
1192 dict[key] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001193 dispatch[SETITEM[0]] = load_setitem
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001194
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001195 def load_setitems(self):
1196 stack = self.stack
1197 mark = self.marker()
1198 dict = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001199 for i in range(mark + 1, len(stack), 2):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001200 dict[stack[i]] = stack[i + 1]
Guido van Rossuma48061a1995-01-10 00:31:14 +00001201
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001202 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001203 dispatch[SETITEMS[0]] = load_setitems
Guido van Rossuma48061a1995-01-10 00:31:14 +00001204
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001205 def load_build(self):
1206 stack = self.stack
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001207 state = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001208 inst = stack[-1]
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001209 setstate = getattr(inst, "__setstate__", None)
1210 if setstate:
1211 setstate(state)
1212 return
1213 slotstate = None
1214 if isinstance(state, tuple) and len(state) == 2:
1215 state, slotstate = state
1216 if state:
Alexandre Vassalottiebfecfd2009-05-25 18:50:33 +00001217 inst_dict = inst.__dict__
Antoine Pitroua9f48a02009-05-02 21:41:14 +00001218 intern = sys.intern
Alexandre Vassalottiebfecfd2009-05-25 18:50:33 +00001219 for k, v in state.items():
1220 if type(k) is str:
1221 inst_dict[intern(k)] = v
1222 else:
1223 inst_dict[k] = v
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001224 if slotstate:
1225 for k, v in slotstate.items():
1226 setattr(inst, k, v)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001227 dispatch[BUILD[0]] = load_build
Guido van Rossuma48061a1995-01-10 00:31:14 +00001228
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001229 def load_mark(self):
1230 self.append(self.mark)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001231 dispatch[MARK[0]] = load_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001232
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001233 def load_stop(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001234 value = self.stack.pop()
Guido van Rossumff871742000-12-13 18:11:56 +00001235 raise _Stop(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001236 dispatch[STOP[0]] = load_stop
Guido van Rossuma48061a1995-01-10 00:31:14 +00001237
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00001238# Encode/decode longs.
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001239
1240def encode_long(x):
Tim Peters91149822003-01-31 03:43:58 +00001241 r"""Encode a long to a two's complement little-endian binary string.
Guido van Rossume2a383d2007-01-15 16:59:06 +00001242 Note that 0 is a special case, returning an empty string, to save a
Tim Peters4b23f2b2003-01-31 16:43:39 +00001243 byte in the LONG1 pickling context.
1244
Guido van Rossume2a383d2007-01-15 16:59:06 +00001245 >>> encode_long(0)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001246 b''
Guido van Rossume2a383d2007-01-15 16:59:06 +00001247 >>> encode_long(255)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001248 b'\xff\x00'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001249 >>> encode_long(32767)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001250 b'\xff\x7f'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001251 >>> encode_long(-256)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001252 b'\x00\xff'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001253 >>> encode_long(-32768)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001254 b'\x00\x80'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001255 >>> encode_long(-128)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001256 b'\x80'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001257 >>> encode_long(127)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001258 b'\x7f'
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001259 >>>
1260 """
Tim Peters91149822003-01-31 03:43:58 +00001261 if x == 0:
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001262 return b''
Alexandre Vassalottid7a3da82010-01-12 01:49:31 +00001263 nbytes = (x.bit_length() >> 3) + 1
1264 result = x.to_bytes(nbytes, byteorder='little', signed=True)
1265 if x < 0 and nbytes > 1:
1266 if result[-1] == 0xff and (result[-2] & 0x80) != 0:
1267 result = result[:-1]
1268 return result
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001269
1270def decode_long(data):
1271 r"""Decode a long from a two's complement little-endian binary string.
Tim Peters4b23f2b2003-01-31 16:43:39 +00001272
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001273 >>> decode_long(b'')
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001274 0
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001275 >>> decode_long(b"\xff\x00")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001276 255
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001277 >>> decode_long(b"\xff\x7f")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001278 32767
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001279 >>> decode_long(b"\x00\xff")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001280 -256
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001281 >>> decode_long(b"\x00\x80")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001282 -32768
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001283 >>> decode_long(b"\x80")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001284 -128
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001285 >>> decode_long(b"\x7f")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001286 127
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001287 """
Alexandre Vassalottid7a3da82010-01-12 01:49:31 +00001288 return int.from_bytes(data, byteorder='little', signed=True)
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001289
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001290# Shorthands
1291
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001292def dump(obj, file, protocol=None, *, fix_imports=True):
1293 Pickler(file, protocol, fix_imports=fix_imports).dump(obj)
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001294
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001295def dumps(obj, protocol=None, *, fix_imports=True):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001296 f = io.BytesIO()
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001297 Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001298 res = f.getvalue()
Guido van Rossum98297ee2007-11-06 21:34:58 +00001299 assert isinstance(res, bytes_types)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001300 return res
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001301
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001302def load(file, *, fix_imports=True, encoding="ASCII", errors="strict"):
1303 return Unpickler(file, fix_imports=fix_imports,
1304 encoding=encoding, errors=errors).load()
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001305
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001306def loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001307 if isinstance(s, str):
1308 raise TypeError("Can't load pickle from unicode string")
1309 file = io.BytesIO(s)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00001310 return Unpickler(file, fix_imports=fix_imports,
1311 encoding=encoding, errors=errors).load()
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001312
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001313# Use the faster _pickle if possible
1314try:
1315 from _pickle import *
1316except ImportError:
1317 Pickler, Unpickler = _Pickler, _Unpickler
1318
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001319# Doctest
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001320def _test():
1321 import doctest
1322 return doctest.testmod()
1323
1324if __name__ == "__main__":
Alexander Belopolsky455f7bd2010-07-27 23:02:38 +00001325 import sys, argparse
1326 parser = argparse.ArgumentParser(
1327 description='display contents of the pickle files')
1328 parser.add_argument(
1329 'pickle_file', type=argparse.FileType('br'),
1330 nargs='*', help='the pickle file')
1331 parser.add_argument(
1332 '-t', '--test', action='store_true',
1333 help='run self-test suite')
1334 parser.add_argument(
1335 '-v', action='store_true',
1336 help='run verbosely; only affects self-test run')
1337 args = parser.parse_args()
1338 if args.test:
1339 _test()
1340 else:
1341 if not args.pickle_file:
1342 parser.print_help()
1343 else:
1344 import pprint
1345 for f in args.pickle_file:
1346 obj = load(f)
1347 pprint.pprint(obj)