blob: 9127f142a905c1a1c1dba06b15cf8c121b382a67 [file] [log] [blame]
Guido van Rossum54f22ed2000-02-04 15:10:34 +00001"""Create portable serialized representations of Python objects.
Guido van Rossuma48061a1995-01-10 00:31:14 +00002
Guido van Rossume467be61997-12-05 19:42:42 +00003See module copy_reg for a mechanism for registering custom picklers.
Tim Peters22a449a2003-01-27 20:16:36 +00004See module pickletools source for extensive comments.
Guido van Rossuma48061a1995-01-10 00:31:14 +00005
Guido van Rossume467be61997-12-05 19:42:42 +00006Classes:
Guido van Rossuma48061a1995-01-10 00:31:14 +00007
Guido van Rossume467be61997-12-05 19:42:42 +00008 Pickler
9 Unpickler
Guido van Rossuma48061a1995-01-10 00:31:14 +000010
Guido van Rossume467be61997-12-05 19:42:42 +000011Functions:
Guido van Rossuma48061a1995-01-10 00:31:14 +000012
Guido van Rossume467be61997-12-05 19:42:42 +000013 dump(object, file)
14 dumps(object) -> string
15 load(file) -> object
16 loads(string) -> object
Guido van Rossuma48061a1995-01-10 00:31:14 +000017
Guido van Rossume467be61997-12-05 19:42:42 +000018Misc variables:
Guido van Rossuma48061a1995-01-10 00:31:14 +000019
Fred Drakefe82acc1998-02-13 03:24:48 +000020 __version__
Guido van Rossume467be61997-12-05 19:42:42 +000021 format_version
22 compatible_formats
Guido van Rossuma48061a1995-01-10 00:31:14 +000023
Guido van Rossuma48061a1995-01-10 00:31:14 +000024"""
25
Guido van Rossum743d17e1998-09-15 20:25:57 +000026__version__ = "$Revision$" # Code version
Guido van Rossuma48061a1995-01-10 00:31:14 +000027
Guido van Rossum13257902007-06-07 23:15:56 +000028from types import FunctionType, BuiltinFunctionType
Guido van Rossum443ada42003-02-18 22:49:10 +000029from copy_reg import dispatch_table
Guido van Rossumd4b920c2003-02-04 01:54:49 +000030from copy_reg import _extension_registry, _inverted_registry, _extension_cache
Guido van Rossumd3703791998-10-22 20:15:36 +000031import marshal
32import sys
33import struct
Skip Montanaro23bafc62001-02-18 03:10:09 +000034import re
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000035import io
Walter Dörwald42748a82007-06-12 16:40:17 +000036import codecs
Guido van Rossuma48061a1995-01-10 00:31:14 +000037
Skip Montanaro352674d2001-02-07 23:14:30 +000038__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
39 "Unpickler", "dump", "dumps", "load", "loads"]
40
Tim Petersc0c12b52003-01-29 00:56:17 +000041# These are purely informational; no code uses these.
Guido van Rossumf29d3d62003-01-27 22:47:53 +000042format_version = "2.0" # File format version we write
43compatible_formats = ["1.0", # Original protocol 0
Guido van Rossumbc64e222003-01-28 16:34:19 +000044 "1.1", # Protocol 0 with INST added
Guido van Rossumf29d3d62003-01-27 22:47:53 +000045 "1.2", # Original protocol 1
46 "1.3", # Protocol 1 with BINFLOAT added
47 "2.0", # Protocol 2
48 ] # Old format versions we can read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000049
Guido van Rossum99603b02007-07-20 00:22:32 +000050# This is the highest protocol number we know how to read.
Tim Peters8587b3c2003-02-13 15:44:41 +000051HIGHEST_PROTOCOL = 2
52
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000053# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
54DEFAULT_PROTOCOL = 2
55
Guido van Rossume0b90422003-01-28 03:17:21 +000056# Why use struct.pack() for pickling but marshal.loads() for
Tim Petersc0c12b52003-01-29 00:56:17 +000057# unpickling? struct.pack() is 40% faster than marshal.dumps(), but
Guido van Rossume0b90422003-01-28 03:17:21 +000058# marshal.loads() is twice as fast as struct.unpack()!
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000059mloads = marshal.loads
Guido van Rossum0c891ce1995-03-14 15:09:05 +000060
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000061class PickleError(Exception):
Neal Norwitzefbb67b2002-05-30 12:12:04 +000062 """A common base class for the other pickling exceptions."""
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000063 pass
64
65class PicklingError(PickleError):
66 """This exception is raised when an unpicklable object is passed to the
67 dump() method.
68
69 """
70 pass
71
72class UnpicklingError(PickleError):
73 """This exception is raised when there is a problem unpickling an object,
74 such as a security violation.
75
76 Note that other exceptions may also be raised during unpickling, including
77 (but not necessarily limited to) AttributeError, EOFError, ImportError,
78 and IndexError.
79
80 """
81 pass
Guido van Rossum7849da81995-03-09 14:08:35 +000082
Tim Petersc0c12b52003-01-29 00:56:17 +000083# An instance of _Stop is raised by Unpickler.load_stop() in response to
84# the STOP opcode, passing the object that is the result of unpickling.
Guido van Rossumff871742000-12-13 18:11:56 +000085class _Stop(Exception):
86 def __init__(self, value):
87 self.value = value
88
Guido van Rossum533dbcf2003-01-28 17:55:05 +000089# Jython has PyStringMap; it's a dict subclass with string keys
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000090try:
91 from org.python.core import PyStringMap
92except ImportError:
93 PyStringMap = None
94
Tim Peters22a449a2003-01-27 20:16:36 +000095# Pickle opcodes. See pickletools.py for extensive docs. The listing
96# here is in kind-of alphabetical order of 1-character pickle code.
97# pickletools groups them by purpose.
Guido van Rossumdbb718f2001-09-21 19:22:34 +000098
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000099MARK = b'(' # push special markobject on stack
100STOP = b'.' # every pickle ends with STOP
101POP = b'0' # discard topmost stack item
102POP_MARK = b'1' # discard stack top through topmost markobject
103DUP = b'2' # duplicate top stack item
104FLOAT = b'F' # push float object; decimal string argument
105INT = b'I' # push integer or bool; decimal string argument
106BININT = b'J' # push four-byte signed int
107BININT1 = b'K' # push 1-byte unsigned int
108LONG = b'L' # push long; decimal string argument
109BININT2 = b'M' # push 2-byte unsigned int
110NONE = b'N' # push None
111PERSID = b'P' # push persistent object; id is taken from string arg
112BINPERSID = b'Q' # " " " ; " " " " stack
113REDUCE = b'R' # apply callable to argtuple, both on stack
114STRING = b'S' # push string; NL-terminated string argument
115BINSTRING = b'T' # push string; counted binary string argument
116SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
117UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
118BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
119APPEND = b'a' # append stack top to list below it
120BUILD = b'b' # call __setstate__ or __dict__.update()
121GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
122DICT = b'd' # build a dict from stack items
123EMPTY_DICT = b'}' # push empty dict
124APPENDS = b'e' # extend list on stack by topmost stack slice
125GET = b'g' # push item from memo on stack; index is string arg
126BINGET = b'h' # " " " " " " ; " " 1-byte arg
127INST = b'i' # build & push class instance
128LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
129LIST = b'l' # build list from topmost stack items
130EMPTY_LIST = b']' # push empty list
131OBJ = b'o' # build & push class instance
132PUT = b'p' # store stack top in memo; index is string arg
133BINPUT = b'q' # " " " " " ; " " 1-byte arg
134LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
135SETITEM = b's' # add key+value pair to dict
136TUPLE = b't' # build tuple from topmost stack items
137EMPTY_TUPLE = b')' # push empty tuple
138SETITEMS = b'u' # modify dict by adding topmost key+value pairs
139BINFLOAT = b'G' # push float; arg is 8-byte float encoding
Tim Peters22a449a2003-01-27 20:16:36 +0000140
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000141TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
142FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
Guido van Rossum77f6a652002-04-03 22:41:51 +0000143
Guido van Rossum586c9e82003-01-29 06:16:12 +0000144# Protocol 2
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000145
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000146PROTO = b'\x80' # identify pickle protocol
147NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
148EXT1 = b'\x82' # push object from extension registry; 1-byte index
149EXT2 = b'\x83' # ditto, but 2-byte index
150EXT4 = b'\x84' # ditto, but 4-byte index
151TUPLE1 = b'\x85' # build 1-tuple from stack top
152TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
153TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
154NEWTRUE = b'\x88' # push True
155NEWFALSE = b'\x89' # push False
156LONG1 = b'\x8a' # push long from < 256 bytes
157LONG4 = b'\x8b' # push really big long
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000158
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000159_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
160
Guido van Rossuma48061a1995-01-10 00:31:14 +0000161
Skip Montanaro23bafc62001-02-18 03:10:09 +0000162__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
163
Guido van Rossum1be31752003-01-28 15:19:53 +0000164
165# Pickling machinery
166
Guido van Rossuma48061a1995-01-10 00:31:14 +0000167class Pickler:
168
Raymond Hettinger3489cad2004-12-05 05:20:42 +0000169 def __init__(self, file, protocol=None):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000170 """This takes a binary file for writing a pickle data stream.
171
172 All protocols now read and write bytes.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000173
Guido van Rossumcf117b02003-02-09 17:19:41 +0000174 The optional protocol argument tells the pickler to use the
175 given protocol; supported protocols are 0, 1, 2. The default
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000176 protocol is 2; it's been supported for many years now.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000177
178 Protocol 1 is more efficient than protocol 0; protocol 2 is
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000179 more efficient than protocol 1.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000180
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000181 Specifying a negative protocol version selects the highest
Tim Peters5bd2a792003-02-01 16:45:06 +0000182 protocol version supported. The higher the protocol used, the
183 more recent the version of Python needed to read the pickle
184 produced.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000185
186 The file parameter must have a write() method that accepts a single
187 string argument. It can thus be an open file object, a StringIO
188 object, or any other custom object that meets this interface.
189
190 """
Guido van Rossumcf117b02003-02-09 17:19:41 +0000191 if protocol is None:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000192 protocol = DEFAULT_PROTOCOL
Guido van Rossumcf117b02003-02-09 17:19:41 +0000193 if protocol < 0:
Tim Peters8587b3c2003-02-13 15:44:41 +0000194 protocol = HIGHEST_PROTOCOL
195 elif not 0 <= protocol <= HIGHEST_PROTOCOL:
196 raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000197 self.write = file.write
198 self.memo = {}
Guido van Rossumcf117b02003-02-09 17:19:41 +0000199 self.proto = int(protocol)
200 self.bin = protocol >= 1
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000201 self.fast = 0
Guido van Rossuma48061a1995-01-10 00:31:14 +0000202
Fred Drake7f781c92002-05-01 20:33:53 +0000203 def clear_memo(self):
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000204 """Clears the pickler's "memo".
205
206 The memo is the data structure that remembers which objects the
Tim Petersb377f8a2003-01-28 00:23:36 +0000207 pickler has already seen, so that shared or recursive objects are
208 pickled by reference and not by value. This method is useful when
209 re-using picklers.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000210
211 """
Fred Drake7f781c92002-05-01 20:33:53 +0000212 self.memo.clear()
213
Guido van Rossum3a41c612003-01-28 15:10:22 +0000214 def dump(self, obj):
Tim Peters5bd2a792003-02-01 16:45:06 +0000215 """Write a pickled representation of obj to the open file."""
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000216 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000217 self.write(PROTO + bytes([self.proto]))
Guido van Rossum3a41c612003-01-28 15:10:22 +0000218 self.save(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000219 self.write(STOP)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000220
Jeremy Hylton3422c992003-01-24 19:29:52 +0000221 def memoize(self, obj):
222 """Store an object in the memo."""
223
Tim Peterse46b73f2003-01-27 21:22:10 +0000224 # The Pickler memo is a dictionary mapping object ids to 2-tuples
225 # that contain the Unpickler memo key and the object being memoized.
226 # The memo key is written to the pickle and will become
Jeremy Hylton3422c992003-01-24 19:29:52 +0000227 # the key in the Unpickler's memo. The object is stored in the
Tim Peterse46b73f2003-01-27 21:22:10 +0000228 # Pickler memo so that transient objects are kept alive during
229 # pickling.
Jeremy Hylton3422c992003-01-24 19:29:52 +0000230
Tim Peterse46b73f2003-01-27 21:22:10 +0000231 # The use of the Unpickler memo length as the memo key is just a
232 # convention. The only requirement is that the memo values be unique.
233 # But there appears no advantage to any other scheme, and this
Tim Peterscbd0a322003-01-28 00:24:43 +0000234 # scheme allows the Unpickler memo to be implemented as a plain (but
Tim Peterse46b73f2003-01-27 21:22:10 +0000235 # growable) array, indexed by memo key.
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000236 if self.fast:
237 return
Guido van Rossum9b40e802003-01-30 06:37:41 +0000238 assert id(obj) not in self.memo
Jeremy Hylton3422c992003-01-24 19:29:52 +0000239 memo_len = len(self.memo)
240 self.write(self.put(memo_len))
Tim Peters518df0d2003-01-28 01:00:38 +0000241 self.memo[id(obj)] = memo_len, obj
Jeremy Hylton3422c992003-01-24 19:29:52 +0000242
Tim Petersbb38e302003-01-27 21:25:41 +0000243 # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000244 def put(self, i, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000245 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000246 if i < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000247 return BINPUT + bytes([i])
Guido van Rossum5c938d02003-01-28 03:03:08 +0000248 else:
249 return LONG_BINPUT + pack("<i", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000250
Guido van Rossum39478e82007-08-27 17:23:59 +0000251 return PUT + repr(i).encode("ascii") + b'\n'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000252
Tim Petersbb38e302003-01-27 21:25:41 +0000253 # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000254 def get(self, i, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000255 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000256 if i < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000257 return BINGET + bytes([i])
Guido van Rossum5c938d02003-01-28 03:03:08 +0000258 else:
259 return LONG_BINGET + pack("<i", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000260
Guido van Rossum39478e82007-08-27 17:23:59 +0000261 return GET + repr(i).encode("ascii") + b'\n'
Tim Peters2344fae2001-01-15 00:50:52 +0000262
Guido van Rossumac5b5d22003-01-28 22:01:16 +0000263 def save(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000264 # Check for persistent id (defined by a subclass)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000265 pid = self.persistent_id(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000266 if pid:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000267 self.save_pers(pid)
268 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000269
Guido van Rossumbc64e222003-01-28 16:34:19 +0000270 # Check the memo
271 x = self.memo.get(id(obj))
272 if x:
273 self.write(self.get(x[0]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000274 return
275
Guido van Rossumbc64e222003-01-28 16:34:19 +0000276 # Check the type dispatch table
Guido van Rossum3a41c612003-01-28 15:10:22 +0000277 t = type(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000278 f = self.dispatch.get(t)
279 if f:
280 f(self, obj) # Call unbound method with explicit self
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000281 return
282
Guido van Rossumbc64e222003-01-28 16:34:19 +0000283 # Check for a class with a custom metaclass; treat as regular class
Tim Petersb32a8312003-01-28 00:48:09 +0000284 try:
Guido van Rossum13257902007-06-07 23:15:56 +0000285 issc = issubclass(t, type)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000286 except TypeError: # t is not a class (old Boost; see SF #502085)
Tim Petersb32a8312003-01-28 00:48:09 +0000287 issc = 0
288 if issc:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000289 self.save_global(obj)
Tim Petersb32a8312003-01-28 00:48:09 +0000290 return
291
Guido van Rossumbc64e222003-01-28 16:34:19 +0000292 # Check copy_reg.dispatch_table
293 reduce = dispatch_table.get(t)
Guido van Rossumc53f0092003-02-18 22:05:12 +0000294 if reduce:
295 rv = reduce(obj)
296 else:
297 # Check for a __reduce_ex__ method, fall back to __reduce__
298 reduce = getattr(obj, "__reduce_ex__", None)
299 if reduce:
300 rv = reduce(self.proto)
301 else:
302 reduce = getattr(obj, "__reduce__", None)
303 if reduce:
304 rv = reduce()
305 else:
306 raise PicklingError("Can't pickle %r object: %r" %
307 (t.__name__, obj))
Tim Petersb32a8312003-01-28 00:48:09 +0000308
Guido van Rossumbc64e222003-01-28 16:34:19 +0000309 # Check for string returned by reduce(), meaning "save as global"
Guido van Rossum1255ed62007-05-04 20:30:19 +0000310 if isinstance(rv, basestring):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000311 self.save_global(obj, rv)
Tim Petersb32a8312003-01-28 00:48:09 +0000312 return
313
Guido van Rossumbc64e222003-01-28 16:34:19 +0000314 # Assert that reduce() returned a tuple
Guido van Rossum13257902007-06-07 23:15:56 +0000315 if not isinstance(rv, tuple):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000316 raise PicklingError("%s must return string or tuple" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000317
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000318 # Assert that it returned an appropriately sized tuple
Guido van Rossumbc64e222003-01-28 16:34:19 +0000319 l = len(rv)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000320 if not (2 <= l <= 5):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000321 raise PicklingError("Tuple returned by %s must have "
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000322 "two to five elements" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000323
Guido van Rossumbc64e222003-01-28 16:34:19 +0000324 # Save the reduce() output and finally memoize the object
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000325 self.save_reduce(obj=obj, *rv)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000326
Guido van Rossum3a41c612003-01-28 15:10:22 +0000327 def persistent_id(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000328 # This exists so a subclass can override it
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000329 return None
330
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000331 def save_pers(self, pid):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000332 # Save a persistent id reference
Tim Petersbd1cdb92003-01-28 01:03:10 +0000333 if self.bin:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000334 self.save(pid)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000335 self.write(BINPERSID)
Tim Petersbd1cdb92003-01-28 01:03:10 +0000336 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000337 self.write(PERSID + str(pid).encode("ascii") + b'\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000338
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000339 def save_reduce(self, func, args, state=None,
340 listitems=None, dictitems=None, obj=None):
Jeremy Hyltone3a565e2003-06-29 16:59:59 +0000341 # This API is called by some subclasses
Guido van Rossumbc64e222003-01-28 16:34:19 +0000342
343 # Assert that args is a tuple or None
Guido van Rossum13257902007-06-07 23:15:56 +0000344 if not isinstance(args, tuple):
Raymond Hettingera6b45cc2004-12-07 07:05:57 +0000345 raise PicklingError("args from reduce() should be a tuple")
Guido van Rossumbc64e222003-01-28 16:34:19 +0000346
347 # Assert that func is callable
Guido van Rossumd59da4b2007-05-22 18:11:13 +0000348 if not hasattr(func, '__call__'):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000349 raise PicklingError("func from reduce should be callable")
350
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000351 save = self.save
Guido van Rossumbc64e222003-01-28 16:34:19 +0000352 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000353
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000354 # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ
355 if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__":
356 # A __reduce__ implementation can direct protocol 2 to
357 # use the more efficient NEWOBJ opcode, while still
358 # allowing protocol 0 and 1 to work normally. For this to
359 # work, the function returned by __reduce__ should be
360 # called __newobj__, and its first argument should be a
361 # new-style class. The implementation for __newobj__
362 # should be as follows, although pickle has no way to
363 # verify this:
364 #
365 # def __newobj__(cls, *args):
366 # return cls.__new__(cls, *args)
367 #
368 # Protocols 0 and 1 will pickle a reference to __newobj__,
369 # while protocol 2 (and above) will pickle a reference to
370 # cls, the remaining args tuple, and the NEWOBJ code,
371 # which calls cls.__new__(cls, *args) at unpickling time
372 # (see load_newobj below). If __reduce__ returns a
373 # three-tuple, the state from the third tuple item will be
374 # pickled regardless of the protocol, calling __setstate__
375 # at unpickling time (see load_build below).
376 #
377 # Note that no standard __newobj__ implementation exists;
378 # you have to provide your own. This is to enforce
379 # compatibility with Python 2.2 (pickles written using
380 # protocol 0 or 1 in Python 2.3 should be unpicklable by
381 # Python 2.2).
382 cls = args[0]
383 if not hasattr(cls, "__new__"):
384 raise PicklingError(
385 "args[0] from __newobj__ args has no __new__")
Guido van Rossumf7f45172003-01-31 17:17:49 +0000386 if obj is not None and cls is not obj.__class__:
387 raise PicklingError(
388 "args[0] from __newobj__ args has the wrong class")
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000389 args = args[1:]
390 save(cls)
391 save(args)
392 write(NEWOBJ)
393 else:
394 save(func)
395 save(args)
396 write(REDUCE)
Tim Peters2344fae2001-01-15 00:50:52 +0000397
Guido van Rossumf7f45172003-01-31 17:17:49 +0000398 if obj is not None:
399 self.memoize(obj)
400
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000401 # More new special cases (that work with older protocols as
402 # well): when __reduce__ returns a tuple with 4 or 5 items,
403 # the 4th and 5th item should be iterators that provide list
404 # items and dict items (as (key, value) tuples), or None.
405
406 if listitems is not None:
407 self._batch_appends(listitems)
408
409 if dictitems is not None:
410 self._batch_setitems(dictitems)
411
Tim Petersc32d8242001-04-10 02:48:53 +0000412 if state is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000413 save(state)
414 write(BUILD)
415
Guido van Rossumbc64e222003-01-28 16:34:19 +0000416 # Methods below this point are dispatched through the dispatch table
417
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000418 dispatch = {}
419
Guido van Rossum3a41c612003-01-28 15:10:22 +0000420 def save_none(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000421 self.write(NONE)
Guido van Rossum13257902007-06-07 23:15:56 +0000422 dispatch[type(None)] = save_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000423
Guido van Rossum3a41c612003-01-28 15:10:22 +0000424 def save_bool(self, obj):
Guido van Rossum7d97d312003-01-28 04:25:27 +0000425 if self.proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000426 self.write(obj and NEWTRUE or NEWFALSE)
Guido van Rossum7d97d312003-01-28 04:25:27 +0000427 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000428 self.write(obj and TRUE or FALSE)
Guido van Rossum77f6a652002-04-03 22:41:51 +0000429 dispatch[bool] = save_bool
430
Guido van Rossum3a41c612003-01-28 15:10:22 +0000431 def save_int(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000432 if self.bin:
Tim Peters44714002001-04-10 05:02:52 +0000433 # If the int is small enough to fit in a signed 4-byte 2's-comp
434 # format, we can store it more efficiently than the general
435 # case.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000436 # First one- and two-byte unsigned ints:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000437 if obj >= 0:
438 if obj <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000439 self.write(BININT1 + bytes([obj]))
Guido van Rossum5c938d02003-01-28 03:03:08 +0000440 return
Guido van Rossum3a41c612003-01-28 15:10:22 +0000441 if obj <= 0xffff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000442 self.write(BININT2 + bytes([obj&0xff, obj>>8]))
Guido van Rossum5c938d02003-01-28 03:03:08 +0000443 return
444 # Next check for 4-byte signed ints:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000445 high_bits = obj >> 31 # note that Python shift sign-extends
Tim Petersd95c2df2003-01-28 03:41:54 +0000446 if high_bits == 0 or high_bits == -1:
Tim Peters44714002001-04-10 05:02:52 +0000447 # All high bits are copies of bit 2**31, so the value
448 # fits in a 4-byte signed int.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000449 self.write(BININT + pack("<i", obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000450 return
Tim Peters44714002001-04-10 05:02:52 +0000451 # Text pickle, or int too big to fit in signed 4-byte format.
Guido van Rossum39478e82007-08-27 17:23:59 +0000452 self.write(INT + repr(obj).encode("ascii") + b'\n')
Guido van Rossumddefaf32007-01-14 03:31:43 +0000453 # XXX save_int is merged into save_long
Guido van Rossum13257902007-06-07 23:15:56 +0000454 # dispatch[int] = save_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000455
Guido van Rossum3a41c612003-01-28 15:10:22 +0000456 def save_long(self, obj, pack=struct.pack):
Guido van Rossumddefaf32007-01-14 03:31:43 +0000457 if self.bin:
458 # If the int is small enough to fit in a signed 4-byte 2's-comp
459 # format, we can store it more efficiently than the general
460 # case.
461 # First one- and two-byte unsigned ints:
462 if obj >= 0:
463 if obj <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000464 self.write(BININT1 + bytes([obj]))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000465 return
466 if obj <= 0xffff:
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000467 self.write(BININT2 + bytes([obj&0xff, obj>>8]))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000468 return
469 # Next check for 4-byte signed ints:
470 high_bits = obj >> 31 # note that Python shift sign-extends
471 if high_bits == 0 or high_bits == -1:
472 # All high bits are copies of bit 2**31, so the value
473 # fits in a 4-byte signed int.
474 self.write(BININT + pack("<i", obj))
475 return
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000476 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000477 encoded = encode_long(obj)
478 n = len(encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000479 if n < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000480 self.write(LONG1 + bytes([n]) + encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000481 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000482 self.write(LONG4 + pack("<i", n) + encoded)
Tim Petersee1a53c2003-02-02 02:57:53 +0000483 return
Guido van Rossum39478e82007-08-27 17:23:59 +0000484 self.write(LONG + repr(obj).encode("ascii") + b'\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000485 dispatch[int] = save_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000486
Guido van Rossum3a41c612003-01-28 15:10:22 +0000487 def save_float(self, obj, pack=struct.pack):
Guido van Rossumd3703791998-10-22 20:15:36 +0000488 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000489 self.write(BINFLOAT + pack('>d', obj))
Guido van Rossumd3703791998-10-22 20:15:36 +0000490 else:
Guido van Rossum39478e82007-08-27 17:23:59 +0000491 self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
Guido van Rossum13257902007-06-07 23:15:56 +0000492 dispatch[float] = save_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000493
Guido van Rossum3a41c612003-01-28 15:10:22 +0000494 def save_string(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000495 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000496 n = len(obj)
Tim Petersbbf63cd2003-01-27 21:15:36 +0000497 if n < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000498 self.write(SHORT_BINSTRING + bytes([n]) + bytes(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000499 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000500 self.write(BINSTRING + pack("<i", n) + bytes(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000501 else:
Guido van Rossumaa588c42007-06-15 03:35:38 +0000502 # Strip leading 's' due to repr() of str8() returning s'...'
Guido van Rossum39478e82007-08-27 17:23:59 +0000503 self.write(STRING + repr(obj).lstrip("s").encode("ascii") + b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000504 self.memoize(obj)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000505 dispatch[str8] = save_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000506
Guido van Rossum3a41c612003-01-28 15:10:22 +0000507 def save_unicode(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000508 if self.bin:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000509 encoded = obj.encode('utf-8')
510 n = len(encoded)
511 self.write(BINUNICODE + pack("<i", n) + encoded)
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000512 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000513 obj = obj.replace("\\", "\\u005c")
514 obj = obj.replace("\n", "\\u000a")
Guido van Rossum1255ed62007-05-04 20:30:19 +0000515 self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) +
516 b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000517 self.memoize(obj)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000518 dispatch[str] = save_unicode
Tim Peters658cba62001-02-09 20:06:00 +0000519
Guido van Rossum3a41c612003-01-28 15:10:22 +0000520 def save_tuple(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000521 write = self.write
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000522 proto = self.proto
523
Guido van Rossum3a41c612003-01-28 15:10:22 +0000524 n = len(obj)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000525 if n == 0:
526 if proto:
527 write(EMPTY_TUPLE)
528 else:
529 write(MARK + TUPLE)
Tim Petersd97da802003-01-28 05:48:29 +0000530 return
531
532 save = self.save
533 memo = self.memo
534 if n <= 3 and proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000535 for element in obj:
Tim Petersd97da802003-01-28 05:48:29 +0000536 save(element)
537 # Subtle. Same as in the big comment below.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000538 if id(obj) in memo:
539 get = self.get(memo[id(obj)][0])
Tim Petersd97da802003-01-28 05:48:29 +0000540 write(POP * n + get)
541 else:
542 write(_tuplesize2code[n])
Guido van Rossum3a41c612003-01-28 15:10:22 +0000543 self.memoize(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000544 return
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000545
Tim Peters1d63c9f2003-02-02 20:29:39 +0000546 # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
Tim Petersff57bff2003-01-28 05:34:53 +0000547 # has more than 3 elements.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000548 write(MARK)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000549 for element in obj:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000550 save(element)
551
Tim Peters1d63c9f2003-02-02 20:29:39 +0000552 if id(obj) in memo:
Tim Petersf558da02003-01-28 02:09:55 +0000553 # Subtle. d was not in memo when we entered save_tuple(), so
554 # the process of saving the tuple's elements must have saved
555 # the tuple itself: the tuple is recursive. The proper action
556 # now is to throw away everything we put on the stack, and
557 # simply GET the tuple (it's already constructed). This check
558 # could have been done in the "for element" loop instead, but
559 # recursive tuples are a rare thing.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000560 get = self.get(memo[id(obj)][0])
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000561 if proto:
Tim Petersf558da02003-01-28 02:09:55 +0000562 write(POP_MARK + get)
563 else: # proto 0 -- POP_MARK not available
Tim Petersd97da802003-01-28 05:48:29 +0000564 write(POP * (n+1) + get)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000565 return
566
Tim Peters1d63c9f2003-02-02 20:29:39 +0000567 # No recursion.
Tim Peters518df0d2003-01-28 01:00:38 +0000568 self.write(TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000569 self.memoize(obj)
Jeremy Hylton3422c992003-01-24 19:29:52 +0000570
Guido van Rossum13257902007-06-07 23:15:56 +0000571 dispatch[tuple] = save_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000572
Tim Petersa6ae9a22003-01-28 16:58:41 +0000573 # save_empty_tuple() isn't used by anything in Python 2.3. However, I
574 # found a Pickler subclass in Zope3 that calls it, so it's not harmless
575 # to remove it.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000576 def save_empty_tuple(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000577 self.write(EMPTY_TUPLE)
578
Guido van Rossum3a41c612003-01-28 15:10:22 +0000579 def save_list(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000580 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000581
Tim Petersc32d8242001-04-10 02:48:53 +0000582 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000583 write(EMPTY_LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000584 else: # proto 0 -- can't use EMPTY_LIST
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000585 write(MARK + LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000586
587 self.memoize(obj)
588 self._batch_appends(iter(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000589
Guido van Rossum13257902007-06-07 23:15:56 +0000590 dispatch[list] = save_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000591
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000592 _BATCHSIZE = 1000
593
594 def _batch_appends(self, items):
595 # Helper to batch up APPENDS sequences
596 save = self.save
597 write = self.write
598
599 if not self.bin:
600 for x in items:
601 save(x)
602 write(APPEND)
603 return
604
Guido van Rossum805365e2007-05-07 22:24:25 +0000605 r = range(self._BATCHSIZE)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000606 while items is not None:
607 tmp = []
608 for i in r:
609 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000610 x = next(items)
Guido van Rossum5aac4e62003-02-06 22:57:00 +0000611 tmp.append(x)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000612 except StopIteration:
613 items = None
614 break
615 n = len(tmp)
616 if n > 1:
617 write(MARK)
618 for x in tmp:
619 save(x)
620 write(APPENDS)
621 elif n:
622 save(tmp[0])
623 write(APPEND)
624 # else tmp is empty, and we're done
625
Guido van Rossum3a41c612003-01-28 15:10:22 +0000626 def save_dict(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000627 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000628
Tim Petersc32d8242001-04-10 02:48:53 +0000629 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000630 write(EMPTY_DICT)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000631 else: # proto 0 -- can't use EMPTY_DICT
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000632 write(MARK + DICT)
633
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000634 self.memoize(obj)
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000635 self._batch_setitems(iter(obj.items()))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000636
Guido van Rossum13257902007-06-07 23:15:56 +0000637 dispatch[dict] = save_dict
638 if PyStringMap is not None:
Jeremy Hylton2b9d0291998-05-27 22:38:22 +0000639 dispatch[PyStringMap] = save_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000640
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000641 def _batch_setitems(self, items):
642 # Helper to batch up SETITEMS sequences; proto >= 1 only
643 save = self.save
644 write = self.write
645
646 if not self.bin:
647 for k, v in items:
648 save(k)
649 save(v)
650 write(SETITEM)
651 return
652
Guido van Rossum805365e2007-05-07 22:24:25 +0000653 r = range(self._BATCHSIZE)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000654 while items is not None:
655 tmp = []
656 for i in r:
657 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000658 tmp.append(next(items))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000659 except StopIteration:
660 items = None
661 break
662 n = len(tmp)
663 if n > 1:
664 write(MARK)
665 for k, v in tmp:
666 save(k)
667 save(v)
668 write(SETITEMS)
669 elif n:
670 k, v = tmp[0]
671 save(k)
672 save(v)
673 write(SETITEM)
674 # else tmp is empty, and we're done
675
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000676 def save_global(self, obj, name=None, pack=struct.pack):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000677 write = self.write
678 memo = self.memo
679
Tim Petersc32d8242001-04-10 02:48:53 +0000680 if name is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000681 name = obj.__name__
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000682
Jeremy Hylton4f0dcc92003-01-31 18:33:18 +0000683 module = getattr(obj, "__module__", None)
684 if module is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000685 module = whichmodule(obj, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000686
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000687 try:
688 __import__(module)
689 mod = sys.modules[module]
690 klass = getattr(mod, name)
691 except (ImportError, KeyError, AttributeError):
692 raise PicklingError(
693 "Can't pickle %r: it's not found as %s.%s" %
Guido van Rossum3a41c612003-01-28 15:10:22 +0000694 (obj, module, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000695 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000696 if klass is not obj:
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000697 raise PicklingError(
698 "Can't pickle %r: it's not the same object as %s.%s" %
Guido van Rossum3a41c612003-01-28 15:10:22 +0000699 (obj, module, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000700
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000701 if self.proto >= 2:
Guido van Rossumd4b920c2003-02-04 01:54:49 +0000702 code = _extension_registry.get((module, name))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000703 if code:
704 assert code > 0
705 if code <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000706 write(EXT1 + bytes([code]))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000707 elif code <= 0xffff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000708 write(EXT2 + bytes([code&0xff, code>>8]))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000709 else:
710 write(EXT4 + pack("<i", code))
711 return
712
Guido van Rossum39478e82007-08-27 17:23:59 +0000713 write(GLOBAL + bytes(module, "utf-8") + b'\n' +
714 bytes(name, "utf-8") + b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000715 self.memoize(obj)
Tim Peters3b769832003-01-28 03:51:36 +0000716
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000717 dispatch[FunctionType] = save_global
718 dispatch[BuiltinFunctionType] = save_global
Guido van Rossum13257902007-06-07 23:15:56 +0000719 dispatch[type] = save_global
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000720
Guido van Rossum1be31752003-01-28 15:19:53 +0000721# Pickling helpers
Guido van Rossuma48061a1995-01-10 00:31:14 +0000722
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000723def _keep_alive(x, memo):
724 """Keeps a reference to the object x in the memo.
725
726 Because we remember objects by their id, we have
727 to assure that possibly temporary objects are kept
728 alive by referencing them.
729 We store a reference at the id of the memo, which should
730 normally not be used unless someone tries to deepcopy
731 the memo itself...
732 """
733 try:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000734 memo[id(memo)].append(x)
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000735 except KeyError:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000736 # aha, this is the first one :-)
737 memo[id(memo)]=[x]
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000738
739
Tim Petersc0c12b52003-01-29 00:56:17 +0000740# A cache for whichmodule(), mapping a function object to the name of
741# the module in which the function was found.
742
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000743classmap = {} # called classmap for backwards compatibility
Guido van Rossuma48061a1995-01-10 00:31:14 +0000744
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000745def whichmodule(func, funcname):
746 """Figure out the module in which a function occurs.
Guido van Rossuma48061a1995-01-10 00:31:14 +0000747
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000748 Search sys.modules for the module.
749 Cache in classmap.
750 Return a module name.
Tim Petersc0c12b52003-01-29 00:56:17 +0000751 If the function cannot be found, return "__main__".
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000752 """
Jeremy Hylton4f0dcc92003-01-31 18:33:18 +0000753 # Python functions should always get an __module__ from their globals.
754 mod = getattr(func, "__module__", None)
755 if mod is not None:
756 return mod
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000757 if func in classmap:
758 return classmap[func]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000759
Guido van Rossum634e53f2007-02-26 07:07:02 +0000760 for name, module in list(sys.modules.items()):
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000761 if module is None:
Jeremy Hylton065a5ab2002-09-19 22:57:26 +0000762 continue # skip dummy package entries
Jeremy Hyltoncc1fccb2003-02-06 16:23:01 +0000763 if name != '__main__' and getattr(module, funcname, None) is func:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000764 break
765 else:
766 name = '__main__'
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000767 classmap[func] = name
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000768 return name
Guido van Rossuma48061a1995-01-10 00:31:14 +0000769
770
Guido van Rossum1be31752003-01-28 15:19:53 +0000771# Unpickling machinery
772
Guido van Rossuma48061a1995-01-10 00:31:14 +0000773class Unpickler:
774
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000775 def __init__(self, file):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000776 """This takes a binary file for reading a pickle data stream.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000777
Tim Peters5bd2a792003-02-01 16:45:06 +0000778 The protocol version of the pickle is detected automatically, so no
779 proto argument is needed.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000780
781 The file-like object must have two methods, a read() method that
782 takes an integer argument, and a readline() method that requires no
783 arguments. Both methods should return a string. Thus file-like
784 object can be a file object opened for reading, a StringIO object,
785 or any other custom object that meets this interface.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000786 """
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000787 try:
788 self.readline = file.readline
789 except AttributeError:
790 self.file = file
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000791 self.read = file.read
792 self.memo = {}
Guido van Rossuma48061a1995-01-10 00:31:14 +0000793
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000794 def readline(self):
795 # XXX Slow but at least correct
796 b = bytes()
797 while True:
798 c = self.file.read(1)
799 if not c:
800 break
801 b += c
802 if c == b'\n':
803 break
804 return b
805
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000806 def load(self):
Guido van Rossum3a41c612003-01-28 15:10:22 +0000807 """Read a pickled object representation from the open file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000808
Guido van Rossum3a41c612003-01-28 15:10:22 +0000809 Return the reconstituted object hierarchy specified in the file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000810 """
Jeremy Hylton20747fa2001-11-09 16:15:04 +0000811 self.mark = object() # any new unique object
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000812 self.stack = []
813 self.append = self.stack.append
814 read = self.read
815 dispatch = self.dispatch
816 try:
817 while 1:
818 key = read(1)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000819 if not key:
820 raise EOFError
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000821 assert isinstance(key, bytes)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000822 dispatch[key[0]](self)
Guido van Rossumb940e112007-01-10 16:19:56 +0000823 except _Stop as stopinst:
Guido van Rossumff871742000-12-13 18:11:56 +0000824 return stopinst.value
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000825
Tim Petersc23d18a2003-01-28 01:41:51 +0000826 # Return largest index k such that self.stack[k] is self.mark.
827 # If the stack doesn't contain a mark, eventually raises IndexError.
828 # This could be sped by maintaining another stack, of indices at which
829 # the mark appears. For that matter, the latter stack would suffice,
830 # and we wouldn't need to push mark objects on self.stack at all.
831 # Doing so is probably a good thing, though, since if the pickle is
832 # corrupt (or hostile) we may get a clue from finding self.mark embedded
833 # in unpickled objects.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000834 def marker(self):
835 stack = self.stack
836 mark = self.mark
837 k = len(stack)-1
838 while stack[k] is not mark: k = k-1
839 return k
840
841 dispatch = {}
842
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000843 def load_proto(self):
844 proto = ord(self.read(1))
845 if not 0 <= proto <= 2:
846 raise ValueError, "unsupported pickle protocol: %d" % proto
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000847 dispatch[PROTO[0]] = load_proto
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000848
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000849 def load_persid(self):
850 pid = self.readline()[:-1]
851 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000852 dispatch[PERSID[0]] = load_persid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000853
854 def load_binpersid(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +0000855 pid = self.stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000856 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000857 dispatch[BINPERSID[0]] = load_binpersid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000858
859 def load_none(self):
860 self.append(None)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000861 dispatch[NONE[0]] = load_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000862
Guido van Rossum7d97d312003-01-28 04:25:27 +0000863 def load_false(self):
864 self.append(False)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000865 dispatch[NEWFALSE[0]] = load_false
Guido van Rossum7d97d312003-01-28 04:25:27 +0000866
867 def load_true(self):
868 self.append(True)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000869 dispatch[NEWTRUE[0]] = load_true
Guido van Rossum7d97d312003-01-28 04:25:27 +0000870
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000871 def load_int(self):
Tim Peters19ef62d2001-08-28 22:21:18 +0000872 data = self.readline()
Guido van Rossume2763392002-04-05 19:30:08 +0000873 if data == FALSE[1:]:
874 val = False
875 elif data == TRUE[1:]:
876 val = True
877 else:
878 try:
879 val = int(data)
880 except ValueError:
Guido van Rossume2a383d2007-01-15 16:59:06 +0000881 val = int(data)
Guido van Rossume2763392002-04-05 19:30:08 +0000882 self.append(val)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000883 dispatch[INT[0]] = load_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000884
885 def load_binint(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000886 self.append(mloads(b'i' + self.read(4)))
887 dispatch[BININT[0]] = load_binint
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000888
889 def load_binint1(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +0000890 self.append(ord(self.read(1)))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000891 dispatch[BININT1[0]] = load_binint1
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000892
893 def load_binint2(self):
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000894 self.append(mloads(b'i' + self.read(2) + b'\000\000'))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000895 dispatch[BININT2[0]] = load_binint2
Tim Peters2344fae2001-01-15 00:50:52 +0000896
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000897 def load_long(self):
Guido van Rossum1255ed62007-05-04 20:30:19 +0000898 self.append(int(str(self.readline()[:-1]), 0))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000899 dispatch[LONG[0]] = load_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000900
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000901 def load_long1(self):
902 n = ord(self.read(1))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000903 data = self.read(n)
904 self.append(decode_long(data))
905 dispatch[LONG1[0]] = load_long1
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000906
907 def load_long4(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000908 n = mloads(b'i' + self.read(4))
909 data = self.read(n)
910 self.append(decode_long(data))
911 dispatch[LONG4[0]] = load_long4
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000912
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000913 def load_float(self):
Guido van Rossumff871742000-12-13 18:11:56 +0000914 self.append(float(self.readline()[:-1]))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000915 dispatch[FLOAT[0]] = load_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000916
Guido van Rossumd3703791998-10-22 20:15:36 +0000917 def load_binfloat(self, unpack=struct.unpack):
918 self.append(unpack('>d', self.read(8))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000919 dispatch[BINFLOAT[0]] = load_binfloat
Guido van Rossumd3703791998-10-22 20:15:36 +0000920
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000921 def load_string(self):
Jeremy Hyltonbe467e52000-09-15 15:14:51 +0000922 rep = self.readline()[:-1]
Tim Petersad5a7712003-01-28 16:23:33 +0000923 for q in "\"'": # double or single quote
Martin v. Löwis8a8da792002-08-14 07:46:28 +0000924 if rep.startswith(q):
925 if not rep.endswith(q):
926 raise ValueError, "insecure string pickle"
927 rep = rep[len(q):-len(q)]
928 break
929 else:
Jeremy Hyltonbe467e52000-09-15 15:14:51 +0000930 raise ValueError, "insecure string pickle"
Guido van Rossumf93254d2007-07-19 22:19:35 +0000931 self.append(str(codecs.escape_decode(rep)[0], "latin-1"))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000932 dispatch[STRING[0]] = load_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000933
934 def load_binstring(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000935 len = mloads(b'i' + self.read(4))
Guido van Rossumf93254d2007-07-19 22:19:35 +0000936 self.append(str(self.read(len), "latin-1"))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000937 dispatch[BINSTRING[0]] = load_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000938
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000939 def load_unicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000940 self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
941 dispatch[UNICODE[0]] = load_unicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000942
943 def load_binunicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000944 len = mloads(b'i' + self.read(4))
945 self.append(str(self.read(len), 'utf-8'))
946 dispatch[BINUNICODE[0]] = load_binunicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000947
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000948 def load_short_binstring(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +0000949 len = ord(self.read(1))
Guido van Rossumf93254d2007-07-19 22:19:35 +0000950 self.append(str(self.read(len), "latin-1"))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000951 dispatch[SHORT_BINSTRING[0]] = load_short_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000952
953 def load_tuple(self):
954 k = self.marker()
955 self.stack[k:] = [tuple(self.stack[k+1:])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000956 dispatch[TUPLE[0]] = load_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000957
958 def load_empty_tuple(self):
959 self.stack.append(())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000960 dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000961
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000962 def load_tuple1(self):
963 self.stack[-1] = (self.stack[-1],)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000964 dispatch[TUPLE1[0]] = load_tuple1
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000965
966 def load_tuple2(self):
967 self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000968 dispatch[TUPLE2[0]] = load_tuple2
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000969
970 def load_tuple3(self):
971 self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000972 dispatch[TUPLE3[0]] = load_tuple3
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000973
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000974 def load_empty_list(self):
975 self.stack.append([])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000976 dispatch[EMPTY_LIST[0]] = load_empty_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000977
978 def load_empty_dictionary(self):
979 self.stack.append({})
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000980 dispatch[EMPTY_DICT[0]] = load_empty_dictionary
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000981
982 def load_list(self):
983 k = self.marker()
984 self.stack[k:] = [self.stack[k+1:]]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000985 dispatch[LIST[0]] = load_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000986
987 def load_dict(self):
988 k = self.marker()
989 d = {}
990 items = self.stack[k+1:]
991 for i in range(0, len(items), 2):
992 key = items[i]
993 value = items[i+1]
994 d[key] = value
995 self.stack[k:] = [d]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000996 dispatch[DICT[0]] = load_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000997
Tim Petersd01c1e92003-01-30 15:41:46 +0000998 # INST and OBJ differ only in how they get a class object. It's not
999 # only sensible to do the rest in a common routine, the two routines
1000 # previously diverged and grew different bugs.
1001 # klass is the class to instantiate, and k points to the topmost mark
1002 # object, following which are the arguments for klass.__init__.
1003 def _instantiate(self, klass, k):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001004 args = tuple(self.stack[k+1:])
1005 del self.stack[k:]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001006 instantiated = 0
Tim Petersd01c1e92003-01-30 15:41:46 +00001007 if (not args and
Guido van Rossum13257902007-06-07 23:15:56 +00001008 isinstance(klass, type) and
Tim Petersd01c1e92003-01-30 15:41:46 +00001009 not hasattr(klass, "__getinitargs__")):
Guido van Rossuma8add0e2007-05-14 22:03:55 +00001010 value = _EmptyClass()
1011 value.__class__ = klass
1012 instantiated = 1
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001013 if not instantiated:
Guido van Rossum743d17e1998-09-15 20:25:57 +00001014 try:
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001015 value = klass(*args)
Guido van Rossumb940e112007-01-10 16:19:56 +00001016 except TypeError as err:
Guido van Rossum743d17e1998-09-15 20:25:57 +00001017 raise TypeError, "in constructor for %s: %s" % (
1018 klass.__name__, str(err)), sys.exc_info()[2]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001019 self.append(value)
Tim Petersd01c1e92003-01-30 15:41:46 +00001020
1021 def load_inst(self):
1022 module = self.readline()[:-1]
1023 name = self.readline()[:-1]
1024 klass = self.find_class(module, name)
1025 self._instantiate(klass, self.marker())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001026 dispatch[INST[0]] = load_inst
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001027
1028 def load_obj(self):
Tim Petersd01c1e92003-01-30 15:41:46 +00001029 # Stack is ... markobject classobject arg1 arg2 ...
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001030 k = self.marker()
Tim Petersd01c1e92003-01-30 15:41:46 +00001031 klass = self.stack.pop(k+1)
1032 self._instantiate(klass, k)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001033 dispatch[OBJ[0]] = load_obj
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001034
Guido van Rossum3a41c612003-01-28 15:10:22 +00001035 def load_newobj(self):
1036 args = self.stack.pop()
1037 cls = self.stack[-1]
1038 obj = cls.__new__(cls, *args)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001039 self.stack[-1] = obj
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001040 dispatch[NEWOBJ[0]] = load_newobj
Guido van Rossum3a41c612003-01-28 15:10:22 +00001041
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001042 def load_global(self):
1043 module = self.readline()[:-1]
1044 name = self.readline()[:-1]
1045 klass = self.find_class(module, name)
1046 self.append(klass)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001047 dispatch[GLOBAL[0]] = load_global
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001048
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001049 def load_ext1(self):
1050 code = ord(self.read(1))
1051 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001052 dispatch[EXT1[0]] = load_ext1
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001053
1054 def load_ext2(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001055 code = mloads(b'i' + self.read(2) + b'\000\000')
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001056 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001057 dispatch[EXT2[0]] = load_ext2
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001058
1059 def load_ext4(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001060 code = mloads(b'i' + self.read(4))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001061 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001062 dispatch[EXT4[0]] = load_ext4
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001063
1064 def get_extension(self, code):
1065 nil = []
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001066 obj = _extension_cache.get(code, nil)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001067 if obj is not nil:
1068 self.append(obj)
1069 return
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001070 key = _inverted_registry.get(code)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001071 if not key:
1072 raise ValueError("unregistered extension code %d" % code)
1073 obj = self.find_class(*key)
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001074 _extension_cache[code] = obj
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001075 self.append(obj)
1076
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001077 def find_class(self, module, name):
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001078 # Subclasses may override this
Guido van Rossum1255ed62007-05-04 20:30:19 +00001079 module = str(module)
1080 name = str(name)
Barry Warsawbf4d9592001-11-15 23:42:58 +00001081 __import__(module)
1082 mod = sys.modules[module]
1083 klass = getattr(mod, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001084 return klass
1085
1086 def load_reduce(self):
1087 stack = self.stack
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001088 args = stack.pop()
1089 func = stack[-1]
Guido van Rossum99603b02007-07-20 00:22:32 +00001090 try:
1091 value = func(*args)
1092 except:
1093 print(sys.exc_info())
1094 print(func, args)
1095 raise
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001096 stack[-1] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001097 dispatch[REDUCE[0]] = load_reduce
Guido van Rossuma48061a1995-01-10 00:31:14 +00001098
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001099 def load_pop(self):
1100 del self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001101 dispatch[POP[0]] = load_pop
Guido van Rossum7b5430f1995-03-04 22:25:21 +00001102
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001103 def load_pop_mark(self):
1104 k = self.marker()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001105 del self.stack[k:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001106 dispatch[POP_MARK[0]] = load_pop_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001107
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001108 def load_dup(self):
Guido van Rossumb1062fc1998-03-31 17:00:46 +00001109 self.append(self.stack[-1])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001110 dispatch[DUP[0]] = load_dup
Guido van Rossuma48061a1995-01-10 00:31:14 +00001111
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001112 def load_get(self):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001113 self.append(self.memo[str8(self.readline())[:-1]])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001114 dispatch[GET[0]] = load_get
Guido van Rossum78536471996-04-12 13:36:27 +00001115
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001116 def load_binget(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +00001117 i = ord(self.read(1))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001118 self.append(self.memo[repr(i)])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001119 dispatch[BINGET[0]] = load_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001120
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001121 def load_long_binget(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001122 i = mloads(b'i' + self.read(4))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001123 self.append(self.memo[repr(i)])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001124 dispatch[LONG_BINGET[0]] = load_long_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001125
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001126 def load_put(self):
Guido van Rossum1255ed62007-05-04 20:30:19 +00001127 self.memo[str(self.readline()[:-1])] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001128 dispatch[PUT[0]] = load_put
Guido van Rossuma48061a1995-01-10 00:31:14 +00001129
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001130 def load_binput(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +00001131 i = ord(self.read(1))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001132 self.memo[repr(i)] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001133 dispatch[BINPUT[0]] = load_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001134
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001135 def load_long_binput(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001136 i = mloads(b'i' + self.read(4))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001137 self.memo[repr(i)] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001138 dispatch[LONG_BINPUT[0]] = load_long_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001139
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001140 def load_append(self):
1141 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001142 value = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001143 list = stack[-1]
1144 list.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001145 dispatch[APPEND[0]] = load_append
Guido van Rossuma48061a1995-01-10 00:31:14 +00001146
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001147 def load_appends(self):
1148 stack = self.stack
1149 mark = self.marker()
1150 list = stack[mark - 1]
Tim Peters209ad952003-01-28 01:44:45 +00001151 list.extend(stack[mark + 1:])
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001152 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001153 dispatch[APPENDS[0]] = load_appends
Tim Peters2344fae2001-01-15 00:50:52 +00001154
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001155 def load_setitem(self):
1156 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001157 value = stack.pop()
1158 key = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001159 dict = stack[-1]
1160 dict[key] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001161 dispatch[SETITEM[0]] = load_setitem
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001162
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001163 def load_setitems(self):
1164 stack = self.stack
1165 mark = self.marker()
1166 dict = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001167 for i in range(mark + 1, len(stack), 2):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001168 dict[stack[i]] = stack[i + 1]
Guido van Rossuma48061a1995-01-10 00:31:14 +00001169
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001170 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001171 dispatch[SETITEMS[0]] = load_setitems
Guido van Rossuma48061a1995-01-10 00:31:14 +00001172
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001173 def load_build(self):
1174 stack = self.stack
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001175 state = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001176 inst = stack[-1]
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001177 setstate = getattr(inst, "__setstate__", None)
1178 if setstate:
1179 setstate(state)
1180 return
1181 slotstate = None
1182 if isinstance(state, tuple) and len(state) == 2:
1183 state, slotstate = state
1184 if state:
Guido van Rossuma8add0e2007-05-14 22:03:55 +00001185 inst.__dict__.update(state)
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001186 if slotstate:
1187 for k, v in slotstate.items():
1188 setattr(inst, k, v)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001189 dispatch[BUILD[0]] = load_build
Guido van Rossuma48061a1995-01-10 00:31:14 +00001190
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001191 def load_mark(self):
1192 self.append(self.mark)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001193 dispatch[MARK[0]] = load_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001194
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001195 def load_stop(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001196 value = self.stack.pop()
Guido van Rossumff871742000-12-13 18:11:56 +00001197 raise _Stop(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001198 dispatch[STOP[0]] = load_stop
Guido van Rossuma48061a1995-01-10 00:31:14 +00001199
Guido van Rossume467be61997-12-05 19:42:42 +00001200# Helper class for load_inst/load_obj
1201
1202class _EmptyClass:
1203 pass
Guido van Rossuma48061a1995-01-10 00:31:14 +00001204
Tim Peters91149822003-01-31 03:43:58 +00001205# Encode/decode longs in linear time.
1206
1207import binascii as _binascii
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001208
1209def encode_long(x):
Tim Peters91149822003-01-31 03:43:58 +00001210 r"""Encode a long to a two's complement little-endian binary string.
Guido van Rossume2a383d2007-01-15 16:59:06 +00001211 Note that 0 is a special case, returning an empty string, to save a
Tim Peters4b23f2b2003-01-31 16:43:39 +00001212 byte in the LONG1 pickling context.
1213
Guido van Rossume2a383d2007-01-15 16:59:06 +00001214 >>> encode_long(0)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001215 b''
Guido van Rossume2a383d2007-01-15 16:59:06 +00001216 >>> encode_long(255)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001217 b'\xff\x00'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001218 >>> encode_long(32767)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001219 b'\xff\x7f'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001220 >>> encode_long(-256)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001221 b'\x00\xff'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001222 >>> encode_long(-32768)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001223 b'\x00\x80'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001224 >>> encode_long(-128)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001225 b'\x80'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001226 >>> encode_long(127)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001227 b'\x7f'
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001228 >>>
1229 """
Tim Peters91149822003-01-31 03:43:58 +00001230
1231 if x == 0:
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001232 return b''
Tim Peters91149822003-01-31 03:43:58 +00001233 if x > 0:
1234 ashex = hex(x)
1235 assert ashex.startswith("0x")
1236 njunkchars = 2 + ashex.endswith('L')
1237 nibbles = len(ashex) - njunkchars
1238 if nibbles & 1:
1239 # need an even # of nibbles for unhexlify
1240 ashex = "0x0" + ashex[2:]
Tim Peters4b23f2b2003-01-31 16:43:39 +00001241 elif int(ashex[2], 16) >= 8:
Tim Peters91149822003-01-31 03:43:58 +00001242 # "looks negative", so need a byte of sign bits
1243 ashex = "0x00" + ashex[2:]
1244 else:
1245 # Build the 256's-complement: (1L << nbytes) + x. The trick is
1246 # to find the number of bytes in linear time (although that should
1247 # really be a constant-time task).
1248 ashex = hex(-x)
1249 assert ashex.startswith("0x")
1250 njunkchars = 2 + ashex.endswith('L')
1251 nibbles = len(ashex) - njunkchars
1252 if nibbles & 1:
Tim Petersee1a53c2003-02-02 02:57:53 +00001253 # Extend to a full byte.
Tim Peters91149822003-01-31 03:43:58 +00001254 nibbles += 1
Tim Peters4b23f2b2003-01-31 16:43:39 +00001255 nbits = nibbles * 4
Guido van Rossume2a383d2007-01-15 16:59:06 +00001256 x += 1 << nbits
Tim Peters91149822003-01-31 03:43:58 +00001257 assert x > 0
1258 ashex = hex(x)
Tim Petersee1a53c2003-02-02 02:57:53 +00001259 njunkchars = 2 + ashex.endswith('L')
1260 newnibbles = len(ashex) - njunkchars
1261 if newnibbles < nibbles:
1262 ashex = "0x" + "0" * (nibbles - newnibbles) + ashex[2:]
1263 if int(ashex[2], 16) < 8:
Tim Peters91149822003-01-31 03:43:58 +00001264 # "looks positive", so need a byte of sign bits
Tim Petersee1a53c2003-02-02 02:57:53 +00001265 ashex = "0xff" + ashex[2:]
Tim Peters91149822003-01-31 03:43:58 +00001266
1267 if ashex.endswith('L'):
1268 ashex = ashex[2:-1]
1269 else:
1270 ashex = ashex[2:]
Tim Petersee1a53c2003-02-02 02:57:53 +00001271 assert len(ashex) & 1 == 0, (x, ashex)
Tim Peters91149822003-01-31 03:43:58 +00001272 binary = _binascii.unhexlify(ashex)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001273 return bytes(binary[::-1])
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001274
1275def decode_long(data):
1276 r"""Decode a long from a two's complement little-endian binary string.
Tim Peters4b23f2b2003-01-31 16:43:39 +00001277
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001278 >>> decode_long(b'')
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001279 0
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001280 >>> decode_long(b"\xff\x00")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001281 255
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001282 >>> decode_long(b"\xff\x7f")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001283 32767
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001284 >>> decode_long(b"\x00\xff")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001285 -256
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001286 >>> decode_long(b"\x00\x80")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001287 -32768
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001288 >>> decode_long(b"\x80")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001289 -128
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001290 >>> decode_long(b"\x7f")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001291 127
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001292 """
Tim Peters91149822003-01-31 03:43:58 +00001293
Tim Peters4b23f2b2003-01-31 16:43:39 +00001294 nbytes = len(data)
1295 if nbytes == 0:
Guido van Rossume2a383d2007-01-15 16:59:06 +00001296 return 0
Tim Peters91149822003-01-31 03:43:58 +00001297 ashex = _binascii.hexlify(data[::-1])
Guido van Rossume2a383d2007-01-15 16:59:06 +00001298 n = int(ashex, 16) # quadratic time before Python 2.3; linear now
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001299 if data[-1] >= 0x80:
Guido van Rossume2a383d2007-01-15 16:59:06 +00001300 n -= 1 << (nbytes * 8)
Tim Peters91149822003-01-31 03:43:58 +00001301 return n
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001302
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001303# Shorthands
1304
Raymond Hettinger3489cad2004-12-05 05:20:42 +00001305def dump(obj, file, protocol=None):
1306 Pickler(file, protocol).dump(obj)
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001307
Raymond Hettinger3489cad2004-12-05 05:20:42 +00001308def dumps(obj, protocol=None):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001309 f = io.BytesIO()
1310 Pickler(f, protocol).dump(obj)
1311 res = f.getvalue()
1312 assert isinstance(res, bytes)
1313 return res
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001314
1315def load(file):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001316 return Unpickler(file).load()
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001317
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001318def loads(s):
1319 if isinstance(s, str):
1320 raise TypeError("Can't load pickle from unicode string")
1321 file = io.BytesIO(s)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001322 return Unpickler(file).load()
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001323
1324# Doctest
1325
1326def _test():
1327 import doctest
1328 return doctest.testmod()
1329
1330if __name__ == "__main__":
1331 _test()