blob: 62658cbbce56ba70910f589dea85c9bf4f2f0abe [file] [log] [blame]
Guido van Rossum54f22ed2000-02-04 15:10:34 +00001"""Create portable serialized representations of Python objects.
Guido van Rossuma48061a1995-01-10 00:31:14 +00002
Guido van Rossume467be61997-12-05 19:42:42 +00003See module cPickle for a (much) faster implementation.
4See module copy_reg for a mechanism for registering custom picklers.
Tim Peters22a449a2003-01-27 20:16:36 +00005See module pickletools source for extensive comments.
Guido van Rossuma48061a1995-01-10 00:31:14 +00006
Guido van Rossume467be61997-12-05 19:42:42 +00007Classes:
Guido van Rossuma48061a1995-01-10 00:31:14 +00008
Guido van Rossume467be61997-12-05 19:42:42 +00009 Pickler
10 Unpickler
Guido van Rossuma48061a1995-01-10 00:31:14 +000011
Guido van Rossume467be61997-12-05 19:42:42 +000012Functions:
Guido van Rossuma48061a1995-01-10 00:31:14 +000013
Guido van Rossume467be61997-12-05 19:42:42 +000014 dump(object, file)
15 dumps(object) -> string
16 load(file) -> object
17 loads(string) -> object
Guido van Rossuma48061a1995-01-10 00:31:14 +000018
Guido van Rossume467be61997-12-05 19:42:42 +000019Misc variables:
Guido van Rossuma48061a1995-01-10 00:31:14 +000020
Fred Drakefe82acc1998-02-13 03:24:48 +000021 __version__
Guido van Rossume467be61997-12-05 19:42:42 +000022 format_version
23 compatible_formats
Guido van Rossuma48061a1995-01-10 00:31:14 +000024
Guido van Rossuma48061a1995-01-10 00:31:14 +000025"""
26
Guido van Rossum743d17e1998-09-15 20:25:57 +000027__version__ = "$Revision$" # Code version
Guido van Rossuma48061a1995-01-10 00:31:14 +000028
29from types import *
Guido van Rossum443ada42003-02-18 22:49:10 +000030from copy_reg import dispatch_table
Guido van Rossumd4b920c2003-02-04 01:54:49 +000031from copy_reg import _extension_registry, _inverted_registry, _extension_cache
Guido van Rossumd3703791998-10-22 20:15:36 +000032import marshal
33import sys
34import struct
Skip Montanaro23bafc62001-02-18 03:10:09 +000035import re
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000036import io
Guido van Rossuma48061a1995-01-10 00:31:14 +000037
Skip Montanaro352674d2001-02-07 23:14:30 +000038__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
39 "Unpickler", "dump", "dumps", "load", "loads"]
40
Tim Petersc0c12b52003-01-29 00:56:17 +000041# These are purely informational; no code uses these.
Guido van Rossumf29d3d62003-01-27 22:47:53 +000042format_version = "2.0" # File format version we write
43compatible_formats = ["1.0", # Original protocol 0
Guido van Rossumbc64e222003-01-28 16:34:19 +000044 "1.1", # Protocol 0 with INST added
Guido van Rossumf29d3d62003-01-27 22:47:53 +000045 "1.2", # Original protocol 1
46 "1.3", # Protocol 1 with BINFLOAT added
47 "2.0", # Protocol 2
48 ] # Old format versions we can read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000049
Tim Peters8587b3c2003-02-13 15:44:41 +000050# Keep in synch with cPickle. This is the highest protocol number we
51# know how to read.
52HIGHEST_PROTOCOL = 2
53
Guido van Rossum2e6a4b32007-05-04 19:56:22 +000054# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
55DEFAULT_PROTOCOL = 2
56
Guido van Rossume0b90422003-01-28 03:17:21 +000057# Why use struct.pack() for pickling but marshal.loads() for
Tim Petersc0c12b52003-01-29 00:56:17 +000058# unpickling? struct.pack() is 40% faster than marshal.dumps(), but
Guido van Rossume0b90422003-01-28 03:17:21 +000059# marshal.loads() is twice as fast as struct.unpack()!
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000060mloads = marshal.loads
Guido van Rossum0c891ce1995-03-14 15:09:05 +000061
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000062class PickleError(Exception):
Neal Norwitzefbb67b2002-05-30 12:12:04 +000063 """A common base class for the other pickling exceptions."""
Raymond Hettingeraef22fb2002-05-29 16:18:42 +000064 pass
65
66class PicklingError(PickleError):
67 """This exception is raised when an unpicklable object is passed to the
68 dump() method.
69
70 """
71 pass
72
73class UnpicklingError(PickleError):
74 """This exception is raised when there is a problem unpickling an object,
75 such as a security violation.
76
77 Note that other exceptions may also be raised during unpickling, including
78 (but not necessarily limited to) AttributeError, EOFError, ImportError,
79 and IndexError.
80
81 """
82 pass
Guido van Rossum7849da81995-03-09 14:08:35 +000083
Tim Petersc0c12b52003-01-29 00:56:17 +000084# An instance of _Stop is raised by Unpickler.load_stop() in response to
85# the STOP opcode, passing the object that is the result of unpickling.
Guido van Rossumff871742000-12-13 18:11:56 +000086class _Stop(Exception):
87 def __init__(self, value):
88 self.value = value
89
Guido van Rossum533dbcf2003-01-28 17:55:05 +000090# Jython has PyStringMap; it's a dict subclass with string keys
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000091try:
92 from org.python.core import PyStringMap
93except ImportError:
94 PyStringMap = None
95
Tim Peters22a449a2003-01-27 20:16:36 +000096# Pickle opcodes. See pickletools.py for extensive docs. The listing
97# here is in kind-of alphabetical order of 1-character pickle code.
98# pickletools groups them by purpose.
Guido van Rossumdbb718f2001-09-21 19:22:34 +000099
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000100MARK = b'(' # push special markobject on stack
101STOP = b'.' # every pickle ends with STOP
102POP = b'0' # discard topmost stack item
103POP_MARK = b'1' # discard stack top through topmost markobject
104DUP = b'2' # duplicate top stack item
105FLOAT = b'F' # push float object; decimal string argument
106INT = b'I' # push integer or bool; decimal string argument
107BININT = b'J' # push four-byte signed int
108BININT1 = b'K' # push 1-byte unsigned int
109LONG = b'L' # push long; decimal string argument
110BININT2 = b'M' # push 2-byte unsigned int
111NONE = b'N' # push None
112PERSID = b'P' # push persistent object; id is taken from string arg
113BINPERSID = b'Q' # " " " ; " " " " stack
114REDUCE = b'R' # apply callable to argtuple, both on stack
115STRING = b'S' # push string; NL-terminated string argument
116BINSTRING = b'T' # push string; counted binary string argument
117SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
118UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
119BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
120APPEND = b'a' # append stack top to list below it
121BUILD = b'b' # call __setstate__ or __dict__.update()
122GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
123DICT = b'd' # build a dict from stack items
124EMPTY_DICT = b'}' # push empty dict
125APPENDS = b'e' # extend list on stack by topmost stack slice
126GET = b'g' # push item from memo on stack; index is string arg
127BINGET = b'h' # " " " " " " ; " " 1-byte arg
128INST = b'i' # build & push class instance
129LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
130LIST = b'l' # build list from topmost stack items
131EMPTY_LIST = b']' # push empty list
132OBJ = b'o' # build & push class instance
133PUT = b'p' # store stack top in memo; index is string arg
134BINPUT = b'q' # " " " " " ; " " 1-byte arg
135LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
136SETITEM = b's' # add key+value pair to dict
137TUPLE = b't' # build tuple from topmost stack items
138EMPTY_TUPLE = b')' # push empty tuple
139SETITEMS = b'u' # modify dict by adding topmost key+value pairs
140BINFLOAT = b'G' # push float; arg is 8-byte float encoding
Tim Peters22a449a2003-01-27 20:16:36 +0000141
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000142TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
143FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
Guido van Rossum77f6a652002-04-03 22:41:51 +0000144
Guido van Rossum586c9e82003-01-29 06:16:12 +0000145# Protocol 2
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000146
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000147PROTO = b'\x80' # identify pickle protocol
148NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
149EXT1 = b'\x82' # push object from extension registry; 1-byte index
150EXT2 = b'\x83' # ditto, but 2-byte index
151EXT4 = b'\x84' # ditto, but 4-byte index
152TUPLE1 = b'\x85' # build 1-tuple from stack top
153TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
154TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
155NEWTRUE = b'\x88' # push True
156NEWFALSE = b'\x89' # push False
157LONG1 = b'\x8a' # push long from < 256 bytes
158LONG4 = b'\x8b' # push really big long
Guido van Rossum5a2d8f52003-01-27 21:44:25 +0000159
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000160_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
161
Guido van Rossuma48061a1995-01-10 00:31:14 +0000162
Skip Montanaro23bafc62001-02-18 03:10:09 +0000163__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
164
Guido van Rossum1be31752003-01-28 15:19:53 +0000165
166# Pickling machinery
167
Guido van Rossuma48061a1995-01-10 00:31:14 +0000168class Pickler:
169
Raymond Hettinger3489cad2004-12-05 05:20:42 +0000170 def __init__(self, file, protocol=None):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000171 """This takes a binary file for writing a pickle data stream.
172
173 All protocols now read and write bytes.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000174
Guido van Rossumcf117b02003-02-09 17:19:41 +0000175 The optional protocol argument tells the pickler to use the
176 given protocol; supported protocols are 0, 1, 2. The default
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000177 protocol is 2; it's been supported for many years now.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000178
179 Protocol 1 is more efficient than protocol 0; protocol 2 is
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000180 more efficient than protocol 1.
Guido van Rossumf29d3d62003-01-27 22:47:53 +0000181
Guido van Rossum7eff63a2003-01-31 19:42:31 +0000182 Specifying a negative protocol version selects the highest
Tim Peters5bd2a792003-02-01 16:45:06 +0000183 protocol version supported. The higher the protocol used, the
184 more recent the version of Python needed to read the pickle
185 produced.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000186
187 The file parameter must have a write() method that accepts a single
188 string argument. It can thus be an open file object, a StringIO
189 object, or any other custom object that meets this interface.
190
191 """
Guido van Rossumcf117b02003-02-09 17:19:41 +0000192 if protocol is None:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000193 protocol = DEFAULT_PROTOCOL
Guido van Rossumcf117b02003-02-09 17:19:41 +0000194 if protocol < 0:
Tim Peters8587b3c2003-02-13 15:44:41 +0000195 protocol = HIGHEST_PROTOCOL
196 elif not 0 <= protocol <= HIGHEST_PROTOCOL:
197 raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000198 self.write = file.write
199 self.memo = {}
Guido van Rossumcf117b02003-02-09 17:19:41 +0000200 self.proto = int(protocol)
201 self.bin = protocol >= 1
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000202 self.fast = 0
Guido van Rossuma48061a1995-01-10 00:31:14 +0000203
Fred Drake7f781c92002-05-01 20:33:53 +0000204 def clear_memo(self):
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000205 """Clears the pickler's "memo".
206
207 The memo is the data structure that remembers which objects the
Tim Petersb377f8a2003-01-28 00:23:36 +0000208 pickler has already seen, so that shared or recursive objects are
209 pickled by reference and not by value. This method is useful when
210 re-using picklers.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000211
212 """
Fred Drake7f781c92002-05-01 20:33:53 +0000213 self.memo.clear()
214
Guido van Rossum3a41c612003-01-28 15:10:22 +0000215 def dump(self, obj):
Tim Peters5bd2a792003-02-01 16:45:06 +0000216 """Write a pickled representation of obj to the open file."""
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000217 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000218 self.write(PROTO + bytes([self.proto]))
Guido van Rossum3a41c612003-01-28 15:10:22 +0000219 self.save(obj)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000220 self.write(STOP)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000221
Jeremy Hylton3422c992003-01-24 19:29:52 +0000222 def memoize(self, obj):
223 """Store an object in the memo."""
224
Tim Peterse46b73f2003-01-27 21:22:10 +0000225 # The Pickler memo is a dictionary mapping object ids to 2-tuples
226 # that contain the Unpickler memo key and the object being memoized.
227 # The memo key is written to the pickle and will become
Jeremy Hylton3422c992003-01-24 19:29:52 +0000228 # the key in the Unpickler's memo. The object is stored in the
Tim Peterse46b73f2003-01-27 21:22:10 +0000229 # Pickler memo so that transient objects are kept alive during
230 # pickling.
Jeremy Hylton3422c992003-01-24 19:29:52 +0000231
Tim Peterse46b73f2003-01-27 21:22:10 +0000232 # The use of the Unpickler memo length as the memo key is just a
233 # convention. The only requirement is that the memo values be unique.
234 # But there appears no advantage to any other scheme, and this
Tim Peterscbd0a322003-01-28 00:24:43 +0000235 # scheme allows the Unpickler memo to be implemented as a plain (but
Tim Peterse46b73f2003-01-27 21:22:10 +0000236 # growable) array, indexed by memo key.
Guido van Rossum5d9113d2003-01-29 17:58:45 +0000237 if self.fast:
238 return
Guido van Rossum9b40e802003-01-30 06:37:41 +0000239 assert id(obj) not in self.memo
Jeremy Hylton3422c992003-01-24 19:29:52 +0000240 memo_len = len(self.memo)
241 self.write(self.put(memo_len))
Tim Peters518df0d2003-01-28 01:00:38 +0000242 self.memo[id(obj)] = memo_len, obj
Jeremy Hylton3422c992003-01-24 19:29:52 +0000243
Tim Petersbb38e302003-01-27 21:25:41 +0000244 # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000245 def put(self, i, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000246 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000247 if i < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000248 return BINPUT + bytes([i])
Guido van Rossum5c938d02003-01-28 03:03:08 +0000249 else:
250 return LONG_BINPUT + pack("<i", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000251
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000252 return PUT + bytes(repr(i)) + b'\n'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000253
Tim Petersbb38e302003-01-27 21:25:41 +0000254 # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000255 def get(self, i, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000256 if self.bin:
Tim Petersc32d8242001-04-10 02:48:53 +0000257 if i < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000258 return BINGET + bytes([i])
Guido van Rossum5c938d02003-01-28 03:03:08 +0000259 else:
260 return LONG_BINGET + pack("<i", i)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000261
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000262 return GET + bytes(repr(i)) + b'\n'
Tim Peters2344fae2001-01-15 00:50:52 +0000263
Guido van Rossumac5b5d22003-01-28 22:01:16 +0000264 def save(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000265 # Check for persistent id (defined by a subclass)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000266 pid = self.persistent_id(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000267 if pid:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000268 self.save_pers(pid)
269 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000270
Guido van Rossumbc64e222003-01-28 16:34:19 +0000271 # Check the memo
272 x = self.memo.get(id(obj))
273 if x:
274 self.write(self.get(x[0]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000275 return
276
Guido van Rossumbc64e222003-01-28 16:34:19 +0000277 # Check the type dispatch table
Guido van Rossum3a41c612003-01-28 15:10:22 +0000278 t = type(obj)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000279 f = self.dispatch.get(t)
280 if f:
281 f(self, obj) # Call unbound method with explicit self
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000282 return
283
Guido van Rossumbc64e222003-01-28 16:34:19 +0000284 # Check for a class with a custom metaclass; treat as regular class
Tim Petersb32a8312003-01-28 00:48:09 +0000285 try:
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000286 issc = issubclass(t, TypeType)
Guido van Rossumbc64e222003-01-28 16:34:19 +0000287 except TypeError: # t is not a class (old Boost; see SF #502085)
Tim Petersb32a8312003-01-28 00:48:09 +0000288 issc = 0
289 if issc:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000290 self.save_global(obj)
Tim Petersb32a8312003-01-28 00:48:09 +0000291 return
292
Guido van Rossumbc64e222003-01-28 16:34:19 +0000293 # Check copy_reg.dispatch_table
294 reduce = dispatch_table.get(t)
Guido van Rossumc53f0092003-02-18 22:05:12 +0000295 if reduce:
296 rv = reduce(obj)
297 else:
298 # Check for a __reduce_ex__ method, fall back to __reduce__
299 reduce = getattr(obj, "__reduce_ex__", None)
300 if reduce:
301 rv = reduce(self.proto)
302 else:
303 reduce = getattr(obj, "__reduce__", None)
304 if reduce:
305 rv = reduce()
306 else:
307 raise PicklingError("Can't pickle %r object: %r" %
308 (t.__name__, obj))
Tim Petersb32a8312003-01-28 00:48:09 +0000309
Guido van Rossumbc64e222003-01-28 16:34:19 +0000310 # Check for string returned by reduce(), meaning "save as global"
Guido van Rossum1255ed62007-05-04 20:30:19 +0000311 if isinstance(rv, basestring):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000312 self.save_global(obj, rv)
Tim Petersb32a8312003-01-28 00:48:09 +0000313 return
314
Guido van Rossumbc64e222003-01-28 16:34:19 +0000315 # Assert that reduce() returned a tuple
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000316 if type(rv) is not TupleType:
Guido van Rossumbc64e222003-01-28 16:34:19 +0000317 raise PicklingError("%s must return string or tuple" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000318
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000319 # Assert that it returned an appropriately sized tuple
Guido van Rossumbc64e222003-01-28 16:34:19 +0000320 l = len(rv)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000321 if not (2 <= l <= 5):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000322 raise PicklingError("Tuple returned by %s must have "
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000323 "two to five elements" % reduce)
Tim Petersb32a8312003-01-28 00:48:09 +0000324
Guido van Rossumbc64e222003-01-28 16:34:19 +0000325 # Save the reduce() output and finally memoize the object
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000326 self.save_reduce(obj=obj, *rv)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000327
Guido van Rossum3a41c612003-01-28 15:10:22 +0000328 def persistent_id(self, obj):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000329 # This exists so a subclass can override it
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000330 return None
331
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000332 def save_pers(self, pid):
Guido van Rossumbc64e222003-01-28 16:34:19 +0000333 # Save a persistent id reference
Tim Petersbd1cdb92003-01-28 01:03:10 +0000334 if self.bin:
Jeremy Hylton5e0f4e72002-11-13 22:01:27 +0000335 self.save(pid)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000336 self.write(BINPERSID)
Tim Petersbd1cdb92003-01-28 01:03:10 +0000337 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000338 self.write(PERSID + bytes(str(pid)) + b'\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000339
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000340 def save_reduce(self, func, args, state=None,
341 listitems=None, dictitems=None, obj=None):
Jeremy Hyltone3a565e2003-06-29 16:59:59 +0000342 # This API is called by some subclasses
Guido van Rossumbc64e222003-01-28 16:34:19 +0000343
344 # Assert that args is a tuple or None
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000345 if not isinstance(args, TupleType):
Raymond Hettingera6b45cc2004-12-07 07:05:57 +0000346 raise PicklingError("args from reduce() should be a tuple")
Guido van Rossumbc64e222003-01-28 16:34:19 +0000347
348 # Assert that func is callable
349 if not callable(func):
350 raise PicklingError("func from reduce should be callable")
351
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000352 save = self.save
Guido van Rossumbc64e222003-01-28 16:34:19 +0000353 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000354
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000355 # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ
356 if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__":
357 # A __reduce__ implementation can direct protocol 2 to
358 # use the more efficient NEWOBJ opcode, while still
359 # allowing protocol 0 and 1 to work normally. For this to
360 # work, the function returned by __reduce__ should be
361 # called __newobj__, and its first argument should be a
362 # new-style class. The implementation for __newobj__
363 # should be as follows, although pickle has no way to
364 # verify this:
365 #
366 # def __newobj__(cls, *args):
367 # return cls.__new__(cls, *args)
368 #
369 # Protocols 0 and 1 will pickle a reference to __newobj__,
370 # while protocol 2 (and above) will pickle a reference to
371 # cls, the remaining args tuple, and the NEWOBJ code,
372 # which calls cls.__new__(cls, *args) at unpickling time
373 # (see load_newobj below). If __reduce__ returns a
374 # three-tuple, the state from the third tuple item will be
375 # pickled regardless of the protocol, calling __setstate__
376 # at unpickling time (see load_build below).
377 #
378 # Note that no standard __newobj__ implementation exists;
379 # you have to provide your own. This is to enforce
380 # compatibility with Python 2.2 (pickles written using
381 # protocol 0 or 1 in Python 2.3 should be unpicklable by
382 # Python 2.2).
383 cls = args[0]
384 if not hasattr(cls, "__new__"):
385 raise PicklingError(
386 "args[0] from __newobj__ args has no __new__")
Guido van Rossumf7f45172003-01-31 17:17:49 +0000387 if obj is not None and cls is not obj.__class__:
388 raise PicklingError(
389 "args[0] from __newobj__ args has the wrong class")
Guido van Rossumd053b4b2003-01-31 16:51:45 +0000390 args = args[1:]
391 save(cls)
392 save(args)
393 write(NEWOBJ)
394 else:
395 save(func)
396 save(args)
397 write(REDUCE)
Tim Peters2344fae2001-01-15 00:50:52 +0000398
Guido van Rossumf7f45172003-01-31 17:17:49 +0000399 if obj is not None:
400 self.memoize(obj)
401
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000402 # More new special cases (that work with older protocols as
403 # well): when __reduce__ returns a tuple with 4 or 5 items,
404 # the 4th and 5th item should be iterators that provide list
405 # items and dict items (as (key, value) tuples), or None.
406
407 if listitems is not None:
408 self._batch_appends(listitems)
409
410 if dictitems is not None:
411 self._batch_setitems(dictitems)
412
Tim Petersc32d8242001-04-10 02:48:53 +0000413 if state is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000414 save(state)
415 write(BUILD)
416
Guido van Rossumbc64e222003-01-28 16:34:19 +0000417 # Methods below this point are dispatched through the dispatch table
418
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000419 dispatch = {}
420
Guido van Rossum3a41c612003-01-28 15:10:22 +0000421 def save_none(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000422 self.write(NONE)
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000423 dispatch[NoneType] = save_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000424
Guido van Rossum3a41c612003-01-28 15:10:22 +0000425 def save_bool(self, obj):
Guido van Rossum7d97d312003-01-28 04:25:27 +0000426 if self.proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000427 self.write(obj and NEWTRUE or NEWFALSE)
Guido van Rossum7d97d312003-01-28 04:25:27 +0000428 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000429 self.write(obj and TRUE or FALSE)
Guido van Rossum77f6a652002-04-03 22:41:51 +0000430 dispatch[bool] = save_bool
431
Guido van Rossum3a41c612003-01-28 15:10:22 +0000432 def save_int(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000433 if self.bin:
Tim Peters44714002001-04-10 05:02:52 +0000434 # If the int is small enough to fit in a signed 4-byte 2's-comp
435 # format, we can store it more efficiently than the general
436 # case.
Guido van Rossum5c938d02003-01-28 03:03:08 +0000437 # First one- and two-byte unsigned ints:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000438 if obj >= 0:
439 if obj <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000440 self.write(BININT1 + bytes([obj]))
Guido van Rossum5c938d02003-01-28 03:03:08 +0000441 return
Guido van Rossum3a41c612003-01-28 15:10:22 +0000442 if obj <= 0xffff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000443 self.write(BININT2 + bytes([obj&0xff, obj>>8]))
Guido van Rossum5c938d02003-01-28 03:03:08 +0000444 return
445 # Next check for 4-byte signed ints:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000446 high_bits = obj >> 31 # note that Python shift sign-extends
Tim Petersd95c2df2003-01-28 03:41:54 +0000447 if high_bits == 0 or high_bits == -1:
Tim Peters44714002001-04-10 05:02:52 +0000448 # All high bits are copies of bit 2**31, so the value
449 # fits in a 4-byte signed int.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000450 self.write(BININT + pack("<i", obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000451 return
Tim Peters44714002001-04-10 05:02:52 +0000452 # Text pickle, or int too big to fit in signed 4-byte format.
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000453 self.write(INT + bytes(repr(obj)) + b'\n')
Guido van Rossumddefaf32007-01-14 03:31:43 +0000454 # XXX save_int is merged into save_long
455 # dispatch[IntType] = save_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000456
Guido van Rossum3a41c612003-01-28 15:10:22 +0000457 def save_long(self, obj, pack=struct.pack):
Guido van Rossumddefaf32007-01-14 03:31:43 +0000458 if self.bin:
459 # If the int is small enough to fit in a signed 4-byte 2's-comp
460 # format, we can store it more efficiently than the general
461 # case.
462 # First one- and two-byte unsigned ints:
463 if obj >= 0:
464 if obj <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000465 self.write(BININT1 + bytes([obj]))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000466 return
467 if obj <= 0xffff:
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000468 self.write(BININT2 + bytes([obj&0xff, obj>>8]))
Guido van Rossumddefaf32007-01-14 03:31:43 +0000469 return
470 # Next check for 4-byte signed ints:
471 high_bits = obj >> 31 # note that Python shift sign-extends
472 if high_bits == 0 or high_bits == -1:
473 # All high bits are copies of bit 2**31, so the value
474 # fits in a 4-byte signed int.
475 self.write(BININT + pack("<i", obj))
476 return
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000477 if self.proto >= 2:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000478 encoded = encode_long(obj)
479 n = len(encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000480 if n < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000481 self.write(LONG1 + bytes([n]) + encoded)
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000482 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000483 self.write(LONG4 + pack("<i", n) + encoded)
Tim Petersee1a53c2003-02-02 02:57:53 +0000484 return
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000485 self.write(LONG + bytes(repr(obj)) + b'\n')
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000486 dispatch[LongType] = save_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000487
Guido van Rossum3a41c612003-01-28 15:10:22 +0000488 def save_float(self, obj, pack=struct.pack):
Guido van Rossumd3703791998-10-22 20:15:36 +0000489 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000490 self.write(BINFLOAT + pack('>d', obj))
Guido van Rossumd3703791998-10-22 20:15:36 +0000491 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000492 self.write(FLOAT + bytes(repr(obj)) + b'\n')
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000493 dispatch[FloatType] = save_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000494
Guido van Rossum3a41c612003-01-28 15:10:22 +0000495 def save_string(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000496 if self.bin:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000497 n = len(obj)
Tim Petersbbf63cd2003-01-27 21:15:36 +0000498 if n < 256:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000499 self.write(SHORT_BINSTRING + bytes([n]) + bytes(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000500 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000501 self.write(BINSTRING + pack("<i", n) + bytes(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000502 else:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000503 self.write(STRING + bytes(repr(obj)) + b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000504 self.memoize(obj)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000505 dispatch[str8] = save_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000506
Guido van Rossum3a41c612003-01-28 15:10:22 +0000507 def save_unicode(self, obj, pack=struct.pack):
Tim Petersc32d8242001-04-10 02:48:53 +0000508 if self.bin:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000509 encoded = obj.encode('utf-8')
510 n = len(encoded)
511 self.write(BINUNICODE + pack("<i", n) + encoded)
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000512 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000513 obj = obj.replace("\\", "\\u005c")
514 obj = obj.replace("\n", "\\u000a")
Guido van Rossum1255ed62007-05-04 20:30:19 +0000515 self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) +
516 b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000517 self.memoize(obj)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000518 dispatch[str] = save_unicode
Tim Peters658cba62001-02-09 20:06:00 +0000519
Guido van Rossum3a41c612003-01-28 15:10:22 +0000520 def save_tuple(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000521 write = self.write
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000522 proto = self.proto
523
Guido van Rossum3a41c612003-01-28 15:10:22 +0000524 n = len(obj)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000525 if n == 0:
526 if proto:
527 write(EMPTY_TUPLE)
528 else:
529 write(MARK + TUPLE)
Tim Petersd97da802003-01-28 05:48:29 +0000530 return
531
532 save = self.save
533 memo = self.memo
534 if n <= 3 and proto >= 2:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000535 for element in obj:
Tim Petersd97da802003-01-28 05:48:29 +0000536 save(element)
537 # Subtle. Same as in the big comment below.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000538 if id(obj) in memo:
539 get = self.get(memo[id(obj)][0])
Tim Petersd97da802003-01-28 05:48:29 +0000540 write(POP * n + get)
541 else:
542 write(_tuplesize2code[n])
Guido van Rossum3a41c612003-01-28 15:10:22 +0000543 self.memoize(obj)
Tim Petersd97da802003-01-28 05:48:29 +0000544 return
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000545
Tim Peters1d63c9f2003-02-02 20:29:39 +0000546 # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
Tim Petersff57bff2003-01-28 05:34:53 +0000547 # has more than 3 elements.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000548 write(MARK)
Guido van Rossum3a41c612003-01-28 15:10:22 +0000549 for element in obj:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000550 save(element)
551
Tim Peters1d63c9f2003-02-02 20:29:39 +0000552 if id(obj) in memo:
Tim Petersf558da02003-01-28 02:09:55 +0000553 # Subtle. d was not in memo when we entered save_tuple(), so
554 # the process of saving the tuple's elements must have saved
555 # the tuple itself: the tuple is recursive. The proper action
556 # now is to throw away everything we put on the stack, and
557 # simply GET the tuple (it's already constructed). This check
558 # could have been done in the "for element" loop instead, but
559 # recursive tuples are a rare thing.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000560 get = self.get(memo[id(obj)][0])
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000561 if proto:
Tim Petersf558da02003-01-28 02:09:55 +0000562 write(POP_MARK + get)
563 else: # proto 0 -- POP_MARK not available
Tim Petersd97da802003-01-28 05:48:29 +0000564 write(POP * (n+1) + get)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000565 return
566
Tim Peters1d63c9f2003-02-02 20:29:39 +0000567 # No recursion.
Tim Peters518df0d2003-01-28 01:00:38 +0000568 self.write(TUPLE)
Tim Peters1d63c9f2003-02-02 20:29:39 +0000569 self.memoize(obj)
Jeremy Hylton3422c992003-01-24 19:29:52 +0000570
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000571 dispatch[TupleType] = save_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000572
Tim Petersa6ae9a22003-01-28 16:58:41 +0000573 # save_empty_tuple() isn't used by anything in Python 2.3. However, I
574 # found a Pickler subclass in Zope3 that calls it, so it's not harmless
575 # to remove it.
Guido van Rossum3a41c612003-01-28 15:10:22 +0000576 def save_empty_tuple(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000577 self.write(EMPTY_TUPLE)
578
Guido van Rossum3a41c612003-01-28 15:10:22 +0000579 def save_list(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000580 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000581
Tim Petersc32d8242001-04-10 02:48:53 +0000582 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000583 write(EMPTY_LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000584 else: # proto 0 -- can't use EMPTY_LIST
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000585 write(MARK + LIST)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000586
587 self.memoize(obj)
588 self._batch_appends(iter(obj))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000589
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000590 dispatch[ListType] = save_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000591
Tim Peters42f08ac2003-02-11 22:43:24 +0000592 # Keep in synch with cPickle's BATCHSIZE. Nothing will break if it gets
593 # out of synch, though.
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000594 _BATCHSIZE = 1000
595
596 def _batch_appends(self, items):
597 # Helper to batch up APPENDS sequences
598 save = self.save
599 write = self.write
600
601 if not self.bin:
602 for x in items:
603 save(x)
604 write(APPEND)
605 return
606
Guido van Rossum805365e2007-05-07 22:24:25 +0000607 r = range(self._BATCHSIZE)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000608 while items is not None:
609 tmp = []
610 for i in r:
611 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000612 x = next(items)
Guido van Rossum5aac4e62003-02-06 22:57:00 +0000613 tmp.append(x)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000614 except StopIteration:
615 items = None
616 break
617 n = len(tmp)
618 if n > 1:
619 write(MARK)
620 for x in tmp:
621 save(x)
622 write(APPENDS)
623 elif n:
624 save(tmp[0])
625 write(APPEND)
626 # else tmp is empty, and we're done
627
Guido van Rossum3a41c612003-01-28 15:10:22 +0000628 def save_dict(self, obj):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000629 write = self.write
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000630
Tim Petersc32d8242001-04-10 02:48:53 +0000631 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000632 write(EMPTY_DICT)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000633 else: # proto 0 -- can't use EMPTY_DICT
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000634 write(MARK + DICT)
635
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000636 self.memoize(obj)
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000637 self._batch_setitems(iter(obj.items()))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000638
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000639 dispatch[DictionaryType] = save_dict
Jeremy Hylton2b9d0291998-05-27 22:38:22 +0000640 if not PyStringMap is None:
641 dispatch[PyStringMap] = save_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000642
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000643 def _batch_setitems(self, items):
644 # Helper to batch up SETITEMS sequences; proto >= 1 only
645 save = self.save
646 write = self.write
647
648 if not self.bin:
649 for k, v in items:
650 save(k)
651 save(v)
652 write(SETITEM)
653 return
654
Guido van Rossum805365e2007-05-07 22:24:25 +0000655 r = range(self._BATCHSIZE)
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000656 while items is not None:
657 tmp = []
658 for i in r:
659 try:
Georg Brandla18af4e2007-04-21 15:47:16 +0000660 tmp.append(next(items))
Guido van Rossum25cb7df2003-01-31 18:53:21 +0000661 except StopIteration:
662 items = None
663 break
664 n = len(tmp)
665 if n > 1:
666 write(MARK)
667 for k, v in tmp:
668 save(k)
669 save(v)
670 write(SETITEMS)
671 elif n:
672 k, v = tmp[0]
673 save(k)
674 save(v)
675 write(SETITEM)
676 # else tmp is empty, and we're done
677
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000678 def save_global(self, obj, name=None, pack=struct.pack):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000679 write = self.write
680 memo = self.memo
681
Tim Petersc32d8242001-04-10 02:48:53 +0000682 if name is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000683 name = obj.__name__
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000684
Jeremy Hylton4f0dcc92003-01-31 18:33:18 +0000685 module = getattr(obj, "__module__", None)
686 if module is None:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000687 module = whichmodule(obj, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000688
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000689 try:
690 __import__(module)
691 mod = sys.modules[module]
692 klass = getattr(mod, name)
693 except (ImportError, KeyError, AttributeError):
694 raise PicklingError(
695 "Can't pickle %r: it's not found as %s.%s" %
Guido van Rossum3a41c612003-01-28 15:10:22 +0000696 (obj, module, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000697 else:
Guido van Rossum3a41c612003-01-28 15:10:22 +0000698 if klass is not obj:
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000699 raise PicklingError(
700 "Can't pickle %r: it's not the same object as %s.%s" %
Guido van Rossum3a41c612003-01-28 15:10:22 +0000701 (obj, module, name))
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000702
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000703 if self.proto >= 2:
Guido van Rossumd4b920c2003-02-04 01:54:49 +0000704 code = _extension_registry.get((module, name))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000705 if code:
706 assert code > 0
707 if code <= 0xff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000708 write(EXT1 + bytes([code]))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000709 elif code <= 0xffff:
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000710 write(EXT2 + bytes([code&0xff, code>>8]))
Guido van Rossum255f3ee2003-01-29 06:14:11 +0000711 else:
712 write(EXT4 + pack("<i", code))
713 return
714
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000715 write(GLOBAL + bytes(module) + b'\n' + bytes(name) + b'\n')
Guido van Rossum3a41c612003-01-28 15:10:22 +0000716 self.memoize(obj)
Tim Peters3b769832003-01-28 03:51:36 +0000717
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000718 dispatch[ClassType] = save_global
719 dispatch[FunctionType] = save_global
720 dispatch[BuiltinFunctionType] = save_global
Raymond Hettingerfe59dc12005-02-07 15:28:45 +0000721 dispatch[TypeType] = save_global
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000722
Guido van Rossum1be31752003-01-28 15:19:53 +0000723# Pickling helpers
Guido van Rossuma48061a1995-01-10 00:31:14 +0000724
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000725def _keep_alive(x, memo):
726 """Keeps a reference to the object x in the memo.
727
728 Because we remember objects by their id, we have
729 to assure that possibly temporary objects are kept
730 alive by referencing them.
731 We store a reference at the id of the memo, which should
732 normally not be used unless someone tries to deepcopy
733 the memo itself...
734 """
735 try:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000736 memo[id(memo)].append(x)
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000737 except KeyError:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000738 # aha, this is the first one :-)
739 memo[id(memo)]=[x]
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000740
741
Tim Petersc0c12b52003-01-29 00:56:17 +0000742# A cache for whichmodule(), mapping a function object to the name of
743# the module in which the function was found.
744
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000745classmap = {} # called classmap for backwards compatibility
Guido van Rossuma48061a1995-01-10 00:31:14 +0000746
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000747def whichmodule(func, funcname):
748 """Figure out the module in which a function occurs.
Guido van Rossuma48061a1995-01-10 00:31:14 +0000749
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000750 Search sys.modules for the module.
751 Cache in classmap.
752 Return a module name.
Tim Petersc0c12b52003-01-29 00:56:17 +0000753 If the function cannot be found, return "__main__".
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000754 """
Jeremy Hylton4f0dcc92003-01-31 18:33:18 +0000755 # Python functions should always get an __module__ from their globals.
756 mod = getattr(func, "__module__", None)
757 if mod is not None:
758 return mod
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000759 if func in classmap:
760 return classmap[func]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000761
Guido van Rossum634e53f2007-02-26 07:07:02 +0000762 for name, module in list(sys.modules.items()):
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000763 if module is None:
Jeremy Hylton065a5ab2002-09-19 22:57:26 +0000764 continue # skip dummy package entries
Jeremy Hyltoncc1fccb2003-02-06 16:23:01 +0000765 if name != '__main__' and getattr(module, funcname, None) is func:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000766 break
767 else:
768 name = '__main__'
Jeremy Hyltonf0cfdf72002-09-19 23:00:12 +0000769 classmap[func] = name
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000770 return name
Guido van Rossuma48061a1995-01-10 00:31:14 +0000771
772
Guido van Rossum1be31752003-01-28 15:19:53 +0000773# Unpickling machinery
774
Guido van Rossuma48061a1995-01-10 00:31:14 +0000775class Unpickler:
776
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000777 def __init__(self, file):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000778 """This takes a binary file for reading a pickle data stream.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000779
Tim Peters5bd2a792003-02-01 16:45:06 +0000780 The protocol version of the pickle is detected automatically, so no
781 proto argument is needed.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000782
783 The file-like object must have two methods, a read() method that
784 takes an integer argument, and a readline() method that requires no
785 arguments. Both methods should return a string. Thus file-like
786 object can be a file object opened for reading, a StringIO object,
787 or any other custom object that meets this interface.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000788 """
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000789 try:
790 self.readline = file.readline
791 except AttributeError:
792 self.file = file
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000793 self.read = file.read
794 self.memo = {}
Guido van Rossuma48061a1995-01-10 00:31:14 +0000795
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000796 def readline(self):
797 # XXX Slow but at least correct
798 b = bytes()
799 while True:
800 c = self.file.read(1)
801 if not c:
802 break
803 b += c
804 if c == b'\n':
805 break
806 return b
807
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000808 def load(self):
Guido van Rossum3a41c612003-01-28 15:10:22 +0000809 """Read a pickled object representation from the open file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000810
Guido van Rossum3a41c612003-01-28 15:10:22 +0000811 Return the reconstituted object hierarchy specified in the file.
Raymond Hettingeraef22fb2002-05-29 16:18:42 +0000812 """
Jeremy Hylton20747fa2001-11-09 16:15:04 +0000813 self.mark = object() # any new unique object
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000814 self.stack = []
815 self.append = self.stack.append
816 read = self.read
817 dispatch = self.dispatch
818 try:
819 while 1:
820 key = read(1)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000821 if not key:
822 raise EOFError
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000823 assert isinstance(key, bytes)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000824 dispatch[key[0]](self)
Guido van Rossumb940e112007-01-10 16:19:56 +0000825 except _Stop as stopinst:
Guido van Rossumff871742000-12-13 18:11:56 +0000826 return stopinst.value
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000827
Tim Petersc23d18a2003-01-28 01:41:51 +0000828 # Return largest index k such that self.stack[k] is self.mark.
829 # If the stack doesn't contain a mark, eventually raises IndexError.
830 # This could be sped by maintaining another stack, of indices at which
831 # the mark appears. For that matter, the latter stack would suffice,
832 # and we wouldn't need to push mark objects on self.stack at all.
833 # Doing so is probably a good thing, though, since if the pickle is
834 # corrupt (or hostile) we may get a clue from finding self.mark embedded
835 # in unpickled objects.
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000836 def marker(self):
837 stack = self.stack
838 mark = self.mark
839 k = len(stack)-1
840 while stack[k] is not mark: k = k-1
841 return k
842
843 dispatch = {}
844
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000845 def load_proto(self):
846 proto = ord(self.read(1))
847 if not 0 <= proto <= 2:
848 raise ValueError, "unsupported pickle protocol: %d" % proto
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000849 dispatch[PROTO[0]] = load_proto
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000850
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000851 def load_persid(self):
852 pid = self.readline()[:-1]
853 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000854 dispatch[PERSID[0]] = load_persid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000855
856 def load_binpersid(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +0000857 pid = self.stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000858 self.append(self.persistent_load(pid))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000859 dispatch[BINPERSID[0]] = load_binpersid
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000860
861 def load_none(self):
862 self.append(None)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000863 dispatch[NONE[0]] = load_none
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000864
Guido van Rossum7d97d312003-01-28 04:25:27 +0000865 def load_false(self):
866 self.append(False)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000867 dispatch[NEWFALSE[0]] = load_false
Guido van Rossum7d97d312003-01-28 04:25:27 +0000868
869 def load_true(self):
870 self.append(True)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000871 dispatch[NEWTRUE[0]] = load_true
Guido van Rossum7d97d312003-01-28 04:25:27 +0000872
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000873 def load_int(self):
Tim Peters19ef62d2001-08-28 22:21:18 +0000874 data = self.readline()
Guido van Rossume2763392002-04-05 19:30:08 +0000875 if data == FALSE[1:]:
876 val = False
877 elif data == TRUE[1:]:
878 val = True
879 else:
880 try:
881 val = int(data)
882 except ValueError:
Guido van Rossume2a383d2007-01-15 16:59:06 +0000883 val = int(data)
Guido van Rossume2763392002-04-05 19:30:08 +0000884 self.append(val)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000885 dispatch[INT[0]] = load_int
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000886
887 def load_binint(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000888 self.append(mloads(b'i' + self.read(4)))
889 dispatch[BININT[0]] = load_binint
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000890
891 def load_binint1(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +0000892 self.append(ord(self.read(1)))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000893 dispatch[BININT1[0]] = load_binint1
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000894
895 def load_binint2(self):
Guido van Rossumcfe5f202007-05-08 21:26:54 +0000896 self.append(mloads(b'i' + self.read(2) + b'\000\000'))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000897 dispatch[BININT2[0]] = load_binint2
Tim Peters2344fae2001-01-15 00:50:52 +0000898
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000899 def load_long(self):
Guido van Rossum1255ed62007-05-04 20:30:19 +0000900 self.append(int(str(self.readline()[:-1]), 0))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000901 dispatch[LONG[0]] = load_long
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000902
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000903 def load_long1(self):
904 n = ord(self.read(1))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000905 data = self.read(n)
906 self.append(decode_long(data))
907 dispatch[LONG1[0]] = load_long1
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000908
909 def load_long4(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000910 n = mloads(b'i' + self.read(4))
911 data = self.read(n)
912 self.append(decode_long(data))
913 dispatch[LONG4[0]] = load_long4
Guido van Rossumd6c9e632003-01-28 03:49:52 +0000914
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000915 def load_float(self):
Guido van Rossumff871742000-12-13 18:11:56 +0000916 self.append(float(self.readline()[:-1]))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000917 dispatch[FLOAT[0]] = load_float
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000918
Guido van Rossumd3703791998-10-22 20:15:36 +0000919 def load_binfloat(self, unpack=struct.unpack):
920 self.append(unpack('>d', self.read(8))[0])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000921 dispatch[BINFLOAT[0]] = load_binfloat
Guido van Rossumd3703791998-10-22 20:15:36 +0000922
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000923 def load_string(self):
Jeremy Hyltonbe467e52000-09-15 15:14:51 +0000924 rep = self.readline()[:-1]
Tim Petersad5a7712003-01-28 16:23:33 +0000925 for q in "\"'": # double or single quote
Martin v. Löwis8a8da792002-08-14 07:46:28 +0000926 if rep.startswith(q):
927 if not rep.endswith(q):
928 raise ValueError, "insecure string pickle"
929 rep = rep[len(q):-len(q)]
930 break
931 else:
Jeremy Hyltonbe467e52000-09-15 15:14:51 +0000932 raise ValueError, "insecure string pickle"
Guido van Rossumb8142c32007-05-08 17:49:10 +0000933 self.append(str8(rep.decode("string-escape")))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000934 dispatch[STRING[0]] = load_string
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000935
936 def load_binstring(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000937 len = mloads(b'i' + self.read(4))
Guido van Rossumb8142c32007-05-08 17:49:10 +0000938 self.append(str8(self.read(len)))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000939 dispatch[BINSTRING[0]] = load_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000940
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000941 def load_unicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000942 self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
943 dispatch[UNICODE[0]] = load_unicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000944
945 def load_binunicode(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000946 len = mloads(b'i' + self.read(4))
947 self.append(str(self.read(len), 'utf-8'))
948 dispatch[BINUNICODE[0]] = load_binunicode
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000949
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000950 def load_short_binstring(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +0000951 len = ord(self.read(1))
Guido van Rossumb8142c32007-05-08 17:49:10 +0000952 self.append(str8(self.read(len)))
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000953 dispatch[SHORT_BINSTRING[0]] = load_short_binstring
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000954
955 def load_tuple(self):
956 k = self.marker()
957 self.stack[k:] = [tuple(self.stack[k+1:])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000958 dispatch[TUPLE[0]] = load_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000959
960 def load_empty_tuple(self):
961 self.stack.append(())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000962 dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000963
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000964 def load_tuple1(self):
965 self.stack[-1] = (self.stack[-1],)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000966 dispatch[TUPLE1[0]] = load_tuple1
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000967
968 def load_tuple2(self):
969 self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000970 dispatch[TUPLE2[0]] = load_tuple2
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000971
972 def load_tuple3(self):
973 self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000974 dispatch[TUPLE3[0]] = load_tuple3
Guido van Rossum44f0ea52003-01-28 04:14:51 +0000975
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000976 def load_empty_list(self):
977 self.stack.append([])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000978 dispatch[EMPTY_LIST[0]] = load_empty_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000979
980 def load_empty_dictionary(self):
981 self.stack.append({})
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000982 dispatch[EMPTY_DICT[0]] = load_empty_dictionary
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000983
984 def load_list(self):
985 k = self.marker()
986 self.stack[k:] = [self.stack[k+1:]]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000987 dispatch[LIST[0]] = load_list
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000988
989 def load_dict(self):
990 k = self.marker()
991 d = {}
992 items = self.stack[k+1:]
993 for i in range(0, len(items), 2):
994 key = items[i]
995 value = items[i+1]
996 d[key] = value
997 self.stack[k:] = [d]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +0000998 dispatch[DICT[0]] = load_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000999
Tim Petersd01c1e92003-01-30 15:41:46 +00001000 # INST and OBJ differ only in how they get a class object. It's not
1001 # only sensible to do the rest in a common routine, the two routines
1002 # previously diverged and grew different bugs.
1003 # klass is the class to instantiate, and k points to the topmost mark
1004 # object, following which are the arguments for klass.__init__.
1005 def _instantiate(self, klass, k):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001006 args = tuple(self.stack[k+1:])
1007 del self.stack[k:]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001008 instantiated = 0
Tim Petersd01c1e92003-01-30 15:41:46 +00001009 if (not args and
1010 type(klass) is ClassType and
1011 not hasattr(klass, "__getinitargs__")):
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001012 try:
1013 value = _EmptyClass()
1014 value.__class__ = klass
Guido van Rossumb19e2a31998-04-13 18:08:45 +00001015 instantiated = 1
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001016 except RuntimeError:
1017 # In restricted execution, assignment to inst.__class__ is
1018 # prohibited
1019 pass
1020 if not instantiated:
Guido van Rossum743d17e1998-09-15 20:25:57 +00001021 try:
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001022 value = klass(*args)
Guido van Rossumb940e112007-01-10 16:19:56 +00001023 except TypeError as err:
Guido van Rossum743d17e1998-09-15 20:25:57 +00001024 raise TypeError, "in constructor for %s: %s" % (
1025 klass.__name__, str(err)), sys.exc_info()[2]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001026 self.append(value)
Tim Petersd01c1e92003-01-30 15:41:46 +00001027
1028 def load_inst(self):
1029 module = self.readline()[:-1]
1030 name = self.readline()[:-1]
1031 klass = self.find_class(module, name)
1032 self._instantiate(klass, self.marker())
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001033 dispatch[INST[0]] = load_inst
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001034
1035 def load_obj(self):
Tim Petersd01c1e92003-01-30 15:41:46 +00001036 # Stack is ... markobject classobject arg1 arg2 ...
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001037 k = self.marker()
Tim Petersd01c1e92003-01-30 15:41:46 +00001038 klass = self.stack.pop(k+1)
1039 self._instantiate(klass, k)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001040 dispatch[OBJ[0]] = load_obj
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001041
Guido van Rossum3a41c612003-01-28 15:10:22 +00001042 def load_newobj(self):
1043 args = self.stack.pop()
1044 cls = self.stack[-1]
1045 obj = cls.__new__(cls, *args)
Guido van Rossum533dbcf2003-01-28 17:55:05 +00001046 self.stack[-1] = obj
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001047 dispatch[NEWOBJ[0]] = load_newobj
Guido van Rossum3a41c612003-01-28 15:10:22 +00001048
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001049 def load_global(self):
1050 module = self.readline()[:-1]
1051 name = self.readline()[:-1]
1052 klass = self.find_class(module, name)
1053 self.append(klass)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001054 dispatch[GLOBAL[0]] = load_global
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001055
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001056 def load_ext1(self):
1057 code = ord(self.read(1))
1058 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001059 dispatch[EXT1[0]] = load_ext1
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001060
1061 def load_ext2(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001062 code = mloads(b'i' + self.read(2) + b'\000\000')
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001063 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001064 dispatch[EXT2[0]] = load_ext2
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001065
1066 def load_ext4(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001067 code = mloads(b'i' + self.read(4))
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001068 self.get_extension(code)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001069 dispatch[EXT4[0]] = load_ext4
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001070
1071 def get_extension(self, code):
1072 nil = []
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001073 obj = _extension_cache.get(code, nil)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001074 if obj is not nil:
1075 self.append(obj)
1076 return
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001077 key = _inverted_registry.get(code)
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001078 if not key:
1079 raise ValueError("unregistered extension code %d" % code)
1080 obj = self.find_class(*key)
Guido van Rossumd4b920c2003-02-04 01:54:49 +00001081 _extension_cache[code] = obj
Guido van Rossum255f3ee2003-01-29 06:14:11 +00001082 self.append(obj)
1083
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001084 def find_class(self, module, name):
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001085 # Subclasses may override this
Guido van Rossum1255ed62007-05-04 20:30:19 +00001086 module = str(module)
1087 name = str(name)
Barry Warsawbf4d9592001-11-15 23:42:58 +00001088 __import__(module)
1089 mod = sys.modules[module]
1090 klass = getattr(mod, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001091 return klass
1092
1093 def load_reduce(self):
1094 stack = self.stack
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001095 args = stack.pop()
1096 func = stack[-1]
Raymond Hettingera6b45cc2004-12-07 07:05:57 +00001097 value = func(*args)
Guido van Rossumb26a97a2003-01-28 22:29:13 +00001098 stack[-1] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001099 dispatch[REDUCE[0]] = load_reduce
Guido van Rossuma48061a1995-01-10 00:31:14 +00001100
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001101 def load_pop(self):
1102 del self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001103 dispatch[POP[0]] = load_pop
Guido van Rossum7b5430f1995-03-04 22:25:21 +00001104
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001105 def load_pop_mark(self):
1106 k = self.marker()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001107 del self.stack[k:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001108 dispatch[POP_MARK[0]] = load_pop_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001109
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001110 def load_dup(self):
Guido van Rossumb1062fc1998-03-31 17:00:46 +00001111 self.append(self.stack[-1])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001112 dispatch[DUP[0]] = load_dup
Guido van Rossuma48061a1995-01-10 00:31:14 +00001113
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001114 def load_get(self):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001115 self.append(self.memo[str8(self.readline())[:-1]])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001116 dispatch[GET[0]] = load_get
Guido van Rossum78536471996-04-12 13:36:27 +00001117
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001118 def load_binget(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +00001119 i = ord(self.read(1))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001120 self.append(self.memo[repr(i)])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001121 dispatch[BINGET[0]] = load_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001122
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001123 def load_long_binget(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001124 i = mloads(b'i' + self.read(4))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001125 self.append(self.memo[repr(i)])
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001126 dispatch[LONG_BINGET[0]] = load_long_binget
Guido van Rossum78536471996-04-12 13:36:27 +00001127
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001128 def load_put(self):
Guido van Rossum1255ed62007-05-04 20:30:19 +00001129 self.memo[str(self.readline()[:-1])] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001130 dispatch[PUT[0]] = load_put
Guido van Rossuma48061a1995-01-10 00:31:14 +00001131
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001132 def load_binput(self):
Tim Petersbbf63cd2003-01-27 21:15:36 +00001133 i = ord(self.read(1))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001134 self.memo[repr(i)] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001135 dispatch[BINPUT[0]] = load_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001136
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001137 def load_long_binput(self):
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001138 i = mloads(b'i' + self.read(4))
Walter Dörwald70a6b492004-02-12 17:35:32 +00001139 self.memo[repr(i)] = self.stack[-1]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001140 dispatch[LONG_BINPUT[0]] = load_long_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +00001141
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001142 def load_append(self):
1143 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001144 value = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001145 list = stack[-1]
1146 list.append(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001147 dispatch[APPEND[0]] = load_append
Guido van Rossuma48061a1995-01-10 00:31:14 +00001148
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001149 def load_appends(self):
1150 stack = self.stack
1151 mark = self.marker()
1152 list = stack[mark - 1]
Tim Peters209ad952003-01-28 01:44:45 +00001153 list.extend(stack[mark + 1:])
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001154 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001155 dispatch[APPENDS[0]] = load_appends
Tim Peters2344fae2001-01-15 00:50:52 +00001156
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001157 def load_setitem(self):
1158 stack = self.stack
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001159 value = stack.pop()
1160 key = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001161 dict = stack[-1]
1162 dict[key] = value
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001163 dispatch[SETITEM[0]] = load_setitem
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001164
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001165 def load_setitems(self):
1166 stack = self.stack
1167 mark = self.marker()
1168 dict = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001169 for i in range(mark + 1, len(stack), 2):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001170 dict[stack[i]] = stack[i + 1]
Guido van Rossuma48061a1995-01-10 00:31:14 +00001171
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001172 del stack[mark:]
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001173 dispatch[SETITEMS[0]] = load_setitems
Guido van Rossuma48061a1995-01-10 00:31:14 +00001174
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001175 def load_build(self):
1176 stack = self.stack
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001177 state = stack.pop()
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001178 inst = stack[-1]
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001179 setstate = getattr(inst, "__setstate__", None)
1180 if setstate:
1181 setstate(state)
1182 return
1183 slotstate = None
1184 if isinstance(state, tuple) and len(state) == 2:
1185 state, slotstate = state
1186 if state:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001187 try:
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001188 inst.__dict__.update(state)
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001189 except RuntimeError:
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001190 # XXX In restricted execution, the instance's __dict__
1191 # is not accessible. Use the old way of unpickling
1192 # the instance variables. This is a semantic
1193 # difference when unpickling in restricted
1194 # vs. unrestricted modes.
Tim Peters080c88b2003-02-15 03:01:11 +00001195 # Note, however, that cPickle has never tried to do the
1196 # .update() business, and always uses
1197 # PyObject_SetItem(inst.__dict__, key, value) in a
1198 # loop over state.items().
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001199 for k, v in state.items():
Guido van Rossum45e2fbc1998-03-26 21:13:24 +00001200 setattr(inst, k, v)
Guido van Rossumac5b5d22003-01-28 22:01:16 +00001201 if slotstate:
1202 for k, v in slotstate.items():
1203 setattr(inst, k, v)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001204 dispatch[BUILD[0]] = load_build
Guido van Rossuma48061a1995-01-10 00:31:14 +00001205
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001206 def load_mark(self):
1207 self.append(self.mark)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001208 dispatch[MARK[0]] = load_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +00001209
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001210 def load_stop(self):
Raymond Hettinger46ac8eb2002-06-30 03:39:14 +00001211 value = self.stack.pop()
Guido van Rossumff871742000-12-13 18:11:56 +00001212 raise _Stop(value)
Guido van Rossum2e6a4b32007-05-04 19:56:22 +00001213 dispatch[STOP[0]] = load_stop
Guido van Rossuma48061a1995-01-10 00:31:14 +00001214
Guido van Rossume467be61997-12-05 19:42:42 +00001215# Helper class for load_inst/load_obj
1216
1217class _EmptyClass:
1218 pass
Guido van Rossuma48061a1995-01-10 00:31:14 +00001219
Tim Peters91149822003-01-31 03:43:58 +00001220# Encode/decode longs in linear time.
1221
1222import binascii as _binascii
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001223
1224def encode_long(x):
Tim Peters91149822003-01-31 03:43:58 +00001225 r"""Encode a long to a two's complement little-endian binary string.
Guido van Rossume2a383d2007-01-15 16:59:06 +00001226 Note that 0 is a special case, returning an empty string, to save a
Tim Peters4b23f2b2003-01-31 16:43:39 +00001227 byte in the LONG1 pickling context.
1228
Guido van Rossume2a383d2007-01-15 16:59:06 +00001229 >>> encode_long(0)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001230 b''
Guido van Rossume2a383d2007-01-15 16:59:06 +00001231 >>> encode_long(255)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001232 b'\xff\x00'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001233 >>> encode_long(32767)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001234 b'\xff\x7f'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001235 >>> encode_long(-256)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001236 b'\x00\xff'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001237 >>> encode_long(-32768)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001238 b'\x00\x80'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001239 >>> encode_long(-128)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001240 b'\x80'
Guido van Rossume2a383d2007-01-15 16:59:06 +00001241 >>> encode_long(127)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001242 b'\x7f'
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001243 >>>
1244 """
Tim Peters91149822003-01-31 03:43:58 +00001245
1246 if x == 0:
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001247 return b''
Tim Peters91149822003-01-31 03:43:58 +00001248 if x > 0:
1249 ashex = hex(x)
1250 assert ashex.startswith("0x")
1251 njunkchars = 2 + ashex.endswith('L')
1252 nibbles = len(ashex) - njunkchars
1253 if nibbles & 1:
1254 # need an even # of nibbles for unhexlify
1255 ashex = "0x0" + ashex[2:]
Tim Peters4b23f2b2003-01-31 16:43:39 +00001256 elif int(ashex[2], 16) >= 8:
Tim Peters91149822003-01-31 03:43:58 +00001257 # "looks negative", so need a byte of sign bits
1258 ashex = "0x00" + ashex[2:]
1259 else:
1260 # Build the 256's-complement: (1L << nbytes) + x. The trick is
1261 # to find the number of bytes in linear time (although that should
1262 # really be a constant-time task).
1263 ashex = hex(-x)
1264 assert ashex.startswith("0x")
1265 njunkchars = 2 + ashex.endswith('L')
1266 nibbles = len(ashex) - njunkchars
1267 if nibbles & 1:
Tim Petersee1a53c2003-02-02 02:57:53 +00001268 # Extend to a full byte.
Tim Peters91149822003-01-31 03:43:58 +00001269 nibbles += 1
Tim Peters4b23f2b2003-01-31 16:43:39 +00001270 nbits = nibbles * 4
Guido van Rossume2a383d2007-01-15 16:59:06 +00001271 x += 1 << nbits
Tim Peters91149822003-01-31 03:43:58 +00001272 assert x > 0
1273 ashex = hex(x)
Tim Petersee1a53c2003-02-02 02:57:53 +00001274 njunkchars = 2 + ashex.endswith('L')
1275 newnibbles = len(ashex) - njunkchars
1276 if newnibbles < nibbles:
1277 ashex = "0x" + "0" * (nibbles - newnibbles) + ashex[2:]
1278 if int(ashex[2], 16) < 8:
Tim Peters91149822003-01-31 03:43:58 +00001279 # "looks positive", so need a byte of sign bits
Tim Petersee1a53c2003-02-02 02:57:53 +00001280 ashex = "0xff" + ashex[2:]
Tim Peters91149822003-01-31 03:43:58 +00001281
1282 if ashex.endswith('L'):
1283 ashex = ashex[2:-1]
1284 else:
1285 ashex = ashex[2:]
Tim Petersee1a53c2003-02-02 02:57:53 +00001286 assert len(ashex) & 1 == 0, (x, ashex)
Tim Peters91149822003-01-31 03:43:58 +00001287 binary = _binascii.unhexlify(ashex)
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001288 return bytes(binary[::-1])
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001289
1290def decode_long(data):
1291 r"""Decode a long from a two's complement little-endian binary string.
Tim Peters4b23f2b2003-01-31 16:43:39 +00001292
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001293 >>> decode_long(b'')
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001294 0
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001295 >>> decode_long(b"\xff\x00")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001296 255
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001297 >>> decode_long(b"\xff\x7f")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001298 32767
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001299 >>> decode_long(b"\x00\xff")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001300 -256
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001301 >>> decode_long(b"\x00\x80")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001302 -32768
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001303 >>> decode_long(b"\x80")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001304 -128
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001305 >>> decode_long(b"\x7f")
Guido van Rossume2b70bc2006-08-18 22:13:04 +00001306 127
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001307 """
Tim Peters91149822003-01-31 03:43:58 +00001308
Tim Peters4b23f2b2003-01-31 16:43:39 +00001309 nbytes = len(data)
1310 if nbytes == 0:
Guido van Rossume2a383d2007-01-15 16:59:06 +00001311 return 0
Tim Peters91149822003-01-31 03:43:58 +00001312 ashex = _binascii.hexlify(data[::-1])
Guido van Rossume2a383d2007-01-15 16:59:06 +00001313 n = int(ashex, 16) # quadratic time before Python 2.3; linear now
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001314 if data[-1] >= 0x80:
Guido van Rossume2a383d2007-01-15 16:59:06 +00001315 n -= 1 << (nbytes * 8)
Tim Peters91149822003-01-31 03:43:58 +00001316 return n
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001317
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001318# Shorthands
1319
Raymond Hettinger3489cad2004-12-05 05:20:42 +00001320def dump(obj, file, protocol=None):
1321 Pickler(file, protocol).dump(obj)
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001322
Raymond Hettinger3489cad2004-12-05 05:20:42 +00001323def dumps(obj, protocol=None):
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001324 f = io.BytesIO()
1325 Pickler(f, protocol).dump(obj)
1326 res = f.getvalue()
1327 assert isinstance(res, bytes)
1328 return res
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001329
1330def load(file):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001331 return Unpickler(file).load()
Guido van Rossum0c891ce1995-03-14 15:09:05 +00001332
Guido van Rossumcfe5f202007-05-08 21:26:54 +00001333def loads(s):
1334 if isinstance(s, str):
1335 raise TypeError("Can't load pickle from unicode string")
1336 file = io.BytesIO(s)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +00001337 return Unpickler(file).load()
Guido van Rossumd6c9e632003-01-28 03:49:52 +00001338
1339# Doctest
1340
1341def _test():
1342 import doctest
1343 return doctest.testmod()
1344
1345if __name__ == "__main__":
1346 _test()