blob: d77e33e844399712b98a1245205cd9b087f76f9a [file] [log] [blame]
Guido van Rossum54f22ed2000-02-04 15:10:34 +00001"""Create portable serialized representations of Python objects.
Guido van Rossuma48061a1995-01-10 00:31:14 +00002
Guido van Rossume467be61997-12-05 19:42:42 +00003See module cPickle for a (much) faster implementation.
4See module copy_reg for a mechanism for registering custom picklers.
Guido van Rossuma48061a1995-01-10 00:31:14 +00005
Guido van Rossume467be61997-12-05 19:42:42 +00006Classes:
Guido van Rossuma48061a1995-01-10 00:31:14 +00007
Guido van Rossume467be61997-12-05 19:42:42 +00008 Pickler
9 Unpickler
Guido van Rossuma48061a1995-01-10 00:31:14 +000010
Guido van Rossume467be61997-12-05 19:42:42 +000011Functions:
Guido van Rossuma48061a1995-01-10 00:31:14 +000012
Guido van Rossume467be61997-12-05 19:42:42 +000013 dump(object, file)
14 dumps(object) -> string
15 load(file) -> object
16 loads(string) -> object
Guido van Rossuma48061a1995-01-10 00:31:14 +000017
Guido van Rossume467be61997-12-05 19:42:42 +000018Misc variables:
Guido van Rossuma48061a1995-01-10 00:31:14 +000019
Fred Drakefe82acc1998-02-13 03:24:48 +000020 __version__
Guido van Rossume467be61997-12-05 19:42:42 +000021 format_version
22 compatible_formats
Guido van Rossuma48061a1995-01-10 00:31:14 +000023
Guido van Rossuma48061a1995-01-10 00:31:14 +000024"""
25
Guido van Rossum743d17e1998-09-15 20:25:57 +000026__version__ = "$Revision$" # Code version
Guido van Rossuma48061a1995-01-10 00:31:14 +000027
28from types import *
Guido van Rossum4fb5b281997-09-12 20:07:24 +000029from copy_reg import dispatch_table, safe_constructors
Guido van Rossumd3703791998-10-22 20:15:36 +000030import marshal
31import sys
32import struct
Skip Montanaro23bafc62001-02-18 03:10:09 +000033import re
Guido van Rossuma48061a1995-01-10 00:31:14 +000034
Skip Montanaro352674d2001-02-07 23:14:30 +000035__all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
36 "Unpickler", "dump", "dumps", "load", "loads"]
37
Guido van Rossumd3703791998-10-22 20:15:36 +000038format_version = "1.3" # File format version we write
39compatible_formats = ["1.0", "1.1", "1.2"] # Old format versions we can read
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000040
41mdumps = marshal.dumps
42mloads = marshal.loads
Guido van Rossum0c891ce1995-03-14 15:09:05 +000043
Guido van Rossum26e89d02000-06-29 16:15:52 +000044class PickleError(Exception): pass
45class PicklingError(PickleError): pass
46class UnpicklingError(PickleError): pass
Guido van Rossum7849da81995-03-09 14:08:35 +000047
Guido van Rossumff871742000-12-13 18:11:56 +000048class _Stop(Exception):
49 def __init__(self, value):
50 self.value = value
51
Jeremy Hylton2b9d0291998-05-27 22:38:22 +000052try:
53 from org.python.core import PyStringMap
54except ImportError:
55 PyStringMap = None
56
Guido van Rossumdbb718f2001-09-21 19:22:34 +000057try:
58 UnicodeType
59except NameError:
60 UnicodeType = None
61
62
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000063MARK = '('
64STOP = '.'
65POP = '0'
66POP_MARK = '1'
67DUP = '2'
68FLOAT = 'F'
69INT = 'I'
70BININT = 'J'
71BININT1 = 'K'
72LONG = 'L'
73BININT2 = 'M'
74NONE = 'N'
75PERSID = 'P'
76BINPERSID = 'Q'
77REDUCE = 'R'
78STRING = 'S'
79BINSTRING = 'T'
80SHORT_BINSTRING = 'U'
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +000081UNICODE = 'V'
82BINUNICODE = 'X'
Guido van Rossumb72cf2d1997-04-09 17:32:51 +000083APPEND = 'a'
84BUILD = 'b'
85GLOBAL = 'c'
86DICT = 'd'
87EMPTY_DICT = '}'
88APPENDS = 'e'
89GET = 'g'
90BINGET = 'h'
91INST = 'i'
92LONG_BINGET = 'j'
93LIST = 'l'
94EMPTY_LIST = ']'
95OBJ = 'o'
96PUT = 'p'
97BINPUT = 'q'
98LONG_BINPUT = 'r'
99SETITEM = 's'
100TUPLE = 't'
101EMPTY_TUPLE = ')'
102SETITEMS = 'u'
Guido van Rossumd3703791998-10-22 20:15:36 +0000103BINFLOAT = 'G'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000104
Skip Montanaro23bafc62001-02-18 03:10:09 +0000105__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
106
Guido van Rossuma48061a1995-01-10 00:31:14 +0000107class Pickler:
108
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000109 def __init__(self, file, bin = 0):
110 self.write = file.write
111 self.memo = {}
112 self.bin = bin
Guido van Rossuma48061a1995-01-10 00:31:14 +0000113
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000114 def dump(self, object):
115 self.save(object)
116 self.write(STOP)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000117
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000118 def put(self, i):
Tim Petersc32d8242001-04-10 02:48:53 +0000119 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000120 s = mdumps(i)[1:]
Tim Petersc32d8242001-04-10 02:48:53 +0000121 if i < 256:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000122 return BINPUT + s[0]
Guido van Rossuma48061a1995-01-10 00:31:14 +0000123
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000124 return LONG_BINPUT + s
Guido van Rossuma48061a1995-01-10 00:31:14 +0000125
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000126 return PUT + `i` + '\n'
Guido van Rossuma48061a1995-01-10 00:31:14 +0000127
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000128 def get(self, i):
Tim Petersc32d8242001-04-10 02:48:53 +0000129 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000130 s = mdumps(i)[1:]
Guido van Rossuma48061a1995-01-10 00:31:14 +0000131
Tim Petersc32d8242001-04-10 02:48:53 +0000132 if i < 256:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000133 return BINGET + s[0]
Guido van Rossuma48061a1995-01-10 00:31:14 +0000134
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000135 return LONG_BINGET + s
Guido van Rossuma48061a1995-01-10 00:31:14 +0000136
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000137 return GET + `i` + '\n'
Tim Peters2344fae2001-01-15 00:50:52 +0000138
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000139 def save(self, object, pers_save = 0):
140 memo = self.memo
Guido van Rossuma48061a1995-01-10 00:31:14 +0000141
Tim Petersc32d8242001-04-10 02:48:53 +0000142 if not pers_save:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000143 pid = self.persistent_id(object)
Tim Petersc32d8242001-04-10 02:48:53 +0000144 if pid is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000145 self.save_pers(pid)
146 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000147
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000148 d = id(object)
Tim Peters2344fae2001-01-15 00:50:52 +0000149
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000150 t = type(object)
Guido van Rossuma48061a1995-01-10 00:31:14 +0000151
Tim Petersc32d8242001-04-10 02:48:53 +0000152 if (t is TupleType) and (len(object) == 0):
153 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000154 self.save_empty_tuple(object)
155 else:
156 self.save_tuple(object)
157 return
Guido van Rossuma48061a1995-01-10 00:31:14 +0000158
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000159 if memo.has_key(d):
160 self.write(self.get(memo[d][0]))
161 return
162
163 try:
164 f = self.dispatch[t]
165 except KeyError:
166 pid = self.inst_persistent_id(object)
167 if pid is not None:
168 self.save_pers(pid)
169 return
170
171 try:
172 reduce = dispatch_table[t]
173 except KeyError:
174 try:
175 reduce = object.__reduce__
176 except AttributeError:
177 raise PicklingError, \
Guido van Rossum08a92cb1999-10-10 21:14:25 +0000178 "can't pickle %s object: %s" % (`t.__name__`,
179 `object`)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000180 else:
181 tup = reduce()
182 else:
183 tup = reduce(object)
184
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000185 if type(tup) is StringType:
186 self.save_global(object, tup)
187 return
Guido van Rossumd1f49841997-12-10 23:40:18 +0000188
Tim Petersc32d8242001-04-10 02:48:53 +0000189 if type(tup) is not TupleType:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000190 raise PicklingError, "Value returned by %s must be a " \
191 "tuple" % reduce
192
193 l = len(tup)
Tim Peters2344fae2001-01-15 00:50:52 +0000194
Tim Petersc32d8242001-04-10 02:48:53 +0000195 if (l != 2) and (l != 3):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000196 raise PicklingError, "tuple returned by %s must contain " \
197 "only two or three elements" % reduce
198
199 callable = tup[0]
200 arg_tup = tup[1]
Tim Peters2344fae2001-01-15 00:50:52 +0000201
Tim Petersc32d8242001-04-10 02:48:53 +0000202 if l > 2:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000203 state = tup[2]
204 else:
205 state = None
206
Guido van Rossumd1f49841997-12-10 23:40:18 +0000207 if type(arg_tup) is not TupleType and arg_tup is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000208 raise PicklingError, "Second element of tuple returned " \
209 "by %s must be a tuple" % reduce
210
Tim Peters2344fae2001-01-15 00:50:52 +0000211 self.save_reduce(callable, arg_tup, state)
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000212 memo_len = len(memo)
213 self.write(self.put(memo_len))
214 memo[d] = (memo_len, object)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000215 return
216
217 f(self, object)
218
219 def persistent_id(self, object):
220 return None
221
222 def inst_persistent_id(self, object):
223 return None
224
225 def save_pers(self, pid):
Tim Petersc32d8242001-04-10 02:48:53 +0000226 if not self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000227 self.write(PERSID + str(pid) + '\n')
228 else:
229 self.save(pid, 1)
230 self.write(BINPERSID)
231
232 def save_reduce(self, callable, arg_tup, state = None):
233 write = self.write
234 save = self.save
235
236 save(callable)
237 save(arg_tup)
238 write(REDUCE)
Tim Peters2344fae2001-01-15 00:50:52 +0000239
Tim Petersc32d8242001-04-10 02:48:53 +0000240 if state is not None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000241 save(state)
242 write(BUILD)
243
244 dispatch = {}
245
246 def save_none(self, object):
247 self.write(NONE)
248 dispatch[NoneType] = save_none
249
250 def save_int(self, object):
Tim Petersc32d8242001-04-10 02:48:53 +0000251 if self.bin:
Tim Peters44714002001-04-10 05:02:52 +0000252 # If the int is small enough to fit in a signed 4-byte 2's-comp
253 # format, we can store it more efficiently than the general
254 # case.
255 high_bits = object >> 31 # note that Python shift sign-extends
256 if high_bits == 0 or high_bits == -1:
257 # All high bits are copies of bit 2**31, so the value
258 # fits in a 4-byte signed int.
259 i = mdumps(object)[1:]
260 assert len(i) == 4
261 if i[-2:] == '\000\000': # fits in 2-byte unsigned int
262 if i[-3] == '\000': # fits in 1-byte unsigned int
263 self.write(BININT1 + i[0])
264 else:
265 self.write(BININT2 + i[:2])
266 else:
267 self.write(BININT + i)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000268 return
Tim Peters44714002001-04-10 05:02:52 +0000269 # Text pickle, or int too big to fit in signed 4-byte format.
270 self.write(INT + `object` + '\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000271 dispatch[IntType] = save_int
272
273 def save_long(self, object):
274 self.write(LONG + `object` + '\n')
275 dispatch[LongType] = save_long
276
Guido van Rossumd3703791998-10-22 20:15:36 +0000277 def save_float(self, object, pack=struct.pack):
278 if self.bin:
279 self.write(BINFLOAT + pack('>d', object))
280 else:
281 self.write(FLOAT + `object` + '\n')
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000282 dispatch[FloatType] = save_float
283
284 def save_string(self, object):
285 d = id(object)
286 memo = self.memo
287
Tim Petersc32d8242001-04-10 02:48:53 +0000288 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000289 l = len(object)
290 s = mdumps(l)[1:]
Tim Petersc32d8242001-04-10 02:48:53 +0000291 if l < 256:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000292 self.write(SHORT_BINSTRING + s[0] + object)
293 else:
294 self.write(BINSTRING + s + object)
295 else:
296 self.write(STRING + `object` + '\n')
297
298 memo_len = len(memo)
299 self.write(self.put(memo_len))
300 memo[d] = (memo_len, object)
301 dispatch[StringType] = save_string
302
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000303 def save_unicode(self, object):
304 d = id(object)
305 memo = self.memo
306
Tim Petersc32d8242001-04-10 02:48:53 +0000307 if self.bin:
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000308 encoding = object.encode('utf-8')
309 l = len(encoding)
310 s = mdumps(l)[1:]
311 self.write(BINUNICODE + s + encoding)
312 else:
Guido van Rossumdbb718f2001-09-21 19:22:34 +0000313 object = object.replace("\\", "\\u005c")
314 object = object.replace("\n", "\\u000a")
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000315 self.write(UNICODE + object.encode('raw-unicode-escape') + '\n')
316
317 memo_len = len(memo)
318 self.write(self.put(memo_len))
319 memo[d] = (memo_len, object)
320 dispatch[UnicodeType] = save_unicode
321
Guido van Rossum31584cb2001-01-22 14:53:29 +0000322 if StringType == UnicodeType:
323 # This is true for Jython
324 def save_string(self, object):
325 d = id(object)
326 memo = self.memo
327 unicode = object.isunicode()
328
Tim Petersc32d8242001-04-10 02:48:53 +0000329 if self.bin:
Guido van Rossum31584cb2001-01-22 14:53:29 +0000330 if unicode:
331 object = object.encode("utf-8")
332 l = len(object)
333 s = mdumps(l)[1:]
Tim Petersc32d8242001-04-10 02:48:53 +0000334 if l < 256 and not unicode:
Guido van Rossum31584cb2001-01-22 14:53:29 +0000335 self.write(SHORT_BINSTRING + s[0] + object)
336 else:
337 if unicode:
338 self.write(BINUNICODE + s + object)
339 else:
340 self.write(BINSTRING + s + object)
341 else:
Tim Peters658cba62001-02-09 20:06:00 +0000342 if unicode:
Guido van Rossumdbb718f2001-09-21 19:22:34 +0000343 object = object.replace("\\", "\\u005c")
344 object = object.replace("\n", "\\u000a")
Guido van Rossum31584cb2001-01-22 14:53:29 +0000345 object = object.encode('raw-unicode-escape')
346 self.write(UNICODE + object + '\n')
347 else:
348 self.write(STRING + `object` + '\n')
349
350 memo_len = len(memo)
351 self.write(self.put(memo_len))
352 memo[d] = (memo_len, object)
353 dispatch[StringType] = save_string
Tim Peters658cba62001-02-09 20:06:00 +0000354
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000355 def save_tuple(self, object):
356
357 write = self.write
358 save = self.save
359 memo = self.memo
360
361 d = id(object)
362
363 write(MARK)
364
365 for element in object:
366 save(element)
367
Tim Petersc32d8242001-04-10 02:48:53 +0000368 if len(object) and memo.has_key(d):
369 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000370 write(POP_MARK + self.get(memo[d][0]))
371 return
Tim Peters2344fae2001-01-15 00:50:52 +0000372
Guido van Rossum599174f1998-03-31 16:30:28 +0000373 write(POP * (len(object) + 1) + self.get(memo[d][0]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000374 return
375
376 memo_len = len(memo)
377 self.write(TUPLE + self.put(memo_len))
378 memo[d] = (memo_len, object)
379 dispatch[TupleType] = save_tuple
380
381 def save_empty_tuple(self, object):
382 self.write(EMPTY_TUPLE)
383
384 def save_list(self, object):
385 d = id(object)
386
387 write = self.write
388 save = self.save
389 memo = self.memo
390
Tim Petersc32d8242001-04-10 02:48:53 +0000391 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000392 write(EMPTY_LIST)
393 else:
394 write(MARK + LIST)
395
396 memo_len = len(memo)
397 write(self.put(memo_len))
398 memo[d] = (memo_len, object)
399
400 using_appends = (self.bin and (len(object) > 1))
401
Tim Petersc32d8242001-04-10 02:48:53 +0000402 if using_appends:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000403 write(MARK)
404
405 for element in object:
406 save(element)
Tim Peters2344fae2001-01-15 00:50:52 +0000407
Tim Petersc32d8242001-04-10 02:48:53 +0000408 if not using_appends:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000409 write(APPEND)
410
Tim Petersc32d8242001-04-10 02:48:53 +0000411 if using_appends:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000412 write(APPENDS)
413 dispatch[ListType] = save_list
414
415 def save_dict(self, object):
416 d = id(object)
417
418 write = self.write
419 save = self.save
420 memo = self.memo
421
Tim Petersc32d8242001-04-10 02:48:53 +0000422 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000423 write(EMPTY_DICT)
424 else:
425 write(MARK + DICT)
426
427 memo_len = len(memo)
428 self.write(self.put(memo_len))
429 memo[d] = (memo_len, object)
430
431 using_setitems = (self.bin and (len(object) > 1))
432
Tim Petersc32d8242001-04-10 02:48:53 +0000433 if using_setitems:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000434 write(MARK)
435
436 items = object.items()
437 for key, value in items:
438 save(key)
439 save(value)
440
Tim Petersc32d8242001-04-10 02:48:53 +0000441 if not using_setitems:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000442 write(SETITEM)
443
Tim Petersc32d8242001-04-10 02:48:53 +0000444 if using_setitems:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000445 write(SETITEMS)
446
447 dispatch[DictionaryType] = save_dict
Jeremy Hylton2b9d0291998-05-27 22:38:22 +0000448 if not PyStringMap is None:
449 dispatch[PyStringMap] = save_dict
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000450
451 def save_inst(self, object):
452 d = id(object)
453 cls = object.__class__
454
455 memo = self.memo
456 write = self.write
457 save = self.save
458
459 if hasattr(object, '__getinitargs__'):
460 args = object.__getinitargs__()
461 len(args) # XXX Assert it's a sequence
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000462 _keep_alive(args, memo)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000463 else:
464 args = ()
465
466 write(MARK)
467
Tim Petersc32d8242001-04-10 02:48:53 +0000468 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000469 save(cls)
470
471 for arg in args:
472 save(arg)
473
474 memo_len = len(memo)
Tim Petersc32d8242001-04-10 02:48:53 +0000475 if self.bin:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000476 write(OBJ + self.put(memo_len))
477 else:
Guido van Rossum4fb5b281997-09-12 20:07:24 +0000478 write(INST + cls.__module__ + '\n' + cls.__name__ + '\n' +
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000479 self.put(memo_len))
480
481 memo[d] = (memo_len, object)
482
483 try:
484 getstate = object.__getstate__
485 except AttributeError:
486 stuff = object.__dict__
487 else:
488 stuff = getstate()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000489 _keep_alive(stuff, memo)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000490 save(stuff)
491 write(BUILD)
492 dispatch[InstanceType] = save_inst
493
494 def save_global(self, object, name = None):
495 write = self.write
496 memo = self.memo
497
Tim Petersc32d8242001-04-10 02:48:53 +0000498 if name is None:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000499 name = object.__name__
500
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000501 try:
502 module = object.__module__
503 except AttributeError:
504 module = whichmodule(object, name)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000505
Guido van Rossumb0a98e92001-08-17 18:49:52 +0000506 try:
507 __import__(module)
508 mod = sys.modules[module]
509 klass = getattr(mod, name)
510 except (ImportError, KeyError, AttributeError):
511 raise PicklingError(
512 "Can't pickle %r: it's not found as %s.%s" %
513 (object, module, name))
514 else:
515 if klass is not object:
516 raise PicklingError(
517 "Can't pickle %r: it's not the same object as %s.%s" %
518 (object, module, name))
519
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000520 memo_len = len(memo)
521 write(GLOBAL + module + '\n' + name + '\n' +
522 self.put(memo_len))
523 memo[id(object)] = (memo_len, object)
524 dispatch[ClassType] = save_global
525 dispatch[FunctionType] = save_global
526 dispatch[BuiltinFunctionType] = save_global
Tim Peters6d6c1a32001-08-02 04:15:00 +0000527 dispatch[TypeType] = save_global
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000528
Guido van Rossuma48061a1995-01-10 00:31:14 +0000529
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000530def _keep_alive(x, memo):
531 """Keeps a reference to the object x in the memo.
532
533 Because we remember objects by their id, we have
534 to assure that possibly temporary objects are kept
535 alive by referencing them.
536 We store a reference at the id of the memo, which should
537 normally not be used unless someone tries to deepcopy
538 the memo itself...
539 """
540 try:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000541 memo[id(memo)].append(x)
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000542 except KeyError:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000543 # aha, this is the first one :-)
544 memo[id(memo)]=[x]
Guido van Rossum5ed5c4c1997-09-03 00:23:54 +0000545
546
Guido van Rossuma48061a1995-01-10 00:31:14 +0000547classmap = {}
548
Guido van Rossum4fb5b281997-09-12 20:07:24 +0000549# This is no longer used to find classes, but still for functions
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000550def whichmodule(cls, clsname):
551 """Figure out the module in which a class occurs.
Guido van Rossuma48061a1995-01-10 00:31:14 +0000552
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000553 Search sys.modules for the module.
554 Cache in classmap.
555 Return a module name.
556 If the class cannot be found, return __main__.
557 """
558 if classmap.has_key(cls):
559 return classmap[cls]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000560
561 for name, module in sys.modules.items():
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000562 if name != '__main__' and \
563 hasattr(module, clsname) and \
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000564 getattr(module, clsname) is cls:
565 break
566 else:
567 name = '__main__'
568 classmap[cls] = name
569 return name
Guido van Rossuma48061a1995-01-10 00:31:14 +0000570
571
572class Unpickler:
573
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000574 def __init__(self, file):
575 self.readline = file.readline
576 self.read = file.read
577 self.memo = {}
Guido van Rossuma48061a1995-01-10 00:31:14 +0000578
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000579 def load(self):
580 self.mark = ['spam'] # Any new unique object
581 self.stack = []
582 self.append = self.stack.append
583 read = self.read
584 dispatch = self.dispatch
585 try:
586 while 1:
587 key = read(1)
588 dispatch[key](self)
Guido van Rossumff871742000-12-13 18:11:56 +0000589 except _Stop, stopinst:
590 return stopinst.value
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000591
592 def marker(self):
593 stack = self.stack
594 mark = self.mark
595 k = len(stack)-1
596 while stack[k] is not mark: k = k-1
597 return k
598
599 dispatch = {}
600
601 def load_eof(self):
602 raise EOFError
603 dispatch[''] = load_eof
604
605 def load_persid(self):
606 pid = self.readline()[:-1]
607 self.append(self.persistent_load(pid))
608 dispatch[PERSID] = load_persid
609
610 def load_binpersid(self):
611 stack = self.stack
Tim Peters2344fae2001-01-15 00:50:52 +0000612
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000613 pid = stack[-1]
614 del stack[-1]
615
616 self.append(self.persistent_load(pid))
617 dispatch[BINPERSID] = load_binpersid
618
619 def load_none(self):
620 self.append(None)
621 dispatch[NONE] = load_none
622
623 def load_int(self):
Tim Peters19ef62d2001-08-28 22:21:18 +0000624 data = self.readline()
625 try:
626 self.append(int(data))
627 except ValueError:
628 self.append(long(data))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000629 dispatch[INT] = load_int
630
631 def load_binint(self):
632 self.append(mloads('i' + self.read(4)))
633 dispatch[BININT] = load_binint
634
635 def load_binint1(self):
636 self.append(mloads('i' + self.read(1) + '\000\000\000'))
637 dispatch[BININT1] = load_binint1
638
639 def load_binint2(self):
640 self.append(mloads('i' + self.read(2) + '\000\000'))
641 dispatch[BININT2] = load_binint2
Tim Peters2344fae2001-01-15 00:50:52 +0000642
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000643 def load_long(self):
Guido van Rossumff871742000-12-13 18:11:56 +0000644 self.append(long(self.readline()[:-1], 0))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000645 dispatch[LONG] = load_long
646
647 def load_float(self):
Guido van Rossumff871742000-12-13 18:11:56 +0000648 self.append(float(self.readline()[:-1]))
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000649 dispatch[FLOAT] = load_float
650
Guido van Rossumd3703791998-10-22 20:15:36 +0000651 def load_binfloat(self, unpack=struct.unpack):
652 self.append(unpack('>d', self.read(8))[0])
653 dispatch[BINFLOAT] = load_binfloat
654
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000655 def load_string(self):
Jeremy Hyltonbe467e52000-09-15 15:14:51 +0000656 rep = self.readline()[:-1]
657 if not self._is_string_secure(rep):
658 raise ValueError, "insecure string pickle"
659 self.append(eval(rep,
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000660 {'__builtins__': {}})) # Let's be careful
661 dispatch[STRING] = load_string
662
Jeremy Hyltonbe467e52000-09-15 15:14:51 +0000663 def _is_string_secure(self, s):
664 """Return true if s contains a string that is safe to eval
665
666 The definition of secure string is based on the implementation
667 in cPickle. s is secure as long as it only contains a quoted
668 string and optional trailing whitespace.
669 """
670 q = s[0]
671 if q not in ("'", '"'):
672 return 0
673 # find the closing quote
674 offset = 1
675 i = None
676 while 1:
677 try:
678 i = s.index(q, offset)
679 except ValueError:
680 # if there is an error the first time, there is no
681 # close quote
682 if offset == 1:
683 return 0
684 if s[i-1] != '\\':
685 break
686 # check to see if this one is escaped
687 nslash = 0
688 j = i - 1
689 while j >= offset and s[j] == '\\':
690 j = j - 1
691 nslash = nslash + 1
692 if nslash % 2 == 0:
693 break
694 offset = i + 1
695 for c in s[i+1:]:
696 if ord(c) > 32:
697 return 0
698 return 1
699
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000700 def load_binstring(self):
701 len = mloads('i' + self.read(4))
702 self.append(self.read(len))
703 dispatch[BINSTRING] = load_binstring
704
Guido van Rossumb5f2f1b2000-03-10 23:20:09 +0000705 def load_unicode(self):
706 self.append(unicode(self.readline()[:-1],'raw-unicode-escape'))
707 dispatch[UNICODE] = load_unicode
708
709 def load_binunicode(self):
710 len = mloads('i' + self.read(4))
711 self.append(unicode(self.read(len),'utf-8'))
712 dispatch[BINUNICODE] = load_binunicode
713
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000714 def load_short_binstring(self):
715 len = mloads('i' + self.read(1) + '\000\000\000')
716 self.append(self.read(len))
717 dispatch[SHORT_BINSTRING] = load_short_binstring
718
719 def load_tuple(self):
720 k = self.marker()
721 self.stack[k:] = [tuple(self.stack[k+1:])]
722 dispatch[TUPLE] = load_tuple
723
724 def load_empty_tuple(self):
725 self.stack.append(())
726 dispatch[EMPTY_TUPLE] = load_empty_tuple
727
728 def load_empty_list(self):
729 self.stack.append([])
730 dispatch[EMPTY_LIST] = load_empty_list
731
732 def load_empty_dictionary(self):
733 self.stack.append({})
734 dispatch[EMPTY_DICT] = load_empty_dictionary
735
736 def load_list(self):
737 k = self.marker()
738 self.stack[k:] = [self.stack[k+1:]]
739 dispatch[LIST] = load_list
740
741 def load_dict(self):
742 k = self.marker()
743 d = {}
744 items = self.stack[k+1:]
745 for i in range(0, len(items), 2):
746 key = items[i]
747 value = items[i+1]
748 d[key] = value
749 self.stack[k:] = [d]
750 dispatch[DICT] = load_dict
751
752 def load_inst(self):
753 k = self.marker()
754 args = tuple(self.stack[k+1:])
755 del self.stack[k:]
756 module = self.readline()[:-1]
757 name = self.readline()[:-1]
758 klass = self.find_class(module, name)
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000759 instantiated = 0
760 if (not args and type(klass) is ClassType and
761 not hasattr(klass, "__getinitargs__")):
762 try:
763 value = _EmptyClass()
764 value.__class__ = klass
Guido van Rossumb19e2a31998-04-13 18:08:45 +0000765 instantiated = 1
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000766 except RuntimeError:
767 # In restricted execution, assignment to inst.__class__ is
768 # prohibited
769 pass
770 if not instantiated:
Guido van Rossum743d17e1998-09-15 20:25:57 +0000771 try:
772 value = apply(klass, args)
773 except TypeError, err:
774 raise TypeError, "in constructor for %s: %s" % (
775 klass.__name__, str(err)), sys.exc_info()[2]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000776 self.append(value)
777 dispatch[INST] = load_inst
778
779 def load_obj(self):
780 stack = self.stack
781 k = self.marker()
782 klass = stack[k + 1]
783 del stack[k + 1]
Tim Peters2344fae2001-01-15 00:50:52 +0000784 args = tuple(stack[k + 1:])
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000785 del stack[k:]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000786 instantiated = 0
787 if (not args and type(klass) is ClassType and
788 not hasattr(klass, "__getinitargs__")):
789 try:
790 value = _EmptyClass()
791 value.__class__ = klass
792 instantiated = 1
793 except RuntimeError:
794 # In restricted execution, assignment to inst.__class__ is
795 # prohibited
796 pass
797 if not instantiated:
798 value = apply(klass, args)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000799 self.append(value)
Tim Peters2344fae2001-01-15 00:50:52 +0000800 dispatch[OBJ] = load_obj
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000801
802 def load_global(self):
803 module = self.readline()[:-1]
804 name = self.readline()[:-1]
805 klass = self.find_class(module, name)
806 self.append(klass)
807 dispatch[GLOBAL] = load_global
808
809 def find_class(self, module, name):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000810 try:
Guido van Rossum397032a1999-03-25 21:58:59 +0000811 __import__(module)
812 mod = sys.modules[module]
813 klass = getattr(mod, name)
814 except (ImportError, KeyError, AttributeError):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000815 raise SystemError, \
816 "Failed to import class %s from module %s" % \
817 (name, module)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000818 return klass
819
820 def load_reduce(self):
821 stack = self.stack
822
823 callable = stack[-2]
824 arg_tup = stack[-1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000825 del stack[-2:]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000826
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000827 if type(callable) is not ClassType:
828 if not safe_constructors.has_key(callable):
829 try:
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000830 safe = callable.__safe_for_unpickling__
831 except AttributeError:
832 safe = None
Guido van Rossuma48061a1995-01-10 00:31:14 +0000833
Tim Petersc32d8242001-04-10 02:48:53 +0000834 if not safe:
Tim Peters2344fae2001-01-15 00:50:52 +0000835 raise UnpicklingError, "%s is not safe for " \
836 "unpickling" % callable
Guido van Rossuma48061a1995-01-10 00:31:14 +0000837
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000838 if arg_tup is None:
839 value = callable.__basicnew__()
840 else:
841 value = apply(callable, arg_tup)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000842 self.append(value)
843 dispatch[REDUCE] = load_reduce
Guido van Rossuma48061a1995-01-10 00:31:14 +0000844
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000845 def load_pop(self):
846 del self.stack[-1]
847 dispatch[POP] = load_pop
Guido van Rossum7b5430f1995-03-04 22:25:21 +0000848
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000849 def load_pop_mark(self):
850 k = self.marker()
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000851 del self.stack[k:]
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000852 dispatch[POP_MARK] = load_pop_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +0000853
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000854 def load_dup(self):
Guido van Rossumb1062fc1998-03-31 17:00:46 +0000855 self.append(self.stack[-1])
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000856 dispatch[DUP] = load_dup
Guido van Rossuma48061a1995-01-10 00:31:14 +0000857
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000858 def load_get(self):
859 self.append(self.memo[self.readline()[:-1]])
860 dispatch[GET] = load_get
Guido van Rossum78536471996-04-12 13:36:27 +0000861
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000862 def load_binget(self):
863 i = mloads('i' + self.read(1) + '\000\000\000')
864 self.append(self.memo[`i`])
865 dispatch[BINGET] = load_binget
Guido van Rossum78536471996-04-12 13:36:27 +0000866
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000867 def load_long_binget(self):
868 i = mloads('i' + self.read(4))
869 self.append(self.memo[`i`])
870 dispatch[LONG_BINGET] = load_long_binget
Guido van Rossum78536471996-04-12 13:36:27 +0000871
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000872 def load_put(self):
873 self.memo[self.readline()[:-1]] = self.stack[-1]
874 dispatch[PUT] = load_put
Guido van Rossuma48061a1995-01-10 00:31:14 +0000875
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000876 def load_binput(self):
877 i = mloads('i' + self.read(1) + '\000\000\000')
878 self.memo[`i`] = self.stack[-1]
879 dispatch[BINPUT] = load_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +0000880
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000881 def load_long_binput(self):
882 i = mloads('i' + self.read(4))
883 self.memo[`i`] = self.stack[-1]
884 dispatch[LONG_BINPUT] = load_long_binput
Guido van Rossuma48061a1995-01-10 00:31:14 +0000885
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000886 def load_append(self):
887 stack = self.stack
888 value = stack[-1]
889 del stack[-1]
890 list = stack[-1]
891 list.append(value)
892 dispatch[APPEND] = load_append
Guido van Rossuma48061a1995-01-10 00:31:14 +0000893
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000894 def load_appends(self):
895 stack = self.stack
896 mark = self.marker()
897 list = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000898 for i in range(mark + 1, len(stack)):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000899 list.append(stack[i])
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000900
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000901 del stack[mark:]
902 dispatch[APPENDS] = load_appends
Tim Peters2344fae2001-01-15 00:50:52 +0000903
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000904 def load_setitem(self):
905 stack = self.stack
906 value = stack[-1]
907 key = stack[-2]
908 del stack[-2:]
909 dict = stack[-1]
910 dict[key] = value
911 dispatch[SETITEM] = load_setitem
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000912
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000913 def load_setitems(self):
914 stack = self.stack
915 mark = self.marker()
916 dict = stack[mark - 1]
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000917 for i in range(mark + 1, len(stack), 2):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000918 dict[stack[i]] = stack[i + 1]
Guido van Rossuma48061a1995-01-10 00:31:14 +0000919
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000920 del stack[mark:]
921 dispatch[SETITEMS] = load_setitems
Guido van Rossuma48061a1995-01-10 00:31:14 +0000922
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000923 def load_build(self):
924 stack = self.stack
925 value = stack[-1]
926 del stack[-1]
927 inst = stack[-1]
928 try:
929 setstate = inst.__setstate__
930 except AttributeError:
Guido van Rossum45e2fbc1998-03-26 21:13:24 +0000931 try:
932 inst.__dict__.update(value)
933 except RuntimeError:
934 # XXX In restricted execution, the instance's __dict__ is not
935 # accessible. Use the old way of unpickling the instance
936 # variables. This is a semantic different when unpickling in
937 # restricted vs. unrestricted modes.
938 for k, v in value.items():
939 setattr(inst, k, v)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000940 else:
941 setstate(value)
942 dispatch[BUILD] = load_build
Guido van Rossuma48061a1995-01-10 00:31:14 +0000943
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000944 def load_mark(self):
945 self.append(self.mark)
946 dispatch[MARK] = load_mark
Guido van Rossuma48061a1995-01-10 00:31:14 +0000947
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000948 def load_stop(self):
949 value = self.stack[-1]
950 del self.stack[-1]
Guido van Rossumff871742000-12-13 18:11:56 +0000951 raise _Stop(value)
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000952 dispatch[STOP] = load_stop
Guido van Rossuma48061a1995-01-10 00:31:14 +0000953
Guido van Rossume467be61997-12-05 19:42:42 +0000954# Helper class for load_inst/load_obj
955
956class _EmptyClass:
957 pass
Guido van Rossuma48061a1995-01-10 00:31:14 +0000958
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000959# Shorthands
960
Jeremy Hyltonabe2c622001-10-15 21:29:28 +0000961try:
962 from cStringIO import StringIO
963except ImportError:
964 from StringIO import StringIO
Guido van Rossumc7c5e691996-07-22 22:26:07 +0000965
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000966def dump(object, file, bin = 0):
967 Pickler(file, bin).dump(object)
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000968
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000969def dumps(object, bin = 0):
970 file = StringIO()
971 Pickler(file, bin).dump(object)
972 return file.getvalue()
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000973
974def load(file):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000975 return Unpickler(file).load()
Guido van Rossum0c891ce1995-03-14 15:09:05 +0000976
977def loads(str):
Guido van Rossumb72cf2d1997-04-09 17:32:51 +0000978 file = StringIO(str)
979 return Unpickler(file).load()