blob: a8d414e713ec064a228d9ca7772321510f74b8e7 [file] [log] [blame]
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001#include "Python.h"
2#include "structmember.h"
3
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004PyDoc_STRVAR(pickle_module_doc,
5"Optimized C implementation for the Python pickle module.");
6
Larry Hastings61272b72014-01-07 12:41:53 -08007/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08008module _pickle
Larry Hastingsc2047262014-01-25 20:43:29 -08009class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
10class _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "&PicklerMemoProxyType"
11class _pickle.Unpickler "UnpicklerObject *" "&Unpickler_Type"
12class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "&UnpicklerMemoProxyType"
Larry Hastings61272b72014-01-07 12:41:53 -080013[clinic start generated code]*/
Serhiy Storchaka1009bf12015-04-03 23:53:51 +030014/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b3e113468a58e6c]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -080015
Alexandre Vassalottica2d6102008-06-12 18:26:05 +000016/* Bump this when new opcodes are added to the pickle protocol. */
17enum {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010018 HIGHEST_PROTOCOL = 4,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +000019 DEFAULT_PROTOCOL = 3
20};
21
Alexandre Vassalottica2d6102008-06-12 18:26:05 +000022/* Pickle opcodes. These must be kept updated with pickle.py.
23 Extensive docs are in pickletools.py. */
24enum opcode {
25 MARK = '(',
26 STOP = '.',
27 POP = '0',
28 POP_MARK = '1',
29 DUP = '2',
30 FLOAT = 'F',
31 INT = 'I',
32 BININT = 'J',
33 BININT1 = 'K',
34 LONG = 'L',
35 BININT2 = 'M',
36 NONE = 'N',
37 PERSID = 'P',
38 BINPERSID = 'Q',
39 REDUCE = 'R',
40 STRING = 'S',
41 BINSTRING = 'T',
42 SHORT_BINSTRING = 'U',
43 UNICODE = 'V',
44 BINUNICODE = 'X',
45 APPEND = 'a',
46 BUILD = 'b',
47 GLOBAL = 'c',
48 DICT = 'd',
49 EMPTY_DICT = '}',
50 APPENDS = 'e',
51 GET = 'g',
52 BINGET = 'h',
53 INST = 'i',
54 LONG_BINGET = 'j',
55 LIST = 'l',
56 EMPTY_LIST = ']',
57 OBJ = 'o',
58 PUT = 'p',
59 BINPUT = 'q',
60 LONG_BINPUT = 'r',
61 SETITEM = 's',
62 TUPLE = 't',
63 EMPTY_TUPLE = ')',
64 SETITEMS = 'u',
65 BINFLOAT = 'G',
66
67 /* Protocol 2. */
68 PROTO = '\x80',
69 NEWOBJ = '\x81',
70 EXT1 = '\x82',
71 EXT2 = '\x83',
72 EXT4 = '\x84',
73 TUPLE1 = '\x85',
74 TUPLE2 = '\x86',
75 TUPLE3 = '\x87',
76 NEWTRUE = '\x88',
77 NEWFALSE = '\x89',
78 LONG1 = '\x8a',
79 LONG4 = '\x8b',
80
81 /* Protocol 3 (Python 3.x) */
82 BINBYTES = 'B',
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +010083 SHORT_BINBYTES = 'C',
84
85 /* Protocol 4 */
86 SHORT_BINUNICODE = '\x8c',
87 BINUNICODE8 = '\x8d',
88 BINBYTES8 = '\x8e',
89 EMPTY_SET = '\x8f',
90 ADDITEMS = '\x90',
91 FROZENSET = '\x91',
92 NEWOBJ_EX = '\x92',
93 STACK_GLOBAL = '\x93',
94 MEMOIZE = '\x94',
95 FRAME = '\x95'
Alexandre Vassalottica2d6102008-06-12 18:26:05 +000096};
97
Alexandre Vassalottica2d6102008-06-12 18:26:05 +000098enum {
99 /* Keep in synch with pickle.Pickler._BATCHSIZE. This is how many elements
100 batch_list/dict() pumps out before doing APPENDS/SETITEMS. Nothing will
101 break if this gets out of synch with pickle.py, but it's unclear that would
102 help anything either. */
103 BATCHSIZE = 1000,
104
105 /* Nesting limit until Pickler, when running in "fast mode", starts
106 checking for self-referential data-structures. */
107 FAST_NESTING_LIMIT = 50,
108
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000109 /* Initial size of the write buffer of Pickler. */
110 WRITE_BUF_SIZE = 4096,
111
Antoine Pitrou04248a82010-10-12 20:51:21 +0000112 /* Prefetch size when unpickling (disabled on unpeekable streams) */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100113 PREFETCH = 8192 * 16,
114
115 FRAME_SIZE_TARGET = 64 * 1024,
116
117 FRAME_HEADER_SIZE = 9
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000118};
119
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800120/*************************************************************************/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000121
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800122/* State of the pickle module, per PEP 3121. */
123typedef struct {
124 /* Exception classes for pickle. */
125 PyObject *PickleError;
126 PyObject *PicklingError;
127 PyObject *UnpicklingError;
Larry Hastings61272b72014-01-07 12:41:53 -0800128
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800129 /* copyreg.dispatch_table, {type_object: pickling_function} */
130 PyObject *dispatch_table;
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000131
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800132 /* For the extension opcodes EXT1, EXT2 and EXT4. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000133
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800134 /* copyreg._extension_registry, {(module_name, function_name): code} */
135 PyObject *extension_registry;
136 /* copyreg._extension_cache, {code: object} */
137 PyObject *extension_cache;
138 /* copyreg._inverted_registry, {code: (module_name, function_name)} */
139 PyObject *inverted_registry;
140
141 /* Import mappings for compatibility with Python 2.x */
142
143 /* _compat_pickle.NAME_MAPPING,
144 {(oldmodule, oldname): (newmodule, newname)} */
145 PyObject *name_mapping_2to3;
146 /* _compat_pickle.IMPORT_MAPPING, {oldmodule: newmodule} */
147 PyObject *import_mapping_2to3;
148 /* Same, but with REVERSE_NAME_MAPPING / REVERSE_IMPORT_MAPPING */
149 PyObject *name_mapping_3to2;
150 PyObject *import_mapping_3to2;
151
152 /* codecs.encode, used for saving bytes in older protocols */
153 PyObject *codecs_encode;
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300154 /* builtins.getattr, used for saving nested names with protocol < 4 */
155 PyObject *getattr;
Serhiy Storchaka0d554d72015-10-10 22:42:18 +0300156 /* functools.partial, used for implementing __newobj_ex__ with protocols
157 2 and 3 */
158 PyObject *partial;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800159} PickleState;
160
161/* Forward declaration of the _pickle module definition. */
162static struct PyModuleDef _picklemodule;
163
164/* Given a module object, get its per-module state. */
165static PickleState *
166_Pickle_GetState(PyObject *module)
167{
168 return (PickleState *)PyModule_GetState(module);
169}
170
171/* Find the module instance imported in the currently running sub-interpreter
172 and get its state. */
173static PickleState *
174_Pickle_GetGlobalState(void)
175{
176 return _Pickle_GetState(PyState_FindModule(&_picklemodule));
177}
178
179/* Clear the given pickle module state. */
180static void
181_Pickle_ClearState(PickleState *st)
182{
183 Py_CLEAR(st->PickleError);
184 Py_CLEAR(st->PicklingError);
185 Py_CLEAR(st->UnpicklingError);
186 Py_CLEAR(st->dispatch_table);
187 Py_CLEAR(st->extension_registry);
188 Py_CLEAR(st->extension_cache);
189 Py_CLEAR(st->inverted_registry);
190 Py_CLEAR(st->name_mapping_2to3);
191 Py_CLEAR(st->import_mapping_2to3);
192 Py_CLEAR(st->name_mapping_3to2);
193 Py_CLEAR(st->import_mapping_3to2);
194 Py_CLEAR(st->codecs_encode);
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300195 Py_CLEAR(st->getattr);
Victor Stinner9ba97df2015-11-17 12:15:07 +0100196 Py_CLEAR(st->partial);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800197}
198
199/* Initialize the given pickle module state. */
200static int
201_Pickle_InitState(PickleState *st)
202{
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300203 PyObject *builtins;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800204 PyObject *copyreg = NULL;
205 PyObject *compat_pickle = NULL;
206 PyObject *codecs = NULL;
Serhiy Storchaka0d554d72015-10-10 22:42:18 +0300207 PyObject *functools = NULL;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800208
Serhiy Storchaka58e41342015-03-31 14:07:24 +0300209 builtins = PyEval_GetBuiltins();
210 if (builtins == NULL)
211 goto error;
212 st->getattr = PyDict_GetItemString(builtins, "getattr");
213 if (st->getattr == NULL)
214 goto error;
215 Py_INCREF(st->getattr);
216
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800217 copyreg = PyImport_ImportModule("copyreg");
218 if (!copyreg)
219 goto error;
220 st->dispatch_table = PyObject_GetAttrString(copyreg, "dispatch_table");
221 if (!st->dispatch_table)
222 goto error;
223 if (!PyDict_CheckExact(st->dispatch_table)) {
224 PyErr_Format(PyExc_RuntimeError,
225 "copyreg.dispatch_table should be a dict, not %.200s",
226 Py_TYPE(st->dispatch_table)->tp_name);
227 goto error;
228 }
229 st->extension_registry = \
230 PyObject_GetAttrString(copyreg, "_extension_registry");
231 if (!st->extension_registry)
232 goto error;
233 if (!PyDict_CheckExact(st->extension_registry)) {
234 PyErr_Format(PyExc_RuntimeError,
235 "copyreg._extension_registry should be a dict, "
236 "not %.200s", Py_TYPE(st->extension_registry)->tp_name);
237 goto error;
238 }
239 st->inverted_registry = \
240 PyObject_GetAttrString(copyreg, "_inverted_registry");
241 if (!st->inverted_registry)
242 goto error;
243 if (!PyDict_CheckExact(st->inverted_registry)) {
244 PyErr_Format(PyExc_RuntimeError,
245 "copyreg._inverted_registry should be a dict, "
246 "not %.200s", Py_TYPE(st->inverted_registry)->tp_name);
247 goto error;
248 }
249 st->extension_cache = PyObject_GetAttrString(copyreg, "_extension_cache");
250 if (!st->extension_cache)
251 goto error;
252 if (!PyDict_CheckExact(st->extension_cache)) {
253 PyErr_Format(PyExc_RuntimeError,
254 "copyreg._extension_cache should be a dict, "
255 "not %.200s", Py_TYPE(st->extension_cache)->tp_name);
256 goto error;
257 }
258 Py_CLEAR(copyreg);
259
260 /* Load the 2.x -> 3.x stdlib module mapping tables */
261 compat_pickle = PyImport_ImportModule("_compat_pickle");
262 if (!compat_pickle)
263 goto error;
264 st->name_mapping_2to3 = \
265 PyObject_GetAttrString(compat_pickle, "NAME_MAPPING");
266 if (!st->name_mapping_2to3)
267 goto error;
268 if (!PyDict_CheckExact(st->name_mapping_2to3)) {
269 PyErr_Format(PyExc_RuntimeError,
270 "_compat_pickle.NAME_MAPPING should be a dict, not %.200s",
271 Py_TYPE(st->name_mapping_2to3)->tp_name);
272 goto error;
273 }
274 st->import_mapping_2to3 = \
275 PyObject_GetAttrString(compat_pickle, "IMPORT_MAPPING");
276 if (!st->import_mapping_2to3)
277 goto error;
278 if (!PyDict_CheckExact(st->import_mapping_2to3)) {
279 PyErr_Format(PyExc_RuntimeError,
280 "_compat_pickle.IMPORT_MAPPING should be a dict, "
281 "not %.200s", Py_TYPE(st->import_mapping_2to3)->tp_name);
282 goto error;
283 }
284 /* ... and the 3.x -> 2.x mapping tables */
285 st->name_mapping_3to2 = \
286 PyObject_GetAttrString(compat_pickle, "REVERSE_NAME_MAPPING");
287 if (!st->name_mapping_3to2)
288 goto error;
289 if (!PyDict_CheckExact(st->name_mapping_3to2)) {
290 PyErr_Format(PyExc_RuntimeError,
291 "_compat_pickle.REVERSE_NAME_MAPPING should be a dict, "
292 "not %.200s", Py_TYPE(st->name_mapping_3to2)->tp_name);
293 goto error;
294 }
295 st->import_mapping_3to2 = \
296 PyObject_GetAttrString(compat_pickle, "REVERSE_IMPORT_MAPPING");
297 if (!st->import_mapping_3to2)
298 goto error;
299 if (!PyDict_CheckExact(st->import_mapping_3to2)) {
300 PyErr_Format(PyExc_RuntimeError,
301 "_compat_pickle.REVERSE_IMPORT_MAPPING should be a dict, "
302 "not %.200s", Py_TYPE(st->import_mapping_3to2)->tp_name);
303 goto error;
304 }
305 Py_CLEAR(compat_pickle);
306
307 codecs = PyImport_ImportModule("codecs");
308 if (codecs == NULL)
309 goto error;
310 st->codecs_encode = PyObject_GetAttrString(codecs, "encode");
311 if (st->codecs_encode == NULL) {
312 goto error;
313 }
314 if (!PyCallable_Check(st->codecs_encode)) {
315 PyErr_Format(PyExc_RuntimeError,
316 "codecs.encode should be a callable, not %.200s",
317 Py_TYPE(st->codecs_encode)->tp_name);
318 goto error;
319 }
320 Py_CLEAR(codecs);
321
Serhiy Storchaka0d554d72015-10-10 22:42:18 +0300322 functools = PyImport_ImportModule("functools");
323 if (!functools)
324 goto error;
325 st->partial = PyObject_GetAttrString(functools, "partial");
326 if (!st->partial)
327 goto error;
328 Py_CLEAR(functools);
329
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800330 return 0;
331
332 error:
333 Py_CLEAR(copyreg);
334 Py_CLEAR(compat_pickle);
335 Py_CLEAR(codecs);
Serhiy Storchaka0d554d72015-10-10 22:42:18 +0300336 Py_CLEAR(functools);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800337 _Pickle_ClearState(st);
338 return -1;
339}
340
341/* Helper for calling a function with a single argument quickly.
342
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800343 This function steals the reference of the given argument. */
344static PyObject *
345_Pickle_FastCall(PyObject *func, PyObject *obj)
346{
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800347 PyObject *result;
348
Victor Stinner559bb6a2016-08-22 22:48:54 +0200349 result = _PyObject_CallArg1(func, obj);
Victor Stinner75210692016-08-19 18:59:15 +0200350 Py_DECREF(obj);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -0800351 return result;
352}
353
354/*************************************************************************/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000355
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000356/* Internal data type used as the unpickling stack. */
357typedef struct {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000358 PyObject_VAR_HEAD
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000359 PyObject **data;
Serhiy Storchaka59fb6342015-12-06 22:01:35 +0200360 int mark_set; /* is MARK set? */
361 Py_ssize_t fence; /* position of top MARK or 0 */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000362 Py_ssize_t allocated; /* number of slots in data allocated */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000363} Pdata;
364
365static void
366Pdata_dealloc(Pdata *self)
367{
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200368 Py_ssize_t i = Py_SIZE(self);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000369 while (--i >= 0) {
370 Py_DECREF(self->data[i]);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000371 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000372 PyMem_FREE(self->data);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000373 PyObject_Del(self);
374}
375
376static PyTypeObject Pdata_Type = {
377 PyVarObject_HEAD_INIT(NULL, 0)
378 "_pickle.Pdata", /*tp_name*/
379 sizeof(Pdata), /*tp_basicsize*/
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +0200380 sizeof(PyObject *), /*tp_itemsize*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000381 (destructor)Pdata_dealloc, /*tp_dealloc*/
382};
383
384static PyObject *
385Pdata_New(void)
386{
387 Pdata *self;
388
389 if (!(self = PyObject_New(Pdata, &Pdata_Type)))
390 return NULL;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000391 Py_SIZE(self) = 0;
Serhiy Storchaka59fb6342015-12-06 22:01:35 +0200392 self->mark_set = 0;
393 self->fence = 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000394 self->allocated = 8;
395 self->data = PyMem_MALLOC(self->allocated * sizeof(PyObject *));
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000396 if (self->data)
397 return (PyObject *)self;
398 Py_DECREF(self);
399 return PyErr_NoMemory();
400}
401
402
403/* Retain only the initial clearto items. If clearto >= the current
404 * number of items, this is a (non-erroneous) NOP.
405 */
406static int
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200407Pdata_clear(Pdata *self, Py_ssize_t clearto)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000408{
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200409 Py_ssize_t i = Py_SIZE(self);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000410
Serhiy Storchaka59fb6342015-12-06 22:01:35 +0200411 assert(clearto >= self->fence);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000412 if (clearto >= i)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000413 return 0;
414
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000415 while (--i >= clearto) {
416 Py_CLEAR(self->data[i]);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000417 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000418 Py_SIZE(self) = clearto;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000419 return 0;
420}
421
422static int
423Pdata_grow(Pdata *self)
424{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000425 PyObject **data = self->data;
Victor Stinnerf13c46c2014-08-17 21:05:55 +0200426 size_t allocated = (size_t)self->allocated;
427 size_t new_allocated;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000428
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000429 new_allocated = (allocated >> 3) + 6;
430 /* check for integer overflow */
Victor Stinnerf13c46c2014-08-17 21:05:55 +0200431 if (new_allocated > (size_t)PY_SSIZE_T_MAX - allocated)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000432 goto nomemory;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000433 new_allocated += allocated;
Benjamin Peterson59b08c12015-06-27 13:41:33 -0500434 PyMem_RESIZE(data, PyObject *, new_allocated);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000435 if (data == NULL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000436 goto nomemory;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000437
438 self->data = data;
Victor Stinnerf13c46c2014-08-17 21:05:55 +0200439 self->allocated = (Py_ssize_t)new_allocated;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000440 return 0;
441
442 nomemory:
443 PyErr_NoMemory();
444 return -1;
445}
446
Serhiy Storchaka59fb6342015-12-06 22:01:35 +0200447static int
448Pdata_stack_underflow(Pdata *self)
449{
450 PickleState *st = _Pickle_GetGlobalState();
451 PyErr_SetString(st->UnpicklingError,
452 self->mark_set ?
453 "unexpected MARK found" :
454 "unpickling stack underflow");
455 return -1;
456}
457
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000458/* D is a Pdata*. Pop the topmost element and store it into V, which
459 * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
460 * is raised and V is set to NULL.
461 */
462static PyObject *
463Pdata_pop(Pdata *self)
464{
Serhiy Storchaka59fb6342015-12-06 22:01:35 +0200465 if (Py_SIZE(self) <= self->fence) {
466 Pdata_stack_underflow(self);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000467 return NULL;
468 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000469 return self->data[--Py_SIZE(self)];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000470}
471#define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0)
472
473static int
474Pdata_push(Pdata *self, PyObject *obj)
475{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000476 if (Py_SIZE(self) == self->allocated && Pdata_grow(self) < 0) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000477 return -1;
478 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000479 self->data[Py_SIZE(self)++] = obj;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000480 return 0;
481}
482
483/* Push an object on stack, transferring its ownership to the stack. */
484#define PDATA_PUSH(D, O, ER) do { \
485 if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
486
487/* Push an object on stack, adding a new reference to the object. */
488#define PDATA_APPEND(D, O, ER) do { \
489 Py_INCREF((O)); \
490 if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
491
492static PyObject *
493Pdata_poptuple(Pdata *self, Py_ssize_t start)
494{
495 PyObject *tuple;
496 Py_ssize_t len, i, j;
497
Serhiy Storchaka59fb6342015-12-06 22:01:35 +0200498 if (start < self->fence) {
499 Pdata_stack_underflow(self);
500 return NULL;
501 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000502 len = Py_SIZE(self) - start;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000503 tuple = PyTuple_New(len);
504 if (tuple == NULL)
505 return NULL;
506 for (i = start, j = 0; j < len; i++, j++)
507 PyTuple_SET_ITEM(tuple, j, self->data[i]);
508
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000509 Py_SIZE(self) = start;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000510 return tuple;
511}
512
513static PyObject *
514Pdata_poplist(Pdata *self, Py_ssize_t start)
515{
516 PyObject *list;
517 Py_ssize_t len, i, j;
518
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000519 len = Py_SIZE(self) - start;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000520 list = PyList_New(len);
521 if (list == NULL)
522 return NULL;
523 for (i = start, j = 0; j < len; i++, j++)
524 PyList_SET_ITEM(list, j, self->data[i]);
525
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000526 Py_SIZE(self) = start;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000527 return list;
528}
529
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000530typedef struct {
531 PyObject *me_key;
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200532 Py_ssize_t me_value;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000533} PyMemoEntry;
534
535typedef struct {
536 Py_ssize_t mt_mask;
537 Py_ssize_t mt_used;
538 Py_ssize_t mt_allocated;
539 PyMemoEntry *mt_table;
540} PyMemoTable;
541
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000542typedef struct PicklerObject {
543 PyObject_HEAD
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000544 PyMemoTable *memo; /* Memo table, keep track of the seen
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000545 objects to support self-referential objects
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000546 pickling. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000547 PyObject *pers_func; /* persistent_id() method, can be NULL */
Antoine Pitrou8d3c2902012-03-04 18:31:48 +0100548 PyObject *dispatch_table; /* private dispatch_table, can be NULL */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000549
550 PyObject *write; /* write() method of the output stream. */
551 PyObject *output_buffer; /* Write into a local bytearray buffer before
552 flushing to the stream. */
553 Py_ssize_t output_len; /* Length of output_buffer. */
554 Py_ssize_t max_output_len; /* Allocation size of output_buffer. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000555 int proto; /* Pickle protocol number, >= 0 */
556 int bin; /* Boolean, true if proto > 0 */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100557 int framing; /* True when framing is enabled, proto >= 4 */
558 Py_ssize_t frame_start; /* Position in output_buffer where the
Martin Pantera90a4a92016-05-30 04:04:50 +0000559 current frame begins. -1 if there
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100560 is no frame currently open. */
561
562 Py_ssize_t buf_size; /* Size of the current buffered pickle data */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000563 int fast; /* Enable fast mode if set to a true value.
564 The fast mode disable the usage of memo,
565 therefore speeding the pickling process by
566 not generating superfluous PUT opcodes. It
567 should not be used if with self-referential
568 objects. */
569 int fast_nesting;
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000570 int fix_imports; /* Indicate whether Pickler should fix
571 the name of globals for Python 2.x. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000572 PyObject *fast_memo;
573} PicklerObject;
574
575typedef struct UnpicklerObject {
576 PyObject_HEAD
577 Pdata *stack; /* Pickle data stack, store unpickled objects. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000578
579 /* The unpickler memo is just an array of PyObject *s. Using a dict
580 is unnecessary, since the keys are contiguous ints. */
581 PyObject **memo;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100582 Py_ssize_t memo_size; /* Capacity of the memo array */
583 Py_ssize_t memo_len; /* Number of objects in the memo */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000584
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000585 PyObject *pers_func; /* persistent_load() method, can be NULL. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000586
587 Py_buffer buffer;
588 char *input_buffer;
589 char *input_line;
590 Py_ssize_t input_len;
591 Py_ssize_t next_read_idx;
Antoine Pitrou04248a82010-10-12 20:51:21 +0000592 Py_ssize_t prefetched_idx; /* index of first prefetched byte */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100593
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000594 PyObject *read; /* read() method of the input stream. */
595 PyObject *readline; /* readline() method of the input stream. */
Antoine Pitrou04248a82010-10-12 20:51:21 +0000596 PyObject *peek; /* peek() method of the input stream, or NULL */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000597
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000598 char *encoding; /* Name of the encoding to be used for
599 decoding strings pickled using Python
600 2.x. The default value is "ASCII" */
601 char *errors; /* Name of errors handling scheme to used when
602 decoding strings. The default value is
603 "strict". */
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -0500604 Py_ssize_t *marks; /* Mark stack, used for unpickling container
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000605 objects. */
606 Py_ssize_t num_marks; /* Number of marks in the mark stack. */
607 Py_ssize_t marks_size; /* Current allocated size of the mark stack. */
Antoine Pitroud9dfaa92009-06-04 20:32:06 +0000608 int proto; /* Protocol of the pickle loaded. */
609 int fix_imports; /* Indicate whether Unpickler should fix
610 the name of globals pickled by Python 2.x. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000611} UnpicklerObject;
612
Serhiy Storchaka3c1f0f12014-01-27 10:34:22 +0200613typedef struct {
614 PyObject_HEAD
615 PicklerObject *pickler; /* Pickler whose memo table we're proxying. */
616} PicklerMemoProxyObject;
617
618typedef struct {
619 PyObject_HEAD
620 UnpicklerObject *unpickler;
621} UnpicklerMemoProxyObject;
622
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000623/* Forward declarations */
624static int save(PicklerObject *, PyObject *, int);
625static int save_reduce(PicklerObject *, PyObject *, PyObject *);
626static PyTypeObject Pickler_Type;
627static PyTypeObject Unpickler_Type;
628
Serhiy Storchaka3c1f0f12014-01-27 10:34:22 +0200629#include "clinic/_pickle.c.h"
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000630
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000631/*************************************************************************
Serhiy Storchaka95949422013-08-27 19:40:23 +0300632 A custom hashtable mapping void* to Python ints. This is used by the pickler
633 for memoization. Using a custom hashtable rather than PyDict allows us to skip
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000634 a bunch of unnecessary object creation. This makes a huge performance
635 difference. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000636
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000637#define MT_MINSIZE 8
638#define PERTURB_SHIFT 5
639
640
641static PyMemoTable *
642PyMemoTable_New(void)
643{
644 PyMemoTable *memo = PyMem_MALLOC(sizeof(PyMemoTable));
645 if (memo == NULL) {
646 PyErr_NoMemory();
647 return NULL;
648 }
649
650 memo->mt_used = 0;
651 memo->mt_allocated = MT_MINSIZE;
652 memo->mt_mask = MT_MINSIZE - 1;
653 memo->mt_table = PyMem_MALLOC(MT_MINSIZE * sizeof(PyMemoEntry));
654 if (memo->mt_table == NULL) {
655 PyMem_FREE(memo);
656 PyErr_NoMemory();
657 return NULL;
658 }
659 memset(memo->mt_table, 0, MT_MINSIZE * sizeof(PyMemoEntry));
660
661 return memo;
662}
663
664static PyMemoTable *
665PyMemoTable_Copy(PyMemoTable *self)
666{
667 Py_ssize_t i;
668 PyMemoTable *new = PyMemoTable_New();
669 if (new == NULL)
670 return NULL;
671
672 new->mt_used = self->mt_used;
673 new->mt_allocated = self->mt_allocated;
674 new->mt_mask = self->mt_mask;
675 /* The table we get from _New() is probably smaller than we wanted.
676 Free it and allocate one that's the right size. */
677 PyMem_FREE(new->mt_table);
Benjamin Peterson59b08c12015-06-27 13:41:33 -0500678 new->mt_table = PyMem_NEW(PyMemoEntry, self->mt_allocated);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000679 if (new->mt_table == NULL) {
680 PyMem_FREE(new);
Victor Stinner42024562013-07-12 00:53:57 +0200681 PyErr_NoMemory();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000682 return NULL;
683 }
684 for (i = 0; i < self->mt_allocated; i++) {
685 Py_XINCREF(self->mt_table[i].me_key);
686 }
687 memcpy(new->mt_table, self->mt_table,
688 sizeof(PyMemoEntry) * self->mt_allocated);
689
690 return new;
691}
692
693static Py_ssize_t
694PyMemoTable_Size(PyMemoTable *self)
695{
696 return self->mt_used;
697}
698
699static int
700PyMemoTable_Clear(PyMemoTable *self)
701{
702 Py_ssize_t i = self->mt_allocated;
703
704 while (--i >= 0) {
705 Py_XDECREF(self->mt_table[i].me_key);
706 }
707 self->mt_used = 0;
708 memset(self->mt_table, 0, self->mt_allocated * sizeof(PyMemoEntry));
709 return 0;
710}
711
712static void
713PyMemoTable_Del(PyMemoTable *self)
714{
715 if (self == NULL)
716 return;
717 PyMemoTable_Clear(self);
718
719 PyMem_FREE(self->mt_table);
720 PyMem_FREE(self);
721}
722
723/* Since entries cannot be deleted from this hashtable, _PyMemoTable_Lookup()
724 can be considerably simpler than dictobject.c's lookdict(). */
725static PyMemoEntry *
726_PyMemoTable_Lookup(PyMemoTable *self, PyObject *key)
727{
728 size_t i;
729 size_t perturb;
730 size_t mask = (size_t)self->mt_mask;
731 PyMemoEntry *table = self->mt_table;
732 PyMemoEntry *entry;
Benjamin Peterson8f67d082010-10-17 20:54:53 +0000733 Py_hash_t hash = (Py_hash_t)key >> 3;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000734
735 i = hash & mask;
736 entry = &table[i];
737 if (entry->me_key == NULL || entry->me_key == key)
738 return entry;
739
740 for (perturb = hash; ; perturb >>= PERTURB_SHIFT) {
741 i = (i << 2) + i + perturb + 1;
742 entry = &table[i & mask];
743 if (entry->me_key == NULL || entry->me_key == key)
744 return entry;
745 }
746 assert(0); /* Never reached */
747 return NULL;
748}
749
750/* Returns -1 on failure, 0 on success. */
751static int
752_PyMemoTable_ResizeTable(PyMemoTable *self, Py_ssize_t min_size)
753{
754 PyMemoEntry *oldtable = NULL;
755 PyMemoEntry *oldentry, *newentry;
756 Py_ssize_t new_size = MT_MINSIZE;
757 Py_ssize_t to_process;
758
759 assert(min_size > 0);
760
761 /* Find the smallest valid table size >= min_size. */
762 while (new_size < min_size && new_size > 0)
763 new_size <<= 1;
764 if (new_size <= 0) {
765 PyErr_NoMemory();
766 return -1;
767 }
768 /* new_size needs to be a power of two. */
769 assert((new_size & (new_size - 1)) == 0);
770
771 /* Allocate new table. */
772 oldtable = self->mt_table;
Benjamin Peterson59b08c12015-06-27 13:41:33 -0500773 self->mt_table = PyMem_NEW(PyMemoEntry, new_size);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000774 if (self->mt_table == NULL) {
Victor Stinner8ca72e22013-07-12 00:53:26 +0200775 self->mt_table = oldtable;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000776 PyErr_NoMemory();
777 return -1;
778 }
779 self->mt_allocated = new_size;
780 self->mt_mask = new_size - 1;
781 memset(self->mt_table, 0, sizeof(PyMemoEntry) * new_size);
782
783 /* Copy entries from the old table. */
784 to_process = self->mt_used;
785 for (oldentry = oldtable; to_process > 0; oldentry++) {
786 if (oldentry->me_key != NULL) {
787 to_process--;
788 /* newentry is a pointer to a chunk of the new
789 mt_table, so we're setting the key:value pair
790 in-place. */
791 newentry = _PyMemoTable_Lookup(self, oldentry->me_key);
792 newentry->me_key = oldentry->me_key;
793 newentry->me_value = oldentry->me_value;
794 }
795 }
796
797 /* Deallocate the old table. */
798 PyMem_FREE(oldtable);
799 return 0;
800}
801
802/* Returns NULL on failure, a pointer to the value otherwise. */
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200803static Py_ssize_t *
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000804PyMemoTable_Get(PyMemoTable *self, PyObject *key)
805{
806 PyMemoEntry *entry = _PyMemoTable_Lookup(self, key);
807 if (entry->me_key == NULL)
808 return NULL;
809 return &entry->me_value;
810}
811
812/* Returns -1 on failure, 0 on success. */
813static int
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200814PyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000815{
816 PyMemoEntry *entry;
817
818 assert(key != NULL);
819
820 entry = _PyMemoTable_Lookup(self, key);
821 if (entry->me_key != NULL) {
822 entry->me_value = value;
823 return 0;
824 }
825 Py_INCREF(key);
826 entry->me_key = key;
827 entry->me_value = value;
828 self->mt_used++;
829
830 /* If we added a key, we can safely resize. Otherwise just return!
831 * If used >= 2/3 size, adjust size. Normally, this quaduples the size.
832 *
833 * Quadrupling the size improves average table sparseness
834 * (reducing collisions) at the cost of some memory. It also halves
835 * the number of expensive resize operations in a growing memo table.
836 *
837 * Very large memo tables (over 50K items) use doubling instead.
838 * This may help applications with severe memory constraints.
839 */
840 if (!(self->mt_used * 3 >= (self->mt_mask + 1) * 2))
841 return 0;
842 return _PyMemoTable_ResizeTable(self,
843 (self->mt_used > 50000 ? 2 : 4) * self->mt_used);
844}
845
846#undef MT_MINSIZE
847#undef PERTURB_SHIFT
848
849/*************************************************************************/
850
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000851
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000852static int
853_Pickler_ClearBuffer(PicklerObject *self)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000854{
Serhiy Storchaka48842712016-04-06 09:45:48 +0300855 Py_XSETREF(self->output_buffer,
Serhiy Storchaka4a1e70f2015-12-27 12:36:18 +0200856 PyBytes_FromStringAndSize(NULL, self->max_output_len));
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000857 if (self->output_buffer == NULL)
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +0000858 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000859 self->output_len = 0;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100860 self->frame_start = -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000861 return 0;
862}
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +0000863
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100864static void
Antoine Pitrou8f2ee6e2013-11-23 21:05:08 +0100865_write_size64(char *out, size_t value)
866{
Victor Stinnerf13c46c2014-08-17 21:05:55 +0200867 size_t i;
Alexandre Vassalotti1048fb52013-11-25 11:35:46 -0800868
Serhiy Storchakafad85aa2015-11-07 15:42:38 +0200869 Py_BUILD_ASSERT(sizeof(size_t) <= 8);
Alexandre Vassalotti1048fb52013-11-25 11:35:46 -0800870
871 for (i = 0; i < sizeof(size_t); i++) {
872 out[i] = (unsigned char)((value >> (8 * i)) & 0xff);
873 }
874 for (i = sizeof(size_t); i < 8; i++) {
875 out[i] = 0;
Alexandre Vassalottided929b2013-11-24 22:41:13 -0800876 }
Antoine Pitrou8f2ee6e2013-11-23 21:05:08 +0100877}
878
879static void
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100880_Pickler_WriteFrameHeader(PicklerObject *self, char *qdata, size_t frame_len)
881{
Antoine Pitrou8f2ee6e2013-11-23 21:05:08 +0100882 qdata[0] = FRAME;
883 _write_size64(qdata + 1, frame_len);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100884}
885
886static int
887_Pickler_CommitFrame(PicklerObject *self)
888{
889 size_t frame_len;
890 char *qdata;
891
892 if (!self->framing || self->frame_start == -1)
893 return 0;
894 frame_len = self->output_len - self->frame_start - FRAME_HEADER_SIZE;
895 qdata = PyBytes_AS_STRING(self->output_buffer) + self->frame_start;
896 _Pickler_WriteFrameHeader(self, qdata, frame_len);
897 self->frame_start = -1;
898 return 0;
899}
900
901static int
902_Pickler_OpcodeBoundary(PicklerObject *self)
903{
904 Py_ssize_t frame_len;
905
906 if (!self->framing || self->frame_start == -1)
907 return 0;
908 frame_len = self->output_len - self->frame_start - FRAME_HEADER_SIZE;
909 if (frame_len >= FRAME_SIZE_TARGET)
910 return _Pickler_CommitFrame(self);
911 else
912 return 0;
913}
914
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000915static PyObject *
916_Pickler_GetString(PicklerObject *self)
917{
918 PyObject *output_buffer = self->output_buffer;
919
920 assert(self->output_buffer != NULL);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100921
922 if (_Pickler_CommitFrame(self))
923 return NULL;
924
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000925 self->output_buffer = NULL;
926 /* Resize down to exact size */
927 if (_PyBytes_Resize(&output_buffer, self->output_len) < 0)
928 return NULL;
929 return output_buffer;
930}
931
932static int
933_Pickler_FlushToFile(PicklerObject *self)
934{
935 PyObject *output, *result;
936
937 assert(self->write != NULL);
938
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100939 /* This will commit the frame first */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000940 output = _Pickler_GetString(self);
941 if (output == NULL)
942 return -1;
943
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -0800944 result = _Pickle_FastCall(self->write, output);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000945 Py_XDECREF(result);
946 return (result == NULL) ? -1 : 0;
947}
948
Antoine Pitrou82be19f2011-08-29 23:09:33 +0200949static Py_ssize_t
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100950_Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000951{
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100952 Py_ssize_t i, n, required;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000953 char *buffer;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100954 int need_new_frame;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000955
956 assert(s != NULL);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100957 need_new_frame = (self->framing && self->frame_start == -1);
958
959 if (need_new_frame)
960 n = data_len + FRAME_HEADER_SIZE;
961 else
962 n = data_len;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000963
964 required = self->output_len + n;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100965 if (required > self->max_output_len) {
966 /* Make place in buffer for the pickle chunk */
967 if (self->output_len >= PY_SSIZE_T_MAX / 2 - n) {
968 PyErr_NoMemory();
969 return -1;
970 }
971 self->max_output_len = (self->output_len + n) / 2 * 3;
972 if (_PyBytes_Resize(&self->output_buffer, self->max_output_len) < 0)
973 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000974 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000975 buffer = PyBytes_AS_STRING(self->output_buffer);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100976 if (need_new_frame) {
977 /* Setup new frame */
978 Py_ssize_t frame_start = self->output_len;
979 self->frame_start = frame_start;
980 for (i = 0; i < FRAME_HEADER_SIZE; i++) {
981 /* Write an invalid value, for debugging */
982 buffer[frame_start + i] = 0xFE;
983 }
984 self->output_len += FRAME_HEADER_SIZE;
985 }
986 if (data_len < 8) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000987 /* This is faster than memcpy when the string is short. */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100988 for (i = 0; i < data_len; i++) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000989 buffer[self->output_len + i] = s[i];
990 }
991 }
992 else {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100993 memcpy(buffer + self->output_len, s, data_len);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000994 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +0100995 self->output_len += data_len;
996 return data_len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +0000997}
998
Antoine Pitrouea99c5c2010-09-09 18:33:21 +0000999static PicklerObject *
1000_Pickler_New(void)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001001{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001002 PicklerObject *self;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001003
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001004 self = PyObject_GC_New(PicklerObject, &Pickler_Type);
1005 if (self == NULL)
1006 return NULL;
1007
1008 self->pers_func = NULL;
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01001009 self->dispatch_table = NULL;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001010 self->write = NULL;
1011 self->proto = 0;
1012 self->bin = 0;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001013 self->framing = 0;
1014 self->frame_start = -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001015 self->fast = 0;
1016 self->fast_nesting = 0;
1017 self->fix_imports = 0;
1018 self->fast_memo = NULL;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001019 self->max_output_len = WRITE_BUF_SIZE;
1020 self->output_len = 0;
Victor Stinner68c8ea22013-07-11 22:56:25 +02001021
1022 self->memo = PyMemoTable_New();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001023 self->output_buffer = PyBytes_FromStringAndSize(NULL,
1024 self->max_output_len);
Victor Stinner68c8ea22013-07-11 22:56:25 +02001025
1026 if (self->memo == NULL || self->output_buffer == NULL) {
Victor Stinnerc31df042013-07-12 00:08:59 +02001027 Py_DECREF(self);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001028 return NULL;
1029 }
1030 return self;
1031}
1032
1033static int
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001034_Pickler_SetProtocol(PicklerObject *self, PyObject *protocol, int fix_imports)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001035{
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001036 long proto;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001037
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001038 if (protocol == NULL || protocol == Py_None) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001039 proto = DEFAULT_PROTOCOL;
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001040 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001041 else {
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001042 proto = PyLong_AsLong(protocol);
1043 if (proto < 0) {
1044 if (proto == -1 && PyErr_Occurred())
1045 return -1;
1046 proto = HIGHEST_PROTOCOL;
1047 }
1048 else if (proto > HIGHEST_PROTOCOL) {
1049 PyErr_Format(PyExc_ValueError, "pickle protocol must be <= %d",
1050 HIGHEST_PROTOCOL);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001051 return -1;
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001052 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001053 }
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08001054 self->proto = (int)proto;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001055 self->bin = proto > 0;
1056 self->fix_imports = fix_imports && proto < 3;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001057 return 0;
1058}
1059
1060/* Returns -1 (with an exception set) on failure, 0 on success. This may
1061 be called once on a freshly created Pickler. */
1062static int
1063_Pickler_SetOutputStream(PicklerObject *self, PyObject *file)
1064{
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02001065 _Py_IDENTIFIER(write);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001066 assert(file != NULL);
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02001067 self->write = _PyObject_GetAttrId(file, &PyId_write);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001068 if (self->write == NULL) {
1069 if (PyErr_ExceptionMatches(PyExc_AttributeError))
1070 PyErr_SetString(PyExc_TypeError,
1071 "file must have a 'write' attribute");
1072 return -1;
1073 }
1074
1075 return 0;
1076}
1077
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001078/* Returns the size of the input on success, -1 on failure. This takes its
1079 own reference to `input`. */
1080static Py_ssize_t
1081_Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input)
1082{
1083 if (self->buffer.buf != NULL)
1084 PyBuffer_Release(&self->buffer);
1085 if (PyObject_GetBuffer(input, &self->buffer, PyBUF_CONTIG_RO) < 0)
1086 return -1;
1087 self->input_buffer = self->buffer.buf;
1088 self->input_len = self->buffer.len;
1089 self->next_read_idx = 0;
Antoine Pitrou04248a82010-10-12 20:51:21 +00001090 self->prefetched_idx = self->input_len;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001091 return self->input_len;
1092}
1093
Antoine Pitrou04248a82010-10-12 20:51:21 +00001094static int
1095_Unpickler_SkipConsumed(UnpicklerObject *self)
1096{
Victor Stinnerb43ad1d2013-10-31 13:38:42 +01001097 Py_ssize_t consumed;
1098 PyObject *r;
Antoine Pitrou04248a82010-10-12 20:51:21 +00001099
Victor Stinnerb43ad1d2013-10-31 13:38:42 +01001100 consumed = self->next_read_idx - self->prefetched_idx;
1101 if (consumed <= 0)
1102 return 0;
1103
1104 assert(self->peek); /* otherwise we did something wrong */
Martin Panter6245cb32016-04-15 02:14:19 +00001105 /* This makes a useless copy... */
Victor Stinnerb43ad1d2013-10-31 13:38:42 +01001106 r = PyObject_CallFunction(self->read, "n", consumed);
1107 if (r == NULL)
1108 return -1;
1109 Py_DECREF(r);
1110
1111 self->prefetched_idx = self->next_read_idx;
Antoine Pitrou04248a82010-10-12 20:51:21 +00001112 return 0;
1113}
1114
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001115static const Py_ssize_t READ_WHOLE_LINE = -1;
1116
1117/* If reading from a file, we need to only pull the bytes we need, since there
1118 may be multiple pickle objects arranged contiguously in the same input
1119 buffer.
1120
1121 If `n` is READ_WHOLE_LINE, read a whole line. Otherwise, read up to `n`
1122 bytes from the input stream/buffer.
1123
1124 Update the unpickler's input buffer with the newly-read data. Returns -1 on
1125 failure; on success, returns the number of bytes read from the file.
1126
1127 On success, self->input_len will be 0; this is intentional so that when
1128 unpickling from a file, the "we've run out of data" code paths will trigger,
1129 causing the Unpickler to go back to the file for more data. Use the returned
1130 size to tell you how much data you can process. */
1131static Py_ssize_t
1132_Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
1133{
1134 PyObject *data;
Serhiy Storchaka6fe39b72013-11-30 23:15:38 +02001135 Py_ssize_t read_size;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001136
1137 assert(self->read != NULL);
Victor Stinner121aab42011-09-29 23:40:53 +02001138
Antoine Pitrou04248a82010-10-12 20:51:21 +00001139 if (_Unpickler_SkipConsumed(self) < 0)
1140 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001141
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08001142 if (n == READ_WHOLE_LINE) {
Victor Stinner559bb6a2016-08-22 22:48:54 +02001143 data = _PyObject_CallNoArg(self->readline);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08001144 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001145 else {
Serhiy Storchaka6fe39b72013-11-30 23:15:38 +02001146 PyObject *len;
1147 /* Prefetch some data without advancing the file pointer, if possible */
1148 if (self->peek && n < PREFETCH) {
1149 len = PyLong_FromSsize_t(PREFETCH);
1150 if (len == NULL)
1151 return -1;
1152 data = _Pickle_FastCall(self->peek, len);
1153 if (data == NULL) {
1154 if (!PyErr_ExceptionMatches(PyExc_NotImplementedError))
1155 return -1;
1156 /* peek() is probably not supported by the given file object */
1157 PyErr_Clear();
1158 Py_CLEAR(self->peek);
1159 }
1160 else {
1161 read_size = _Unpickler_SetStringInput(self, data);
1162 Py_DECREF(data);
1163 self->prefetched_idx = 0;
1164 if (n <= read_size)
1165 return n;
1166 }
1167 }
1168 len = PyLong_FromSsize_t(n);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001169 if (len == NULL)
1170 return -1;
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08001171 data = _Pickle_FastCall(self->read, len);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001172 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001173 if (data == NULL)
1174 return -1;
1175
Serhiy Storchaka6fe39b72013-11-30 23:15:38 +02001176 read_size = _Unpickler_SetStringInput(self, data);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001177 Py_DECREF(data);
1178 return read_size;
1179}
1180
Victor Stinner19ed27e2016-05-20 11:42:37 +02001181/* Don't call it directly: use _Unpickler_Read() */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001182static Py_ssize_t
Victor Stinner19ed27e2016-05-20 11:42:37 +02001183_Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001184{
Antoine Pitrou04248a82010-10-12 20:51:21 +00001185 Py_ssize_t num_read;
1186
Benjamin Peterson6aa15642015-09-27 01:16:03 -07001187 *s = NULL;
Benjamin Petersone48cf7e2015-09-26 00:08:34 -07001188 if (self->next_read_idx > PY_SSIZE_T_MAX - n) {
1189 PickleState *st = _Pickle_GetGlobalState();
1190 PyErr_SetString(st->UnpicklingError,
1191 "read would overflow (invalid bytecode)");
1192 return -1;
1193 }
Victor Stinner19ed27e2016-05-20 11:42:37 +02001194
1195 /* This case is handled by the _Unpickler_Read() macro for efficiency */
1196 assert(self->next_read_idx + n > self->input_len);
1197
Antoine Pitrou04248a82010-10-12 20:51:21 +00001198 if (!self->read) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001199 PyErr_Format(PyExc_EOFError, "Ran out of input");
Amaury Forgeot d'Arc3e4e72f2008-11-11 20:05:06 +00001200 return -1;
1201 }
Antoine Pitrou04248a82010-10-12 20:51:21 +00001202 num_read = _Unpickler_ReadFromFile(self, n);
1203 if (num_read < 0)
1204 return -1;
1205 if (num_read < n) {
1206 PyErr_Format(PyExc_EOFError, "Ran out of input");
1207 return -1;
1208 }
1209 *s = self->input_buffer;
1210 self->next_read_idx = n;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001211 return n;
1212}
1213
Victor Stinner19ed27e2016-05-20 11:42:37 +02001214/* Read `n` bytes from the unpickler's data source, storing the result in `*s`.
1215
1216 This should be used for all data reads, rather than accessing the unpickler's
1217 input buffer directly. This method deals correctly with reading from input
1218 streams, which the input buffer doesn't deal with.
1219
1220 Note that when reading from a file-like object, self->next_read_idx won't
1221 be updated (it should remain at 0 for the entire unpickling process). You
1222 should use this function's return value to know how many bytes you can
1223 consume.
1224
1225 Returns -1 (with an exception set) on failure. On success, return the
1226 number of chars read. */
1227#define _Unpickler_Read(self, s, n) \
Victor Stinnerda230562016-05-20 21:16:59 +02001228 (((n) <= (self)->input_len - (self)->next_read_idx) \
Victor Stinner19ed27e2016-05-20 11:42:37 +02001229 ? (*(s) = (self)->input_buffer + (self)->next_read_idx, \
1230 (self)->next_read_idx += (n), \
1231 (n)) \
1232 : _Unpickler_ReadImpl(self, (s), (n)))
1233
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001234static Py_ssize_t
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001235_Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len,
1236 char **result)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001237{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001238 char *input_line = PyMem_Realloc(self->input_line, len + 1);
Victor Stinner42024562013-07-12 00:53:57 +02001239 if (input_line == NULL) {
1240 PyErr_NoMemory();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001241 return -1;
Victor Stinner42024562013-07-12 00:53:57 +02001242 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001243
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001244 memcpy(input_line, line, len);
1245 input_line[len] = '\0';
1246 self->input_line = input_line;
1247 *result = self->input_line;
1248 return len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001249}
1250
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001251/* Read a line from the input stream/buffer. If we run off the end of the input
1252 before hitting \n, return the data we found.
1253
1254 Returns the number of chars read, or -1 on failure. */
1255static Py_ssize_t
1256_Unpickler_Readline(UnpicklerObject *self, char **result)
1257{
1258 Py_ssize_t i, num_read;
1259
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001260 for (i = self->next_read_idx; i < self->input_len; i++) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001261 if (self->input_buffer[i] == '\n') {
1262 char *line_start = self->input_buffer + self->next_read_idx;
1263 num_read = i - self->next_read_idx + 1;
1264 self->next_read_idx = i + 1;
1265 return _Unpickler_CopyLine(self, line_start, num_read, result);
1266 }
1267 }
1268 if (self->read) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001269 num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE);
1270 if (num_read < 0)
1271 return -1;
Antoine Pitrou04248a82010-10-12 20:51:21 +00001272 self->next_read_idx = num_read;
Antoine Pitrouf6c7a852011-08-11 21:04:02 +02001273 return _Unpickler_CopyLine(self, self->input_buffer, num_read, result);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001274 }
Victor Stinner121aab42011-09-29 23:40:53 +02001275
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001276 /* If we get here, we've run off the end of the input string. Return the
1277 remaining string and let the caller figure it out. */
1278 *result = self->input_buffer + self->next_read_idx;
1279 num_read = i - self->next_read_idx;
1280 self->next_read_idx = i;
1281 return num_read;
1282}
1283
1284/* Returns -1 (with an exception set) on failure, 0 on success. The memo array
1285 will be modified in place. */
1286static int
1287_Unpickler_ResizeMemoList(UnpicklerObject *self, Py_ssize_t new_size)
1288{
1289 Py_ssize_t i;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001290
1291 assert(new_size > self->memo_size);
1292
Benjamin Peterson59b08c12015-06-27 13:41:33 -05001293 PyMem_RESIZE(self->memo, PyObject *, new_size);
1294 if (self->memo == NULL) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001295 PyErr_NoMemory();
1296 return -1;
1297 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001298 for (i = self->memo_size; i < new_size; i++)
1299 self->memo[i] = NULL;
1300 self->memo_size = new_size;
1301 return 0;
1302}
1303
1304/* Returns NULL if idx is out of bounds. */
1305static PyObject *
1306_Unpickler_MemoGet(UnpicklerObject *self, Py_ssize_t idx)
1307{
1308 if (idx < 0 || idx >= self->memo_size)
1309 return NULL;
1310
1311 return self->memo[idx];
1312}
1313
1314/* Returns -1 (with an exception set) on failure, 0 on success.
1315 This takes its own reference to `value`. */
1316static int
1317_Unpickler_MemoPut(UnpicklerObject *self, Py_ssize_t idx, PyObject *value)
1318{
1319 PyObject *old_item;
1320
1321 if (idx >= self->memo_size) {
1322 if (_Unpickler_ResizeMemoList(self, idx * 2) < 0)
1323 return -1;
1324 assert(idx < self->memo_size);
1325 }
1326 Py_INCREF(value);
1327 old_item = self->memo[idx];
1328 self->memo[idx] = value;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001329 if (old_item != NULL) {
1330 Py_DECREF(old_item);
1331 }
1332 else {
1333 self->memo_len++;
1334 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001335 return 0;
1336}
1337
1338static PyObject **
1339_Unpickler_NewMemo(Py_ssize_t new_size)
1340{
Benjamin Peterson59b08c12015-06-27 13:41:33 -05001341 PyObject **memo = PyMem_NEW(PyObject *, new_size);
Victor Stinner42024562013-07-12 00:53:57 +02001342 if (memo == NULL) {
1343 PyErr_NoMemory();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001344 return NULL;
Victor Stinner42024562013-07-12 00:53:57 +02001345 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001346 memset(memo, 0, new_size * sizeof(PyObject *));
1347 return memo;
1348}
1349
1350/* Free the unpickler's memo, taking care to decref any items left in it. */
1351static void
1352_Unpickler_MemoCleanup(UnpicklerObject *self)
1353{
1354 Py_ssize_t i;
1355 PyObject **memo = self->memo;
1356
1357 if (self->memo == NULL)
1358 return;
1359 self->memo = NULL;
1360 i = self->memo_size;
1361 while (--i >= 0) {
1362 Py_XDECREF(memo[i]);
1363 }
1364 PyMem_FREE(memo);
1365}
1366
1367static UnpicklerObject *
1368_Unpickler_New(void)
1369{
1370 UnpicklerObject *self;
1371
1372 self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type);
1373 if (self == NULL)
1374 return NULL;
1375
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001376 self->pers_func = NULL;
1377 self->input_buffer = NULL;
1378 self->input_line = NULL;
1379 self->input_len = 0;
1380 self->next_read_idx = 0;
Antoine Pitrou04248a82010-10-12 20:51:21 +00001381 self->prefetched_idx = 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001382 self->read = NULL;
1383 self->readline = NULL;
Antoine Pitrou04248a82010-10-12 20:51:21 +00001384 self->peek = NULL;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001385 self->encoding = NULL;
1386 self->errors = NULL;
1387 self->marks = NULL;
1388 self->num_marks = 0;
1389 self->marks_size = 0;
1390 self->proto = 0;
1391 self->fix_imports = 0;
Victor Stinner68c8ea22013-07-11 22:56:25 +02001392 memset(&self->buffer, 0, sizeof(Py_buffer));
1393 self->memo_size = 32;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001394 self->memo_len = 0;
Victor Stinner68c8ea22013-07-11 22:56:25 +02001395 self->memo = _Unpickler_NewMemo(self->memo_size);
1396 self->stack = (Pdata *)Pdata_New();
1397
1398 if (self->memo == NULL || self->stack == NULL) {
1399 Py_DECREF(self);
1400 return NULL;
1401 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001402
1403 return self;
1404}
1405
1406/* Returns -1 (with an exception set) on failure, 0 on success. This may
1407 be called once on a freshly created Pickler. */
1408static int
1409_Unpickler_SetInputStream(UnpicklerObject *self, PyObject *file)
1410{
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02001411 _Py_IDENTIFIER(peek);
1412 _Py_IDENTIFIER(read);
1413 _Py_IDENTIFIER(readline);
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02001414
1415 self->peek = _PyObject_GetAttrId(file, &PyId_peek);
Antoine Pitrou04248a82010-10-12 20:51:21 +00001416 if (self->peek == NULL) {
1417 if (PyErr_ExceptionMatches(PyExc_AttributeError))
1418 PyErr_Clear();
1419 else
1420 return -1;
1421 }
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02001422 self->read = _PyObject_GetAttrId(file, &PyId_read);
1423 self->readline = _PyObject_GetAttrId(file, &PyId_readline);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001424 if (self->readline == NULL || self->read == NULL) {
1425 if (PyErr_ExceptionMatches(PyExc_AttributeError))
1426 PyErr_SetString(PyExc_TypeError,
1427 "file must have 'read' and 'readline' attributes");
1428 Py_CLEAR(self->read);
1429 Py_CLEAR(self->readline);
Antoine Pitrou04248a82010-10-12 20:51:21 +00001430 Py_CLEAR(self->peek);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001431 return -1;
1432 }
1433 return 0;
1434}
1435
1436/* Returns -1 (with an exception set) on failure, 0 on success. This may
1437 be called once on a freshly created Pickler. */
1438static int
1439_Unpickler_SetInputEncoding(UnpicklerObject *self,
1440 const char *encoding,
1441 const char *errors)
1442{
1443 if (encoding == NULL)
1444 encoding = "ASCII";
1445 if (errors == NULL)
1446 errors = "strict";
1447
Victor Stinner49fc8ec2013-07-07 23:30:24 +02001448 self->encoding = _PyMem_Strdup(encoding);
1449 self->errors = _PyMem_Strdup(errors);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001450 if (self->encoding == NULL || self->errors == NULL) {
1451 PyErr_NoMemory();
1452 return -1;
1453 }
1454 return 0;
1455}
1456
1457/* Generate a GET opcode for an object stored in the memo. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001458static int
1459memo_get(PicklerObject *self, PyObject *key)
1460{
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001461 Py_ssize_t *value;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001462 char pdata[30];
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001463 Py_ssize_t len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001464
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001465 value = PyMemoTable_Get(self->memo, key);
1466 if (value == NULL) {
1467 PyErr_SetObject(PyExc_KeyError, key);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001468 return -1;
1469 }
1470
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001471 if (!self->bin) {
1472 pdata[0] = GET;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001473 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1,
1474 "%" PY_FORMAT_SIZE_T "d\n", *value);
1475 len = strlen(pdata);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001476 }
1477 else {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001478 if (*value < 256) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001479 pdata[0] = BINGET;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001480 pdata[1] = (unsigned char)(*value & 0xff);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001481 len = 2;
1482 }
Serhiy Storchaka67c719b2014-09-05 10:10:23 +03001483 else if ((size_t)*value <= 0xffffffffUL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001484 pdata[0] = LONG_BINGET;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001485 pdata[1] = (unsigned char)(*value & 0xff);
1486 pdata[2] = (unsigned char)((*value >> 8) & 0xff);
1487 pdata[3] = (unsigned char)((*value >> 16) & 0xff);
1488 pdata[4] = (unsigned char)((*value >> 24) & 0xff);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001489 len = 5;
1490 }
1491 else { /* unlikely */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08001492 PickleState *st = _Pickle_GetGlobalState();
1493 PyErr_SetString(st->PicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001494 "memo id too large for LONG_BINGET");
1495 return -1;
1496 }
1497 }
1498
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001499 if (_Pickler_Write(self, pdata, len) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001500 return -1;
1501
1502 return 0;
1503}
1504
1505/* Store an object in the memo, assign it a new unique ID based on the number
1506 of objects currently stored in the memo and generate a PUT opcode. */
1507static int
1508memo_put(PicklerObject *self, PyObject *obj)
1509{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001510 char pdata[30];
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001511 Py_ssize_t len;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001512 Py_ssize_t idx;
1513
1514 const char memoize_op = MEMOIZE;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001515
1516 if (self->fast)
1517 return 0;
1518
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001519 idx = PyMemoTable_Size(self->memo);
1520 if (PyMemoTable_Set(self->memo, obj, idx) < 0)
1521 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001522
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001523 if (self->proto >= 4) {
1524 if (_Pickler_Write(self, &memoize_op, 1) < 0)
1525 return -1;
1526 return 0;
1527 }
1528 else if (!self->bin) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001529 pdata[0] = PUT;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001530 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001531 "%" PY_FORMAT_SIZE_T "d\n", idx);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001532 len = strlen(pdata);
1533 }
1534 else {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001535 if (idx < 256) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001536 pdata[0] = BINPUT;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001537 pdata[1] = (unsigned char)idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001538 len = 2;
1539 }
Serhiy Storchaka67c719b2014-09-05 10:10:23 +03001540 else if ((size_t)idx <= 0xffffffffUL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001541 pdata[0] = LONG_BINPUT;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001542 pdata[1] = (unsigned char)(idx & 0xff);
1543 pdata[2] = (unsigned char)((idx >> 8) & 0xff);
1544 pdata[3] = (unsigned char)((idx >> 16) & 0xff);
1545 pdata[4] = (unsigned char)((idx >> 24) & 0xff);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001546 len = 5;
1547 }
1548 else { /* unlikely */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08001549 PickleState *st = _Pickle_GetGlobalState();
1550 PyErr_SetString(st->PicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001551 "memo id too large for LONG_BINPUT");
1552 return -1;
1553 }
1554 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001555 if (_Pickler_Write(self, pdata, len) < 0)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001556 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001557
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001558 return 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001559}
1560
1561static PyObject *
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001562get_dotted_path(PyObject *obj, PyObject *name) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001563 _Py_static_string(PyId_dot, ".");
1564 _Py_static_string(PyId_locals, "<locals>");
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001565 PyObject *dotted_path;
1566 Py_ssize_t i, n;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001567
1568 dotted_path = PyUnicode_Split(name, _PyUnicode_FromId(&PyId_dot), -1);
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001569 if (dotted_path == NULL)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001570 return NULL;
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001571 n = PyList_GET_SIZE(dotted_path);
1572 assert(n >= 1);
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001573 for (i = 0; i < n; i++) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001574 PyObject *subpath = PyList_GET_ITEM(dotted_path, i);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001575 PyObject *result = PyUnicode_RichCompare(
1576 subpath, _PyUnicode_FromId(&PyId_locals), Py_EQ);
1577 int is_equal = (result == Py_True);
1578 assert(PyBool_Check(result));
1579 Py_DECREF(result);
1580 if (is_equal) {
Antoine Pitrou6cd5eda2014-12-02 00:20:03 +01001581 if (obj == NULL)
1582 PyErr_Format(PyExc_AttributeError,
1583 "Can't pickle local object %R", name);
1584 else
1585 PyErr_Format(PyExc_AttributeError,
1586 "Can't pickle local attribute %R on %R", name, obj);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001587 Py_DECREF(dotted_path);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001588 return NULL;
1589 }
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001590 }
1591 return dotted_path;
1592}
1593
1594static PyObject *
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001595get_deep_attribute(PyObject *obj, PyObject *names, PyObject **pparent)
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001596{
1597 Py_ssize_t i, n;
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001598 PyObject *parent = NULL;
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001599
1600 assert(PyList_CheckExact(names));
1601 Py_INCREF(obj);
1602 n = PyList_GET_SIZE(names);
1603 for (i = 0; i < n; i++) {
1604 PyObject *name = PyList_GET_ITEM(names, i);
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001605 Py_XDECREF(parent);
1606 parent = obj;
1607 obj = PyObject_GetAttr(parent, name);
1608 if (obj == NULL) {
1609 Py_DECREF(parent);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001610 return NULL;
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001611 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001612 }
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001613 if (pparent != NULL)
1614 *pparent = parent;
1615 else
1616 Py_XDECREF(parent);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001617 return obj;
1618}
1619
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001620static void
1621reformat_attribute_error(PyObject *obj, PyObject *name)
1622{
1623 if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
1624 PyErr_Clear();
1625 PyErr_Format(PyExc_AttributeError,
1626 "Can't get attribute %R on %R", name, obj);
1627 }
1628}
1629
1630
1631static PyObject *
1632getattribute(PyObject *obj, PyObject *name, int allow_qualname)
1633{
1634 PyObject *dotted_path, *attr;
1635
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001636 if (allow_qualname) {
1637 dotted_path = get_dotted_path(obj, name);
1638 if (dotted_path == NULL)
1639 return NULL;
1640 attr = get_deep_attribute(obj, dotted_path, NULL);
1641 Py_DECREF(dotted_path);
1642 }
1643 else
1644 attr = PyObject_GetAttr(obj, name);
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001645 if (attr == NULL)
1646 reformat_attribute_error(obj, name);
1647 return attr;
1648}
1649
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001650static PyObject *
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001651whichmodule(PyObject *global, PyObject *dotted_path)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001652{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001653 PyObject *module_name;
1654 PyObject *modules_dict;
1655 PyObject *module;
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001656 Py_ssize_t i;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001657 _Py_IDENTIFIER(__module__);
1658 _Py_IDENTIFIER(modules);
1659 _Py_IDENTIFIER(__main__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001660
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001661 module_name = _PyObject_GetAttrId(global, &PyId___module__);
1662
1663 if (module_name == NULL) {
1664 if (!PyErr_ExceptionMatches(PyExc_AttributeError))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001665 return NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001666 PyErr_Clear();
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001667 }
1668 else {
1669 /* In some rare cases (e.g., bound methods of extension types),
1670 __module__ can be None. If it is so, then search sys.modules for
1671 the module of global. */
1672 if (module_name != Py_None)
1673 return module_name;
1674 Py_CLEAR(module_name);
1675 }
1676 assert(module_name == NULL);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001677
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001678 /* Fallback on walking sys.modules */
Victor Stinnerbb520202013-11-06 22:40:41 +01001679 modules_dict = _PySys_GetObjectId(&PyId_modules);
Victor Stinner1e53bba2013-07-16 22:26:05 +02001680 if (modules_dict == NULL) {
1681 PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001682 return NULL;
Victor Stinner1e53bba2013-07-16 22:26:05 +02001683 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001684
1685 i = 0;
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001686 while (PyDict_Next(modules_dict, &i, &module_name, &module)) {
1687 PyObject *candidate;
1688 if (PyUnicode_Check(module_name) &&
1689 !PyUnicode_CompareWithASCIIString(module_name, "__main__"))
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001690 continue;
1691 if (module == Py_None)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001692 continue;
1693
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001694 candidate = get_deep_attribute(module, dotted_path, NULL);
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001695 if (candidate == NULL) {
Serhiy Storchaka58e41342015-03-31 14:07:24 +03001696 if (!PyErr_ExceptionMatches(PyExc_AttributeError))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001697 return NULL;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001698 PyErr_Clear();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001699 continue;
1700 }
1701
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001702 if (candidate == global) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001703 Py_INCREF(module_name);
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001704 Py_DECREF(candidate);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001705 return module_name;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001706 }
Antoine Pitroufce60ea2014-10-23 22:47:50 +02001707 Py_DECREF(candidate);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001708 }
1709
1710 /* If no module is found, use __main__. */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01001711 module_name = _PyUnicode_FromId(&PyId___main__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001712 Py_INCREF(module_name);
1713 return module_name;
1714}
1715
1716/* fast_save_enter() and fast_save_leave() are guards against recursive
1717 objects when Pickler is used with the "fast mode" (i.e., with object
1718 memoization disabled). If the nesting of a list or dict object exceed
1719 FAST_NESTING_LIMIT, these guards will start keeping an internal
1720 reference to the seen list or dict objects and check whether these objects
1721 are recursive. These are not strictly necessary, since save() has a
1722 hard-coded recursion limit, but they give a nicer error message than the
1723 typical RuntimeError. */
1724static int
1725fast_save_enter(PicklerObject *self, PyObject *obj)
1726{
1727 /* if fast_nesting < 0, we're doing an error exit. */
1728 if (++self->fast_nesting >= FAST_NESTING_LIMIT) {
1729 PyObject *key = NULL;
1730 if (self->fast_memo == NULL) {
1731 self->fast_memo = PyDict_New();
1732 if (self->fast_memo == NULL) {
1733 self->fast_nesting = -1;
1734 return 0;
1735 }
1736 }
1737 key = PyLong_FromVoidPtr(obj);
1738 if (key == NULL)
1739 return 0;
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08001740 if (PyDict_GetItemWithError(self->fast_memo, key)) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001741 Py_DECREF(key);
1742 PyErr_Format(PyExc_ValueError,
1743 "fast mode: can't pickle cyclic objects "
1744 "including object type %.200s at %p",
1745 obj->ob_type->tp_name, obj);
1746 self->fast_nesting = -1;
1747 return 0;
1748 }
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08001749 if (PyErr_Occurred()) {
1750 return 0;
1751 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001752 if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {
1753 Py_DECREF(key);
1754 self->fast_nesting = -1;
1755 return 0;
1756 }
1757 Py_DECREF(key);
1758 }
1759 return 1;
1760}
1761
1762static int
1763fast_save_leave(PicklerObject *self, PyObject *obj)
1764{
1765 if (self->fast_nesting-- >= FAST_NESTING_LIMIT) {
1766 PyObject *key = PyLong_FromVoidPtr(obj);
1767 if (key == NULL)
1768 return 0;
1769 if (PyDict_DelItem(self->fast_memo, key) < 0) {
1770 Py_DECREF(key);
1771 return 0;
1772 }
1773 Py_DECREF(key);
1774 }
1775 return 1;
1776}
1777
1778static int
1779save_none(PicklerObject *self, PyObject *obj)
1780{
1781 const char none_op = NONE;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001782 if (_Pickler_Write(self, &none_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001783 return -1;
1784
1785 return 0;
1786}
1787
1788static int
1789save_bool(PicklerObject *self, PyObject *obj)
1790{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001791 if (self->proto >= 2) {
Alexandre Vassalotti8a67f522013-11-24 21:40:18 -08001792 const char bool_op = (obj == Py_True) ? NEWTRUE : NEWFALSE;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001793 if (_Pickler_Write(self, &bool_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001794 return -1;
1795 }
Alexandre Vassalotti8a67f522013-11-24 21:40:18 -08001796 else {
1797 /* These aren't opcodes -- they're ways to pickle bools before protocol 2
1798 * so that unpicklers written before bools were introduced unpickle them
1799 * as ints, but unpicklers after can recognize that bools were intended.
1800 * Note that protocol 2 added direct ways to pickle bools.
1801 */
1802 const char *bool_str = (obj == Py_True) ? "I01\n" : "I00\n";
1803 if (_Pickler_Write(self, bool_str, strlen(bool_str)) < 0)
1804 return -1;
1805 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001806 return 0;
1807}
1808
1809static int
Alexandre Vassalottided929b2013-11-24 22:41:13 -08001810save_long(PicklerObject *self, PyObject *obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001811{
Alexandre Vassalottided929b2013-11-24 22:41:13 -08001812 PyObject *repr = NULL;
1813 Py_ssize_t size;
1814 long val;
1815 int status = 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001816
Alexandre Vassalottided929b2013-11-24 22:41:13 -08001817 const char long_op = LONG;
1818
1819 val= PyLong_AsLong(obj);
1820 if (val == -1 && PyErr_Occurred()) {
1821 /* out of range for int pickling */
1822 PyErr_Clear();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001823 }
Alexandre Vassalottided929b2013-11-24 22:41:13 -08001824 else if (self->bin &&
1825 (sizeof(long) <= 4 ||
Alexandre Vassalotti1048fb52013-11-25 11:35:46 -08001826 (val <= 0x7fffffffL && val >= (-0x7fffffffL - 1)))) {
Larry Hastings61272b72014-01-07 12:41:53 -08001827 /* result fits in a signed 4-byte integer.
Alexandre Vassalotti1048fb52013-11-25 11:35:46 -08001828
1829 Note: we can't use -0x80000000L in the above condition because some
1830 compilers (e.g., MSVC) will promote 0x80000000L to an unsigned type
1831 before applying the unary minus when sizeof(long) <= 4. The
1832 resulting value stays unsigned which is commonly not what we want,
1833 so MSVC happily warns us about it. However, that result would have
1834 been fine because we guard for sizeof(long) <= 4 which turns the
1835 condition true in that particular case. */
Alexandre Vassalottided929b2013-11-24 22:41:13 -08001836 char pdata[32];
1837 Py_ssize_t len = 0;
1838
1839 pdata[1] = (unsigned char)(val & 0xff);
1840 pdata[2] = (unsigned char)((val >> 8) & 0xff);
1841 pdata[3] = (unsigned char)((val >> 16) & 0xff);
1842 pdata[4] = (unsigned char)((val >> 24) & 0xff);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001843
1844 if ((pdata[4] == 0) && (pdata[3] == 0)) {
1845 if (pdata[2] == 0) {
1846 pdata[0] = BININT1;
1847 len = 2;
1848 }
1849 else {
1850 pdata[0] = BININT2;
1851 len = 3;
1852 }
1853 }
1854 else {
1855 pdata[0] = BININT;
1856 len = 5;
1857 }
1858
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001859 if (_Pickler_Write(self, pdata, len) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001860 return -1;
Alexandre Vassalottided929b2013-11-24 22:41:13 -08001861
1862 return 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001863 }
1864
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001865 if (self->proto >= 2) {
1866 /* Linear-time pickling. */
1867 size_t nbits;
1868 size_t nbytes;
1869 unsigned char *pdata;
1870 char header[5];
1871 int i;
1872 int sign = _PyLong_Sign(obj);
1873
1874 if (sign == 0) {
1875 header[0] = LONG1;
1876 header[1] = 0; /* It's 0 -- an empty bytestring. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001877 if (_Pickler_Write(self, header, 2) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001878 goto error;
1879 return 0;
1880 }
1881 nbits = _PyLong_NumBits(obj);
1882 if (nbits == (size_t)-1 && PyErr_Occurred())
1883 goto error;
1884 /* How many bytes do we need? There are nbits >> 3 full
1885 * bytes of data, and nbits & 7 leftover bits. If there
1886 * are any leftover bits, then we clearly need another
1887 * byte. Wnat's not so obvious is that we *probably*
1888 * need another byte even if there aren't any leftovers:
1889 * the most-significant bit of the most-significant byte
1890 * acts like a sign bit, and it's usually got a sense
Serhiy Storchaka95949422013-08-27 19:40:23 +03001891 * opposite of the one we need. The exception is ints
1892 * of the form -(2**(8*j-1)) for j > 0. Such an int is
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001893 * its own 256's-complement, so has the right sign bit
1894 * even without the extra byte. That's a pain to check
1895 * for in advance, though, so we always grab an extra
1896 * byte at the start, and cut it back later if possible.
1897 */
1898 nbytes = (nbits >> 3) + 1;
Antoine Pitroubf6ecf92012-11-24 20:40:21 +01001899 if (nbytes > 0x7fffffffL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001900 PyErr_SetString(PyExc_OverflowError,
Serhiy Storchaka95949422013-08-27 19:40:23 +03001901 "int too large to pickle");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001902 goto error;
1903 }
Neal Norwitz6ae2eb22008-08-24 23:50:08 +00001904 repr = PyBytes_FromStringAndSize(NULL, (Py_ssize_t)nbytes);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001905 if (repr == NULL)
1906 goto error;
Neal Norwitz6ae2eb22008-08-24 23:50:08 +00001907 pdata = (unsigned char *)PyBytes_AS_STRING(repr);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001908 i = _PyLong_AsByteArray((PyLongObject *)obj,
1909 pdata, nbytes,
1910 1 /* little endian */ , 1 /* signed */ );
1911 if (i < 0)
1912 goto error;
Serhiy Storchaka95949422013-08-27 19:40:23 +03001913 /* If the int is negative, this may be a byte more than
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001914 * needed. This is so iff the MSB is all redundant sign
1915 * bits.
1916 */
1917 if (sign < 0 &&
Victor Stinner121aab42011-09-29 23:40:53 +02001918 nbytes > 1 &&
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001919 pdata[nbytes - 1] == 0xff &&
1920 (pdata[nbytes - 2] & 0x80) != 0) {
1921 nbytes--;
1922 }
1923
1924 if (nbytes < 256) {
1925 header[0] = LONG1;
1926 header[1] = (unsigned char)nbytes;
1927 size = 2;
1928 }
1929 else {
1930 header[0] = LONG4;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001931 size = (Py_ssize_t) nbytes;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001932 for (i = 1; i < 5; i++) {
1933 header[i] = (unsigned char)(size & 0xff);
1934 size >>= 8;
1935 }
1936 size = 5;
1937 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001938 if (_Pickler_Write(self, header, size) < 0 ||
1939 _Pickler_Write(self, (char *)pdata, (int)nbytes) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001940 goto error;
1941 }
1942 else {
1943 char *string;
1944
Mark Dickinson8dd05142009-01-20 20:43:58 +00001945 /* proto < 2: write the repr and newline. This is quadratic-time (in
1946 the number of digits), in both directions. We add a trailing 'L'
1947 to the repr, for compatibility with Python 2.x. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001948
1949 repr = PyObject_Repr(obj);
1950 if (repr == NULL)
1951 goto error;
1952
Marc-André Lemburg4cc0f242008-08-07 18:54:33 +00001953 string = _PyUnicode_AsStringAndSize(repr, &size);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001954 if (string == NULL)
1955 goto error;
1956
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001957 if (_Pickler_Write(self, &long_op, 1) < 0 ||
1958 _Pickler_Write(self, string, size) < 0 ||
1959 _Pickler_Write(self, "L\n", 2) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001960 goto error;
1961 }
1962
1963 if (0) {
1964 error:
1965 status = -1;
1966 }
1967 Py_XDECREF(repr);
1968
1969 return status;
1970}
1971
1972static int
1973save_float(PicklerObject *self, PyObject *obj)
1974{
1975 double x = PyFloat_AS_DOUBLE((PyFloatObject *)obj);
1976
1977 if (self->bin) {
1978 char pdata[9];
1979 pdata[0] = BINFLOAT;
1980 if (_PyFloat_Pack8(x, (unsigned char *)&pdata[1], 0) < 0)
1981 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001982 if (_Pickler_Write(self, pdata, 9) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001983 return -1;
Victor Stinner121aab42011-09-29 23:40:53 +02001984 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001985 else {
Eric Smith0923d1d2009-04-16 20:16:10 +00001986 int result = -1;
1987 char *buf = NULL;
1988 char op = FLOAT;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00001989
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001990 if (_Pickler_Write(self, &op, 1) < 0)
Eric Smith0923d1d2009-04-16 20:16:10 +00001991 goto done;
1992
Serhiy Storchakac86ca262015-02-15 14:18:32 +02001993 buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL);
Eric Smith0923d1d2009-04-16 20:16:10 +00001994 if (!buf) {
1995 PyErr_NoMemory();
1996 goto done;
1997 }
1998
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00001999 if (_Pickler_Write(self, buf, strlen(buf)) < 0)
Eric Smith0923d1d2009-04-16 20:16:10 +00002000 goto done;
2001
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002002 if (_Pickler_Write(self, "\n", 1) < 0)
Eric Smith0923d1d2009-04-16 20:16:10 +00002003 goto done;
2004
2005 result = 0;
2006done:
2007 PyMem_Free(buf);
2008 return result;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002009 }
2010
2011 return 0;
2012}
2013
2014static int
2015save_bytes(PicklerObject *self, PyObject *obj)
2016{
2017 if (self->proto < 3) {
2018 /* Older pickle protocols do not have an opcode for pickling bytes
2019 objects. Therefore, we need to fake the copy protocol (i.e.,
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002020 the __reduce__ method) to permit bytes object unpickling.
2021
2022 Here we use a hack to be compatible with Python 2. Since in Python
2023 2 'bytes' is just an alias for 'str' (which has different
2024 parameters than the actual bytes object), we use codecs.encode
2025 to create the appropriate 'str' object when unpickled using
2026 Python 2 *and* the appropriate 'bytes' object when unpickled
2027 using Python 3. Again this is a hack and we don't need to do this
2028 with newer protocols. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002029 PyObject *reduce_value = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002030 int status;
2031
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002032 if (PyBytes_GET_SIZE(obj) == 0) {
2033 reduce_value = Py_BuildValue("(O())", (PyObject*)&PyBytes_Type);
2034 }
2035 else {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08002036 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002037 PyObject *unicode_str =
2038 PyUnicode_DecodeLatin1(PyBytes_AS_STRING(obj),
2039 PyBytes_GET_SIZE(obj),
2040 "strict");
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002041 _Py_IDENTIFIER(latin1);
2042
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002043 if (unicode_str == NULL)
2044 return -1;
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002045 reduce_value = Py_BuildValue("(O(OO))",
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08002046 st->codecs_encode, unicode_str,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002047 _PyUnicode_FromId(&PyId_latin1));
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002048 Py_DECREF(unicode_str);
2049 }
2050
2051 if (reduce_value == NULL)
2052 return -1;
2053
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002054 /* save_reduce() will memoize the object automatically. */
2055 status = save_reduce(self, reduce_value, obj);
2056 Py_DECREF(reduce_value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002057 return status;
2058 }
2059 else {
2060 Py_ssize_t size;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002061 char header[9];
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002062 Py_ssize_t len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002063
Alexandre Vassalotti3bfc65a2011-12-13 13:08:09 -05002064 size = PyBytes_GET_SIZE(obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002065 if (size < 0)
2066 return -1;
2067
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002068 if (size <= 0xff) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002069 header[0] = SHORT_BINBYTES;
2070 header[1] = (unsigned char)size;
2071 len = 2;
2072 }
Serhiy Storchaka67c719b2014-09-05 10:10:23 +03002073 else if ((size_t)size <= 0xffffffffUL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002074 header[0] = BINBYTES;
2075 header[1] = (unsigned char)(size & 0xff);
2076 header[2] = (unsigned char)((size >> 8) & 0xff);
2077 header[3] = (unsigned char)((size >> 16) & 0xff);
2078 header[4] = (unsigned char)((size >> 24) & 0xff);
2079 len = 5;
2080 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002081 else if (self->proto >= 4) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002082 header[0] = BINBYTES8;
Alexandre Vassalotti1048fb52013-11-25 11:35:46 -08002083 _write_size64(header + 1, size);
Alexandre Vassalotti6e73ff12013-12-05 19:29:32 -08002084 len = 9;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002085 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002086 else {
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002087 PyErr_SetString(PyExc_OverflowError,
Serhiy Storchakaf8def282013-02-16 17:29:56 +02002088 "cannot serialize a bytes object larger than 4 GiB");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002089 return -1; /* string too large */
2090 }
2091
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002092 if (_Pickler_Write(self, header, len) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002093 return -1;
2094
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002095 if (_Pickler_Write(self, PyBytes_AS_STRING(obj), size) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002096 return -1;
2097
2098 if (memo_put(self, obj) < 0)
2099 return -1;
2100
2101 return 0;
2102 }
2103}
2104
2105/* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
2106 backslash and newline characters to \uXXXX escapes. */
2107static PyObject *
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002108raw_unicode_escape(PyObject *obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002109{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002110 char *p;
Victor Stinner049e5092014-08-17 22:20:00 +02002111 Py_ssize_t i, size;
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002112 void *data;
2113 unsigned int kind;
Victor Stinner358af132015-10-12 22:36:57 +02002114 _PyBytesWriter writer;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002115
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002116 if (PyUnicode_READY(obj))
2117 return NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002118
Victor Stinner358af132015-10-12 22:36:57 +02002119 _PyBytesWriter_Init(&writer);
2120
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002121 size = PyUnicode_GET_LENGTH(obj);
2122 data = PyUnicode_DATA(obj);
2123 kind = PyUnicode_KIND(obj);
Victor Stinner121aab42011-09-29 23:40:53 +02002124
Victor Stinner358af132015-10-12 22:36:57 +02002125 p = _PyBytesWriter_Alloc(&writer, size);
2126 if (p == NULL)
2127 goto error;
2128 writer.overallocate = 1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002129
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002130 for (i=0; i < size; i++) {
2131 Py_UCS4 ch = PyUnicode_READ(kind, data, i);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002132 /* Map 32-bit characters to '\Uxxxxxxxx' */
2133 if (ch >= 0x10000) {
Raymond Hettinger15f44ab2016-08-30 10:47:49 -07002134 /* -1: subtract 1 preallocated byte */
Victor Stinner358af132015-10-12 22:36:57 +02002135 p = _PyBytesWriter_Prepare(&writer, p, 10-1);
2136 if (p == NULL)
2137 goto error;
2138
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002139 *p++ = '\\';
2140 *p++ = 'U';
Victor Stinnerf5cff562011-10-14 02:13:11 +02002141 *p++ = Py_hexdigits[(ch >> 28) & 0xf];
2142 *p++ = Py_hexdigits[(ch >> 24) & 0xf];
2143 *p++ = Py_hexdigits[(ch >> 20) & 0xf];
2144 *p++ = Py_hexdigits[(ch >> 16) & 0xf];
2145 *p++ = Py_hexdigits[(ch >> 12) & 0xf];
2146 *p++ = Py_hexdigits[(ch >> 8) & 0xf];
2147 *p++ = Py_hexdigits[(ch >> 4) & 0xf];
2148 *p++ = Py_hexdigits[ch & 15];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002149 }
Victor Stinner358af132015-10-12 22:36:57 +02002150 /* Map 16-bit characters, '\\' and '\n' to '\uxxxx' */
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002151 else if (ch >= 256 || ch == '\\' || ch == '\n') {
Raymond Hettinger15f44ab2016-08-30 10:47:49 -07002152 /* -1: subtract 1 preallocated byte */
Victor Stinner358af132015-10-12 22:36:57 +02002153 p = _PyBytesWriter_Prepare(&writer, p, 6-1);
2154 if (p == NULL)
2155 goto error;
2156
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002157 *p++ = '\\';
2158 *p++ = 'u';
Victor Stinnerf5cff562011-10-14 02:13:11 +02002159 *p++ = Py_hexdigits[(ch >> 12) & 0xf];
2160 *p++ = Py_hexdigits[(ch >> 8) & 0xf];
2161 *p++ = Py_hexdigits[(ch >> 4) & 0xf];
2162 *p++ = Py_hexdigits[ch & 15];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002163 }
Alexandre Vassalotti554d8782008-12-27 07:32:41 +00002164 /* Copy everything else as-is */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002165 else
2166 *p++ = (char) ch;
2167 }
Victor Stinner358af132015-10-12 22:36:57 +02002168
2169 return _PyBytesWriter_Finish(&writer, p);
2170
2171error:
2172 _PyBytesWriter_Dealloc(&writer);
2173 return NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002174}
2175
2176static int
Serhiy Storchakaef1585e2015-12-25 20:01:53 +02002177write_utf8(PicklerObject *self, const char *data, Py_ssize_t size)
Antoine Pitrou299978d2013-04-07 17:38:11 +02002178{
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002179 char header[9];
2180 Py_ssize_t len;
Antoine Pitrou299978d2013-04-07 17:38:11 +02002181
Victor Stinnerf13c46c2014-08-17 21:05:55 +02002182 assert(size >= 0);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002183 if (size <= 0xff && self->proto >= 4) {
2184 header[0] = SHORT_BINUNICODE;
2185 header[1] = (unsigned char)(size & 0xff);
2186 len = 2;
2187 }
Victor Stinnerf13c46c2014-08-17 21:05:55 +02002188 else if ((size_t)size <= 0xffffffffUL) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002189 header[0] = BINUNICODE;
2190 header[1] = (unsigned char)(size & 0xff);
2191 header[2] = (unsigned char)((size >> 8) & 0xff);
2192 header[3] = (unsigned char)((size >> 16) & 0xff);
2193 header[4] = (unsigned char)((size >> 24) & 0xff);
2194 len = 5;
2195 }
2196 else if (self->proto >= 4) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002197 header[0] = BINUNICODE8;
Alexandre Vassalotti1048fb52013-11-25 11:35:46 -08002198 _write_size64(header + 1, size);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002199 len = 9;
2200 }
2201 else {
Antoine Pitrou299978d2013-04-07 17:38:11 +02002202 PyErr_SetString(PyExc_OverflowError,
Antoine Pitrou4b7b0f02013-04-07 23:46:52 +02002203 "cannot serialize a string larger than 4GiB");
Antoine Pitrou299978d2013-04-07 17:38:11 +02002204 return -1;
2205 }
Antoine Pitrou299978d2013-04-07 17:38:11 +02002206
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002207 if (_Pickler_Write(self, header, len) < 0)
Antoine Pitrou299978d2013-04-07 17:38:11 +02002208 return -1;
Antoine Pitrou299978d2013-04-07 17:38:11 +02002209 if (_Pickler_Write(self, data, size) < 0)
2210 return -1;
2211
2212 return 0;
2213}
2214
2215static int
2216write_unicode_binary(PicklerObject *self, PyObject *obj)
2217{
2218 PyObject *encoded = NULL;
2219 Py_ssize_t size;
2220 char *data;
2221 int r;
2222
2223 if (PyUnicode_READY(obj))
2224 return -1;
2225
2226 data = PyUnicode_AsUTF8AndSize(obj, &size);
2227 if (data != NULL)
2228 return write_utf8(self, data, size);
2229
2230 /* Issue #8383: for strings with lone surrogates, fallback on the
2231 "surrogatepass" error handler. */
2232 PyErr_Clear();
2233 encoded = PyUnicode_AsEncodedString(obj, "utf-8", "surrogatepass");
2234 if (encoded == NULL)
2235 return -1;
2236
2237 r = write_utf8(self, PyBytes_AS_STRING(encoded),
2238 PyBytes_GET_SIZE(encoded));
2239 Py_DECREF(encoded);
2240 return r;
2241}
2242
2243static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002244save_unicode(PicklerObject *self, PyObject *obj)
2245{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002246 if (self->bin) {
Antoine Pitrou299978d2013-04-07 17:38:11 +02002247 if (write_unicode_binary(self, obj) < 0)
2248 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002249 }
2250 else {
Antoine Pitrou299978d2013-04-07 17:38:11 +02002251 PyObject *encoded;
2252 Py_ssize_t size;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002253 const char unicode_op = UNICODE;
2254
Victor Stinnerc806fdc2011-09-29 23:50:23 +02002255 encoded = raw_unicode_escape(obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002256 if (encoded == NULL)
Antoine Pitrou299978d2013-04-07 17:38:11 +02002257 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002258
Antoine Pitrou299978d2013-04-07 17:38:11 +02002259 if (_Pickler_Write(self, &unicode_op, 1) < 0) {
2260 Py_DECREF(encoded);
2261 return -1;
2262 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002263
2264 size = PyBytes_GET_SIZE(encoded);
Antoine Pitrou299978d2013-04-07 17:38:11 +02002265 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), size) < 0) {
2266 Py_DECREF(encoded);
2267 return -1;
2268 }
2269 Py_DECREF(encoded);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002270
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002271 if (_Pickler_Write(self, "\n", 1) < 0)
Antoine Pitrou299978d2013-04-07 17:38:11 +02002272 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002273 }
2274 if (memo_put(self, obj) < 0)
Antoine Pitrou299978d2013-04-07 17:38:11 +02002275 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002276
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002277 return 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002278}
2279
2280/* A helper for save_tuple. Push the len elements in tuple t on the stack. */
2281static int
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002282store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002283{
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002284 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002285
2286 assert(PyTuple_Size(t) == len);
2287
2288 for (i = 0; i < len; i++) {
2289 PyObject *element = PyTuple_GET_ITEM(t, i);
2290
2291 if (element == NULL)
2292 return -1;
2293 if (save(self, element, 0) < 0)
2294 return -1;
2295 }
2296
2297 return 0;
2298}
2299
2300/* Tuples are ubiquitous in the pickle protocols, so many techniques are
2301 * used across protocols to minimize the space needed to pickle them.
2302 * Tuples are also the only builtin immutable type that can be recursive
2303 * (a tuple can be reached from itself), and that requires some subtle
2304 * magic so that it works in all cases. IOW, this is a long routine.
2305 */
2306static int
2307save_tuple(PicklerObject *self, PyObject *obj)
2308{
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002309 Py_ssize_t len, i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002310
2311 const char mark_op = MARK;
2312 const char tuple_op = TUPLE;
2313 const char pop_op = POP;
2314 const char pop_mark_op = POP_MARK;
2315 const char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
2316
2317 if ((len = PyTuple_Size(obj)) < 0)
2318 return -1;
2319
2320 if (len == 0) {
2321 char pdata[2];
2322
2323 if (self->proto) {
2324 pdata[0] = EMPTY_TUPLE;
2325 len = 1;
2326 }
2327 else {
2328 pdata[0] = MARK;
2329 pdata[1] = TUPLE;
2330 len = 2;
2331 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002332 if (_Pickler_Write(self, pdata, len) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002333 return -1;
2334 return 0;
2335 }
2336
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002337 /* The tuple isn't in the memo now. If it shows up there after
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002338 * saving the tuple elements, the tuple must be recursive, in
2339 * which case we'll pop everything we put on the stack, and fetch
2340 * its value from the memo.
2341 */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002342 if (len <= 3 && self->proto >= 2) {
2343 /* Use TUPLE{1,2,3} opcodes. */
2344 if (store_tuple_elements(self, obj, len) < 0)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002345 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002346
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002347 if (PyMemoTable_Get(self->memo, obj)) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002348 /* pop the len elements */
2349 for (i = 0; i < len; i++)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002350 if (_Pickler_Write(self, &pop_op, 1) < 0)
2351 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002352 /* fetch from memo */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002353 if (memo_get(self, obj) < 0)
2354 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002355
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002356 return 0;
2357 }
2358 else { /* Not recursive. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002359 if (_Pickler_Write(self, len2opcode + len, 1) < 0)
2360 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002361 }
2362 goto memoize;
2363 }
2364
2365 /* proto < 2 and len > 0, or proto >= 2 and len > 3.
2366 * Generate MARK e1 e2 ... TUPLE
2367 */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002368 if (_Pickler_Write(self, &mark_op, 1) < 0)
2369 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002370
2371 if (store_tuple_elements(self, obj, len) < 0)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002372 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002373
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002374 if (PyMemoTable_Get(self->memo, obj)) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002375 /* pop the stack stuff we pushed */
2376 if (self->bin) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002377 if (_Pickler_Write(self, &pop_mark_op, 1) < 0)
2378 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002379 }
2380 else {
2381 /* Note that we pop one more than len, to remove
2382 * the MARK too.
2383 */
2384 for (i = 0; i <= len; i++)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002385 if (_Pickler_Write(self, &pop_op, 1) < 0)
2386 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002387 }
2388 /* fetch from memo */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002389 if (memo_get(self, obj) < 0)
2390 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002391
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002392 return 0;
2393 }
2394 else { /* Not recursive. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002395 if (_Pickler_Write(self, &tuple_op, 1) < 0)
2396 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002397 }
2398
2399 memoize:
2400 if (memo_put(self, obj) < 0)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002401 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002402
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002403 return 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002404}
2405
2406/* iter is an iterator giving items, and we batch up chunks of
2407 * MARK item item ... item APPENDS
2408 * opcode sequences. Calling code should have arranged to first create an
2409 * empty list, or list-like object, for the APPENDS to operate on.
2410 * Returns 0 on success, <0 on error.
2411 */
2412static int
2413batch_list(PicklerObject *self, PyObject *iter)
2414{
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002415 PyObject *obj = NULL;
2416 PyObject *firstitem = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002417 int i, n;
2418
2419 const char mark_op = MARK;
2420 const char append_op = APPEND;
2421 const char appends_op = APPENDS;
2422
2423 assert(iter != NULL);
2424
2425 /* XXX: I think this function could be made faster by avoiding the
2426 iterator interface and fetching objects directly from list using
2427 PyList_GET_ITEM.
2428 */
2429
2430 if (self->proto == 0) {
2431 /* APPENDS isn't available; do one at a time. */
2432 for (;;) {
2433 obj = PyIter_Next(iter);
2434 if (obj == NULL) {
2435 if (PyErr_Occurred())
2436 return -1;
2437 break;
2438 }
2439 i = save(self, obj, 0);
2440 Py_DECREF(obj);
2441 if (i < 0)
2442 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002443 if (_Pickler_Write(self, &append_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002444 return -1;
2445 }
2446 return 0;
2447 }
2448
2449 /* proto > 0: write in batches of BATCHSIZE. */
2450 do {
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002451 /* Get first item */
2452 firstitem = PyIter_Next(iter);
2453 if (firstitem == NULL) {
2454 if (PyErr_Occurred())
2455 goto error;
2456
2457 /* nothing more to add */
2458 break;
2459 }
2460
2461 /* Try to get a second item */
2462 obj = PyIter_Next(iter);
2463 if (obj == NULL) {
2464 if (PyErr_Occurred())
2465 goto error;
2466
2467 /* Only one item to write */
2468 if (save(self, firstitem, 0) < 0)
2469 goto error;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002470 if (_Pickler_Write(self, &append_op, 1) < 0)
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002471 goto error;
2472 Py_CLEAR(firstitem);
2473 break;
2474 }
2475
2476 /* More than one item to write */
2477
2478 /* Pump out MARK, items, APPENDS. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002479 if (_Pickler_Write(self, &mark_op, 1) < 0)
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002480 goto error;
2481
2482 if (save(self, firstitem, 0) < 0)
2483 goto error;
2484 Py_CLEAR(firstitem);
2485 n = 1;
2486
2487 /* Fetch and save up to BATCHSIZE items */
2488 while (obj) {
2489 if (save(self, obj, 0) < 0)
2490 goto error;
2491 Py_CLEAR(obj);
2492 n += 1;
2493
2494 if (n == BATCHSIZE)
2495 break;
2496
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002497 obj = PyIter_Next(iter);
2498 if (obj == NULL) {
2499 if (PyErr_Occurred())
2500 goto error;
2501 break;
2502 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002503 }
2504
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002505 if (_Pickler_Write(self, &appends_op, 1) < 0)
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002506 goto error;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002507
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002508 } while (n == BATCHSIZE);
2509 return 0;
2510
2511 error:
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002512 Py_XDECREF(firstitem);
2513 Py_XDECREF(obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002514 return -1;
2515}
2516
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002517/* This is a variant of batch_list() above, specialized for lists (with no
2518 * support for list subclasses). Like batch_list(), we batch up chunks of
2519 * MARK item item ... item APPENDS
2520 * opcode sequences. Calling code should have arranged to first create an
2521 * empty list, or list-like object, for the APPENDS to operate on.
2522 * Returns 0 on success, -1 on error.
2523 *
2524 * This version is considerably faster than batch_list(), if less general.
2525 *
2526 * Note that this only works for protocols > 0.
2527 */
2528static int
2529batch_list_exact(PicklerObject *self, PyObject *obj)
2530{
2531 PyObject *item = NULL;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002532 Py_ssize_t this_batch, total;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002533
2534 const char append_op = APPEND;
2535 const char appends_op = APPENDS;
2536 const char mark_op = MARK;
2537
2538 assert(obj != NULL);
2539 assert(self->proto > 0);
2540 assert(PyList_CheckExact(obj));
2541
2542 if (PyList_GET_SIZE(obj) == 1) {
2543 item = PyList_GET_ITEM(obj, 0);
2544 if (save(self, item, 0) < 0)
2545 return -1;
2546 if (_Pickler_Write(self, &append_op, 1) < 0)
2547 return -1;
2548 return 0;
2549 }
2550
2551 /* Write in batches of BATCHSIZE. */
2552 total = 0;
2553 do {
2554 this_batch = 0;
2555 if (_Pickler_Write(self, &mark_op, 1) < 0)
2556 return -1;
2557 while (total < PyList_GET_SIZE(obj)) {
2558 item = PyList_GET_ITEM(obj, total);
2559 if (save(self, item, 0) < 0)
2560 return -1;
2561 total++;
2562 if (++this_batch == BATCHSIZE)
2563 break;
2564 }
2565 if (_Pickler_Write(self, &appends_op, 1) < 0)
2566 return -1;
2567
2568 } while (total < PyList_GET_SIZE(obj));
2569
2570 return 0;
2571}
2572
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002573static int
2574save_list(PicklerObject *self, PyObject *obj)
2575{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002576 char header[3];
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002577 Py_ssize_t len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002578 int status = 0;
2579
2580 if (self->fast && !fast_save_enter(self, obj))
2581 goto error;
2582
2583 /* Create an empty list. */
2584 if (self->bin) {
2585 header[0] = EMPTY_LIST;
2586 len = 1;
2587 }
2588 else {
2589 header[0] = MARK;
2590 header[1] = LIST;
2591 len = 2;
2592 }
2593
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002594 if (_Pickler_Write(self, header, len) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002595 goto error;
2596
2597 /* Get list length, and bow out early if empty. */
2598 if ((len = PyList_Size(obj)) < 0)
2599 goto error;
2600
2601 if (memo_put(self, obj) < 0)
2602 goto error;
2603
2604 if (len != 0) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002605 /* Materialize the list elements. */
2606 if (PyList_CheckExact(obj) && self->proto > 0) {
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00002607 if (Py_EnterRecursiveCall(" while pickling an object"))
2608 goto error;
2609 status = batch_list_exact(self, obj);
2610 Py_LeaveRecursiveCall();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002611 } else {
2612 PyObject *iter = PyObject_GetIter(obj);
2613 if (iter == NULL)
2614 goto error;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002615
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00002616 if (Py_EnterRecursiveCall(" while pickling an object")) {
2617 Py_DECREF(iter);
2618 goto error;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002619 }
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00002620 status = batch_list(self, iter);
2621 Py_LeaveRecursiveCall();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002622 Py_DECREF(iter);
2623 }
2624 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002625 if (0) {
2626 error:
2627 status = -1;
2628 }
2629
2630 if (self->fast && !fast_save_leave(self, obj))
2631 status = -1;
2632
2633 return status;
2634}
2635
2636/* iter is an iterator giving (key, value) pairs, and we batch up chunks of
2637 * MARK key value ... key value SETITEMS
2638 * opcode sequences. Calling code should have arranged to first create an
2639 * empty dict, or dict-like object, for the SETITEMS to operate on.
2640 * Returns 0 on success, <0 on error.
2641 *
2642 * This is very much like batch_list(). The difference between saving
2643 * elements directly, and picking apart two-tuples, is so long-winded at
2644 * the C level, though, that attempts to combine these routines were too
2645 * ugly to bear.
2646 */
2647static int
2648batch_dict(PicklerObject *self, PyObject *iter)
2649{
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002650 PyObject *obj = NULL;
2651 PyObject *firstitem = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002652 int i, n;
2653
2654 const char mark_op = MARK;
2655 const char setitem_op = SETITEM;
2656 const char setitems_op = SETITEMS;
2657
2658 assert(iter != NULL);
2659
2660 if (self->proto == 0) {
2661 /* SETITEMS isn't available; do one at a time. */
2662 for (;;) {
2663 obj = PyIter_Next(iter);
2664 if (obj == NULL) {
2665 if (PyErr_Occurred())
2666 return -1;
2667 break;
2668 }
2669 if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
2670 PyErr_SetString(PyExc_TypeError, "dict items "
2671 "iterator must return 2-tuples");
2672 return -1;
2673 }
2674 i = save(self, PyTuple_GET_ITEM(obj, 0), 0);
2675 if (i >= 0)
2676 i = save(self, PyTuple_GET_ITEM(obj, 1), 0);
2677 Py_DECREF(obj);
2678 if (i < 0)
2679 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002680 if (_Pickler_Write(self, &setitem_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002681 return -1;
2682 }
2683 return 0;
2684 }
2685
2686 /* proto > 0: write in batches of BATCHSIZE. */
2687 do {
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002688 /* Get first item */
2689 firstitem = PyIter_Next(iter);
2690 if (firstitem == NULL) {
2691 if (PyErr_Occurred())
2692 goto error;
2693
2694 /* nothing more to add */
2695 break;
2696 }
2697 if (!PyTuple_Check(firstitem) || PyTuple_Size(firstitem) != 2) {
2698 PyErr_SetString(PyExc_TypeError, "dict items "
2699 "iterator must return 2-tuples");
2700 goto error;
2701 }
2702
2703 /* Try to get a second item */
2704 obj = PyIter_Next(iter);
2705 if (obj == NULL) {
2706 if (PyErr_Occurred())
2707 goto error;
2708
2709 /* Only one item to write */
2710 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
2711 goto error;
2712 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
2713 goto error;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002714 if (_Pickler_Write(self, &setitem_op, 1) < 0)
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002715 goto error;
2716 Py_CLEAR(firstitem);
2717 break;
2718 }
2719
2720 /* More than one item to write */
2721
2722 /* Pump out MARK, items, SETITEMS. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002723 if (_Pickler_Write(self, &mark_op, 1) < 0)
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002724 goto error;
2725
2726 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
2727 goto error;
2728 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
2729 goto error;
2730 Py_CLEAR(firstitem);
2731 n = 1;
2732
2733 /* Fetch and save up to BATCHSIZE items */
2734 while (obj) {
2735 if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
2736 PyErr_SetString(PyExc_TypeError, "dict items "
2737 "iterator must return 2-tuples");
2738 goto error;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002739 }
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002740 if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 ||
2741 save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0)
2742 goto error;
2743 Py_CLEAR(obj);
2744 n += 1;
2745
2746 if (n == BATCHSIZE)
2747 break;
2748
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002749 obj = PyIter_Next(iter);
2750 if (obj == NULL) {
2751 if (PyErr_Occurred())
2752 goto error;
2753 break;
2754 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002755 }
2756
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002757 if (_Pickler_Write(self, &setitems_op, 1) < 0)
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002758 goto error;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002759
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002760 } while (n == BATCHSIZE);
2761 return 0;
2762
2763 error:
Amaury Forgeot d'Arcfb1a5eb2008-09-11 21:03:37 +00002764 Py_XDECREF(firstitem);
2765 Py_XDECREF(obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002766 return -1;
2767}
2768
Collin Winter5c9b02d2009-05-25 05:43:30 +00002769/* This is a variant of batch_dict() above that specializes for dicts, with no
2770 * support for dict subclasses. Like batch_dict(), we batch up chunks of
2771 * MARK key value ... key value SETITEMS
2772 * opcode sequences. Calling code should have arranged to first create an
2773 * empty dict, or dict-like object, for the SETITEMS to operate on.
2774 * Returns 0 on success, -1 on error.
2775 *
2776 * Note that this currently doesn't work for protocol 0.
2777 */
2778static int
2779batch_dict_exact(PicklerObject *self, PyObject *obj)
2780{
2781 PyObject *key = NULL, *value = NULL;
2782 int i;
2783 Py_ssize_t dict_size, ppos = 0;
2784
Alexandre Vassalottif70b1292009-05-25 18:00:52 +00002785 const char mark_op = MARK;
2786 const char setitem_op = SETITEM;
2787 const char setitems_op = SETITEMS;
Collin Winter5c9b02d2009-05-25 05:43:30 +00002788
2789 assert(obj != NULL);
2790 assert(self->proto > 0);
2791
2792 dict_size = PyDict_Size(obj);
2793
2794 /* Special-case len(d) == 1 to save space. */
2795 if (dict_size == 1) {
2796 PyDict_Next(obj, &ppos, &key, &value);
2797 if (save(self, key, 0) < 0)
2798 return -1;
2799 if (save(self, value, 0) < 0)
2800 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002801 if (_Pickler_Write(self, &setitem_op, 1) < 0)
Collin Winter5c9b02d2009-05-25 05:43:30 +00002802 return -1;
2803 return 0;
2804 }
2805
2806 /* Write in batches of BATCHSIZE. */
2807 do {
2808 i = 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002809 if (_Pickler_Write(self, &mark_op, 1) < 0)
Collin Winter5c9b02d2009-05-25 05:43:30 +00002810 return -1;
2811 while (PyDict_Next(obj, &ppos, &key, &value)) {
2812 if (save(self, key, 0) < 0)
2813 return -1;
2814 if (save(self, value, 0) < 0)
2815 return -1;
2816 if (++i == BATCHSIZE)
2817 break;
2818 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002819 if (_Pickler_Write(self, &setitems_op, 1) < 0)
Collin Winter5c9b02d2009-05-25 05:43:30 +00002820 return -1;
2821 if (PyDict_Size(obj) != dict_size) {
2822 PyErr_Format(
2823 PyExc_RuntimeError,
2824 "dictionary changed size during iteration");
2825 return -1;
2826 }
2827
2828 } while (i == BATCHSIZE);
2829 return 0;
2830}
2831
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002832static int
2833save_dict(PicklerObject *self, PyObject *obj)
2834{
2835 PyObject *items, *iter;
2836 char header[3];
Antoine Pitrou82be19f2011-08-29 23:09:33 +02002837 Py_ssize_t len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002838 int status = 0;
2839
2840 if (self->fast && !fast_save_enter(self, obj))
2841 goto error;
2842
2843 /* Create an empty dict. */
2844 if (self->bin) {
2845 header[0] = EMPTY_DICT;
2846 len = 1;
2847 }
2848 else {
2849 header[0] = MARK;
2850 header[1] = DICT;
2851 len = 2;
2852 }
2853
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00002854 if (_Pickler_Write(self, header, len) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002855 goto error;
2856
2857 /* Get dict size, and bow out early if empty. */
2858 if ((len = PyDict_Size(obj)) < 0)
2859 goto error;
2860
2861 if (memo_put(self, obj) < 0)
2862 goto error;
2863
2864 if (len != 0) {
2865 /* Save the dict items. */
Collin Winter5c9b02d2009-05-25 05:43:30 +00002866 if (PyDict_CheckExact(obj) && self->proto > 0) {
2867 /* We can take certain shortcuts if we know this is a dict and
2868 not a dict subclass. */
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00002869 if (Py_EnterRecursiveCall(" while pickling an object"))
2870 goto error;
2871 status = batch_dict_exact(self, obj);
2872 Py_LeaveRecursiveCall();
Collin Winter5c9b02d2009-05-25 05:43:30 +00002873 } else {
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02002874 _Py_IDENTIFIER(items);
Martin v. Löwisafe55bb2011-10-09 10:38:36 +02002875
2876 items = _PyObject_CallMethodId(obj, &PyId_items, "()");
Collin Winter5c9b02d2009-05-25 05:43:30 +00002877 if (items == NULL)
2878 goto error;
2879 iter = PyObject_GetIter(items);
2880 Py_DECREF(items);
2881 if (iter == NULL)
2882 goto error;
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00002883 if (Py_EnterRecursiveCall(" while pickling an object")) {
2884 Py_DECREF(iter);
2885 goto error;
2886 }
Collin Winter5c9b02d2009-05-25 05:43:30 +00002887 status = batch_dict(self, iter);
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00002888 Py_LeaveRecursiveCall();
Collin Winter5c9b02d2009-05-25 05:43:30 +00002889 Py_DECREF(iter);
2890 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00002891 }
2892
2893 if (0) {
2894 error:
2895 status = -1;
2896 }
2897
2898 if (self->fast && !fast_save_leave(self, obj))
2899 status = -1;
2900
2901 return status;
2902}
2903
2904static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01002905save_set(PicklerObject *self, PyObject *obj)
2906{
2907 PyObject *item;
2908 int i;
2909 Py_ssize_t set_size, ppos = 0;
2910 Py_hash_t hash;
2911
2912 const char empty_set_op = EMPTY_SET;
2913 const char mark_op = MARK;
2914 const char additems_op = ADDITEMS;
2915
2916 if (self->proto < 4) {
2917 PyObject *items;
2918 PyObject *reduce_value;
2919 int status;
2920
2921 items = PySequence_List(obj);
2922 if (items == NULL) {
2923 return -1;
2924 }
2925 reduce_value = Py_BuildValue("(O(O))", (PyObject*)&PySet_Type, items);
2926 Py_DECREF(items);
2927 if (reduce_value == NULL) {
2928 return -1;
2929 }
2930 /* save_reduce() will memoize the object automatically. */
2931 status = save_reduce(self, reduce_value, obj);
2932 Py_DECREF(reduce_value);
2933 return status;
2934 }
2935
2936 if (_Pickler_Write(self, &empty_set_op, 1) < 0)
2937 return -1;
2938
2939 if (memo_put(self, obj) < 0)
2940 return -1;
2941
2942 set_size = PySet_GET_SIZE(obj);
2943 if (set_size == 0)
2944 return 0; /* nothing to do */
2945
2946 /* Write in batches of BATCHSIZE. */
2947 do {
2948 i = 0;
2949 if (_Pickler_Write(self, &mark_op, 1) < 0)
2950 return -1;
2951 while (_PySet_NextEntry(obj, &ppos, &item, &hash)) {
2952 if (save(self, item, 0) < 0)
2953 return -1;
2954 if (++i == BATCHSIZE)
2955 break;
2956 }
2957 if (_Pickler_Write(self, &additems_op, 1) < 0)
2958 return -1;
2959 if (PySet_GET_SIZE(obj) != set_size) {
2960 PyErr_Format(
2961 PyExc_RuntimeError,
2962 "set changed size during iteration");
2963 return -1;
2964 }
2965 } while (i == BATCHSIZE);
2966
2967 return 0;
2968}
2969
2970static int
2971save_frozenset(PicklerObject *self, PyObject *obj)
2972{
2973 PyObject *iter;
2974
2975 const char mark_op = MARK;
2976 const char frozenset_op = FROZENSET;
2977
2978 if (self->fast && !fast_save_enter(self, obj))
2979 return -1;
2980
2981 if (self->proto < 4) {
2982 PyObject *items;
2983 PyObject *reduce_value;
2984 int status;
2985
2986 items = PySequence_List(obj);
2987 if (items == NULL) {
2988 return -1;
2989 }
2990 reduce_value = Py_BuildValue("(O(O))", (PyObject*)&PyFrozenSet_Type,
2991 items);
2992 Py_DECREF(items);
2993 if (reduce_value == NULL) {
2994 return -1;
2995 }
2996 /* save_reduce() will memoize the object automatically. */
2997 status = save_reduce(self, reduce_value, obj);
2998 Py_DECREF(reduce_value);
2999 return status;
3000 }
3001
3002 if (_Pickler_Write(self, &mark_op, 1) < 0)
3003 return -1;
3004
3005 iter = PyObject_GetIter(obj);
Christian Heimesb3d3ee42013-11-23 21:01:40 +01003006 if (iter == NULL) {
Christian Heimes74d8d632013-11-23 21:05:31 +01003007 return -1;
Christian Heimesb3d3ee42013-11-23 21:01:40 +01003008 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003009 for (;;) {
3010 PyObject *item;
3011
3012 item = PyIter_Next(iter);
3013 if (item == NULL) {
3014 if (PyErr_Occurred()) {
3015 Py_DECREF(iter);
3016 return -1;
3017 }
3018 break;
3019 }
3020 if (save(self, item, 0) < 0) {
3021 Py_DECREF(item);
3022 Py_DECREF(iter);
3023 return -1;
3024 }
3025 Py_DECREF(item);
3026 }
3027 Py_DECREF(iter);
3028
3029 /* If the object is already in the memo, this means it is
3030 recursive. In this case, throw away everything we put on the
3031 stack, and fetch the object back from the memo. */
3032 if (PyMemoTable_Get(self->memo, obj)) {
3033 const char pop_mark_op = POP_MARK;
3034
3035 if (_Pickler_Write(self, &pop_mark_op, 1) < 0)
3036 return -1;
3037 if (memo_get(self, obj) < 0)
3038 return -1;
3039 return 0;
3040 }
3041
3042 if (_Pickler_Write(self, &frozenset_op, 1) < 0)
3043 return -1;
3044 if (memo_put(self, obj) < 0)
3045 return -1;
3046
3047 return 0;
3048}
3049
3050static int
3051fix_imports(PyObject **module_name, PyObject **global_name)
3052{
3053 PyObject *key;
3054 PyObject *item;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003055 PickleState *st = _Pickle_GetGlobalState();
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003056
3057 key = PyTuple_Pack(2, *module_name, *global_name);
3058 if (key == NULL)
3059 return -1;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003060 item = PyDict_GetItemWithError(st->name_mapping_3to2, key);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003061 Py_DECREF(key);
3062 if (item) {
3063 PyObject *fixed_module_name;
3064 PyObject *fixed_global_name;
3065
3066 if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
3067 PyErr_Format(PyExc_RuntimeError,
3068 "_compat_pickle.REVERSE_NAME_MAPPING values "
3069 "should be 2-tuples, not %.200s",
3070 Py_TYPE(item)->tp_name);
3071 return -1;
3072 }
3073 fixed_module_name = PyTuple_GET_ITEM(item, 0);
3074 fixed_global_name = PyTuple_GET_ITEM(item, 1);
3075 if (!PyUnicode_Check(fixed_module_name) ||
3076 !PyUnicode_Check(fixed_global_name)) {
3077 PyErr_Format(PyExc_RuntimeError,
3078 "_compat_pickle.REVERSE_NAME_MAPPING values "
3079 "should be pairs of str, not (%.200s, %.200s)",
3080 Py_TYPE(fixed_module_name)->tp_name,
3081 Py_TYPE(fixed_global_name)->tp_name);
3082 return -1;
3083 }
3084
3085 Py_CLEAR(*module_name);
3086 Py_CLEAR(*global_name);
3087 Py_INCREF(fixed_module_name);
3088 Py_INCREF(fixed_global_name);
3089 *module_name = fixed_module_name;
3090 *global_name = fixed_global_name;
Serhiy Storchakabfe18242015-03-31 13:12:37 +03003091 return 0;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003092 }
3093 else if (PyErr_Occurred()) {
3094 return -1;
3095 }
3096
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003097 item = PyDict_GetItemWithError(st->import_mapping_3to2, *module_name);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003098 if (item) {
3099 if (!PyUnicode_Check(item)) {
3100 PyErr_Format(PyExc_RuntimeError,
3101 "_compat_pickle.REVERSE_IMPORT_MAPPING values "
3102 "should be strings, not %.200s",
3103 Py_TYPE(item)->tp_name);
3104 return -1;
3105 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003106 Py_INCREF(item);
Serhiy Storchaka48842712016-04-06 09:45:48 +03003107 Py_XSETREF(*module_name, item);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003108 }
3109 else if (PyErr_Occurred()) {
3110 return -1;
3111 }
3112
3113 return 0;
3114}
3115
3116static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003117save_global(PicklerObject *self, PyObject *obj, PyObject *name)
3118{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003119 PyObject *global_name = NULL;
3120 PyObject *module_name = NULL;
3121 PyObject *module = NULL;
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003122 PyObject *parent = NULL;
3123 PyObject *dotted_path = NULL;
3124 PyObject *lastname = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003125 PyObject *cls;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003126 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003127 int status = 0;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003128 _Py_IDENTIFIER(__name__);
3129 _Py_IDENTIFIER(__qualname__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003130
3131 const char global_op = GLOBAL;
3132
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003133 if (name) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003134 Py_INCREF(name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003135 global_name = name;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003136 }
3137 else {
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003138 global_name = _PyObject_GetAttrId(obj, &PyId___qualname__);
3139 if (global_name == NULL) {
3140 if (!PyErr_ExceptionMatches(PyExc_AttributeError))
3141 goto error;
3142 PyErr_Clear();
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003143 }
3144 if (global_name == NULL) {
3145 global_name = _PyObject_GetAttrId(obj, &PyId___name__);
3146 if (global_name == NULL)
3147 goto error;
3148 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003149 }
3150
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003151 dotted_path = get_dotted_path(module, global_name);
3152 if (dotted_path == NULL)
3153 goto error;
3154 module_name = whichmodule(obj, dotted_path);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003155 if (module_name == NULL)
3156 goto error;
3157
3158 /* XXX: Change to use the import C API directly with level=0 to disallow
3159 relative imports.
3160
3161 XXX: PyImport_ImportModuleLevel could be used. However, this bypasses
3162 builtins.__import__. Therefore, _pickle, unlike pickle.py, will ignore
3163 custom import functions (IMHO, this would be a nice security
3164 feature). The import C API would need to be extended to support the
3165 extra parameters of __import__ to fix that. */
3166 module = PyImport_Import(module_name);
3167 if (module == NULL) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003168 PyErr_Format(st->PicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003169 "Can't pickle %R: import of module %R failed",
3170 obj, module_name);
3171 goto error;
3172 }
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003173 lastname = PyList_GET_ITEM(dotted_path, PyList_GET_SIZE(dotted_path)-1);
3174 Py_INCREF(lastname);
3175 cls = get_deep_attribute(module, dotted_path, &parent);
3176 Py_CLEAR(dotted_path);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003177 if (cls == NULL) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003178 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003179 "Can't pickle %R: attribute lookup %S on %S failed",
3180 obj, global_name, module_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003181 goto error;
3182 }
3183 if (cls != obj) {
3184 Py_DECREF(cls);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003185 PyErr_Format(st->PicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003186 "Can't pickle %R: it's not the same object as %S.%S",
3187 obj, module_name, global_name);
3188 goto error;
3189 }
3190 Py_DECREF(cls);
3191
3192 if (self->proto >= 2) {
3193 /* See whether this is in the extension registry, and if
3194 * so generate an EXT opcode.
3195 */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003196 PyObject *extension_key;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003197 PyObject *code_obj; /* extension code as Python object */
3198 long code; /* extension code as C value */
3199 char pdata[5];
Antoine Pitrou82be19f2011-08-29 23:09:33 +02003200 Py_ssize_t n;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003201
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003202 extension_key = PyTuple_Pack(2, module_name, global_name);
3203 if (extension_key == NULL) {
3204 goto error;
3205 }
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08003206 code_obj = PyDict_GetItemWithError(st->extension_registry,
3207 extension_key);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003208 Py_DECREF(extension_key);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003209 /* The object is not registered in the extension registry.
3210 This is the most likely code path. */
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08003211 if (code_obj == NULL) {
3212 if (PyErr_Occurred()) {
3213 goto error;
3214 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003215 goto gen_global;
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08003216 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003217
3218 /* XXX: pickle.py doesn't check neither the type, nor the range
3219 of the value returned by the extension_registry. It should for
3220 consistency. */
3221
3222 /* Verify code_obj has the right type and value. */
3223 if (!PyLong_Check(code_obj)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003224 PyErr_Format(st->PicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003225 "Can't pickle %R: extension code %R isn't an integer",
3226 obj, code_obj);
3227 goto error;
3228 }
3229 code = PyLong_AS_LONG(code_obj);
3230 if (code <= 0 || code > 0x7fffffffL) {
Antoine Pitrou82be19f2011-08-29 23:09:33 +02003231 if (!PyErr_Occurred())
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003232 PyErr_Format(st->PicklingError, "Can't pickle %R: extension "
3233 "code %ld is out of range", obj, code);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003234 goto error;
3235 }
3236
3237 /* Generate an EXT opcode. */
3238 if (code <= 0xff) {
3239 pdata[0] = EXT1;
3240 pdata[1] = (unsigned char)code;
3241 n = 2;
3242 }
3243 else if (code <= 0xffff) {
3244 pdata[0] = EXT2;
3245 pdata[1] = (unsigned char)(code & 0xff);
3246 pdata[2] = (unsigned char)((code >> 8) & 0xff);
3247 n = 3;
3248 }
3249 else {
3250 pdata[0] = EXT4;
3251 pdata[1] = (unsigned char)(code & 0xff);
3252 pdata[2] = (unsigned char)((code >> 8) & 0xff);
3253 pdata[3] = (unsigned char)((code >> 16) & 0xff);
3254 pdata[4] = (unsigned char)((code >> 24) & 0xff);
3255 n = 5;
3256 }
3257
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003258 if (_Pickler_Write(self, pdata, n) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003259 goto error;
3260 }
3261 else {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003262 gen_global:
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003263 if (parent == module) {
3264 Py_INCREF(lastname);
3265 Py_DECREF(global_name);
3266 global_name = lastname;
3267 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003268 if (self->proto >= 4) {
3269 const char stack_global_op = STACK_GLOBAL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003270
Christian Heimese8b1ba12013-11-23 21:13:39 +01003271 if (save(self, module_name, 0) < 0)
3272 goto error;
3273 if (save(self, global_name, 0) < 0)
3274 goto error;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003275
3276 if (_Pickler_Write(self, &stack_global_op, 1) < 0)
3277 goto error;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003278 }
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003279 else if (parent != module) {
3280 PickleState *st = _Pickle_GetGlobalState();
3281 PyObject *reduce_value = Py_BuildValue("(O(OO))",
3282 st->getattr, parent, lastname);
3283 status = save_reduce(self, reduce_value, NULL);
3284 Py_DECREF(reduce_value);
3285 if (status < 0)
3286 goto error;
3287 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003288 else {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003289 /* Generate a normal global opcode if we are using a pickle
3290 protocol < 4, or if the object is not registered in the
3291 extension registry. */
3292 PyObject *encoded;
3293 PyObject *(*unicode_encoder)(PyObject *);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003294
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003295 if (_Pickler_Write(self, &global_op, 1) < 0)
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00003296 goto error;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003297
3298 /* For protocol < 3 and if the user didn't request against doing
3299 so, we convert module names to the old 2.x module names. */
3300 if (self->proto < 3 && self->fix_imports) {
3301 if (fix_imports(&module_name, &global_name) < 0) {
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00003302 goto error;
3303 }
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00003304 }
3305
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003306 /* Since Python 3.0 now supports non-ASCII identifiers, we encode
3307 both the module name and the global name using UTF-8. We do so
3308 only when we are using the pickle protocol newer than version
3309 3. This is to ensure compatibility with older Unpickler running
3310 on Python 2.x. */
3311 if (self->proto == 3) {
3312 unicode_encoder = PyUnicode_AsUTF8String;
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00003313 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003314 else {
3315 unicode_encoder = PyUnicode_AsASCIIString;
3316 }
3317 encoded = unicode_encoder(module_name);
3318 if (encoded == NULL) {
3319 if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003320 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003321 "can't pickle module identifier '%S' using "
3322 "pickle protocol %i",
3323 module_name, self->proto);
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00003324 goto error;
3325 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003326 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded),
3327 PyBytes_GET_SIZE(encoded)) < 0) {
3328 Py_DECREF(encoded);
3329 goto error;
3330 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003331 Py_DECREF(encoded);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003332 if(_Pickler_Write(self, "\n", 1) < 0)
3333 goto error;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003334
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003335 /* Save the name of the module. */
3336 encoded = unicode_encoder(global_name);
3337 if (encoded == NULL) {
3338 if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003339 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003340 "can't pickle global identifier '%S' using "
3341 "pickle protocol %i",
3342 global_name, self->proto);
3343 goto error;
3344 }
3345 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded),
3346 PyBytes_GET_SIZE(encoded)) < 0) {
3347 Py_DECREF(encoded);
3348 goto error;
3349 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003350 Py_DECREF(encoded);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003351 if (_Pickler_Write(self, "\n", 1) < 0)
3352 goto error;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003353 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003354 /* Memoize the object. */
3355 if (memo_put(self, obj) < 0)
3356 goto error;
3357 }
3358
3359 if (0) {
3360 error:
3361 status = -1;
3362 }
3363 Py_XDECREF(module_name);
3364 Py_XDECREF(global_name);
3365 Py_XDECREF(module);
Serhiy Storchaka58e41342015-03-31 14:07:24 +03003366 Py_XDECREF(parent);
3367 Py_XDECREF(dotted_path);
3368 Py_XDECREF(lastname);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003369
3370 return status;
3371}
3372
3373static int
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -08003374save_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton)
3375{
3376 PyObject *reduce_value;
3377 int status;
3378
3379 reduce_value = Py_BuildValue("O(O)", &PyType_Type, singleton);
3380 if (reduce_value == NULL) {
3381 return -1;
3382 }
3383 status = save_reduce(self, reduce_value, obj);
3384 Py_DECREF(reduce_value);
3385 return status;
3386}
3387
3388static int
3389save_type(PicklerObject *self, PyObject *obj)
3390{
Alexandre Vassalotti65846c62013-11-30 17:55:48 -08003391 if (obj == (PyObject *)&_PyNone_Type) {
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -08003392 return save_singleton_type(self, obj, Py_None);
3393 }
3394 else if (obj == (PyObject *)&PyEllipsis_Type) {
3395 return save_singleton_type(self, obj, Py_Ellipsis);
3396 }
Alexandre Vassalotti65846c62013-11-30 17:55:48 -08003397 else if (obj == (PyObject *)&_PyNotImplemented_Type) {
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -08003398 return save_singleton_type(self, obj, Py_NotImplemented);
3399 }
3400 return save_global(self, obj, NULL);
3401}
3402
3403static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003404save_pers(PicklerObject *self, PyObject *obj, PyObject *func)
3405{
3406 PyObject *pid = NULL;
3407 int status = 0;
3408
3409 const char persid_op = PERSID;
3410 const char binpersid_op = BINPERSID;
3411
3412 Py_INCREF(obj);
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08003413 pid = _Pickle_FastCall(func, obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003414 if (pid == NULL)
3415 return -1;
3416
3417 if (pid != Py_None) {
3418 if (self->bin) {
3419 if (save(self, pid, 1) < 0 ||
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003420 _Pickler_Write(self, &binpersid_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003421 goto error;
3422 }
3423 else {
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003424 PyObject *pid_str;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003425
3426 pid_str = PyObject_Str(pid);
3427 if (pid_str == NULL)
3428 goto error;
3429
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003430 /* XXX: Should it check whether the pid contains embedded
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003431 newlines? */
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003432 if (!PyUnicode_IS_ASCII(pid_str)) {
3433 PyErr_SetString(_Pickle_GetGlobalState()->PicklingError,
3434 "persistent IDs in protocol 0 must be "
3435 "ASCII strings");
3436 Py_DECREF(pid_str);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003437 goto error;
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003438 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003439
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003440 if (_Pickler_Write(self, &persid_op, 1) < 0 ||
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003441 _Pickler_Write(self, PyUnicode_DATA(pid_str),
3442 PyUnicode_GET_LENGTH(pid_str)) < 0 ||
3443 _Pickler_Write(self, "\n", 1) < 0) {
3444 Py_DECREF(pid_str);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003445 goto error;
Serhiy Storchakadec25af2016-07-17 11:24:17 +03003446 }
3447 Py_DECREF(pid_str);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003448 }
3449 status = 1;
3450 }
3451
3452 if (0) {
3453 error:
3454 status = -1;
3455 }
3456 Py_XDECREF(pid);
3457
3458 return status;
3459}
3460
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003461static PyObject *
3462get_class(PyObject *obj)
3463{
3464 PyObject *cls;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003465 _Py_IDENTIFIER(__class__);
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003466
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003467 cls = _PyObject_GetAttrId(obj, &PyId___class__);
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003468 if (cls == NULL) {
3469 if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
3470 PyErr_Clear();
3471 cls = (PyObject *) Py_TYPE(obj);
3472 Py_INCREF(cls);
3473 }
3474 }
3475 return cls;
3476}
3477
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003478/* We're saving obj, and args is the 2-thru-5 tuple returned by the
3479 * appropriate __reduce__ method for obj.
3480 */
3481static int
3482save_reduce(PicklerObject *self, PyObject *args, PyObject *obj)
3483{
3484 PyObject *callable;
3485 PyObject *argtup;
3486 PyObject *state = NULL;
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003487 PyObject *listitems = Py_None;
3488 PyObject *dictitems = Py_None;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003489 PickleState *st = _Pickle_GetGlobalState();
Hirokazu Yamamotob46a6332008-11-04 00:35:10 +00003490 Py_ssize_t size;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003491 int use_newobj = 0, use_newobj_ex = 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003492
3493 const char reduce_op = REDUCE;
3494 const char build_op = BUILD;
3495 const char newobj_op = NEWOBJ;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003496 const char newobj_ex_op = NEWOBJ_EX;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003497
Hirokazu Yamamotob46a6332008-11-04 00:35:10 +00003498 size = PyTuple_Size(args);
3499 if (size < 2 || size > 5) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003500 PyErr_SetString(st->PicklingError, "tuple returned by "
Hirokazu Yamamotob46a6332008-11-04 00:35:10 +00003501 "__reduce__ must contain 2 through 5 elements");
3502 return -1;
3503 }
3504
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003505 if (!PyArg_UnpackTuple(args, "save_reduce", 2, 5,
3506 &callable, &argtup, &state, &listitems, &dictitems))
3507 return -1;
3508
3509 if (!PyCallable_Check(callable)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003510 PyErr_SetString(st->PicklingError, "first item of the tuple "
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003511 "returned by __reduce__ must be callable");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003512 return -1;
3513 }
3514 if (!PyTuple_Check(argtup)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003515 PyErr_SetString(st->PicklingError, "second item of the tuple "
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003516 "returned by __reduce__ must be a tuple");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003517 return -1;
3518 }
3519
3520 if (state == Py_None)
3521 state = NULL;
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003522
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003523 if (listitems == Py_None)
3524 listitems = NULL;
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003525 else if (!PyIter_Check(listitems)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003526 PyErr_Format(st->PicklingError, "fourth element of the tuple "
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003527 "returned by __reduce__ must be an iterator, not %s",
3528 Py_TYPE(listitems)->tp_name);
3529 return -1;
3530 }
3531
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003532 if (dictitems == Py_None)
3533 dictitems = NULL;
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003534 else if (!PyIter_Check(dictitems)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003535 PyErr_Format(st->PicklingError, "fifth element of the tuple "
Amaury Forgeot d'Arc424b4812008-10-30 22:25:31 +00003536 "returned by __reduce__ must be an iterator, not %s",
3537 Py_TYPE(dictitems)->tp_name);
3538 return -1;
3539 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003540
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003541 if (self->proto >= 2) {
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003542 PyObject *name;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003543 _Py_IDENTIFIER(__name__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003544
Victor Stinner804e05e2013-11-14 01:26:17 +01003545 name = _PyObject_GetAttrId(callable, &PyId___name__);
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003546 if (name == NULL) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003547 if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003548 return -1;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003549 }
3550 PyErr_Clear();
3551 }
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02003552 else if (PyUnicode_Check(name)) {
Serhiy Storchaka0d554d72015-10-10 22:42:18 +03003553 _Py_IDENTIFIER(__newobj_ex__);
3554 use_newobj_ex = PyUnicode_Compare(
3555 name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0;
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02003556 if (!use_newobj_ex) {
3557 _Py_IDENTIFIER(__newobj__);
3558 use_newobj = PyUnicode_Compare(
3559 name, _PyUnicode_FromId(&PyId___newobj__)) == 0;
3560 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003561 }
Serhiy Storchaka707b5cc2014-12-16 19:43:46 +02003562 Py_XDECREF(name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003563 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003564
3565 if (use_newobj_ex) {
3566 PyObject *cls;
3567 PyObject *args;
3568 PyObject *kwargs;
3569
3570 if (Py_SIZE(argtup) != 3) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003571 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003572 "length of the NEWOBJ_EX argument tuple must be "
3573 "exactly 3, not %zd", Py_SIZE(argtup));
3574 return -1;
3575 }
3576
3577 cls = PyTuple_GET_ITEM(argtup, 0);
3578 if (!PyType_Check(cls)) {
Larry Hastings61272b72014-01-07 12:41:53 -08003579 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003580 "first item from NEWOBJ_EX argument tuple must "
3581 "be a class, not %.200s", Py_TYPE(cls)->tp_name);
3582 return -1;
3583 }
3584 args = PyTuple_GET_ITEM(argtup, 1);
3585 if (!PyTuple_Check(args)) {
Larry Hastings61272b72014-01-07 12:41:53 -08003586 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003587 "second item from NEWOBJ_EX argument tuple must "
3588 "be a tuple, not %.200s", Py_TYPE(args)->tp_name);
3589 return -1;
3590 }
3591 kwargs = PyTuple_GET_ITEM(argtup, 2);
3592 if (!PyDict_Check(kwargs)) {
Larry Hastings61272b72014-01-07 12:41:53 -08003593 PyErr_Format(st->PicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003594 "third item from NEWOBJ_EX argument tuple must "
3595 "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name);
3596 return -1;
3597 }
3598
Serhiy Storchaka0d554d72015-10-10 22:42:18 +03003599 if (self->proto >= 4) {
3600 if (save(self, cls, 0) < 0 ||
3601 save(self, args, 0) < 0 ||
3602 save(self, kwargs, 0) < 0 ||
3603 _Pickler_Write(self, &newobj_ex_op, 1) < 0) {
3604 return -1;
3605 }
3606 }
3607 else {
3608 PyObject *newargs;
3609 PyObject *cls_new;
3610 Py_ssize_t i;
3611 _Py_IDENTIFIER(__new__);
3612
3613 newargs = PyTuple_New(Py_SIZE(args) + 2);
3614 if (newargs == NULL)
3615 return -1;
3616
3617 cls_new = _PyObject_GetAttrId(cls, &PyId___new__);
3618 if (cls_new == NULL) {
3619 Py_DECREF(newargs);
3620 return -1;
3621 }
3622 PyTuple_SET_ITEM(newargs, 0, cls_new);
3623 Py_INCREF(cls);
3624 PyTuple_SET_ITEM(newargs, 1, cls);
3625 for (i = 0; i < Py_SIZE(args); i++) {
3626 PyObject *item = PyTuple_GET_ITEM(args, i);
3627 Py_INCREF(item);
3628 PyTuple_SET_ITEM(newargs, i + 2, item);
3629 }
3630
3631 callable = PyObject_Call(st->partial, newargs, kwargs);
3632 Py_DECREF(newargs);
3633 if (callable == NULL)
3634 return -1;
3635
3636 newargs = PyTuple_New(0);
3637 if (newargs == NULL) {
3638 Py_DECREF(callable);
3639 return -1;
3640 }
3641
3642 if (save(self, callable, 0) < 0 ||
3643 save(self, newargs, 0) < 0 ||
3644 _Pickler_Write(self, &reduce_op, 1) < 0) {
3645 Py_DECREF(newargs);
3646 Py_DECREF(callable);
3647 return -1;
3648 }
3649 Py_DECREF(newargs);
3650 Py_DECREF(callable);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003651 }
3652 }
3653 else if (use_newobj) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003654 PyObject *cls;
3655 PyObject *newargtup;
3656 PyObject *obj_class;
3657 int p;
3658
3659 /* Sanity checks. */
3660 if (Py_SIZE(argtup) < 1) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003661 PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003662 return -1;
3663 }
3664
3665 cls = PyTuple_GET_ITEM(argtup, 0);
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003666 if (!PyType_Check(cls)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003667 PyErr_SetString(st->PicklingError, "args[0] from "
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003668 "__newobj__ args is not a type");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003669 return -1;
3670 }
3671
3672 if (obj != NULL) {
Antoine Pitrou16c4ce12011-03-11 21:30:43 +01003673 obj_class = get_class(obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003674 p = obj_class != cls; /* true iff a problem */
3675 Py_DECREF(obj_class);
3676 if (p) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003677 PyErr_SetString(st->PicklingError, "args[0] from "
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003678 "__newobj__ args has the wrong class");
3679 return -1;
3680 }
3681 }
3682 /* XXX: These calls save() are prone to infinite recursion. Imagine
3683 what happen if the value returned by the __reduce__() method of
3684 some extension type contains another object of the same type. Ouch!
3685
3686 Here is a quick example, that I ran into, to illustrate what I
3687 mean:
3688
3689 >>> import pickle, copyreg
3690 >>> copyreg.dispatch_table.pop(complex)
3691 >>> pickle.dumps(1+2j)
3692 Traceback (most recent call last):
3693 ...
Yury Selivanovf488fb42015-07-03 01:04:23 -04003694 RecursionError: maximum recursion depth exceeded
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003695
3696 Removing the complex class from copyreg.dispatch_table made the
3697 __reduce_ex__() method emit another complex object:
3698
3699 >>> (1+1j).__reduce_ex__(2)
3700 (<function __newobj__ at 0xb7b71c3c>,
3701 (<class 'complex'>, (1+1j)), None, None, None)
3702
3703 Thus when save() was called on newargstup (the 2nd item) recursion
3704 ensued. Of course, the bug was in the complex class which had a
3705 broken __getnewargs__() that emitted another complex object. But,
3706 the point, here, is it is quite easy to end up with a broken reduce
3707 function. */
3708
3709 /* Save the class and its __new__ arguments. */
3710 if (save(self, cls, 0) < 0)
3711 return -1;
3712
3713 newargtup = PyTuple_GetSlice(argtup, 1, Py_SIZE(argtup));
3714 if (newargtup == NULL)
3715 return -1;
3716
3717 p = save(self, newargtup, 0);
3718 Py_DECREF(newargtup);
3719 if (p < 0)
3720 return -1;
3721
3722 /* Add NEWOBJ opcode. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003723 if (_Pickler_Write(self, &newobj_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003724 return -1;
3725 }
3726 else { /* Not using NEWOBJ. */
3727 if (save(self, callable, 0) < 0 ||
3728 save(self, argtup, 0) < 0 ||
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003729 _Pickler_Write(self, &reduce_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003730 return -1;
3731 }
3732
3733 /* obj can be NULL when save_reduce() is used directly. A NULL obj means
3734 the caller do not want to memoize the object. Not particularly useful,
3735 but that is to mimic the behavior save_reduce() in pickle.py when
3736 obj is None. */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003737 if (obj != NULL) {
3738 /* If the object is already in the memo, this means it is
3739 recursive. In this case, throw away everything we put on the
3740 stack, and fetch the object back from the memo. */
3741 if (PyMemoTable_Get(self->memo, obj)) {
3742 const char pop_op = POP;
3743
3744 if (_Pickler_Write(self, &pop_op, 1) < 0)
3745 return -1;
3746 if (memo_get(self, obj) < 0)
3747 return -1;
3748
3749 return 0;
3750 }
3751 else if (memo_put(self, obj) < 0)
3752 return -1;
3753 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003754
3755 if (listitems && batch_list(self, listitems) < 0)
3756 return -1;
3757
3758 if (dictitems && batch_dict(self, dictitems) < 0)
3759 return -1;
3760
3761 if (state) {
Victor Stinner121aab42011-09-29 23:40:53 +02003762 if (save(self, state, 0) < 0 ||
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003763 _Pickler_Write(self, &build_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003764 return -1;
3765 }
3766
3767 return 0;
3768}
3769
3770static int
3771save(PicklerObject *self, PyObject *obj, int pers_save)
3772{
3773 PyTypeObject *type;
3774 PyObject *reduce_func = NULL;
3775 PyObject *reduce_value = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003776 int status = 0;
3777
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08003778 if (_Pickler_OpcodeBoundary(self) < 0)
3779 return -1;
3780
Antoine Pitroue6d4c5b2011-01-23 17:12:25 +00003781 if (Py_EnterRecursiveCall(" while pickling an object"))
Alexandre Vassalottidff18342008-07-13 18:48:30 +00003782 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003783
3784 /* The extra pers_save argument is necessary to avoid calling save_pers()
3785 on its returned object. */
3786 if (!pers_save && self->pers_func) {
3787 /* save_pers() returns:
3788 -1 to signal an error;
3789 0 if it did nothing successfully;
3790 1 if a persistent id was saved.
3791 */
3792 if ((status = save_pers(self, obj, self->pers_func)) != 0)
3793 goto done;
3794 }
3795
3796 type = Py_TYPE(obj);
3797
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003798 /* The old cPickle had an optimization that used switch-case statement
3799 dispatching on the first letter of the type name. This has was removed
3800 since benchmarks shown that this optimization was actually slowing
3801 things down. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003802
3803 /* Atom types; these aren't memoized, so don't check the memo. */
3804
3805 if (obj == Py_None) {
3806 status = save_none(self, obj);
3807 goto done;
3808 }
3809 else if (obj == Py_False || obj == Py_True) {
3810 status = save_bool(self, obj);
3811 goto done;
3812 }
3813 else if (type == &PyLong_Type) {
3814 status = save_long(self, obj);
3815 goto done;
3816 }
3817 else if (type == &PyFloat_Type) {
3818 status = save_float(self, obj);
3819 goto done;
3820 }
3821
3822 /* Check the memo to see if it has the object. If so, generate
3823 a GET (or BINGET) opcode, instead of pickling the object
3824 once again. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003825 if (PyMemoTable_Get(self->memo, obj)) {
3826 if (memo_get(self, obj) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003827 goto error;
3828 goto done;
3829 }
3830
3831 if (type == &PyBytes_Type) {
3832 status = save_bytes(self, obj);
3833 goto done;
3834 }
3835 else if (type == &PyUnicode_Type) {
3836 status = save_unicode(self, obj);
3837 goto done;
3838 }
3839 else if (type == &PyDict_Type) {
3840 status = save_dict(self, obj);
3841 goto done;
3842 }
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003843 else if (type == &PySet_Type) {
3844 status = save_set(self, obj);
3845 goto done;
3846 }
3847 else if (type == &PyFrozenSet_Type) {
3848 status = save_frozenset(self, obj);
3849 goto done;
3850 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003851 else if (type == &PyList_Type) {
3852 status = save_list(self, obj);
3853 goto done;
3854 }
3855 else if (type == &PyTuple_Type) {
3856 status = save_tuple(self, obj);
3857 goto done;
3858 }
3859 else if (type == &PyType_Type) {
Alexandre Vassalotti19b6fa62013-11-30 16:06:39 -08003860 status = save_type(self, obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003861 goto done;
3862 }
3863 else if (type == &PyFunction_Type) {
3864 status = save_global(self, obj, NULL);
Alexandre Vassalottifc912852013-11-24 03:07:35 -08003865 goto done;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003866 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003867
3868 /* XXX: This part needs some unit tests. */
3869
3870 /* Get a reduction callable, and call it. This may come from
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003871 * self.dispatch_table, copyreg.dispatch_table, the object's
3872 * __reduce_ex__ method, or the object's __reduce__ method.
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003873 */
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003874 if (self->dispatch_table == NULL) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003875 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08003876 reduce_func = PyDict_GetItemWithError(st->dispatch_table,
3877 (PyObject *)type);
3878 if (reduce_func == NULL) {
3879 if (PyErr_Occurred()) {
3880 goto error;
3881 }
3882 } else {
3883 /* PyDict_GetItemWithError() returns a borrowed reference.
3884 Increase the reference count to be consistent with
3885 PyObject_GetItem and _PyObject_GetAttrId used below. */
3886 Py_INCREF(reduce_func);
3887 }
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003888 } else {
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08003889 reduce_func = PyObject_GetItem(self->dispatch_table,
3890 (PyObject *)type);
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01003891 if (reduce_func == NULL) {
3892 if (PyErr_ExceptionMatches(PyExc_KeyError))
3893 PyErr_Clear();
3894 else
3895 goto error;
3896 }
3897 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003898 if (reduce_func != NULL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003899 Py_INCREF(obj);
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08003900 reduce_value = _Pickle_FastCall(reduce_func, obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003901 }
Antoine Pitrouffd41d92011-10-04 09:23:04 +02003902 else if (PyType_IsSubtype(type, &PyType_Type)) {
3903 status = save_global(self, obj, NULL);
3904 goto done;
3905 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003906 else {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003907 _Py_IDENTIFIER(__reduce__);
3908 _Py_IDENTIFIER(__reduce_ex__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003909
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003910
3911 /* XXX: If the __reduce__ method is defined, __reduce_ex__ is
3912 automatically defined as __reduce__. While this is convenient, this
3913 make it impossible to know which method was actually called. Of
3914 course, this is not a big deal. But still, it would be nice to let
3915 the user know which method was called when something go
3916 wrong. Incidentally, this means if __reduce_ex__ is not defined, we
3917 don't actually have to check for a __reduce__ method. */
3918
3919 /* Check for a __reduce_ex__ method. */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003920 reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce_ex__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003921 if (reduce_func != NULL) {
3922 PyObject *proto;
3923 proto = PyLong_FromLong(self->proto);
3924 if (proto != NULL) {
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08003925 reduce_value = _Pickle_FastCall(reduce_func, proto);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003926 }
3927 }
3928 else {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003929 PickleState *st = _Pickle_GetGlobalState();
3930
3931 if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003932 PyErr_Clear();
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003933 }
3934 else {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003935 goto error;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003936 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003937 /* Check for a __reduce__ method. */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003938 reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003939 if (reduce_func != NULL) {
Victor Stinner559bb6a2016-08-22 22:48:54 +02003940 reduce_value = _PyObject_CallNoArg(reduce_func);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003941 }
3942 else {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003943 PyErr_Format(st->PicklingError,
3944 "can't pickle '%.200s' object: %R",
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003945 type->tp_name, obj);
3946 goto error;
3947 }
3948 }
3949 }
3950
3951 if (reduce_value == NULL)
3952 goto error;
3953
3954 if (PyUnicode_Check(reduce_value)) {
3955 status = save_global(self, obj, reduce_value);
3956 goto done;
3957 }
3958
3959 if (!PyTuple_Check(reduce_value)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08003960 PickleState *st = _Pickle_GetGlobalState();
3961 PyErr_SetString(st->PicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003962 "__reduce__ must return a string or tuple");
3963 goto error;
3964 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003965
3966 status = save_reduce(self, reduce_value, obj);
3967
3968 if (0) {
3969 error:
3970 status = -1;
3971 }
3972 done:
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08003973
Alexandre Vassalottidff18342008-07-13 18:48:30 +00003974 Py_LeaveRecursiveCall();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003975 Py_XDECREF(reduce_func);
3976 Py_XDECREF(reduce_value);
3977
3978 return status;
3979}
3980
3981static int
3982dump(PicklerObject *self, PyObject *obj)
3983{
3984 const char stop_op = STOP;
3985
3986 if (self->proto >= 2) {
3987 char header[2];
3988
3989 header[0] = PROTO;
3990 assert(self->proto >= 0 && self->proto < 256);
3991 header[1] = (unsigned char)self->proto;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003992 if (_Pickler_Write(self, header, 2) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003993 return -1;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01003994 if (self->proto >= 4)
3995 self->framing = 1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00003996 }
3997
3998 if (save(self, obj, 0) < 0 ||
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00003999 _Pickler_Write(self, &stop_op, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004000 return -1;
4001
4002 return 0;
4003}
4004
Larry Hastings61272b72014-01-07 12:41:53 -08004005/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004006
4007_pickle.Pickler.clear_memo
4008
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004009Clears the pickler's "memo".
4010
4011The memo is the data structure that remembers which objects the
4012pickler has already seen, so that shared or recursive objects are
4013pickled by reference and not by value. This method is useful when
4014re-using picklers.
Larry Hastings61272b72014-01-07 12:41:53 -08004015[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004016
Larry Hastings3cceb382014-01-04 11:09:09 -08004017static PyObject *
4018_pickle_Pickler_clear_memo_impl(PicklerObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08004019/*[clinic end generated code: output=8665c8658aaa094b input=01bdad52f3d93e56]*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004020{
4021 if (self->memo)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004022 PyMemoTable_Clear(self->memo);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004023
4024 Py_RETURN_NONE;
4025}
4026
Larry Hastings61272b72014-01-07 12:41:53 -08004027/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004028
4029_pickle.Pickler.dump
4030
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004031 obj: object
4032 /
4033
4034Write a pickled representation of the given object to the open file.
Larry Hastings61272b72014-01-07 12:41:53 -08004035[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004036
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004037static PyObject *
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004038_pickle_Pickler_dump(PicklerObject *self, PyObject *obj)
Larry Hastings581ee362014-01-28 05:00:08 -08004039/*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004040{
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00004041 /* Check whether the Pickler was initialized correctly (issue3664).
4042 Developers often forget to call __init__() in their subclasses, which
4043 would trigger a segfault without this check. */
4044 if (self->write == NULL) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004045 PickleState *st = _Pickle_GetGlobalState();
4046 PyErr_Format(st->PicklingError,
Amaury Forgeot d'Arc87eee632008-10-17 20:15:53 +00004047 "Pickler.__init__() was not called by %s.__init__()",
4048 Py_TYPE(self)->tp_name);
4049 return NULL;
4050 }
4051
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004052 if (_Pickler_ClearBuffer(self) < 0)
4053 return NULL;
4054
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004055 if (dump(self, obj) < 0)
4056 return NULL;
4057
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004058 if (_Pickler_FlushToFile(self) < 0)
4059 return NULL;
4060
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004061 Py_RETURN_NONE;
4062}
4063
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +02004064/*[clinic input]
4065
4066_pickle.Pickler.__sizeof__ -> Py_ssize_t
4067
4068Returns size in memory, in bytes.
4069[clinic start generated code]*/
4070
4071static Py_ssize_t
4072_pickle_Pickler___sizeof___impl(PicklerObject *self)
4073/*[clinic end generated code: output=106edb3123f332e1 input=8cbbec9bd5540d42]*/
4074{
4075 Py_ssize_t res, s;
4076
Serhiy Storchaka5c4064e2015-12-19 20:05:25 +02004077 res = _PyObject_SIZE(Py_TYPE(self));
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +02004078 if (self->memo != NULL) {
4079 res += sizeof(PyMemoTable);
4080 res += self->memo->mt_allocated * sizeof(PyMemoEntry);
4081 }
4082 if (self->output_buffer != NULL) {
4083 s = _PySys_GetSizeOf(self->output_buffer);
4084 if (s == -1)
4085 return -1;
4086 res += s;
4087 }
4088 return res;
4089}
4090
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004091static struct PyMethodDef Pickler_methods[] = {
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004092 _PICKLE_PICKLER_DUMP_METHODDEF
4093 _PICKLE_PICKLER_CLEAR_MEMO_METHODDEF
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +02004094 _PICKLE_PICKLER___SIZEOF___METHODDEF
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004095 {NULL, NULL} /* sentinel */
4096};
4097
4098static void
4099Pickler_dealloc(PicklerObject *self)
4100{
4101 PyObject_GC_UnTrack(self);
4102
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004103 Py_XDECREF(self->output_buffer);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004104 Py_XDECREF(self->write);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004105 Py_XDECREF(self->pers_func);
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01004106 Py_XDECREF(self->dispatch_table);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004107 Py_XDECREF(self->fast_memo);
4108
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004109 PyMemoTable_Del(self->memo);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004110
4111 Py_TYPE(self)->tp_free((PyObject *)self);
4112}
4113
4114static int
4115Pickler_traverse(PicklerObject *self, visitproc visit, void *arg)
4116{
4117 Py_VISIT(self->write);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004118 Py_VISIT(self->pers_func);
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01004119 Py_VISIT(self->dispatch_table);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004120 Py_VISIT(self->fast_memo);
4121 return 0;
4122}
4123
4124static int
4125Pickler_clear(PicklerObject *self)
4126{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004127 Py_CLEAR(self->output_buffer);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004128 Py_CLEAR(self->write);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004129 Py_CLEAR(self->pers_func);
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01004130 Py_CLEAR(self->dispatch_table);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004131 Py_CLEAR(self->fast_memo);
4132
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004133 if (self->memo != NULL) {
4134 PyMemoTable *memo = self->memo;
4135 self->memo = NULL;
4136 PyMemoTable_Del(memo);
4137 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004138 return 0;
4139}
4140
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004141
Larry Hastings61272b72014-01-07 12:41:53 -08004142/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004143
4144_pickle.Pickler.__init__
4145
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004146 file: object
4147 protocol: object = NULL
4148 fix_imports: bool = True
4149
4150This takes a binary file for writing a pickle data stream.
4151
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004152The optional *protocol* argument tells the pickler to use the given
4153protocol; supported protocols are 0, 1, 2, 3 and 4. The default
4154protocol is 3; a backward-incompatible protocol designed for Python 3.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004155
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004156Specifying a negative protocol version selects the highest protocol
4157version supported. The higher the protocol used, the more recent the
4158version of Python needed to read the pickle produced.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004159
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004160The *file* argument must have a write() method that accepts a single
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004161bytes argument. It can thus be a file object opened for binary
Martin Panter7462b6492015-11-02 03:37:02 +00004162writing, an io.BytesIO instance, or any other custom object that meets
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004163this interface.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004164
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004165If *fix_imports* is True and protocol is less than 3, pickle will try
4166to map the new Python 3 names to the old module names used in Python
41672, so that the pickle data stream is readable with Python 2.
Larry Hastings61272b72014-01-07 12:41:53 -08004168[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004169
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004170static int
Larry Hastings89964c42015-04-14 18:07:59 -04004171_pickle_Pickler___init___impl(PicklerObject *self, PyObject *file,
4172 PyObject *protocol, int fix_imports)
Martin Panter2eb819f2015-11-02 04:04:57 +00004173/*[clinic end generated code: output=b5f31078dab17fb0 input=4faabdbc763c2389]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004174{
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02004175 _Py_IDENTIFIER(persistent_id);
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01004176 _Py_IDENTIFIER(dispatch_table);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004177
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004178 /* In case of multiple __init__() calls, clear previous content. */
4179 if (self->write != NULL)
4180 (void)Pickler_clear(self);
4181
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004182 if (_Pickler_SetProtocol(self, protocol, fix_imports) < 0)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004183 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004184
4185 if (_Pickler_SetOutputStream(self, file) < 0)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004186 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004187
4188 /* memo and output_buffer may have already been created in _Pickler_New */
4189 if (self->memo == NULL) {
4190 self->memo = PyMemoTable_New();
4191 if (self->memo == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004192 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004193 }
4194 self->output_len = 0;
4195 if (self->output_buffer == NULL) {
4196 self->max_output_len = WRITE_BUF_SIZE;
4197 self->output_buffer = PyBytes_FromStringAndSize(NULL,
4198 self->max_output_len);
4199 if (self->output_buffer == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004200 return -1;
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00004201 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004202
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00004203 self->fast = 0;
4204 self->fast_nesting = 0;
4205 self->fast_memo = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004206 self->pers_func = NULL;
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02004207 if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_id)) {
4208 self->pers_func = _PyObject_GetAttrId((PyObject *)self,
4209 &PyId_persistent_id);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004210 if (self->pers_func == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004211 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004212 }
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01004213 self->dispatch_table = NULL;
4214 if (_PyObject_HasAttrId((PyObject *)self, &PyId_dispatch_table)) {
4215 self->dispatch_table = _PyObject_GetAttrId((PyObject *)self,
4216 &PyId_dispatch_table);
4217 if (self->dispatch_table == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004218 return -1;
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004219 }
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004220
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004221 return 0;
4222}
4223
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004224
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004225/* Define a proxy object for the Pickler's internal memo object. This is to
4226 * avoid breaking code like:
4227 * pickler.memo.clear()
4228 * and
4229 * pickler.memo = saved_memo
4230 * Is this a good idea? Not really, but we don't want to break code that uses
4231 * it. Note that we don't implement the entire mapping API here. This is
4232 * intentional, as these should be treated as black-box implementation details.
4233 */
4234
Larry Hastings61272b72014-01-07 12:41:53 -08004235/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004236_pickle.PicklerMemoProxy.clear
4237
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004238Remove all items from memo.
Larry Hastings61272b72014-01-07 12:41:53 -08004239[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004240
Larry Hastings3cceb382014-01-04 11:09:09 -08004241static PyObject *
4242_pickle_PicklerMemoProxy_clear_impl(PicklerMemoProxyObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08004243/*[clinic end generated code: output=5fb9370d48ae8b05 input=ccc186dacd0f1405]*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004244{
4245 if (self->pickler->memo)
4246 PyMemoTable_Clear(self->pickler->memo);
4247 Py_RETURN_NONE;
4248}
4249
Larry Hastings61272b72014-01-07 12:41:53 -08004250/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004251_pickle.PicklerMemoProxy.copy
4252
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004253Copy the memo to a new object.
Larry Hastings61272b72014-01-07 12:41:53 -08004254[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004255
Larry Hastings3cceb382014-01-04 11:09:09 -08004256static PyObject *
4257_pickle_PicklerMemoProxy_copy_impl(PicklerMemoProxyObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08004258/*[clinic end generated code: output=bb83a919d29225ef input=b73043485ac30b36]*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004259{
4260 Py_ssize_t i;
4261 PyMemoTable *memo;
4262 PyObject *new_memo = PyDict_New();
4263 if (new_memo == NULL)
4264 return NULL;
4265
4266 memo = self->pickler->memo;
4267 for (i = 0; i < memo->mt_allocated; ++i) {
4268 PyMemoEntry entry = memo->mt_table[i];
4269 if (entry.me_key != NULL) {
4270 int status;
4271 PyObject *key, *value;
4272
4273 key = PyLong_FromVoidPtr(entry.me_key);
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004274 value = Py_BuildValue("nO", entry.me_value, entry.me_key);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004275
4276 if (key == NULL || value == NULL) {
4277 Py_XDECREF(key);
4278 Py_XDECREF(value);
4279 goto error;
4280 }
4281 status = PyDict_SetItem(new_memo, key, value);
4282 Py_DECREF(key);
4283 Py_DECREF(value);
4284 if (status < 0)
4285 goto error;
4286 }
4287 }
4288 return new_memo;
4289
4290 error:
4291 Py_XDECREF(new_memo);
4292 return NULL;
4293}
4294
Larry Hastings61272b72014-01-07 12:41:53 -08004295/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004296_pickle.PicklerMemoProxy.__reduce__
4297
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004298Implement pickle support.
Larry Hastings61272b72014-01-07 12:41:53 -08004299[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004300
Larry Hastings3cceb382014-01-04 11:09:09 -08004301static PyObject *
4302_pickle_PicklerMemoProxy___reduce___impl(PicklerMemoProxyObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08004303/*[clinic end generated code: output=bebba1168863ab1d input=2f7c540e24b7aae4]*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004304{
4305 PyObject *reduce_value, *dict_args;
Larry Hastings3cceb382014-01-04 11:09:09 -08004306 PyObject *contents = _pickle_PicklerMemoProxy_copy_impl(self);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004307 if (contents == NULL)
4308 return NULL;
4309
4310 reduce_value = PyTuple_New(2);
4311 if (reduce_value == NULL) {
4312 Py_DECREF(contents);
4313 return NULL;
4314 }
4315 dict_args = PyTuple_New(1);
4316 if (dict_args == NULL) {
4317 Py_DECREF(contents);
4318 Py_DECREF(reduce_value);
4319 return NULL;
4320 }
4321 PyTuple_SET_ITEM(dict_args, 0, contents);
4322 Py_INCREF((PyObject *)&PyDict_Type);
4323 PyTuple_SET_ITEM(reduce_value, 0, (PyObject *)&PyDict_Type);
4324 PyTuple_SET_ITEM(reduce_value, 1, dict_args);
4325 return reduce_value;
4326}
4327
4328static PyMethodDef picklerproxy_methods[] = {
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004329 _PICKLE_PICKLERMEMOPROXY_CLEAR_METHODDEF
4330 _PICKLE_PICKLERMEMOPROXY_COPY_METHODDEF
4331 _PICKLE_PICKLERMEMOPROXY___REDUCE___METHODDEF
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004332 {NULL, NULL} /* sentinel */
4333};
4334
4335static void
4336PicklerMemoProxy_dealloc(PicklerMemoProxyObject *self)
4337{
4338 PyObject_GC_UnTrack(self);
4339 Py_XDECREF(self->pickler);
4340 PyObject_GC_Del((PyObject *)self);
4341}
4342
4343static int
4344PicklerMemoProxy_traverse(PicklerMemoProxyObject *self,
4345 visitproc visit, void *arg)
4346{
4347 Py_VISIT(self->pickler);
4348 return 0;
4349}
4350
4351static int
4352PicklerMemoProxy_clear(PicklerMemoProxyObject *self)
4353{
4354 Py_CLEAR(self->pickler);
4355 return 0;
4356}
4357
4358static PyTypeObject PicklerMemoProxyType = {
4359 PyVarObject_HEAD_INIT(NULL, 0)
4360 "_pickle.PicklerMemoProxy", /*tp_name*/
4361 sizeof(PicklerMemoProxyObject), /*tp_basicsize*/
4362 0,
4363 (destructor)PicklerMemoProxy_dealloc, /* tp_dealloc */
4364 0, /* tp_print */
4365 0, /* tp_getattr */
4366 0, /* tp_setattr */
4367 0, /* tp_compare */
4368 0, /* tp_repr */
4369 0, /* tp_as_number */
4370 0, /* tp_as_sequence */
4371 0, /* tp_as_mapping */
Georg Brandlf038b322010-10-18 07:35:09 +00004372 PyObject_HashNotImplemented, /* tp_hash */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004373 0, /* tp_call */
4374 0, /* tp_str */
4375 PyObject_GenericGetAttr, /* tp_getattro */
4376 PyObject_GenericSetAttr, /* tp_setattro */
4377 0, /* tp_as_buffer */
4378 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
4379 0, /* tp_doc */
4380 (traverseproc)PicklerMemoProxy_traverse, /* tp_traverse */
4381 (inquiry)PicklerMemoProxy_clear, /* tp_clear */
4382 0, /* tp_richcompare */
4383 0, /* tp_weaklistoffset */
4384 0, /* tp_iter */
4385 0, /* tp_iternext */
4386 picklerproxy_methods, /* tp_methods */
4387};
4388
4389static PyObject *
4390PicklerMemoProxy_New(PicklerObject *pickler)
4391{
4392 PicklerMemoProxyObject *self;
4393
4394 self = PyObject_GC_New(PicklerMemoProxyObject, &PicklerMemoProxyType);
4395 if (self == NULL)
4396 return NULL;
4397 Py_INCREF(pickler);
4398 self->pickler = pickler;
4399 PyObject_GC_Track(self);
4400 return (PyObject *)self;
4401}
4402
4403/*****************************************************************************/
4404
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004405static PyObject *
4406Pickler_get_memo(PicklerObject *self)
4407{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004408 return PicklerMemoProxy_New(self);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004409}
4410
4411static int
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004412Pickler_set_memo(PicklerObject *self, PyObject *obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004413{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004414 PyMemoTable *new_memo = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004415
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004416 if (obj == NULL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004417 PyErr_SetString(PyExc_TypeError,
4418 "attribute deletion is not supported");
4419 return -1;
4420 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004421
4422 if (Py_TYPE(obj) == &PicklerMemoProxyType) {
4423 PicklerObject *pickler =
4424 ((PicklerMemoProxyObject *)obj)->pickler;
4425
4426 new_memo = PyMemoTable_Copy(pickler->memo);
4427 if (new_memo == NULL)
4428 return -1;
4429 }
4430 else if (PyDict_Check(obj)) {
4431 Py_ssize_t i = 0;
4432 PyObject *key, *value;
4433
4434 new_memo = PyMemoTable_New();
4435 if (new_memo == NULL)
4436 return -1;
4437
4438 while (PyDict_Next(obj, &i, &key, &value)) {
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004439 Py_ssize_t memo_id;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004440 PyObject *memo_obj;
4441
4442 if (!PyTuple_Check(value) || Py_SIZE(value) != 2) {
4443 PyErr_SetString(PyExc_TypeError,
4444 "'memo' values must be 2-item tuples");
4445 goto error;
4446 }
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004447 memo_id = PyLong_AsSsize_t(PyTuple_GET_ITEM(value, 0));
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004448 if (memo_id == -1 && PyErr_Occurred())
4449 goto error;
4450 memo_obj = PyTuple_GET_ITEM(value, 1);
4451 if (PyMemoTable_Set(new_memo, memo_obj, memo_id) < 0)
4452 goto error;
4453 }
4454 }
4455 else {
4456 PyErr_Format(PyExc_TypeError,
Serhiy Storchakab6a9c972016-04-17 09:39:28 +03004457 "'memo' attribute must be a PicklerMemoProxy object"
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004458 "or dict, not %.200s", Py_TYPE(obj)->tp_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004459 return -1;
4460 }
4461
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004462 PyMemoTable_Del(self->memo);
4463 self->memo = new_memo;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004464
4465 return 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004466
4467 error:
4468 if (new_memo)
4469 PyMemoTable_Del(new_memo);
4470 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004471}
4472
4473static PyObject *
4474Pickler_get_persid(PicklerObject *self)
4475{
4476 if (self->pers_func == NULL)
4477 PyErr_SetString(PyExc_AttributeError, "persistent_id");
4478 else
4479 Py_INCREF(self->pers_func);
4480 return self->pers_func;
4481}
4482
4483static int
4484Pickler_set_persid(PicklerObject *self, PyObject *value)
4485{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004486 if (value == NULL) {
4487 PyErr_SetString(PyExc_TypeError,
4488 "attribute deletion is not supported");
4489 return -1;
4490 }
4491 if (!PyCallable_Check(value)) {
4492 PyErr_SetString(PyExc_TypeError,
4493 "persistent_id must be a callable taking one argument");
4494 return -1;
4495 }
4496
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004497 Py_INCREF(value);
Serhiy Storchakaec397562016-04-06 09:50:03 +03004498 Py_XSETREF(self->pers_func, value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004499
4500 return 0;
4501}
4502
4503static PyMemberDef Pickler_members[] = {
4504 {"bin", T_INT, offsetof(PicklerObject, bin)},
4505 {"fast", T_INT, offsetof(PicklerObject, fast)},
Antoine Pitrou8d3c2902012-03-04 18:31:48 +01004506 {"dispatch_table", T_OBJECT_EX, offsetof(PicklerObject, dispatch_table)},
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004507 {NULL}
4508};
4509
4510static PyGetSetDef Pickler_getsets[] = {
4511 {"memo", (getter)Pickler_get_memo,
4512 (setter)Pickler_set_memo},
4513 {"persistent_id", (getter)Pickler_get_persid,
4514 (setter)Pickler_set_persid},
4515 {NULL}
4516};
4517
4518static PyTypeObject Pickler_Type = {
4519 PyVarObject_HEAD_INIT(NULL, 0)
4520 "_pickle.Pickler" , /*tp_name*/
4521 sizeof(PicklerObject), /*tp_basicsize*/
4522 0, /*tp_itemsize*/
4523 (destructor)Pickler_dealloc, /*tp_dealloc*/
4524 0, /*tp_print*/
4525 0, /*tp_getattr*/
4526 0, /*tp_setattr*/
Mark Dickinsone94c6792009-02-02 20:36:42 +00004527 0, /*tp_reserved*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004528 0, /*tp_repr*/
4529 0, /*tp_as_number*/
4530 0, /*tp_as_sequence*/
4531 0, /*tp_as_mapping*/
4532 0, /*tp_hash*/
4533 0, /*tp_call*/
4534 0, /*tp_str*/
4535 0, /*tp_getattro*/
4536 0, /*tp_setattro*/
4537 0, /*tp_as_buffer*/
4538 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08004539 _pickle_Pickler___init____doc__, /*tp_doc*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004540 (traverseproc)Pickler_traverse, /*tp_traverse*/
4541 (inquiry)Pickler_clear, /*tp_clear*/
4542 0, /*tp_richcompare*/
4543 0, /*tp_weaklistoffset*/
4544 0, /*tp_iter*/
4545 0, /*tp_iternext*/
4546 Pickler_methods, /*tp_methods*/
4547 Pickler_members, /*tp_members*/
4548 Pickler_getsets, /*tp_getset*/
4549 0, /*tp_base*/
4550 0, /*tp_dict*/
4551 0, /*tp_descr_get*/
4552 0, /*tp_descr_set*/
4553 0, /*tp_dictoffset*/
Larry Hastingsb7ccb202014-01-18 23:50:21 -08004554 _pickle_Pickler___init__, /*tp_init*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004555 PyType_GenericAlloc, /*tp_alloc*/
4556 PyType_GenericNew, /*tp_new*/
4557 PyObject_GC_Del, /*tp_free*/
4558 0, /*tp_is_gc*/
4559};
4560
Victor Stinner121aab42011-09-29 23:40:53 +02004561/* Temporary helper for calling self.find_class().
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004562
4563 XXX: It would be nice to able to avoid Python function call overhead, by
4564 using directly the C version of find_class(), when find_class() is not
4565 overridden by a subclass. Although, this could become rather hackish. A
4566 simpler optimization would be to call the C function when self is not a
4567 subclass instance. */
4568static PyObject *
4569find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name)
4570{
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02004571 _Py_IDENTIFIER(find_class);
Martin v. Löwisafe55bb2011-10-09 10:38:36 +02004572
4573 return _PyObject_CallMethodId((PyObject *)self, &PyId_find_class, "OO",
4574 module_name, global_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004575}
4576
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004577static Py_ssize_t
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004578marker(UnpicklerObject *self)
4579{
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02004580 Py_ssize_t mark;
4581
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004582 if (self->num_marks < 1) {
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02004583 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004584 PyErr_SetString(st->UnpicklingError, "could not find MARK");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004585 return -1;
4586 }
4587
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02004588 mark = self->marks[--self->num_marks];
4589 self->stack->mark_set = self->num_marks != 0;
4590 self->stack->fence = self->num_marks ?
4591 self->marks[self->num_marks - 1] : 0;
4592 return mark;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004593}
4594
4595static int
4596load_none(UnpicklerObject *self)
4597{
4598 PDATA_APPEND(self->stack, Py_None, -1);
4599 return 0;
4600}
4601
4602static int
4603bad_readline(void)
4604{
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004605 PickleState *st = _Pickle_GetGlobalState();
4606 PyErr_SetString(st->UnpicklingError, "pickle data was truncated");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004607 return -1;
4608}
4609
4610static int
4611load_int(UnpicklerObject *self)
4612{
4613 PyObject *value;
4614 char *endptr, *s;
4615 Py_ssize_t len;
4616 long x;
4617
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004618 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004619 return -1;
4620 if (len < 2)
4621 return bad_readline();
4622
4623 errno = 0;
Victor Stinner121aab42011-09-29 23:40:53 +02004624 /* XXX: Should the base argument of strtol() be explicitly set to 10?
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004625 XXX(avassalotti): Should this uses PyOS_strtol()? */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004626 x = strtol(s, &endptr, 0);
4627
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004628 if (errno || (*endptr != '\n' && *endptr != '\0')) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004629 /* Hm, maybe we've got something long. Let's try reading
Serhiy Storchaka95949422013-08-27 19:40:23 +03004630 * it as a Python int object. */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004631 errno = 0;
4632 /* XXX: Same thing about the base here. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004633 value = PyLong_FromString(s, NULL, 0);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004634 if (value == NULL) {
4635 PyErr_SetString(PyExc_ValueError,
4636 "could not convert string to int");
4637 return -1;
4638 }
4639 }
4640 else {
4641 if (len == 3 && (x == 0 || x == 1)) {
4642 if ((value = PyBool_FromLong(x)) == NULL)
4643 return -1;
4644 }
4645 else {
4646 if ((value = PyLong_FromLong(x)) == NULL)
4647 return -1;
4648 }
4649 }
4650
4651 PDATA_PUSH(self->stack, value, -1);
4652 return 0;
4653}
4654
4655static int
4656load_bool(UnpicklerObject *self, PyObject *boolean)
4657{
4658 assert(boolean == Py_True || boolean == Py_False);
4659 PDATA_APPEND(self->stack, boolean, -1);
4660 return 0;
4661}
4662
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004663/* s contains x bytes of an unsigned little-endian integer. Return its value
4664 * as a C Py_ssize_t, or -1 if it's higher than PY_SSIZE_T_MAX.
4665 */
4666static Py_ssize_t
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004667calc_binsize(char *bytes, int nbytes)
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004668{
4669 unsigned char *s = (unsigned char *)bytes;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004670 int i;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004671 size_t x = 0;
4672
Serhiy Storchakae0606192015-09-29 22:10:07 +03004673 if (nbytes > (int)sizeof(size_t)) {
4674 /* Check for integer overflow. BINBYTES8 and BINUNICODE8 opcodes
4675 * have 64-bit size that can't be represented on 32-bit platform.
4676 */
4677 for (i = (int)sizeof(size_t); i < nbytes; i++) {
4678 if (s[i])
4679 return -1;
4680 }
4681 nbytes = (int)sizeof(size_t);
4682 }
4683 for (i = 0; i < nbytes; i++) {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004684 x |= (size_t) s[i] << (8 * i);
4685 }
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004686
4687 if (x > PY_SSIZE_T_MAX)
4688 return -1;
4689 else
4690 return (Py_ssize_t) x;
4691}
4692
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004693/* s contains x bytes of a little-endian integer. Return its value as a
4694 * C int. Obscure: when x is 1 or 2, this is an unsigned little-endian
Serhiy Storchaka6a7b3a72016-04-17 08:32:47 +03004695 * int, but when x is 4 it's a signed one. This is a historical source
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004696 * of x-platform bugs.
4697 */
4698static long
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004699calc_binint(char *bytes, int nbytes)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004700{
4701 unsigned char *s = (unsigned char *)bytes;
Victor Stinnerf13c46c2014-08-17 21:05:55 +02004702 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004703 long x = 0;
4704
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004705 for (i = 0; i < nbytes; i++) {
4706 x |= (long)s[i] << (8 * i);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004707 }
4708
4709 /* Unlike BININT1 and BININT2, BININT (more accurately BININT4)
4710 * is signed, so on a box with longs bigger than 4 bytes we need
4711 * to extend a BININT's sign bit to the full width.
4712 */
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004713 if (SIZEOF_LONG > 4 && nbytes == 4) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004714 x |= -(x & (1L << 31));
4715 }
4716
4717 return x;
4718}
4719
4720static int
4721load_binintx(UnpicklerObject *self, char *s, int size)
4722{
4723 PyObject *value;
4724 long x;
4725
4726 x = calc_binint(s, size);
4727
4728 if ((value = PyLong_FromLong(x)) == NULL)
4729 return -1;
4730
4731 PDATA_PUSH(self->stack, value, -1);
4732 return 0;
4733}
4734
4735static int
4736load_binint(UnpicklerObject *self)
4737{
4738 char *s;
4739
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004740 if (_Unpickler_Read(self, &s, 4) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004741 return -1;
4742
4743 return load_binintx(self, s, 4);
4744}
4745
4746static int
4747load_binint1(UnpicklerObject *self)
4748{
4749 char *s;
4750
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004751 if (_Unpickler_Read(self, &s, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004752 return -1;
4753
4754 return load_binintx(self, s, 1);
4755}
4756
4757static int
4758load_binint2(UnpicklerObject *self)
4759{
4760 char *s;
4761
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004762 if (_Unpickler_Read(self, &s, 2) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004763 return -1;
4764
4765 return load_binintx(self, s, 2);
4766}
4767
4768static int
4769load_long(UnpicklerObject *self)
4770{
4771 PyObject *value;
Alexandre Vassalotti446f7ff2009-01-23 04:43:46 +00004772 char *s;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004773 Py_ssize_t len;
4774
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004775 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004776 return -1;
4777 if (len < 2)
4778 return bad_readline();
4779
Mark Dickinson8dd05142009-01-20 20:43:58 +00004780 /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove
4781 the 'L' before calling PyLong_FromString. In order to maintain
4782 compatibility with Python 3.0.0, we don't actually *require*
4783 the 'L' to be present. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004784 if (s[len-2] == 'L')
Alexandre Vassalotti446f7ff2009-01-23 04:43:46 +00004785 s[len-2] = '\0';
Alexandre Vassalottie4bccb72009-01-24 01:47:57 +00004786 /* XXX: Should the base argument explicitly set to 10? */
4787 value = PyLong_FromString(s, NULL, 0);
Mark Dickinson8dd05142009-01-20 20:43:58 +00004788 if (value == NULL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004789 return -1;
4790
4791 PDATA_PUSH(self->stack, value, -1);
4792 return 0;
4793}
4794
4795/* 'size' bytes contain the # of bytes of little-endian 256's-complement
4796 * data following.
4797 */
4798static int
4799load_counted_long(UnpicklerObject *self, int size)
4800{
4801 PyObject *value;
4802 char *nbytes;
4803 char *pdata;
4804
4805 assert(size == 1 || size == 4);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004806 if (_Unpickler_Read(self, &nbytes, size) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004807 return -1;
4808
4809 size = calc_binint(nbytes, size);
4810 if (size < 0) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004811 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004812 /* Corrupt or hostile pickle -- we never write one like this */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004813 PyErr_SetString(st->UnpicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004814 "LONG pickle has negative byte count");
4815 return -1;
4816 }
4817
4818 if (size == 0)
4819 value = PyLong_FromLong(0L);
4820 else {
4821 /* Read the raw little-endian bytes and convert. */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004822 if (_Unpickler_Read(self, &pdata, size) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004823 return -1;
4824 value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size,
4825 1 /* little endian */ , 1 /* signed */ );
4826 }
4827 if (value == NULL)
4828 return -1;
4829 PDATA_PUSH(self->stack, value, -1);
4830 return 0;
4831}
4832
4833static int
4834load_float(UnpicklerObject *self)
4835{
4836 PyObject *value;
4837 char *endptr, *s;
4838 Py_ssize_t len;
4839 double d;
4840
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004841 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004842 return -1;
4843 if (len < 2)
4844 return bad_readline();
4845
4846 errno = 0;
Mark Dickinson725bfd82009-05-03 20:33:40 +00004847 d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError);
4848 if (d == -1.0 && PyErr_Occurred())
4849 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004850 if ((endptr[0] != '\n') && (endptr[0] != '\0')) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004851 PyErr_SetString(PyExc_ValueError, "could not convert string to float");
4852 return -1;
4853 }
Mark Dickinson725bfd82009-05-03 20:33:40 +00004854 value = PyFloat_FromDouble(d);
4855 if (value == NULL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004856 return -1;
4857
4858 PDATA_PUSH(self->stack, value, -1);
4859 return 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004860}
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004861
4862static int
4863load_binfloat(UnpicklerObject *self)
4864{
4865 PyObject *value;
4866 double x;
4867 char *s;
4868
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004869 if (_Unpickler_Read(self, &s, 8) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004870 return -1;
4871
4872 x = _PyFloat_Unpack8((unsigned char *)s, 0);
4873 if (x == -1.0 && PyErr_Occurred())
4874 return -1;
4875
4876 if ((value = PyFloat_FromDouble(x)) == NULL)
4877 return -1;
4878
4879 PDATA_PUSH(self->stack, value, -1);
4880 return 0;
4881}
4882
4883static int
4884load_string(UnpicklerObject *self)
4885{
4886 PyObject *bytes;
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004887 PyObject *obj;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004888 Py_ssize_t len;
4889 char *s, *p;
4890
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00004891 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004892 return -1;
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07004893 /* Strip the newline */
4894 len--;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004895 /* Strip outermost quotes */
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07004896 if (len >= 2 && s[0] == s[len - 1] && (s[0] == '\'' || s[0] == '"')) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004897 p = s + 1;
4898 len -= 2;
4899 }
4900 else {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08004901 PickleState *st = _Pickle_GetGlobalState();
4902 PyErr_SetString(st->UnpicklingError,
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07004903 "the STRING opcode argument must be quoted");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004904 return -1;
4905 }
Alexandre Vassalotti7c5e0942013-04-15 23:14:55 -07004906 assert(len >= 0);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004907
4908 /* Use the PyBytes API to decode the string, since that is what is used
4909 to encode, and then coerce the result to Unicode. */
4910 bytes = PyBytes_DecodeEscape(p, len, NULL, 0, NULL);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004911 if (bytes == NULL)
4912 return -1;
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004913
4914 /* Leave the Python 2.x strings as bytes if the *encoding* given to the
4915 Unpickler was 'bytes'. Otherwise, convert them to unicode. */
4916 if (strcmp(self->encoding, "bytes") == 0) {
4917 obj = bytes;
4918 }
4919 else {
4920 obj = PyUnicode_FromEncodedObject(bytes, self->encoding, self->errors);
4921 Py_DECREF(bytes);
4922 if (obj == NULL) {
4923 return -1;
4924 }
4925 }
4926
4927 PDATA_PUSH(self->stack, obj, -1);
4928 return 0;
4929}
4930
4931static int
4932load_counted_binstring(UnpicklerObject *self, int nbytes)
4933{
4934 PyObject *obj;
4935 Py_ssize_t size;
4936 char *s;
4937
4938 if (_Unpickler_Read(self, &s, nbytes) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004939 return -1;
4940
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08004941 size = calc_binsize(s, nbytes);
4942 if (size < 0) {
4943 PickleState *st = _Pickle_GetGlobalState();
4944 PyErr_Format(st->UnpicklingError,
4945 "BINSTRING exceeds system's maximum size of %zd bytes",
4946 PY_SSIZE_T_MAX);
4947 return -1;
4948 }
4949
4950 if (_Unpickler_Read(self, &s, size) < 0)
4951 return -1;
4952
4953 /* Convert Python 2.x strings to bytes if the *encoding* given to the
4954 Unpickler was 'bytes'. Otherwise, convert them to unicode. */
4955 if (strcmp(self->encoding, "bytes") == 0) {
4956 obj = PyBytes_FromStringAndSize(s, size);
4957 }
4958 else {
4959 obj = PyUnicode_Decode(s, size, self->encoding, self->errors);
4960 }
4961 if (obj == NULL) {
4962 return -1;
4963 }
4964
4965 PDATA_PUSH(self->stack, obj, -1);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004966 return 0;
4967}
4968
4969static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004970load_counted_binbytes(UnpicklerObject *self, int nbytes)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004971{
4972 PyObject *bytes;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004973 Py_ssize_t size;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004974 char *s;
4975
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004976 if (_Unpickler_Read(self, &s, nbytes) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004977 return -1;
4978
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004979 size = calc_binsize(s, nbytes);
4980 if (size < 0) {
Antoine Pitrou82be19f2011-08-29 23:09:33 +02004981 PyErr_Format(PyExc_OverflowError,
4982 "BINBYTES exceeds system's maximum size of %zd bytes",
Alexandre Vassalotticc757172013-04-14 02:25:10 -07004983 PY_SSIZE_T_MAX);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004984 return -1;
4985 }
4986
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004987 if (_Unpickler_Read(self, &s, size) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004988 return -1;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01004989
4990 bytes = PyBytes_FromStringAndSize(s, size);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004991 if (bytes == NULL)
4992 return -1;
4993
4994 PDATA_PUSH(self->stack, bytes, -1);
4995 return 0;
4996}
4997
4998static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00004999load_unicode(UnpicklerObject *self)
5000{
5001 PyObject *str;
5002 Py_ssize_t len;
5003 char *s;
5004
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005005 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005006 return -1;
5007 if (len < 1)
5008 return bad_readline();
5009
5010 str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL);
5011 if (str == NULL)
5012 return -1;
5013
5014 PDATA_PUSH(self->stack, str, -1);
5015 return 0;
5016}
5017
5018static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005019load_counted_binunicode(UnpicklerObject *self, int nbytes)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005020{
5021 PyObject *str;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005022 Py_ssize_t size;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005023 char *s;
5024
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005025 if (_Unpickler_Read(self, &s, nbytes) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005026 return -1;
5027
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005028 size = calc_binsize(s, nbytes);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005029 if (size < 0) {
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005030 PyErr_Format(PyExc_OverflowError,
5031 "BINUNICODE exceeds system's maximum size of %zd bytes",
Alexandre Vassalotticc757172013-04-14 02:25:10 -07005032 PY_SSIZE_T_MAX);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005033 return -1;
5034 }
5035
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005036 if (_Unpickler_Read(self, &s, size) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005037 return -1;
5038
Victor Stinner485fb562010-04-13 11:07:24 +00005039 str = PyUnicode_DecodeUTF8(s, size, "surrogatepass");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005040 if (str == NULL)
5041 return -1;
5042
5043 PDATA_PUSH(self->stack, str, -1);
5044 return 0;
5045}
5046
5047static int
Victor Stinner21b47112016-03-14 18:09:39 +01005048load_counted_tuple(UnpicklerObject *self, Py_ssize_t len)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005049{
5050 PyObject *tuple;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005051
Serhiy Storchakaa49de6b2015-11-25 15:01:53 +02005052 if (Py_SIZE(self->stack) < len)
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005053 return Pdata_stack_underflow(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005054
Serhiy Storchakaa49de6b2015-11-25 15:01:53 +02005055 tuple = Pdata_poptuple(self->stack, Py_SIZE(self->stack) - len);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005056 if (tuple == NULL)
5057 return -1;
5058 PDATA_PUSH(self->stack, tuple, -1);
5059 return 0;
5060}
5061
5062static int
Serhiy Storchakaa49de6b2015-11-25 15:01:53 +02005063load_tuple(UnpicklerObject *self)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005064{
Serhiy Storchakaa49de6b2015-11-25 15:01:53 +02005065 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005066
Serhiy Storchakaa49de6b2015-11-25 15:01:53 +02005067 if ((i = marker(self)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005068 return -1;
5069
Serhiy Storchakaa49de6b2015-11-25 15:01:53 +02005070 return load_counted_tuple(self, Py_SIZE(self->stack) - i);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005071}
5072
5073static int
5074load_empty_list(UnpicklerObject *self)
5075{
5076 PyObject *list;
5077
5078 if ((list = PyList_New(0)) == NULL)
5079 return -1;
5080 PDATA_PUSH(self->stack, list, -1);
5081 return 0;
5082}
5083
5084static int
5085load_empty_dict(UnpicklerObject *self)
5086{
5087 PyObject *dict;
5088
5089 if ((dict = PyDict_New()) == NULL)
5090 return -1;
5091 PDATA_PUSH(self->stack, dict, -1);
5092 return 0;
5093}
5094
5095static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005096load_empty_set(UnpicklerObject *self)
5097{
5098 PyObject *set;
5099
5100 if ((set = PySet_New(NULL)) == NULL)
5101 return -1;
5102 PDATA_PUSH(self->stack, set, -1);
5103 return 0;
5104}
5105
5106static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005107load_list(UnpicklerObject *self)
5108{
5109 PyObject *list;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005110 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005111
5112 if ((i = marker(self)) < 0)
5113 return -1;
5114
5115 list = Pdata_poplist(self->stack, i);
5116 if (list == NULL)
5117 return -1;
5118 PDATA_PUSH(self->stack, list, -1);
5119 return 0;
5120}
5121
5122static int
5123load_dict(UnpicklerObject *self)
5124{
5125 PyObject *dict, *key, *value;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005126 Py_ssize_t i, j, k;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005127
5128 if ((i = marker(self)) < 0)
5129 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005130 j = Py_SIZE(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005131
5132 if ((dict = PyDict_New()) == NULL)
5133 return -1;
5134
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005135 if ((j - i) % 2 != 0) {
5136 PickleState *st = _Pickle_GetGlobalState();
5137 PyErr_SetString(st->UnpicklingError, "odd number of items for DICT");
Serhiy Storchaka3ac53802015-12-07 11:32:00 +02005138 Py_DECREF(dict);
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005139 return -1;
5140 }
5141
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005142 for (k = i + 1; k < j; k += 2) {
5143 key = self->stack->data[k - 1];
5144 value = self->stack->data[k];
5145 if (PyDict_SetItem(dict, key, value) < 0) {
5146 Py_DECREF(dict);
5147 return -1;
5148 }
5149 }
5150 Pdata_clear(self->stack, i);
5151 PDATA_PUSH(self->stack, dict, -1);
5152 return 0;
5153}
5154
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005155static int
5156load_frozenset(UnpicklerObject *self)
5157{
5158 PyObject *items;
5159 PyObject *frozenset;
5160 Py_ssize_t i;
5161
5162 if ((i = marker(self)) < 0)
5163 return -1;
5164
5165 items = Pdata_poptuple(self->stack, i);
5166 if (items == NULL)
5167 return -1;
5168
5169 frozenset = PyFrozenSet_New(items);
5170 Py_DECREF(items);
5171 if (frozenset == NULL)
5172 return -1;
5173
5174 PDATA_PUSH(self->stack, frozenset, -1);
5175 return 0;
5176}
5177
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005178static PyObject *
5179instantiate(PyObject *cls, PyObject *args)
5180{
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00005181 PyObject *result = NULL;
Martin v. Löwis1c67dd92011-10-14 15:16:45 +02005182 _Py_IDENTIFIER(__getinitargs__);
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00005183 /* Caller must assure args are a tuple. Normally, args come from
5184 Pdata_poptuple which packs objects from the top of the stack
5185 into a newly created tuple. */
5186 assert(PyTuple_Check(args));
5187 if (Py_SIZE(args) > 0 || !PyType_Check(cls) ||
Martin v. Löwis1c67dd92011-10-14 15:16:45 +02005188 _PyObject_HasAttrId(cls, &PyId___getinitargs__)) {
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00005189 result = PyObject_CallObject(cls, args);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005190 }
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00005191 else {
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02005192 _Py_IDENTIFIER(__new__);
Martin v. Löwisafe55bb2011-10-09 10:38:36 +02005193
5194 result = _PyObject_CallMethodId(cls, &PyId___new__, "O", cls);
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00005195 }
5196 return result;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005197}
5198
5199static int
5200load_obj(UnpicklerObject *self)
5201{
5202 PyObject *cls, *args, *obj = NULL;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005203 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005204
5205 if ((i = marker(self)) < 0)
5206 return -1;
5207
Serhiy Storchakae9b30742015-11-23 15:17:43 +02005208 if (Py_SIZE(self->stack) - i < 1)
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005209 return Pdata_stack_underflow(self->stack);
Serhiy Storchakae9b30742015-11-23 15:17:43 +02005210
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005211 args = Pdata_poptuple(self->stack, i + 1);
5212 if (args == NULL)
5213 return -1;
5214
5215 PDATA_POP(self->stack, cls);
5216 if (cls) {
5217 obj = instantiate(cls, args);
5218 Py_DECREF(cls);
5219 }
5220 Py_DECREF(args);
5221 if (obj == NULL)
5222 return -1;
5223
5224 PDATA_PUSH(self->stack, obj, -1);
5225 return 0;
5226}
5227
5228static int
5229load_inst(UnpicklerObject *self)
5230{
5231 PyObject *cls = NULL;
5232 PyObject *args = NULL;
5233 PyObject *obj = NULL;
5234 PyObject *module_name;
5235 PyObject *class_name;
5236 Py_ssize_t len;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005237 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005238 char *s;
5239
5240 if ((i = marker(self)) < 0)
5241 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005242 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005243 return -1;
5244 if (len < 2)
5245 return bad_readline();
5246
5247 /* Here it is safe to use PyUnicode_DecodeASCII(), even though non-ASCII
5248 identifiers are permitted in Python 3.0, since the INST opcode is only
5249 supported by older protocols on Python 2.x. */
5250 module_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
5251 if (module_name == NULL)
5252 return -1;
5253
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005254 if ((len = _Unpickler_Readline(self, &s)) >= 0) {
Serhiy Storchakaca28eba2015-12-01 00:18:23 +02005255 if (len < 2) {
5256 Py_DECREF(module_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005257 return bad_readline();
Serhiy Storchakaca28eba2015-12-01 00:18:23 +02005258 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005259 class_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00005260 if (class_name != NULL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005261 cls = find_class(self, module_name, class_name);
5262 Py_DECREF(class_name);
5263 }
5264 }
5265 Py_DECREF(module_name);
5266
5267 if (cls == NULL)
5268 return -1;
5269
5270 if ((args = Pdata_poptuple(self->stack, i)) != NULL) {
5271 obj = instantiate(cls, args);
5272 Py_DECREF(args);
5273 }
5274 Py_DECREF(cls);
5275
5276 if (obj == NULL)
5277 return -1;
5278
5279 PDATA_PUSH(self->stack, obj, -1);
5280 return 0;
5281}
5282
5283static int
5284load_newobj(UnpicklerObject *self)
5285{
5286 PyObject *args = NULL;
5287 PyObject *clsraw = NULL;
5288 PyTypeObject *cls; /* clsraw cast to its true type */
5289 PyObject *obj;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005290 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005291
5292 /* Stack is ... cls argtuple, and we want to call
5293 * cls.__new__(cls, *argtuple).
5294 */
5295 PDATA_POP(self->stack, args);
5296 if (args == NULL)
5297 goto error;
5298 if (!PyTuple_Check(args)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005299 PyErr_SetString(st->UnpicklingError,
5300 "NEWOBJ expected an arg " "tuple.");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005301 goto error;
5302 }
5303
5304 PDATA_POP(self->stack, clsraw);
5305 cls = (PyTypeObject *)clsraw;
5306 if (cls == NULL)
5307 goto error;
5308 if (!PyType_Check(cls)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005309 PyErr_SetString(st->UnpicklingError, "NEWOBJ class argument "
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005310 "isn't a type object");
5311 goto error;
5312 }
5313 if (cls->tp_new == NULL) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005314 PyErr_SetString(st->UnpicklingError, "NEWOBJ class argument "
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005315 "has NULL tp_new");
5316 goto error;
5317 }
5318
5319 /* Call __new__. */
5320 obj = cls->tp_new(cls, args, NULL);
5321 if (obj == NULL)
5322 goto error;
5323
5324 Py_DECREF(args);
5325 Py_DECREF(clsraw);
5326 PDATA_PUSH(self->stack, obj, -1);
5327 return 0;
5328
5329 error:
5330 Py_XDECREF(args);
5331 Py_XDECREF(clsraw);
5332 return -1;
5333}
5334
5335static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005336load_newobj_ex(UnpicklerObject *self)
5337{
5338 PyObject *cls, *args, *kwargs;
5339 PyObject *obj;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005340 PickleState *st = _Pickle_GetGlobalState();
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005341
5342 PDATA_POP(self->stack, kwargs);
5343 if (kwargs == NULL) {
5344 return -1;
5345 }
5346 PDATA_POP(self->stack, args);
5347 if (args == NULL) {
5348 Py_DECREF(kwargs);
5349 return -1;
5350 }
5351 PDATA_POP(self->stack, cls);
5352 if (cls == NULL) {
5353 Py_DECREF(kwargs);
5354 Py_DECREF(args);
5355 return -1;
5356 }
Larry Hastings61272b72014-01-07 12:41:53 -08005357
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005358 if (!PyType_Check(cls)) {
5359 Py_DECREF(kwargs);
5360 Py_DECREF(args);
Larry Hastings61272b72014-01-07 12:41:53 -08005361 PyErr_Format(st->UnpicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005362 "NEWOBJ_EX class argument must be a type, not %.200s",
5363 Py_TYPE(cls)->tp_name);
Benjamin Peterson80f78a32015-07-02 16:18:38 -05005364 Py_DECREF(cls);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005365 return -1;
5366 }
5367
5368 if (((PyTypeObject *)cls)->tp_new == NULL) {
5369 Py_DECREF(kwargs);
5370 Py_DECREF(args);
5371 Py_DECREF(cls);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005372 PyErr_SetString(st->UnpicklingError,
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005373 "NEWOBJ_EX class argument doesn't have __new__");
5374 return -1;
5375 }
5376 obj = ((PyTypeObject *)cls)->tp_new((PyTypeObject *)cls, args, kwargs);
5377 Py_DECREF(kwargs);
5378 Py_DECREF(args);
5379 Py_DECREF(cls);
5380 if (obj == NULL) {
5381 return -1;
5382 }
5383 PDATA_PUSH(self->stack, obj, -1);
5384 return 0;
5385}
5386
5387static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005388load_global(UnpicklerObject *self)
5389{
5390 PyObject *global = NULL;
5391 PyObject *module_name;
5392 PyObject *global_name;
5393 Py_ssize_t len;
5394 char *s;
5395
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005396 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005397 return -1;
5398 if (len < 2)
5399 return bad_readline();
5400 module_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
5401 if (!module_name)
5402 return -1;
5403
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005404 if ((len = _Unpickler_Readline(self, &s)) >= 0) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005405 if (len < 2) {
5406 Py_DECREF(module_name);
5407 return bad_readline();
5408 }
5409 global_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
5410 if (global_name) {
5411 global = find_class(self, module_name, global_name);
5412 Py_DECREF(global_name);
5413 }
5414 }
5415 Py_DECREF(module_name);
5416
5417 if (global == NULL)
5418 return -1;
5419 PDATA_PUSH(self->stack, global, -1);
5420 return 0;
5421}
5422
5423static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005424load_stack_global(UnpicklerObject *self)
5425{
5426 PyObject *global;
5427 PyObject *module_name;
5428 PyObject *global_name;
5429
5430 PDATA_POP(self->stack, global_name);
5431 PDATA_POP(self->stack, module_name);
5432 if (module_name == NULL || !PyUnicode_CheckExact(module_name) ||
5433 global_name == NULL || !PyUnicode_CheckExact(global_name)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005434 PickleState *st = _Pickle_GetGlobalState();
5435 PyErr_SetString(st->UnpicklingError, "STACK_GLOBAL requires str");
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005436 Py_XDECREF(global_name);
5437 Py_XDECREF(module_name);
5438 return -1;
5439 }
5440 global = find_class(self, module_name, global_name);
5441 Py_DECREF(global_name);
5442 Py_DECREF(module_name);
5443 if (global == NULL)
5444 return -1;
5445 PDATA_PUSH(self->stack, global, -1);
5446 return 0;
5447}
5448
5449static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005450load_persid(UnpicklerObject *self)
5451{
5452 PyObject *pid;
5453 Py_ssize_t len;
5454 char *s;
5455
5456 if (self->pers_func) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005457 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005458 return -1;
Alexandre Vassalotti896414f2013-11-30 13:52:35 -08005459 if (len < 1)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005460 return bad_readline();
5461
Serhiy Storchakadec25af2016-07-17 11:24:17 +03005462 pid = PyUnicode_DecodeASCII(s, len - 1, "strict");
5463 if (pid == NULL) {
5464 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
5465 PyErr_SetString(_Pickle_GetGlobalState()->UnpicklingError,
5466 "persistent IDs in protocol 0 must be "
5467 "ASCII strings");
5468 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005469 return -1;
Serhiy Storchakadec25af2016-07-17 11:24:17 +03005470 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005471
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08005472 /* This does not leak since _Pickle_FastCall() steals the reference
5473 to pid first. */
5474 pid = _Pickle_FastCall(self->pers_func, pid);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005475 if (pid == NULL)
5476 return -1;
5477
5478 PDATA_PUSH(self->stack, pid, -1);
5479 return 0;
5480 }
5481 else {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005482 PickleState *st = _Pickle_GetGlobalState();
5483 PyErr_SetString(st->UnpicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005484 "A load persistent id instruction was encountered,\n"
5485 "but no persistent_load function was specified.");
5486 return -1;
5487 }
5488}
5489
5490static int
5491load_binpersid(UnpicklerObject *self)
5492{
5493 PyObject *pid;
5494
5495 if (self->pers_func) {
5496 PDATA_POP(self->stack, pid);
5497 if (pid == NULL)
5498 return -1;
5499
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08005500 /* This does not leak since _Pickle_FastCall() steals the
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005501 reference to pid first. */
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08005502 pid = _Pickle_FastCall(self->pers_func, pid);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005503 if (pid == NULL)
5504 return -1;
5505
5506 PDATA_PUSH(self->stack, pid, -1);
5507 return 0;
5508 }
5509 else {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005510 PickleState *st = _Pickle_GetGlobalState();
5511 PyErr_SetString(st->UnpicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005512 "A load persistent id instruction was encountered,\n"
5513 "but no persistent_load function was specified.");
5514 return -1;
5515 }
5516}
5517
5518static int
5519load_pop(UnpicklerObject *self)
5520{
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005521 Py_ssize_t len = Py_SIZE(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005522
5523 /* Note that we split the (pickle.py) stack into two stacks,
5524 * an object stack and a mark stack. We have to be clever and
5525 * pop the right one. We do this by looking at the top of the
Collin Winter8ca69de2009-05-26 16:53:41 +00005526 * mark stack first, and only signalling a stack underflow if
5527 * the object stack is empty and the mark stack doesn't match
5528 * our expectations.
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005529 */
Collin Winter8ca69de2009-05-26 16:53:41 +00005530 if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005531 self->num_marks--;
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005532 self->stack->mark_set = self->num_marks != 0;
5533 self->stack->fence = self->num_marks ?
5534 self->marks[self->num_marks - 1] : 0;
5535 } else if (len <= self->stack->fence)
5536 return Pdata_stack_underflow(self->stack);
5537 else {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005538 len--;
5539 Py_DECREF(self->stack->data[len]);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005540 Py_SIZE(self->stack) = len;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005541 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005542 return 0;
5543}
5544
5545static int
5546load_pop_mark(UnpicklerObject *self)
5547{
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005548 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005549
5550 if ((i = marker(self)) < 0)
5551 return -1;
5552
5553 Pdata_clear(self->stack, i);
5554
5555 return 0;
5556}
5557
5558static int
5559load_dup(UnpicklerObject *self)
5560{
5561 PyObject *last;
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005562 Py_ssize_t len = Py_SIZE(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005563
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005564 if (len <= self->stack->fence)
5565 return Pdata_stack_underflow(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005566 last = self->stack->data[len - 1];
5567 PDATA_APPEND(self->stack, last, -1);
5568 return 0;
5569}
5570
5571static int
5572load_get(UnpicklerObject *self)
5573{
5574 PyObject *key, *value;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005575 Py_ssize_t idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005576 Py_ssize_t len;
5577 char *s;
5578
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005579 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005580 return -1;
5581 if (len < 2)
5582 return bad_readline();
5583
5584 key = PyLong_FromString(s, NULL, 10);
5585 if (key == NULL)
5586 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005587 idx = PyLong_AsSsize_t(key);
5588 if (idx == -1 && PyErr_Occurred()) {
5589 Py_DECREF(key);
5590 return -1;
5591 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005592
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005593 value = _Unpickler_MemoGet(self, idx);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005594 if (value == NULL) {
5595 if (!PyErr_Occurred())
5596 PyErr_SetObject(PyExc_KeyError, key);
5597 Py_DECREF(key);
5598 return -1;
5599 }
5600 Py_DECREF(key);
5601
5602 PDATA_APPEND(self->stack, value, -1);
5603 return 0;
5604}
5605
5606static int
5607load_binget(UnpicklerObject *self)
5608{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005609 PyObject *value;
5610 Py_ssize_t idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005611 char *s;
5612
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005613 if (_Unpickler_Read(self, &s, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005614 return -1;
5615
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005616 idx = Py_CHARMASK(s[0]);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005617
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005618 value = _Unpickler_MemoGet(self, idx);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005619 if (value == NULL) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005620 PyObject *key = PyLong_FromSsize_t(idx);
Christian Heimes9ee5c372013-07-26 22:45:00 +02005621 if (key != NULL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005622 PyErr_SetObject(PyExc_KeyError, key);
Christian Heimes9ee5c372013-07-26 22:45:00 +02005623 Py_DECREF(key);
5624 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005625 return -1;
5626 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005627
5628 PDATA_APPEND(self->stack, value, -1);
5629 return 0;
5630}
5631
5632static int
5633load_long_binget(UnpicklerObject *self)
5634{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005635 PyObject *value;
5636 Py_ssize_t idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005637 char *s;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005638
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005639 if (_Unpickler_Read(self, &s, 4) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005640 return -1;
5641
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005642 idx = calc_binsize(s, 4);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005643
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005644 value = _Unpickler_MemoGet(self, idx);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005645 if (value == NULL) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005646 PyObject *key = PyLong_FromSsize_t(idx);
Christian Heimes9ee5c372013-07-26 22:45:00 +02005647 if (key != NULL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005648 PyErr_SetObject(PyExc_KeyError, key);
Christian Heimes9ee5c372013-07-26 22:45:00 +02005649 Py_DECREF(key);
5650 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005651 return -1;
5652 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005653
5654 PDATA_APPEND(self->stack, value, -1);
5655 return 0;
5656}
5657
5658/* Push an object from the extension registry (EXT[124]). nbytes is
5659 * the number of bytes following the opcode, holding the index (code) value.
5660 */
5661static int
5662load_extension(UnpicklerObject *self, int nbytes)
5663{
5664 char *codebytes; /* the nbytes bytes after the opcode */
5665 long code; /* calc_binint returns long */
5666 PyObject *py_code; /* code as a Python int */
5667 PyObject *obj; /* the object to push */
5668 PyObject *pair; /* (module_name, class_name) */
5669 PyObject *module_name, *class_name;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005670 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005671
5672 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005673 if (_Unpickler_Read(self, &codebytes, nbytes) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005674 return -1;
5675 code = calc_binint(codebytes, nbytes);
5676 if (code <= 0) { /* note that 0 is forbidden */
5677 /* Corrupt or hostile pickle. */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005678 PyErr_SetString(st->UnpicklingError, "EXT specifies code <= 0");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005679 return -1;
5680 }
5681
5682 /* Look for the code in the cache. */
5683 py_code = PyLong_FromLong(code);
5684 if (py_code == NULL)
5685 return -1;
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08005686 obj = PyDict_GetItemWithError(st->extension_cache, py_code);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005687 if (obj != NULL) {
5688 /* Bingo. */
5689 Py_DECREF(py_code);
5690 PDATA_APPEND(self->stack, obj, -1);
5691 return 0;
5692 }
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08005693 if (PyErr_Occurred()) {
5694 Py_DECREF(py_code);
5695 return -1;
5696 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005697
5698 /* Look up the (module_name, class_name) pair. */
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08005699 pair = PyDict_GetItemWithError(st->inverted_registry, py_code);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005700 if (pair == NULL) {
5701 Py_DECREF(py_code);
Alexandre Vassalotti567eba12013-11-28 17:09:16 -08005702 if (!PyErr_Occurred()) {
5703 PyErr_Format(PyExc_ValueError, "unregistered extension "
5704 "code %ld", code);
5705 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005706 return -1;
5707 }
5708 /* Since the extension registry is manipulable via Python code,
5709 * confirm that pair is really a 2-tuple of strings.
5710 */
5711 if (!PyTuple_Check(pair) || PyTuple_Size(pair) != 2 ||
5712 !PyUnicode_Check(module_name = PyTuple_GET_ITEM(pair, 0)) ||
5713 !PyUnicode_Check(class_name = PyTuple_GET_ITEM(pair, 1))) {
5714 Py_DECREF(py_code);
5715 PyErr_Format(PyExc_ValueError, "_inverted_registry[%ld] "
5716 "isn't a 2-tuple of strings", code);
5717 return -1;
5718 }
5719 /* Load the object. */
5720 obj = find_class(self, module_name, class_name);
5721 if (obj == NULL) {
5722 Py_DECREF(py_code);
5723 return -1;
5724 }
5725 /* Cache code -> obj. */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005726 code = PyDict_SetItem(st->extension_cache, py_code, obj);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005727 Py_DECREF(py_code);
5728 if (code < 0) {
5729 Py_DECREF(obj);
5730 return -1;
5731 }
5732 PDATA_PUSH(self->stack, obj, -1);
5733 return 0;
5734}
5735
5736static int
5737load_put(UnpicklerObject *self)
5738{
5739 PyObject *key, *value;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005740 Py_ssize_t idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005741 Py_ssize_t len;
5742 char *s;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005743
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005744 if ((len = _Unpickler_Readline(self, &s)) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005745 return -1;
5746 if (len < 2)
5747 return bad_readline();
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005748 if (Py_SIZE(self->stack) <= self->stack->fence)
5749 return Pdata_stack_underflow(self->stack);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005750 value = self->stack->data[Py_SIZE(self->stack) - 1];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005751
5752 key = PyLong_FromString(s, NULL, 10);
5753 if (key == NULL)
5754 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005755 idx = PyLong_AsSsize_t(key);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005756 Py_DECREF(key);
Antoine Pitrou55549ec2011-08-30 00:27:10 +02005757 if (idx < 0) {
5758 if (!PyErr_Occurred())
5759 PyErr_SetString(PyExc_ValueError,
5760 "negative PUT argument");
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005761 return -1;
Antoine Pitrou55549ec2011-08-30 00:27:10 +02005762 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005763
5764 return _Unpickler_MemoPut(self, idx, value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005765}
5766
5767static int
5768load_binput(UnpicklerObject *self)
5769{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005770 PyObject *value;
5771 Py_ssize_t idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005772 char *s;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005773
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005774 if (_Unpickler_Read(self, &s, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005775 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005776
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005777 if (Py_SIZE(self->stack) <= self->stack->fence)
5778 return Pdata_stack_underflow(self->stack);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005779 value = self->stack->data[Py_SIZE(self->stack) - 1];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005780
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005781 idx = Py_CHARMASK(s[0]);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005782
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005783 return _Unpickler_MemoPut(self, idx, value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005784}
5785
5786static int
5787load_long_binput(UnpicklerObject *self)
5788{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005789 PyObject *value;
5790 Py_ssize_t idx;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005791 char *s;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005792
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005793 if (_Unpickler_Read(self, &s, 4) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005794 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005795
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005796 if (Py_SIZE(self->stack) <= self->stack->fence)
5797 return Pdata_stack_underflow(self->stack);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005798 value = self->stack->data[Py_SIZE(self->stack) - 1];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005799
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005800 idx = calc_binsize(s, 4);
Antoine Pitrou55549ec2011-08-30 00:27:10 +02005801 if (idx < 0) {
5802 PyErr_SetString(PyExc_ValueError,
5803 "negative LONG_BINPUT argument");
5804 return -1;
5805 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005806
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005807 return _Unpickler_MemoPut(self, idx, value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005808}
5809
5810static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005811load_memoize(UnpicklerObject *self)
5812{
5813 PyObject *value;
5814
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005815 if (Py_SIZE(self->stack) <= self->stack->fence)
5816 return Pdata_stack_underflow(self->stack);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005817 value = self->stack->data[Py_SIZE(self->stack) - 1];
5818
5819 return _Unpickler_MemoPut(self, self->memo_len, value);
5820}
5821
5822static int
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005823do_append(UnpicklerObject *self, Py_ssize_t x)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005824{
5825 PyObject *value;
5826 PyObject *list;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005827 Py_ssize_t len, i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005828
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005829 len = Py_SIZE(self->stack);
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005830 if (x > len || x <= self->stack->fence)
5831 return Pdata_stack_underflow(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005832 if (len == x) /* nothing to do */
5833 return 0;
5834
5835 list = self->stack->data[x - 1];
5836
5837 if (PyList_Check(list)) {
5838 PyObject *slice;
5839 Py_ssize_t list_len;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005840 int ret;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005841
5842 slice = Pdata_poplist(self->stack, x);
5843 if (!slice)
5844 return -1;
5845 list_len = PyList_GET_SIZE(list);
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005846 ret = PyList_SetSlice(list, list_len, list_len, slice);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005847 Py_DECREF(slice);
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005848 return ret;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005849 }
5850 else {
5851 PyObject *append_func;
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02005852 _Py_IDENTIFIER(append);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005853
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02005854 append_func = _PyObject_GetAttrId(list, &PyId_append);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005855 if (append_func == NULL)
5856 return -1;
5857 for (i = x; i < len; i++) {
5858 PyObject *result;
5859
5860 value = self->stack->data[i];
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08005861 result = _Pickle_FastCall(append_func, value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005862 if (result == NULL) {
5863 Pdata_clear(self->stack, i + 1);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005864 Py_SIZE(self->stack) = x;
Alexandre Vassalotti637c7c42013-04-20 21:28:21 -07005865 Py_DECREF(append_func);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005866 return -1;
5867 }
5868 Py_DECREF(result);
5869 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005870 Py_SIZE(self->stack) = x;
Alexandre Vassalotti637c7c42013-04-20 21:28:21 -07005871 Py_DECREF(append_func);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005872 }
5873
5874 return 0;
5875}
5876
5877static int
5878load_append(UnpicklerObject *self)
5879{
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005880 if (Py_SIZE(self->stack) - 1 <= self->stack->fence)
5881 return Pdata_stack_underflow(self->stack);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005882 return do_append(self, Py_SIZE(self->stack) - 1);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005883}
5884
5885static int
5886load_appends(UnpicklerObject *self)
5887{
Serhiy Storchakae9b30742015-11-23 15:17:43 +02005888 Py_ssize_t i = marker(self);
5889 if (i < 0)
5890 return -1;
5891 return do_append(self, i);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005892}
5893
5894static int
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005895do_setitems(UnpicklerObject *self, Py_ssize_t x)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005896{
5897 PyObject *value, *key;
5898 PyObject *dict;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02005899 Py_ssize_t len, i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005900 int status = 0;
5901
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005902 len = Py_SIZE(self->stack);
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005903 if (x > len || x <= self->stack->fence)
5904 return Pdata_stack_underflow(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005905 if (len == x) /* nothing to do */
5906 return 0;
Victor Stinner121aab42011-09-29 23:40:53 +02005907 if ((len - x) % 2 != 0) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005908 PickleState *st = _Pickle_GetGlobalState();
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005909 /* Currupt or hostile pickle -- we never write one like this. */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08005910 PyErr_SetString(st->UnpicklingError,
5911 "odd number of items for SETITEMS");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005912 return -1;
5913 }
5914
5915 /* Here, dict does not actually need to be a PyDict; it could be anything
5916 that supports the __setitem__ attribute. */
5917 dict = self->stack->data[x - 1];
5918
5919 for (i = x + 1; i < len; i += 2) {
5920 key = self->stack->data[i - 1];
5921 value = self->stack->data[i];
5922 if (PyObject_SetItem(dict, key, value) < 0) {
5923 status = -1;
5924 break;
5925 }
5926 }
5927
5928 Pdata_clear(self->stack, x);
5929 return status;
5930}
5931
5932static int
5933load_setitem(UnpicklerObject *self)
5934{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00005935 return do_setitems(self, Py_SIZE(self->stack) - 2);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005936}
5937
5938static int
5939load_setitems(UnpicklerObject *self)
5940{
Serhiy Storchakae9b30742015-11-23 15:17:43 +02005941 Py_ssize_t i = marker(self);
5942 if (i < 0)
5943 return -1;
5944 return do_setitems(self, i);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00005945}
5946
5947static int
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005948load_additems(UnpicklerObject *self)
5949{
5950 PyObject *set;
5951 Py_ssize_t mark, len, i;
5952
5953 mark = marker(self);
Serhiy Storchakae9b30742015-11-23 15:17:43 +02005954 if (mark < 0)
5955 return -1;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005956 len = Py_SIZE(self->stack);
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02005957 if (mark > len || mark <= self->stack->fence)
5958 return Pdata_stack_underflow(self->stack);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005959 if (len == mark) /* nothing to do */
5960 return 0;
5961
5962 set = self->stack->data[mark - 1];
5963
5964 if (PySet_Check(set)) {
5965 PyObject *items;
5966 int status;
5967
5968 items = Pdata_poptuple(self->stack, mark);
5969 if (items == NULL)
5970 return -1;
5971
5972 status = _PySet_Update(set, items);
5973 Py_DECREF(items);
5974 return status;
5975 }
5976 else {
5977 PyObject *add_func;
5978 _Py_IDENTIFIER(add);
5979
5980 add_func = _PyObject_GetAttrId(set, &PyId_add);
5981 if (add_func == NULL)
5982 return -1;
5983 for (i = mark; i < len; i++) {
5984 PyObject *result;
5985 PyObject *item;
5986
5987 item = self->stack->data[i];
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08005988 result = _Pickle_FastCall(add_func, item);
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01005989 if (result == NULL) {
5990 Pdata_clear(self->stack, i + 1);
5991 Py_SIZE(self->stack) = mark;
5992 return -1;
5993 }
5994 Py_DECREF(result);
5995 }
5996 Py_SIZE(self->stack) = mark;
5997 }
5998
5999 return 0;
6000}
6001
6002static int
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006003load_build(UnpicklerObject *self)
6004{
6005 PyObject *state, *inst, *slotstate;
6006 PyObject *setstate;
6007 int status = 0;
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02006008 _Py_IDENTIFIER(__setstate__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006009
6010 /* Stack is ... instance, state. We want to leave instance at
6011 * the stack top, possibly mutated via instance.__setstate__(state).
6012 */
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02006013 if (Py_SIZE(self->stack) - 2 < self->stack->fence)
6014 return Pdata_stack_underflow(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006015
6016 PDATA_POP(self->stack, state);
6017 if (state == NULL)
6018 return -1;
6019
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006020 inst = self->stack->data[Py_SIZE(self->stack) - 1];
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006021
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02006022 setstate = _PyObject_GetAttrId(inst, &PyId___setstate__);
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00006023 if (setstate == NULL) {
6024 if (PyErr_ExceptionMatches(PyExc_AttributeError))
6025 PyErr_Clear();
Antoine Pitroud79dc622008-09-05 00:03:33 +00006026 else {
6027 Py_DECREF(state);
Alexandre Vassalotti1f9d9072008-08-15 03:07:47 +00006028 return -1;
Antoine Pitroud79dc622008-09-05 00:03:33 +00006029 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006030 }
6031 else {
6032 PyObject *result;
6033
6034 /* The explicit __setstate__ is responsible for everything. */
Alexandre Vassalotti20c28c12013-11-27 02:26:54 -08006035 result = _Pickle_FastCall(setstate, state);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006036 Py_DECREF(setstate);
6037 if (result == NULL)
6038 return -1;
6039 Py_DECREF(result);
6040 return 0;
6041 }
6042
6043 /* A default __setstate__. First see whether state embeds a
6044 * slot state dict too (a proto 2 addition).
6045 */
6046 if (PyTuple_Check(state) && Py_SIZE(state) == 2) {
6047 PyObject *tmp = state;
6048
6049 state = PyTuple_GET_ITEM(tmp, 0);
6050 slotstate = PyTuple_GET_ITEM(tmp, 1);
6051 Py_INCREF(state);
6052 Py_INCREF(slotstate);
6053 Py_DECREF(tmp);
6054 }
6055 else
6056 slotstate = NULL;
6057
6058 /* Set inst.__dict__ from the state dict (if any). */
6059 if (state != Py_None) {
6060 PyObject *dict;
Antoine Pitroua9f48a02009-05-02 21:41:14 +00006061 PyObject *d_key, *d_value;
6062 Py_ssize_t i;
Martin v. Löwisbd928fe2011-10-14 10:20:37 +02006063 _Py_IDENTIFIER(__dict__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006064
6065 if (!PyDict_Check(state)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006066 PickleState *st = _Pickle_GetGlobalState();
6067 PyErr_SetString(st->UnpicklingError, "state is not a dictionary");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006068 goto error;
6069 }
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02006070 dict = _PyObject_GetAttrId(inst, &PyId___dict__);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006071 if (dict == NULL)
6072 goto error;
6073
Antoine Pitroua9f48a02009-05-02 21:41:14 +00006074 i = 0;
6075 while (PyDict_Next(state, &i, &d_key, &d_value)) {
6076 /* normally the keys for instance attributes are
6077 interned. we should try to do that here. */
6078 Py_INCREF(d_key);
6079 if (PyUnicode_CheckExact(d_key))
6080 PyUnicode_InternInPlace(&d_key);
6081 if (PyObject_SetItem(dict, d_key, d_value) < 0) {
6082 Py_DECREF(d_key);
6083 goto error;
6084 }
6085 Py_DECREF(d_key);
6086 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006087 Py_DECREF(dict);
6088 }
6089
6090 /* Also set instance attributes from the slotstate dict (if any). */
6091 if (slotstate != NULL) {
6092 PyObject *d_key, *d_value;
6093 Py_ssize_t i;
6094
6095 if (!PyDict_Check(slotstate)) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006096 PickleState *st = _Pickle_GetGlobalState();
6097 PyErr_SetString(st->UnpicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006098 "slot state is not a dictionary");
6099 goto error;
6100 }
6101 i = 0;
6102 while (PyDict_Next(slotstate, &i, &d_key, &d_value)) {
6103 if (PyObject_SetAttr(inst, d_key, d_value) < 0)
6104 goto error;
6105 }
6106 }
6107
6108 if (0) {
6109 error:
6110 status = -1;
6111 }
6112
6113 Py_DECREF(state);
6114 Py_XDECREF(slotstate);
6115 return status;
6116}
6117
6118static int
6119load_mark(UnpicklerObject *self)
6120{
6121
6122 /* Note that we split the (pickle.py) stack into two stacks, an
6123 * object stack and a mark stack. Here we push a mark onto the
6124 * mark stack.
6125 */
6126
6127 if ((self->num_marks + 1) >= self->marks_size) {
6128 size_t alloc;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006129
6130 /* Use the size_t type to check for overflow. */
6131 alloc = ((size_t)self->num_marks << 1) + 20;
Antoine Pitrou82be19f2011-08-29 23:09:33 +02006132 if (alloc > (PY_SSIZE_T_MAX / sizeof(Py_ssize_t)) ||
Alexandre Vassalotti7634ff52008-06-13 02:16:06 +00006133 alloc <= ((size_t)self->num_marks + 1)) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006134 PyErr_NoMemory();
6135 return -1;
6136 }
6137
6138 if (self->marks == NULL)
Benjamin Peterson59b08c12015-06-27 13:41:33 -05006139 self->marks = PyMem_NEW(Py_ssize_t, alloc);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006140 else
Benjamin Peterson59b08c12015-06-27 13:41:33 -05006141 PyMem_RESIZE(self->marks, Py_ssize_t, alloc);
6142 if (self->marks == NULL) {
6143 self->marks_size = 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006144 PyErr_NoMemory();
6145 return -1;
6146 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006147 self->marks_size = (Py_ssize_t)alloc;
6148 }
6149
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02006150 self->stack->mark_set = 1;
6151 self->marks[self->num_marks++] = self->stack->fence = Py_SIZE(self->stack);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006152
6153 return 0;
6154}
6155
6156static int
6157load_reduce(UnpicklerObject *self)
6158{
6159 PyObject *callable = NULL;
6160 PyObject *argtup = NULL;
6161 PyObject *obj = NULL;
6162
6163 PDATA_POP(self->stack, argtup);
6164 if (argtup == NULL)
6165 return -1;
6166 PDATA_POP(self->stack, callable);
6167 if (callable) {
Alexander Belopolskyd92f0402010-07-17 22:50:45 +00006168 obj = PyObject_CallObject(callable, argtup);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006169 Py_DECREF(callable);
6170 }
6171 Py_DECREF(argtup);
6172
6173 if (obj == NULL)
6174 return -1;
6175
6176 PDATA_PUSH(self->stack, obj, -1);
6177 return 0;
6178}
6179
6180/* Just raises an error if we don't know the protocol specified. PROTO
6181 * is the first opcode for protocols >= 2.
6182 */
6183static int
6184load_proto(UnpicklerObject *self)
6185{
6186 char *s;
6187 int i;
6188
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006189 if (_Unpickler_Read(self, &s, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006190 return -1;
6191
6192 i = (unsigned char)s[0];
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006193 if (i <= HIGHEST_PROTOCOL) {
6194 self->proto = i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006195 return 0;
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006196 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006197
6198 PyErr_Format(PyExc_ValueError, "unsupported pickle protocol: %d", i);
6199 return -1;
6200}
6201
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08006202static int
6203load_frame(UnpicklerObject *self)
6204{
6205 char *s;
6206 Py_ssize_t frame_len;
6207
6208 if (_Unpickler_Read(self, &s, 8) < 0)
6209 return -1;
6210
6211 frame_len = calc_binsize(s, 8);
6212 if (frame_len < 0) {
6213 PyErr_Format(PyExc_OverflowError,
6214 "FRAME length exceeds system's maximum of %zd bytes",
6215 PY_SSIZE_T_MAX);
6216 return -1;
6217 }
6218
6219 if (_Unpickler_Read(self, &s, frame_len) < 0)
6220 return -1;
6221
6222 /* Rewind to start of frame */
6223 self->next_read_idx -= frame_len;
6224 return 0;
6225}
6226
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006227static PyObject *
6228load(UnpicklerObject *self)
6229{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006230 PyObject *value = NULL;
Christian Heimes27ea78b2014-01-27 01:03:53 +01006231 char *s = NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006232
6233 self->num_marks = 0;
Serhiy Storchaka59fb6342015-12-06 22:01:35 +02006234 self->stack->mark_set = 0;
6235 self->stack->fence = 0;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006236 self->proto = 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006237 if (Py_SIZE(self->stack))
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006238 Pdata_clear(self->stack, 0);
6239
6240 /* Convenient macros for the dispatch while-switch loop just below. */
6241#define OP(opcode, load_func) \
6242 case opcode: if (load_func(self) < 0) break; continue;
6243
6244#define OP_ARG(opcode, load_func, arg) \
6245 case opcode: if (load_func(self, (arg)) < 0) break; continue;
6246
6247 while (1) {
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006248 if (_Unpickler_Read(self, &s, 1) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006249 break;
6250
6251 switch ((enum opcode)s[0]) {
6252 OP(NONE, load_none)
6253 OP(BININT, load_binint)
6254 OP(BININT1, load_binint1)
6255 OP(BININT2, load_binint2)
6256 OP(INT, load_int)
6257 OP(LONG, load_long)
6258 OP_ARG(LONG1, load_counted_long, 1)
6259 OP_ARG(LONG4, load_counted_long, 4)
6260 OP(FLOAT, load_float)
6261 OP(BINFLOAT, load_binfloat)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006262 OP_ARG(SHORT_BINBYTES, load_counted_binbytes, 1)
6263 OP_ARG(BINBYTES, load_counted_binbytes, 4)
6264 OP_ARG(BINBYTES8, load_counted_binbytes, 8)
6265 OP_ARG(SHORT_BINSTRING, load_counted_binstring, 1)
6266 OP_ARG(BINSTRING, load_counted_binstring, 4)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006267 OP(STRING, load_string)
6268 OP(UNICODE, load_unicode)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006269 OP_ARG(SHORT_BINUNICODE, load_counted_binunicode, 1)
6270 OP_ARG(BINUNICODE, load_counted_binunicode, 4)
6271 OP_ARG(BINUNICODE8, load_counted_binunicode, 8)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006272 OP_ARG(EMPTY_TUPLE, load_counted_tuple, 0)
6273 OP_ARG(TUPLE1, load_counted_tuple, 1)
6274 OP_ARG(TUPLE2, load_counted_tuple, 2)
6275 OP_ARG(TUPLE3, load_counted_tuple, 3)
6276 OP(TUPLE, load_tuple)
6277 OP(EMPTY_LIST, load_empty_list)
6278 OP(LIST, load_list)
6279 OP(EMPTY_DICT, load_empty_dict)
6280 OP(DICT, load_dict)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006281 OP(EMPTY_SET, load_empty_set)
6282 OP(ADDITEMS, load_additems)
6283 OP(FROZENSET, load_frozenset)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006284 OP(OBJ, load_obj)
6285 OP(INST, load_inst)
6286 OP(NEWOBJ, load_newobj)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006287 OP(NEWOBJ_EX, load_newobj_ex)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006288 OP(GLOBAL, load_global)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006289 OP(STACK_GLOBAL, load_stack_global)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006290 OP(APPEND, load_append)
6291 OP(APPENDS, load_appends)
6292 OP(BUILD, load_build)
6293 OP(DUP, load_dup)
6294 OP(BINGET, load_binget)
6295 OP(LONG_BINGET, load_long_binget)
6296 OP(GET, load_get)
6297 OP(MARK, load_mark)
6298 OP(BINPUT, load_binput)
6299 OP(LONG_BINPUT, load_long_binput)
6300 OP(PUT, load_put)
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006301 OP(MEMOIZE, load_memoize)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006302 OP(POP, load_pop)
6303 OP(POP_MARK, load_pop_mark)
6304 OP(SETITEM, load_setitem)
6305 OP(SETITEMS, load_setitems)
6306 OP(PERSID, load_persid)
6307 OP(BINPERSID, load_binpersid)
6308 OP(REDUCE, load_reduce)
6309 OP(PROTO, load_proto)
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08006310 OP(FRAME, load_frame)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006311 OP_ARG(EXT1, load_extension, 1)
6312 OP_ARG(EXT2, load_extension, 2)
6313 OP_ARG(EXT4, load_extension, 4)
6314 OP_ARG(NEWTRUE, load_bool, Py_True)
6315 OP_ARG(NEWFALSE, load_bool, Py_False)
6316
6317 case STOP:
6318 break;
6319
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006320 default:
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006321 if (s[0] == '\0') {
Benjamin Petersonadde86d2011-09-23 13:41:41 -04006322 PyErr_SetNone(PyExc_EOFError);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006323 }
6324 else {
6325 PickleState *st = _Pickle_GetGlobalState();
6326 PyErr_Format(st->UnpicklingError,
Benjamin Petersonadde86d2011-09-23 13:41:41 -04006327 "invalid load key, '%c'.", s[0]);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006328 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006329 return NULL;
6330 }
6331
6332 break; /* and we are done! */
6333 }
6334
Alexandre Vassalottib6a2f2a2013-11-23 20:30:03 -08006335 if (PyErr_Occurred()) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006336 return NULL;
6337 }
6338
Victor Stinner2ae57e32013-10-31 13:39:23 +01006339 if (_Unpickler_SkipConsumed(self) < 0)
6340 return NULL;
6341
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006342 PDATA_POP(self->stack, value);
6343 return value;
6344}
6345
Larry Hastings61272b72014-01-07 12:41:53 -08006346/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006347
6348_pickle.Unpickler.load
6349
6350Load a pickle.
6351
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006352Read a pickled object representation from the open file object given
6353in the constructor, and return the reconstituted object hierarchy
6354specified therein.
Larry Hastings61272b72014-01-07 12:41:53 -08006355[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006356
Larry Hastings3cceb382014-01-04 11:09:09 -08006357static PyObject *
Larry Hastingsc2047262014-01-25 20:43:29 -08006358_pickle_Unpickler_load_impl(UnpicklerObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08006359/*[clinic end generated code: output=fdcc488aad675b14 input=acbb91a42fa9b7b9]*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006360{
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006361 UnpicklerObject *unpickler = (UnpicklerObject*)self;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006362
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006363 /* Check whether the Unpickler was initialized correctly. This prevents
6364 segfaulting if a subclass overridden __init__ with a function that does
6365 not call Unpickler.__init__(). Here, we simply ensure that self->read
6366 is not NULL. */
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006367 if (unpickler->read == NULL) {
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006368 PickleState *st = _Pickle_GetGlobalState();
6369 PyErr_Format(st->UnpicklingError,
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006370 "Unpickler.__init__() was not called by %s.__init__()",
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006371 Py_TYPE(unpickler)->tp_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006372 return NULL;
6373 }
6374
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006375 return load(unpickler);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006376}
6377
6378/* The name of find_class() is misleading. In newer pickle protocols, this
6379 function is used for loading any global (i.e., functions), not just
6380 classes. The name is kept only for backward compatibility. */
6381
Larry Hastings61272b72014-01-07 12:41:53 -08006382/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006383
6384_pickle.Unpickler.find_class
6385
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006386 module_name: object
6387 global_name: object
6388 /
6389
6390Return an object from a specified module.
6391
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006392If necessary, the module will be imported. Subclasses may override
6393this method (e.g. to restrict unpickling of arbitrary classes and
6394functions).
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006395
6396This method is called whenever a class or a function object is
6397needed. Both arguments passed are str objects.
Larry Hastings61272b72014-01-07 12:41:53 -08006398[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006399
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006400static PyObject *
Larry Hastings89964c42015-04-14 18:07:59 -04006401_pickle_Unpickler_find_class_impl(UnpicklerObject *self,
6402 PyObject *module_name,
6403 PyObject *global_name)
6404/*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006405{
6406 PyObject *global;
6407 PyObject *modules_dict;
6408 PyObject *module;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006409 _Py_IDENTIFIER(modules);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006410
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006411 /* Try to map the old names used in Python 2.x to the new ones used in
6412 Python 3.x. We do this only with old pickle protocols and when the
6413 user has not disabled the feature. */
6414 if (self->proto < 3 && self->fix_imports) {
6415 PyObject *key;
6416 PyObject *item;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006417 PickleState *st = _Pickle_GetGlobalState();
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006418
6419 /* Check if the global (i.e., a function or a class) was renamed
6420 or moved to another module. */
6421 key = PyTuple_Pack(2, module_name, global_name);
6422 if (key == NULL)
6423 return NULL;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08006424 item = PyDict_GetItemWithError(st->name_mapping_2to3, key);
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006425 Py_DECREF(key);
6426 if (item) {
6427 if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
6428 PyErr_Format(PyExc_RuntimeError,
6429 "_compat_pickle.NAME_MAPPING values should be "
6430 "2-tuples, not %.200s", Py_TYPE(item)->tp_name);
6431 return NULL;
6432 }
6433 module_name = PyTuple_GET_ITEM(item, 0);
6434 global_name = PyTuple_GET_ITEM(item, 1);
6435 if (!PyUnicode_Check(module_name) ||
6436 !PyUnicode_Check(global_name)) {
6437 PyErr_Format(PyExc_RuntimeError,
6438 "_compat_pickle.NAME_MAPPING values should be "
6439 "pairs of str, not (%.200s, %.200s)",
6440 Py_TYPE(module_name)->tp_name,
6441 Py_TYPE(global_name)->tp_name);
6442 return NULL;
6443 }
6444 }
6445 else if (PyErr_Occurred()) {
6446 return NULL;
6447 }
Serhiy Storchakabfe18242015-03-31 13:12:37 +03006448 else {
6449 /* Check if the module was renamed. */
6450 item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name);
6451 if (item) {
6452 if (!PyUnicode_Check(item)) {
6453 PyErr_Format(PyExc_RuntimeError,
6454 "_compat_pickle.IMPORT_MAPPING values should be "
6455 "strings, not %.200s", Py_TYPE(item)->tp_name);
6456 return NULL;
6457 }
6458 module_name = item;
6459 }
6460 else if (PyErr_Occurred()) {
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006461 return NULL;
6462 }
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006463 }
6464 }
6465
Victor Stinnerbb520202013-11-06 22:40:41 +01006466 modules_dict = _PySys_GetObjectId(&PyId_modules);
Victor Stinner1e53bba2013-07-16 22:26:05 +02006467 if (modules_dict == NULL) {
6468 PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006469 return NULL;
Victor Stinner1e53bba2013-07-16 22:26:05 +02006470 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006471
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006472 module = PyDict_GetItemWithError(modules_dict, module_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006473 if (module == NULL) {
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006474 if (PyErr_Occurred())
6475 return NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006476 module = PyImport_Import(module_name);
6477 if (module == NULL)
6478 return NULL;
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006479 global = getattribute(module, global_name, self->proto >= 4);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006480 Py_DECREF(module);
6481 }
Victor Stinner121aab42011-09-29 23:40:53 +02006482 else {
Antoine Pitrouc9dc4a22013-11-23 18:59:12 +01006483 global = getattribute(module, global_name, self->proto >= 4);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006484 }
6485 return global;
6486}
6487
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +02006488/*[clinic input]
6489
6490_pickle.Unpickler.__sizeof__ -> Py_ssize_t
6491
6492Returns size in memory, in bytes.
6493[clinic start generated code]*/
6494
6495static Py_ssize_t
6496_pickle_Unpickler___sizeof___impl(UnpicklerObject *self)
6497/*[clinic end generated code: output=119d9d03ad4c7651 input=13333471fdeedf5e]*/
6498{
6499 Py_ssize_t res;
6500
Serhiy Storchaka5c4064e2015-12-19 20:05:25 +02006501 res = _PyObject_SIZE(Py_TYPE(self));
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +02006502 if (self->memo != NULL)
6503 res += self->memo_size * sizeof(PyObject *);
6504 if (self->marks != NULL)
6505 res += self->marks_size * sizeof(Py_ssize_t);
6506 if (self->input_line != NULL)
6507 res += strlen(self->input_line) + 1;
6508 if (self->encoding != NULL)
6509 res += strlen(self->encoding) + 1;
6510 if (self->errors != NULL)
6511 res += strlen(self->errors) + 1;
6512 return res;
6513}
6514
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006515static struct PyMethodDef Unpickler_methods[] = {
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006516 _PICKLE_UNPICKLER_LOAD_METHODDEF
6517 _PICKLE_UNPICKLER_FIND_CLASS_METHODDEF
Serhiy Storchaka5bbd2312014-12-16 19:39:08 +02006518 _PICKLE_UNPICKLER___SIZEOF___METHODDEF
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006519 {NULL, NULL} /* sentinel */
6520};
6521
6522static void
6523Unpickler_dealloc(UnpicklerObject *self)
6524{
6525 PyObject_GC_UnTrack((PyObject *)self);
6526 Py_XDECREF(self->readline);
6527 Py_XDECREF(self->read);
Antoine Pitrou04248a82010-10-12 20:51:21 +00006528 Py_XDECREF(self->peek);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006529 Py_XDECREF(self->stack);
6530 Py_XDECREF(self->pers_func);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006531 if (self->buffer.buf != NULL) {
6532 PyBuffer_Release(&self->buffer);
6533 self->buffer.buf = NULL;
6534 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006535
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006536 _Unpickler_MemoCleanup(self);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006537 PyMem_Free(self->marks);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006538 PyMem_Free(self->input_line);
Victor Stinner49fc8ec2013-07-07 23:30:24 +02006539 PyMem_Free(self->encoding);
6540 PyMem_Free(self->errors);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006541
6542 Py_TYPE(self)->tp_free((PyObject *)self);
6543}
6544
6545static int
6546Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg)
6547{
6548 Py_VISIT(self->readline);
6549 Py_VISIT(self->read);
Antoine Pitrou04248a82010-10-12 20:51:21 +00006550 Py_VISIT(self->peek);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006551 Py_VISIT(self->stack);
6552 Py_VISIT(self->pers_func);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006553 return 0;
6554}
6555
6556static int
6557Unpickler_clear(UnpicklerObject *self)
6558{
6559 Py_CLEAR(self->readline);
6560 Py_CLEAR(self->read);
Antoine Pitrou04248a82010-10-12 20:51:21 +00006561 Py_CLEAR(self->peek);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006562 Py_CLEAR(self->stack);
6563 Py_CLEAR(self->pers_func);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006564 if (self->buffer.buf != NULL) {
6565 PyBuffer_Release(&self->buffer);
6566 self->buffer.buf = NULL;
6567 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006568
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006569 _Unpickler_MemoCleanup(self);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006570 PyMem_Free(self->marks);
6571 self->marks = NULL;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006572 PyMem_Free(self->input_line);
6573 self->input_line = NULL;
Victor Stinner49fc8ec2013-07-07 23:30:24 +02006574 PyMem_Free(self->encoding);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006575 self->encoding = NULL;
Victor Stinner49fc8ec2013-07-07 23:30:24 +02006576 PyMem_Free(self->errors);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006577 self->errors = NULL;
6578
6579 return 0;
6580}
6581
Larry Hastings61272b72014-01-07 12:41:53 -08006582/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006583
6584_pickle.Unpickler.__init__
6585
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006586 file: object
6587 *
6588 fix_imports: bool = True
6589 encoding: str = 'ASCII'
6590 errors: str = 'strict'
6591
6592This takes a binary file for reading a pickle data stream.
6593
6594The protocol version of the pickle is detected automatically, so no
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006595protocol argument is needed. Bytes past the pickled object's
6596representation are ignored.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006597
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006598The argument *file* must have two methods, a read() method that takes
6599an integer argument, and a readline() method that requires no
6600arguments. Both methods should return bytes. Thus *file* can be a
Martin Panter7462b6492015-11-02 03:37:02 +00006601binary file object opened for reading, an io.BytesIO object, or any
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006602other custom object that meets this interface.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006603
6604Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
Martin Panter46f50722016-05-26 05:35:26 +00006605which are used to control compatibility support for pickle stream
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006606generated by Python 2. If *fix_imports* is True, pickle will try to
6607map the old Python 2 names to the new names used in Python 3. The
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006608*encoding* and *errors* tell pickle how to decode 8-bit string
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08006609instances pickled by Python 2; these default to 'ASCII' and 'strict',
6610respectively. The *encoding* can be 'bytes' to read these 8-bit
6611string instances as bytes objects.
Larry Hastings61272b72014-01-07 12:41:53 -08006612[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006613
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006614static int
Larry Hastings89964c42015-04-14 18:07:59 -04006615_pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file,
6616 int fix_imports, const char *encoding,
6617 const char *errors)
Martin Panter46f50722016-05-26 05:35:26 +00006618/*[clinic end generated code: output=e2c8ce748edc57b0 input=f9b7da04f5f4f335]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006619{
Martin v. Löwis1c67dd92011-10-14 15:16:45 +02006620 _Py_IDENTIFIER(persistent_load);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006621
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006622 /* In case of multiple __init__() calls, clear previous content. */
6623 if (self->read != NULL)
6624 (void)Unpickler_clear(self);
6625
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006626 if (_Unpickler_SetInputStream(self, file) < 0)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006627 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006628
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006629 if (_Unpickler_SetInputEncoding(self, encoding, errors) < 0)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006630 return -1;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006631
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006632 self->fix_imports = fix_imports;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006633 if (self->fix_imports == -1)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006634 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006635
Martin v. Löwis1c67dd92011-10-14 15:16:45 +02006636 if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_load)) {
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +02006637 self->pers_func = _PyObject_GetAttrId((PyObject *)self,
6638 &PyId_persistent_load);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006639 if (self->pers_func == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006640 return 1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006641 }
6642 else {
6643 self->pers_func = NULL;
6644 }
6645
6646 self->stack = (Pdata *)Pdata_New();
6647 if (self->stack == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006648 return 1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006649
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006650 self->memo_size = 32;
6651 self->memo = _Unpickler_NewMemo(self->memo_size);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006652 if (self->memo == NULL)
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006653 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006654
Antoine Pitroud9dfaa92009-06-04 20:32:06 +00006655 self->proto = 0;
Alexandre Vassalotti0e7aa8c2009-04-03 04:17:41 +00006656
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006657 return 0;
6658}
6659
Larry Hastingsb7ccb202014-01-18 23:50:21 -08006660
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006661/* Define a proxy object for the Unpickler's internal memo object. This is to
6662 * avoid breaking code like:
6663 * unpickler.memo.clear()
6664 * and
6665 * unpickler.memo = saved_memo
6666 * Is this a good idea? Not really, but we don't want to break code that uses
6667 * it. Note that we don't implement the entire mapping API here. This is
6668 * intentional, as these should be treated as black-box implementation details.
6669 *
6670 * We do, however, have to implement pickling/unpickling support because of
Victor Stinner121aab42011-09-29 23:40:53 +02006671 * real-world code like cvs2svn.
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006672 */
6673
Larry Hastings61272b72014-01-07 12:41:53 -08006674/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006675_pickle.UnpicklerMemoProxy.clear
6676
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006677Remove all items from memo.
Larry Hastings61272b72014-01-07 12:41:53 -08006678[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006679
Larry Hastings3cceb382014-01-04 11:09:09 -08006680static PyObject *
6681_pickle_UnpicklerMemoProxy_clear_impl(UnpicklerMemoProxyObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08006682/*[clinic end generated code: output=d20cd43f4ba1fb1f input=b1df7c52e7afd9bd]*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006683{
6684 _Unpickler_MemoCleanup(self->unpickler);
6685 self->unpickler->memo = _Unpickler_NewMemo(self->unpickler->memo_size);
6686 if (self->unpickler->memo == NULL)
6687 return NULL;
6688 Py_RETURN_NONE;
6689}
6690
Larry Hastings61272b72014-01-07 12:41:53 -08006691/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006692_pickle.UnpicklerMemoProxy.copy
6693
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006694Copy the memo to a new object.
Larry Hastings61272b72014-01-07 12:41:53 -08006695[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006696
Larry Hastings3cceb382014-01-04 11:09:09 -08006697static PyObject *
6698_pickle_UnpicklerMemoProxy_copy_impl(UnpicklerMemoProxyObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08006699/*[clinic end generated code: output=e12af7e9bc1e4c77 input=97769247ce032c1d]*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006700{
6701 Py_ssize_t i;
6702 PyObject *new_memo = PyDict_New();
6703 if (new_memo == NULL)
6704 return NULL;
6705
6706 for (i = 0; i < self->unpickler->memo_size; i++) {
6707 int status;
6708 PyObject *key, *value;
6709
6710 value = self->unpickler->memo[i];
6711 if (value == NULL)
6712 continue;
6713
6714 key = PyLong_FromSsize_t(i);
6715 if (key == NULL)
6716 goto error;
6717 status = PyDict_SetItem(new_memo, key, value);
6718 Py_DECREF(key);
6719 if (status < 0)
6720 goto error;
6721 }
6722 return new_memo;
6723
6724error:
6725 Py_DECREF(new_memo);
6726 return NULL;
6727}
6728
Larry Hastings61272b72014-01-07 12:41:53 -08006729/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006730_pickle.UnpicklerMemoProxy.__reduce__
6731
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006732Implement pickling support.
Larry Hastings61272b72014-01-07 12:41:53 -08006733[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006734
Larry Hastings3cceb382014-01-04 11:09:09 -08006735static PyObject *
6736_pickle_UnpicklerMemoProxy___reduce___impl(UnpicklerMemoProxyObject *self)
Larry Hastings581ee362014-01-28 05:00:08 -08006737/*[clinic end generated code: output=6da34ac048d94cca input=6920862413407199]*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006738{
6739 PyObject *reduce_value;
6740 PyObject *constructor_args;
Larry Hastings3cceb382014-01-04 11:09:09 -08006741 PyObject *contents = _pickle_UnpicklerMemoProxy_copy_impl(self);
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006742 if (contents == NULL)
6743 return NULL;
6744
6745 reduce_value = PyTuple_New(2);
6746 if (reduce_value == NULL) {
6747 Py_DECREF(contents);
6748 return NULL;
6749 }
6750 constructor_args = PyTuple_New(1);
6751 if (constructor_args == NULL) {
6752 Py_DECREF(contents);
6753 Py_DECREF(reduce_value);
6754 return NULL;
6755 }
6756 PyTuple_SET_ITEM(constructor_args, 0, contents);
6757 Py_INCREF((PyObject *)&PyDict_Type);
6758 PyTuple_SET_ITEM(reduce_value, 0, (PyObject *)&PyDict_Type);
6759 PyTuple_SET_ITEM(reduce_value, 1, constructor_args);
6760 return reduce_value;
6761}
6762
6763static PyMethodDef unpicklerproxy_methods[] = {
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006764 _PICKLE_UNPICKLERMEMOPROXY_CLEAR_METHODDEF
6765 _PICKLE_UNPICKLERMEMOPROXY_COPY_METHODDEF
6766 _PICKLE_UNPICKLERMEMOPROXY___REDUCE___METHODDEF
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006767 {NULL, NULL} /* sentinel */
6768};
6769
6770static void
6771UnpicklerMemoProxy_dealloc(UnpicklerMemoProxyObject *self)
6772{
6773 PyObject_GC_UnTrack(self);
6774 Py_XDECREF(self->unpickler);
6775 PyObject_GC_Del((PyObject *)self);
6776}
6777
6778static int
6779UnpicklerMemoProxy_traverse(UnpicklerMemoProxyObject *self,
6780 visitproc visit, void *arg)
6781{
6782 Py_VISIT(self->unpickler);
6783 return 0;
6784}
6785
6786static int
6787UnpicklerMemoProxy_clear(UnpicklerMemoProxyObject *self)
6788{
6789 Py_CLEAR(self->unpickler);
6790 return 0;
6791}
6792
6793static PyTypeObject UnpicklerMemoProxyType = {
6794 PyVarObject_HEAD_INIT(NULL, 0)
6795 "_pickle.UnpicklerMemoProxy", /*tp_name*/
6796 sizeof(UnpicklerMemoProxyObject), /*tp_basicsize*/
6797 0,
6798 (destructor)UnpicklerMemoProxy_dealloc, /* tp_dealloc */
6799 0, /* tp_print */
6800 0, /* tp_getattr */
6801 0, /* tp_setattr */
6802 0, /* tp_compare */
6803 0, /* tp_repr */
6804 0, /* tp_as_number */
6805 0, /* tp_as_sequence */
6806 0, /* tp_as_mapping */
Georg Brandlf038b322010-10-18 07:35:09 +00006807 PyObject_HashNotImplemented, /* tp_hash */
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006808 0, /* tp_call */
6809 0, /* tp_str */
6810 PyObject_GenericGetAttr, /* tp_getattro */
6811 PyObject_GenericSetAttr, /* tp_setattro */
6812 0, /* tp_as_buffer */
6813 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
6814 0, /* tp_doc */
6815 (traverseproc)UnpicklerMemoProxy_traverse, /* tp_traverse */
6816 (inquiry)UnpicklerMemoProxy_clear, /* tp_clear */
6817 0, /* tp_richcompare */
6818 0, /* tp_weaklistoffset */
6819 0, /* tp_iter */
6820 0, /* tp_iternext */
6821 unpicklerproxy_methods, /* tp_methods */
6822};
6823
6824static PyObject *
6825UnpicklerMemoProxy_New(UnpicklerObject *unpickler)
6826{
6827 UnpicklerMemoProxyObject *self;
6828
6829 self = PyObject_GC_New(UnpicklerMemoProxyObject,
6830 &UnpicklerMemoProxyType);
6831 if (self == NULL)
6832 return NULL;
6833 Py_INCREF(unpickler);
6834 self->unpickler = unpickler;
6835 PyObject_GC_Track(self);
6836 return (PyObject *)self;
6837}
6838
6839/*****************************************************************************/
6840
6841
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006842static PyObject *
6843Unpickler_get_memo(UnpicklerObject *self)
6844{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006845 return UnpicklerMemoProxy_New(self);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006846}
6847
6848static int
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006849Unpickler_set_memo(UnpicklerObject *self, PyObject *obj)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006850{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006851 PyObject **new_memo;
6852 Py_ssize_t new_memo_size = 0;
6853 Py_ssize_t i;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006854
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006855 if (obj == NULL) {
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006856 PyErr_SetString(PyExc_TypeError,
6857 "attribute deletion is not supported");
6858 return -1;
6859 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006860
6861 if (Py_TYPE(obj) == &UnpicklerMemoProxyType) {
6862 UnpicklerObject *unpickler =
6863 ((UnpicklerMemoProxyObject *)obj)->unpickler;
6864
6865 new_memo_size = unpickler->memo_size;
6866 new_memo = _Unpickler_NewMemo(new_memo_size);
6867 if (new_memo == NULL)
6868 return -1;
6869
6870 for (i = 0; i < new_memo_size; i++) {
6871 Py_XINCREF(unpickler->memo[i]);
6872 new_memo[i] = unpickler->memo[i];
6873 }
6874 }
6875 else if (PyDict_Check(obj)) {
6876 Py_ssize_t i = 0;
6877 PyObject *key, *value;
6878
6879 new_memo_size = PyDict_Size(obj);
6880 new_memo = _Unpickler_NewMemo(new_memo_size);
6881 if (new_memo == NULL)
6882 return -1;
6883
6884 while (PyDict_Next(obj, &i, &key, &value)) {
6885 Py_ssize_t idx;
6886 if (!PyLong_Check(key)) {
6887 PyErr_SetString(PyExc_TypeError,
6888 "memo key must be integers");
6889 goto error;
6890 }
6891 idx = PyLong_AsSsize_t(key);
6892 if (idx == -1 && PyErr_Occurred())
6893 goto error;
Christian Heimesa24b4d22013-07-01 15:17:45 +02006894 if (idx < 0) {
6895 PyErr_SetString(PyExc_ValueError,
Christian Heimes80878792013-07-01 15:23:39 +02006896 "memo key must be positive integers.");
Christian Heimesa24b4d22013-07-01 15:17:45 +02006897 goto error;
6898 }
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006899 if (_Unpickler_MemoPut(self, idx, value) < 0)
6900 goto error;
6901 }
6902 }
6903 else {
6904 PyErr_Format(PyExc_TypeError,
6905 "'memo' attribute must be an UnpicklerMemoProxy object"
6906 "or dict, not %.200s", Py_TYPE(obj)->tp_name);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006907 return -1;
6908 }
6909
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006910 _Unpickler_MemoCleanup(self);
6911 self->memo_size = new_memo_size;
6912 self->memo = new_memo;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006913
6914 return 0;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006915
6916 error:
6917 if (new_memo_size) {
6918 i = new_memo_size;
6919 while (--i >= 0) {
6920 Py_XDECREF(new_memo[i]);
6921 }
6922 PyMem_FREE(new_memo);
6923 }
6924 return -1;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006925}
6926
6927static PyObject *
6928Unpickler_get_persload(UnpicklerObject *self)
6929{
6930 if (self->pers_func == NULL)
6931 PyErr_SetString(PyExc_AttributeError, "persistent_load");
6932 else
6933 Py_INCREF(self->pers_func);
6934 return self->pers_func;
6935}
6936
6937static int
6938Unpickler_set_persload(UnpicklerObject *self, PyObject *value)
6939{
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006940 if (value == NULL) {
6941 PyErr_SetString(PyExc_TypeError,
6942 "attribute deletion is not supported");
6943 return -1;
6944 }
6945 if (!PyCallable_Check(value)) {
6946 PyErr_SetString(PyExc_TypeError,
6947 "persistent_load must be a callable taking "
6948 "one argument");
6949 return -1;
6950 }
6951
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006952 Py_INCREF(value);
Serhiy Storchakaec397562016-04-06 09:50:03 +03006953 Py_XSETREF(self->pers_func, value);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006954
6955 return 0;
6956}
6957
6958static PyGetSetDef Unpickler_getsets[] = {
6959 {"memo", (getter)Unpickler_get_memo, (setter)Unpickler_set_memo},
6960 {"persistent_load", (getter)Unpickler_get_persload,
6961 (setter)Unpickler_set_persload},
6962 {NULL}
6963};
6964
6965static PyTypeObject Unpickler_Type = {
6966 PyVarObject_HEAD_INIT(NULL, 0)
6967 "_pickle.Unpickler", /*tp_name*/
6968 sizeof(UnpicklerObject), /*tp_basicsize*/
6969 0, /*tp_itemsize*/
6970 (destructor)Unpickler_dealloc, /*tp_dealloc*/
6971 0, /*tp_print*/
6972 0, /*tp_getattr*/
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00006973 0, /*tp_setattr*/
Mark Dickinsone94c6792009-02-02 20:36:42 +00006974 0, /*tp_reserved*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006975 0, /*tp_repr*/
6976 0, /*tp_as_number*/
6977 0, /*tp_as_sequence*/
6978 0, /*tp_as_mapping*/
6979 0, /*tp_hash*/
6980 0, /*tp_call*/
6981 0, /*tp_str*/
6982 0, /*tp_getattro*/
6983 0, /*tp_setattro*/
6984 0, /*tp_as_buffer*/
6985 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08006986 _pickle_Unpickler___init____doc__, /*tp_doc*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00006987 (traverseproc)Unpickler_traverse, /*tp_traverse*/
6988 (inquiry)Unpickler_clear, /*tp_clear*/
6989 0, /*tp_richcompare*/
6990 0, /*tp_weaklistoffset*/
6991 0, /*tp_iter*/
6992 0, /*tp_iternext*/
6993 Unpickler_methods, /*tp_methods*/
6994 0, /*tp_members*/
6995 Unpickler_getsets, /*tp_getset*/
6996 0, /*tp_base*/
6997 0, /*tp_dict*/
6998 0, /*tp_descr_get*/
6999 0, /*tp_descr_set*/
7000 0, /*tp_dictoffset*/
Larry Hastingsb7ccb202014-01-18 23:50:21 -08007001 _pickle_Unpickler___init__, /*tp_init*/
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007002 PyType_GenericAlloc, /*tp_alloc*/
7003 PyType_GenericNew, /*tp_new*/
7004 PyObject_GC_Del, /*tp_free*/
7005 0, /*tp_is_gc*/
7006};
7007
Larry Hastings61272b72014-01-07 12:41:53 -08007008/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007009
7010_pickle.dump
7011
7012 obj: object
7013 file: object
7014 protocol: object = NULL
7015 *
7016 fix_imports: bool = True
7017
7018Write a pickled representation of obj to the open file object file.
7019
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007020This is equivalent to ``Pickler(file, protocol).dump(obj)``, but may
7021be more efficient.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007022
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007023The optional *protocol* argument tells the pickler to use the given
7024protocol supported protocols are 0, 1, 2, 3 and 4. The default
7025protocol is 3; a backward-incompatible protocol designed for Python 3.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007026
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007027Specifying a negative protocol version selects the highest protocol
7028version supported. The higher the protocol used, the more recent the
7029version of Python needed to read the pickle produced.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007030
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007031The *file* argument must have a write() method that accepts a single
7032bytes argument. It can thus be a file object opened for binary
Martin Panter7462b6492015-11-02 03:37:02 +00007033writing, an io.BytesIO instance, or any other custom object that meets
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007034this interface.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007035
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007036If *fix_imports* is True and protocol is less than 3, pickle will try
7037to map the new Python 3 names to the old module names used in Python
70382, so that the pickle data stream is readable with Python 2.
Larry Hastings61272b72014-01-07 12:41:53 -08007039[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007040
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007041static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007042_pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file,
Larry Hastings89964c42015-04-14 18:07:59 -04007043 PyObject *protocol, int fix_imports)
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007044/*[clinic end generated code: output=a4774d5fde7d34de input=830f8a64cef6f042]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007045{
7046 PicklerObject *pickler = _Pickler_New();
7047
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007048 if (pickler == NULL)
7049 return NULL;
7050
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007051 if (_Pickler_SetProtocol(pickler, protocol, fix_imports) < 0)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007052 goto error;
7053
7054 if (_Pickler_SetOutputStream(pickler, file) < 0)
7055 goto error;
7056
7057 if (dump(pickler, obj) < 0)
7058 goto error;
7059
7060 if (_Pickler_FlushToFile(pickler) < 0)
7061 goto error;
7062
7063 Py_DECREF(pickler);
7064 Py_RETURN_NONE;
7065
7066 error:
7067 Py_XDECREF(pickler);
7068 return NULL;
7069}
7070
Larry Hastings61272b72014-01-07 12:41:53 -08007071/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007072
7073_pickle.dumps
7074
7075 obj: object
7076 protocol: object = NULL
7077 *
7078 fix_imports: bool = True
7079
7080Return the pickled representation of the object as a bytes object.
7081
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007082The optional *protocol* argument tells the pickler to use the given
7083protocol; supported protocols are 0, 1, 2, 3 and 4. The default
7084protocol is 3; a backward-incompatible protocol designed for Python 3.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007085
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007086Specifying a negative protocol version selects the highest protocol
7087version supported. The higher the protocol used, the more recent the
7088version of Python needed to read the pickle produced.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007089
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007090If *fix_imports* is True and *protocol* is less than 3, pickle will
7091try to map the new Python 3 names to the old module names used in
7092Python 2, so that the pickle data stream is readable with Python 2.
Larry Hastings61272b72014-01-07 12:41:53 -08007093[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007094
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007095static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007096_pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol,
Larry Hastings89964c42015-04-14 18:07:59 -04007097 int fix_imports)
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007098/*[clinic end generated code: output=d75d5cda456fd261 input=293dbeda181580b7]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007099{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007100 PyObject *result;
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007101 PicklerObject *pickler = _Pickler_New();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007102
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007103 if (pickler == NULL)
7104 return NULL;
7105
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007106 if (_Pickler_SetProtocol(pickler, protocol, fix_imports) < 0)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007107 goto error;
7108
7109 if (dump(pickler, obj) < 0)
7110 goto error;
7111
7112 result = _Pickler_GetString(pickler);
7113 Py_DECREF(pickler);
7114 return result;
7115
7116 error:
7117 Py_XDECREF(pickler);
7118 return NULL;
7119}
7120
Larry Hastings61272b72014-01-07 12:41:53 -08007121/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007122
7123_pickle.load
7124
7125 file: object
7126 *
7127 fix_imports: bool = True
7128 encoding: str = 'ASCII'
7129 errors: str = 'strict'
7130
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007131Read and return an object from the pickle data stored in a file.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007132
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007133This is equivalent to ``Unpickler(file).load()``, but may be more
7134efficient.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007135
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007136The protocol version of the pickle is detected automatically, so no
7137protocol argument is needed. Bytes past the pickled object's
7138representation are ignored.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007139
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007140The argument *file* must have two methods, a read() method that takes
7141an integer argument, and a readline() method that requires no
7142arguments. Both methods should return bytes. Thus *file* can be a
Martin Panter7462b6492015-11-02 03:37:02 +00007143binary file object opened for reading, an io.BytesIO object, or any
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007144other custom object that meets this interface.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007145
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007146Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
Martin Panter46f50722016-05-26 05:35:26 +00007147which are used to control compatibility support for pickle stream
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007148generated by Python 2. If *fix_imports* is True, pickle will try to
7149map the old Python 2 names to the new names used in Python 3. The
7150*encoding* and *errors* tell pickle how to decode 8-bit string
7151instances pickled by Python 2; these default to 'ASCII' and 'strict',
7152respectively. The *encoding* can be 'bytes' to read these 8-bit
7153string instances as bytes objects.
Larry Hastings61272b72014-01-07 12:41:53 -08007154[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007155
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007156static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007157_pickle_load_impl(PyObject *module, PyObject *file, int fix_imports,
Larry Hastings89964c42015-04-14 18:07:59 -04007158 const char *encoding, const char *errors)
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007159/*[clinic end generated code: output=69e298160285199e input=01b44dd3fc07afa7]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007160{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007161 PyObject *result;
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007162 UnpicklerObject *unpickler = _Unpickler_New();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007163
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007164 if (unpickler == NULL)
7165 return NULL;
7166
7167 if (_Unpickler_SetInputStream(unpickler, file) < 0)
7168 goto error;
7169
7170 if (_Unpickler_SetInputEncoding(unpickler, encoding, errors) < 0)
7171 goto error;
7172
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007173 unpickler->fix_imports = fix_imports;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007174
7175 result = load(unpickler);
7176 Py_DECREF(unpickler);
7177 return result;
7178
7179 error:
7180 Py_XDECREF(unpickler);
7181 return NULL;
7182}
7183
Larry Hastings61272b72014-01-07 12:41:53 -08007184/*[clinic input]
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007185
7186_pickle.loads
7187
7188 data: object
7189 *
7190 fix_imports: bool = True
7191 encoding: str = 'ASCII'
7192 errors: str = 'strict'
7193
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007194Read and return an object from the given pickle data.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007195
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007196The protocol version of the pickle is detected automatically, so no
7197protocol argument is needed. Bytes past the pickled object's
7198representation are ignored.
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007199
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007200Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
Martin Panter46f50722016-05-26 05:35:26 +00007201which are used to control compatibility support for pickle stream
Alexandre Vassalottid05c9ff2013-12-07 01:09:27 -08007202generated by Python 2. If *fix_imports* is True, pickle will try to
7203map the old Python 2 names to the new names used in Python 3. The
7204*encoding* and *errors* tell pickle how to decode 8-bit string
7205instances pickled by Python 2; these default to 'ASCII' and 'strict',
7206respectively. The *encoding* can be 'bytes' to read these 8-bit
7207string instances as bytes objects.
Larry Hastings61272b72014-01-07 12:41:53 -08007208[clinic start generated code]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007209
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007210static PyObject *
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007211_pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports,
Larry Hastings89964c42015-04-14 18:07:59 -04007212 const char *encoding, const char *errors)
Serhiy Storchaka1a2b24f2016-07-07 17:35:15 +03007213/*[clinic end generated code: output=1e7cb2343f2c440f input=70605948a719feb9]*/
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007214{
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007215 PyObject *result;
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007216 UnpicklerObject *unpickler = _Unpickler_New();
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007217
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007218 if (unpickler == NULL)
7219 return NULL;
7220
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007221 if (_Unpickler_SetStringInput(unpickler, data) < 0)
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007222 goto error;
7223
7224 if (_Unpickler_SetInputEncoding(unpickler, encoding, errors) < 0)
7225 goto error;
7226
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007227 unpickler->fix_imports = fix_imports;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007228
7229 result = load(unpickler);
7230 Py_DECREF(unpickler);
7231 return result;
7232
7233 error:
7234 Py_XDECREF(unpickler);
7235 return NULL;
7236}
7237
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007238static struct PyMethodDef pickle_methods[] = {
Alexandre Vassalottied8c9062013-11-24 12:25:48 -08007239 _PICKLE_DUMP_METHODDEF
7240 _PICKLE_DUMPS_METHODDEF
7241 _PICKLE_LOAD_METHODDEF
7242 _PICKLE_LOADS_METHODDEF
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007243 {NULL, NULL} /* sentinel */
7244};
7245
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007246static int
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007247pickle_clear(PyObject *m)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007248{
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007249 _Pickle_ClearState(_Pickle_GetState(m));
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007250 return 0;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007251}
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007252
Stefan Krahf483b0f2013-12-14 13:43:10 +01007253static void
7254pickle_free(PyObject *m)
7255{
7256 _Pickle_ClearState(_Pickle_GetState(m));
7257}
7258
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007259static int
7260pickle_traverse(PyObject *m, visitproc visit, void *arg)
7261{
7262 PickleState *st = _Pickle_GetState(m);
7263 Py_VISIT(st->PickleError);
7264 Py_VISIT(st->PicklingError);
7265 Py_VISIT(st->UnpicklingError);
7266 Py_VISIT(st->dispatch_table);
7267 Py_VISIT(st->extension_registry);
7268 Py_VISIT(st->extension_cache);
7269 Py_VISIT(st->inverted_registry);
7270 Py_VISIT(st->name_mapping_2to3);
7271 Py_VISIT(st->import_mapping_2to3);
7272 Py_VISIT(st->name_mapping_3to2);
7273 Py_VISIT(st->import_mapping_3to2);
7274 Py_VISIT(st->codecs_encode);
Serhiy Storchaka58e41342015-03-31 14:07:24 +03007275 Py_VISIT(st->getattr);
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007276 return 0;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007277}
7278
7279static struct PyModuleDef _picklemodule = {
7280 PyModuleDef_HEAD_INIT,
Stefan Krahf483b0f2013-12-14 13:43:10 +01007281 "_pickle", /* m_name */
7282 pickle_module_doc, /* m_doc */
7283 sizeof(PickleState), /* m_size */
7284 pickle_methods, /* m_methods */
7285 NULL, /* m_reload */
7286 pickle_traverse, /* m_traverse */
7287 pickle_clear, /* m_clear */
7288 (freefunc)pickle_free /* m_free */
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007289};
7290
7291PyMODINIT_FUNC
7292PyInit__pickle(void)
7293{
7294 PyObject *m;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007295 PickleState *st;
7296
7297 m = PyState_FindModule(&_picklemodule);
7298 if (m) {
7299 Py_INCREF(m);
7300 return m;
7301 }
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007302
7303 if (PyType_Ready(&Unpickler_Type) < 0)
7304 return NULL;
7305 if (PyType_Ready(&Pickler_Type) < 0)
7306 return NULL;
7307 if (PyType_Ready(&Pdata_Type) < 0)
7308 return NULL;
Antoine Pitrouea99c5c2010-09-09 18:33:21 +00007309 if (PyType_Ready(&PicklerMemoProxyType) < 0)
7310 return NULL;
7311 if (PyType_Ready(&UnpicklerMemoProxyType) < 0)
7312 return NULL;
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007313
7314 /* Create the module and add the functions. */
7315 m = PyModule_Create(&_picklemodule);
7316 if (m == NULL)
7317 return NULL;
7318
Antoine Pitrou8391cf42011-07-15 21:01:21 +02007319 Py_INCREF(&Pickler_Type);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007320 if (PyModule_AddObject(m, "Pickler", (PyObject *)&Pickler_Type) < 0)
7321 return NULL;
Antoine Pitrou8391cf42011-07-15 21:01:21 +02007322 Py_INCREF(&Unpickler_Type);
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007323 if (PyModule_AddObject(m, "Unpickler", (PyObject *)&Unpickler_Type) < 0)
7324 return NULL;
7325
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007326 st = _Pickle_GetState(m);
7327
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007328 /* Initialize the exceptions. */
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007329 st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL);
7330 if (st->PickleError == NULL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007331 return NULL;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007332 st->PicklingError = \
7333 PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL);
7334 if (st->PicklingError == NULL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007335 return NULL;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007336 st->UnpicklingError = \
7337 PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL);
7338 if (st->UnpicklingError == NULL)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007339 return NULL;
7340
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007341 Py_INCREF(st->PickleError);
7342 if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007343 return NULL;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007344 Py_INCREF(st->PicklingError);
7345 if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007346 return NULL;
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007347 Py_INCREF(st->UnpicklingError);
7348 if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007349 return NULL;
7350
Alexandre Vassalotti23bdd832013-11-27 19:36:52 -08007351 if (_Pickle_InitState(st) < 0)
Alexandre Vassalottica2d6102008-06-12 18:26:05 +00007352 return NULL;
7353
7354 return m;
7355}