bpo-41052: Fix pickling heap types implemented in C with protocols 0 and 1 (GH-22870)
diff --git a/Lib/copyreg.py b/Lib/copyreg.py
index dfc463c..7ab8c12 100644
--- a/Lib/copyreg.py
+++ b/Lib/copyreg.py
@@ -48,6 +48,7 @@ def _reconstructor(cls, base, state):
return obj
_HEAPTYPE = 1<<9
+_new_type = type(int.__new__)
# Python code for object.__reduce_ex__ for protocols 0 and 1
@@ -57,6 +58,9 @@ def _reduce_ex(self, proto):
for base in cls.__mro__:
if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
break
+ new = base.__new__
+ if isinstance(new, _new_type) and new.__self__ is base:
+ break
else:
base = object # not really reachable
if base is object:
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index fb972a3..ae288f5 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1969,6 +1969,17 @@ def test_newobj_proxies(self):
self.assertEqual(B(x), B(y), detail)
self.assertEqual(x.__dict__, y.__dict__, detail)
+ def test_newobj_overridden_new(self):
+ # Test that Python class with C implemented __new__ is pickleable
+ for proto in protocols:
+ x = MyIntWithNew2(1)
+ x.foo = 42
+ s = self.dumps(x, proto)
+ y = self.loads(s)
+ self.assertIs(type(y), MyIntWithNew2)
+ self.assertEqual(int(y), 1)
+ self.assertEqual(y.foo, 42)
+
def test_newobj_not_class(self):
# Issue 24552
global SimpleNewObj
@@ -3089,6 +3100,13 @@ class MyFrozenSet(frozenset):
MyStr, MyUnicode,
MyTuple, MyList, MyDict, MySet, MyFrozenSet]
+class MyIntWithNew(int):
+ def __new__(cls, value):
+ raise AssertionError
+
+class MyIntWithNew2(MyIntWithNew):
+ __new__ = int.__new__
+
class SlotList(MyList):
__slots__ = ["foo"]
diff --git a/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst b/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst
new file mode 100644
index 0000000..528e90e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst
@@ -0,0 +1,2 @@
+Pickling heap types implemented in C with protocols 0 and 1 raises now an
+error instead of producing incorrect data.
diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
index effb0de..bfcdac6 100644
--- a/Modules/_bz2module.c
+++ b/Modules/_bz2module.c
@@ -272,21 +272,6 @@ _bz2_BZ2Compressor_flush_impl(BZ2Compressor *self)
return result;
}
-/*[clinic input]
-_bz2.BZ2Compressor.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_bz2_BZ2Compressor___reduce___impl(BZ2Compressor *self)
-/*[clinic end generated code: output=d13db66ae043e141 input=e09bccef0e6731b2]*/
-{
- PyErr_Format(PyExc_TypeError,
- "cannot pickle %s object",
- Py_TYPE(self)->tp_name);
- return NULL;
-}
-
static void*
BZ2_Malloc(void* ctx, int items, int size)
{
@@ -399,7 +384,6 @@ BZ2Compressor_traverse(BZ2Compressor *self, visitproc visit, void *arg)
static PyMethodDef BZ2Compressor_methods[] = {
_BZ2_BZ2COMPRESSOR_COMPRESS_METHODDEF
_BZ2_BZ2COMPRESSOR_FLUSH_METHODDEF
- _BZ2_BZ2COMPRESSOR___REDUCE___METHODDEF
{NULL}
};
@@ -642,21 +626,6 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data,
return result;
}
-/*[clinic input]
-_bz2.BZ2Decompressor.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_bz2_BZ2Decompressor___reduce___impl(BZ2Decompressor *self)
-/*[clinic end generated code: output=f6a40650813f482e input=8db9175a609fdd43]*/
-{
- PyErr_Format(PyExc_TypeError,
- "cannot pickle %s object",
- Py_TYPE(self)->tp_name);
- return NULL;
-}
-
/* Argument Clinic is not used since the Argument Clinic always want to
check the type which would be wrong here */
static int
@@ -746,7 +715,6 @@ BZ2Decompressor_traverse(BZ2Decompressor *self, visitproc visit, void *arg)
static PyMethodDef BZ2Decompressor_methods[] = {
_BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF
- _BZ2_BZ2DECOMPRESSOR___REDUCE___METHODDEF
{NULL}
};
diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c
index 24e1d6c..b01f630 100644
--- a/Modules/_lzmamodule.c
+++ b/Modules/_lzmamodule.c
@@ -825,24 +825,9 @@ Compressor_dealloc(Compressor *self)
Py_DECREF(tp);
}
-/*[clinic input]
-_lzma.LZMACompressor.__reduce__
-[clinic start generated code]*/
-
-static PyObject *
-_lzma_LZMACompressor___reduce___impl(Compressor *self)
-/*[clinic end generated code: output=b49a0538d1cad752 input=6be52aba16b513c1]*/
-{
- PyErr_Format(PyExc_TypeError,
- "cannot pickle %s object",
- Py_TYPE(self)->tp_name);
- return NULL;
-}
-
static PyMethodDef Compressor_methods[] = {
_LZMA_LZMACOMPRESSOR_COMPRESS_METHODDEF
_LZMA_LZMACOMPRESSOR_FLUSH_METHODDEF
- _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF
{NULL}
};
@@ -1309,23 +1294,8 @@ Decompressor_traverse(Decompressor *self, visitproc visit, void *arg)
return 0;
}
-/*[clinic input]
-_lzma.LZMADecompressor.__reduce__
-[clinic start generated code]*/
-
-static PyObject *
-_lzma_LZMADecompressor___reduce___impl(Decompressor *self)
-/*[clinic end generated code: output=2611fff0104a9c30 input=b9882e030aecd9a5]*/
-{
- PyErr_Format(PyExc_TypeError,
- "cannot pickle %s object",
- Py_TYPE(self)->tp_name);
- return NULL;
-}
-
static PyMethodDef Decompressor_methods[] = {
_LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF
- _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF
{NULL}
};
diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
index b8bc044..d338708 100644
--- a/Modules/_randommodule.c
+++ b/Modules/_randommodule.c
@@ -536,29 +536,12 @@ random_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
}
-/*[clinic input]
-
-_random.Random.__reduce__
-
-[clinic start generated code]*/
-
-static PyObject *
-_random_Random___reduce___impl(RandomObject *self)
-/*[clinic end generated code: output=ddea0dcdb60ffd6d input=bd38ec35fd157e0f]*/
-{
- PyErr_Format(PyExc_TypeError,
- "cannot pickle %s object",
- Py_TYPE(self)->tp_name);
- return NULL;
-}
-
static PyMethodDef random_methods[] = {
_RANDOM_RANDOM_RANDOM_METHODDEF
_RANDOM_RANDOM_SEED_METHODDEF
_RANDOM_RANDOM_GETSTATE_METHODDEF
_RANDOM_RANDOM_SETSTATE_METHODDEF
_RANDOM_RANDOM_GETRANDBITS_METHODDEF
- _RANDOM_RANDOM___REDUCE___METHODDEF
{NULL, NULL} /* sentinel */
};
diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h
index ff67d34..71ad0b1 100644
--- a/Modules/clinic/_bz2module.c.h
+++ b/Modules/clinic/_bz2module.c.h
@@ -65,23 +65,6 @@ _bz2_BZ2Compressor_flush(BZ2Compressor *self, PyObject *Py_UNUSED(ignored))
return _bz2_BZ2Compressor_flush_impl(self);
}
-PyDoc_STRVAR(_bz2_BZ2Compressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _BZ2_BZ2COMPRESSOR___REDUCE___METHODDEF \
- {"__reduce__", (PyCFunction)_bz2_BZ2Compressor___reduce__, METH_NOARGS, _bz2_BZ2Compressor___reduce____doc__},
-
-static PyObject *
-_bz2_BZ2Compressor___reduce___impl(BZ2Compressor *self);
-
-static PyObject *
-_bz2_BZ2Compressor___reduce__(BZ2Compressor *self, PyObject *Py_UNUSED(ignored))
-{
- return _bz2_BZ2Compressor___reduce___impl(self);
-}
-
PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__,
"decompress($self, /, data, max_length=-1)\n"
"--\n"
@@ -156,21 +139,4 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py
return return_value;
}
-
-PyDoc_STRVAR(_bz2_BZ2Decompressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _BZ2_BZ2DECOMPRESSOR___REDUCE___METHODDEF \
- {"__reduce__", (PyCFunction)_bz2_BZ2Decompressor___reduce__, METH_NOARGS, _bz2_BZ2Decompressor___reduce____doc__},
-
-static PyObject *
-_bz2_BZ2Decompressor___reduce___impl(BZ2Decompressor *self);
-
-static PyObject *
-_bz2_BZ2Decompressor___reduce__(BZ2Decompressor *self, PyObject *Py_UNUSED(ignored))
-{
- return _bz2_BZ2Decompressor___reduce___impl(self);
-}
-/*[clinic end generated code: output=001f31fdacb4cb01 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=ed10705d7a9fd598 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h
index e15cc0c..526031a 100644
--- a/Modules/clinic/_lzmamodule.c.h
+++ b/Modules/clinic/_lzmamodule.c.h
@@ -65,23 +65,6 @@ _lzma_LZMACompressor_flush(Compressor *self, PyObject *Py_UNUSED(ignored))
return _lzma_LZMACompressor_flush_impl(self);
}
-PyDoc_STRVAR(_lzma_LZMACompressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _LZMA_LZMACOMPRESSOR___REDUCE___METHODDEF \
- {"__reduce__", (PyCFunction)_lzma_LZMACompressor___reduce__, METH_NOARGS, _lzma_LZMACompressor___reduce____doc__},
-
-static PyObject *
-_lzma_LZMACompressor___reduce___impl(Compressor *self);
-
-static PyObject *
-_lzma_LZMACompressor___reduce__(Compressor *self, PyObject *Py_UNUSED(ignored))
-{
- return _lzma_LZMACompressor___reduce___impl(self);
-}
-
PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__,
"decompress($self, /, data, max_length=-1)\n"
"--\n"
@@ -228,23 +211,6 @@ _lzma_LZMADecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs
return return_value;
}
-PyDoc_STRVAR(_lzma_LZMADecompressor___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _LZMA_LZMADECOMPRESSOR___REDUCE___METHODDEF \
- {"__reduce__", (PyCFunction)_lzma_LZMADecompressor___reduce__, METH_NOARGS, _lzma_LZMADecompressor___reduce____doc__},
-
-static PyObject *
-_lzma_LZMADecompressor___reduce___impl(Decompressor *self);
-
-static PyObject *
-_lzma_LZMADecompressor___reduce__(Decompressor *self, PyObject *Py_UNUSED(ignored))
-{
- return _lzma_LZMADecompressor___reduce___impl(self);
-}
-
PyDoc_STRVAR(_lzma_is_check_supported__doc__,
"is_check_supported($module, check_id, /)\n"
"--\n"
@@ -320,4 +286,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz
return return_value;
}
-/*[clinic end generated code: output=d89b6159e98544be input=a9049054013a1b77]*/
+/*[clinic end generated code: output=867b9e334053b679 input=a9049054013a1b77]*/
diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h
index 3322a37..b3cd435 100644
--- a/Modules/clinic/_randommodule.c.h
+++ b/Modules/clinic/_randommodule.c.h
@@ -109,21 +109,4 @@ _random_Random_getrandbits(RandomObject *self, PyObject *arg)
exit:
return return_value;
}
-
-PyDoc_STRVAR(_random_Random___reduce____doc__,
-"__reduce__($self, /)\n"
-"--\n"
-"\n");
-
-#define _RANDOM_RANDOM___REDUCE___METHODDEF \
- {"__reduce__", (PyCFunction)_random_Random___reduce__, METH_NOARGS, _random_Random___reduce____doc__},
-
-static PyObject *
-_random_Random___reduce___impl(RandomObject *self);
-
-static PyObject *
-_random_Random___reduce__(RandomObject *self, PyObject *Py_UNUSED(ignored))
-{
- return _random_Random___reduce___impl(self);
-}
-/*[clinic end generated code: output=450f0961c2c92389 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=cc8a23b2757dc6ba input=a9049054013a1b77]*/