Issue #25262. Added support for BINBYTES8 opcode in Python implementation of
unpickler. Highest 32 bits of 64-bit size for BINUNICODE8 and BINBYTES8
opcodes no longer silently ignored on 32-bit platforms in C implementation.
diff --git a/Lib/pickle.py b/Lib/pickle.py
index 6c26c5e..e93057a 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -1205,6 +1205,14 @@
self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
dispatch[BINUNICODE8[0]] = load_binunicode8
+ def load_binbytes8(self):
+ len, = unpack('<Q', self.read(8))
+ if len > maxsize:
+ raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
+ "of %d bytes" % maxsize)
+ self.append(self.read(len))
+ dispatch[BINBYTES8[0]] = load_binbytes8
+
def load_short_binstring(self):
len = self.read(1)[0]
data = self.read(len)
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 9a2b24b..2ef48e6 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -857,6 +857,26 @@
self.assert_is_copy([(100,), (100,)],
self.loads(b'((Kdtp0\nh\x00l.))'))
+ def test_binbytes8(self):
+ dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
+ self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
+
+ def test_binunicode8(self):
+ dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
+ self.assertEqual(self.loads(dumped), '\u20ac\x00')
+
+ @requires_32b
+ def test_large_32b_binbytes8(self):
+ dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
+ with self.assertRaises((pickle.UnpicklingError, OverflowError)):
+ self.loads(dumped)
+
+ @requires_32b
+ def test_large_32b_binunicode8(self):
+ dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
+ with self.assertRaises((pickle.UnpicklingError, OverflowError)):
+ self.loads(dumped)
+
def test_get(self):
pickled = b'((lp100000\ng100000\nt.'
unpickled = self.loads(pickled)
diff --git a/Misc/NEWS b/Misc/NEWS
index a2c4020..b5307cf 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -21,6 +21,10 @@
Library
-------
+- Issue #25262. Added support for BINBYTES8 opcode in Python implementation of
+ unpickler. Highest 32 bits of 64-bit size for BINUNICODE8 and BINBYTES8
+ opcodes no longer silently ignored on 32-bit platforms in C implementation.
+
- Issue #25034: Fix string.Formatter problem with auto-numbering and
nested format_specs. Patch by Anthon van der Neut.
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index 123f14b..f3b73f1 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -4606,10 +4606,20 @@
calc_binsize(char *bytes, int nbytes)
{
unsigned char *s = (unsigned char *)bytes;
- Py_ssize_t i;
+ int i;
size_t x = 0;
- for (i = 0; i < nbytes && (size_t)i < sizeof(size_t); i++) {
+ if (nbytes > (int)sizeof(size_t)) {
+ /* Check for integer overflow. BINBYTES8 and BINUNICODE8 opcodes
+ * have 64-bit size that can't be represented on 32-bit platform.
+ */
+ for (i = (int)sizeof(size_t); i < nbytes; i++) {
+ if (s[i])
+ return -1;
+ }
+ nbytes = (int)sizeof(size_t);
+ }
+ for (i = 0; i < nbytes; i++) {
x |= (size_t) s[i] << (8 * i);
}