Issue #27130: Fix handling of buffers exceeding (U)INT_MAX in “zlib” module
Ported from patches by Xiang Zhang, Nadeem Vawda, and myself.
diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
index 96cc3a7..c556060 100644
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -1,4 +1,5 @@
import unittest
+from test import test_support as support
from test.test_support import TESTFN, run_unittest, import_module, unlink, requires
import binascii
import pickle
@@ -80,6 +81,16 @@
zlib.crc32('spam', (2**31)))
+# Issue #10276 - check that inputs >=4GB are handled correctly.
+class ChecksumBigBufferTestCase(unittest.TestCase):
+
+ @precisionbigmemtest(size=_4G + 4, memuse=1, dry_run=False)
+ def test_big_buffer(self, size):
+ data = b"nyan" * (_1G + 1)
+ self.assertEqual(zlib.crc32(data) & 0xFFFFFFFF, 1044521549)
+ self.assertEqual(zlib.adler32(data) & 0xFFFFFFFF, 2256789997)
+
+
class ExceptionTestCase(unittest.TestCase):
# make sure we generate some expected errors
def test_badlevel(self):
@@ -104,6 +115,15 @@
self.assertRaises(ValueError, zlib.decompressobj().flush, 0)
self.assertRaises(ValueError, zlib.decompressobj().flush, -1)
+ @support.cpython_only
+ def test_overflow(self):
+ with self.assertRaisesRegexp(OverflowError, 'int too large'):
+ zlib.decompress(b'', 15, sys.maxsize + 1)
+ with self.assertRaisesRegexp(OverflowError, 'int too large'):
+ zlib.decompressobj().decompress(b'', sys.maxsize + 1)
+ with self.assertRaisesRegexp(OverflowError, 'int too large'):
+ zlib.decompressobj().flush(sys.maxsize + 1)
+
class BaseCompressTestCase(object):
def check_big_compress_buffer(self, size, compress_func):
@@ -167,6 +187,28 @@
def test_big_decompress_buffer(self, size):
self.check_big_decompress_buffer(size, zlib.decompress)
+ @precisionbigmemtest(size=_4G, memuse=1)
+ def test_large_bufsize(self, size):
+ # Test decompress(bufsize) parameter greater than the internal limit
+ data = HAMLET_SCENE * 10
+ compressed = zlib.compress(data, 1)
+ self.assertEqual(zlib.decompress(compressed, 15, size), data)
+
+ def test_custom_bufsize(self):
+ data = HAMLET_SCENE * 10
+ compressed = zlib.compress(data, 1)
+ self.assertEqual(zlib.decompress(compressed, 15, CustomInt()), data)
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @precisionbigmemtest(size=_4G + 100, memuse=4)
+ def test_64bit_compress(self, size):
+ data = b'x' * size
+ try:
+ comp = zlib.compress(data, 0)
+ self.assertEqual(zlib.decompress(comp), data)
+ finally:
+ comp = data = None
+
class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
# Test compression object
@@ -318,6 +360,22 @@
self.assertRaises(ValueError, dco.decompress, "", -1)
self.assertEqual('', dco.unconsumed_tail)
+ def test_maxlen_large(self):
+ # Sizes up to sys.maxsize should be accepted, although zlib is
+ # internally limited to expressing sizes with unsigned int
+ data = HAMLET_SCENE * 10
+ DEFAULTALLOC = 16 * 1024
+ self.assertGreater(len(data), DEFAULTALLOC)
+ compressed = zlib.compress(data, 1)
+ dco = zlib.decompressobj()
+ self.assertEqual(dco.decompress(compressed, sys.maxsize), data)
+
+ def test_maxlen_custom(self):
+ data = HAMLET_SCENE * 10
+ compressed = zlib.compress(data, 1)
+ dco = zlib.decompressobj()
+ self.assertEqual(dco.decompress(compressed, CustomInt()), data[:100])
+
def test_clear_unconsumed_tail(self):
# Issue #12050: calling decompress() without providing max_length
# should clear the unconsumed_tail attribute.
@@ -416,6 +474,22 @@
data = zlib.compress(input2)
self.assertEqual(dco.flush(), input1[1:])
+ @precisionbigmemtest(size=_4G, memuse=1)
+ def test_flush_large_length(self, size):
+ # Test flush(length) parameter greater than internal limit UINT_MAX
+ input = HAMLET_SCENE * 10
+ data = zlib.compress(input, 1)
+ dco = zlib.decompressobj()
+ dco.decompress(data, 1)
+ self.assertEqual(dco.flush(size), input[1:])
+
+ def test_flush_custom_length(self):
+ input = HAMLET_SCENE * 10
+ data = zlib.compress(input, 1)
+ dco = zlib.decompressobj()
+ dco.decompress(data, 1)
+ self.assertEqual(dco.flush(CustomInt()), input[1:])
+
@requires_Compress_copy
def test_compresscopy(self):
# Test copying a compression object
@@ -527,6 +601,46 @@
decompress = lambda s: d.decompress(s) + d.flush()
self.check_big_decompress_buffer(size, decompress)
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @precisionbigmemtest(size=_4G + 100, memuse=4)
+ def test_64bit_compress(self, size):
+ data = b'x' * size
+ co = zlib.compressobj(0)
+ do = zlib.decompressobj()
+ try:
+ comp = co.compress(data) + co.flush()
+ uncomp = do.decompress(comp) + do.flush()
+ self.assertEqual(uncomp, data)
+ finally:
+ comp = uncomp = data = None
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @precisionbigmemtest(size=_4G + 100, memuse=3)
+ def test_large_unused_data(self, size):
+ data = b'abcdefghijklmnop'
+ unused = b'x' * size
+ comp = zlib.compress(data) + unused
+ do = zlib.decompressobj()
+ try:
+ uncomp = do.decompress(comp) + do.flush()
+ self.assertEqual(unused, do.unused_data)
+ self.assertEqual(uncomp, data)
+ finally:
+ unused = comp = do = None
+
+ @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform')
+ @precisionbigmemtest(size=_4G + 100, memuse=5)
+ def test_large_unconsumed_tail(self, size):
+ data = b'x' * size
+ do = zlib.decompressobj()
+ try:
+ comp = zlib.compress(data, 0)
+ uncomp = do.decompress(comp, 1) + do.flush()
+ self.assertEqual(uncomp, data)
+ self.assertEqual(do.unconsumed_tail, b'')
+ finally:
+ comp = uncomp = data = None
+
def test_wbits(self):
co = zlib.compressobj(1, zlib.DEFLATED, 15)
zlib15 = co.compress(HAMLET_SCENE) + co.flush()
@@ -658,9 +772,15 @@
"""
+class CustomInt:
+ def __int__(self):
+ return 100
+
+
def test_main():
run_unittest(
ChecksumTestCase,
+ ChecksumBigBufferTestCase,
ExceptionTestCase,
CompressTestCase,
CompressObjectTestCase