blob: 7cd1d7c992aee72195aa327f09dcb791940abae0 [file] [log] [blame]
Guido van Rossum7d9ea502003-02-03 20:45:52 +00001import unittest
Benjamin Petersonee8712c2008-05-20 21:35:26 +00002from test import support
Christian Heimesd5e2b6f2008-03-19 21:50:51 +00003import binascii
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00004import random
Antoine Pitrouf3d22752011-02-21 18:09:00 +00005import sys
Antoine Pitrou94190bb2011-10-04 10:22:36 +02006from test.support import bigmemtest, _1G, _4G
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00007
R. David Murraya21e4ca2009-03-31 23:16:50 +00008zlib = support.import_module('zlib')
9
Serhiy Storchaka43767632013-11-03 21:31:38 +020010requires_Compress_copy = unittest.skipUnless(
11 hasattr(zlib.compressobj(), "copy"),
12 'requires Compress.copy()')
13requires_Decompress_copy = unittest.skipUnless(
14 hasattr(zlib.decompressobj(), "copy"),
15 'requires Decompress.copy()')
16
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000017
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020018class VersionTestCase(unittest.TestCase):
19
20 def test_library_version(self):
Nadeem Vawda131c7072012-01-25 23:16:50 +020021 # Test that the major version of the actual library in use matches the
22 # major version that we were compiled against. We can't guarantee that
23 # the minor versions will match (even on the machine on which the module
24 # was compiled), and the API is stable between minor versions, so
Nadeem Vawdad770fe42012-01-28 17:32:47 +020025 # testing only the major versions avoids spurious failures.
Nadeem Vawda131c7072012-01-25 23:16:50 +020026 self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0])
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020027
28
Guido van Rossum7d9ea502003-02-03 20:45:52 +000029class ChecksumTestCase(unittest.TestCase):
30 # checksum test cases
31 def test_crc32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000032 self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000033 self.assertTrue(zlib.crc32(b"abc", 0xffffffff))
Andrew M. Kuchlingfcfc8d52001-08-10 15:50:11 +000034
Guido van Rossum7d9ea502003-02-03 20:45:52 +000035 def test_crc32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000036 self.assertEqual(zlib.crc32(b"", 0), 0)
37 self.assertEqual(zlib.crc32(b"", 1), 1)
38 self.assertEqual(zlib.crc32(b"", 432), 432)
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000039
Guido van Rossum7d9ea502003-02-03 20:45:52 +000040 def test_adler32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000041 self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000042 self.assertTrue(zlib.adler32(b"abc", 0xffffffff))
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000043
Guido van Rossum7d9ea502003-02-03 20:45:52 +000044 def test_adler32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000045 self.assertEqual(zlib.adler32(b"", 0), 0)
46 self.assertEqual(zlib.adler32(b"", 1), 1)
47 self.assertEqual(zlib.adler32(b"", 432), 432)
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000048
Guido van Rossum7d9ea502003-02-03 20:45:52 +000049 def assertEqual32(self, seen, expected):
50 # 32-bit values masked -- checksums on 32- vs 64- bit machines
51 # This is important if bit 31 (0x08000000L) is set.
Guido van Rossume2a383d2007-01-15 16:59:06 +000052 self.assertEqual(seen & 0x0FFFFFFFF, expected & 0x0FFFFFFFF)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000053
54 def test_penguins(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000055 self.assertEqual32(zlib.crc32(b"penguin", 0), 0x0e5c1a120)
56 self.assertEqual32(zlib.crc32(b"penguin", 1), 0x43b6aa94)
57 self.assertEqual32(zlib.adler32(b"penguin", 0), 0x0bcf02f6)
58 self.assertEqual32(zlib.adler32(b"penguin", 1), 0x0bd602f7)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000059
Guido van Rossum776152b2007-05-22 22:44:07 +000060 self.assertEqual(zlib.crc32(b"penguin"), zlib.crc32(b"penguin", 0))
61 self.assertEqual(zlib.adler32(b"penguin"),zlib.adler32(b"penguin",1))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000062
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000063 def test_crc32_adler32_unsigned(self):
Antoine Pitrou77b338b2009-12-14 18:00:06 +000064 foo = b'abcdefghijklmnop'
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000065 # explicitly test signed behavior
Gregory P. Smith27275032008-03-20 06:20:09 +000066 self.assertEqual(zlib.crc32(foo), 2486878355)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000067 self.assertEqual(zlib.crc32(b'spam'), 1138425661)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000068 self.assertEqual(zlib.adler32(foo+foo), 3573550353)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000069 self.assertEqual(zlib.adler32(b'spam'), 72286642)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000070
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000071 def test_same_as_binascii_crc32(self):
Martin v. Löwis15b16a32008-12-02 06:00:15 +000072 foo = b'abcdefghijklmnop'
Gregory P. Smith27275032008-03-20 06:20:09 +000073 crc = 2486878355
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000074 self.assertEqual(binascii.crc32(foo), crc)
75 self.assertEqual(zlib.crc32(foo), crc)
Martin v. Löwis15b16a32008-12-02 06:00:15 +000076 self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000077
78
Antoine Pitrouf3d22752011-02-21 18:09:00 +000079# Issue #10276 - check that inputs >=4GB are handled correctly.
80class ChecksumBigBufferTestCase(unittest.TestCase):
81
Nadeem Vawdabc8c8172012-02-23 14:16:15 +020082 @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
83 def test_big_buffer(self, size):
Nadeem Vawdab063a482012-02-23 13:36:25 +020084 data = b"nyan" * (_1G + 1)
85 self.assertEqual(zlib.crc32(data), 1044521549)
86 self.assertEqual(zlib.adler32(data), 2256789997)
Antoine Pitrouf3d22752011-02-21 18:09:00 +000087
Christian Heimesb186d002008-03-18 15:15:01 +000088
Guido van Rossum7d9ea502003-02-03 20:45:52 +000089class ExceptionTestCase(unittest.TestCase):
90 # make sure we generate some expected errors
Guido van Rossum8ce8a782007-11-01 19:42:39 +000091 def test_badlevel(self):
92 # specifying compression level out of range causes an error
93 # (but -1 is Z_DEFAULT_COMPRESSION and apparently the zlib
94 # accepts 0 too)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000095 self.assertRaises(zlib.error, zlib.compress, b'ERROR', 10)
96
97 def test_badargs(self):
98 self.assertRaises(TypeError, zlib.adler32)
99 self.assertRaises(TypeError, zlib.crc32)
100 self.assertRaises(TypeError, zlib.compress)
101 self.assertRaises(TypeError, zlib.decompress)
102 for arg in (42, None, '', 'abc', (), []):
103 self.assertRaises(TypeError, zlib.adler32, arg)
104 self.assertRaises(TypeError, zlib.crc32, arg)
105 self.assertRaises(TypeError, zlib.compress, arg)
106 self.assertRaises(TypeError, zlib.decompress, arg)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000107
108 def test_badcompressobj(self):
109 # verify failure on building compress object with bad params
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000110 self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0)
Guido van Rossum8ce8a782007-11-01 19:42:39 +0000111 # specifying total bits too large causes an error
112 self.assertRaises(ValueError,
113 zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000114
115 def test_baddecompressobj(self):
116 # verify failure on building decompress object with bad params
Antoine Pitrou90ee4df2010-04-06 17:23:13 +0000117 self.assertRaises(ValueError, zlib.decompressobj, -1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000118
Christian Heimes5e696852008-04-09 08:37:03 +0000119 def test_decompressobj_badflush(self):
120 # verify failure on calling decompressobj.flush with bad params
121 self.assertRaises(ValueError, zlib.decompressobj().flush, 0)
122 self.assertRaises(ValueError, zlib.decompressobj().flush, -1)
123
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000124
Antoine Pitrou89562712010-05-07 17:04:02 +0000125class BaseCompressTestCase(object):
126 def check_big_compress_buffer(self, size, compress_func):
127 _1M = 1024 * 1024
128 fmt = "%%0%dx" % (2 * _1M)
129 # Generate 10MB worth of random, and expand it by repeating it.
130 # The assumption is that zlib's memory is not big enough to exploit
131 # such spread out redundancy.
132 data = b''.join([random.getrandbits(8 * _1M).to_bytes(_1M, 'little')
133 for i in range(10)])
134 data = data * (size // len(data) + 1)
135 try:
136 compress_func(data)
137 finally:
138 # Release memory
139 data = None
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000140
Antoine Pitrou89562712010-05-07 17:04:02 +0000141 def check_big_decompress_buffer(self, size, decompress_func):
142 data = b'x' * size
143 try:
144 compressed = zlib.compress(data, 1)
145 finally:
146 # Release memory
147 data = None
148 data = decompress_func(compressed)
149 # Sanity check
150 try:
151 self.assertEqual(len(data), size)
152 self.assertEqual(len(data.strip(b'x')), 0)
153 finally:
154 data = None
155
156
157class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000158 # Test compression in one go (whole message compression)
159 def test_speech(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000160 x = zlib.compress(HAMLET_SCENE)
161 self.assertEqual(zlib.decompress(x), HAMLET_SCENE)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000162
163 def test_speech128(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000164 # compress more data
165 data = HAMLET_SCENE * 128
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000166 x = zlib.compress(data)
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000167 self.assertEqual(zlib.compress(bytearray(data)), x)
168 for ob in x, bytearray(x):
169 self.assertEqual(zlib.decompress(ob), data)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000170
Antoine Pitrou53b21662010-05-11 23:46:02 +0000171 def test_incomplete_stream(self):
172 # An useful error message is given
173 x = zlib.compress(HAMLET_SCENE)
Ezio Melottied3a7d22010-12-01 02:32:32 +0000174 self.assertRaisesRegex(zlib.error,
Antoine Pitrou53b21662010-05-11 23:46:02 +0000175 "Error -5 while decompressing data: incomplete or truncated stream",
176 zlib.decompress, x[:-1])
177
Antoine Pitrou89562712010-05-07 17:04:02 +0000178 # Memory use of the following functions takes into account overallocation
179
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200180 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000181 def test_big_compress_buffer(self, size):
182 compress = lambda s: zlib.compress(s, 1)
183 self.check_big_compress_buffer(size, compress)
184
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200185 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000186 def test_big_decompress_buffer(self, size):
187 self.check_big_decompress_buffer(size, zlib.decompress)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000188
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200189 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000190 def test_length_overflow(self, size):
Victor Stinner8848c7a2011-01-04 02:07:36 +0000191 data = b'x' * size
192 try:
193 self.assertRaises(OverflowError, zlib.compress, data, 1)
Nadeem Vawda154bdf92011-05-14 23:07:36 +0200194 self.assertRaises(OverflowError, zlib.decompress, data)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000195 finally:
196 data = None
197
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000198
Antoine Pitrou89562712010-05-07 17:04:02 +0000199class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000200 # Test compression object
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000201 def test_pair(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000202 # straightforward compress/decompress objects
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000203 datasrc = HAMLET_SCENE * 128
204 datazip = zlib.compress(datasrc)
205 # should compress both bytes and bytearray data
206 for data in (datasrc, bytearray(datasrc)):
207 co = zlib.compressobj()
208 x1 = co.compress(data)
209 x2 = co.flush()
210 self.assertRaises(zlib.error, co.flush) # second flush should not work
211 self.assertEqual(x1 + x2, datazip)
212 for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
213 dco = zlib.decompressobj()
214 y1 = dco.decompress(v1 + v2)
215 y2 = dco.flush()
216 self.assertEqual(data, y1 + y2)
217 self.assertIsInstance(dco.unconsumed_tail, bytes)
218 self.assertIsInstance(dco.unused_data, bytes)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000219
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000220 def test_compressoptions(self):
221 # specify lots of options to compressobj()
222 level = 2
223 method = zlib.DEFLATED
224 wbits = -12
Martin Panterbf19d162015-09-09 01:01:13 +0000225 memLevel = 9
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000226 strategy = zlib.Z_FILTERED
Martin Panterbf19d162015-09-09 01:01:13 +0000227 co = zlib.compressobj(level, method, wbits, memLevel, strategy)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000228 x1 = co.compress(HAMLET_SCENE)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000229 x2 = co.flush()
230 dco = zlib.decompressobj(wbits)
231 y1 = dco.decompress(x1 + x2)
232 y2 = dco.flush()
Neil Schemenauer6412b122004-06-05 19:34:28 +0000233 self.assertEqual(HAMLET_SCENE, y1 + y2)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000234
Martin Panterbf19d162015-09-09 01:01:13 +0000235 # keyword arguments should also be supported
236 zlib.compressobj(level=level, method=method, wbits=wbits,
237 memLevel=memLevel, strategy=strategy, zdict=b"")
238
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000239 def test_compressincremental(self):
240 # compress object in steps, decompress object as one-shot
Neil Schemenauer6412b122004-06-05 19:34:28 +0000241 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000242 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000243 bufs = []
244 for i in range(0, len(data), 256):
245 bufs.append(co.compress(data[i:i+256]))
246 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000247 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000248
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000249 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000250 y1 = dco.decompress(b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000251 y2 = dco.flush()
252 self.assertEqual(data, y1 + y2)
253
Neil Schemenauer6412b122004-06-05 19:34:28 +0000254 def test_decompinc(self, flush=False, source=None, cx=256, dcx=64):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000255 # compress object in steps, decompress object in steps
Neil Schemenauer6412b122004-06-05 19:34:28 +0000256 source = source or HAMLET_SCENE
257 data = source * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000258 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000259 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000260 for i in range(0, len(data), cx):
261 bufs.append(co.compress(data[i:i+cx]))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000262 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000263 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000264
Gregory P. Smith693fc462008-09-06 20:13:06 +0000265 decombuf = zlib.decompress(combuf)
266 # Test type of return value
Ezio Melottie9615932010-01-24 19:26:24 +0000267 self.assertIsInstance(decombuf, bytes)
Gregory P. Smith693fc462008-09-06 20:13:06 +0000268
269 self.assertEqual(data, decombuf)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000270
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000271 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000272 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000273 for i in range(0, len(combuf), dcx):
274 bufs.append(dco.decompress(combuf[i:i+dcx]))
Guido van Rossum776152b2007-05-22 22:44:07 +0000275 self.assertEqual(b'', dco.unconsumed_tail, ########
276 "(A) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000277 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000278 self.assertEqual(b'', dco.unused_data)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000279 if flush:
280 bufs.append(dco.flush())
281 else:
282 while True:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000283 chunk = dco.decompress(b'')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000284 if chunk:
285 bufs.append(chunk)
286 else:
287 break
Guido van Rossum776152b2007-05-22 22:44:07 +0000288 self.assertEqual(b'', dco.unconsumed_tail, ########
289 "(B) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000290 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000291 self.assertEqual(b'', dco.unused_data)
Guido van Rossum776152b2007-05-22 22:44:07 +0000292 self.assertEqual(data, b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000293 # Failure means: "decompressobj with init options failed"
294
Neil Schemenauer6412b122004-06-05 19:34:28 +0000295 def test_decompincflush(self):
296 self.test_decompinc(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000297
Neil Schemenauer6412b122004-06-05 19:34:28 +0000298 def test_decompimax(self, source=None, cx=256, dcx=64):
299 # compress in steps, decompress in length-restricted steps
300 source = source or HAMLET_SCENE
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000301 # Check a decompression object with max_length specified
Neil Schemenauer6412b122004-06-05 19:34:28 +0000302 data = source * 128
303 co = zlib.compressobj()
304 bufs = []
305 for i in range(0, len(data), cx):
306 bufs.append(co.compress(data[i:i+cx]))
307 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000308 combuf = b''.join(bufs)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000309 self.assertEqual(data, zlib.decompress(combuf),
310 'compressed data failure')
311
312 dco = zlib.decompressobj()
313 bufs = []
314 cb = combuf
315 while cb:
316 #max_length = 1 + len(cb)//10
317 chunk = dco.decompress(cb, dcx)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000318 self.assertFalse(len(chunk) > dcx,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000319 'chunk too big (%d>%d)' % (len(chunk), dcx))
320 bufs.append(chunk)
321 cb = dco.unconsumed_tail
322 bufs.append(dco.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000323 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000324
325 def test_decompressmaxlen(self, flush=False):
326 # Check a decompression object with max_length specified
327 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000328 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000329 bufs = []
330 for i in range(0, len(data), 256):
331 bufs.append(co.compress(data[i:i+256]))
332 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000333 combuf = b''.join(bufs)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000334 self.assertEqual(data, zlib.decompress(combuf),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000335 'compressed data failure')
336
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000337 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000338 bufs = []
339 cb = combuf
340 while cb:
Guido van Rossumf3594102003-02-27 18:39:18 +0000341 max_length = 1 + len(cb)//10
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000342 chunk = dco.decompress(cb, max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000343 self.assertFalse(len(chunk) > max_length,
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000344 'chunk too big (%d>%d)' % (len(chunk),max_length))
345 bufs.append(chunk)
346 cb = dco.unconsumed_tail
Neil Schemenauer6412b122004-06-05 19:34:28 +0000347 if flush:
348 bufs.append(dco.flush())
349 else:
350 while chunk:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000351 chunk = dco.decompress(b'', max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000352 self.assertFalse(len(chunk) > max_length,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000353 'chunk too big (%d>%d)' % (len(chunk),max_length))
354 bufs.append(chunk)
Guido van Rossum776152b2007-05-22 22:44:07 +0000355 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000356
Neil Schemenauer6412b122004-06-05 19:34:28 +0000357 def test_decompressmaxlenflush(self):
358 self.test_decompressmaxlen(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000359
360 def test_maxlenmisc(self):
361 # Misc tests of max_length
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000362 dco = zlib.decompressobj()
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000363 self.assertRaises(ValueError, dco.decompress, b"", -1)
Guido van Rossum776152b2007-05-22 22:44:07 +0000364 self.assertEqual(b'', dco.unconsumed_tail)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000365
Nadeem Vawda7619e882011-05-14 14:05:20 +0200366 def test_clear_unconsumed_tail(self):
367 # Issue #12050: calling decompress() without providing max_length
368 # should clear the unconsumed_tail attribute.
369 cdata = b"x\x9cKLJ\x06\x00\x02M\x01" # "abc"
370 dco = zlib.decompressobj()
371 ddata = dco.decompress(cdata, 1)
372 ddata += dco.decompress(dco.unconsumed_tail)
373 self.assertEqual(dco.unconsumed_tail, b"")
374
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000375 def test_flushes(self):
376 # Test flush() with the various options, using all the
377 # different levels in order to provide more variations.
378 sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH']
379 sync_opt = [getattr(zlib, opt) for opt in sync_opt
380 if hasattr(zlib, opt)]
Neil Schemenauer6412b122004-06-05 19:34:28 +0000381 data = HAMLET_SCENE * 8
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000382
383 for sync in sync_opt:
384 for level in range(10):
385 obj = zlib.compressobj( level )
386 a = obj.compress( data[:3000] )
387 b = obj.flush( sync )
388 c = obj.compress( data[3000:] )
389 d = obj.flush()
Guido van Rossum776152b2007-05-22 22:44:07 +0000390 self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000391 data, ("Decompress failed: flush "
392 "mode=%i, level=%i") % (sync, level))
393 del obj
394
Serhiy Storchaka43767632013-11-03 21:31:38 +0200395 @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'),
396 'requires zlib.Z_SYNC_FLUSH')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000397 def test_odd_flush(self):
398 # Test for odd flushing bugs noted in 2.0, and hopefully fixed in 2.1
399 import random
Serhiy Storchaka43767632013-11-03 21:31:38 +0200400 # Testing on 17K of "random" data
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000401
Serhiy Storchaka43767632013-11-03 21:31:38 +0200402 # Create compressor and decompressor objects
403 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
404 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000405
Serhiy Storchaka43767632013-11-03 21:31:38 +0200406 # Try 17K of data
407 # generate random data stream
408 try:
409 # In 2.3 and later, WichmannHill is the RNG of the bug report
410 gen = random.WichmannHill()
411 except AttributeError:
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000412 try:
Serhiy Storchaka43767632013-11-03 21:31:38 +0200413 # 2.2 called it Random
414 gen = random.Random()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000415 except AttributeError:
Serhiy Storchaka43767632013-11-03 21:31:38 +0200416 # others might simply have a single RNG
417 gen = random
418 gen.seed(1)
419 data = genblock(1, 17 * 1024, generator=gen)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000420
Serhiy Storchaka43767632013-11-03 21:31:38 +0200421 # compress, sync-flush, and decompress
422 first = co.compress(data)
423 second = co.flush(zlib.Z_SYNC_FLUSH)
424 expanded = dco.decompress(first + second)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000425
Serhiy Storchaka43767632013-11-03 21:31:38 +0200426 # if decompressed data is different from the input data, choke.
427 self.assertEqual(expanded, data, "17K random source doesn't match")
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000428
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000429 def test_empty_flush(self):
430 # Test that calling .flush() on unused objects works.
431 # (Bug #1083110 -- calling .flush() on decompress objects
432 # caused a core dump.)
433
434 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000435 self.assertTrue(co.flush()) # Returns a zlib header
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000436 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000437 self.assertEqual(dco.flush(), b"") # Returns nothing
Tim Peters5a9fb3c2005-01-07 16:01:32 +0000438
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200439 def test_dictionary(self):
440 h = HAMLET_SCENE
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200441 # Build a simulated dictionary out of the words in HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200442 words = h.split()
443 random.shuffle(words)
444 zdict = b''.join(words)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200445 # Use it to compress HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200446 co = zlib.compressobj(zdict=zdict)
447 cd = co.compress(h) + co.flush()
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200448 # Verify that it will decompress with the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200449 dco = zlib.decompressobj(zdict=zdict)
450 self.assertEqual(dco.decompress(cd) + dco.flush(), h)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200451 # Verify that it fails when not given the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200452 dco = zlib.decompressobj()
453 self.assertRaises(zlib.error, dco.decompress, cd)
454
455 def test_dictionary_streaming(self):
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200456 # This simulates the reuse of a compressor object for compressing
457 # several separate data streams.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200458 co = zlib.compressobj(zdict=HAMLET_SCENE)
459 do = zlib.decompressobj(zdict=HAMLET_SCENE)
460 piece = HAMLET_SCENE[1000:1500]
461 d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH)
462 d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH)
463 d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH)
464 self.assertEqual(do.decompress(d0), piece)
465 self.assertEqual(do.decompress(d1), piece[100:])
466 self.assertEqual(do.decompress(d2), piece[:-100])
467
Antoine Pitrouc09c92f2010-05-11 23:36:40 +0000468 def test_decompress_incomplete_stream(self):
469 # This is 'foo', deflated
470 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'
471 # For the record
472 self.assertEqual(zlib.decompress(x), b'foo')
473 self.assertRaises(zlib.error, zlib.decompress, x[:-5])
474 # Omitting the stream end works with decompressor objects
475 # (see issue #8672).
476 dco = zlib.decompressobj()
477 y = dco.decompress(x[:-5])
478 y += dco.flush()
479 self.assertEqual(y, b'foo')
480
Nadeem Vawda1c385462011-08-13 15:22:40 +0200481 def test_decompress_eof(self):
482 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
483 dco = zlib.decompressobj()
484 self.assertFalse(dco.eof)
485 dco.decompress(x[:-5])
486 self.assertFalse(dco.eof)
487 dco.decompress(x[-5:])
488 self.assertTrue(dco.eof)
489 dco.flush()
490 self.assertTrue(dco.eof)
491
492 def test_decompress_eof_incomplete_stream(self):
493 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
494 dco = zlib.decompressobj()
495 self.assertFalse(dco.eof)
496 dco.decompress(x[:-5])
497 self.assertFalse(dco.eof)
498 dco.flush()
499 self.assertFalse(dco.eof)
500
Nadeem Vawda39079942012-11-05 00:37:42 +0100501 def test_decompress_unused_data(self):
502 # Repeated calls to decompress() after EOF should accumulate data in
503 # dco.unused_data, instead of just storing the arg to the last call.
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100504 source = b'abcdefghijklmnopqrstuvwxyz'
505 remainder = b'0123456789'
506 y = zlib.compress(source)
507 x = y + remainder
508 for maxlen in 0, 1000:
509 for step in 1, 2, len(y), len(x):
510 dco = zlib.decompressobj()
511 data = b''
512 for i in range(0, len(x), step):
513 if i < len(y):
514 self.assertEqual(dco.unused_data, b'')
515 if maxlen == 0:
516 data += dco.decompress(x[i : i + step])
517 self.assertEqual(dco.unconsumed_tail, b'')
518 else:
519 data += dco.decompress(
520 dco.unconsumed_tail + x[i : i + step], maxlen)
521 data += dco.flush()
Nadeem Vawdadd1253a2012-11-11 02:21:22 +0100522 self.assertTrue(dco.eof)
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100523 self.assertEqual(data, source)
524 self.assertEqual(dco.unconsumed_tail, b'')
525 self.assertEqual(dco.unused_data, remainder)
Nadeem Vawda39079942012-11-05 00:37:42 +0100526
Nadeem Vawda7ee95552012-11-11 03:15:32 +0100527 def test_flush_with_freed_input(self):
528 # Issue #16411: decompressor accesses input to last decompress() call
529 # in flush(), even if this object has been freed in the meanwhile.
530 input1 = b'abcdefghijklmnopqrstuvwxyz'
531 input2 = b'QWERTYUIOPASDFGHJKLZXCVBNM'
532 data = zlib.compress(input1)
533 dco = zlib.decompressobj()
534 dco.decompress(data, 1)
535 del data
536 data = zlib.compress(input2)
537 self.assertEqual(dco.flush(), input1[1:])
538
Serhiy Storchaka43767632013-11-03 21:31:38 +0200539 @requires_Compress_copy
540 def test_compresscopy(self):
541 # Test copying a compression object
542 data0 = HAMLET_SCENE
543 data1 = bytes(str(HAMLET_SCENE, "ascii").swapcase(), "ascii")
544 c0 = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
545 bufs0 = []
546 bufs0.append(c0.compress(data0))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000547
Serhiy Storchaka43767632013-11-03 21:31:38 +0200548 c1 = c0.copy()
549 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000550
Serhiy Storchaka43767632013-11-03 21:31:38 +0200551 bufs0.append(c0.compress(data0))
552 bufs0.append(c0.flush())
553 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000554
Serhiy Storchaka43767632013-11-03 21:31:38 +0200555 bufs1.append(c1.compress(data1))
556 bufs1.append(c1.flush())
557 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000558
Serhiy Storchaka43767632013-11-03 21:31:38 +0200559 self.assertEqual(zlib.decompress(s0),data0+data0)
560 self.assertEqual(zlib.decompress(s1),data0+data1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000561
Serhiy Storchaka43767632013-11-03 21:31:38 +0200562 @requires_Compress_copy
563 def test_badcompresscopy(self):
564 # Test copying a compression object in an inconsistent state
565 c = zlib.compressobj()
566 c.compress(HAMLET_SCENE)
567 c.flush()
568 self.assertRaises(ValueError, c.copy)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000569
Serhiy Storchaka43767632013-11-03 21:31:38 +0200570 @requires_Decompress_copy
571 def test_decompresscopy(self):
572 # Test copying a decompression object
573 data = HAMLET_SCENE
574 comp = zlib.compress(data)
575 # Test type of return value
576 self.assertIsInstance(comp, bytes)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000577
Serhiy Storchaka43767632013-11-03 21:31:38 +0200578 d0 = zlib.decompressobj()
579 bufs0 = []
580 bufs0.append(d0.decompress(comp[:32]))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000581
Serhiy Storchaka43767632013-11-03 21:31:38 +0200582 d1 = d0.copy()
583 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000584
Serhiy Storchaka43767632013-11-03 21:31:38 +0200585 bufs0.append(d0.decompress(comp[32:]))
586 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000587
Serhiy Storchaka43767632013-11-03 21:31:38 +0200588 bufs1.append(d1.decompress(comp[32:]))
589 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000590
Serhiy Storchaka43767632013-11-03 21:31:38 +0200591 self.assertEqual(s0,s1)
592 self.assertEqual(s0,data)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000593
Serhiy Storchaka43767632013-11-03 21:31:38 +0200594 @requires_Decompress_copy
595 def test_baddecompresscopy(self):
596 # Test copying a compression object in an inconsistent state
597 data = zlib.compress(HAMLET_SCENE)
598 d = zlib.decompressobj()
599 d.decompress(data)
600 d.flush()
601 self.assertRaises(ValueError, d.copy)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000602
Antoine Pitrou89562712010-05-07 17:04:02 +0000603 # Memory use of the following functions takes into account overallocation
604
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200605 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000606 def test_big_compress_buffer(self, size):
607 c = zlib.compressobj(1)
608 compress = lambda s: c.compress(s) + c.flush()
609 self.check_big_compress_buffer(size, compress)
610
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200611 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000612 def test_big_decompress_buffer(self, size):
613 d = zlib.decompressobj()
614 decompress = lambda s: d.decompress(s) + d.flush()
615 self.check_big_decompress_buffer(size, decompress)
616
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200617 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200618 def test_length_overflow(self, size):
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200619 data = b'x' * size
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200620 c = zlib.compressobj(1)
621 d = zlib.decompressobj()
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200622 try:
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200623 self.assertRaises(OverflowError, c.compress, data)
624 self.assertRaises(OverflowError, d.decompress, data)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200625 finally:
626 data = None
627
Antoine Pitrou89562712010-05-07 17:04:02 +0000628
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000629def genblock(seed, length, step=1024, generator=random):
630 """length-byte stream of random data from a seed (in step-byte blocks)."""
631 if seed is not None:
632 generator.seed(seed)
633 randint = generator.randint
634 if length < step or step < 2:
635 step = length
Guido van Rossum776152b2007-05-22 22:44:07 +0000636 blocks = bytes()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000637 for i in range(0, length, step):
Guido van Rossum776152b2007-05-22 22:44:07 +0000638 blocks += bytes(randint(0, 255) for x in range(step))
639 return blocks
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000640
641
642
643def choose_lines(source, number, seed=None, generator=random):
644 """Return a list of number lines randomly chosen from the source"""
645 if seed is not None:
646 generator.seed(seed)
647 sources = source.split('\n')
648 return [generator.choice(sources) for n in range(number)]
649
650
651
Guido van Rossum776152b2007-05-22 22:44:07 +0000652HAMLET_SCENE = b"""
Fred Drake004d5e62000-10-23 17:22:08 +0000653LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000654
655 O, fear me not.
656 I stay too long: but here my father comes.
657
658 Enter POLONIUS
659
660 A double blessing is a double grace,
661 Occasion smiles upon a second leave.
662
Fred Drake004d5e62000-10-23 17:22:08 +0000663LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000664
665 Yet here, Laertes! aboard, aboard, for shame!
666 The wind sits in the shoulder of your sail,
667 And you are stay'd for. There; my blessing with thee!
668 And these few precepts in thy memory
669 See thou character. Give thy thoughts no tongue,
670 Nor any unproportioned thought his act.
671 Be thou familiar, but by no means vulgar.
672 Those friends thou hast, and their adoption tried,
673 Grapple them to thy soul with hoops of steel;
674 But do not dull thy palm with entertainment
675 Of each new-hatch'd, unfledged comrade. Beware
676 Of entrance to a quarrel, but being in,
677 Bear't that the opposed may beware of thee.
678 Give every man thy ear, but few thy voice;
679 Take each man's censure, but reserve thy judgment.
680 Costly thy habit as thy purse can buy,
681 But not express'd in fancy; rich, not gaudy;
682 For the apparel oft proclaims the man,
683 And they in France of the best rank and station
684 Are of a most select and generous chief in that.
685 Neither a borrower nor a lender be;
686 For loan oft loses both itself and friend,
687 And borrowing dulls the edge of husbandry.
688 This above all: to thine ownself be true,
689 And it must follow, as the night the day,
690 Thou canst not then be false to any man.
691 Farewell: my blessing season this in thee!
692
Fred Drake004d5e62000-10-23 17:22:08 +0000693LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000694
695 Most humbly do I take my leave, my lord.
696
Fred Drake004d5e62000-10-23 17:22:08 +0000697LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000698
699 The time invites you; go; your servants tend.
700
Fred Drake004d5e62000-10-23 17:22:08 +0000701LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000702
703 Farewell, Ophelia; and remember well
704 What I have said to you.
705
Fred Drake004d5e62000-10-23 17:22:08 +0000706OPHELIA
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000707
708 'Tis in my memory lock'd,
709 And you yourself shall keep the key of it.
710
Fred Drake004d5e62000-10-23 17:22:08 +0000711LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000712
713 Farewell.
714"""
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000715
716
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000717if __name__ == "__main__":
Zachary Ware38c707e2015-04-13 15:00:43 -0500718 unittest.main()