blob: 7154e139bc98caa10578d71ef8461aa3abfd55b2 [file] [log] [blame]
Guido van Rossum7d9ea502003-02-03 20:45:52 +00001import unittest
Benjamin Petersonee8712c2008-05-20 21:35:26 +00002from test import support
Christian Heimesd5e2b6f2008-03-19 21:50:51 +00003import binascii
Serhiy Storchakad7a44152015-11-12 11:23:04 +02004import pickle
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00005import random
Antoine Pitrouf3d22752011-02-21 18:09:00 +00006import sys
Antoine Pitrou94190bb2011-10-04 10:22:36 +02007from test.support import bigmemtest, _1G, _4G
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00008
R. David Murraya21e4ca2009-03-31 23:16:50 +00009zlib = support.import_module('zlib')
10
Serhiy Storchaka43767632013-11-03 21:31:38 +020011requires_Compress_copy = unittest.skipUnless(
12 hasattr(zlib.compressobj(), "copy"),
13 'requires Compress.copy()')
14requires_Decompress_copy = unittest.skipUnless(
15 hasattr(zlib.decompressobj(), "copy"),
16 'requires Decompress.copy()')
17
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000018
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020019class VersionTestCase(unittest.TestCase):
20
21 def test_library_version(self):
Nadeem Vawda131c7072012-01-25 23:16:50 +020022 # Test that the major version of the actual library in use matches the
23 # major version that we were compiled against. We can't guarantee that
24 # the minor versions will match (even on the machine on which the module
25 # was compiled), and the API is stable between minor versions, so
Nadeem Vawdad770fe42012-01-28 17:32:47 +020026 # testing only the major versions avoids spurious failures.
Nadeem Vawda131c7072012-01-25 23:16:50 +020027 self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0])
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020028
29
Guido van Rossum7d9ea502003-02-03 20:45:52 +000030class ChecksumTestCase(unittest.TestCase):
31 # checksum test cases
32 def test_crc32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000033 self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000034 self.assertTrue(zlib.crc32(b"abc", 0xffffffff))
Andrew M. Kuchlingfcfc8d52001-08-10 15:50:11 +000035
Guido van Rossum7d9ea502003-02-03 20:45:52 +000036 def test_crc32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000037 self.assertEqual(zlib.crc32(b"", 0), 0)
38 self.assertEqual(zlib.crc32(b"", 1), 1)
39 self.assertEqual(zlib.crc32(b"", 432), 432)
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000040
Guido van Rossum7d9ea502003-02-03 20:45:52 +000041 def test_adler32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000042 self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000043 self.assertTrue(zlib.adler32(b"abc", 0xffffffff))
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000044
Guido van Rossum7d9ea502003-02-03 20:45:52 +000045 def test_adler32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000046 self.assertEqual(zlib.adler32(b"", 0), 0)
47 self.assertEqual(zlib.adler32(b"", 1), 1)
48 self.assertEqual(zlib.adler32(b"", 432), 432)
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000049
Guido van Rossum7d9ea502003-02-03 20:45:52 +000050 def assertEqual32(self, seen, expected):
51 # 32-bit values masked -- checksums on 32- vs 64- bit machines
52 # This is important if bit 31 (0x08000000L) is set.
Guido van Rossume2a383d2007-01-15 16:59:06 +000053 self.assertEqual(seen & 0x0FFFFFFFF, expected & 0x0FFFFFFFF)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000054
55 def test_penguins(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000056 self.assertEqual32(zlib.crc32(b"penguin", 0), 0x0e5c1a120)
57 self.assertEqual32(zlib.crc32(b"penguin", 1), 0x43b6aa94)
58 self.assertEqual32(zlib.adler32(b"penguin", 0), 0x0bcf02f6)
59 self.assertEqual32(zlib.adler32(b"penguin", 1), 0x0bd602f7)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000060
Guido van Rossum776152b2007-05-22 22:44:07 +000061 self.assertEqual(zlib.crc32(b"penguin"), zlib.crc32(b"penguin", 0))
62 self.assertEqual(zlib.adler32(b"penguin"),zlib.adler32(b"penguin",1))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000063
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000064 def test_crc32_adler32_unsigned(self):
Antoine Pitrou77b338b2009-12-14 18:00:06 +000065 foo = b'abcdefghijklmnop'
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000066 # explicitly test signed behavior
Gregory P. Smith27275032008-03-20 06:20:09 +000067 self.assertEqual(zlib.crc32(foo), 2486878355)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000068 self.assertEqual(zlib.crc32(b'spam'), 1138425661)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000069 self.assertEqual(zlib.adler32(foo+foo), 3573550353)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000070 self.assertEqual(zlib.adler32(b'spam'), 72286642)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000071
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000072 def test_same_as_binascii_crc32(self):
Martin v. Löwis15b16a32008-12-02 06:00:15 +000073 foo = b'abcdefghijklmnop'
Gregory P. Smith27275032008-03-20 06:20:09 +000074 crc = 2486878355
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000075 self.assertEqual(binascii.crc32(foo), crc)
76 self.assertEqual(zlib.crc32(foo), crc)
Martin v. Löwis15b16a32008-12-02 06:00:15 +000077 self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000078
79
Antoine Pitrouf3d22752011-02-21 18:09:00 +000080# Issue #10276 - check that inputs >=4GB are handled correctly.
81class ChecksumBigBufferTestCase(unittest.TestCase):
82
Nadeem Vawdabc8c8172012-02-23 14:16:15 +020083 @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
84 def test_big_buffer(self, size):
Nadeem Vawdab063a482012-02-23 13:36:25 +020085 data = b"nyan" * (_1G + 1)
86 self.assertEqual(zlib.crc32(data), 1044521549)
87 self.assertEqual(zlib.adler32(data), 2256789997)
Antoine Pitrouf3d22752011-02-21 18:09:00 +000088
Christian Heimesb186d002008-03-18 15:15:01 +000089
Guido van Rossum7d9ea502003-02-03 20:45:52 +000090class ExceptionTestCase(unittest.TestCase):
91 # make sure we generate some expected errors
Guido van Rossum8ce8a782007-11-01 19:42:39 +000092 def test_badlevel(self):
93 # specifying compression level out of range causes an error
94 # (but -1 is Z_DEFAULT_COMPRESSION and apparently the zlib
95 # accepts 0 too)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000096 self.assertRaises(zlib.error, zlib.compress, b'ERROR', 10)
97
98 def test_badargs(self):
99 self.assertRaises(TypeError, zlib.adler32)
100 self.assertRaises(TypeError, zlib.crc32)
101 self.assertRaises(TypeError, zlib.compress)
102 self.assertRaises(TypeError, zlib.decompress)
103 for arg in (42, None, '', 'abc', (), []):
104 self.assertRaises(TypeError, zlib.adler32, arg)
105 self.assertRaises(TypeError, zlib.crc32, arg)
106 self.assertRaises(TypeError, zlib.compress, arg)
107 self.assertRaises(TypeError, zlib.decompress, arg)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000108
109 def test_badcompressobj(self):
110 # verify failure on building compress object with bad params
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000111 self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0)
Guido van Rossum8ce8a782007-11-01 19:42:39 +0000112 # specifying total bits too large causes an error
113 self.assertRaises(ValueError,
114 zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000115
116 def test_baddecompressobj(self):
117 # verify failure on building decompress object with bad params
Antoine Pitrou90ee4df2010-04-06 17:23:13 +0000118 self.assertRaises(ValueError, zlib.decompressobj, -1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000119
Christian Heimes5e696852008-04-09 08:37:03 +0000120 def test_decompressobj_badflush(self):
121 # verify failure on calling decompressobj.flush with bad params
122 self.assertRaises(ValueError, zlib.decompressobj().flush, 0)
123 self.assertRaises(ValueError, zlib.decompressobj().flush, -1)
124
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000125
Antoine Pitrou89562712010-05-07 17:04:02 +0000126class BaseCompressTestCase(object):
127 def check_big_compress_buffer(self, size, compress_func):
128 _1M = 1024 * 1024
129 fmt = "%%0%dx" % (2 * _1M)
130 # Generate 10MB worth of random, and expand it by repeating it.
131 # The assumption is that zlib's memory is not big enough to exploit
132 # such spread out redundancy.
133 data = b''.join([random.getrandbits(8 * _1M).to_bytes(_1M, 'little')
134 for i in range(10)])
135 data = data * (size // len(data) + 1)
136 try:
137 compress_func(data)
138 finally:
139 # Release memory
140 data = None
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000141
Antoine Pitrou89562712010-05-07 17:04:02 +0000142 def check_big_decompress_buffer(self, size, decompress_func):
143 data = b'x' * size
144 try:
145 compressed = zlib.compress(data, 1)
146 finally:
147 # Release memory
148 data = None
149 data = decompress_func(compressed)
150 # Sanity check
151 try:
152 self.assertEqual(len(data), size)
153 self.assertEqual(len(data.strip(b'x')), 0)
154 finally:
155 data = None
156
157
158class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000159 # Test compression in one go (whole message compression)
160 def test_speech(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000161 x = zlib.compress(HAMLET_SCENE)
162 self.assertEqual(zlib.decompress(x), HAMLET_SCENE)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000163
164 def test_speech128(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000165 # compress more data
166 data = HAMLET_SCENE * 128
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000167 x = zlib.compress(data)
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000168 self.assertEqual(zlib.compress(bytearray(data)), x)
169 for ob in x, bytearray(x):
170 self.assertEqual(zlib.decompress(ob), data)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000171
Antoine Pitrou53b21662010-05-11 23:46:02 +0000172 def test_incomplete_stream(self):
173 # An useful error message is given
174 x = zlib.compress(HAMLET_SCENE)
Ezio Melottied3a7d22010-12-01 02:32:32 +0000175 self.assertRaisesRegex(zlib.error,
Antoine Pitrou53b21662010-05-11 23:46:02 +0000176 "Error -5 while decompressing data: incomplete or truncated stream",
177 zlib.decompress, x[:-1])
178
Antoine Pitrou89562712010-05-07 17:04:02 +0000179 # Memory use of the following functions takes into account overallocation
180
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200181 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000182 def test_big_compress_buffer(self, size):
183 compress = lambda s: zlib.compress(s, 1)
184 self.check_big_compress_buffer(size, compress)
185
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200186 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000187 def test_big_decompress_buffer(self, size):
188 self.check_big_decompress_buffer(size, zlib.decompress)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000189
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200190 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000191 def test_length_overflow(self, size):
Victor Stinner8848c7a2011-01-04 02:07:36 +0000192 data = b'x' * size
193 try:
194 self.assertRaises(OverflowError, zlib.compress, data, 1)
Nadeem Vawda154bdf92011-05-14 23:07:36 +0200195 self.assertRaises(OverflowError, zlib.decompress, data)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000196 finally:
197 data = None
198
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000199
Antoine Pitrou89562712010-05-07 17:04:02 +0000200class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000201 # Test compression object
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000202 def test_pair(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000203 # straightforward compress/decompress objects
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000204 datasrc = HAMLET_SCENE * 128
205 datazip = zlib.compress(datasrc)
206 # should compress both bytes and bytearray data
207 for data in (datasrc, bytearray(datasrc)):
208 co = zlib.compressobj()
209 x1 = co.compress(data)
210 x2 = co.flush()
211 self.assertRaises(zlib.error, co.flush) # second flush should not work
212 self.assertEqual(x1 + x2, datazip)
213 for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
214 dco = zlib.decompressobj()
215 y1 = dco.decompress(v1 + v2)
216 y2 = dco.flush()
217 self.assertEqual(data, y1 + y2)
218 self.assertIsInstance(dco.unconsumed_tail, bytes)
219 self.assertIsInstance(dco.unused_data, bytes)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000220
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000221 def test_compressoptions(self):
222 # specify lots of options to compressobj()
223 level = 2
224 method = zlib.DEFLATED
225 wbits = -12
Martin Panterbf19d162015-09-09 01:01:13 +0000226 memLevel = 9
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000227 strategy = zlib.Z_FILTERED
Martin Panterbf19d162015-09-09 01:01:13 +0000228 co = zlib.compressobj(level, method, wbits, memLevel, strategy)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000229 x1 = co.compress(HAMLET_SCENE)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000230 x2 = co.flush()
231 dco = zlib.decompressobj(wbits)
232 y1 = dco.decompress(x1 + x2)
233 y2 = dco.flush()
Neil Schemenauer6412b122004-06-05 19:34:28 +0000234 self.assertEqual(HAMLET_SCENE, y1 + y2)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000235
Martin Panterbf19d162015-09-09 01:01:13 +0000236 # keyword arguments should also be supported
237 zlib.compressobj(level=level, method=method, wbits=wbits,
238 memLevel=memLevel, strategy=strategy, zdict=b"")
239
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000240 def test_compressincremental(self):
241 # compress object in steps, decompress object as one-shot
Neil Schemenauer6412b122004-06-05 19:34:28 +0000242 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000243 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000244 bufs = []
245 for i in range(0, len(data), 256):
246 bufs.append(co.compress(data[i:i+256]))
247 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000248 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000249
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000250 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000251 y1 = dco.decompress(b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000252 y2 = dco.flush()
253 self.assertEqual(data, y1 + y2)
254
Neil Schemenauer6412b122004-06-05 19:34:28 +0000255 def test_decompinc(self, flush=False, source=None, cx=256, dcx=64):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000256 # compress object in steps, decompress object in steps
Neil Schemenauer6412b122004-06-05 19:34:28 +0000257 source = source or HAMLET_SCENE
258 data = source * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000259 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000260 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000261 for i in range(0, len(data), cx):
262 bufs.append(co.compress(data[i:i+cx]))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000263 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000264 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000265
Gregory P. Smith693fc462008-09-06 20:13:06 +0000266 decombuf = zlib.decompress(combuf)
267 # Test type of return value
Ezio Melottie9615932010-01-24 19:26:24 +0000268 self.assertIsInstance(decombuf, bytes)
Gregory P. Smith693fc462008-09-06 20:13:06 +0000269
270 self.assertEqual(data, decombuf)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000271
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000272 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000273 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000274 for i in range(0, len(combuf), dcx):
275 bufs.append(dco.decompress(combuf[i:i+dcx]))
Guido van Rossum776152b2007-05-22 22:44:07 +0000276 self.assertEqual(b'', dco.unconsumed_tail, ########
277 "(A) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000278 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000279 self.assertEqual(b'', dco.unused_data)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000280 if flush:
281 bufs.append(dco.flush())
282 else:
283 while True:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000284 chunk = dco.decompress(b'')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000285 if chunk:
286 bufs.append(chunk)
287 else:
288 break
Guido van Rossum776152b2007-05-22 22:44:07 +0000289 self.assertEqual(b'', dco.unconsumed_tail, ########
290 "(B) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000291 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000292 self.assertEqual(b'', dco.unused_data)
Guido van Rossum776152b2007-05-22 22:44:07 +0000293 self.assertEqual(data, b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000294 # Failure means: "decompressobj with init options failed"
295
Neil Schemenauer6412b122004-06-05 19:34:28 +0000296 def test_decompincflush(self):
297 self.test_decompinc(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000298
Neil Schemenauer6412b122004-06-05 19:34:28 +0000299 def test_decompimax(self, source=None, cx=256, dcx=64):
300 # compress in steps, decompress in length-restricted steps
301 source = source or HAMLET_SCENE
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000302 # Check a decompression object with max_length specified
Neil Schemenauer6412b122004-06-05 19:34:28 +0000303 data = source * 128
304 co = zlib.compressobj()
305 bufs = []
306 for i in range(0, len(data), cx):
307 bufs.append(co.compress(data[i:i+cx]))
308 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000309 combuf = b''.join(bufs)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000310 self.assertEqual(data, zlib.decompress(combuf),
311 'compressed data failure')
312
313 dco = zlib.decompressobj()
314 bufs = []
315 cb = combuf
316 while cb:
317 #max_length = 1 + len(cb)//10
318 chunk = dco.decompress(cb, dcx)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000319 self.assertFalse(len(chunk) > dcx,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000320 'chunk too big (%d>%d)' % (len(chunk), dcx))
321 bufs.append(chunk)
322 cb = dco.unconsumed_tail
323 bufs.append(dco.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000324 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000325
326 def test_decompressmaxlen(self, flush=False):
327 # Check a decompression object with max_length specified
328 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000329 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000330 bufs = []
331 for i in range(0, len(data), 256):
332 bufs.append(co.compress(data[i:i+256]))
333 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000334 combuf = b''.join(bufs)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000335 self.assertEqual(data, zlib.decompress(combuf),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000336 'compressed data failure')
337
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000338 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000339 bufs = []
340 cb = combuf
341 while cb:
Guido van Rossumf3594102003-02-27 18:39:18 +0000342 max_length = 1 + len(cb)//10
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000343 chunk = dco.decompress(cb, max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000344 self.assertFalse(len(chunk) > max_length,
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000345 'chunk too big (%d>%d)' % (len(chunk),max_length))
346 bufs.append(chunk)
347 cb = dco.unconsumed_tail
Neil Schemenauer6412b122004-06-05 19:34:28 +0000348 if flush:
349 bufs.append(dco.flush())
350 else:
351 while chunk:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000352 chunk = dco.decompress(b'', max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000353 self.assertFalse(len(chunk) > max_length,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000354 'chunk too big (%d>%d)' % (len(chunk),max_length))
355 bufs.append(chunk)
Guido van Rossum776152b2007-05-22 22:44:07 +0000356 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000357
Neil Schemenauer6412b122004-06-05 19:34:28 +0000358 def test_decompressmaxlenflush(self):
359 self.test_decompressmaxlen(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000360
361 def test_maxlenmisc(self):
362 # Misc tests of max_length
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000363 dco = zlib.decompressobj()
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000364 self.assertRaises(ValueError, dco.decompress, b"", -1)
Guido van Rossum776152b2007-05-22 22:44:07 +0000365 self.assertEqual(b'', dco.unconsumed_tail)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000366
Nadeem Vawda7619e882011-05-14 14:05:20 +0200367 def test_clear_unconsumed_tail(self):
368 # Issue #12050: calling decompress() without providing max_length
369 # should clear the unconsumed_tail attribute.
370 cdata = b"x\x9cKLJ\x06\x00\x02M\x01" # "abc"
371 dco = zlib.decompressobj()
372 ddata = dco.decompress(cdata, 1)
373 ddata += dco.decompress(dco.unconsumed_tail)
374 self.assertEqual(dco.unconsumed_tail, b"")
375
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000376 def test_flushes(self):
377 # Test flush() with the various options, using all the
378 # different levels in order to provide more variations.
379 sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH']
380 sync_opt = [getattr(zlib, opt) for opt in sync_opt
381 if hasattr(zlib, opt)]
Neil Schemenauer6412b122004-06-05 19:34:28 +0000382 data = HAMLET_SCENE * 8
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000383
384 for sync in sync_opt:
385 for level in range(10):
386 obj = zlib.compressobj( level )
387 a = obj.compress( data[:3000] )
388 b = obj.flush( sync )
389 c = obj.compress( data[3000:] )
390 d = obj.flush()
Guido van Rossum776152b2007-05-22 22:44:07 +0000391 self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000392 data, ("Decompress failed: flush "
393 "mode=%i, level=%i") % (sync, level))
394 del obj
395
Serhiy Storchaka43767632013-11-03 21:31:38 +0200396 @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'),
397 'requires zlib.Z_SYNC_FLUSH')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000398 def test_odd_flush(self):
399 # Test for odd flushing bugs noted in 2.0, and hopefully fixed in 2.1
400 import random
Serhiy Storchaka43767632013-11-03 21:31:38 +0200401 # Testing on 17K of "random" data
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000402
Serhiy Storchaka43767632013-11-03 21:31:38 +0200403 # Create compressor and decompressor objects
404 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
405 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000406
Serhiy Storchaka43767632013-11-03 21:31:38 +0200407 # Try 17K of data
408 # generate random data stream
409 try:
410 # In 2.3 and later, WichmannHill is the RNG of the bug report
411 gen = random.WichmannHill()
412 except AttributeError:
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000413 try:
Serhiy Storchaka43767632013-11-03 21:31:38 +0200414 # 2.2 called it Random
415 gen = random.Random()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000416 except AttributeError:
Serhiy Storchaka43767632013-11-03 21:31:38 +0200417 # others might simply have a single RNG
418 gen = random
419 gen.seed(1)
420 data = genblock(1, 17 * 1024, generator=gen)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000421
Serhiy Storchaka43767632013-11-03 21:31:38 +0200422 # compress, sync-flush, and decompress
423 first = co.compress(data)
424 second = co.flush(zlib.Z_SYNC_FLUSH)
425 expanded = dco.decompress(first + second)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000426
Serhiy Storchaka43767632013-11-03 21:31:38 +0200427 # if decompressed data is different from the input data, choke.
428 self.assertEqual(expanded, data, "17K random source doesn't match")
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000429
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000430 def test_empty_flush(self):
431 # Test that calling .flush() on unused objects works.
432 # (Bug #1083110 -- calling .flush() on decompress objects
433 # caused a core dump.)
434
435 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000436 self.assertTrue(co.flush()) # Returns a zlib header
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000437 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000438 self.assertEqual(dco.flush(), b"") # Returns nothing
Tim Peters5a9fb3c2005-01-07 16:01:32 +0000439
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200440 def test_dictionary(self):
441 h = HAMLET_SCENE
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200442 # Build a simulated dictionary out of the words in HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200443 words = h.split()
444 random.shuffle(words)
445 zdict = b''.join(words)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200446 # Use it to compress HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200447 co = zlib.compressobj(zdict=zdict)
448 cd = co.compress(h) + co.flush()
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200449 # Verify that it will decompress with the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200450 dco = zlib.decompressobj(zdict=zdict)
451 self.assertEqual(dco.decompress(cd) + dco.flush(), h)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200452 # Verify that it fails when not given the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200453 dco = zlib.decompressobj()
454 self.assertRaises(zlib.error, dco.decompress, cd)
455
456 def test_dictionary_streaming(self):
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200457 # This simulates the reuse of a compressor object for compressing
458 # several separate data streams.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200459 co = zlib.compressobj(zdict=HAMLET_SCENE)
460 do = zlib.decompressobj(zdict=HAMLET_SCENE)
461 piece = HAMLET_SCENE[1000:1500]
462 d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH)
463 d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH)
464 d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH)
465 self.assertEqual(do.decompress(d0), piece)
466 self.assertEqual(do.decompress(d1), piece[100:])
467 self.assertEqual(do.decompress(d2), piece[:-100])
468
Antoine Pitrouc09c92f2010-05-11 23:36:40 +0000469 def test_decompress_incomplete_stream(self):
470 # This is 'foo', deflated
471 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'
472 # For the record
473 self.assertEqual(zlib.decompress(x), b'foo')
474 self.assertRaises(zlib.error, zlib.decompress, x[:-5])
475 # Omitting the stream end works with decompressor objects
476 # (see issue #8672).
477 dco = zlib.decompressobj()
478 y = dco.decompress(x[:-5])
479 y += dco.flush()
480 self.assertEqual(y, b'foo')
481
Nadeem Vawda1c385462011-08-13 15:22:40 +0200482 def test_decompress_eof(self):
483 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
484 dco = zlib.decompressobj()
485 self.assertFalse(dco.eof)
486 dco.decompress(x[:-5])
487 self.assertFalse(dco.eof)
488 dco.decompress(x[-5:])
489 self.assertTrue(dco.eof)
490 dco.flush()
491 self.assertTrue(dco.eof)
492
493 def test_decompress_eof_incomplete_stream(self):
494 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
495 dco = zlib.decompressobj()
496 self.assertFalse(dco.eof)
497 dco.decompress(x[:-5])
498 self.assertFalse(dco.eof)
499 dco.flush()
500 self.assertFalse(dco.eof)
501
Nadeem Vawda39079942012-11-05 00:37:42 +0100502 def test_decompress_unused_data(self):
503 # Repeated calls to decompress() after EOF should accumulate data in
504 # dco.unused_data, instead of just storing the arg to the last call.
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100505 source = b'abcdefghijklmnopqrstuvwxyz'
506 remainder = b'0123456789'
507 y = zlib.compress(source)
508 x = y + remainder
509 for maxlen in 0, 1000:
510 for step in 1, 2, len(y), len(x):
511 dco = zlib.decompressobj()
512 data = b''
513 for i in range(0, len(x), step):
514 if i < len(y):
515 self.assertEqual(dco.unused_data, b'')
516 if maxlen == 0:
517 data += dco.decompress(x[i : i + step])
518 self.assertEqual(dco.unconsumed_tail, b'')
519 else:
520 data += dco.decompress(
521 dco.unconsumed_tail + x[i : i + step], maxlen)
522 data += dco.flush()
Nadeem Vawdadd1253a2012-11-11 02:21:22 +0100523 self.assertTrue(dco.eof)
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100524 self.assertEqual(data, source)
525 self.assertEqual(dco.unconsumed_tail, b'')
526 self.assertEqual(dco.unused_data, remainder)
Nadeem Vawda39079942012-11-05 00:37:42 +0100527
Nadeem Vawda7ee95552012-11-11 03:15:32 +0100528 def test_flush_with_freed_input(self):
529 # Issue #16411: decompressor accesses input to last decompress() call
530 # in flush(), even if this object has been freed in the meanwhile.
531 input1 = b'abcdefghijklmnopqrstuvwxyz'
532 input2 = b'QWERTYUIOPASDFGHJKLZXCVBNM'
533 data = zlib.compress(input1)
534 dco = zlib.decompressobj()
535 dco.decompress(data, 1)
536 del data
537 data = zlib.compress(input2)
538 self.assertEqual(dco.flush(), input1[1:])
539
Serhiy Storchaka43767632013-11-03 21:31:38 +0200540 @requires_Compress_copy
541 def test_compresscopy(self):
542 # Test copying a compression object
543 data0 = HAMLET_SCENE
544 data1 = bytes(str(HAMLET_SCENE, "ascii").swapcase(), "ascii")
545 c0 = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
546 bufs0 = []
547 bufs0.append(c0.compress(data0))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000548
Serhiy Storchaka43767632013-11-03 21:31:38 +0200549 c1 = c0.copy()
550 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000551
Serhiy Storchaka43767632013-11-03 21:31:38 +0200552 bufs0.append(c0.compress(data0))
553 bufs0.append(c0.flush())
554 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000555
Serhiy Storchaka43767632013-11-03 21:31:38 +0200556 bufs1.append(c1.compress(data1))
557 bufs1.append(c1.flush())
558 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000559
Serhiy Storchaka43767632013-11-03 21:31:38 +0200560 self.assertEqual(zlib.decompress(s0),data0+data0)
561 self.assertEqual(zlib.decompress(s1),data0+data1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000562
Serhiy Storchaka43767632013-11-03 21:31:38 +0200563 @requires_Compress_copy
564 def test_badcompresscopy(self):
565 # Test copying a compression object in an inconsistent state
566 c = zlib.compressobj()
567 c.compress(HAMLET_SCENE)
568 c.flush()
569 self.assertRaises(ValueError, c.copy)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000570
Serhiy Storchaka43767632013-11-03 21:31:38 +0200571 @requires_Decompress_copy
572 def test_decompresscopy(self):
573 # Test copying a decompression object
574 data = HAMLET_SCENE
575 comp = zlib.compress(data)
576 # Test type of return value
577 self.assertIsInstance(comp, bytes)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000578
Serhiy Storchaka43767632013-11-03 21:31:38 +0200579 d0 = zlib.decompressobj()
580 bufs0 = []
581 bufs0.append(d0.decompress(comp[:32]))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000582
Serhiy Storchaka43767632013-11-03 21:31:38 +0200583 d1 = d0.copy()
584 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000585
Serhiy Storchaka43767632013-11-03 21:31:38 +0200586 bufs0.append(d0.decompress(comp[32:]))
587 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000588
Serhiy Storchaka43767632013-11-03 21:31:38 +0200589 bufs1.append(d1.decompress(comp[32:]))
590 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000591
Serhiy Storchaka43767632013-11-03 21:31:38 +0200592 self.assertEqual(s0,s1)
593 self.assertEqual(s0,data)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000594
Serhiy Storchaka43767632013-11-03 21:31:38 +0200595 @requires_Decompress_copy
596 def test_baddecompresscopy(self):
597 # Test copying a compression object in an inconsistent state
598 data = zlib.compress(HAMLET_SCENE)
599 d = zlib.decompressobj()
600 d.decompress(data)
601 d.flush()
602 self.assertRaises(ValueError, d.copy)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000603
Serhiy Storchakad7a44152015-11-12 11:23:04 +0200604 def test_compresspickle(self):
605 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
606 with self.assertRaises((TypeError, pickle.PicklingError)):
607 pickle.dumps(zlib.compressobj(zlib.Z_BEST_COMPRESSION), proto)
608
609 def test_decompresspickle(self):
610 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
611 with self.assertRaises((TypeError, pickle.PicklingError)):
612 pickle.dumps(zlib.decompressobj(), proto)
613
Antoine Pitrou89562712010-05-07 17:04:02 +0000614 # Memory use of the following functions takes into account overallocation
615
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200616 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000617 def test_big_compress_buffer(self, size):
618 c = zlib.compressobj(1)
619 compress = lambda s: c.compress(s) + c.flush()
620 self.check_big_compress_buffer(size, compress)
621
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200622 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000623 def test_big_decompress_buffer(self, size):
624 d = zlib.decompressobj()
625 decompress = lambda s: d.decompress(s) + d.flush()
626 self.check_big_decompress_buffer(size, decompress)
627
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200628 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200629 def test_length_overflow(self, size):
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200630 data = b'x' * size
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200631 c = zlib.compressobj(1)
632 d = zlib.decompressobj()
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200633 try:
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200634 self.assertRaises(OverflowError, c.compress, data)
635 self.assertRaises(OverflowError, d.decompress, data)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200636 finally:
637 data = None
638
Antoine Pitrou89562712010-05-07 17:04:02 +0000639
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000640def genblock(seed, length, step=1024, generator=random):
641 """length-byte stream of random data from a seed (in step-byte blocks)."""
642 if seed is not None:
643 generator.seed(seed)
644 randint = generator.randint
645 if length < step or step < 2:
646 step = length
Guido van Rossum776152b2007-05-22 22:44:07 +0000647 blocks = bytes()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000648 for i in range(0, length, step):
Guido van Rossum776152b2007-05-22 22:44:07 +0000649 blocks += bytes(randint(0, 255) for x in range(step))
650 return blocks
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000651
652
653
654def choose_lines(source, number, seed=None, generator=random):
655 """Return a list of number lines randomly chosen from the source"""
656 if seed is not None:
657 generator.seed(seed)
658 sources = source.split('\n')
659 return [generator.choice(sources) for n in range(number)]
660
661
662
Guido van Rossum776152b2007-05-22 22:44:07 +0000663HAMLET_SCENE = b"""
Fred Drake004d5e62000-10-23 17:22:08 +0000664LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000665
666 O, fear me not.
667 I stay too long: but here my father comes.
668
669 Enter POLONIUS
670
671 A double blessing is a double grace,
672 Occasion smiles upon a second leave.
673
Fred Drake004d5e62000-10-23 17:22:08 +0000674LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000675
676 Yet here, Laertes! aboard, aboard, for shame!
677 The wind sits in the shoulder of your sail,
678 And you are stay'd for. There; my blessing with thee!
679 And these few precepts in thy memory
680 See thou character. Give thy thoughts no tongue,
681 Nor any unproportioned thought his act.
682 Be thou familiar, but by no means vulgar.
683 Those friends thou hast, and their adoption tried,
684 Grapple them to thy soul with hoops of steel;
685 But do not dull thy palm with entertainment
686 Of each new-hatch'd, unfledged comrade. Beware
687 Of entrance to a quarrel, but being in,
688 Bear't that the opposed may beware of thee.
689 Give every man thy ear, but few thy voice;
690 Take each man's censure, but reserve thy judgment.
691 Costly thy habit as thy purse can buy,
692 But not express'd in fancy; rich, not gaudy;
693 For the apparel oft proclaims the man,
694 And they in France of the best rank and station
695 Are of a most select and generous chief in that.
696 Neither a borrower nor a lender be;
697 For loan oft loses both itself and friend,
698 And borrowing dulls the edge of husbandry.
699 This above all: to thine ownself be true,
700 And it must follow, as the night the day,
701 Thou canst not then be false to any man.
702 Farewell: my blessing season this in thee!
703
Fred Drake004d5e62000-10-23 17:22:08 +0000704LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000705
706 Most humbly do I take my leave, my lord.
707
Fred Drake004d5e62000-10-23 17:22:08 +0000708LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000709
710 The time invites you; go; your servants tend.
711
Fred Drake004d5e62000-10-23 17:22:08 +0000712LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000713
714 Farewell, Ophelia; and remember well
715 What I have said to you.
716
Fred Drake004d5e62000-10-23 17:22:08 +0000717OPHELIA
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000718
719 'Tis in my memory lock'd,
720 And you yourself shall keep the key of it.
721
Fred Drake004d5e62000-10-23 17:22:08 +0000722LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000723
724 Farewell.
725"""
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000726
727
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000728if __name__ == "__main__":
Zachary Ware38c707e2015-04-13 15:00:43 -0500729 unittest.main()