blob: 2f6f84042f13c61248748d5e600399a7ed024a56 [file] [log] [blame]
Guido van Rossum7d9ea502003-02-03 20:45:52 +00001import unittest
Benjamin Petersonee8712c2008-05-20 21:35:26 +00002from test import support
Christian Heimesd5e2b6f2008-03-19 21:50:51 +00003import binascii
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00004import random
Antoine Pitrouf3d22752011-02-21 18:09:00 +00005import sys
Antoine Pitrou94190bb2011-10-04 10:22:36 +02006from test.support import bigmemtest, _1G, _4G
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00007
R. David Murraya21e4ca2009-03-31 23:16:50 +00008zlib = support.import_module('zlib')
9
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000010
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020011class VersionTestCase(unittest.TestCase):
12
13 def test_library_version(self):
Nadeem Vawda131c7072012-01-25 23:16:50 +020014 # Test that the major version of the actual library in use matches the
15 # major version that we were compiled against. We can't guarantee that
16 # the minor versions will match (even on the machine on which the module
17 # was compiled), and the API is stable between minor versions, so
Nadeem Vawdad770fe42012-01-28 17:32:47 +020018 # testing only the major versions avoids spurious failures.
Nadeem Vawda131c7072012-01-25 23:16:50 +020019 self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0])
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020020
21
Guido van Rossum7d9ea502003-02-03 20:45:52 +000022class ChecksumTestCase(unittest.TestCase):
23 # checksum test cases
24 def test_crc32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000025 self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000026 self.assertTrue(zlib.crc32(b"abc", 0xffffffff))
Andrew M. Kuchlingfcfc8d52001-08-10 15:50:11 +000027
Guido van Rossum7d9ea502003-02-03 20:45:52 +000028 def test_crc32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000029 self.assertEqual(zlib.crc32(b"", 0), 0)
30 self.assertEqual(zlib.crc32(b"", 1), 1)
31 self.assertEqual(zlib.crc32(b"", 432), 432)
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000032
Guido van Rossum7d9ea502003-02-03 20:45:52 +000033 def test_adler32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000034 self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000035 self.assertTrue(zlib.adler32(b"abc", 0xffffffff))
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000036
Guido van Rossum7d9ea502003-02-03 20:45:52 +000037 def test_adler32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000038 self.assertEqual(zlib.adler32(b"", 0), 0)
39 self.assertEqual(zlib.adler32(b"", 1), 1)
40 self.assertEqual(zlib.adler32(b"", 432), 432)
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000041
Guido van Rossum7d9ea502003-02-03 20:45:52 +000042 def assertEqual32(self, seen, expected):
43 # 32-bit values masked -- checksums on 32- vs 64- bit machines
44 # This is important if bit 31 (0x08000000L) is set.
Guido van Rossume2a383d2007-01-15 16:59:06 +000045 self.assertEqual(seen & 0x0FFFFFFFF, expected & 0x0FFFFFFFF)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000046
47 def test_penguins(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000048 self.assertEqual32(zlib.crc32(b"penguin", 0), 0x0e5c1a120)
49 self.assertEqual32(zlib.crc32(b"penguin", 1), 0x43b6aa94)
50 self.assertEqual32(zlib.adler32(b"penguin", 0), 0x0bcf02f6)
51 self.assertEqual32(zlib.adler32(b"penguin", 1), 0x0bd602f7)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000052
Guido van Rossum776152b2007-05-22 22:44:07 +000053 self.assertEqual(zlib.crc32(b"penguin"), zlib.crc32(b"penguin", 0))
54 self.assertEqual(zlib.adler32(b"penguin"),zlib.adler32(b"penguin",1))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000055
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000056 def test_crc32_adler32_unsigned(self):
Antoine Pitrou77b338b2009-12-14 18:00:06 +000057 foo = b'abcdefghijklmnop'
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000058 # explicitly test signed behavior
Gregory P. Smith27275032008-03-20 06:20:09 +000059 self.assertEqual(zlib.crc32(foo), 2486878355)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000060 self.assertEqual(zlib.crc32(b'spam'), 1138425661)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000061 self.assertEqual(zlib.adler32(foo+foo), 3573550353)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000062 self.assertEqual(zlib.adler32(b'spam'), 72286642)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000063
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000064 def test_same_as_binascii_crc32(self):
Martin v. Löwis15b16a32008-12-02 06:00:15 +000065 foo = b'abcdefghijklmnop'
Gregory P. Smith27275032008-03-20 06:20:09 +000066 crc = 2486878355
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000067 self.assertEqual(binascii.crc32(foo), crc)
68 self.assertEqual(zlib.crc32(foo), crc)
Martin v. Löwis15b16a32008-12-02 06:00:15 +000069 self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000070
71
Antoine Pitrouf3d22752011-02-21 18:09:00 +000072# Issue #10276 - check that inputs >=4GB are handled correctly.
73class ChecksumBigBufferTestCase(unittest.TestCase):
74
Nadeem Vawdabc8c8172012-02-23 14:16:15 +020075 @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
76 def test_big_buffer(self, size):
Nadeem Vawdab063a482012-02-23 13:36:25 +020077 data = b"nyan" * (_1G + 1)
78 self.assertEqual(zlib.crc32(data), 1044521549)
79 self.assertEqual(zlib.adler32(data), 2256789997)
Antoine Pitrouf3d22752011-02-21 18:09:00 +000080
Christian Heimesb186d002008-03-18 15:15:01 +000081
Guido van Rossum7d9ea502003-02-03 20:45:52 +000082class ExceptionTestCase(unittest.TestCase):
83 # make sure we generate some expected errors
Guido van Rossum8ce8a782007-11-01 19:42:39 +000084 def test_badlevel(self):
85 # specifying compression level out of range causes an error
86 # (but -1 is Z_DEFAULT_COMPRESSION and apparently the zlib
87 # accepts 0 too)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000088 self.assertRaises(zlib.error, zlib.compress, b'ERROR', 10)
89
90 def test_badargs(self):
91 self.assertRaises(TypeError, zlib.adler32)
92 self.assertRaises(TypeError, zlib.crc32)
93 self.assertRaises(TypeError, zlib.compress)
94 self.assertRaises(TypeError, zlib.decompress)
95 for arg in (42, None, '', 'abc', (), []):
96 self.assertRaises(TypeError, zlib.adler32, arg)
97 self.assertRaises(TypeError, zlib.crc32, arg)
98 self.assertRaises(TypeError, zlib.compress, arg)
99 self.assertRaises(TypeError, zlib.decompress, arg)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000100
101 def test_badcompressobj(self):
102 # verify failure on building compress object with bad params
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000103 self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0)
Guido van Rossum8ce8a782007-11-01 19:42:39 +0000104 # specifying total bits too large causes an error
105 self.assertRaises(ValueError,
106 zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000107
108 def test_baddecompressobj(self):
109 # verify failure on building decompress object with bad params
Antoine Pitrou90ee4df2010-04-06 17:23:13 +0000110 self.assertRaises(ValueError, zlib.decompressobj, -1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000111
Christian Heimes5e696852008-04-09 08:37:03 +0000112 def test_decompressobj_badflush(self):
113 # verify failure on calling decompressobj.flush with bad params
114 self.assertRaises(ValueError, zlib.decompressobj().flush, 0)
115 self.assertRaises(ValueError, zlib.decompressobj().flush, -1)
116
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000117
Antoine Pitrou89562712010-05-07 17:04:02 +0000118class BaseCompressTestCase(object):
119 def check_big_compress_buffer(self, size, compress_func):
120 _1M = 1024 * 1024
121 fmt = "%%0%dx" % (2 * _1M)
122 # Generate 10MB worth of random, and expand it by repeating it.
123 # The assumption is that zlib's memory is not big enough to exploit
124 # such spread out redundancy.
125 data = b''.join([random.getrandbits(8 * _1M).to_bytes(_1M, 'little')
126 for i in range(10)])
127 data = data * (size // len(data) + 1)
128 try:
129 compress_func(data)
130 finally:
131 # Release memory
132 data = None
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000133
Antoine Pitrou89562712010-05-07 17:04:02 +0000134 def check_big_decompress_buffer(self, size, decompress_func):
135 data = b'x' * size
136 try:
137 compressed = zlib.compress(data, 1)
138 finally:
139 # Release memory
140 data = None
141 data = decompress_func(compressed)
142 # Sanity check
143 try:
144 self.assertEqual(len(data), size)
145 self.assertEqual(len(data.strip(b'x')), 0)
146 finally:
147 data = None
148
149
150class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000151 # Test compression in one go (whole message compression)
152 def test_speech(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000153 x = zlib.compress(HAMLET_SCENE)
154 self.assertEqual(zlib.decompress(x), HAMLET_SCENE)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000155
156 def test_speech128(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000157 # compress more data
158 data = HAMLET_SCENE * 128
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000159 x = zlib.compress(data)
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000160 self.assertEqual(zlib.compress(bytearray(data)), x)
161 for ob in x, bytearray(x):
162 self.assertEqual(zlib.decompress(ob), data)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000163
Antoine Pitrou53b21662010-05-11 23:46:02 +0000164 def test_incomplete_stream(self):
165 # An useful error message is given
166 x = zlib.compress(HAMLET_SCENE)
Ezio Melottied3a7d22010-12-01 02:32:32 +0000167 self.assertRaisesRegex(zlib.error,
Antoine Pitrou53b21662010-05-11 23:46:02 +0000168 "Error -5 while decompressing data: incomplete or truncated stream",
169 zlib.decompress, x[:-1])
170
Antoine Pitrou89562712010-05-07 17:04:02 +0000171 # Memory use of the following functions takes into account overallocation
172
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200173 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000174 def test_big_compress_buffer(self, size):
175 compress = lambda s: zlib.compress(s, 1)
176 self.check_big_compress_buffer(size, compress)
177
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200178 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000179 def test_big_decompress_buffer(self, size):
180 self.check_big_decompress_buffer(size, zlib.decompress)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000181
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200182 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000183 def test_length_overflow(self, size):
Victor Stinner8848c7a2011-01-04 02:07:36 +0000184 data = b'x' * size
185 try:
186 self.assertRaises(OverflowError, zlib.compress, data, 1)
Nadeem Vawda154bdf92011-05-14 23:07:36 +0200187 self.assertRaises(OverflowError, zlib.decompress, data)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000188 finally:
189 data = None
190
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000191
Antoine Pitrou89562712010-05-07 17:04:02 +0000192class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000193 # Test compression object
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000194 def test_pair(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000195 # straightforward compress/decompress objects
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000196 datasrc = HAMLET_SCENE * 128
197 datazip = zlib.compress(datasrc)
198 # should compress both bytes and bytearray data
199 for data in (datasrc, bytearray(datasrc)):
200 co = zlib.compressobj()
201 x1 = co.compress(data)
202 x2 = co.flush()
203 self.assertRaises(zlib.error, co.flush) # second flush should not work
204 self.assertEqual(x1 + x2, datazip)
205 for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
206 dco = zlib.decompressobj()
207 y1 = dco.decompress(v1 + v2)
208 y2 = dco.flush()
209 self.assertEqual(data, y1 + y2)
210 self.assertIsInstance(dco.unconsumed_tail, bytes)
211 self.assertIsInstance(dco.unused_data, bytes)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000212
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000213 def test_compressoptions(self):
214 # specify lots of options to compressobj()
215 level = 2
216 method = zlib.DEFLATED
217 wbits = -12
218 memlevel = 9
219 strategy = zlib.Z_FILTERED
220 co = zlib.compressobj(level, method, wbits, memlevel, strategy)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000221 x1 = co.compress(HAMLET_SCENE)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000222 x2 = co.flush()
223 dco = zlib.decompressobj(wbits)
224 y1 = dco.decompress(x1 + x2)
225 y2 = dco.flush()
Neil Schemenauer6412b122004-06-05 19:34:28 +0000226 self.assertEqual(HAMLET_SCENE, y1 + y2)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000227
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000228 def test_compressincremental(self):
229 # compress object in steps, decompress object as one-shot
Neil Schemenauer6412b122004-06-05 19:34:28 +0000230 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000231 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000232 bufs = []
233 for i in range(0, len(data), 256):
234 bufs.append(co.compress(data[i:i+256]))
235 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000236 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000237
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000238 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000239 y1 = dco.decompress(b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000240 y2 = dco.flush()
241 self.assertEqual(data, y1 + y2)
242
Neil Schemenauer6412b122004-06-05 19:34:28 +0000243 def test_decompinc(self, flush=False, source=None, cx=256, dcx=64):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000244 # compress object in steps, decompress object in steps
Neil Schemenauer6412b122004-06-05 19:34:28 +0000245 source = source or HAMLET_SCENE
246 data = source * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000247 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000248 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000249 for i in range(0, len(data), cx):
250 bufs.append(co.compress(data[i:i+cx]))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000251 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000252 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000253
Gregory P. Smith693fc462008-09-06 20:13:06 +0000254 decombuf = zlib.decompress(combuf)
255 # Test type of return value
Ezio Melottie9615932010-01-24 19:26:24 +0000256 self.assertIsInstance(decombuf, bytes)
Gregory P. Smith693fc462008-09-06 20:13:06 +0000257
258 self.assertEqual(data, decombuf)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000259
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000260 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000261 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000262 for i in range(0, len(combuf), dcx):
263 bufs.append(dco.decompress(combuf[i:i+dcx]))
Guido van Rossum776152b2007-05-22 22:44:07 +0000264 self.assertEqual(b'', dco.unconsumed_tail, ########
265 "(A) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000266 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000267 self.assertEqual(b'', dco.unused_data)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000268 if flush:
269 bufs.append(dco.flush())
270 else:
271 while True:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000272 chunk = dco.decompress(b'')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000273 if chunk:
274 bufs.append(chunk)
275 else:
276 break
Guido van Rossum776152b2007-05-22 22:44:07 +0000277 self.assertEqual(b'', dco.unconsumed_tail, ########
278 "(B) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000279 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000280 self.assertEqual(b'', dco.unused_data)
Guido van Rossum776152b2007-05-22 22:44:07 +0000281 self.assertEqual(data, b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000282 # Failure means: "decompressobj with init options failed"
283
Neil Schemenauer6412b122004-06-05 19:34:28 +0000284 def test_decompincflush(self):
285 self.test_decompinc(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000286
Neil Schemenauer6412b122004-06-05 19:34:28 +0000287 def test_decompimax(self, source=None, cx=256, dcx=64):
288 # compress in steps, decompress in length-restricted steps
289 source = source or HAMLET_SCENE
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000290 # Check a decompression object with max_length specified
Neil Schemenauer6412b122004-06-05 19:34:28 +0000291 data = source * 128
292 co = zlib.compressobj()
293 bufs = []
294 for i in range(0, len(data), cx):
295 bufs.append(co.compress(data[i:i+cx]))
296 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000297 combuf = b''.join(bufs)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000298 self.assertEqual(data, zlib.decompress(combuf),
299 'compressed data failure')
300
301 dco = zlib.decompressobj()
302 bufs = []
303 cb = combuf
304 while cb:
305 #max_length = 1 + len(cb)//10
306 chunk = dco.decompress(cb, dcx)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000307 self.assertFalse(len(chunk) > dcx,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000308 'chunk too big (%d>%d)' % (len(chunk), dcx))
309 bufs.append(chunk)
310 cb = dco.unconsumed_tail
311 bufs.append(dco.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000312 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000313
314 def test_decompressmaxlen(self, flush=False):
315 # Check a decompression object with max_length specified
316 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000317 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000318 bufs = []
319 for i in range(0, len(data), 256):
320 bufs.append(co.compress(data[i:i+256]))
321 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000322 combuf = b''.join(bufs)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000323 self.assertEqual(data, zlib.decompress(combuf),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000324 'compressed data failure')
325
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000326 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000327 bufs = []
328 cb = combuf
329 while cb:
Guido van Rossumf3594102003-02-27 18:39:18 +0000330 max_length = 1 + len(cb)//10
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000331 chunk = dco.decompress(cb, max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000332 self.assertFalse(len(chunk) > max_length,
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000333 'chunk too big (%d>%d)' % (len(chunk),max_length))
334 bufs.append(chunk)
335 cb = dco.unconsumed_tail
Neil Schemenauer6412b122004-06-05 19:34:28 +0000336 if flush:
337 bufs.append(dco.flush())
338 else:
339 while chunk:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000340 chunk = dco.decompress(b'', max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000341 self.assertFalse(len(chunk) > max_length,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000342 'chunk too big (%d>%d)' % (len(chunk),max_length))
343 bufs.append(chunk)
Guido van Rossum776152b2007-05-22 22:44:07 +0000344 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000345
Neil Schemenauer6412b122004-06-05 19:34:28 +0000346 def test_decompressmaxlenflush(self):
347 self.test_decompressmaxlen(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000348
349 def test_maxlenmisc(self):
350 # Misc tests of max_length
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000351 dco = zlib.decompressobj()
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000352 self.assertRaises(ValueError, dco.decompress, b"", -1)
Guido van Rossum776152b2007-05-22 22:44:07 +0000353 self.assertEqual(b'', dco.unconsumed_tail)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000354
Nadeem Vawda7619e882011-05-14 14:05:20 +0200355 def test_clear_unconsumed_tail(self):
356 # Issue #12050: calling decompress() without providing max_length
357 # should clear the unconsumed_tail attribute.
358 cdata = b"x\x9cKLJ\x06\x00\x02M\x01" # "abc"
359 dco = zlib.decompressobj()
360 ddata = dco.decompress(cdata, 1)
361 ddata += dco.decompress(dco.unconsumed_tail)
362 self.assertEqual(dco.unconsumed_tail, b"")
363
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000364 def test_flushes(self):
365 # Test flush() with the various options, using all the
366 # different levels in order to provide more variations.
367 sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH']
368 sync_opt = [getattr(zlib, opt) for opt in sync_opt
369 if hasattr(zlib, opt)]
Neil Schemenauer6412b122004-06-05 19:34:28 +0000370 data = HAMLET_SCENE * 8
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000371
372 for sync in sync_opt:
373 for level in range(10):
374 obj = zlib.compressobj( level )
375 a = obj.compress( data[:3000] )
376 b = obj.flush( sync )
377 c = obj.compress( data[3000:] )
378 d = obj.flush()
Guido van Rossum776152b2007-05-22 22:44:07 +0000379 self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000380 data, ("Decompress failed: flush "
381 "mode=%i, level=%i") % (sync, level))
382 del obj
383
384 def test_odd_flush(self):
385 # Test for odd flushing bugs noted in 2.0, and hopefully fixed in 2.1
386 import random
387
388 if hasattr(zlib, 'Z_SYNC_FLUSH'):
389 # Testing on 17K of "random" data
390
391 # Create compressor and decompressor objects
Neil Schemenauer6412b122004-06-05 19:34:28 +0000392 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000393 dco = zlib.decompressobj()
394
395 # Try 17K of data
396 # generate random data stream
397 try:
398 # In 2.3 and later, WichmannHill is the RNG of the bug report
399 gen = random.WichmannHill()
400 except AttributeError:
401 try:
402 # 2.2 called it Random
403 gen = random.Random()
404 except AttributeError:
405 # others might simply have a single RNG
406 gen = random
407 gen.seed(1)
408 data = genblock(1, 17 * 1024, generator=gen)
409
410 # compress, sync-flush, and decompress
411 first = co.compress(data)
412 second = co.flush(zlib.Z_SYNC_FLUSH)
413 expanded = dco.decompress(first + second)
414
415 # if decompressed data is different from the input data, choke.
416 self.assertEqual(expanded, data, "17K random source doesn't match")
417
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000418 def test_empty_flush(self):
419 # Test that calling .flush() on unused objects works.
420 # (Bug #1083110 -- calling .flush() on decompress objects
421 # caused a core dump.)
422
423 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000424 self.assertTrue(co.flush()) # Returns a zlib header
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000425 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000426 self.assertEqual(dco.flush(), b"") # Returns nothing
Tim Peters5a9fb3c2005-01-07 16:01:32 +0000427
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200428 def test_dictionary(self):
429 h = HAMLET_SCENE
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200430 # Build a simulated dictionary out of the words in HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200431 words = h.split()
432 random.shuffle(words)
433 zdict = b''.join(words)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200434 # Use it to compress HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200435 co = zlib.compressobj(zdict=zdict)
436 cd = co.compress(h) + co.flush()
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200437 # Verify that it will decompress with the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200438 dco = zlib.decompressobj(zdict=zdict)
439 self.assertEqual(dco.decompress(cd) + dco.flush(), h)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200440 # Verify that it fails when not given the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200441 dco = zlib.decompressobj()
442 self.assertRaises(zlib.error, dco.decompress, cd)
443
444 def test_dictionary_streaming(self):
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200445 # This simulates the reuse of a compressor object for compressing
446 # several separate data streams.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200447 co = zlib.compressobj(zdict=HAMLET_SCENE)
448 do = zlib.decompressobj(zdict=HAMLET_SCENE)
449 piece = HAMLET_SCENE[1000:1500]
450 d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH)
451 d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH)
452 d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH)
453 self.assertEqual(do.decompress(d0), piece)
454 self.assertEqual(do.decompress(d1), piece[100:])
455 self.assertEqual(do.decompress(d2), piece[:-100])
456
Antoine Pitrouc09c92f2010-05-11 23:36:40 +0000457 def test_decompress_incomplete_stream(self):
458 # This is 'foo', deflated
459 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'
460 # For the record
461 self.assertEqual(zlib.decompress(x), b'foo')
462 self.assertRaises(zlib.error, zlib.decompress, x[:-5])
463 # Omitting the stream end works with decompressor objects
464 # (see issue #8672).
465 dco = zlib.decompressobj()
466 y = dco.decompress(x[:-5])
467 y += dco.flush()
468 self.assertEqual(y, b'foo')
469
Nadeem Vawda1c385462011-08-13 15:22:40 +0200470 def test_decompress_eof(self):
471 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
472 dco = zlib.decompressobj()
473 self.assertFalse(dco.eof)
474 dco.decompress(x[:-5])
475 self.assertFalse(dco.eof)
476 dco.decompress(x[-5:])
477 self.assertTrue(dco.eof)
478 dco.flush()
479 self.assertTrue(dco.eof)
480
481 def test_decompress_eof_incomplete_stream(self):
482 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
483 dco = zlib.decompressobj()
484 self.assertFalse(dco.eof)
485 dco.decompress(x[:-5])
486 self.assertFalse(dco.eof)
487 dco.flush()
488 self.assertFalse(dco.eof)
489
Nadeem Vawda39079942012-11-05 00:37:42 +0100490 def test_decompress_unused_data(self):
491 # Repeated calls to decompress() after EOF should accumulate data in
492 # dco.unused_data, instead of just storing the arg to the last call.
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100493 source = b'abcdefghijklmnopqrstuvwxyz'
494 remainder = b'0123456789'
495 y = zlib.compress(source)
496 x = y + remainder
497 for maxlen in 0, 1000:
498 for step in 1, 2, len(y), len(x):
499 dco = zlib.decompressobj()
500 data = b''
501 for i in range(0, len(x), step):
502 if i < len(y):
503 self.assertEqual(dco.unused_data, b'')
504 if maxlen == 0:
505 data += dco.decompress(x[i : i + step])
506 self.assertEqual(dco.unconsumed_tail, b'')
507 else:
508 data += dco.decompress(
509 dco.unconsumed_tail + x[i : i + step], maxlen)
510 data += dco.flush()
Nadeem Vawdadd1253a2012-11-11 02:21:22 +0100511 self.assertTrue(dco.eof)
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100512 self.assertEqual(data, source)
513 self.assertEqual(dco.unconsumed_tail, b'')
514 self.assertEqual(dco.unused_data, remainder)
Nadeem Vawda39079942012-11-05 00:37:42 +0100515
Nadeem Vawda7ee95552012-11-11 03:15:32 +0100516 def test_flush_with_freed_input(self):
517 # Issue #16411: decompressor accesses input to last decompress() call
518 # in flush(), even if this object has been freed in the meanwhile.
519 input1 = b'abcdefghijklmnopqrstuvwxyz'
520 input2 = b'QWERTYUIOPASDFGHJKLZXCVBNM'
521 data = zlib.compress(input1)
522 dco = zlib.decompressobj()
523 dco.decompress(data, 1)
524 del data
525 data = zlib.compress(input2)
526 self.assertEqual(dco.flush(), input1[1:])
527
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000528 if hasattr(zlib.compressobj(), "copy"):
529 def test_compresscopy(self):
530 # Test copying a compression object
531 data0 = HAMLET_SCENE
Guido van Rossum776152b2007-05-22 22:44:07 +0000532 data1 = bytes(str(HAMLET_SCENE, "ascii").swapcase(), "ascii")
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000533 c0 = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
534 bufs0 = []
535 bufs0.append(c0.compress(data0))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000536
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000537 c1 = c0.copy()
538 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000539
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000540 bufs0.append(c0.compress(data0))
541 bufs0.append(c0.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000542 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000543
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000544 bufs1.append(c1.compress(data1))
545 bufs1.append(c1.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000546 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000547
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000548 self.assertEqual(zlib.decompress(s0),data0+data0)
549 self.assertEqual(zlib.decompress(s1),data0+data1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000550
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000551 def test_badcompresscopy(self):
552 # Test copying a compression object in an inconsistent state
553 c = zlib.compressobj()
554 c.compress(HAMLET_SCENE)
555 c.flush()
556 self.assertRaises(ValueError, c.copy)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000557
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000558 if hasattr(zlib.decompressobj(), "copy"):
559 def test_decompresscopy(self):
560 # Test copying a decompression object
561 data = HAMLET_SCENE
562 comp = zlib.compress(data)
Gregory P. Smith693fc462008-09-06 20:13:06 +0000563 # Test type of return value
Ezio Melottie9615932010-01-24 19:26:24 +0000564 self.assertIsInstance(comp, bytes)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000565
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000566 d0 = zlib.decompressobj()
567 bufs0 = []
568 bufs0.append(d0.decompress(comp[:32]))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000569
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000570 d1 = d0.copy()
571 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000572
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000573 bufs0.append(d0.decompress(comp[32:]))
Guido van Rossum776152b2007-05-22 22:44:07 +0000574 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000575
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000576 bufs1.append(d1.decompress(comp[32:]))
Guido van Rossum776152b2007-05-22 22:44:07 +0000577 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000578
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000579 self.assertEqual(s0,s1)
580 self.assertEqual(s0,data)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000581
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000582 def test_baddecompresscopy(self):
583 # Test copying a compression object in an inconsistent state
584 data = zlib.compress(HAMLET_SCENE)
585 d = zlib.decompressobj()
586 d.decompress(data)
587 d.flush()
588 self.assertRaises(ValueError, d.copy)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000589
Antoine Pitrou89562712010-05-07 17:04:02 +0000590 # Memory use of the following functions takes into account overallocation
591
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200592 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000593 def test_big_compress_buffer(self, size):
594 c = zlib.compressobj(1)
595 compress = lambda s: c.compress(s) + c.flush()
596 self.check_big_compress_buffer(size, compress)
597
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200598 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000599 def test_big_decompress_buffer(self, size):
600 d = zlib.decompressobj()
601 decompress = lambda s: d.decompress(s) + d.flush()
602 self.check_big_decompress_buffer(size, decompress)
603
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200604 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200605 def test_length_overflow(self, size):
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200606 data = b'x' * size
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200607 c = zlib.compressobj(1)
608 d = zlib.decompressobj()
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200609 try:
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200610 self.assertRaises(OverflowError, c.compress, data)
611 self.assertRaises(OverflowError, d.decompress, data)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200612 finally:
613 data = None
614
Antoine Pitrou89562712010-05-07 17:04:02 +0000615
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000616def genblock(seed, length, step=1024, generator=random):
617 """length-byte stream of random data from a seed (in step-byte blocks)."""
618 if seed is not None:
619 generator.seed(seed)
620 randint = generator.randint
621 if length < step or step < 2:
622 step = length
Guido van Rossum776152b2007-05-22 22:44:07 +0000623 blocks = bytes()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000624 for i in range(0, length, step):
Guido van Rossum776152b2007-05-22 22:44:07 +0000625 blocks += bytes(randint(0, 255) for x in range(step))
626 return blocks
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000627
628
629
630def choose_lines(source, number, seed=None, generator=random):
631 """Return a list of number lines randomly chosen from the source"""
632 if seed is not None:
633 generator.seed(seed)
634 sources = source.split('\n')
635 return [generator.choice(sources) for n in range(number)]
636
637
638
Guido van Rossum776152b2007-05-22 22:44:07 +0000639HAMLET_SCENE = b"""
Fred Drake004d5e62000-10-23 17:22:08 +0000640LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000641
642 O, fear me not.
643 I stay too long: but here my father comes.
644
645 Enter POLONIUS
646
647 A double blessing is a double grace,
648 Occasion smiles upon a second leave.
649
Fred Drake004d5e62000-10-23 17:22:08 +0000650LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000651
652 Yet here, Laertes! aboard, aboard, for shame!
653 The wind sits in the shoulder of your sail,
654 And you are stay'd for. There; my blessing with thee!
655 And these few precepts in thy memory
656 See thou character. Give thy thoughts no tongue,
657 Nor any unproportioned thought his act.
658 Be thou familiar, but by no means vulgar.
659 Those friends thou hast, and their adoption tried,
660 Grapple them to thy soul with hoops of steel;
661 But do not dull thy palm with entertainment
662 Of each new-hatch'd, unfledged comrade. Beware
663 Of entrance to a quarrel, but being in,
664 Bear't that the opposed may beware of thee.
665 Give every man thy ear, but few thy voice;
666 Take each man's censure, but reserve thy judgment.
667 Costly thy habit as thy purse can buy,
668 But not express'd in fancy; rich, not gaudy;
669 For the apparel oft proclaims the man,
670 And they in France of the best rank and station
671 Are of a most select and generous chief in that.
672 Neither a borrower nor a lender be;
673 For loan oft loses both itself and friend,
674 And borrowing dulls the edge of husbandry.
675 This above all: to thine ownself be true,
676 And it must follow, as the night the day,
677 Thou canst not then be false to any man.
678 Farewell: my blessing season this in thee!
679
Fred Drake004d5e62000-10-23 17:22:08 +0000680LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000681
682 Most humbly do I take my leave, my lord.
683
Fred Drake004d5e62000-10-23 17:22:08 +0000684LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000685
686 The time invites you; go; your servants tend.
687
Fred Drake004d5e62000-10-23 17:22:08 +0000688LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000689
690 Farewell, Ophelia; and remember well
691 What I have said to you.
692
Fred Drake004d5e62000-10-23 17:22:08 +0000693OPHELIA
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000694
695 'Tis in my memory lock'd,
696 And you yourself shall keep the key of it.
697
Fred Drake004d5e62000-10-23 17:22:08 +0000698LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000699
700 Farewell.
701"""
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000702
703
704def test_main():
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000705 support.run_unittest(
Nadeem Vawda64d25dd2011-09-12 00:04:13 +0200706 VersionTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000707 ChecksumTestCase,
Antoine Pitrouf3d22752011-02-21 18:09:00 +0000708 ChecksumBigBufferTestCase,
Walter Dörwald21d3a322003-05-01 17:45:56 +0000709 ExceptionTestCase,
710 CompressTestCase,
711 CompressObjectTestCase
712 )
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000713
714if __name__ == "__main__":
Guido van Rossum776152b2007-05-22 22:44:07 +0000715 unittest.main() # XXX
716 ###test_main()