blob: bd935ede1874208643b333657a32d070a68357b3 [file] [log] [blame]
Guido van Rossum7d9ea502003-02-03 20:45:52 +00001import unittest
Benjamin Petersonee8712c2008-05-20 21:35:26 +00002from test import support
Christian Heimesd5e2b6f2008-03-19 21:50:51 +00003import binascii
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00004import random
Antoine Pitrouf3d22752011-02-21 18:09:00 +00005import sys
Antoine Pitrou94190bb2011-10-04 10:22:36 +02006from test.support import bigmemtest, _1G, _4G
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +00007
R. David Murraya21e4ca2009-03-31 23:16:50 +00008zlib = support.import_module('zlib')
9
Serhiy Storchaka43767632013-11-03 21:31:38 +020010requires_Compress_copy = unittest.skipUnless(
11 hasattr(zlib.compressobj(), "copy"),
12 'requires Compress.copy()')
13requires_Decompress_copy = unittest.skipUnless(
14 hasattr(zlib.decompressobj(), "copy"),
15 'requires Decompress.copy()')
16
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000017
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020018class VersionTestCase(unittest.TestCase):
19
20 def test_library_version(self):
Nadeem Vawda131c7072012-01-25 23:16:50 +020021 # Test that the major version of the actual library in use matches the
22 # major version that we were compiled against. We can't guarantee that
23 # the minor versions will match (even on the machine on which the module
24 # was compiled), and the API is stable between minor versions, so
Nadeem Vawdad770fe42012-01-28 17:32:47 +020025 # testing only the major versions avoids spurious failures.
Nadeem Vawda131c7072012-01-25 23:16:50 +020026 self.assertEqual(zlib.ZLIB_RUNTIME_VERSION[0], zlib.ZLIB_VERSION[0])
Nadeem Vawda64d25dd2011-09-12 00:04:13 +020027
28
Guido van Rossum7d9ea502003-02-03 20:45:52 +000029class ChecksumTestCase(unittest.TestCase):
30 # checksum test cases
31 def test_crc32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000032 self.assertEqual(zlib.crc32(b""), zlib.crc32(b"", 0))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000033 self.assertTrue(zlib.crc32(b"abc", 0xffffffff))
Andrew M. Kuchlingfcfc8d52001-08-10 15:50:11 +000034
Guido van Rossum7d9ea502003-02-03 20:45:52 +000035 def test_crc32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000036 self.assertEqual(zlib.crc32(b"", 0), 0)
37 self.assertEqual(zlib.crc32(b"", 1), 1)
38 self.assertEqual(zlib.crc32(b"", 432), 432)
Andrew M. Kuchling9a0f98e2001-02-21 02:17:01 +000039
Guido van Rossum7d9ea502003-02-03 20:45:52 +000040 def test_adler32start(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000041 self.assertEqual(zlib.adler32(b""), zlib.adler32(b"", 1))
Benjamin Petersonc9c0f202009-06-30 23:06:06 +000042 self.assertTrue(zlib.adler32(b"abc", 0xffffffff))
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000043
Guido van Rossum7d9ea502003-02-03 20:45:52 +000044 def test_adler32empty(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000045 self.assertEqual(zlib.adler32(b"", 0), 0)
46 self.assertEqual(zlib.adler32(b"", 1), 1)
47 self.assertEqual(zlib.adler32(b"", 432), 432)
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +000048
Guido van Rossum7d9ea502003-02-03 20:45:52 +000049 def assertEqual32(self, seen, expected):
50 # 32-bit values masked -- checksums on 32- vs 64- bit machines
51 # This is important if bit 31 (0x08000000L) is set.
Guido van Rossume2a383d2007-01-15 16:59:06 +000052 self.assertEqual(seen & 0x0FFFFFFFF, expected & 0x0FFFFFFFF)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000053
54 def test_penguins(self):
Guido van Rossum776152b2007-05-22 22:44:07 +000055 self.assertEqual32(zlib.crc32(b"penguin", 0), 0x0e5c1a120)
56 self.assertEqual32(zlib.crc32(b"penguin", 1), 0x43b6aa94)
57 self.assertEqual32(zlib.adler32(b"penguin", 0), 0x0bcf02f6)
58 self.assertEqual32(zlib.adler32(b"penguin", 1), 0x0bd602f7)
Guido van Rossum7d9ea502003-02-03 20:45:52 +000059
Guido van Rossum776152b2007-05-22 22:44:07 +000060 self.assertEqual(zlib.crc32(b"penguin"), zlib.crc32(b"penguin", 0))
61 self.assertEqual(zlib.adler32(b"penguin"),zlib.adler32(b"penguin",1))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000062
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000063 def test_crc32_adler32_unsigned(self):
Antoine Pitrou77b338b2009-12-14 18:00:06 +000064 foo = b'abcdefghijklmnop'
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000065 # explicitly test signed behavior
Gregory P. Smith27275032008-03-20 06:20:09 +000066 self.assertEqual(zlib.crc32(foo), 2486878355)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000067 self.assertEqual(zlib.crc32(b'spam'), 1138425661)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000068 self.assertEqual(zlib.adler32(foo+foo), 3573550353)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000069 self.assertEqual(zlib.adler32(b'spam'), 72286642)
Gregory P. Smithab0d8a12008-03-17 20:24:09 +000070
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000071 def test_same_as_binascii_crc32(self):
Martin v. Löwis15b16a32008-12-02 06:00:15 +000072 foo = b'abcdefghijklmnop'
Gregory P. Smith27275032008-03-20 06:20:09 +000073 crc = 2486878355
Christian Heimesd5e2b6f2008-03-19 21:50:51 +000074 self.assertEqual(binascii.crc32(foo), crc)
75 self.assertEqual(zlib.crc32(foo), crc)
Martin v. Löwis15b16a32008-12-02 06:00:15 +000076 self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
Guido van Rossum7d9ea502003-02-03 20:45:52 +000077
78
Antoine Pitrouf3d22752011-02-21 18:09:00 +000079# Issue #10276 - check that inputs >=4GB are handled correctly.
80class ChecksumBigBufferTestCase(unittest.TestCase):
81
Nadeem Vawdabc8c8172012-02-23 14:16:15 +020082 @bigmemtest(size=_4G + 4, memuse=1, dry_run=False)
83 def test_big_buffer(self, size):
Nadeem Vawdab063a482012-02-23 13:36:25 +020084 data = b"nyan" * (_1G + 1)
85 self.assertEqual(zlib.crc32(data), 1044521549)
86 self.assertEqual(zlib.adler32(data), 2256789997)
Antoine Pitrouf3d22752011-02-21 18:09:00 +000087
Christian Heimesb186d002008-03-18 15:15:01 +000088
Guido van Rossum7d9ea502003-02-03 20:45:52 +000089class ExceptionTestCase(unittest.TestCase):
90 # make sure we generate some expected errors
Guido van Rossum8ce8a782007-11-01 19:42:39 +000091 def test_badlevel(self):
92 # specifying compression level out of range causes an error
93 # (but -1 is Z_DEFAULT_COMPRESSION and apparently the zlib
94 # accepts 0 too)
Antoine Pitrou77b338b2009-12-14 18:00:06 +000095 self.assertRaises(zlib.error, zlib.compress, b'ERROR', 10)
96
97 def test_badargs(self):
98 self.assertRaises(TypeError, zlib.adler32)
99 self.assertRaises(TypeError, zlib.crc32)
100 self.assertRaises(TypeError, zlib.compress)
101 self.assertRaises(TypeError, zlib.decompress)
102 for arg in (42, None, '', 'abc', (), []):
103 self.assertRaises(TypeError, zlib.adler32, arg)
104 self.assertRaises(TypeError, zlib.crc32, arg)
105 self.assertRaises(TypeError, zlib.compress, arg)
106 self.assertRaises(TypeError, zlib.decompress, arg)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000107
108 def test_badcompressobj(self):
109 # verify failure on building compress object with bad params
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000110 self.assertRaises(ValueError, zlib.compressobj, 1, zlib.DEFLATED, 0)
Guido van Rossum8ce8a782007-11-01 19:42:39 +0000111 # specifying total bits too large causes an error
112 self.assertRaises(ValueError,
113 zlib.compressobj, 1, zlib.DEFLATED, zlib.MAX_WBITS + 1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000114
115 def test_baddecompressobj(self):
116 # verify failure on building decompress object with bad params
Antoine Pitrou90ee4df2010-04-06 17:23:13 +0000117 self.assertRaises(ValueError, zlib.decompressobj, -1)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000118
Christian Heimes5e696852008-04-09 08:37:03 +0000119 def test_decompressobj_badflush(self):
120 # verify failure on calling decompressobj.flush with bad params
121 self.assertRaises(ValueError, zlib.decompressobj().flush, 0)
122 self.assertRaises(ValueError, zlib.decompressobj().flush, -1)
123
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000124
Antoine Pitrou89562712010-05-07 17:04:02 +0000125class BaseCompressTestCase(object):
126 def check_big_compress_buffer(self, size, compress_func):
127 _1M = 1024 * 1024
128 fmt = "%%0%dx" % (2 * _1M)
129 # Generate 10MB worth of random, and expand it by repeating it.
130 # The assumption is that zlib's memory is not big enough to exploit
131 # such spread out redundancy.
132 data = b''.join([random.getrandbits(8 * _1M).to_bytes(_1M, 'little')
133 for i in range(10)])
134 data = data * (size // len(data) + 1)
135 try:
136 compress_func(data)
137 finally:
138 # Release memory
139 data = None
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000140
Antoine Pitrou89562712010-05-07 17:04:02 +0000141 def check_big_decompress_buffer(self, size, decompress_func):
142 data = b'x' * size
143 try:
144 compressed = zlib.compress(data, 1)
145 finally:
146 # Release memory
147 data = None
148 data = decompress_func(compressed)
149 # Sanity check
150 try:
151 self.assertEqual(len(data), size)
152 self.assertEqual(len(data.strip(b'x')), 0)
153 finally:
154 data = None
155
156
157class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000158 # Test compression in one go (whole message compression)
159 def test_speech(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000160 x = zlib.compress(HAMLET_SCENE)
161 self.assertEqual(zlib.decompress(x), HAMLET_SCENE)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000162
163 def test_speech128(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000164 # compress more data
165 data = HAMLET_SCENE * 128
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000166 x = zlib.compress(data)
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000167 self.assertEqual(zlib.compress(bytearray(data)), x)
168 for ob in x, bytearray(x):
169 self.assertEqual(zlib.decompress(ob), data)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000170
Antoine Pitrou53b21662010-05-11 23:46:02 +0000171 def test_incomplete_stream(self):
172 # An useful error message is given
173 x = zlib.compress(HAMLET_SCENE)
Ezio Melottied3a7d22010-12-01 02:32:32 +0000174 self.assertRaisesRegex(zlib.error,
Antoine Pitrou53b21662010-05-11 23:46:02 +0000175 "Error -5 while decompressing data: incomplete or truncated stream",
176 zlib.decompress, x[:-1])
177
Antoine Pitrou89562712010-05-07 17:04:02 +0000178 # Memory use of the following functions takes into account overallocation
179
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200180 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000181 def test_big_compress_buffer(self, size):
182 compress = lambda s: zlib.compress(s, 1)
183 self.check_big_compress_buffer(size, compress)
184
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200185 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000186 def test_big_decompress_buffer(self, size):
187 self.check_big_decompress_buffer(size, zlib.decompress)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000188
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200189 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000190 def test_length_overflow(self, size):
Victor Stinner8848c7a2011-01-04 02:07:36 +0000191 data = b'x' * size
192 try:
193 self.assertRaises(OverflowError, zlib.compress, data, 1)
Nadeem Vawda154bdf92011-05-14 23:07:36 +0200194 self.assertRaises(OverflowError, zlib.decompress, data)
Victor Stinner8848c7a2011-01-04 02:07:36 +0000195 finally:
196 data = None
197
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000198
Antoine Pitrou89562712010-05-07 17:04:02 +0000199class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000200 # Test compression object
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000201 def test_pair(self):
Neil Schemenauer6412b122004-06-05 19:34:28 +0000202 # straightforward compress/decompress objects
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000203 datasrc = HAMLET_SCENE * 128
204 datazip = zlib.compress(datasrc)
205 # should compress both bytes and bytearray data
206 for data in (datasrc, bytearray(datasrc)):
207 co = zlib.compressobj()
208 x1 = co.compress(data)
209 x2 = co.flush()
210 self.assertRaises(zlib.error, co.flush) # second flush should not work
211 self.assertEqual(x1 + x2, datazip)
212 for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))):
213 dco = zlib.decompressobj()
214 y1 = dco.decompress(v1 + v2)
215 y2 = dco.flush()
216 self.assertEqual(data, y1 + y2)
217 self.assertIsInstance(dco.unconsumed_tail, bytes)
218 self.assertIsInstance(dco.unused_data, bytes)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000219
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000220 def test_compressoptions(self):
221 # specify lots of options to compressobj()
222 level = 2
223 method = zlib.DEFLATED
224 wbits = -12
225 memlevel = 9
226 strategy = zlib.Z_FILTERED
227 co = zlib.compressobj(level, method, wbits, memlevel, strategy)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000228 x1 = co.compress(HAMLET_SCENE)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000229 x2 = co.flush()
230 dco = zlib.decompressobj(wbits)
231 y1 = dco.decompress(x1 + x2)
232 y2 = dco.flush()
Neil Schemenauer6412b122004-06-05 19:34:28 +0000233 self.assertEqual(HAMLET_SCENE, y1 + y2)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000234
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000235 def test_compressincremental(self):
236 # compress object in steps, decompress object as one-shot
Neil Schemenauer6412b122004-06-05 19:34:28 +0000237 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000238 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000239 bufs = []
240 for i in range(0, len(data), 256):
241 bufs.append(co.compress(data[i:i+256]))
242 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000243 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000244
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000245 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000246 y1 = dco.decompress(b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000247 y2 = dco.flush()
248 self.assertEqual(data, y1 + y2)
249
Neil Schemenauer6412b122004-06-05 19:34:28 +0000250 def test_decompinc(self, flush=False, source=None, cx=256, dcx=64):
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000251 # compress object in steps, decompress object in steps
Neil Schemenauer6412b122004-06-05 19:34:28 +0000252 source = source or HAMLET_SCENE
253 data = source * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000254 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000255 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000256 for i in range(0, len(data), cx):
257 bufs.append(co.compress(data[i:i+cx]))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000258 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000259 combuf = b''.join(bufs)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000260
Gregory P. Smith693fc462008-09-06 20:13:06 +0000261 decombuf = zlib.decompress(combuf)
262 # Test type of return value
Ezio Melottie9615932010-01-24 19:26:24 +0000263 self.assertIsInstance(decombuf, bytes)
Gregory P. Smith693fc462008-09-06 20:13:06 +0000264
265 self.assertEqual(data, decombuf)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000266
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000267 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000268 bufs = []
Neil Schemenauer6412b122004-06-05 19:34:28 +0000269 for i in range(0, len(combuf), dcx):
270 bufs.append(dco.decompress(combuf[i:i+dcx]))
Guido van Rossum776152b2007-05-22 22:44:07 +0000271 self.assertEqual(b'', dco.unconsumed_tail, ########
272 "(A) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000273 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000274 self.assertEqual(b'', dco.unused_data)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000275 if flush:
276 bufs.append(dco.flush())
277 else:
278 while True:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000279 chunk = dco.decompress(b'')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000280 if chunk:
281 bufs.append(chunk)
282 else:
283 break
Guido van Rossum776152b2007-05-22 22:44:07 +0000284 self.assertEqual(b'', dco.unconsumed_tail, ########
285 "(B) uct should be b'': not %d long" %
Neil Schemenauer6412b122004-06-05 19:34:28 +0000286 len(dco.unconsumed_tail))
Amaury Forgeot d'Arce43d33a2008-07-02 20:50:16 +0000287 self.assertEqual(b'', dco.unused_data)
Guido van Rossum776152b2007-05-22 22:44:07 +0000288 self.assertEqual(data, b''.join(bufs))
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000289 # Failure means: "decompressobj with init options failed"
290
Neil Schemenauer6412b122004-06-05 19:34:28 +0000291 def test_decompincflush(self):
292 self.test_decompinc(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000293
Neil Schemenauer6412b122004-06-05 19:34:28 +0000294 def test_decompimax(self, source=None, cx=256, dcx=64):
295 # compress in steps, decompress in length-restricted steps
296 source = source or HAMLET_SCENE
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000297 # Check a decompression object with max_length specified
Neil Schemenauer6412b122004-06-05 19:34:28 +0000298 data = source * 128
299 co = zlib.compressobj()
300 bufs = []
301 for i in range(0, len(data), cx):
302 bufs.append(co.compress(data[i:i+cx]))
303 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000304 combuf = b''.join(bufs)
Neil Schemenauer6412b122004-06-05 19:34:28 +0000305 self.assertEqual(data, zlib.decompress(combuf),
306 'compressed data failure')
307
308 dco = zlib.decompressobj()
309 bufs = []
310 cb = combuf
311 while cb:
312 #max_length = 1 + len(cb)//10
313 chunk = dco.decompress(cb, dcx)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000314 self.assertFalse(len(chunk) > dcx,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000315 'chunk too big (%d>%d)' % (len(chunk), dcx))
316 bufs.append(chunk)
317 cb = dco.unconsumed_tail
318 bufs.append(dco.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000319 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Neil Schemenauer6412b122004-06-05 19:34:28 +0000320
321 def test_decompressmaxlen(self, flush=False):
322 # Check a decompression object with max_length specified
323 data = HAMLET_SCENE * 128
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000324 co = zlib.compressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000325 bufs = []
326 for i in range(0, len(data), 256):
327 bufs.append(co.compress(data[i:i+256]))
328 bufs.append(co.flush())
Guido van Rossum776152b2007-05-22 22:44:07 +0000329 combuf = b''.join(bufs)
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000330 self.assertEqual(data, zlib.decompress(combuf),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000331 'compressed data failure')
332
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000333 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000334 bufs = []
335 cb = combuf
336 while cb:
Guido van Rossumf3594102003-02-27 18:39:18 +0000337 max_length = 1 + len(cb)//10
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000338 chunk = dco.decompress(cb, max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000339 self.assertFalse(len(chunk) > max_length,
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000340 'chunk too big (%d>%d)' % (len(chunk),max_length))
341 bufs.append(chunk)
342 cb = dco.unconsumed_tail
Neil Schemenauer6412b122004-06-05 19:34:28 +0000343 if flush:
344 bufs.append(dco.flush())
345 else:
346 while chunk:
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000347 chunk = dco.decompress(b'', max_length)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000348 self.assertFalse(len(chunk) > max_length,
Neil Schemenauer6412b122004-06-05 19:34:28 +0000349 'chunk too big (%d>%d)' % (len(chunk),max_length))
350 bufs.append(chunk)
Guido van Rossum776152b2007-05-22 22:44:07 +0000351 self.assertEqual(data, b''.join(bufs), 'Wrong data retrieved')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000352
Neil Schemenauer6412b122004-06-05 19:34:28 +0000353 def test_decompressmaxlenflush(self):
354 self.test_decompressmaxlen(flush=True)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000355
356 def test_maxlenmisc(self):
357 # Misc tests of max_length
Neil Schemenauer94afd3e2004-06-05 19:02:52 +0000358 dco = zlib.decompressobj()
Antoine Pitrou77b338b2009-12-14 18:00:06 +0000359 self.assertRaises(ValueError, dco.decompress, b"", -1)
Guido van Rossum776152b2007-05-22 22:44:07 +0000360 self.assertEqual(b'', dco.unconsumed_tail)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000361
Nadeem Vawda7619e882011-05-14 14:05:20 +0200362 def test_clear_unconsumed_tail(self):
363 # Issue #12050: calling decompress() without providing max_length
364 # should clear the unconsumed_tail attribute.
365 cdata = b"x\x9cKLJ\x06\x00\x02M\x01" # "abc"
366 dco = zlib.decompressobj()
367 ddata = dco.decompress(cdata, 1)
368 ddata += dco.decompress(dco.unconsumed_tail)
369 self.assertEqual(dco.unconsumed_tail, b"")
370
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000371 def test_flushes(self):
372 # Test flush() with the various options, using all the
373 # different levels in order to provide more variations.
374 sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH']
375 sync_opt = [getattr(zlib, opt) for opt in sync_opt
376 if hasattr(zlib, opt)]
Neil Schemenauer6412b122004-06-05 19:34:28 +0000377 data = HAMLET_SCENE * 8
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000378
379 for sync in sync_opt:
380 for level in range(10):
381 obj = zlib.compressobj( level )
382 a = obj.compress( data[:3000] )
383 b = obj.flush( sync )
384 c = obj.compress( data[3000:] )
385 d = obj.flush()
Guido van Rossum776152b2007-05-22 22:44:07 +0000386 self.assertEqual(zlib.decompress(b''.join([a,b,c,d])),
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000387 data, ("Decompress failed: flush "
388 "mode=%i, level=%i") % (sync, level))
389 del obj
390
Serhiy Storchaka43767632013-11-03 21:31:38 +0200391 @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'),
392 'requires zlib.Z_SYNC_FLUSH')
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000393 def test_odd_flush(self):
394 # Test for odd flushing bugs noted in 2.0, and hopefully fixed in 2.1
395 import random
Serhiy Storchaka43767632013-11-03 21:31:38 +0200396 # Testing on 17K of "random" data
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000397
Serhiy Storchaka43767632013-11-03 21:31:38 +0200398 # Create compressor and decompressor objects
399 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
400 dco = zlib.decompressobj()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000401
Serhiy Storchaka43767632013-11-03 21:31:38 +0200402 # Try 17K of data
403 # generate random data stream
404 try:
405 # In 2.3 and later, WichmannHill is the RNG of the bug report
406 gen = random.WichmannHill()
407 except AttributeError:
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000408 try:
Serhiy Storchaka43767632013-11-03 21:31:38 +0200409 # 2.2 called it Random
410 gen = random.Random()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000411 except AttributeError:
Serhiy Storchaka43767632013-11-03 21:31:38 +0200412 # others might simply have a single RNG
413 gen = random
414 gen.seed(1)
415 data = genblock(1, 17 * 1024, generator=gen)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000416
Serhiy Storchaka43767632013-11-03 21:31:38 +0200417 # compress, sync-flush, and decompress
418 first = co.compress(data)
419 second = co.flush(zlib.Z_SYNC_FLUSH)
420 expanded = dco.decompress(first + second)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000421
Serhiy Storchaka43767632013-11-03 21:31:38 +0200422 # if decompressed data is different from the input data, choke.
423 self.assertEqual(expanded, data, "17K random source doesn't match")
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000424
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000425 def test_empty_flush(self):
426 # Test that calling .flush() on unused objects works.
427 # (Bug #1083110 -- calling .flush() on decompress objects
428 # caused a core dump.)
429
430 co = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
Benjamin Petersonc9c0f202009-06-30 23:06:06 +0000431 self.assertTrue(co.flush()) # Returns a zlib header
Andrew M. Kuchling3b585b32004-12-28 20:10:48 +0000432 dco = zlib.decompressobj()
Guido van Rossum776152b2007-05-22 22:44:07 +0000433 self.assertEqual(dco.flush(), b"") # Returns nothing
Tim Peters5a9fb3c2005-01-07 16:01:32 +0000434
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200435 def test_dictionary(self):
436 h = HAMLET_SCENE
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200437 # Build a simulated dictionary out of the words in HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200438 words = h.split()
439 random.shuffle(words)
440 zdict = b''.join(words)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200441 # Use it to compress HAMLET.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200442 co = zlib.compressobj(zdict=zdict)
443 cd = co.compress(h) + co.flush()
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200444 # Verify that it will decompress with the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200445 dco = zlib.decompressobj(zdict=zdict)
446 self.assertEqual(dco.decompress(cd) + dco.flush(), h)
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200447 # Verify that it fails when not given the dictionary.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200448 dco = zlib.decompressobj()
449 self.assertRaises(zlib.error, dco.decompress, cd)
450
451 def test_dictionary_streaming(self):
Nadeem Vawdacf5e1d82012-06-22 00:35:57 +0200452 # This simulates the reuse of a compressor object for compressing
453 # several separate data streams.
Nadeem Vawdafd8a8382012-06-21 02:13:12 +0200454 co = zlib.compressobj(zdict=HAMLET_SCENE)
455 do = zlib.decompressobj(zdict=HAMLET_SCENE)
456 piece = HAMLET_SCENE[1000:1500]
457 d0 = co.compress(piece) + co.flush(zlib.Z_SYNC_FLUSH)
458 d1 = co.compress(piece[100:]) + co.flush(zlib.Z_SYNC_FLUSH)
459 d2 = co.compress(piece[:-100]) + co.flush(zlib.Z_SYNC_FLUSH)
460 self.assertEqual(do.decompress(d0), piece)
461 self.assertEqual(do.decompress(d1), piece[100:])
462 self.assertEqual(do.decompress(d2), piece[:-100])
463
Antoine Pitrouc09c92f2010-05-11 23:36:40 +0000464 def test_decompress_incomplete_stream(self):
465 # This is 'foo', deflated
466 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E'
467 # For the record
468 self.assertEqual(zlib.decompress(x), b'foo')
469 self.assertRaises(zlib.error, zlib.decompress, x[:-5])
470 # Omitting the stream end works with decompressor objects
471 # (see issue #8672).
472 dco = zlib.decompressobj()
473 y = dco.decompress(x[:-5])
474 y += dco.flush()
475 self.assertEqual(y, b'foo')
476
Nadeem Vawda1c385462011-08-13 15:22:40 +0200477 def test_decompress_eof(self):
478 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
479 dco = zlib.decompressobj()
480 self.assertFalse(dco.eof)
481 dco.decompress(x[:-5])
482 self.assertFalse(dco.eof)
483 dco.decompress(x[-5:])
484 self.assertTrue(dco.eof)
485 dco.flush()
486 self.assertTrue(dco.eof)
487
488 def test_decompress_eof_incomplete_stream(self):
489 x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo'
490 dco = zlib.decompressobj()
491 self.assertFalse(dco.eof)
492 dco.decompress(x[:-5])
493 self.assertFalse(dco.eof)
494 dco.flush()
495 self.assertFalse(dco.eof)
496
Nadeem Vawda39079942012-11-05 00:37:42 +0100497 def test_decompress_unused_data(self):
498 # Repeated calls to decompress() after EOF should accumulate data in
499 # dco.unused_data, instead of just storing the arg to the last call.
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100500 source = b'abcdefghijklmnopqrstuvwxyz'
501 remainder = b'0123456789'
502 y = zlib.compress(source)
503 x = y + remainder
504 for maxlen in 0, 1000:
505 for step in 1, 2, len(y), len(x):
506 dco = zlib.decompressobj()
507 data = b''
508 for i in range(0, len(x), step):
509 if i < len(y):
510 self.assertEqual(dco.unused_data, b'')
511 if maxlen == 0:
512 data += dco.decompress(x[i : i + step])
513 self.assertEqual(dco.unconsumed_tail, b'')
514 else:
515 data += dco.decompress(
516 dco.unconsumed_tail + x[i : i + step], maxlen)
517 data += dco.flush()
Nadeem Vawdadd1253a2012-11-11 02:21:22 +0100518 self.assertTrue(dco.eof)
Nadeem Vawdaee7889d2012-11-11 02:14:36 +0100519 self.assertEqual(data, source)
520 self.assertEqual(dco.unconsumed_tail, b'')
521 self.assertEqual(dco.unused_data, remainder)
Nadeem Vawda39079942012-11-05 00:37:42 +0100522
Nadeem Vawda7ee95552012-11-11 03:15:32 +0100523 def test_flush_with_freed_input(self):
524 # Issue #16411: decompressor accesses input to last decompress() call
525 # in flush(), even if this object has been freed in the meanwhile.
526 input1 = b'abcdefghijklmnopqrstuvwxyz'
527 input2 = b'QWERTYUIOPASDFGHJKLZXCVBNM'
528 data = zlib.compress(input1)
529 dco = zlib.decompressobj()
530 dco.decompress(data, 1)
531 del data
532 data = zlib.compress(input2)
533 self.assertEqual(dco.flush(), input1[1:])
534
Serhiy Storchaka43767632013-11-03 21:31:38 +0200535 @requires_Compress_copy
536 def test_compresscopy(self):
537 # Test copying a compression object
538 data0 = HAMLET_SCENE
539 data1 = bytes(str(HAMLET_SCENE, "ascii").swapcase(), "ascii")
540 c0 = zlib.compressobj(zlib.Z_BEST_COMPRESSION)
541 bufs0 = []
542 bufs0.append(c0.compress(data0))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000543
Serhiy Storchaka43767632013-11-03 21:31:38 +0200544 c1 = c0.copy()
545 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000546
Serhiy Storchaka43767632013-11-03 21:31:38 +0200547 bufs0.append(c0.compress(data0))
548 bufs0.append(c0.flush())
549 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000550
Serhiy Storchaka43767632013-11-03 21:31:38 +0200551 bufs1.append(c1.compress(data1))
552 bufs1.append(c1.flush())
553 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000554
Serhiy Storchaka43767632013-11-03 21:31:38 +0200555 self.assertEqual(zlib.decompress(s0),data0+data0)
556 self.assertEqual(zlib.decompress(s1),data0+data1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000557
Serhiy Storchaka43767632013-11-03 21:31:38 +0200558 @requires_Compress_copy
559 def test_badcompresscopy(self):
560 # Test copying a compression object in an inconsistent state
561 c = zlib.compressobj()
562 c.compress(HAMLET_SCENE)
563 c.flush()
564 self.assertRaises(ValueError, c.copy)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000565
Serhiy Storchaka43767632013-11-03 21:31:38 +0200566 @requires_Decompress_copy
567 def test_decompresscopy(self):
568 # Test copying a decompression object
569 data = HAMLET_SCENE
570 comp = zlib.compress(data)
571 # Test type of return value
572 self.assertIsInstance(comp, bytes)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000573
Serhiy Storchaka43767632013-11-03 21:31:38 +0200574 d0 = zlib.decompressobj()
575 bufs0 = []
576 bufs0.append(d0.decompress(comp[:32]))
Thomas Wouters477c8d52006-05-27 19:21:47 +0000577
Serhiy Storchaka43767632013-11-03 21:31:38 +0200578 d1 = d0.copy()
579 bufs1 = bufs0[:]
Thomas Wouters477c8d52006-05-27 19:21:47 +0000580
Serhiy Storchaka43767632013-11-03 21:31:38 +0200581 bufs0.append(d0.decompress(comp[32:]))
582 s0 = b''.join(bufs0)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000583
Serhiy Storchaka43767632013-11-03 21:31:38 +0200584 bufs1.append(d1.decompress(comp[32:]))
585 s1 = b''.join(bufs1)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000586
Serhiy Storchaka43767632013-11-03 21:31:38 +0200587 self.assertEqual(s0,s1)
588 self.assertEqual(s0,data)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000589
Serhiy Storchaka43767632013-11-03 21:31:38 +0200590 @requires_Decompress_copy
591 def test_baddecompresscopy(self):
592 # Test copying a compression object in an inconsistent state
593 data = zlib.compress(HAMLET_SCENE)
594 d = zlib.decompressobj()
595 d.decompress(data)
596 d.flush()
597 self.assertRaises(ValueError, d.copy)
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000598
Antoine Pitrou89562712010-05-07 17:04:02 +0000599 # Memory use of the following functions takes into account overallocation
600
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200601 @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
Antoine Pitrou89562712010-05-07 17:04:02 +0000602 def test_big_compress_buffer(self, size):
603 c = zlib.compressobj(1)
604 compress = lambda s: c.compress(s) + c.flush()
605 self.check_big_compress_buffer(size, compress)
606
Antoine Pitrou94190bb2011-10-04 10:22:36 +0200607 @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
Antoine Pitrou89562712010-05-07 17:04:02 +0000608 def test_big_decompress_buffer(self, size):
609 d = zlib.decompressobj()
610 decompress = lambda s: d.decompress(s) + d.flush()
611 self.check_big_decompress_buffer(size, decompress)
612
Nadeem Vawda197e22c2012-02-23 14:23:17 +0200613 @bigmemtest(size=_4G + 100, memuse=1, dry_run=False)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200614 def test_length_overflow(self, size):
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200615 data = b'x' * size
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200616 c = zlib.compressobj(1)
617 d = zlib.decompressobj()
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200618 try:
Nadeem Vawda1161a9c2011-05-15 00:48:24 +0200619 self.assertRaises(OverflowError, c.compress, data)
620 self.assertRaises(OverflowError, d.decompress, data)
Nadeem Vawda0c3d96a2011-05-15 00:19:50 +0200621 finally:
622 data = None
623
Antoine Pitrou89562712010-05-07 17:04:02 +0000624
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000625def genblock(seed, length, step=1024, generator=random):
626 """length-byte stream of random data from a seed (in step-byte blocks)."""
627 if seed is not None:
628 generator.seed(seed)
629 randint = generator.randint
630 if length < step or step < 2:
631 step = length
Guido van Rossum776152b2007-05-22 22:44:07 +0000632 blocks = bytes()
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000633 for i in range(0, length, step):
Guido van Rossum776152b2007-05-22 22:44:07 +0000634 blocks += bytes(randint(0, 255) for x in range(step))
635 return blocks
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000636
637
638
639def choose_lines(source, number, seed=None, generator=random):
640 """Return a list of number lines randomly chosen from the source"""
641 if seed is not None:
642 generator.seed(seed)
643 sources = source.split('\n')
644 return [generator.choice(sources) for n in range(number)]
645
646
647
Guido van Rossum776152b2007-05-22 22:44:07 +0000648HAMLET_SCENE = b"""
Fred Drake004d5e62000-10-23 17:22:08 +0000649LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000650
651 O, fear me not.
652 I stay too long: but here my father comes.
653
654 Enter POLONIUS
655
656 A double blessing is a double grace,
657 Occasion smiles upon a second leave.
658
Fred Drake004d5e62000-10-23 17:22:08 +0000659LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000660
661 Yet here, Laertes! aboard, aboard, for shame!
662 The wind sits in the shoulder of your sail,
663 And you are stay'd for. There; my blessing with thee!
664 And these few precepts in thy memory
665 See thou character. Give thy thoughts no tongue,
666 Nor any unproportioned thought his act.
667 Be thou familiar, but by no means vulgar.
668 Those friends thou hast, and their adoption tried,
669 Grapple them to thy soul with hoops of steel;
670 But do not dull thy palm with entertainment
671 Of each new-hatch'd, unfledged comrade. Beware
672 Of entrance to a quarrel, but being in,
673 Bear't that the opposed may beware of thee.
674 Give every man thy ear, but few thy voice;
675 Take each man's censure, but reserve thy judgment.
676 Costly thy habit as thy purse can buy,
677 But not express'd in fancy; rich, not gaudy;
678 For the apparel oft proclaims the man,
679 And they in France of the best rank and station
680 Are of a most select and generous chief in that.
681 Neither a borrower nor a lender be;
682 For loan oft loses both itself and friend,
683 And borrowing dulls the edge of husbandry.
684 This above all: to thine ownself be true,
685 And it must follow, as the night the day,
686 Thou canst not then be false to any man.
687 Farewell: my blessing season this in thee!
688
Fred Drake004d5e62000-10-23 17:22:08 +0000689LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000690
691 Most humbly do I take my leave, my lord.
692
Fred Drake004d5e62000-10-23 17:22:08 +0000693LORD POLONIUS
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000694
695 The time invites you; go; your servants tend.
696
Fred Drake004d5e62000-10-23 17:22:08 +0000697LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000698
699 Farewell, Ophelia; and remember well
700 What I have said to you.
701
Fred Drake004d5e62000-10-23 17:22:08 +0000702OPHELIA
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000703
704 'Tis in my memory lock'd,
705 And you yourself shall keep the key of it.
706
Fred Drake004d5e62000-10-23 17:22:08 +0000707LAERTES
Jeremy Hylton6eb4b6a1997-08-15 15:59:43 +0000708
709 Farewell.
710"""
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000711
712
Guido van Rossum7d9ea502003-02-03 20:45:52 +0000713if __name__ == "__main__":
Zachary Ware38c707e2015-04-13 15:00:43 -0500714 unittest.main()