blob: 26f5958fae1d5d169124220a9e64b50f8252f4df [file] [log] [blame]
Jean-Paul Calderone8671c852011-03-02 19:26:20 -05001# Copyright (C) Jean-Paul Calderone
2# Copyright (C) Twisted Matrix Laboratories.
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -04003# See LICENSE for details.
4
5"""
6Helpers for the OpenSSL test suite, largely copied from
7U{Twisted<http://twistedmatrix.com/>}.
8"""
9
10import shutil
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -080011import traceback
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -040012import os, os.path
13from tempfile import mktemp
14from unittest import TestCase
Rick Dean47262da2009-07-08 16:17:17 -050015import sys
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -040016
Jean-Paul Calderonec86bb7d2013-12-29 10:25:59 -050017from OpenSSL._util import exception_from_error_queue
18from OpenSSL.crypto import Error
Jean-Paul Calderone88f38b22009-07-16 16:25:19 -040019
Konstantinos Koukopoulos410d0422014-01-30 20:15:25 +020020try:
21 import memdbg
22except Exception:
23 class _memdbg(object): heap = None
24 memdbg = _memdbg()
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -080025
Jean-Paul Calderone4f0467a2014-01-11 11:58:41 -050026from OpenSSL._util import ffi, lib, byte_string as b
Jean-Paul Calderone9e4eeae2010-08-22 21:32:52 -040027
Jean-Paul Calderone210c0f32015-04-12 09:20:31 -040028
29# This is the UTF-8 encoding of the SNOWMAN unicode code point.
30NON_ASCII = b("\xe2\x98\x83").decode("utf-8")
31
32
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -040033class TestCase(TestCase):
34 """
Jonathan Ballet648875f2011-07-16 14:14:58 +090035 :py:class:`TestCase` adds useful testing functionality beyond what is available
36 from the standard library :py:class:`unittest.TestCase`.
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -040037 """
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080038 def run(self, result):
Jean-Paul Calderone68703ed2013-03-04 12:23:44 -080039 run = super(TestCase, self).run
40 if memdbg.heap is None:
41 return run(result)
42
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080043 # Run the test as usual
44 before = set(memdbg.heap)
Jean-Paul Calderone68703ed2013-03-04 12:23:44 -080045 run(result)
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080046
47 # Clean up some long-lived allocations so they won't be reported as
48 # memory leaks.
Jean-Paul Calderone9227c472013-12-31 13:47:36 -050049 lib.CRYPTO_cleanup_all_ex_data()
50 lib.ERR_remove_thread_state(ffi.NULL)
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080051 after = set(memdbg.heap)
52
53 if not after - before:
54 # No leaks, fast succeed
55 return
56
57 if result.wasSuccessful():
58 # If it passed, run it again with memory debugging
59 before = set(memdbg.heap)
Jean-Paul Calderone68703ed2013-03-04 12:23:44 -080060 run(result)
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080061
62 # Clean up some long-lived allocations so they won't be reported as
63 # memory leaks.
Jean-Paul Calderone3f93d212014-01-01 12:36:53 -050064 lib.CRYPTO_cleanup_all_ex_data()
65 lib.ERR_remove_thread_state(ffi.NULL)
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080066
67 after = set(memdbg.heap)
68
69 self._reportLeaks(after - before, result)
Jean-Paul Calderonef6745b32013-03-01 15:08:46 -080070
Jean-Paul Calderone40732ff2013-03-01 20:53:50 -080071
Jean-Paul Calderone855331d2013-03-03 10:21:43 -080072 def _reportLeaks(self, leaks, result):
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -080073 def format_leak(p):
74 stacks = memdbg.heap[p]
75 # Eventually look at multiple stacks for the realloc() case. For
76 # now just look at the original allocation location.
Jean-Paul Calderonec2e8b412013-03-02 16:27:55 -080077 (size, python_stack, c_stack) = stacks[0]
Jean-Paul Calderonef6745b32013-03-01 15:08:46 -080078
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -080079 stack = traceback.format_list(python_stack)[:-1]
Jean-Paul Calderonef6745b32013-03-01 15:08:46 -080080
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -080081 # c_stack looks something like this (interesting parts indicated
82 # with inserted arrows not part of the data):
83 #
84 # /home/exarkun/Projects/pyOpenSSL/branches/use-opentls/__pycache__/_cffi__x89095113xb9185b9b.so(+0x12cf) [0x7fe2e20582cf]
85 # /home/exarkun/Projects/cpython/2.7/python(PyCFunction_Call+0x8b) [0x56265a]
86 # /home/exarkun/Projects/cpython/2.7/python() [0x4d5f52]
87 # /home/exarkun/Projects/cpython/2.7/python(PyEval_EvalFrameEx+0x753b) [0x4d0e1e]
88 # /home/exarkun/Projects/cpython/2.7/python() [0x4d6419]
89 # /home/exarkun/Projects/cpython/2.7/python() [0x4d6129]
90 # /home/exarkun/Projects/cpython/2.7/python(PyEval_EvalFrameEx+0x753b) [0x4d0e1e]
91 # /home/exarkun/Projects/cpython/2.7/python(PyEval_EvalCodeEx+0x1043) [0x4d3726]
92 # /home/exarkun/Projects/cpython/2.7/python() [0x55fd51]
93 # /home/exarkun/Projects/cpython/2.7/python(PyObject_Call+0x7e) [0x420ee6]
94 # /home/exarkun/Projects/cpython/2.7/python(PyEval_CallObjectWithKeywords+0x158) [0x4d56ec]
95 # /home/exarkun/.local/lib/python2.7/site-packages/cffi-0.5-py2.7-linux-x86_64.egg/_cffi_backend.so(+0xe96e) [0x7fe2e38be96e]
96 # /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_closure_unix64_inner+0x1b9) [0x7fe2e36ad819]
97 # /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_closure_unix64+0x46) [0x7fe2e36adb7c]
98 # /lib/x86_64-linux-gnu/libcrypto.so.1.0.0(CRYPTO_malloc+0x64) [0x7fe2e1cef784] <------ end interesting
99 # /lib/x86_64-linux-gnu/libcrypto.so.1.0.0(lh_insert+0x16b) [0x7fe2e1d6a24b] .
100 # /lib/x86_64-linux-gnu/libcrypto.so.1.0.0(+0x61c18) [0x7fe2e1cf0c18] .
101 # /lib/x86_64-linux-gnu/libcrypto.so.1.0.0(+0x625ec) [0x7fe2e1cf15ec] .
102 # /lib/x86_64-linux-gnu/libcrypto.so.1.0.0(DSA_new_method+0xe6) [0x7fe2e1d524d6] .
103 # /lib/x86_64-linux-gnu/libcrypto.so.1.0.0(DSA_generate_parameters+0x3a) [0x7fe2e1d5364a] <------ begin interesting
104 # /home/exarkun/Projects/opentls/trunk/tls/c/__pycache__/_cffi__x305d4698xb539baaa.so(+0x1f397) [0x7fe2df84d397]
105 # /home/exarkun/Projects/cpython/2.7/python(PyCFunction_Call+0x8b) [0x56265a]
106 # /home/exarkun/Projects/cpython/2.7/python() [0x4d5f52]
107 # /home/exarkun/Projects/cpython/2.7/python(PyEval_EvalFrameEx+0x753b) [0x4d0e1e]
108 # /home/exarkun/Projects/cpython/2.7/python() [0x4d6419]
109 # ...
110 #
111 # Notice the stack is upside down compared to a Python traceback.
112 # Identify the start and end of interesting bits and stuff it into the stack we report.
Jean-Paul Calderonef6745b32013-03-01 15:08:46 -0800113
Jean-Paul Calderonec2e8b412013-03-02 16:27:55 -0800114 saved = list(c_stack)
115
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -0800116 # Figure the first interesting frame will be after a the cffi-compiled module
Jean-Paul Calderonec2e8b412013-03-02 16:27:55 -0800117 while c_stack and '/__pycache__/_cffi__' not in c_stack[-1]:
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -0800118 c_stack.pop()
Jean-Paul Calderonef6745b32013-03-01 15:08:46 -0800119
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -0800120 # Figure the last interesting frame will always be CRYPTO_malloc,
121 # since that's where we hooked in to things.
Jean-Paul Calderonec2e8b412013-03-02 16:27:55 -0800122 while c_stack and 'CRYPTO_malloc' not in c_stack[0] and 'CRYPTO_realloc' not in c_stack[0]:
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -0800123 c_stack.pop(0)
124
Jean-Paul Calderonec2e8b412013-03-02 16:27:55 -0800125 if c_stack:
126 c_stack.reverse()
127 else:
128 c_stack = saved[::-1]
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -0800129 stack.extend([frame + "\n" for frame in c_stack])
130
Jean-Paul Calderone2beac532013-03-03 17:30:36 -0800131 stack.insert(0, "Leaked (%s) at:\n")
Jean-Paul Calderone68a6f8f2013-03-01 17:56:22 -0800132 return "".join(stack)
133
Jean-Paul Calderone855331d2013-03-03 10:21:43 -0800134 if leaks:
Jean-Paul Calderone2beac532013-03-03 17:30:36 -0800135 unique_leaks = {}
Jean-Paul Calderone855331d2013-03-03 10:21:43 -0800136 for p in leaks:
Jean-Paul Calderone2beac532013-03-03 17:30:36 -0800137 size = memdbg.heap[p][-1][0]
138 new_leak = format_leak(p)
139 if new_leak not in unique_leaks:
140 unique_leaks[new_leak] = [(size, p)]
141 else:
142 unique_leaks[new_leak].append((size, p))
143 memdbg.free(p)
144
145 for (stack, allocs) in unique_leaks.iteritems():
146 allocs_accum = []
147 for (size, pointer) in allocs:
148
Jean-Paul Calderone9227c472013-12-31 13:47:36 -0500149 addr = int(ffi.cast('uintptr_t', pointer))
Jean-Paul Calderone2beac532013-03-03 17:30:36 -0800150 allocs_accum.append("%d@0x%x" % (size, addr))
151 allocs_report = ", ".join(sorted(allocs_accum))
152
Jean-Paul Calderone855331d2013-03-03 10:21:43 -0800153 result.addError(
154 self,
Jean-Paul Calderone2beac532013-03-03 17:30:36 -0800155 (None, Exception(stack % (allocs_report,)), None))
Jean-Paul Calderonef6745b32013-03-01 15:08:46 -0800156
Jean-Paul Calderone855331d2013-03-03 10:21:43 -0800157
158 def tearDown(self):
159 """
160 Clean up any files or directories created using :py:meth:`TestCase.mktemp`.
161 Subclasses must invoke this method if they override it or the
162 cleanup will not occur.
163 """
Jean-Paul Calderonebf37f0f2010-07-31 14:56:20 -0400164 if False and self._temporaryFiles is not None:
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400165 for temp in self._temporaryFiles:
166 if os.path.isdir(temp):
167 shutil.rmtree(temp)
168 elif os.path.exists(temp):
169 os.unlink(temp)
Jean-Paul Calderone1206daf2009-07-16 16:07:42 -0400170 try:
Jean-Paul Calderonec86bb7d2013-12-29 10:25:59 -0500171 exception_from_error_queue(Error)
Jean-Paul Calderone24b64592010-08-12 10:43:09 -0400172 except Error:
173 e = sys.exc_info()[1]
Jean-Paul Calderone1206daf2009-07-16 16:07:42 -0400174 if e.args != ([],):
175 self.fail("Left over errors in OpenSSL error queue: " + repr(e))
176
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400177
Jean-Paul Calderone8fb53182013-12-30 08:35:49 -0500178 def assertIsInstance(self, instance, classOrTuple, message=None):
179 """
180 Fail if C{instance} is not an instance of the given class or of
181 one of the given classes.
182
183 @param instance: the object to test the type (first argument of the
184 C{isinstance} call).
185 @type instance: any.
186 @param classOrTuple: the class or classes to test against (second
187 argument of the C{isinstance} call).
188 @type classOrTuple: class, type, or tuple.
189
190 @param message: Custom text to include in the exception text if the
191 assertion fails.
192 """
193 if not isinstance(instance, classOrTuple):
194 if message is None:
195 suffix = ""
196 else:
197 suffix = ": " + message
198 self.fail("%r is not an instance of %s%s" % (
199 instance, classOrTuple, suffix))
200
Jean-Paul Calderoneabfbab62013-02-09 21:25:02 -0800201
Jean-Paul Calderone060a57e2011-05-04 18:02:49 -0400202 def failUnlessIn(self, containee, container, msg=None):
203 """
Jonathan Ballet648875f2011-07-16 14:14:58 +0900204 Fail the test if :py:data:`containee` is not found in :py:data:`container`.
Jean-Paul Calderone060a57e2011-05-04 18:02:49 -0400205
Jonathan Ballet648875f2011-07-16 14:14:58 +0900206 :param containee: the value that should be in :py:class:`container`
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900207 :param container: a sequence type, or in the case of a mapping type,
Jean-Paul Calderone060a57e2011-05-04 18:02:49 -0400208 will follow semantics of 'if key in dict.keys()'
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900209 :param msg: if msg is None, then the failure message will be
Jean-Paul Calderone060a57e2011-05-04 18:02:49 -0400210 '%r not in %r' % (first, second)
211 """
212 if containee not in container:
213 raise self.failureException(msg or "%r not in %r"
214 % (containee, container))
215 return containee
216 assertIn = failUnlessIn
217
Jean-Paul Calderone15f36442014-05-01 07:58:02 -0400218 def assertNotIn(self, containee, container, msg=None):
219 """
220 Fail the test if C{containee} is found in C{container}.
221
222 @param containee: the value that should not be in C{container}
223 @param container: a sequence type, or in the case of a mapping type,
224 will follow semantics of 'if key in dict.keys()'
225 @param msg: if msg is None, then the failure message will be
226 '%r in %r' % (first, second)
227 """
228 if containee in container:
229 raise self.failureException(msg or "%r in %r"
230 % (containee, container))
231 return containee
232 failIfIn = assertNotIn
233
234
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400235 def failUnlessIdentical(self, first, second, msg=None):
236 """
Jonathan Ballet648875f2011-07-16 14:14:58 +0900237 Fail the test if :py:data:`first` is not :py:data:`second`. This is an
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400238 obect-identity-equality test, not an object equality
Jonathan Ballet648875f2011-07-16 14:14:58 +0900239 (i.e. :py:func:`__eq__`) test.
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400240
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900241 :param msg: if msg is None, then the failure message will be
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400242 '%r is not %r' % (first, second)
243 """
244 if first is not second:
245 raise self.failureException(msg or '%r is not %r' % (first, second))
246 return first
247 assertIdentical = failUnlessIdentical
248
249
250 def failIfIdentical(self, first, second, msg=None):
251 """
Jonathan Ballet648875f2011-07-16 14:14:58 +0900252 Fail the test if :py:data:`first` is :py:data:`second`. This is an
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400253 obect-identity-equality test, not an object equality
Jonathan Ballet648875f2011-07-16 14:14:58 +0900254 (i.e. :py:func:`__eq__`) test.
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400255
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900256 :param msg: if msg is None, then the failure message will be
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400257 '%r is %r' % (first, second)
258 """
259 if first is second:
260 raise self.failureException(msg or '%r is %r' % (first, second))
261 return first
262 assertNotIdentical = failIfIdentical
263
264
265 def failUnlessRaises(self, exception, f, *args, **kwargs):
266 """
Jonathan Ballet648875f2011-07-16 14:14:58 +0900267 Fail the test unless calling the function :py:data:`f` with the given
268 :py:data:`args` and :py:data:`kwargs` raises :py:data:`exception`. The
269 failure will report the traceback and call stack of the unexpected
270 exception.
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400271
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900272 :param exception: exception type that is to be expected
273 :param f: the function to call
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400274
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900275 :return: The raised exception instance, if it is of the given type.
276 :raise self.failureException: Raised if the function call does
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400277 not raise an exception or if it raises an exception of a
278 different type.
279 """
280 try:
281 result = f(*args, **kwargs)
Jean-Paul Calderone24b64592010-08-12 10:43:09 -0400282 except exception:
283 inst = sys.exc_info()[1]
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400284 return inst
285 except:
Rick Dean47262da2009-07-08 16:17:17 -0500286 raise self.failureException('%s raised instead of %s'
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400287 % (sys.exc_info()[0],
288 exception.__name__,
Rick Dean47262da2009-07-08 16:17:17 -0500289 ))
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400290 else:
291 raise self.failureException('%s not raised (%r returned)'
292 % (exception.__name__, result))
293 assertRaises = failUnlessRaises
294
295
296 _temporaryFiles = None
297 def mktemp(self):
298 """
299 Pathetic substitute for twisted.trial.unittest.TestCase.mktemp.
300 """
301 if self._temporaryFiles is None:
302 self._temporaryFiles = []
Jean-Paul Calderonef9fb8922014-01-11 08:35:54 -0500303 temp = b(mktemp(dir="."))
Jean-Paul Calderone0ef63ed2009-07-05 13:05:45 -0400304 self._temporaryFiles.append(temp)
305 return temp
306
307
Jean-Paul Calderone68649052009-07-17 21:14:27 -0400308 # Other stuff
309 def assertConsistentType(self, theType, name, *constructionArgs):
310 """
Jonathan Ballet648875f2011-07-16 14:14:58 +0900311 Perform various assertions about :py:data:`theType` to ensure that it is a
Jean-Paul Calderone68649052009-07-17 21:14:27 -0400312 well-defined type. This is useful for extension types, where it's
313 pretty easy to do something wacky. If something about the type is
314 unusual, an exception will be raised.
315
Jonathan Ballet78b92a22011-07-16 08:07:26 +0900316 :param theType: The type object about which to make assertions.
317 :param name: A string giving the name of the type.
Jonathan Ballet648875f2011-07-16 14:14:58 +0900318 :param constructionArgs: Positional arguments to use with :py:data:`theType` to
Jean-Paul Calderone68649052009-07-17 21:14:27 -0400319 create an instance of it.
320 """
321 self.assertEqual(theType.__name__, name)
322 self.assertTrue(isinstance(theType, type))
323 instance = theType(*constructionArgs)
324 self.assertIdentical(type(instance), theType)
Jean-Paul Calderone9c7f0692014-04-30 18:17:19 -0400325
326
327
328class EqualityTestsMixin(object):
329 """
330 A mixin defining tests for the standard implementation of C{==} and C{!=}.
331 """
332 def anInstance(self):
333 """
334 Return an instance of the class under test. Each call to this method
335 must return a different object. All objects returned must be equal to
336 each other.
337 """
338 raise NotImplementedError()
339
340
341 def anotherInstance(self):
342 """
343 Return an instance of the class under test. Each call to this method
344 must return a different object. The objects must not be equal to the
345 objects returned by C{anInstance}. They may or may not be equal to
346 each other (they will not be compared against each other).
347 """
348 raise NotImplementedError()
349
350
351 def test_identicalEq(self):
352 """
353 An object compares equal to itself using the C{==} operator.
354 """
355 o = self.anInstance()
356 self.assertTrue(o == o)
357
358
359 def test_identicalNe(self):
360 """
361 An object doesn't compare not equal to itself using the C{!=} operator.
362 """
363 o = self.anInstance()
364 self.assertFalse(o != o)
365
366
367 def test_sameEq(self):
368 """
369 Two objects that are equal to each other compare equal to each other
370 using the C{==} operator.
371 """
372 a = self.anInstance()
373 b = self.anInstance()
374 self.assertTrue(a == b)
375
376
377 def test_sameNe(self):
378 """
379 Two objects that are equal to each other do not compare not equal to
380 each other using the C{!=} operator.
381 """
382 a = self.anInstance()
383 b = self.anInstance()
384 self.assertFalse(a != b)
385
386
387 def test_differentEq(self):
388 """
389 Two objects that are not equal to each other do not compare equal to
390 each other using the C{==} operator.
391 """
392 a = self.anInstance()
393 b = self.anotherInstance()
394 self.assertFalse(a == b)
395
396
397 def test_differentNe(self):
398 """
399 Two objects that are not equal to each other compare not equal to each
400 other using the C{!=} operator.
401 """
402 a = self.anInstance()
403 b = self.anotherInstance()
404 self.assertTrue(a != b)
405
406
407 def test_anotherTypeEq(self):
408 """
409 The object does not compare equal to an object of an unrelated type
410 (which does not implement the comparison) using the C{==} operator.
411 """
412 a = self.anInstance()
413 b = object()
414 self.assertFalse(a == b)
415
416
417 def test_anotherTypeNe(self):
418 """
419 The object compares not equal to an object of an unrelated type (which
420 does not implement the comparison) using the C{!=} operator.
421 """
422 a = self.anInstance()
423 b = object()
424 self.assertTrue(a != b)
425
426
427 def test_delegatedEq(self):
428 """
429 The result of comparison using C{==} is delegated to the right-hand
430 operand if it is of an unrelated type.
431 """
432 class Delegate(object):
433 def __eq__(self, other):
434 # Do something crazy and obvious.
435 return [self]
436
437 a = self.anInstance()
438 b = Delegate()
439 self.assertEqual(a == b, [b])
440
441
442 def test_delegateNe(self):
443 """
444 The result of comparison using C{!=} is delegated to the right-hand
445 operand if it is of an unrelated type.
446 """
447 class Delegate(object):
448 def __ne__(self, other):
449 # Do something crazy and obvious.
450 return [self]
451
452 a = self.anInstance()
453 b = Delegate()
454 self.assertEqual(a != b, [b])