blob: 8db90a2a8cd7b943b3c2575f8881c46ac2e38f99 [file] [log] [blame]
Brett Cannonf1cfb622003-05-04 21:15:27 +00001"""Supporting definitions for the Python regression tests."""
Guido van Rossum3bead091992-01-27 17:00:37 +00002
Benjamin Petersonee8712c2008-05-20 21:35:26 +00003if __name__ != 'test.support':
4 raise ImportError('support must be imported from the test package')
Barry Warsaw408b6d32002-07-30 23:27:12 +00005
Guido van Rossumd8faa362007-04-27 19:54:29 +00006import contextlib
7import errno
Benjamin Petersonfa0d7032009-06-01 22:42:33 +00008import functools
Benjamin Peterson8cc7d882009-06-01 23:14:51 +00009import gc
Guido van Rossumd8faa362007-04-27 19:54:29 +000010import socket
Fred Drakecd1b1dd2001-03-21 18:26:33 +000011import sys
Guido van Rossumd8faa362007-04-27 19:54:29 +000012import os
Benjamin Petersone549ead2009-03-28 21:42:05 +000013import platform
Christian Heimes23daade02008-02-25 12:39:23 +000014import shutil
Thomas Wouters902d6eb2007-01-09 23:18:33 +000015import warnings
Guido van Rossumd8faa362007-04-27 19:54:29 +000016import unittest
R. David Murraya21e4ca2009-03-31 23:16:50 +000017import importlib
Raymond Hettinger57d1a882011-02-23 00:46:28 +000018import collections.abc
Florent Xiclunab14930c2010-03-13 15:26:44 +000019import re
Brian Curtind40e6f72010-07-08 21:39:08 +000020import subprocess
Barry Warsaw28a691b2010-04-17 00:19:56 +000021import imp
Benjamin Petersona6590e82010-04-11 21:22:10 +000022import time
Benjamin Peterson65c66ab2010-10-29 21:31:35 +000023import sysconfig
Antoine Pitroub9c73e82011-07-29 23:53:38 +020024import fnmatch
Vinay Sajip129fd042010-12-10 08:19:38 +000025import logging.handlers
Antoine Pitrou75e78b62011-10-04 11:51:23 +020026import struct
Hynek Schlawacke02ba102012-05-23 11:22:44 +020027import tempfile
Martin v. Löwis33f79972012-07-29 16:33:05 +020028import _testcapi
Benjamin Peterson65c66ab2010-10-29 21:31:35 +000029
Victor Stinner45df8202010-04-28 22:31:17 +000030try:
Antoine Pitrou707f2282011-07-15 22:29:44 +020031 import _thread, threading
Victor Stinner45df8202010-04-28 22:31:17 +000032except ImportError:
33 _thread = None
Antoine Pitrou707f2282011-07-15 22:29:44 +020034 threading = None
35try:
36 import multiprocessing.process
37except ImportError:
38 multiprocessing = None
39
Antoine Pitrou75e78b62011-10-04 11:51:23 +020040try:
Ezio Melotticad648c2011-05-19 21:25:10 +030041 import zlib
42except ImportError:
43 zlib = None
44
Martin v. Löwisf6b16a42012-05-01 07:58:44 +020045try:
Serhiy Storchaka8b562922013-06-17 15:38:50 +030046 import gzip
47except ImportError:
48 gzip = None
49
50try:
Martin v. Löwisf6b16a42012-05-01 07:58:44 +020051 import bz2
52except ImportError:
53 bz2 = None
54
Martin v. Löwis7fb79fc2012-05-13 10:06:36 +020055try:
56 import lzma
57except ImportError:
58 lzma = None
59
Barry Warsaw28a691b2010-04-17 00:19:56 +000060__all__ = [
Hynek Schlawacke02ba102012-05-23 11:22:44 +020061 "Error", "TestFailed", "ResourceDenied", "import_module", "verbose",
62 "use_resources", "max_memuse", "record_original_stdout",
Barry Warsaw28a691b2010-04-17 00:19:56 +000063 "get_original_stdout", "unload", "unlink", "rmtree", "forget",
Charles-François Natali87b3c922011-10-03 19:40:37 +020064 "is_resource_enabled", "requires", "requires_freebsd_version",
Hynek Schlawacke02ba102012-05-23 11:22:44 +020065 "requires_linux_version", "requires_mac_ver", "find_unused_port",
66 "bind_port", "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD",
67 "temp_cwd", "findfile", "create_empty_file", "sortdict",
68 "check_syntax_error", "open_urlresource", "check_warnings", "CleanImport",
69 "EnvironmentVarGuard", "TransientResource", "captured_stdout",
70 "captured_stdin", "captured_stderr", "time_out", "socket_peer_reset",
71 "ioerror_peer_reset", "run_with_locale", 'temp_umask',
Antoine Pitrou9c39f3c2011-04-28 19:18:10 +020072 "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest",
73 "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
74 "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail",
75 "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754",
Benjamin Peterson31dc3732011-05-08 15:34:24 -050076 "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink",
Hynek Schlawacke02ba102012-05-23 11:22:44 +020077 "skip_unless_xattr", "import_fresh_module", "requires_zlib",
78 "PIPE_MAX_SIZE", "failfast", "anticipate_failure", "run_with_tz",
Serhiy Storchaka8b562922013-06-17 15:38:50 +030079 "requires_gzip", "requires_bz2", "requires_lzma", "suppress_crash_popup",
Antoine Pitrou4d7979b2010-09-07 21:22:56 +000080 ]
Florent Xiclunaf089fd62010-03-19 14:25:03 +000081
Fred Drake1790dd42000-07-24 06:55:00 +000082class Error(Exception):
Fred Drake004d5e62000-10-23 17:22:08 +000083 """Base class for regression test exceptions."""
Fred Drake1790dd42000-07-24 06:55:00 +000084
85class TestFailed(Error):
Fred Drake004d5e62000-10-23 17:22:08 +000086 """Test failed."""
Fred Drake1790dd42000-07-24 06:55:00 +000087
Benjamin Petersone549ead2009-03-28 21:42:05 +000088class ResourceDenied(unittest.SkipTest):
Fred Drake9a0db072003-02-03 15:19:30 +000089 """Test skipped because it requested a disallowed resource.
90
91 This is raised when a test calls requires() for a resource that
92 has not be enabled. It is used to distinguish between expected
93 and unexpected skips.
94 """
95
Nick Coghlanfce769e2009-04-11 14:30:59 +000096@contextlib.contextmanager
97def _ignore_deprecated_imports(ignore=True):
98 """Context manager to suppress package and module deprecation
99 warnings when importing them.
100
101 If ignore is False, this context manager has no effect."""
102 if ignore:
103 with warnings.catch_warnings():
104 warnings.filterwarnings("ignore", ".+ (module|package)",
105 DeprecationWarning)
106 yield
107 else:
108 yield
109
110
Benjamin Peterson699adb92008-05-08 22:27:58 +0000111def import_module(name, deprecated=False):
R. David Murraya21e4ca2009-03-31 23:16:50 +0000112 """Import and return the module to be tested, raising SkipTest if
113 it is not available.
114
115 If deprecated is True, any module or package deprecation messages
116 will be suppressed."""
Nick Coghlanfce769e2009-04-11 14:30:59 +0000117 with _ignore_deprecated_imports(deprecated):
Benjamin Peterson699adb92008-05-08 22:27:58 +0000118 try:
Nick Coghlanfce769e2009-04-11 14:30:59 +0000119 return importlib.import_module(name)
R. David Murraya21e4ca2009-03-31 23:16:50 +0000120 except ImportError as msg:
121 raise unittest.SkipTest(str(msg))
Nick Coghlanfce769e2009-04-11 14:30:59 +0000122
123
Nick Coghlan47384702009-04-22 16:13:36 +0000124def _save_and_remove_module(name, orig_modules):
125 """Helper function to save and remove a module from sys.modules
126
Ezio Melotti199e0852011-05-09 06:41:55 +0300127 Raise ImportError if the module can't be imported."""
Ezio Melottifec3ad12011-05-14 06:02:25 +0300128 # try to import the module and raise an error if it can't be imported
129 if name not in sys.modules:
Ezio Melotti199e0852011-05-09 06:41:55 +0300130 __import__(name)
Nick Coghlan47384702009-04-22 16:13:36 +0000131 del sys.modules[name]
Ezio Melottifec3ad12011-05-14 06:02:25 +0300132 for modname in list(sys.modules):
133 if modname == name or modname.startswith(name + '.'):
134 orig_modules[modname] = sys.modules[modname]
135 del sys.modules[modname]
Nick Coghlan47384702009-04-22 16:13:36 +0000136
137def _save_and_block_module(name, orig_modules):
138 """Helper function to save and block a module in sys.modules
139
Ezio Melotti199e0852011-05-09 06:41:55 +0300140 Return True if the module was in sys.modules, False otherwise."""
Nick Coghlan47384702009-04-22 16:13:36 +0000141 saved = True
142 try:
143 orig_modules[name] = sys.modules[name]
144 except KeyError:
145 saved = False
Alexander Belopolsky903396e2010-07-13 14:50:16 +0000146 sys.modules[name] = None
Nick Coghlan47384702009-04-22 16:13:36 +0000147 return saved
148
149
Nick Coghlan2496f332011-09-19 20:26:31 +1000150def anticipate_failure(condition):
151 """Decorator to mark a test that is known to be broken in some cases
152
153 Any use of this decorator should have a comment identifying the
154 associated tracker issue.
155 """
156 if condition:
157 return unittest.expectedFailure
158 return lambda f: f
159
160
Nick Coghlan47384702009-04-22 16:13:36 +0000161def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
Nick Coghlanfce769e2009-04-11 14:30:59 +0000162 """Imports and returns a module, deliberately bypassing the sys.modules cache
163 and importing a fresh copy of the module. Once the import is complete,
164 the sys.modules cache is restored to its original state.
165
Nick Coghlan47384702009-04-22 16:13:36 +0000166 Modules named in fresh are also imported anew if needed by the import.
Ezio Melotti199e0852011-05-09 06:41:55 +0300167 If one of these modules can't be imported, None is returned.
Nick Coghlan47384702009-04-22 16:13:36 +0000168
169 Importing of modules named in blocked is prevented while the fresh import
Nick Coghlanfce769e2009-04-11 14:30:59 +0000170 takes place.
171
172 If deprecated is True, any module or package deprecation messages
173 will be suppressed."""
Ezio Melotti6b60fb92011-05-14 06:47:51 +0300174 # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
175 # to make sure that this utility function is working as expected
Nick Coghlanfce769e2009-04-11 14:30:59 +0000176 with _ignore_deprecated_imports(deprecated):
Nick Coghlan47384702009-04-22 16:13:36 +0000177 # Keep track of modules saved for later restoration as well
178 # as those which just need a blocking entry removed
Nick Coghlanfce769e2009-04-11 14:30:59 +0000179 orig_modules = {}
Nick Coghlan47384702009-04-22 16:13:36 +0000180 names_to_remove = []
181 _save_and_remove_module(name, orig_modules)
Nick Coghlanfce769e2009-04-11 14:30:59 +0000182 try:
Nick Coghlan47384702009-04-22 16:13:36 +0000183 for fresh_name in fresh:
184 _save_and_remove_module(fresh_name, orig_modules)
185 for blocked_name in blocked:
186 if not _save_and_block_module(blocked_name, orig_modules):
187 names_to_remove.append(blocked_name)
188 fresh_module = importlib.import_module(name)
Ezio Melotti199e0852011-05-09 06:41:55 +0300189 except ImportError:
190 fresh_module = None
Nick Coghlanfce769e2009-04-11 14:30:59 +0000191 finally:
Nick Coghlan47384702009-04-22 16:13:36 +0000192 for orig_name, module in orig_modules.items():
193 sys.modules[orig_name] = module
194 for name_to_remove in names_to_remove:
195 del sys.modules[name_to_remove]
196 return fresh_module
Nick Coghlanfce769e2009-04-11 14:30:59 +0000197
Benjamin Peterson699adb92008-05-08 22:27:58 +0000198
R. David Murraya21e4ca2009-03-31 23:16:50 +0000199def get_attribute(obj, name):
200 """Get an attribute, raising SkipTest if AttributeError is raised."""
201 try:
202 attribute = getattr(obj, name)
203 except AttributeError:
Éric Araujo4300f692011-10-05 01:50:22 +0200204 raise unittest.SkipTest("object %r has no attribute %r" % (obj, name))
R. David Murraya21e4ca2009-03-31 23:16:50 +0000205 else:
206 return attribute
207
Barry Warsawc0fb6052001-08-20 22:29:23 +0000208verbose = 1 # Flag set to 0 by regrtest.py
Thomas Wouters477c8d52006-05-27 19:21:47 +0000209use_resources = None # Flag set to [] by regrtest.py
210max_memuse = 0 # Disable bigmem tests (they will still be run with
211 # small sizes, to make sure they work.)
Neal Norwitz3ce5d922008-08-24 07:08:55 +0000212real_max_memuse = 0
Antoine Pitrou216a3bc2011-07-23 22:33:39 +0200213failfast = False
Antoine Pitroub9c73e82011-07-29 23:53:38 +0200214match_tests = None
Guido van Rossum531661c1996-12-20 02:58:22 +0000215
Tim Peters8dee8092001-09-25 20:05:11 +0000216# _original_stdout is meant to hold stdout at the time regrtest began.
217# This may be "the real" stdout, or IDLE's emulation of stdout, or whatever.
218# The point is to have some flavor of stdout the user can actually see.
219_original_stdout = None
220def record_original_stdout(stdout):
221 global _original_stdout
222 _original_stdout = stdout
223
224def get_original_stdout():
225 return _original_stdout or sys.stdout
226
Guido van Rossum3bead091992-01-27 17:00:37 +0000227def unload(name):
Fred Drake004d5e62000-10-23 17:22:08 +0000228 try:
229 del sys.modules[name]
230 except KeyError:
231 pass
Guido van Rossum3bead091992-01-27 17:00:37 +0000232
Brian Curtin6f5c5cb2012-08-13 17:05:57 -0500233if sys.platform.startswith("win"):
234 def _waitfor(func, pathname, waitall=False):
235 # Peform the operation
236 func(pathname)
237 # Now setup the wait loop
238 if waitall:
239 dirname = pathname
240 else:
241 dirname, name = os.path.split(pathname)
242 dirname = dirname or '.'
243 # Check for `pathname` to be removed from the filesystem.
244 # The exponential backoff of the timeout amounts to a total
245 # of ~1 second after which the deletion is probably an error
246 # anyway.
247 # Testing on a i7@4.3GHz shows that usually only 1 iteration is
248 # required when contention occurs.
249 timeout = 0.001
250 while timeout < 1.0:
251 # Note we are only testing for the existance of the file(s) in
252 # the contents of the directory regardless of any security or
253 # access rights. If we have made it this far, we have sufficient
254 # permissions to do that much using Python's equivalent of the
255 # Windows API FindFirstFile.
256 # Other Windows APIs can fail or give incorrect results when
257 # dealing with files that are pending deletion.
258 L = os.listdir(dirname)
259 if not (L if waitall else name in L):
260 return
261 # Increase the timeout and try again
262 time.sleep(timeout)
263 timeout *= 2
264 warnings.warn('tests may fail, delete still pending for ' + pathname,
265 RuntimeWarning, stacklevel=4)
266
267 def _unlink(filename):
268 _waitfor(os.unlink, filename)
269
270 def _rmdir(dirname):
271 _waitfor(os.rmdir, dirname)
272
273 def _rmtree(path):
274 def _rmtree_inner(path):
275 for name in os.listdir(path):
276 fullname = os.path.join(path, name)
277 if os.path.isdir(fullname):
278 _waitfor(_rmtree_inner, fullname, waitall=True)
279 os.rmdir(fullname)
280 else:
281 os.unlink(fullname)
282 _waitfor(_rmtree_inner, path, waitall=True)
283 _waitfor(os.rmdir, path)
284else:
285 _unlink = os.unlink
286 _rmdir = os.rmdir
287 _rmtree = shutil.rmtree
288
Neal Norwitz0e17f8c2006-01-23 07:51:27 +0000289def unlink(filename):
Neal Norwitz0e17f8c2006-01-23 07:51:27 +0000290 try:
Brian Curtin6f5c5cb2012-08-13 17:05:57 -0500291 _unlink(filename)
Barry Warsaw28a691b2010-04-17 00:19:56 +0000292 except OSError as error:
293 # The filename need not exist.
Michael Foord2d9c2d52010-05-04 22:29:10 +0000294 if error.errno not in (errno.ENOENT, errno.ENOTDIR):
Barry Warsaw28a691b2010-04-17 00:19:56 +0000295 raise
Neal Norwitz0e17f8c2006-01-23 07:51:27 +0000296
Brian Curtin6f5c5cb2012-08-13 17:05:57 -0500297def rmdir(dirname):
298 try:
299 _rmdir(dirname)
300 except OSError as error:
301 # The directory need not exist.
302 if error.errno != errno.ENOENT:
303 raise
304
Christian Heimes23daade02008-02-25 12:39:23 +0000305def rmtree(path):
306 try:
Brian Curtin6f5c5cb2012-08-13 17:05:57 -0500307 _rmtree(path)
Barry Warsaw28a691b2010-04-17 00:19:56 +0000308 except OSError as error:
Antoine Pitrou7a18d212011-08-30 18:34:47 +0200309 if error.errno != errno.ENOENT:
Christian Heimes23daade02008-02-25 12:39:23 +0000310 raise
311
Barry Warsaw28a691b2010-04-17 00:19:56 +0000312def make_legacy_pyc(source):
313 """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
314
315 The choice of .pyc or .pyo extension is done based on the __debug__ flag
316 value.
317
318 :param source: The file system path to the source file. The source file
319 does not need to exist, however the PEP 3147 pyc file must exist.
320 :return: The file system path to the legacy pyc file.
321 """
322 pyc_file = imp.cache_from_source(source)
323 up_one = os.path.dirname(os.path.abspath(source))
324 legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o'))
325 os.rename(pyc_file, legacy_pyc)
326 return legacy_pyc
327
Guido van Rossum3bead091992-01-27 17:00:37 +0000328def forget(modname):
Barry Warsaw28a691b2010-04-17 00:19:56 +0000329 """'Forget' a module was ever imported.
330
331 This removes the module from sys.modules and deletes any PEP 3147 or
332 legacy .pyc and .pyo files.
333 """
Fred Drake004d5e62000-10-23 17:22:08 +0000334 unload(modname)
Fred Drake004d5e62000-10-23 17:22:08 +0000335 for dirname in sys.path:
Barry Warsaw28a691b2010-04-17 00:19:56 +0000336 source = os.path.join(dirname, modname + '.py')
337 # It doesn't matter if they exist or not, unlink all possible
338 # combinations of PEP 3147 and legacy pyc and pyo files.
339 unlink(source + 'c')
340 unlink(source + 'o')
341 unlink(imp.cache_from_source(source, debug_override=True))
342 unlink(imp.cache_from_source(source, debug_override=False))
Guido van Rossum3bead091992-01-27 17:00:37 +0000343
Antoine Pitroud20a5f62011-02-26 15:58:05 +0000344# On some platforms, should not run gui test even if it is allowed
345# in `use_resources'.
346if sys.platform.startswith('win'):
347 import ctypes
348 import ctypes.wintypes
349 def _is_gui_available():
350 UOI_FLAGS = 1
351 WSF_VISIBLE = 0x0001
352 class USEROBJECTFLAGS(ctypes.Structure):
353 _fields_ = [("fInherit", ctypes.wintypes.BOOL),
354 ("fReserved", ctypes.wintypes.BOOL),
355 ("dwFlags", ctypes.wintypes.DWORD)]
356 dll = ctypes.windll.user32
357 h = dll.GetProcessWindowStation()
358 if not h:
359 raise ctypes.WinError()
360 uof = USEROBJECTFLAGS()
361 needed = ctypes.wintypes.DWORD()
362 res = dll.GetUserObjectInformationW(h,
363 UOI_FLAGS,
364 ctypes.byref(uof),
365 ctypes.sizeof(uof),
366 ctypes.byref(needed))
367 if not res:
368 raise ctypes.WinError()
369 return bool(uof.dwFlags & WSF_VISIBLE)
370else:
371 def _is_gui_available():
372 return True
373
Tim Petersb4ee4eb2002-12-04 03:26:57 +0000374def is_resource_enabled(resource):
Brett Cannonf1cfb622003-05-04 21:15:27 +0000375 """Test whether a resource is enabled. Known resources are set by
376 regrtest.py."""
Tim Petersb4ee4eb2002-12-04 03:26:57 +0000377 return use_resources is not None and resource in use_resources
378
Barry Warsawc0fb6052001-08-20 22:29:23 +0000379def requires(resource, msg=None):
Brett Cannonf1cfb622003-05-04 21:15:27 +0000380 """Raise ResourceDenied if the specified resource is not available.
381
382 If the caller's module is __main__ then automatically return True. The
Barry Warsaw28a691b2010-04-17 00:19:56 +0000383 possibility of False being returned occurs when regrtest.py is
384 executing.
385 """
Antoine Pitroud20a5f62011-02-26 15:58:05 +0000386 if resource == 'gui' and not _is_gui_available():
387 raise unittest.SkipTest("Cannot use the 'gui' resource")
Skip Montanarod839ecd2003-04-24 19:06:57 +0000388 # see if the caller's module is __main__ - if so, treat as if
389 # the resource was set
Benjamin Petersone549ead2009-03-28 21:42:05 +0000390 if sys._getframe(1).f_globals.get("__name__") == "__main__":
Skip Montanarod839ecd2003-04-24 19:06:57 +0000391 return
Tim Petersb4ee4eb2002-12-04 03:26:57 +0000392 if not is_resource_enabled(resource):
Barry Warsawc0fb6052001-08-20 22:29:23 +0000393 if msg is None:
Éric Araujoaf5bacf2011-07-15 17:50:15 +0200394 msg = "Use of the %r resource not enabled" % resource
Fred Drake9a0db072003-02-03 15:19:30 +0000395 raise ResourceDenied(msg)
Barry Warsawc0fb6052001-08-20 22:29:23 +0000396
Charles-François Natali87b3c922011-10-03 19:40:37 +0200397def _requires_unix_version(sysname, min_version):
398 """Decorator raising SkipTest if the OS is `sysname` and the version is less
399 than `min_version`.
Charles-François Natali239bb962011-06-03 12:55:15 +0200400
Charles-François Natali87b3c922011-10-03 19:40:37 +0200401 For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if
402 the FreeBSD version is less than 7.2.
Charles-François Natali239bb962011-06-03 12:55:15 +0200403 """
404 def decorator(func):
405 @functools.wraps(func)
406 def wrapper(*args, **kw):
Charles-François Natali87b3c922011-10-03 19:40:37 +0200407 if platform.system() == sysname:
Charles-François Natali239bb962011-06-03 12:55:15 +0200408 version_txt = platform.release().split('-', 1)[0]
409 try:
410 version = tuple(map(int, version_txt.split('.')))
411 except ValueError:
412 pass
413 else:
414 if version < min_version:
415 min_version_txt = '.'.join(map(str, min_version))
416 raise unittest.SkipTest(
Charles-François Natali87b3c922011-10-03 19:40:37 +0200417 "%s version %s or higher required, not %s"
418 % (sysname, min_version_txt, version_txt))
Charles-François Natali239bb962011-06-03 12:55:15 +0200419 return wrapper
420 return decorator
Victor Stinnerfea0f4d2011-05-24 00:24:19 +0200421
Charles-François Natali87b3c922011-10-03 19:40:37 +0200422def requires_freebsd_version(*min_version):
423 """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is
424 less than `min_version`.
425
426 For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD
427 version is less than 7.2.
428 """
429 return _requires_unix_version('FreeBSD', min_version)
430
431def requires_linux_version(*min_version):
432 """Decorator raising SkipTest if the OS is Linux and the Linux version is
433 less than `min_version`.
434
435 For example, @requires_linux_version(2, 6, 32) raises SkipTest if the Linux
436 version is less than 2.6.32.
437 """
438 return _requires_unix_version('Linux', min_version)
439
Victor Stinnerfce92332011-06-01 12:28:04 +0200440def requires_mac_ver(*min_version):
Victor Stinner88701e22011-06-01 13:13:04 +0200441 """Decorator raising SkipTest if the OS is Mac OS X and the OS X
442 version if less than min_version.
Victor Stinnerfce92332011-06-01 12:28:04 +0200443
Victor Stinner88701e22011-06-01 13:13:04 +0200444 For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version
445 is lesser than 10.5.
Victor Stinnerfce92332011-06-01 12:28:04 +0200446 """
Victor Stinner88701e22011-06-01 13:13:04 +0200447 def decorator(func):
448 @functools.wraps(func)
449 def wrapper(*args, **kw):
450 if sys.platform == 'darwin':
451 version_txt = platform.mac_ver()[0]
452 try:
453 version = tuple(map(int, version_txt.split('.')))
454 except ValueError:
455 pass
456 else:
457 if version < min_version:
458 min_version_txt = '.'.join(map(str, min_version))
459 raise unittest.SkipTest(
460 "Mac OS X %s or higher required, not %s"
461 % (min_version_txt, version_txt))
462 return func(*args, **kw)
463 wrapper.min_version = min_version
464 return wrapper
465 return decorator
466
Victor Stinnerfce92332011-06-01 12:28:04 +0200467
Christian Heimes5e696852008-04-09 08:37:03 +0000468HOST = 'localhost'
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000469
Christian Heimes5e696852008-04-09 08:37:03 +0000470def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
471 """Returns an unused port that should be suitable for binding. This is
472 achieved by creating a temporary socket with the same family and type as
473 the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to
474 the specified host address (defaults to 0.0.0.0) with the port set to 0,
475 eliciting an unused ephemeral port from the OS. The temporary socket is
476 then closed and deleted, and the ephemeral port is returned.
477
478 Either this method or bind_port() should be used for any tests where a
479 server socket needs to be bound to a particular port for the duration of
480 the test. Which one to use depends on whether the calling code is creating
481 a python socket, or if an unused port needs to be provided in a constructor
482 or passed to an external program (i.e. the -accept argument to openssl's
483 s_server mode). Always prefer bind_port() over find_unused_port() where
484 possible. Hard coded ports should *NEVER* be used. As soon as a server
485 socket is bound to a hard coded port, the ability to run multiple instances
486 of the test simultaneously on the same host is compromised, which makes the
487 test a ticking time bomb in a buildbot environment. On Unix buildbots, this
488 may simply manifest as a failed test, which can be recovered from without
489 intervention in most cases, but on Windows, the entire python process can
490 completely and utterly wedge, requiring someone to log in to the buildbot
491 and manually kill the affected process.
492
493 (This is easy to reproduce on Windows, unfortunately, and can be traced to
494 the SO_REUSEADDR socket option having different semantics on Windows versus
495 Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind,
496 listen and then accept connections on identical host/ports. An EADDRINUSE
497 socket.error will be raised at some point (depending on the platform and
498 the order bind and listen were called on each socket).
499
500 However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE
501 will ever be raised when attempting to bind two identical host/ports. When
502 accept() is called on each socket, the second caller's process will steal
503 the port from the first caller, leaving them both in an awkwardly wedged
504 state where they'll no longer respond to any signals or graceful kills, and
505 must be forcibly killed via OpenProcess()/TerminateProcess().
506
507 The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option
508 instead of SO_REUSEADDR, which effectively affords the same semantics as
509 SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open
510 Source world compared to Windows ones, this is a common mistake. A quick
511 look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when
512 openssl.exe is called with the 's_server' option, for example. See
513 http://bugs.python.org/issue2550 for more info. The following site also
514 has a very thorough description about the implications of both REUSEADDR
515 and EXCLUSIVEADDRUSE on Windows:
516 http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx)
517
518 XXX: although this approach is a vast improvement on previous attempts to
519 elicit unused ports, it rests heavily on the assumption that the ephemeral
520 port returned to us by the OS won't immediately be dished back out to some
521 other process when we close and delete our temporary socket but before our
522 calling code has a chance to bind the returned port. We can deal with this
523 issue if/when we come across it.
524 """
525
526 tempsock = socket.socket(family, socktype)
527 port = bind_port(tempsock)
528 tempsock.close()
529 del tempsock
530 return port
531
532def bind_port(sock, host=HOST):
533 """Bind the socket to a free port and return the port number. Relies on
534 ephemeral ports in order to ensure we are using an unbound port. This is
535 important as many tests may be running simultaneously, especially in a
536 buildbot environment. This method raises an exception if the sock.family
537 is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
538 or SO_REUSEPORT set on it. Tests should *never* set these socket options
539 for TCP/IP sockets. The only case for setting these options is testing
540 multicasting via multiple UDP sockets.
541
542 Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
543 on Windows), it will be set on the socket. This will prevent anyone else
544 from bind()'ing to our host/port for the duration of the test.
545 """
546
547 if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
548 if hasattr(socket, 'SO_REUSEADDR'):
549 if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
550 raise TestFailed("tests should never set the SO_REUSEADDR " \
551 "socket option on TCP/IP sockets!")
552 if hasattr(socket, 'SO_REUSEPORT'):
553 if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
554 raise TestFailed("tests should never set the SO_REUSEPORT " \
555 "socket option on TCP/IP sockets!")
556 if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
557 sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
558
559 sock.bind((host, 0))
560 port = sock.getsockname()[1]
561 return port
Thomas Wouters0e3f5912006-08-11 14:57:12 +0000562
Antoine Pitrou9c39f3c2011-04-28 19:18:10 +0200563def _is_ipv6_enabled():
564 """Check whether IPv6 is enabled on this host."""
565 if socket.has_ipv6:
Ross Lagerwall121d59f2012-07-07 18:40:32 +0200566 sock = None
Antoine Pitrou9c39f3c2011-04-28 19:18:10 +0200567 try:
568 sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
569 sock.bind(('::1', 0))
Ross Lagerwall121d59f2012-07-07 18:40:32 +0200570 return True
Antoine Pitrou9c39f3c2011-04-28 19:18:10 +0200571 except (socket.error, socket.gaierror):
572 pass
Ross Lagerwall121d59f2012-07-07 18:40:32 +0200573 finally:
574 if sock:
575 sock.close()
Antoine Pitrou9c39f3c2011-04-28 19:18:10 +0200576 return False
577
578IPV6_ENABLED = _is_ipv6_enabled()
579
Charles-François Natali2d517212011-05-29 16:36:44 +0200580
Antoine Pitroue1a16742013-04-24 23:31:38 +0200581# A constant likely larger than the underlying OS pipe buffer size, to
582# make writes blocking.
583# Windows limit seems to be around 512 B, and many Unix kernels have a
584# 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure.
585# (see issue #17835 for a discussion of this number).
586PIPE_MAX_SIZE = 4 *1024 * 1024 + 1
Charles-François Natali2d517212011-05-29 16:36:44 +0200587
588
Eric Smithf24a0d92010-12-04 13:32:18 +0000589# decorator for skipping tests on non-IEEE 754 platforms
590requires_IEEE_754 = unittest.skipUnless(
591 float.__getformat__("double").startswith("IEEE"),
592 "test requires IEEE 754 doubles")
593
Ezio Melotticad648c2011-05-19 21:25:10 +0300594requires_zlib = unittest.skipUnless(zlib, 'requires zlib')
595
Serhiy Storchaka8b562922013-06-17 15:38:50 +0300596requires_gzip = unittest.skipUnless(gzip, 'requires gzip')
597
Martin v. Löwisf6b16a42012-05-01 07:58:44 +0200598requires_bz2 = unittest.skipUnless(bz2, 'requires bz2')
599
Martin v. Löwis7fb79fc2012-05-13 10:06:36 +0200600requires_lzma = unittest.skipUnless(lzma, 'requires lzma')
601
Finn Bock57bc5fa2002-11-01 18:02:03 +0000602is_jython = sys.platform.startswith('java')
603
Barry Warsaw559f6682001-03-23 18:04:02 +0000604# Filename used for testing
605if os.name == 'java':
606 # Jython disallows @ in module names
607 TESTFN = '$test'
Martin v. Löwisa94568a2003-05-10 07:36:56 +0000608else:
Barry Warsaw559f6682001-03-23 18:04:02 +0000609 TESTFN = '@test'
Walter Dörwald9b775532007-06-08 14:30:53 +0000610
Antoine Pitrou88909542009-06-29 13:54:42 +0000611# Disambiguate TESTFN for parallel testing, while letting it remain a valid
612# module name.
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000613TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
Antoine Pitrou88909542009-06-29 13:54:42 +0000614
Victor Stinnere667e982012-11-12 01:23:15 +0100615# FS_NONASCII: non-ASCII character encodable by os.fsencode(),
616# or None if there is no such character.
617FS_NONASCII = None
618for character in (
619 # First try printable and common characters to have a readable filename.
620 # For each character, the encoding list are just example of encodings able
621 # to encode the character (the list is not exhaustive).
622
623 # U+00E6 (Latin Small Letter Ae): cp1252, iso-8859-1
624 '\u00E6',
625 # U+0130 (Latin Capital Letter I With Dot Above): cp1254, iso8859_3
626 '\u0130',
627 # U+0141 (Latin Capital Letter L With Stroke): cp1250, cp1257
628 '\u0141',
629 # U+03C6 (Greek Small Letter Phi): cp1253
630 '\u03C6',
631 # U+041A (Cyrillic Capital Letter Ka): cp1251
632 '\u041A',
633 # U+05D0 (Hebrew Letter Alef): Encodable to cp424
634 '\u05D0',
635 # U+060C (Arabic Comma): cp864, cp1006, iso8859_6, mac_arabic
636 '\u060C',
637 # U+062A (Arabic Letter Teh): cp720
638 '\u062A',
639 # U+0E01 (Thai Character Ko Kai): cp874
640 '\u0E01',
641
642 # Then try more "special" characters. "special" because they may be
643 # interpreted or displayed differently depending on the exact locale
644 # encoding and the font.
645
646 # U+00A0 (No-Break Space)
647 '\u00A0',
648 # U+20AC (Euro Sign)
649 '\u20AC',
650):
651 try:
652 os.fsdecode(os.fsencode(character))
653 except UnicodeError:
654 pass
655 else:
656 FS_NONASCII = character
657 break
Michael Foord2d9c2d52010-05-04 22:29:10 +0000658
Victor Stinnerd91df1a2010-08-18 10:56:19 +0000659# TESTFN_UNICODE is a non-ascii filename
660TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
Victor Stinner74a833f2010-08-18 21:06:23 +0000661if sys.platform == 'darwin':
662 # In Mac OS X's VFS API file names are, by definition, canonically
663 # decomposed Unicode, encoded using UTF-8. See QA1173:
664 # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
665 import unicodedata
666 TESTFN_UNICODE = unicodedata.normalize('NFD', TESTFN_UNICODE)
Antoine Pitrou88909542009-06-29 13:54:42 +0000667TESTFN_ENCODING = sys.getfilesystemencoding()
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000668
Victor Stinner09c449c2010-08-13 22:23:24 +0000669# TESTFN_UNENCODABLE is a filename (str type) that should *not* be able to be
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000670# encoded by the filesystem encoding (in strict mode). It can be None if we
671# cannot generate such filename.
Victor Stinnera0241c82010-08-15 19:28:21 +0000672TESTFN_UNENCODABLE = None
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000673if os.name in ('nt', 'ce'):
Victor Stinnera0241c82010-08-15 19:28:21 +0000674 # skip win32s (0) or Windows 9x/ME (1)
675 if sys.getwindowsversion().platform >= 2:
Victor Stinner8ce7df62010-09-10 11:19:59 +0000676 # Different kinds of characters from various languages to minimize the
677 # probability that the whole name is encodable to MBCS (issue #9819)
678 TESTFN_UNENCODABLE = TESTFN + "-\u5171\u0141\u2661\u0363\uDC80"
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000679 try:
Victor Stinner09c449c2010-08-13 22:23:24 +0000680 TESTFN_UNENCODABLE.encode(TESTFN_ENCODING)
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000681 except UnicodeEncodeError:
682 pass
683 else:
684 print('WARNING: The filename %r CAN be encoded by the filesystem encoding (%s). '
685 'Unicode filename tests may not be effective'
Victor Stinner09c449c2010-08-13 22:23:24 +0000686 % (TESTFN_UNENCODABLE, TESTFN_ENCODING))
687 TESTFN_UNENCODABLE = None
Victor Stinnera0241c82010-08-15 19:28:21 +0000688# Mac OS X denies unencodable filenames (invalid utf-8)
Victor Stinner03c9e1d2010-08-14 17:35:20 +0000689elif sys.platform != 'darwin':
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000690 try:
691 # ascii and utf-8 cannot encode the byte 0xff
692 b'\xff'.decode(TESTFN_ENCODING)
693 except UnicodeDecodeError:
694 # 0xff will be encoded using the surrogate character u+DCFF
Victor Stinner09c449c2010-08-13 22:23:24 +0000695 TESTFN_UNENCODABLE = TESTFN \
Victor Stinner3d85a6f2010-08-13 13:02:04 +0000696 + b'-\xff'.decode(TESTFN_ENCODING, 'surrogateescape')
697 else:
698 # File system encoding (eg. ISO-8859-* encodings) can encode
699 # the byte 0xff. Skip some unicode filename tests.
Victor Stinnera0241c82010-08-15 19:28:21 +0000700 pass
Neal Norwitz26a1eef2002-11-03 00:35:53 +0000701
Victor Stinnere667e982012-11-12 01:23:15 +0100702# TESTFN_UNDECODABLE is a filename (bytes type) that should *not* be able to be
703# decoded from the filesystem encoding (in strict mode). It can be None if we
704# cannot generate such filename (ex: the latin1 encoding can decode any byte
705# sequence). On UNIX, TESTFN_UNDECODABLE can be decoded by os.fsdecode() thanks
706# to the surrogateescape error handler (PEP 383), but not from the filesystem
707# encoding in strict mode.
708TESTFN_UNDECODABLE = None
709for name in (
710 # b'\xff' is not decodable by os.fsdecode() with code page 932. Windows
711 # accepts it to create a file or a directory, or don't accept to enter to
712 # such directory (when the bytes name is used). So test b'\xe7' first: it is
713 # not decodable from cp932.
714 b'\xe7w\xf0',
715 # undecodable from ASCII, UTF-8
716 b'\xff',
717 # undecodable from iso8859-3, iso8859-6, iso8859-7, cp424, iso8859-8, cp856
718 # and cp857
719 b'\xae\xd5'
720 # undecodable from UTF-8 (UNIX and Mac OS X)
721 b'\xed\xb2\x80', b'\xed\xb4\x80',
Victor Stinner0af71aa2013-01-03 01:50:30 +0100722 # undecodable from shift_jis, cp869, cp874, cp932, cp1250, cp1251, cp1252,
723 # cp1253, cp1254, cp1255, cp1257, cp1258
724 b'\x81\x98',
Victor Stinnere667e982012-11-12 01:23:15 +0100725):
726 try:
727 name.decode(TESTFN_ENCODING)
728 except UnicodeDecodeError:
729 TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
730 break
731
732if FS_NONASCII:
733 TESTFN_NONASCII = TESTFN + '-' + FS_NONASCII
734else:
735 TESTFN_NONASCII = None
736
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000737# Save the initial cwd
738SAVEDCWD = os.getcwd()
739
740@contextlib.contextmanager
Nick Coghland26c18a2010-08-17 13:06:11 +0000741def temp_cwd(name='tempcwd', quiet=False, path=None):
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000742 """
Nick Coghland26c18a2010-08-17 13:06:11 +0000743 Context manager that temporarily changes the CWD.
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000744
Nick Coghland26c18a2010-08-17 13:06:11 +0000745 An existing path may be provided as *path*, in which case this
746 function makes no changes to the file system.
747
748 Otherwise, the new CWD is created in the current directory and it's
749 named *name*. If *quiet* is False (default) and it's not possible to
750 create or change the CWD, an error is raised. If it's True, only a
751 warning is raised and the original CWD is used.
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000752 """
753 saved_dir = os.getcwd()
754 is_temporary = False
Nick Coghland26c18a2010-08-17 13:06:11 +0000755 if path is None:
756 path = name
757 try:
758 os.mkdir(name)
759 is_temporary = True
760 except OSError:
761 if not quiet:
762 raise
763 warnings.warn('tests may fail, unable to create temp CWD ' + name,
764 RuntimeWarning, stacklevel=3)
Neal Norwitz26a1eef2002-11-03 00:35:53 +0000765 try:
Nick Coghland26c18a2010-08-17 13:06:11 +0000766 os.chdir(path)
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000767 except OSError:
768 if not quiet:
769 raise
Ezio Melottie418d762012-09-21 16:48:12 +0300770 warnings.warn('tests may fail, unable to change the CWD to ' + path,
Ezio Melotti184bdfb2010-02-18 09:37:05 +0000771 RuntimeWarning, stacklevel=3)
772 try:
773 yield os.getcwd()
774 finally:
775 os.chdir(saved_dir)
776 if is_temporary:
777 rmtree(name)
778
Guido van Rossuma8f7e592001-03-13 09:31:07 +0000779
Eli Bendersky6c519992011-07-23 08:48:53 +0300780if hasattr(os, "umask"):
781 @contextlib.contextmanager
782 def temp_umask(umask):
783 """Context manager that temporarily sets the process umask."""
784 oldmask = os.umask(umask)
785 try:
786 yield
787 finally:
788 os.umask(oldmask)
Barry Warsaw28a691b2010-04-17 00:19:56 +0000789
790
Florent Xiclunaf15351d2010-03-13 23:24:31 +0000791def findfile(file, here=__file__, subdir=None):
Brett Cannonf1cfb622003-05-04 21:15:27 +0000792 """Try to find a file on sys.path and the working directory. If it is not
793 found the argument passed to the function is returned (this does not
794 necessarily signal failure; could still be the legitimate path)."""
Fred Drake004d5e62000-10-23 17:22:08 +0000795 if os.path.isabs(file):
796 return file
Florent Xiclunaf15351d2010-03-13 23:24:31 +0000797 if subdir is not None:
798 file = os.path.join(subdir, file)
Fred Drake004d5e62000-10-23 17:22:08 +0000799 path = sys.path
800 path = [os.path.dirname(here)] + path
801 for dn in path:
802 fn = os.path.join(dn, file)
803 if os.path.exists(fn): return fn
804 return file
Marc-André Lemburg36619082001-01-17 19:11:13 +0000805
Victor Stinnerbf816222011-06-30 23:25:47 +0200806def create_empty_file(filename):
807 """Create an empty file. If the file already exists, truncate it."""
808 fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
809 os.close(fd)
810
Tim Peters2f228e72001-05-13 00:19:31 +0000811def sortdict(dict):
812 "Like repr(dict), but in sorted order."
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000813 items = sorted(dict.items())
Tim Peters2f228e72001-05-13 00:19:31 +0000814 reprpairs = ["%r: %r" % pair for pair in items]
815 withcommas = ", ".join(reprpairs)
816 return "{%s}" % withcommas
817
Benjamin Peterson7522c742009-01-19 21:00:09 +0000818def make_bad_fd():
819 """
820 Create an invalid file descriptor by opening and closing a file and return
821 its fd.
822 """
823 file = open(TESTFN, "wb")
824 try:
825 return file.fileno()
826 finally:
827 file.close()
828 unlink(TESTFN)
829
Thomas Wouters89f507f2006-12-13 04:49:30 +0000830def check_syntax_error(testcase, statement):
Benjamin Petersone549ead2009-03-28 21:42:05 +0000831 testcase.assertRaises(SyntaxError, compile, statement,
832 '<test string>', 'exec')
Fred Drakecd1b1dd2001-03-21 18:26:33 +0000833
Martin v. Löwis234a34a2007-08-30 20:58:02 +0000834def open_urlresource(url, *args, **kw):
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000835 import urllib.request, urllib.parse
Fred Drakecd1b1dd2001-03-21 18:26:33 +0000836
Florent Xiclunaf089fd62010-03-19 14:25:03 +0000837 check = kw.pop('check', None)
838
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000839 filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL!
Hye-Shik Changaaa2f1d2005-12-10 17:44:27 +0000840
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +0000841 fn = os.path.join(os.path.dirname(__file__), "data", filename)
Florent Xiclunaf089fd62010-03-19 14:25:03 +0000842
843 def check_valid_file(fn):
844 f = open(fn, *args, **kw)
845 if check is None:
846 return f
847 elif check(f):
848 f.seek(0)
849 return f
850 f.close()
851
Alexandre Vassalotti711ed4a2009-07-17 10:42:05 +0000852 if os.path.exists(fn):
Florent Xiclunaf089fd62010-03-19 14:25:03 +0000853 f = check_valid_file(fn)
854 if f is not None:
855 return f
856 unlink(fn)
857
858 # Verify the requirement before downloading the file
859 requires('urlfetch')
Hye-Shik Changaaa2f1d2005-12-10 17:44:27 +0000860
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000861 print('\tfetching %s ...' % url, file=get_original_stdout())
Antoine Pitroufd0680b2009-11-01 22:13:48 +0000862 f = urllib.request.urlopen(url, timeout=15)
863 try:
864 with open(fn, "wb") as out:
865 s = f.read()
866 while s:
867 out.write(s)
868 s = f.read()
869 finally:
870 f.close()
Florent Xiclunaf089fd62010-03-19 14:25:03 +0000871
872 f = check_valid_file(fn)
873 if f is not None:
874 return f
Éric Araujoaf5bacf2011-07-15 17:50:15 +0200875 raise TestFailed('invalid resource %r' % fn)
Florent Xiclunaf089fd62010-03-19 14:25:03 +0000876
Thomas Wouters9fe394c2007-02-05 01:24:16 +0000877
Benjamin Petersonfcf5d632008-10-16 23:24:44 +0000878class WarningsRecorder(object):
879 """Convenience wrapper for the warnings list returned on
880 entry to the warnings.catch_warnings() context manager.
Guido van Rossumd8faa362007-04-27 19:54:29 +0000881 """
Benjamin Petersonfcf5d632008-10-16 23:24:44 +0000882 def __init__(self, warnings_list):
Florent Xiclunab14930c2010-03-13 15:26:44 +0000883 self._warnings = warnings_list
884 self._last = 0
Benjamin Petersonfcf5d632008-10-16 23:24:44 +0000885
886 def __getattr__(self, attr):
Florent Xiclunab14930c2010-03-13 15:26:44 +0000887 if len(self._warnings) > self._last:
888 return getattr(self._warnings[-1], attr)
Benjamin Petersonfcf5d632008-10-16 23:24:44 +0000889 elif attr in warnings.WarningMessage._WARNING_DETAILS:
890 return None
891 raise AttributeError("%r has no attribute %r" % (self, attr))
892
Florent Xiclunab14930c2010-03-13 15:26:44 +0000893 @property
894 def warnings(self):
895 return self._warnings[self._last:]
896
Benjamin Petersonfcf5d632008-10-16 23:24:44 +0000897 def reset(self):
Florent Xiclunab14930c2010-03-13 15:26:44 +0000898 self._last = len(self._warnings)
899
900
901def _filterwarnings(filters, quiet=False):
902 """Catch the warnings, then check if all the expected
903 warnings have been raised and re-raise unexpected warnings.
904 If 'quiet' is True, only re-raise the unexpected warnings.
905 """
906 # Clear the warning registry of the calling module
907 # in order to re-raise the warnings.
908 frame = sys._getframe(2)
909 registry = frame.f_globals.get('__warningregistry__')
910 if registry:
911 registry.clear()
912 with warnings.catch_warnings(record=True) as w:
913 # Set filter "always" to record all warnings. Because
914 # test_warnings swap the module, we need to look up in
915 # the sys.modules dictionary.
916 sys.modules['warnings'].simplefilter("always")
917 yield WarningsRecorder(w)
918 # Filter the recorded warnings
Antoine Pitrou31e08a42010-10-29 11:54:03 +0000919 reraise = list(w)
Florent Xiclunab14930c2010-03-13 15:26:44 +0000920 missing = []
921 for msg, cat in filters:
922 seen = False
Antoine Pitrou31e08a42010-10-29 11:54:03 +0000923 for w in reraise[:]:
924 warning = w.message
Florent Xiclunab14930c2010-03-13 15:26:44 +0000925 # Filter out the matching messages
Antoine Pitrou31e08a42010-10-29 11:54:03 +0000926 if (re.match(msg, str(warning), re.I) and
927 issubclass(warning.__class__, cat)):
Florent Xiclunab14930c2010-03-13 15:26:44 +0000928 seen = True
Antoine Pitrou31e08a42010-10-29 11:54:03 +0000929 reraise.remove(w)
Florent Xiclunab14930c2010-03-13 15:26:44 +0000930 if not seen and not quiet:
931 # This filter caught nothing
932 missing.append((msg, cat.__name__))
933 if reraise:
Antoine Pitrou31e08a42010-10-29 11:54:03 +0000934 raise AssertionError("unhandled warning %s" % reraise[0])
Florent Xiclunab14930c2010-03-13 15:26:44 +0000935 if missing:
936 raise AssertionError("filter (%r, %s) did not catch any warning" %
937 missing[0])
938
Benjamin Petersonfcf5d632008-10-16 23:24:44 +0000939
940@contextlib.contextmanager
Florent Xiclunab14930c2010-03-13 15:26:44 +0000941def check_warnings(*filters, **kwargs):
942 """Context manager to silence warnings.
943
944 Accept 2-tuples as positional arguments:
945 ("message regexp", WarningCategory)
946
947 Optional argument:
948 - if 'quiet' is True, it does not fail if a filter catches nothing
Florent Xicluna53b506be2010-03-18 20:00:57 +0000949 (default True without argument,
950 default False if some filters are defined)
Florent Xiclunab14930c2010-03-13 15:26:44 +0000951
952 Without argument, it defaults to:
Florent Xicluna53b506be2010-03-18 20:00:57 +0000953 check_warnings(("", Warning), quiet=True)
Florent Xiclunab14930c2010-03-13 15:26:44 +0000954 """
Florent Xicluna53b506be2010-03-18 20:00:57 +0000955 quiet = kwargs.get('quiet')
Florent Xiclunab14930c2010-03-13 15:26:44 +0000956 if not filters:
957 filters = (("", Warning),)
Florent Xicluna53b506be2010-03-18 20:00:57 +0000958 # Preserve backward compatibility
959 if quiet is None:
960 quiet = True
961 return _filterwarnings(filters, quiet)
Guido van Rossumd8faa362007-04-27 19:54:29 +0000962
Alexandre Vassalotti5f8ced22008-05-16 00:03:33 +0000963
964class CleanImport(object):
965 """Context manager to force import to return a new module reference.
966
967 This is useful for testing module-level behaviours, such as
Nick Coghlanb1304932008-07-13 12:25:08 +0000968 the emission of a DeprecationWarning on import.
Alexandre Vassalotti5f8ced22008-05-16 00:03:33 +0000969
970 Use like this:
971
972 with CleanImport("foo"):
Brett Cannonddb5e702010-02-03 22:16:11 +0000973 importlib.import_module("foo") # new reference
Alexandre Vassalotti5f8ced22008-05-16 00:03:33 +0000974 """
975
976 def __init__(self, *module_names):
977 self.original_modules = sys.modules.copy()
978 for module_name in module_names:
979 if module_name in sys.modules:
980 module = sys.modules[module_name]
981 # It is possible that module_name is just an alias for
982 # another module (e.g. stub for modules renamed in 3.x).
983 # In that case, we also need delete the real module to clear
984 # the import cache.
985 if module.__name__ != module_name:
986 del sys.modules[module.__name__]
987 del sys.modules[module_name]
988
989 def __enter__(self):
990 return self
991
992 def __exit__(self, *ignore_exc):
993 sys.modules.update(self.original_modules)
994
995
Raymond Hettinger57d1a882011-02-23 00:46:28 +0000996class EnvironmentVarGuard(collections.abc.MutableMapping):
Thomas Wouters902d6eb2007-01-09 23:18:33 +0000997
998 """Class to help protect the environment variable properly. Can be used as
999 a context manager."""
1000
1001 def __init__(self):
Walter Dörwald155374d2009-05-01 19:58:58 +00001002 self._environ = os.environ
Walter Dörwald4ba80132009-04-25 12:48:43 +00001003 self._changed = {}
Thomas Wouters902d6eb2007-01-09 23:18:33 +00001004
Walter Dörwald155374d2009-05-01 19:58:58 +00001005 def __getitem__(self, envvar):
1006 return self._environ[envvar]
1007
1008 def __setitem__(self, envvar, value):
Walter Dörwald4ba80132009-04-25 12:48:43 +00001009 # Remember the initial value on the first access
1010 if envvar not in self._changed:
Walter Dörwald155374d2009-05-01 19:58:58 +00001011 self._changed[envvar] = self._environ.get(envvar)
1012 self._environ[envvar] = value
1013
1014 def __delitem__(self, envvar):
1015 # Remember the initial value on the first access
1016 if envvar not in self._changed:
1017 self._changed[envvar] = self._environ.get(envvar)
1018 if envvar in self._environ:
1019 del self._environ[envvar]
1020
1021 def keys(self):
1022 return self._environ.keys()
1023
1024 def __iter__(self):
1025 return iter(self._environ)
1026
1027 def __len__(self):
1028 return len(self._environ)
1029
1030 def set(self, envvar, value):
1031 self[envvar] = value
Thomas Wouters902d6eb2007-01-09 23:18:33 +00001032
1033 def unset(self, envvar):
Walter Dörwald155374d2009-05-01 19:58:58 +00001034 del self[envvar]
Thomas Wouters902d6eb2007-01-09 23:18:33 +00001035
1036 def __enter__(self):
1037 return self
1038
1039 def __exit__(self, *ignore_exc):
Walter Dörwald4ba80132009-04-25 12:48:43 +00001040 for (k, v) in self._changed.items():
1041 if v is None:
Walter Dörwald155374d2009-05-01 19:58:58 +00001042 if k in self._environ:
1043 del self._environ[k]
Walter Dörwald4ba80132009-04-25 12:48:43 +00001044 else:
Walter Dörwald155374d2009-05-01 19:58:58 +00001045 self._environ[k] = v
Nick Coghlan6ead5522009-10-18 13:19:33 +00001046 os.environ = self._environ
1047
1048
1049class DirsOnSysPath(object):
1050 """Context manager to temporarily add directories to sys.path.
1051
1052 This makes a copy of sys.path, appends any directories given
1053 as positional arguments, then reverts sys.path to the copied
1054 settings when the context ends.
1055
1056 Note that *all* sys.path modifications in the body of the
1057 context manager, including replacement of the object,
1058 will be reverted at the end of the block.
1059 """
1060
1061 def __init__(self, *paths):
1062 self.original_value = sys.path[:]
1063 self.original_object = sys.path
1064 sys.path.extend(paths)
1065
1066 def __enter__(self):
1067 return self
1068
1069 def __exit__(self, *ignore_exc):
1070 sys.path = self.original_object
1071 sys.path[:] = self.original_value
Walter Dörwald155374d2009-05-01 19:58:58 +00001072
Thomas Wouters902d6eb2007-01-09 23:18:33 +00001073
Guido van Rossumd8faa362007-04-27 19:54:29 +00001074class TransientResource(object):
1075
1076 """Raise ResourceDenied if an exception is raised while the context manager
1077 is in effect that matches the specified exception and attributes."""
1078
1079 def __init__(self, exc, **kwargs):
1080 self.exc = exc
1081 self.attrs = kwargs
1082
1083 def __enter__(self):
1084 return self
1085
1086 def __exit__(self, type_=None, value=None, traceback=None):
1087 """If type_ is a subclass of self.exc and value has attributes matching
1088 self.attrs, raise ResourceDenied. Otherwise let the exception
1089 propagate (if any)."""
1090 if type_ is not None and issubclass(self.exc, type_):
1091 for attr, attr_value in self.attrs.items():
1092 if not hasattr(value, attr):
1093 break
1094 if getattr(value, attr) != attr_value:
1095 break
1096 else:
1097 raise ResourceDenied("an optional resource is not available")
1098
Raymond Hettinger686057b2009-06-04 00:11:54 +00001099# Context managers that raise ResourceDenied when various issues
1100# with the Internet connection manifest themselves as exceptions.
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001101# XXX deprecate these and use transient_internet() instead
Raymond Hettinger686057b2009-06-04 00:11:54 +00001102time_out = TransientResource(IOError, errno=errno.ETIMEDOUT)
1103socket_peer_reset = TransientResource(socket.error, errno=errno.ECONNRESET)
1104ioerror_peer_reset = TransientResource(IOError, errno=errno.ECONNRESET)
Guido van Rossumd8faa362007-04-27 19:54:29 +00001105
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001106
Thomas Woutersed03b412007-08-28 21:37:11 +00001107@contextlib.contextmanager
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001108def transient_internet(resource_name, *, timeout=30.0, errnos=()):
Antoine Pitroufec12ff2010-04-21 19:46:23 +00001109 """Return a context manager that raises ResourceDenied when various issues
1110 with the Internet connection manifest themselves as exceptions."""
Antoine Pitrou8bc09032010-09-07 21:09:09 +00001111 default_errnos = [
1112 ('ECONNREFUSED', 111),
1113 ('ECONNRESET', 104),
Antoine Pitrou5d938cb2011-01-08 10:28:11 +00001114 ('EHOSTUNREACH', 113),
Antoine Pitrou8bc09032010-09-07 21:09:09 +00001115 ('ENETUNREACH', 101),
1116 ('ETIMEDOUT', 110),
1117 ]
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001118 default_gai_errnos = [
Antoine Pitrou4875c462011-07-09 02:31:24 +02001119 ('EAI_AGAIN', -3),
Charles-François Natali13859bf2011-12-10 13:16:44 +01001120 ('EAI_FAIL', -4),
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001121 ('EAI_NONAME', -2),
1122 ('EAI_NODATA', -5),
Antoine Pitrou390ea0f2011-04-29 00:44:33 +02001123 # Encountered when trying to resolve IPv6-only hostnames
1124 ('WSANO_DATA', 11004),
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001125 ]
Antoine Pitrou8bc09032010-09-07 21:09:09 +00001126
Éric Araujoaf5bacf2011-07-15 17:50:15 +02001127 denied = ResourceDenied("Resource %r is not available" % resource_name)
Antoine Pitrou8bc09032010-09-07 21:09:09 +00001128 captured_errnos = errnos
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001129 gai_errnos = []
Antoine Pitrou8bc09032010-09-07 21:09:09 +00001130 if not captured_errnos:
1131 captured_errnos = [getattr(errno, name, num)
1132 for (name, num) in default_errnos]
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001133 gai_errnos = [getattr(socket, name, num)
1134 for (name, num) in default_gai_errnos]
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001135
1136 def filter_error(err):
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001137 n = getattr(err, 'errno', None)
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001138 if (isinstance(err, socket.timeout) or
Antoine Pitrou2673c5b2010-09-07 21:43:31 +00001139 (isinstance(err, socket.gaierror) and n in gai_errnos) or
1140 n in captured_errnos):
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001141 if not verbose:
1142 sys.stderr.write(denied.args[0] + "\n")
1143 raise denied from err
1144
1145 old_timeout = socket.getdefaulttimeout()
1146 try:
1147 if timeout is not None:
1148 socket.setdefaulttimeout(timeout)
Antoine Pitroufec12ff2010-04-21 19:46:23 +00001149 yield
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001150 except IOError as err:
Antoine Pitrou8bc09032010-09-07 21:09:09 +00001151 # urllib can wrap original socket errors multiple times (!), we must
1152 # unwrap to get at the original error.
1153 while True:
1154 a = err.args
1155 if len(a) >= 1 and isinstance(a[0], IOError):
1156 err = a[0]
1157 # The error can also be wrapped as args[1]:
1158 # except socket.error as msg:
1159 # raise IOError('socket error', msg).with_traceback(sys.exc_info()[2])
1160 elif len(a) >= 2 and isinstance(a[1], IOError):
1161 err = a[1]
1162 else:
1163 break
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001164 filter_error(err)
Antoine Pitroua88c83c2010-09-07 20:42:19 +00001165 raise
1166 # XXX should we catch generic exceptions and look for their
1167 # __cause__ or __context__?
1168 finally:
1169 socket.setdefaulttimeout(old_timeout)
Antoine Pitroufec12ff2010-04-21 19:46:23 +00001170
1171
1172@contextlib.contextmanager
Benjamin Petersonad9d48d2008-04-02 21:49:44 +00001173def captured_output(stream_name):
Ezio Melotti07352b02011-05-14 14:51:18 +03001174 """Return a context manager used by captured_stdout/stdin/stderr
Ezio Melottifc778fd2011-05-14 08:22:47 +03001175 that temporarily replaces the sys stream *stream_name* with a StringIO."""
Thomas Woutersed03b412007-08-28 21:37:11 +00001176 import io
Benjamin Petersonad9d48d2008-04-02 21:49:44 +00001177 orig_stdout = getattr(sys, stream_name)
1178 setattr(sys, stream_name, io.StringIO())
Christian Heimes81ee3ef2008-05-04 22:42:01 +00001179 try:
1180 yield getattr(sys, stream_name)
1181 finally:
1182 setattr(sys, stream_name, orig_stdout)
Benjamin Petersonad9d48d2008-04-02 21:49:44 +00001183
1184def captured_stdout():
Ezio Melottifc778fd2011-05-14 08:22:47 +03001185 """Capture the output of sys.stdout:
1186
1187 with captured_stdout() as s:
1188 print("hello")
1189 self.assertEqual(s.getvalue(), "hello")
1190 """
Benjamin Petersonad9d48d2008-04-02 21:49:44 +00001191 return captured_output("stdout")
Thomas Woutersed03b412007-08-28 21:37:11 +00001192
Nick Coghlan6b22f3f2010-12-12 15:24:21 +00001193def captured_stderr():
1194 return captured_output("stderr")
1195
Nick Coghlan6ead5522009-10-18 13:19:33 +00001196def captured_stdin():
1197 return captured_output("stdin")
1198
Ezio Melotti07352b02011-05-14 14:51:18 +03001199
Benjamin Petersone549ead2009-03-28 21:42:05 +00001200def gc_collect():
1201 """Force as many objects as possible to be collected.
1202
1203 In non-CPython implementations of Python, this is needed because timely
1204 deallocation is not guaranteed by the garbage collector. (Even in CPython
1205 this can be the case in case of reference cycles.) This means that __del__
1206 methods may be called later than expected and weakrefs may remain alive for
1207 longer than expected. This function tries its best to force all garbage
1208 objects to disappear.
1209 """
Benjamin Petersone549ead2009-03-28 21:42:05 +00001210 gc.collect()
Benjamin Petersona6590e82010-04-11 21:22:10 +00001211 if is_jython:
1212 time.sleep(0.1)
Benjamin Petersone549ead2009-03-28 21:42:05 +00001213 gc.collect()
1214 gc.collect()
1215
Benjamin Peterson31f4beb2011-07-14 12:48:01 -05001216@contextlib.contextmanager
1217def disable_gc():
1218 have_gc = gc.isenabled()
1219 gc.disable()
1220 try:
1221 yield
1222 finally:
1223 if have_gc:
1224 gc.enable()
1225
Thomas Woutersed03b412007-08-28 21:37:11 +00001226
Benjamin Peterson65c66ab2010-10-29 21:31:35 +00001227def python_is_optimized():
1228 """Find if Python was built with optimizations."""
Antoine Pitrou64474542010-10-31 11:34:47 +00001229 cflags = sysconfig.get_config_var('PY_CFLAGS') or ''
Benjamin Peterson65c66ab2010-10-29 21:31:35 +00001230 final_opt = ""
1231 for opt in cflags.split():
1232 if opt.startswith('-O'):
1233 final_opt = opt
Eli Bendersky6c519992011-07-23 08:48:53 +03001234 return final_opt != '' and final_opt != '-O0'
Benjamin Peterson65c66ab2010-10-29 21:31:35 +00001235
1236
Martin v. Löwis2b168442012-07-29 16:38:45 +02001237_header = 'nP'
1238_align = '0n'
Martin v. Löwis33f79972012-07-29 16:33:05 +02001239if hasattr(sys, "gettotalrefcount"):
1240 _header = '2P' + _header
Martin v. Löwis2b168442012-07-29 16:38:45 +02001241 _align = '0P'
1242_vheader = _header + 'n'
Martin v. Löwis33f79972012-07-29 16:33:05 +02001243
1244def calcobjsize(fmt):
Martin v. Löwis2b168442012-07-29 16:38:45 +02001245 return struct.calcsize(_header + fmt + _align)
Martin v. Löwis33f79972012-07-29 16:33:05 +02001246
1247def calcvobjsize(fmt):
Martin v. Löwis2b168442012-07-29 16:38:45 +02001248 return struct.calcsize(_vheader + fmt + _align)
Martin v. Löwis33f79972012-07-29 16:33:05 +02001249
1250
1251_TPFLAGS_HAVE_GC = 1<<14
1252_TPFLAGS_HEAPTYPE = 1<<9
1253
1254def check_sizeof(test, o, size):
1255 result = sys.getsizeof(o)
1256 # add GC header size
1257 if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\
1258 ((type(o) != type) and (type(o).__flags__ & _TPFLAGS_HAVE_GC))):
1259 size += _testcapi.SIZEOF_PYGC_HEAD
1260 msg = 'wrong size for %s: got %d, expected %d' \
1261 % (type(o), result, size)
1262 test.assertEqual(result, size, msg)
1263
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001264#=======================================================================
Thomas Wouters477c8d52006-05-27 19:21:47 +00001265# Decorator for running a function in a different locale, correctly resetting
1266# it afterwards.
1267
1268def run_with_locale(catstr, *locales):
1269 def decorator(func):
1270 def inner(*args, **kwds):
1271 try:
1272 import locale
1273 category = getattr(locale, catstr)
1274 orig_locale = locale.setlocale(category)
1275 except AttributeError:
1276 # if the test author gives us an invalid category string
1277 raise
1278 except:
1279 # cannot retrieve original locale, so do nothing
1280 locale = orig_locale = None
1281 else:
1282 for loc in locales:
1283 try:
1284 locale.setlocale(category, loc)
1285 break
1286 except:
1287 pass
1288
1289 # now run the function, resetting the locale on exceptions
1290 try:
1291 return func(*args, **kwds)
1292 finally:
1293 if locale and orig_locale:
1294 locale.setlocale(category, orig_locale)
Neal Norwitz221085d2007-02-25 20:55:47 +00001295 inner.__name__ = func.__name__
Thomas Wouters477c8d52006-05-27 19:21:47 +00001296 inner.__doc__ = func.__doc__
1297 return inner
1298 return decorator
1299
1300#=======================================================================
Alexander Belopolsky2420d832012-04-29 15:56:49 -04001301# Decorator for running a function in a specific timezone, correctly
1302# resetting it afterwards.
1303
1304def run_with_tz(tz):
1305 def decorator(func):
1306 def inner(*args, **kwds):
Alexander Belopolskyb8f02b52012-04-29 18:16:46 -04001307 try:
1308 tzset = time.tzset
1309 except AttributeError:
1310 raise unittest.SkipTest("tzset required")
Alexander Belopolsky2420d832012-04-29 15:56:49 -04001311 if 'TZ' in os.environ:
1312 orig_tz = os.environ['TZ']
1313 else:
1314 orig_tz = None
1315 os.environ['TZ'] = tz
Alexander Belopolskyb8f02b52012-04-29 18:16:46 -04001316 tzset()
Alexander Belopolsky2420d832012-04-29 15:56:49 -04001317
1318 # now run the function, resetting the tz on exceptions
1319 try:
1320 return func(*args, **kwds)
1321 finally:
Benjamin Petersonb29614e2012-10-09 11:16:03 -04001322 if orig_tz is None:
Alexander Belopolsky2420d832012-04-29 15:56:49 -04001323 del os.environ['TZ']
1324 else:
1325 os.environ['TZ'] = orig_tz
1326 time.tzset()
1327
1328 inner.__name__ = func.__name__
1329 inner.__doc__ = func.__doc__
1330 return inner
1331 return decorator
1332
1333#=======================================================================
Georg Brandldb028442008-02-05 20:48:58 +00001334# Big-memory-test support. Separate from 'resources' because memory use
1335# should be configurable.
Thomas Wouters477c8d52006-05-27 19:21:47 +00001336
1337# Some handy shorthands. Note that these are used for byte-limits as well
1338# as size-limits, in the various bigmem tests
1339_1M = 1024*1024
1340_1G = 1024 * _1M
1341_2G = 2 * _1G
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001342_4G = 4 * _1G
Thomas Wouters477c8d52006-05-27 19:21:47 +00001343
Thomas Woutersd2cf20e2007-08-30 22:57:53 +00001344MAX_Py_ssize_t = sys.maxsize
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001345
Thomas Wouters477c8d52006-05-27 19:21:47 +00001346def set_memlimit(limit):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001347 global max_memuse
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001348 global real_max_memuse
Thomas Wouters477c8d52006-05-27 19:21:47 +00001349 sizes = {
1350 'k': 1024,
1351 'm': _1M,
1352 'g': _1G,
1353 't': 1024*_1G,
1354 }
1355 m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit,
1356 re.IGNORECASE | re.VERBOSE)
1357 if m is None:
1358 raise ValueError('Invalid memory limit %r' % (limit,))
1359 memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001360 real_max_memuse = memlimit
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001361 if memlimit > MAX_Py_ssize_t:
1362 memlimit = MAX_Py_ssize_t
1363 if memlimit < _2G - 1:
Thomas Wouters477c8d52006-05-27 19:21:47 +00001364 raise ValueError('Memory limit %r too low to be useful' % (limit,))
1365 max_memuse = memlimit
1366
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001367class _MemoryWatchdog:
1368 """An object which periodically watches the process' memory consumption
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001369 and prints it out.
1370 """
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001371
1372 def __init__(self):
1373 self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
1374 self.started = False
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001375
1376 def start(self):
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001377 try:
Charles-François Natali55bce632012-03-24 10:06:23 +01001378 f = open(self.procfile, 'r')
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001379 except OSError as e:
1380 warnings.warn('/proc not available for stats: {}'.format(e),
1381 RuntimeWarning)
1382 sys.stderr.flush()
1383 return
Charles-François Natali55bce632012-03-24 10:06:23 +01001384
1385 watchdog_script = findfile("memory_watchdog.py")
1386 self.mem_watchdog = subprocess.Popen([sys.executable, watchdog_script],
1387 stdin=f, stderr=subprocess.DEVNULL)
1388 f.close()
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001389 self.started = True
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001390
1391 def stop(self):
Charles-François Natali55bce632012-03-24 10:06:23 +01001392 if self.started:
1393 self.mem_watchdog.terminate()
1394 self.mem_watchdog.wait()
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001395
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001396
1397def bigmemtest(size, memuse, dry_run=True):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001398 """Decorator for bigmem tests.
1399
1400 'minsize' is the minimum useful size for the test (in arbitrary,
1401 test-interpreted units.) 'memuse' is the number of 'bytes per size' for
Antoine Pitrouaca5fa72011-01-12 21:19:59 +00001402 the test, or a good estimate of it.
Thomas Wouters477c8d52006-05-27 19:21:47 +00001403
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001404 if 'dry_run' is False, it means the test doesn't support dummy runs
1405 when -M is not specified.
Thomas Wouters518b5ae2011-03-25 11:42:37 +01001406 """
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001407 def decorator(f):
1408 def wrapper(self):
Antoine Pitrou7cdb4952009-03-07 23:40:49 +00001409 size = wrapper.size
1410 memuse = wrapper.memuse
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001411 if not real_max_memuse:
1412 maxsize = 5147
1413 else:
1414 maxsize = size
1415
Antoine Pitrou82be19f2011-08-29 23:09:33 +02001416 if ((real_max_memuse or not dry_run)
1417 and real_max_memuse < maxsize * memuse):
1418 raise unittest.SkipTest(
1419 "not enough memory: %.1fG minimum needed"
1420 % (size * memuse / (1024 ** 3)))
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001421
Charles-François Natali55bce632012-03-24 10:06:23 +01001422 if real_max_memuse and verbose:
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001423 print()
1424 print(" ... expected peak memory use: {peak:.1f}G"
1425 .format(peak=size * memuse / (1024 ** 3)))
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001426 watchdog = _MemoryWatchdog()
1427 watchdog.start()
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001428 else:
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001429 watchdog = None
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001430
1431 try:
1432 return f(self, maxsize)
1433 finally:
Antoine Pitrou75e78b62011-10-04 11:51:23 +02001434 if watchdog:
1435 watchdog.stop()
Antoine Pitrou94190bb2011-10-04 10:22:36 +02001436
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001437 wrapper.size = size
1438 wrapper.memuse = memuse
Neal Norwitz3ce5d922008-08-24 07:08:55 +00001439 return wrapper
1440 return decorator
1441
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001442def bigaddrspacetest(f):
1443 """Decorator for tests that fill the address space."""
1444 def wrapper(self):
1445 if max_memuse < MAX_Py_ssize_t:
Antoine Pitrou98c62bd2011-01-12 21:58:39 +00001446 if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31:
Antoine Pitroue0d3f8a2011-01-12 21:50:44 +00001447 raise unittest.SkipTest(
1448 "not enough memory: try a 32-bit build instead")
1449 else:
1450 raise unittest.SkipTest(
1451 "not enough memory: %.1fG minimum needed"
1452 % (MAX_Py_ssize_t / (1024 ** 3)))
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001453 else:
1454 return f(self)
1455 return wrapper
1456
Thomas Wouters477c8d52006-05-27 19:21:47 +00001457#=======================================================================
Guido van Rossumd8faa362007-04-27 19:54:29 +00001458# unittest integration.
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001459
Steve Purcell5ddd1a82001-03-22 08:45:36 +00001460class BasicTestRunner:
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001461 def run(self, test):
Steve Purcell5ddd1a82001-03-22 08:45:36 +00001462 result = unittest.TestResult()
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001463 test(result)
1464 return result
1465
Benjamin Petersone549ead2009-03-28 21:42:05 +00001466def _id(obj):
1467 return obj
1468
1469def requires_resource(resource):
Antoine Pitroud20a5f62011-02-26 15:58:05 +00001470 if resource == 'gui' and not _is_gui_available():
1471 return unittest.skip("resource 'gui' is not available")
Antoine Pitrou5bc4fa72010-10-14 15:34:31 +00001472 if is_resource_enabled(resource):
Benjamin Petersone549ead2009-03-28 21:42:05 +00001473 return _id
1474 else:
1475 return unittest.skip("resource {0!r} is not enabled".format(resource))
1476
1477def cpython_only(test):
1478 """
1479 Decorator for tests only applicable on CPython.
1480 """
1481 return impl_detail(cpython=True)(test)
1482
1483def impl_detail(msg=None, **guards):
1484 if check_impl_detail(**guards):
1485 return _id
1486 if msg is None:
1487 guardnames, default = _parse_guards(guards)
1488 if default:
1489 msg = "implementation detail not available on {0}"
1490 else:
1491 msg = "implementation detail specific to {0}"
1492 guardnames = sorted(guardnames.keys())
1493 msg = msg.format(' or '.join(guardnames))
1494 return unittest.skip(msg)
1495
1496def _parse_guards(guards):
1497 # Returns a tuple ({platform_name: run_me}, default_value)
1498 if not guards:
1499 return ({'cpython': True}, False)
Eric Smith886b40a2009-04-26 21:26:45 +00001500 is_true = list(guards.values())[0]
1501 assert list(guards.values()) == [is_true] * len(guards) # all True or all False
Benjamin Petersone549ead2009-03-28 21:42:05 +00001502 return (guards, not is_true)
1503
1504# Use the following check to guard CPython's implementation-specific tests --
1505# or to run them only on the implementation(s) guarded by the arguments.
1506def check_impl_detail(**guards):
1507 """This function returns True or False depending on the host platform.
1508 Examples:
1509 if check_impl_detail(): # only on CPython (default)
1510 if check_impl_detail(jython=True): # only on Jython
1511 if check_impl_detail(cpython=False): # everywhere except on CPython
1512 """
1513 guards, default = _parse_guards(guards)
1514 return guards.get(platform.python_implementation().lower(), default)
1515
1516
Brett Cannon31f59292011-02-21 19:29:56 +00001517def no_tracing(func):
1518 """Decorator to temporarily turn off tracing for the duration of a test."""
1519 if not hasattr(sys, 'gettrace'):
1520 return func
1521 else:
1522 @functools.wraps(func)
1523 def wrapper(*args, **kwargs):
1524 original_trace = sys.gettrace()
1525 try:
1526 sys.settrace(None)
1527 return func(*args, **kwargs)
1528 finally:
1529 sys.settrace(original_trace)
1530 return wrapper
1531
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001532
Brett Cannon7a540732011-02-22 03:04:06 +00001533def refcount_test(test):
1534 """Decorator for tests which involve reference counting.
1535
1536 To start, the decorator does not run the test if is not run by CPython.
1537 After that, any trace function is unset during the test to prevent
1538 unexpected refcounts caused by the trace function.
1539
1540 """
1541 return no_tracing(cpython_only(test))
1542
1543
Antoine Pitroub9c73e82011-07-29 23:53:38 +02001544def _filter_suite(suite, pred):
1545 """Recursively filter test cases in a suite based on a predicate."""
1546 newtests = []
1547 for test in suite._tests:
1548 if isinstance(test, unittest.TestSuite):
1549 _filter_suite(test, pred)
1550 newtests.append(test)
1551 else:
1552 if pred(test):
1553 newtests.append(test)
1554 suite._tests = newtests
1555
Guido van Rossumd8faa362007-04-27 19:54:29 +00001556def _run_suite(suite):
Barry Warsawc88425e2001-09-20 06:31:22 +00001557 """Run tests from a unittest.TestSuite-derived class."""
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001558 if verbose:
Antoine Pitrou216a3bc2011-07-23 22:33:39 +02001559 runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
1560 failfast=failfast)
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001561 else:
Steve Purcell5ddd1a82001-03-22 08:45:36 +00001562 runner = BasicTestRunner()
Fred Drakecd1b1dd2001-03-21 18:26:33 +00001563
Steve Purcell5ddd1a82001-03-22 08:45:36 +00001564 result = runner.run(suite)
1565 if not result.wasSuccessful():
Fred Drake14f6c182001-07-16 18:51:32 +00001566 if len(result.errors) == 1 and not result.failures:
1567 err = result.errors[0][1]
1568 elif len(result.failures) == 1 and not result.errors:
1569 err = result.failures[0][1]
1570 else:
R. David Murray723357e2009-10-19 18:06:17 +00001571 err = "multiple errors occurred"
1572 if not verbose: err += "; run in verbose mode for details"
Tim Peters2d84f2c2001-09-08 03:37:56 +00001573 raise TestFailed(err)
Tim Petersa0a62222001-09-09 06:12:01 +00001574
Barry Warsawc10d6902001-09-20 06:30:41 +00001575
Walter Dörwald21d3a322003-05-01 17:45:56 +00001576def run_unittest(*classes):
1577 """Run tests from unittest.TestCase-derived classes."""
Guido van Rossumd8faa362007-04-27 19:54:29 +00001578 valid_types = (unittest.TestSuite, unittest.TestCase)
Raymond Hettinger9dcbbea2003-04-27 07:54:23 +00001579 suite = unittest.TestSuite()
Walter Dörwald21d3a322003-05-01 17:45:56 +00001580 for cls in classes:
Guido van Rossum3172c5d2007-10-16 18:12:55 +00001581 if isinstance(cls, str):
Guido van Rossumd8faa362007-04-27 19:54:29 +00001582 if cls in sys.modules:
1583 suite.addTest(unittest.findTestCases(sys.modules[cls]))
1584 else:
1585 raise ValueError("str arguments must be keys in sys.modules")
1586 elif isinstance(cls, valid_types):
Raymond Hettinger21d99872003-07-16 02:59:32 +00001587 suite.addTest(cls)
1588 else:
1589 suite.addTest(unittest.makeSuite(cls))
Antoine Pitroub9c73e82011-07-29 23:53:38 +02001590 def case_pred(test):
1591 if match_tests is None:
1592 return True
1593 for name in test.id().split("."):
1594 if fnmatch.fnmatchcase(name, match_tests):
1595 return True
1596 return False
1597 _filter_suite(suite, case_pred)
Guido van Rossumd8faa362007-04-27 19:54:29 +00001598 _run_suite(suite)
Raymond Hettinger9dcbbea2003-04-27 07:54:23 +00001599
Serhiy Storchaka9d0add02013-01-27 19:47:45 +02001600#=======================================================================
1601# Check for the presence of docstrings.
1602
1603HAVE_DOCSTRINGS = (check_impl_detail(cpython=False) or
1604 sys.platform == 'win32' or
1605 sysconfig.get_config_var('WITH_DOC_STRINGS'))
1606
1607requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS,
1608 "test requires docstrings")
1609
Barry Warsawc10d6902001-09-20 06:30:41 +00001610
Tim Petersa0a62222001-09-09 06:12:01 +00001611#=======================================================================
1612# doctest driver.
1613
Stefan Krah1919b7e2012-03-21 18:25:23 +01001614def run_doctest(module, verbosity=None, optionflags=0):
Tim Peters17111f32001-10-03 04:08:26 +00001615 """Run doctest on the given module. Return (#failures, #tests).
Tim Petersa0a62222001-09-09 06:12:01 +00001616
1617 If optional argument verbosity is not specified (or is None), pass
Benjamin Petersonee8712c2008-05-20 21:35:26 +00001618 support's belief about verbosity on to doctest. Else doctest's
Tim Petersbea3fb82001-09-10 01:39:21 +00001619 usual behavior is used (it searches sys.argv for -v).
Tim Petersa0a62222001-09-09 06:12:01 +00001620 """
1621
1622 import doctest
1623
1624 if verbosity is None:
1625 verbosity = verbose
1626 else:
1627 verbosity = None
1628
Stefan Krah1919b7e2012-03-21 18:25:23 +01001629 f, t = doctest.testmod(module, verbose=verbosity, optionflags=optionflags)
Victor Stinnerbddc4d42011-06-29 15:52:46 +02001630 if f:
1631 raise TestFailed("%d of %d doctests failed" % (f, t))
Raymond Hettinger35b34bd2003-05-17 00:58:33 +00001632 if verbose:
Georg Brandldb028442008-02-05 20:48:58 +00001633 print('doctest (%s) ... %d tests with zero failures' %
1634 (module.__name__, t))
Raymond Hettinger35b34bd2003-05-17 00:58:33 +00001635 return f, t
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001636
Antoine Pitrou060cee22009-11-13 16:29:04 +00001637
1638#=======================================================================
1639# Support for saving and restoring the imported modules.
1640
1641def modules_setup():
1642 return sys.modules.copy(),
1643
1644def modules_cleanup(oldmodules):
1645 # Encoders/decoders are registered permanently within the internal
1646 # codec cache. If we destroy the corresponding modules their
1647 # globals will be set to None which will trip up the cached functions.
1648 encodings = [(k, v) for k, v in sys.modules.items()
1649 if k.startswith('encodings.')]
1650 sys.modules.clear()
1651 sys.modules.update(encodings)
Nick Coghlan90be5fb2011-01-11 10:05:20 +00001652 # XXX: This kind of problem can affect more than just encodings. In particular
Eric Smitha3e8f3d2011-01-11 10:24:34 +00001653 # extension modules (such as _ssl) don't cope with reloading properly.
Nick Coghlan90be5fb2011-01-11 10:05:20 +00001654 # Really, test modules should be cleaning out the test specific modules they
1655 # know they added (ala test_runpy) rather than relying on this function (as
1656 # test_importhooks and test_pkg do currently).
1657 # Implicitly imported *real* modules should be left alone (see issue 10556).
Antoine Pitrou060cee22009-11-13 16:29:04 +00001658 sys.modules.update(oldmodules)
1659
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001660#=======================================================================
1661# Threading support to prevent reporting refleaks when running regrtest.py -R
1662
Antoine Pitrou65c9c642009-10-30 17:25:12 +00001663# NOTE: we use thread._count() rather than threading.enumerate() (or the
1664# moral equivalent thereof) because a threading.Thread object is still alive
1665# until its __bootstrap() method has returned, even after it has been
1666# unregistered from the threading module.
1667# thread._count(), on the other hand, only gets decremented *after* the
1668# __bootstrap() method has returned, which gives us reliable reference counts
1669# at the end of a test run.
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001670
Antoine Pitrou65c9c642009-10-30 17:25:12 +00001671def threading_setup():
Victor Stinner45df8202010-04-28 22:31:17 +00001672 if _thread:
Antoine Pitrou707f2282011-07-15 22:29:44 +02001673 return _thread._count(), threading._dangling.copy()
Victor Stinner45df8202010-04-28 22:31:17 +00001674 else:
Antoine Pitrou707f2282011-07-15 22:29:44 +02001675 return 1, ()
Antoine Pitrou65c9c642009-10-30 17:25:12 +00001676
Antoine Pitrou707f2282011-07-15 22:29:44 +02001677def threading_cleanup(*original_values):
Victor Stinner45df8202010-04-28 22:31:17 +00001678 if not _thread:
1679 return
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001680 _MAX_COUNT = 10
Antoine Pitrou65c9c642009-10-30 17:25:12 +00001681 for count in range(_MAX_COUNT):
Antoine Pitrou707f2282011-07-15 22:29:44 +02001682 values = _thread._count(), threading._dangling
1683 if values == original_values:
Antoine Pitrou65c9c642009-10-30 17:25:12 +00001684 break
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001685 time.sleep(0.1)
Antoine Pitrou707f2282011-07-15 22:29:44 +02001686 gc_collect()
Antoine Pitrou65c9c642009-10-30 17:25:12 +00001687 # XXX print a warning in case of failure?
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001688
Benjamin Petersonfa0d7032009-06-01 22:42:33 +00001689def reap_threads(func):
Victor Stinner45df8202010-04-28 22:31:17 +00001690 """Use this function when threads are being used. This will
1691 ensure that the threads are cleaned up even when the test fails.
1692 If threading is unavailable this function does nothing.
1693 """
1694 if not _thread:
1695 return func
1696
Benjamin Petersonfa0d7032009-06-01 22:42:33 +00001697 @functools.wraps(func)
1698 def decorator(*args):
1699 key = threading_setup()
1700 try:
1701 return func(*args)
1702 finally:
1703 threading_cleanup(*key)
1704 return decorator
1705
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001706def reap_children():
1707 """Use this function at the end of test_main() whenever sub-processes
1708 are started. This will help ensure that no extra children (zombies)
1709 stick around to hog resources and create problems when looking
1710 for refleaks.
1711 """
1712
1713 # Reap all our dead child processes so we don't leave zombies around.
1714 # These hog resources and might be causing some of the buildbots to die.
Thomas Wouters0e3f5912006-08-11 14:57:12 +00001715 if hasattr(os, 'waitpid'):
1716 any_process = -1
1717 while True:
1718 try:
1719 # This will raise an exception on Windows. That's ok.
1720 pid, status = os.waitpid(any_process, os.WNOHANG)
1721 if pid == 0:
1722 break
1723 except:
1724 break
Collin Winterf2bf2b32010-03-17 00:41:56 +00001725
1726@contextlib.contextmanager
1727def swap_attr(obj, attr, new_val):
1728 """Temporary swap out an attribute with a new object.
1729
1730 Usage:
1731 with swap_attr(obj, "attr", 5):
1732 ...
1733
1734 This will set obj.attr to 5 for the duration of the with: block,
1735 restoring the old value at the end of the block. If `attr` doesn't
1736 exist on `obj`, it will be created and then deleted at the end of the
1737 block.
1738 """
1739 if hasattr(obj, attr):
1740 real_val = getattr(obj, attr)
1741 setattr(obj, attr, new_val)
1742 try:
1743 yield
1744 finally:
1745 setattr(obj, attr, real_val)
1746 else:
1747 setattr(obj, attr, new_val)
1748 try:
1749 yield
1750 finally:
1751 delattr(obj, attr)
1752
1753@contextlib.contextmanager
1754def swap_item(obj, item, new_val):
1755 """Temporary swap out an item with a new object.
1756
1757 Usage:
1758 with swap_item(obj, "item", 5):
1759 ...
1760
1761 This will set obj["item"] to 5 for the duration of the with: block,
1762 restoring the old value at the end of the block. If `item` doesn't
1763 exist on `obj`, it will be created and then deleted at the end of the
1764 block.
1765 """
1766 if item in obj:
1767 real_val = obj[item]
1768 obj[item] = new_val
1769 try:
1770 yield
1771 finally:
1772 obj[item] = real_val
1773 else:
1774 obj[item] = new_val
1775 try:
1776 yield
1777 finally:
1778 del obj[item]
Antoine Pitrou62f68ed2010-08-04 11:48:56 +00001779
1780def strip_python_stderr(stderr):
1781 """Strip the stderr of a Python process from potential debug output
1782 emitted by the interpreter.
1783
1784 This will typically be run on the result of the communicate() method
1785 of a subprocess.Popen object.
1786 """
Richard Oudkerk73d9a292012-06-14 15:30:10 +01001787 stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
Antoine Pitrou62f68ed2010-08-04 11:48:56 +00001788 return stderr
Antoine Pitrou1b03f2c2010-10-14 11:12:00 +00001789
1790def args_from_interpreter_flags():
1791 """Return a list of command-line arguments reproducing the current
Brett Cannonb880c152011-03-15 16:03:09 -04001792 settings in sys.flags and sys.warnoptions."""
Antoine Pitrouebdcd852012-05-18 18:33:07 +02001793 return subprocess._args_from_interpreter_flags()
Vinay Sajip129fd042010-12-10 08:19:38 +00001794
1795#============================================================
1796# Support for assertions about logging.
1797#============================================================
1798
1799class TestHandler(logging.handlers.BufferingHandler):
1800 def __init__(self, matcher):
1801 # BufferingHandler takes a "capacity" argument
1802 # so as to know when to flush. As we're overriding
1803 # shouldFlush anyway, we can set a capacity of zero.
1804 # You can call flush() manually to clear out the
1805 # buffer.
1806 logging.handlers.BufferingHandler.__init__(self, 0)
1807 self.matcher = matcher
1808
1809 def shouldFlush(self):
1810 return False
1811
1812 def emit(self, record):
1813 self.format(record)
1814 self.buffer.append(record.__dict__)
1815
1816 def matches(self, **kwargs):
1817 """
1818 Look for a saved dict whose keys/values match the supplied arguments.
1819 """
1820 result = False
1821 for d in self.buffer:
1822 if self.matcher.matches(d, **kwargs):
1823 result = True
1824 break
1825 return result
1826
1827class Matcher(object):
1828
1829 _partial_matches = ('msg', 'message')
1830
1831 def matches(self, d, **kwargs):
1832 """
1833 Try to match a single dict with the supplied arguments.
1834
1835 Keys whose values are strings and which are in self._partial_matches
1836 will be checked for partial (i.e. substring) matches. You can extend
1837 this scheme to (for example) do regular expression matching, etc.
1838 """
1839 result = True
1840 for k in kwargs:
1841 v = kwargs[k]
1842 dv = d.get(k)
1843 if not self.match_value(k, dv, v):
1844 result = False
1845 break
1846 return result
1847
1848 def match_value(self, k, dv, v):
1849 """
1850 Try to match a single stored value (dv) with a supplied value (v).
1851 """
1852 if type(v) != type(dv):
1853 result = False
1854 elif type(dv) is not str or k not in self._partial_matches:
1855 result = (v == dv)
1856 else:
1857 result = dv.find(v) >= 0
1858 return result
Brian Curtin3b4499c2010-12-28 14:31:47 +00001859
1860
1861_can_symlink = None
1862def can_symlink():
1863 global _can_symlink
1864 if _can_symlink is not None:
1865 return _can_symlink
Brett Cannonee877a02011-03-15 17:32:14 -04001866 symlink_path = TESTFN + "can_symlink"
Brian Curtin3b4499c2010-12-28 14:31:47 +00001867 try:
Brett Cannonee877a02011-03-15 17:32:14 -04001868 os.symlink(TESTFN, symlink_path)
Brian Curtin3b4499c2010-12-28 14:31:47 +00001869 can = True
Brian Curtind25aef52011-06-13 15:16:04 -05001870 except (OSError, NotImplementedError, AttributeError):
Brian Curtin3b4499c2010-12-28 14:31:47 +00001871 can = False
Victor Stinner62ec61f2011-06-07 12:17:15 +02001872 else:
1873 os.remove(symlink_path)
Brian Curtin3b4499c2010-12-28 14:31:47 +00001874 _can_symlink = can
1875 return can
1876
1877def skip_unless_symlink(test):
1878 """Skip decorator for tests that require functional symlink"""
1879 ok = can_symlink()
1880 msg = "Requires functional symlink implementation"
1881 return test if ok else unittest.skip(msg)(test)
Antoine Pitroue8706232011-03-15 21:05:36 +01001882
Antoine Pitrou424246f2012-05-12 19:02:01 +02001883_can_xattr = None
1884def can_xattr():
1885 global _can_xattr
1886 if _can_xattr is not None:
1887 return _can_xattr
1888 if not hasattr(os, "setxattr"):
1889 can = False
1890 else:
Hynek Schlawacke02ba102012-05-23 11:22:44 +02001891 tmp_fp, tmp_name = tempfile.mkstemp()
Antoine Pitrou424246f2012-05-12 19:02:01 +02001892 try:
1893 with open(TESTFN, "wb") as fp:
1894 try:
Hynek Schlawacke02ba102012-05-23 11:22:44 +02001895 # TESTFN & tempfile may use different file systems with
1896 # different capabilities
Larry Hastings9cf065c2012-06-22 16:30:09 -07001897 os.setxattr(tmp_fp, b"user.test", b"")
1898 os.setxattr(fp.fileno(), b"user.test", b"")
Antoine Pitrou424246f2012-05-12 19:02:01 +02001899 # Kernels < 2.6.39 don't respect setxattr flags.
1900 kernel_version = platform.release()
1901 m = re.match("2.6.(\d{1,2})", kernel_version)
1902 can = m is None or int(m.group(1)) >= 39
1903 except OSError:
1904 can = False
1905 finally:
1906 unlink(TESTFN)
Hynek Schlawacke02ba102012-05-23 11:22:44 +02001907 unlink(tmp_name)
Antoine Pitrou424246f2012-05-12 19:02:01 +02001908 _can_xattr = can
1909 return can
1910
1911def skip_unless_xattr(test):
1912 """Skip decorator for tests that require functional extended attributes"""
1913 ok = can_xattr()
1914 msg = "no non-broken extended attribute support"
1915 return test if ok else unittest.skip(msg)(test)
1916
Ezio Melotti25a40452013-03-05 20:26:17 +02001917
1918if sys.platform.startswith('win'):
1919 @contextlib.contextmanager
1920 def suppress_crash_popup():
1921 """Disable Windows Error Reporting dialogs using SetErrorMode."""
1922 # see http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621%28v=vs.85%29.aspx
Ezio Melottie8463422013-03-07 18:37:13 +02001923 # GetErrorMode is not available on Windows XP and Windows Server 2003,
1924 # but SetErrorMode returns the previous value, so we can use that
Ezio Melotti25a40452013-03-05 20:26:17 +02001925 import ctypes
1926 k32 = ctypes.windll.kernel32
Ezio Melotti25a40452013-03-05 20:26:17 +02001927 SEM_NOGPFAULTERRORBOX = 0x02
Ezio Melottie8463422013-03-07 18:37:13 +02001928 old_error_mode = k32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
Ezio Melotti25a40452013-03-05 20:26:17 +02001929 k32.SetErrorMode(old_error_mode | SEM_NOGPFAULTERRORBOX)
1930 try:
1931 yield
1932 finally:
1933 k32.SetErrorMode(old_error_mode)
1934else:
1935 # this is a no-op for other platforms
1936 @contextlib.contextmanager
1937 def suppress_crash_popup():
1938 yield
1939
1940
Antoine Pitrou2c50a092011-03-15 21:02:59 +01001941def patch(test_instance, object_to_patch, attr_name, new_value):
1942 """Override 'object_to_patch'.'attr_name' with 'new_value'.
1943
1944 Also, add a cleanup procedure to 'test_instance' to restore
1945 'object_to_patch' value for 'attr_name'.
1946 The 'attr_name' should be a valid attribute for 'object_to_patch'.
1947
1948 """
1949 # check that 'attr_name' is a real attribute for 'object_to_patch'
1950 # will raise AttributeError if it does not exist
1951 getattr(object_to_patch, attr_name)
1952
1953 # keep a copy of the old value
1954 attr_is_local = False
1955 try:
1956 old_value = object_to_patch.__dict__[attr_name]
1957 except (AttributeError, KeyError):
1958 old_value = getattr(object_to_patch, attr_name, None)
1959 else:
1960 attr_is_local = True
1961
1962 # restore the value when the test is done
1963 def cleanup():
1964 if attr_is_local:
1965 setattr(object_to_patch, attr_name, old_value)
1966 else:
1967 delattr(object_to_patch, attr_name)
1968
1969 test_instance.addCleanup(cleanup)
1970
1971 # actually override the attribute
1972 setattr(object_to_patch, attr_name, new_value)