Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 1 | import collections |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 2 | import faulthandler |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 3 | import functools |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 4 | import importlib |
| 5 | import io |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 6 | import os |
| 7 | import sys |
| 8 | import time |
| 9 | import traceback |
| 10 | import unittest |
| 11 | from test import support |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 12 | from test.libregrtest.refleak import dash_R, clear_caches |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 13 | from test.libregrtest.save_env import saved_test_environment |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 14 | from test.libregrtest.utils import print_warning |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 15 | |
| 16 | |
| 17 | # Test result constants. |
| 18 | PASSED = 1 |
| 19 | FAILED = 0 |
| 20 | ENV_CHANGED = -1 |
| 21 | SKIPPED = -2 |
| 22 | RESOURCE_DENIED = -3 |
| 23 | INTERRUPTED = -4 |
| 24 | CHILD_ERROR = -5 # error in a child process |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 25 | TEST_DID_NOT_RUN = -6 # error in a child process |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 26 | |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 27 | _FORMAT_TEST_RESULT = { |
| 28 | PASSED: '%s passed', |
| 29 | FAILED: '%s failed', |
| 30 | ENV_CHANGED: '%s failed (env changed)', |
| 31 | SKIPPED: '%s skipped', |
| 32 | RESOURCE_DENIED: '%s skipped (resource denied)', |
| 33 | INTERRUPTED: '%s interrupted', |
| 34 | CHILD_ERROR: '%s crashed', |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 35 | TEST_DID_NOT_RUN: '%s run no tests', |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 36 | } |
| 37 | |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 38 | # Minimum duration of a test to display its duration or to mention that |
| 39 | # the test is running in background |
| 40 | PROGRESS_MIN_TIME = 30.0 # seconds |
| 41 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 42 | # small set of tests to determine if we have a basically functioning interpreter |
| 43 | # (i.e. if any of these fail, then anything else is likely to follow) |
| 44 | STDTESTS = [ |
| 45 | 'test_grammar', |
| 46 | 'test_opcodes', |
| 47 | 'test_dict', |
| 48 | 'test_builtin', |
| 49 | 'test_exceptions', |
| 50 | 'test_types', |
| 51 | 'test_unittest', |
| 52 | 'test_doctest', |
| 53 | 'test_doctest2', |
| 54 | 'test_support' |
| 55 | ] |
| 56 | |
| 57 | # set of tests that we don't want to be executed when using regrtest |
| 58 | NOTTESTS = set() |
| 59 | |
| 60 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 61 | # used by --findleaks, store for gc.garbage |
| 62 | found_garbage = [] |
| 63 | |
| 64 | |
| 65 | def format_test_result(result): |
| 66 | fmt = _FORMAT_TEST_RESULT.get(result.result, "%s") |
| 67 | return fmt % result.test_name |
| 68 | |
| 69 | |
| 70 | def findtestdir(path=None): |
| 71 | return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 72 | |
| 73 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 74 | def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): |
| 75 | """Return a list of all applicable test modules.""" |
| 76 | testdir = findtestdir(testdir) |
| 77 | names = os.listdir(testdir) |
| 78 | tests = [] |
| 79 | others = set(stdtests) | nottests |
| 80 | for name in names: |
| 81 | mod, ext = os.path.splitext(name) |
| 82 | if mod[:5] == "test_" and ext in (".py", "") and mod not in others: |
| 83 | tests.append(mod) |
| 84 | return stdtests + sorted(tests) |
| 85 | |
| 86 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 87 | def get_abs_module(ns, test_name): |
| 88 | if test_name.startswith('test.') or ns.testdir: |
| 89 | return test_name |
mlouielu | a49c935 | 2017-06-16 17:36:19 +0800 | [diff] [blame] | 90 | else: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 91 | # Import it from the test package |
| 92 | return 'test.' + test_name |
mlouielu | a49c935 | 2017-06-16 17:36:19 +0800 | [diff] [blame] | 93 | |
| 94 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 95 | TestResult = collections.namedtuple('TestResult', |
| 96 | 'test_name result test_time xml_data') |
| 97 | |
| 98 | def _runtest(ns, test_name): |
| 99 | # Handle faulthandler timeout, capture stdout+stderr, XML serialization |
| 100 | # and measure time. |
| 101 | |
| 102 | output_on_failure = ns.verbose3 |
| 103 | |
| 104 | use_timeout = (ns.timeout is not None) |
| 105 | if use_timeout: |
| 106 | faulthandler.dump_traceback_later(ns.timeout, exit=True) |
| 107 | |
| 108 | start_time = time.perf_counter() |
| 109 | try: |
| 110 | support.set_match_tests(ns.match_tests) |
| 111 | support.junit_xml_list = xml_list = [] if ns.xmlpath else None |
| 112 | if ns.failfast: |
| 113 | support.failfast = True |
| 114 | |
| 115 | if output_on_failure: |
| 116 | support.verbose = True |
| 117 | |
| 118 | stream = io.StringIO() |
| 119 | orig_stdout = sys.stdout |
| 120 | orig_stderr = sys.stderr |
| 121 | try: |
| 122 | sys.stdout = stream |
| 123 | sys.stderr = stream |
| 124 | result = _runtest_inner(ns, test_name, |
| 125 | display_failure=False) |
| 126 | if result != PASSED: |
| 127 | output = stream.getvalue() |
| 128 | orig_stderr.write(output) |
| 129 | orig_stderr.flush() |
| 130 | finally: |
| 131 | sys.stdout = orig_stdout |
| 132 | sys.stderr = orig_stderr |
| 133 | else: |
| 134 | # Tell tests to be moderately quiet |
| 135 | support.verbose = ns.verbose |
| 136 | |
| 137 | result = _runtest_inner(ns, test_name, |
| 138 | display_failure=not ns.verbose) |
| 139 | |
| 140 | if xml_list: |
| 141 | import xml.etree.ElementTree as ET |
| 142 | xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list] |
| 143 | else: |
| 144 | xml_data = None |
| 145 | |
| 146 | test_time = time.perf_counter() - start_time |
| 147 | |
| 148 | return TestResult(test_name, result, test_time, xml_data) |
| 149 | finally: |
| 150 | if use_timeout: |
| 151 | faulthandler.cancel_dump_traceback_later() |
| 152 | support.junit_xml_list = None |
| 153 | |
| 154 | |
| 155 | def runtest(ns, test_name): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 156 | """Run a single test. |
| 157 | |
Victor Stinner | ab98367 | 2016-08-22 14:28:52 +0200 | [diff] [blame] | 158 | ns -- regrtest namespace of options |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 159 | test_name -- the name of the test |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 160 | |
Steve Dower | d0f49d2 | 2018-09-18 09:10:26 -0700 | [diff] [blame] | 161 | Returns the tuple (result, test_time, xml_data), where result is one |
| 162 | of the constants: |
Victor Stinner | ab98367 | 2016-08-22 14:28:52 +0200 | [diff] [blame] | 163 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 164 | INTERRUPTED KeyboardInterrupt |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 165 | RESOURCE_DENIED test skipped because resource denied |
| 166 | SKIPPED test skipped for some other reason |
| 167 | ENV_CHANGED test failed because it changed the execution environment |
| 168 | FAILED test failed |
| 169 | PASSED test passed |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 170 | EMPTY_TEST_SUITE test ran no subtests. |
Steve Dower | d0f49d2 | 2018-09-18 09:10:26 -0700 | [diff] [blame] | 171 | |
| 172 | If ns.xmlpath is not None, xml_data is a list containing each |
| 173 | generated testsuite element. |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 174 | """ |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 175 | try: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 176 | return _runtest(ns, test_name) |
| 177 | except: |
| 178 | if not ns.pgo: |
| 179 | msg = traceback.format_exc() |
| 180 | print(f"test {test_name} crashed -- {msg}", |
| 181 | file=sys.stderr, flush=True) |
| 182 | return TestResult(test_name, FAILED, 0.0, None) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 183 | |
| 184 | |
Victor Stinner | e3510d7 | 2017-08-09 17:44:33 +0200 | [diff] [blame] | 185 | def post_test_cleanup(): |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 186 | support.gc_collect() |
Victor Stinner | e3510d7 | 2017-08-09 17:44:33 +0200 | [diff] [blame] | 187 | support.reap_children() |
| 188 | |
| 189 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 190 | def _test_module(the_module): |
| 191 | loader = unittest.TestLoader() |
| 192 | tests = loader.loadTestsFromModule(the_module) |
| 193 | for error in loader.errors: |
| 194 | print(error, file=sys.stderr) |
| 195 | if loader.errors: |
| 196 | raise Exception("errors while loading tests") |
| 197 | support.run_unittest(tests) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 198 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 199 | |
| 200 | def _runtest_inner2(ns, test_name): |
| 201 | # Load the test function, run the test function, handle huntrleaks |
| 202 | # and findleaks to detect leaks |
| 203 | |
| 204 | abstest = get_abs_module(ns, test_name) |
| 205 | |
| 206 | # remove the module from sys.module to reload it if it was already imported |
| 207 | support.unload(abstest) |
| 208 | |
| 209 | the_module = importlib.import_module(abstest) |
| 210 | |
| 211 | # If the test has a test_main, that will run the appropriate |
| 212 | # tests. If not, use normal unittest test loading. |
| 213 | test_runner = getattr(the_module, "test_main", None) |
| 214 | if test_runner is None: |
| 215 | test_runner = functools.partial(_test_module, the_module) |
| 216 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 217 | try: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 218 | if ns.huntrleaks: |
| 219 | # Return True if the test leaked references |
| 220 | refleak = dash_R(ns, test_name, test_runner) |
| 221 | else: |
| 222 | test_runner() |
| 223 | refleak = False |
| 224 | finally: |
| 225 | cleanup_test_droppings(test_name, ns.verbose) |
| 226 | |
| 227 | if ns.findleaks: |
| 228 | import gc |
| 229 | support.gc_collect() |
| 230 | if gc.garbage: |
| 231 | import gc |
| 232 | gc.garbage = [1] |
| 233 | print_warning(f"{test_name} created {len(gc.garbage)} " |
| 234 | f"uncollectable object(s).") |
| 235 | # move the uncollectable objects somewhere, |
| 236 | # so we don't see them again |
| 237 | found_garbage.extend(gc.garbage) |
| 238 | gc.garbage.clear() |
| 239 | support.environment_altered = True |
| 240 | |
| 241 | post_test_cleanup() |
| 242 | |
| 243 | return refleak |
| 244 | |
| 245 | |
| 246 | def _runtest_inner(ns, test_name, display_failure=True): |
| 247 | # Detect environment changes, handle exceptions. |
| 248 | |
| 249 | # Reset the environment_altered flag to detect if a test altered |
| 250 | # the environment |
| 251 | support.environment_altered = False |
| 252 | |
| 253 | if ns.pgo: |
| 254 | display_failure = False |
| 255 | |
| 256 | try: |
Serhiy Storchaka | 8391026 | 2016-11-11 11:46:44 +0200 | [diff] [blame] | 257 | clear_caches() |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 258 | |
| 259 | with saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: |
| 260 | refleak = _runtest_inner2(ns, test_name) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 261 | except support.ResourceDenied as msg: |
Victor Stinner | ab98367 | 2016-08-22 14:28:52 +0200 | [diff] [blame] | 262 | if not ns.quiet and not ns.pgo: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 263 | print(f"{test_name} skipped -- {msg}", flush=True) |
| 264 | return RESOURCE_DENIED |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 265 | except unittest.SkipTest as msg: |
Victor Stinner | ab98367 | 2016-08-22 14:28:52 +0200 | [diff] [blame] | 266 | if not ns.quiet and not ns.pgo: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 267 | print(f"{test_name} skipped -- {msg}", flush=True) |
| 268 | return SKIPPED |
| 269 | except support.TestFailed as exc: |
| 270 | msg = f"test {test_name} failed" |
| 271 | if display_failure: |
| 272 | msg = f"{msg} -- {exc}" |
| 273 | print(msg, file=sys.stderr, flush=True) |
| 274 | return FAILED |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 275 | except support.TestDidNotRun: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 276 | return TEST_DID_NOT_RUN |
| 277 | except KeyboardInterrupt: |
Victor Stinner | 3cde440 | 2019-04-26 08:40:25 +0200 | [diff] [blame^] | 278 | print() |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 279 | return INTERRUPTED |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 280 | except: |
Victor Stinner | ab98367 | 2016-08-22 14:28:52 +0200 | [diff] [blame] | 281 | if not ns.pgo: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 282 | msg = traceback.format_exc() |
| 283 | print(f"test {test_name} crashed -- {msg}", |
| 284 | file=sys.stderr, flush=True) |
| 285 | return FAILED |
| 286 | |
| 287 | if refleak: |
| 288 | return FAILED |
| 289 | if environment.changed: |
| 290 | return ENV_CHANGED |
| 291 | return PASSED |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 292 | |
| 293 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 294 | def cleanup_test_droppings(test_name, verbose): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 295 | # First kill any dangling references to open files etc. |
| 296 | # This can also issue some ResourceWarnings which would otherwise get |
| 297 | # triggered during the following test run, and possibly produce failures. |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 298 | support.gc_collect() |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 299 | |
| 300 | # Try to clean up junk commonly left behind. While tests shouldn't leave |
| 301 | # any files or directories behind, when a test fails that can be tedious |
| 302 | # for it to arrange. The consequences can be especially nasty on Windows, |
| 303 | # since if a test leaves a file open, it cannot be deleted by name (while |
| 304 | # there's nothing we can do about that here either, we can display the |
| 305 | # name of the offending test, which is a real help). |
| 306 | for name in (support.TESTFN, |
| 307 | "db_home", |
| 308 | ): |
| 309 | if not os.path.exists(name): |
| 310 | continue |
| 311 | |
| 312 | if os.path.isdir(name): |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 313 | import shutil |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 314 | kind, nuker = "directory", shutil.rmtree |
| 315 | elif os.path.isfile(name): |
| 316 | kind, nuker = "file", os.unlink |
| 317 | else: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 318 | raise RuntimeError(f"os.path says {name!r} exists but is neither " |
| 319 | f"directory nor file") |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 320 | |
| 321 | if verbose: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 322 | print_warning("%r left behind %s %r" % (test_name, kind, name)) |
| 323 | support.environment_altered = True |
| 324 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 325 | try: |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 326 | import stat |
Anthony Sottile | 8377cd4 | 2019-02-25 14:32:27 -0800 | [diff] [blame] | 327 | # fix possible permissions problems that might prevent cleanup |
| 328 | os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 329 | nuker(name) |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 330 | except Exception as exc: |
| 331 | print_warning(f"{test_name} left behind {kind} {name!r} " |
| 332 | f"and it couldn't be removed: {exc}") |