Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 1 | import datetime |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 2 | import faulthandler |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 3 | import os |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 4 | import platform |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 5 | import random |
| 6 | import re |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 7 | import sys |
| 8 | import sysconfig |
| 9 | import tempfile |
| 10 | import textwrap |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 11 | import time |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 12 | from test.libregrtest.cmdline import _parse_args |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 13 | from test.libregrtest.runtest import ( |
Victor Stinner | 6f20a2e | 2015-09-30 02:32:11 +0200 | [diff] [blame] | 14 | findtests, runtest, |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 15 | STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 16 | INTERRUPTED, CHILD_ERROR, |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 17 | PROGRESS_MIN_TIME, format_test_result) |
Victor Stinner | a204502 | 2015-09-30 02:17:28 +0200 | [diff] [blame] | 18 | from test.libregrtest.setup import setup_tests |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 19 | from test import support |
| 20 | try: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 21 | import gc |
| 22 | except ImportError: |
| 23 | gc = None |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 24 | |
| 25 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 26 | # When tests are run from the Python build directory, it is best practice |
| 27 | # to keep the test files in a subfolder. This eases the cleanup of leftover |
| 28 | # files using the "make distclean" command. |
| 29 | if sysconfig.is_python_build(): |
| 30 | TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') |
| 31 | else: |
| 32 | TEMPDIR = tempfile.gettempdir() |
| 33 | TEMPDIR = os.path.abspath(TEMPDIR) |
| 34 | |
| 35 | |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 36 | def format_duration(seconds): |
| 37 | if seconds < 1.0: |
| 38 | return '%.0f ms' % (seconds * 1e3) |
| 39 | if seconds < 60.0: |
| 40 | return '%.0f sec' % seconds |
| 41 | |
| 42 | minutes, seconds = divmod(seconds, 60.0) |
| 43 | return '%.0f min %.0f sec' % (minutes, seconds) |
| 44 | |
| 45 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 46 | class Regrtest: |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 47 | """Execute a test suite. |
| 48 | |
| 49 | This also parses command-line options and modifies its behavior |
| 50 | accordingly. |
| 51 | |
| 52 | tests -- a list of strings containing test names (optional) |
| 53 | testdir -- the directory in which to look for tests (optional) |
| 54 | |
| 55 | Users other than the Python test suite will certainly want to |
| 56 | specify testdir; if it's omitted, the directory containing the |
| 57 | Python test suite is searched for. |
| 58 | |
| 59 | If the tests argument is omitted, the tests listed on the |
| 60 | command-line will be used. If that's empty, too, then all *.py |
| 61 | files beginning with test_ will be used. |
| 62 | |
| 63 | The other default arguments (verbose, quiet, exclude, |
| 64 | single, randomize, findleaks, use_resources, trace, coverdir, |
| 65 | print_slow, and random_seed) allow programmers calling main() |
| 66 | directly to set the values that would normally be set by flags |
| 67 | on the command line. |
| 68 | """ |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 69 | def __init__(self): |
| 70 | # Namespace of command line options |
| 71 | self.ns = None |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 72 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 73 | # tests |
| 74 | self.tests = [] |
| 75 | self.selected = [] |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 76 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 77 | # test results |
| 78 | self.good = [] |
| 79 | self.bad = [] |
| 80 | self.skipped = [] |
| 81 | self.resource_denieds = [] |
| 82 | self.environment_changed = [] |
| 83 | self.interrupted = False |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 84 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 85 | # used by --slow |
| 86 | self.test_times = [] |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 87 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 88 | # used by --coverage, trace.Trace instance |
| 89 | self.tracer = None |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 90 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 91 | # used by --findleaks, store for gc.garbage |
| 92 | self.found_garbage = [] |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 93 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 94 | # used to display the progress bar "[ 3/100]" |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 95 | self.start_time = time.monotonic() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 96 | self.test_count = '' |
| 97 | self.test_count_width = 1 |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 98 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 99 | # used by --single |
| 100 | self.next_single_test = None |
| 101 | self.next_single_filename = None |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 102 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 103 | def accumulate_result(self, test, result): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 104 | ok, test_time = result |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 105 | if ok not in (CHILD_ERROR, INTERRUPTED): |
| 106 | self.test_times.append((test_time, test)) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 107 | if ok == PASSED: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 108 | self.good.append(test) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 109 | elif ok == FAILED: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 110 | self.bad.append(test) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 111 | elif ok == ENV_CHANGED: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 112 | self.environment_changed.append(test) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 113 | elif ok == SKIPPED: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 114 | self.skipped.append(test) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 115 | elif ok == RESOURCE_DENIED: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 116 | self.skipped.append(test) |
| 117 | self.resource_denieds.append(test) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 118 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 119 | def display_progress(self, test_index, test): |
| 120 | if self.ns.quiet: |
| 121 | return |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 122 | if self.bad and not self.ns.pgo: |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 123 | fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 124 | else: |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 125 | fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 126 | test_time = time.monotonic() - self.start_time |
| 127 | test_time = datetime.timedelta(seconds=int(test_time)) |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 128 | line = fmt.format(count_width=self.test_count_width, |
| 129 | test_index=test_index, |
| 130 | test_count=self.test_count, |
| 131 | nbad=len(self.bad), |
| 132 | test_name=test, |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 133 | time=test_time) |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 134 | print(line, flush=True) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 135 | |
Victor Stinner | 234cbef | 2015-09-30 01:13:53 +0200 | [diff] [blame] | 136 | def parse_args(self, kwargs): |
| 137 | ns = _parse_args(sys.argv[1:], **kwargs) |
| 138 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 139 | if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): |
| 140 | print("Warning: The timeout option requires " |
| 141 | "faulthandler.dump_traceback_later", file=sys.stderr) |
| 142 | ns.timeout = None |
| 143 | |
Victor Stinner | 234cbef | 2015-09-30 01:13:53 +0200 | [diff] [blame] | 144 | if ns.threshold is not None and gc is None: |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 145 | print('No GC available, ignore --threshold.', file=sys.stderr) |
Victor Stinner | 234cbef | 2015-09-30 01:13:53 +0200 | [diff] [blame] | 146 | ns.threshold = None |
| 147 | |
| 148 | if ns.findleaks: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 149 | if gc is not None: |
| 150 | # Uncomment the line below to report garbage that is not |
| 151 | # freeable by reference counting alone. By default only |
| 152 | # garbage that is not collectable by the GC is reported. |
| 153 | pass |
| 154 | #gc.set_debug(gc.DEBUG_SAVEALL) |
| 155 | else: |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 156 | print('No GC available, disabling --findleaks', |
| 157 | file=sys.stderr) |
Victor Stinner | 234cbef | 2015-09-30 01:13:53 +0200 | [diff] [blame] | 158 | ns.findleaks = False |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 159 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 160 | # Strip .py extensions. |
Victor Stinner | 234cbef | 2015-09-30 01:13:53 +0200 | [diff] [blame] | 161 | removepy(ns.args) |
| 162 | |
| 163 | return ns |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 164 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 165 | def find_tests(self, tests): |
| 166 | self.tests = tests |
| 167 | |
| 168 | if self.ns.single: |
| 169 | self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') |
| 170 | try: |
| 171 | with open(self.next_single_filename, 'r') as fp: |
| 172 | next_test = fp.read().strip() |
| 173 | self.tests = [next_test] |
| 174 | except OSError: |
| 175 | pass |
| 176 | |
| 177 | if self.ns.fromfile: |
| 178 | self.tests = [] |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 179 | # regex to match 'test_builtin' in line: |
| 180 | # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' |
| 181 | regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' |
| 182 | r'(?:\[[0-9/ ]+\] *)?' |
| 183 | r'(test_[a-zA-Z0-9_]+)') |
| 184 | regex = re.compile(regex) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 185 | with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 186 | for line in fp: |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 187 | line = line.strip() |
| 188 | if line.startswith('#'): |
| 189 | continue |
| 190 | match = regex.match(line) |
| 191 | if match is None: |
| 192 | continue |
| 193 | self.tests.append(match.group(1)) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 194 | |
| 195 | removepy(self.tests) |
| 196 | |
| 197 | stdtests = STDTESTS[:] |
| 198 | nottests = NOTTESTS.copy() |
| 199 | if self.ns.exclude: |
| 200 | for arg in self.ns.args: |
| 201 | if arg in stdtests: |
| 202 | stdtests.remove(arg) |
| 203 | nottests.add(arg) |
| 204 | self.ns.args = [] |
| 205 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 206 | # if testdir is set, then we are not running the python tests suite, so |
| 207 | # don't add default tests to be executed or skipped (pass empty values) |
| 208 | if self.ns.testdir: |
| 209 | alltests = findtests(self.ns.testdir, list(), set()) |
| 210 | else: |
| 211 | alltests = findtests(self.ns.testdir, stdtests, nottests) |
| 212 | |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 213 | if not self.ns.fromfile: |
| 214 | self.selected = self.tests or self.ns.args or alltests |
| 215 | else: |
| 216 | self.selected = self.tests |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 217 | if self.ns.single: |
| 218 | self.selected = self.selected[:1] |
| 219 | try: |
| 220 | pos = alltests.index(self.selected[0]) |
| 221 | self.next_single_test = alltests[pos + 1] |
| 222 | except IndexError: |
| 223 | pass |
| 224 | |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 225 | # Remove all the selected tests that precede start if it's set. |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 226 | if self.ns.start: |
| 227 | try: |
| 228 | del self.selected[:self.selected.index(self.ns.start)] |
| 229 | except ValueError: |
Victor Stinner | 6448b80 | 2015-09-29 23:43:33 +0200 | [diff] [blame] | 230 | print("Couldn't find starting test (%s), using all tests" |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 231 | % self.ns.start, file=sys.stderr) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 232 | |
| 233 | if self.ns.randomize: |
| 234 | if self.ns.random_seed is None: |
| 235 | self.ns.random_seed = random.randrange(10000000) |
| 236 | random.seed(self.ns.random_seed) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 237 | random.shuffle(self.selected) |
| 238 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 239 | def list_tests(self): |
| 240 | for name in self.selected: |
| 241 | print(name) |
| 242 | |
Victor Stinner | 6f20a2e | 2015-09-30 02:32:11 +0200 | [diff] [blame] | 243 | def rerun_failed_tests(self): |
| 244 | self.ns.verbose = True |
| 245 | self.ns.failfast = False |
| 246 | self.ns.verbose3 = False |
| 247 | self.ns.match_tests = None |
| 248 | |
| 249 | print("Re-running failed tests in verbose mode") |
| 250 | for test in self.bad[:]: |
| 251 | print("Re-running test %r in verbose mode" % test, flush=True) |
| 252 | try: |
| 253 | self.ns.verbose = True |
| 254 | ok = runtest(self.ns, test) |
| 255 | except KeyboardInterrupt: |
Victor Stinner | c5a01f8 | 2016-08-17 16:00:12 +0200 | [diff] [blame] | 256 | self.interrupted = True |
Victor Stinner | 6f20a2e | 2015-09-30 02:32:11 +0200 | [diff] [blame] | 257 | # print a newline separate from the ^C |
| 258 | print() |
| 259 | break |
| 260 | else: |
| 261 | if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: |
| 262 | self.bad.remove(test) |
| 263 | else: |
| 264 | if self.bad: |
| 265 | print(count(len(self.bad), 'test'), "failed again:") |
| 266 | printlist(self.bad) |
| 267 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 268 | def display_result(self): |
| 269 | if self.interrupted: |
| 270 | # print a newline after ^C |
| 271 | print() |
| 272 | print("Test suite interrupted by signal SIGINT.") |
Victor Stinner | 6448b80 | 2015-09-29 23:43:33 +0200 | [diff] [blame] | 273 | executed = set(self.good) | set(self.bad) | set(self.skipped) |
| 274 | omitted = set(self.selected) - executed |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 275 | print(count(len(omitted), "test"), "omitted:") |
| 276 | printlist(omitted) |
| 277 | |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 278 | # If running the test suite for PGO then no one cares about |
| 279 | # results. |
| 280 | if self.ns.pgo: |
| 281 | return |
| 282 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 283 | if self.good and not self.ns.quiet: |
Victor Stinner | 6448b80 | 2015-09-29 23:43:33 +0200 | [diff] [blame] | 284 | if (not self.bad |
| 285 | and not self.skipped |
| 286 | and not self.interrupted |
| 287 | and len(self.good) > 1): |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 288 | print("All", end=' ') |
| 289 | print(count(len(self.good), "test"), "OK.") |
| 290 | |
| 291 | if self.ns.print_slow: |
| 292 | self.test_times.sort(reverse=True) |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 293 | print() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 294 | print("10 slowest tests:") |
| 295 | for time, test in self.test_times[:10]: |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 296 | print("- %s: %s" % (test, format_duration(time))) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 297 | |
| 298 | if self.bad: |
Victor Stinner | 8f00319 | 2016-08-17 15:42:21 +0200 | [diff] [blame] | 299 | print() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 300 | print(count(len(self.bad), "test"), "failed:") |
| 301 | printlist(self.bad) |
| 302 | |
| 303 | if self.environment_changed: |
Victor Stinner | 8f00319 | 2016-08-17 15:42:21 +0200 | [diff] [blame] | 304 | print() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 305 | print("{} altered the execution environment:".format( |
| 306 | count(len(self.environment_changed), "test"))) |
| 307 | printlist(self.environment_changed) |
| 308 | |
| 309 | if self.skipped and not self.ns.quiet: |
Victor Stinner | 8f00319 | 2016-08-17 15:42:21 +0200 | [diff] [blame] | 310 | print() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 311 | print(count(len(self.skipped), "test"), "skipped:") |
| 312 | printlist(self.skipped) |
| 313 | |
Victor Stinner | bd1a72c | 2015-09-29 23:36:27 +0200 | [diff] [blame] | 314 | def run_tests_sequential(self): |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 315 | if self.ns.trace: |
| 316 | import trace |
Victor Stinner | a53a818 | 2015-10-01 00:53:09 +0200 | [diff] [blame] | 317 | self.tracer = trace.Trace(trace=False, count=True) |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 318 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 319 | save_modules = sys.modules.keys() |
| 320 | |
Victor Stinner | 2b60b72 | 2016-03-24 11:55:29 +0100 | [diff] [blame] | 321 | print("Run tests sequentially") |
| 322 | |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 323 | previous_test = None |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 324 | for test_index, test in enumerate(self.tests, 1): |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 325 | start_time = time.monotonic() |
| 326 | |
| 327 | text = test |
| 328 | if previous_test: |
| 329 | text = '%s -- %s' % (text, previous_test) |
| 330 | self.display_progress(test_index, text) |
| 331 | |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 332 | if self.tracer: |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 333 | # If we're tracing code coverage, then we don't exit with status |
| 334 | # if on a false return value from main. |
Victor Stinner | 6f20a2e | 2015-09-30 02:32:11 +0200 | [diff] [blame] | 335 | cmd = ('result = runtest(self.ns, test); ' |
| 336 | 'self.accumulate_result(test, result)') |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 337 | ns = dict(locals()) |
| 338 | self.tracer.runctx(cmd, globals=globals(), locals=ns) |
| 339 | result = ns['result'] |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 340 | else: |
| 341 | try: |
Victor Stinner | 6f20a2e | 2015-09-30 02:32:11 +0200 | [diff] [blame] | 342 | result = runtest(self.ns, test) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 343 | except KeyboardInterrupt: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 344 | self.interrupted = True |
Victor Stinner | c5a01f8 | 2016-08-17 16:00:12 +0200 | [diff] [blame] | 345 | self.accumulate_result(test, (INTERRUPTED, None)) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 346 | break |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 347 | else: |
| 348 | self.accumulate_result(test, result) |
Victor Stinner | bd1a72c | 2015-09-29 23:36:27 +0200 | [diff] [blame] | 349 | |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 350 | previous_test = format_test_result(test, result[0]) |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 351 | test_time = time.monotonic() - start_time |
| 352 | if test_time >= PROGRESS_MIN_TIME: |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 353 | previous_test = "%s in %s" % (previous_test, format_duration(test_time)) |
Victor Stinner | 1b8b423 | 2016-05-20 13:37:40 +0200 | [diff] [blame] | 354 | elif result[0] == PASSED: |
| 355 | # be quiet: say nothing if the test passed shortly |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 356 | previous_test = None |
| 357 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 358 | if self.ns.findleaks: |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 359 | gc.collect() |
| 360 | if gc.garbage: |
| 361 | print("Warning: test created", len(gc.garbage), end=' ') |
| 362 | print("uncollectable object(s).") |
| 363 | # move the uncollectable objects somewhere so we don't see |
| 364 | # them again |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 365 | self.found_garbage.extend(gc.garbage) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 366 | del gc.garbage[:] |
Victor Stinner | bd1a72c | 2015-09-29 23:36:27 +0200 | [diff] [blame] | 367 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 368 | # Unload the newly imported modules (best effort finalization) |
| 369 | for module in sys.modules.keys(): |
| 370 | if module not in save_modules and module.startswith("test."): |
| 371 | support.unload(module) |
| 372 | |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 373 | if previous_test: |
| 374 | print(previous_test) |
| 375 | |
Victor Stinner | b408435 | 2015-09-30 02:39:22 +0200 | [diff] [blame] | 376 | def _test_forever(self, tests): |
| 377 | while True: |
| 378 | for test in tests: |
| 379 | yield test |
| 380 | if self.bad: |
| 381 | return |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 382 | |
Victor Stinner | b408435 | 2015-09-30 02:39:22 +0200 | [diff] [blame] | 383 | def run_tests(self): |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 384 | # For a partial run, we do not need to clutter the output. |
| 385 | if (self.ns.verbose |
| 386 | or self.ns.header |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 387 | or not (self.ns.pgo or self.ns.quiet or self.ns.single |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 388 | or self.tests or self.ns.args)): |
| 389 | # Print basic platform information |
| 390 | print("==", platform.python_implementation(), *sys.version.split()) |
| 391 | print("== ", platform.platform(aliased=True), |
| 392 | "%s-endian" % sys.byteorder) |
| 393 | print("== ", "hash algorithm:", sys.hash_info.algorithm, |
| 394 | "64bit" if sys.maxsize > 2**32 else "32bit") |
| 395 | print("== ", os.getcwd()) |
| 396 | print("Testing with flags:", sys.flags) |
| 397 | |
| 398 | if self.ns.randomize: |
| 399 | print("Using random seed", self.ns.random_seed) |
| 400 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 401 | if self.ns.forever: |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 402 | self.tests = self._test_forever(list(self.selected)) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 403 | self.test_count = '' |
| 404 | self.test_count_width = 3 |
| 405 | else: |
| 406 | self.tests = iter(self.selected) |
| 407 | self.test_count = '/{}'.format(len(self.selected)) |
| 408 | self.test_count_width = len(self.test_count) - 1 |
| 409 | |
| 410 | if self.ns.use_mp: |
Victor Stinner | 56e05dd | 2015-09-29 23:15:38 +0200 | [diff] [blame] | 411 | from test.libregrtest.runtest_mp import run_tests_multiprocess |
| 412 | run_tests_multiprocess(self) |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 413 | else: |
Victor Stinner | bd1a72c | 2015-09-29 23:36:27 +0200 | [diff] [blame] | 414 | self.run_tests_sequential() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 415 | |
| 416 | def finalize(self): |
| 417 | if self.next_single_filename: |
| 418 | if self.next_single_test: |
| 419 | with open(self.next_single_filename, 'w') as fp: |
| 420 | fp.write(self.next_single_test + '\n') |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 421 | else: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 422 | os.unlink(self.next_single_filename) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 423 | |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 424 | if self.tracer: |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 425 | r = self.tracer.results() |
| 426 | r.write_results(show_missing=True, summary=True, |
| 427 | coverdir=self.ns.coverdir) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 428 | |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 429 | print() |
| 430 | duration = time.monotonic() - self.start_time |
| 431 | print("Total duration: %s" % format_duration(duration)) |
Victor Stinner | 24f949e | 2016-03-22 15:14:09 +0100 | [diff] [blame] | 432 | |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 433 | if self.bad: |
| 434 | result = "FAILURE" |
| 435 | elif self.interrupted: |
| 436 | result = "INTERRUPTED" |
| 437 | else: |
| 438 | result = "SUCCESS" |
Victor Stinner | f2abf5c | 2016-08-19 17:54:25 +0200 | [diff] [blame] | 439 | print("Tests result: %s" % result) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 440 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 441 | if self.ns.runleaks: |
| 442 | os.system("leaks %d" % os.getpid()) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 443 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 444 | def main(self, tests=None, **kwargs): |
Victor Stinner | 3aac0ad | 2016-03-24 17:53:20 +0100 | [diff] [blame] | 445 | global TEMPDIR |
| 446 | |
| 447 | if sysconfig.is_python_build(): |
| 448 | try: |
| 449 | os.mkdir(TEMPDIR) |
| 450 | except FileExistsError: |
| 451 | pass |
| 452 | |
| 453 | # Define a writable temp dir that will be used as cwd while running |
| 454 | # the tests. The name of the dir includes the pid to allow parallel |
| 455 | # testing (see the -j option). |
| 456 | test_cwd = 'test_python_{}'.format(os.getpid()) |
| 457 | test_cwd = os.path.join(TEMPDIR, test_cwd) |
| 458 | |
| 459 | # Run the tests in a context manager that temporarily changes the CWD to a |
| 460 | # temporary and writable directory. If it's not possible to create or |
| 461 | # change the CWD, the original CWD will be used. The original CWD is |
| 462 | # available from support.SAVEDCWD. |
| 463 | with support.temp_cwd(test_cwd, quiet=True): |
| 464 | self._main(tests, kwargs) |
| 465 | |
| 466 | def _main(self, tests, kwargs): |
Victor Stinner | 234cbef | 2015-09-30 01:13:53 +0200 | [diff] [blame] | 467 | self.ns = self.parse_args(kwargs) |
| 468 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 469 | if self.ns.slaveargs is not None: |
Victor Stinner | 56e05dd | 2015-09-29 23:15:38 +0200 | [diff] [blame] | 470 | from test.libregrtest.runtest_mp import run_tests_slave |
| 471 | run_tests_slave(self.ns.slaveargs) |
Victor Stinner | ecef622 | 2015-09-30 01:39:28 +0200 | [diff] [blame] | 472 | |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 473 | if self.ns.wait: |
| 474 | input("Press any key to continue...") |
| 475 | |
Steve Dower | 22d0698 | 2016-09-06 19:38:15 -0700 | [diff] [blame^] | 476 | support.PGO = self.ns.pgo |
| 477 | |
Victor Stinner | a204502 | 2015-09-30 02:17:28 +0200 | [diff] [blame] | 478 | setup_tests(self.ns) |
Victor Stinner | ecef622 | 2015-09-30 01:39:28 +0200 | [diff] [blame] | 479 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 480 | self.find_tests(tests) |
Victor Stinner | c7eab05 | 2015-09-30 00:59:35 +0200 | [diff] [blame] | 481 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 482 | if self.ns.list_tests: |
| 483 | self.list_tests() |
| 484 | sys.exit(0) |
| 485 | |
| 486 | self.run_tests() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 487 | self.display_result() |
Victor Stinner | 6f20a2e | 2015-09-30 02:32:11 +0200 | [diff] [blame] | 488 | |
| 489 | if self.ns.verbose2 and self.bad: |
| 490 | self.rerun_failed_tests() |
| 491 | |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 492 | self.finalize() |
| 493 | sys.exit(len(self.bad) > 0 or self.interrupted) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 494 | |
| 495 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 496 | def removepy(names): |
| 497 | if not names: |
| 498 | return |
| 499 | for idx, name in enumerate(names): |
| 500 | basename, ext = os.path.splitext(name) |
| 501 | if ext == '.py': |
| 502 | names[idx] = basename |
| 503 | |
| 504 | |
| 505 | def count(n, word): |
| 506 | if n == 1: |
| 507 | return "%d %s" % (n, word) |
| 508 | else: |
| 509 | return "%d %ss" % (n, word) |
| 510 | |
| 511 | |
| 512 | def printlist(x, width=70, indent=4): |
| 513 | """Print the elements of iterable x to stdout. |
| 514 | |
| 515 | Optional arg width (default 70) is the maximum line length. |
| 516 | Optional arg indent (default 4) is the number of blanks with which to |
| 517 | begin each line. |
| 518 | """ |
| 519 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 520 | blanks = ' ' * indent |
| 521 | # Print the sorted list: 'x' may be a '--random' list or a set() |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 522 | print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, |
| 523 | initial_indent=blanks, subsequent_indent=blanks)) |
| 524 | |
| 525 | |
| 526 | def main(tests=None, **kwargs): |
Victor Stinner | 3aac0ad | 2016-03-24 17:53:20 +0100 | [diff] [blame] | 527 | """Run the Python suite.""" |
Victor Stinner | dad20e4 | 2015-09-29 22:48:52 +0200 | [diff] [blame] | 528 | Regrtest().main(tests=tests, **kwargs) |