[2.7] bpo-30283: regrtest: backport test_slow_interrupted() and test_coverage() (#1541)
* bpo-30283: regrtest: backport test_coverage()
* Add --coverage option, the option was already described in the doc
* When coverage is used, regrtest now pass all options to runtest()
and calls also accumulate_result() (as done when coverage is not
used).
* bpo-25260: Fix coverage on Windows: remove the list of ignored
directories.
* bpo-30283: regrtest: backport test_slow_interrupted()
* Fix regrtest to report interrupted tests as omitted rather than
failed.
* bpo-25260: Fix coverage on Windows: remove the list of ignored
directories.
* bpo-30283: Fix test_regrtest on Visual Studio 2008
Skip Tools\buildbot\test.bat and PCbuild\rt.bat if Python was not
compiled in PCbuild (but compiled in PC\VS9.0\ for example).
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index 37781fc..48779fc 100755
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -316,7 +316,7 @@
'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',
'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo',
- 'failfast', 'match=', 'testdir=', 'list-tests'])
+ 'failfast', 'match=', 'testdir=', 'list-tests', 'coverage'])
except getopt.error, msg:
usage(2, msg)
@@ -531,8 +531,7 @@
if trace:
import trace
- tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
- trace=False, count=True)
+ tracer = trace.Trace(trace=False, count=True)
test_times = []
test_support.use_resources = use_resources
@@ -544,7 +543,7 @@
test_times.append((test_time, test))
if ok == PASSED:
good.append(test)
- elif ok == FAILED:
+ elif ok in (FAILED, CHILD_ERROR):
bad.append(test)
elif ok == ENV_CHANGED:
environment_changed.append(test)
@@ -553,9 +552,8 @@
elif ok == RESOURCE_DENIED:
skipped.append(test)
resource_denieds.append(test)
- else:
- # CHILD_ERROR
- bad.append(test)
+ elif ok != INTERRUPTED:
+ raise ValueError("invalid test result: %r" % ok)
if forever:
def test_forever(tests=list(selected)):
@@ -740,19 +738,26 @@
if previous_test:
text = '%s -- %s' % (text, previous_test)
display_progress(test_index, text)
+
+ def local_runtest():
+ result = runtest(test, verbose, quiet, huntrleaks, None, pgo,
+ failfast=failfast,
+ match_tests=match_tests,
+ testdir=testdir)
+ accumulate_result(test, result)
+ return result
+
+ start_time = time.time()
if trace:
# If we're tracing code coverage, then we don't exit with status
# if on a false return value from main.
- tracer.runctx('runtest(test, verbose, quiet, testdir=testdir)',
- globals=globals(), locals=vars())
+ ns = dict(locals())
+ tracer.runctx('result = local_runtest()',
+ globals=globals(), locals=ns)
+ result = ns['result']
else:
- start_time = time.time()
try:
- result = runtest(test, verbose, quiet, huntrleaks, None, pgo,
- failfast=failfast,
- match_tests=match_tests,
- testdir=testdir)
- accumulate_result(test, result)
+ result = local_runtest()
if verbose3 and result[0] == FAILED:
if not pgo:
print "Re-running test %r in verbose mode" % test
@@ -764,14 +769,14 @@
except:
raise
- previous_test = format_test_result(test, result[0])
- test_time = time.time() - start_time
- if test_time >= PROGRESS_MIN_TIME:
- previous_test = "%s in %s" % (previous_test,
- format_duration(test_time))
- elif result[0] == PASSED:
- # be quiet: say nothing if the test passed shortly
- previous_test = None
+ test_time = time.time() - start_time
+ previous_test = format_test_result(test, result[0])
+ if test_time >= PROGRESS_MIN_TIME:
+ previous_test = "%s in %s" % (previous_test,
+ format_duration(test_time))
+ elif result[0] == PASSED:
+ # be quiet: say nothing if the test passed shortly
+ previous_test = None
if findleaks:
gc.collect()
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 211d1b5..fa23220 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -255,10 +255,22 @@
proc = self.run_command(args)
self.check_output(proc.stdout)
- @unittest.skipUnless(sysconfig.is_python_build(),
- 'test.bat script is not installed')
+ def need_pcbuild(self):
+ exe = os.path.normpath(os.path.abspath(sys.executable))
+ parts = exe.split(os.path.sep)
+ if len(parts) < 3:
+ # it's not a python build, python is likely to be installed
+ return
+
+ build_dir = parts[-3]
+ if build_dir.lower() != 'pcbuild':
+ self.skipTest("Tools/buildbot/test.bat requires PCbuild build, "
+ "found %s" % build_dir)
+
@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
def test_tools_buildbot_test(self):
+ self.need_pcbuild()
+
# Tools\buildbot\test.bat
script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
test_args = ['--testdir=%s' % self.tmptestdir]
@@ -272,6 +284,8 @@
@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
def test_pcbuild_rt(self):
+ self.need_pcbuild()
+
# PCbuild\rt.bat
script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
rt_args = ["-q"] # Quick, don't run tests twice
@@ -392,6 +406,37 @@
% (self.TESTNAME_REGEX, len(tests)))
self.check_line(output, regex)
+ def test_slow_interrupted(self):
+ # Issue #25373: test --slowest with an interrupted test
+ code = TEST_INTERRUPTED
+ test = self.create_test("sigint", code=code)
+
+ try:
+ import threading
+ tests = (False, True)
+ except ImportError:
+ tests = (False,)
+ for multiprocessing in tests:
+ if multiprocessing:
+ args = ("--slowest", "-j2", test)
+ else:
+ args = ("--slowest", test)
+ output = self.run_tests(*args, exitcode=1)
+ self.check_executed_tests(output, test,
+ omitted=test, interrupted=True)
+
+ regex = ('10 slowest tests:\n')
+ self.check_line(output, regex)
+
+ def test_coverage(self):
+ # test --coverage
+ test = self.create_test('coverage')
+ output = self.run_tests("--coverage", test)
+ self.check_executed_tests(output, [test])
+ regex = (r'lines +cov% +module +\(path\)\n'
+ r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+')
+ self.check_line(output, regex)
+
def test_forever(self):
# test --forever
code = textwrap.dedent("""