bpo-44708: Only re-run test methods that match names of previously failing test methods (GH-27287) (GH-27290)
* Move to a static argparse.Namespace subclass
* Roughly annotate runtest.py
* Refactor libregrtest to use lossless test result objects
* Only re-run test methods that match names of previously failing test methods
* Adopt tests to cover test method name matching
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
(cherry picked from commit f1afef5e0d93d66fbf3c9aaeab8b3b8da9617583)
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 75fa6f2..3780fee 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -15,6 +15,7 @@
import sysconfig
import tempfile
import textwrap
+import time
import unittest
from test import libregrtest
from test import support
@@ -414,7 +415,7 @@ def parse_executed_tests(self, output):
def check_executed_tests(self, output, tests, skipped=(), failed=(),
env_changed=(), omitted=(),
- rerun=(), no_test_ran=(),
+ rerun={}, no_test_ran=(),
randomize=False, interrupted=False,
fail_env_changed=False):
if isinstance(tests, str):
@@ -427,8 +428,6 @@ def check_executed_tests(self, output, tests, skipped=(), failed=(),
env_changed = [env_changed]
if isinstance(omitted, str):
omitted = [omitted]
- if isinstance(rerun, str):
- rerun = [rerun]
if isinstance(no_test_ran, str):
no_test_ran = [no_test_ran]
@@ -466,12 +465,12 @@ def list_regex(line_format, tests):
self.check_line(output, regex)
if rerun:
- regex = list_regex('%s re-run test%s', rerun)
+ regex = list_regex('%s re-run test%s', rerun.keys())
self.check_line(output, regex)
regex = LOG_PREFIX + r"Re-running failed tests in verbose mode"
self.check_line(output, regex)
- for test_name in rerun:
- regex = LOG_PREFIX + f"Re-running {test_name} in verbose mode"
+ for name, match in rerun.items():
+ regex = LOG_PREFIX + f"Re-running {name} in verbose mode \\(matching: {match}\\)"
self.check_line(output, regex)
if no_test_ran:
@@ -549,11 +548,10 @@ def run_python(self, args, **kw):
class CheckActualTests(BaseTestCase):
- """
- Check that regrtest appears to find the expected set of tests.
- """
-
def test_finds_expected_number_of_tests(self):
+ """
+ Check that regrtest appears to find the expected set of tests.
+ """
args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests']
output = self.run_python(args)
rough_number_of_tests_found = len(output.splitlines())
@@ -1081,15 +1079,18 @@ def test_rerun_fail(self):
import unittest
class Tests(unittest.TestCase):
- def test_bug(self):
- # test always fail
+ def test_succeed(self):
+ return
+
+ def test_fail_always(self):
+ # test that always fails
self.fail("bug")
""")
testname = self.create_test(code=code)
output = self.run_tests("-w", testname, exitcode=2)
self.check_executed_tests(output, [testname],
- failed=testname, rerun=testname)
+ failed=testname, rerun={testname: "test_fail_always"})
def test_rerun_success(self):
# FAILURE then SUCCESS
@@ -1098,7 +1099,8 @@ def test_rerun_success(self):
import unittest
class Tests(unittest.TestCase):
- failed = False
+ def test_succeed(self):
+ return
def test_fail_once(self):
if not hasattr(builtins, '_test_failed'):
@@ -1109,7 +1111,7 @@ def test_fail_once(self):
output = self.run_tests("-w", testname, exitcode=0)
self.check_executed_tests(output, [testname],
- rerun=testname)
+ rerun={testname: "test_fail_once"})
def test_no_tests_ran(self):
code = textwrap.dedent("""