bpo-44708: Only re-run test methods that match names of previously failing test methods (GH-27287) (GH-27290)
* Move to a static argparse.Namespace subclass
* Roughly annotate runtest.py
* Refactor libregrtest to use lossless test result objects
* Only re-run test methods that match names of previously failing test methods
* Adopt tests to cover test method name matching
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
(cherry picked from commit f1afef5e0d93d66fbf3c9aaeab8b3b8da9617583)
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index 927c470..489ab98 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -1,4 +1,3 @@
-import collections
import faulthandler
import functools
import gc
@@ -12,33 +11,109 @@
from test import support
from test.support import os_helper
-from test.libregrtest.utils import clear_caches
+from test.libregrtest.cmdline import Namespace
from test.libregrtest.save_env import saved_test_environment
-from test.libregrtest.utils import format_duration, print_warning
+from test.libregrtest.utils import clear_caches, format_duration, print_warning
-# Test result constants.
-PASSED = 1
-FAILED = 0
-ENV_CHANGED = -1
-SKIPPED = -2
-RESOURCE_DENIED = -3
-INTERRUPTED = -4
-CHILD_ERROR = -5 # error in a child process
-TEST_DID_NOT_RUN = -6
-TIMEOUT = -7
+class TestResult:
+ def __init__(
+ self,
+ name: str,
+ duration_sec: float = 0.0,
+ xml_data: list[str] | None = None,
+ ) -> None:
+ self.name = name
+ self.duration_sec = duration_sec
+ self.xml_data = xml_data
-_FORMAT_TEST_RESULT = {
- PASSED: '%s passed',
- FAILED: '%s failed',
- ENV_CHANGED: '%s failed (env changed)',
- SKIPPED: '%s skipped',
- RESOURCE_DENIED: '%s skipped (resource denied)',
- INTERRUPTED: '%s interrupted',
- CHILD_ERROR: '%s crashed',
- TEST_DID_NOT_RUN: '%s run no tests',
- TIMEOUT: '%s timed out',
-}
+ def __str__(self) -> str:
+ return f"{self.name} finished"
+
+
+class Passed(TestResult):
+ def __str__(self) -> str:
+ return f"{self.name} passed"
+
+
+class Failed(TestResult):
+ def __init__(
+ self,
+ name: str,
+ duration_sec: float = 0.0,
+ xml_data: list[str] | None = None,
+ errors: list[tuple[str, str]] | None = None,
+ failures: list[tuple[str, str]] | None = None,
+ ) -> None:
+ super().__init__(name, duration_sec=duration_sec, xml_data=xml_data)
+ self.errors = errors
+ self.failures = failures
+
+ def __str__(self) -> str:
+ if self.errors and self.failures:
+ le = len(self.errors)
+ lf = len(self.failures)
+ error_s = "error" + ("s" if le > 1 else "")
+ failure_s = "failure" + ("s" if lf > 1 else "")
+ return f"{self.name} failed ({le} {error_s}, {lf} {failure_s})"
+
+ if self.errors:
+ le = len(self.errors)
+ error_s = "error" + ("s" if le > 1 else "")
+ return f"{self.name} failed ({le} {error_s})"
+
+ if self.failures:
+ lf = len(self.failures)
+ failure_s = "failure" + ("s" if lf > 1 else "")
+ return f"{self.name} failed ({lf} {failure_s})"
+
+ return f"{self.name} failed"
+
+
+class UncaughtException(Failed):
+ def __str__(self) -> str:
+ return f"{self.name} failed (uncaught exception)"
+
+
+class EnvChanged(Failed):
+ def __str__(self) -> str:
+ return f"{self.name} failed (env changed)"
+
+
+class RefLeak(Failed):
+ def __str__(self) -> str:
+ return f"{self.name} failed (reference leak)"
+
+
+class Skipped(TestResult):
+ def __str__(self) -> str:
+ return f"{self.name} skipped"
+
+
+class ResourceDenied(Skipped):
+ def __str__(self) -> str:
+ return f"{self.name} skipped (resource denied)"
+
+
+class Interrupted(TestResult):
+ def __str__(self) -> str:
+ return f"{self.name} interrupted"
+
+
+class ChildError(Failed):
+ def __str__(self) -> str:
+ return f"{self.name} crashed"
+
+
+class DidNotRun(TestResult):
+ def __str__(self) -> str:
+ return f"{self.name} ran no tests"
+
+
+class Timeout(Failed):
+ def __str__(self) -> str:
+ return f"{self.name} timed out ({format_duration(self.duration_sec)})"
+
# Minimum duration of a test to display its duration or to mention that
# the test is running in background
@@ -67,21 +142,10 @@
FOUND_GARBAGE = []
-def is_failed(result, ns):
- ok = result.result
- if ok in (PASSED, RESOURCE_DENIED, SKIPPED, TEST_DID_NOT_RUN):
- return False
- if ok == ENV_CHANGED:
+def is_failed(result: TestResult, ns: Namespace) -> bool:
+ if isinstance(result, EnvChanged):
return ns.fail_env_changed
- return True
-
-
-def format_test_result(result):
- fmt = _FORMAT_TEST_RESULT.get(result.result, "%s")
- text = fmt % result.test_name
- if result.result == TIMEOUT:
- text = '%s (%s)' % (text, format_duration(result.test_time))
- return text
+ return isinstance(result, Failed)
def findtestdir(path=None):
@@ -101,7 +165,7 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
return stdtests + sorted(tests)
-def get_abs_module(ns, test_name):
+def get_abs_module(ns: Namespace, test_name: str) -> str:
if test_name.startswith('test.') or ns.testdir:
return test_name
else:
@@ -109,10 +173,7 @@ def get_abs_module(ns, test_name):
return 'test.' + test_name
-TestResult = collections.namedtuple('TestResult',
- 'test_name result test_time xml_data')
-
-def _runtest(ns, test_name):
+def _runtest(ns: Namespace, test_name: str) -> TestResult:
# Handle faulthandler timeout, capture stdout+stderr, XML serialization
# and measure time.
@@ -140,7 +201,7 @@ def _runtest(ns, test_name):
sys.stderr = stream
result = _runtest_inner(ns, test_name,
display_failure=False)
- if result != PASSED:
+ if not isinstance(result, Passed):
output = stream.getvalue()
orig_stderr.write(output)
orig_stderr.flush()
@@ -156,36 +217,26 @@ def _runtest(ns, test_name):
if xml_list:
import xml.etree.ElementTree as ET
- xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list]
- else:
- xml_data = None
+ result.xml_data = [
+ ET.tostring(x).decode('us-ascii')
+ for x in xml_list
+ ]
- test_time = time.perf_counter() - start_time
-
- return TestResult(test_name, result, test_time, xml_data)
+ result.duration_sec = time.perf_counter() - start_time
+ return result
finally:
if use_timeout:
faulthandler.cancel_dump_traceback_later()
support.junit_xml_list = None
-def runtest(ns, test_name):
+def runtest(ns: Namespace, test_name: str) -> TestResult:
"""Run a single test.
ns -- regrtest namespace of options
test_name -- the name of the test
- Returns the tuple (result, test_time, xml_data), where result is one
- of the constants:
-
- INTERRUPTED KeyboardInterrupt
- RESOURCE_DENIED test skipped because resource denied
- SKIPPED test skipped for some other reason
- ENV_CHANGED test failed because it changed the execution environment
- FAILED test failed
- PASSED test passed
- EMPTY_TEST_SUITE test ran no subtests.
- TIMEOUT test timed out.
+ Returns a TestResult sub-class depending on the kind of result received.
If ns.xmlpath is not None, xml_data is a list containing each
generated testsuite element.
@@ -197,7 +248,7 @@ def runtest(ns, test_name):
msg = traceback.format_exc()
print(f"test {test_name} crashed -- {msg}",
file=sys.stderr, flush=True)
- return TestResult(test_name, FAILED, 0.0, None)
+ return Failed(test_name)
def _test_module(the_module):
@@ -210,11 +261,11 @@ def _test_module(the_module):
support.run_unittest(tests)
-def save_env(ns, test_name):
+def save_env(ns: Namespace, test_name: str):
return saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo)
-def _runtest_inner2(ns, test_name):
+def _runtest_inner2(ns: Namespace, test_name: str) -> bool:
# Load the test function, run the test function, handle huntrleaks
# and findleaks to detect leaks
@@ -265,7 +316,9 @@ def _runtest_inner2(ns, test_name):
return refleak
-def _runtest_inner(ns, test_name, display_failure=True):
+def _runtest_inner(
+ ns: Namespace, test_name: str, display_failure: bool = True
+) -> TestResult:
# Detect environment changes, handle exceptions.
# Reset the environment_altered flag to detect if a test altered
@@ -283,37 +336,43 @@ def _runtest_inner(ns, test_name, display_failure=True):
except support.ResourceDenied as msg:
if not ns.quiet and not ns.pgo:
print(f"{test_name} skipped -- {msg}", flush=True)
- return RESOURCE_DENIED
+ return ResourceDenied(test_name)
except unittest.SkipTest as msg:
if not ns.quiet and not ns.pgo:
print(f"{test_name} skipped -- {msg}", flush=True)
- return SKIPPED
+ return Skipped(test_name)
+ except support.TestFailedWithDetails as exc:
+ msg = f"test {test_name} failed"
+ if display_failure:
+ msg = f"{msg} -- {exc}"
+ print(msg, file=sys.stderr, flush=True)
+ return Failed(test_name, errors=exc.errors, failures=exc.failures)
except support.TestFailed as exc:
msg = f"test {test_name} failed"
if display_failure:
msg = f"{msg} -- {exc}"
print(msg, file=sys.stderr, flush=True)
- return FAILED
+ return Failed(test_name)
except support.TestDidNotRun:
- return TEST_DID_NOT_RUN
+ return DidNotRun(test_name)
except KeyboardInterrupt:
print()
- return INTERRUPTED
+ return Interrupted(test_name)
except:
if not ns.pgo:
msg = traceback.format_exc()
print(f"test {test_name} crashed -- {msg}",
file=sys.stderr, flush=True)
- return FAILED
+ return UncaughtException(test_name)
if refleak:
- return FAILED
+ return RefLeak(test_name)
if support.environment_altered:
- return ENV_CHANGED
- return PASSED
+ return EnvChanged(test_name)
+ return Passed(test_name)
-def cleanup_test_droppings(test_name, verbose):
+def cleanup_test_droppings(test_name: str, verbose: int) -> None:
# First kill any dangling references to open files etc.
# This can also issue some ResourceWarnings which would otherwise get
# triggered during the following test run, and possibly produce failures.