bpo-37667: Add regression test for regrtest. (GH-14929)
Verify that it appears to find roughly the right number of tests in the stdlib's testsuite.
(cherry picked from commit e95ac20103437d8099fb91cd4468ab030852f3b7)
Co-authored-by: Gregory P. Smith <greg@krypto.org>
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 2b082b0..b569100 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -6,6 +6,7 @@
import contextlib
import faulthandler
+import glob
import io
import os.path
import platform
@@ -529,6 +530,31 @@
return proc.stdout
+class CheckActualTests(BaseTestCase):
+ """
+ Check that regrtest appears to find the expected set of tests.
+ """
+
+ def test_finds_expected_number_of_tests(self):
+ args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests']
+ output = self.run_python(args)
+ rough_number_of_tests_found = len(output.splitlines())
+ actual_testsuite_glob = os.path.join(os.path.dirname(__file__),
+ 'test*.py')
+ rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob))
+ # We're not trying to duplicate test finding logic in here,
+ # just give a rough estimate of how many there should be and
+ # be near that. This is a regression test to prevent mishaps
+ # such as https://bugs.python.org/issue37667 in the future.
+ # If you need to change the values in here during some
+ # mythical future test suite reorganization, don't go
+ # overboard with logic and keep that goal in mind.
+ self.assertGreater(rough_number_of_tests_found,
+ rough_counted_test_py_files*9//10,
+ msg='Unexpectedly low number of tests found in:\n'
+ f'{", ".join(output.splitlines())}')
+
+
class ProgramsTestCase(BaseTestCase):
"""
Test various ways to run the Python test suite. Use options close