bpo-30540: regrtest: add --matchfile option (#1909)
* Add a new option taking a filename to get a list of test names to
filter tests.
* support.match_tests becomes a list.
* Modify run_unittest() to accept to match the whole test identifier,
not just a part of a test identifier.
For example, the following command only runs test_default_timeout()
of the BarrierTests class of test_threading:
$ ./python -m test -v test_threading -m test.test_threading.BarrierTests.test_default_timeout
Remove also some empty lines from test_regrtest.py to make flake8
tool happy.
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index 8b658a4..7f66997 100644
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -117,6 +117,13 @@
To enable all resources except one, use '-uall,-<resource>'. For
example, to run all the tests except for the gui tests, give the
option '-uall,-gui'.
+
+--matchfile filters tests using a text file, one pattern per line.
+Pattern examples:
+
+- test method: test_stat_attributes
+- test class: FileTests
+- test identifier: test_os.FileTests.test_stat_attributes
"""
@@ -189,8 +196,12 @@
help='single step through a set of tests.' +
more_details)
group.add_argument('-m', '--match', metavar='PAT',
- dest='match_tests',
+ dest='match_tests', action='append',
help='match test cases and methods with glob pattern PAT')
+ group.add_argument('--matchfile', metavar='FILENAME',
+ dest='match_filename',
+ help='similar to --match but get patterns from a '
+ 'text file, one pattern per line')
group.add_argument('-G', '--failfast', action='store_true',
help='fail as soon as a test fails (only with -v or -W)')
group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@@ -350,5 +361,12 @@
print("WARNING: Disable --verbose3 because it's incompatible with "
"--huntrleaks: see http://bugs.python.org/issue27103",
file=sys.stderr)
+ if ns.match_filename:
+ if ns.match_tests is None:
+ ns.match_tests = []
+ filename = os.path.join(support.SAVEDCWD, ns.match_filename)
+ with open(filename) as fp:
+ for line in fp:
+ ns.match_tests.append(line.strip())
return ns
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index d8d599b..0a3d4d9 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1922,9 +1922,15 @@
def case_pred(test):
if match_tests is None:
return True
- for name in test.id().split("."):
- if fnmatch.fnmatchcase(name, match_tests):
+ test_id = test.id()
+
+ for match_test in match_tests:
+ if fnmatch.fnmatchcase(test_id, match_test):
return True
+
+ for name in test_id.split("."):
+ if fnmatch.fnmatchcase(name, match_test):
+ return True
return False
_filter_suite(suite, case_pred)
_run_suite(suite)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index d349260..c189b4a 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -159,9 +159,24 @@
for opt in '-m', '--match':
with self.subTest(opt=opt):
ns = libregrtest._parse_args([opt, 'pattern'])
- self.assertEqual(ns.match_tests, 'pattern')
+ self.assertEqual(ns.match_tests, ['pattern'])
self.checkError([opt], 'expected one argument')
+ ns = libregrtest._parse_args(['-m', 'pattern1',
+ '-m', 'pattern2'])
+ self.assertEqual(ns.match_tests, ['pattern1', 'pattern2'])
+
+ self.addCleanup(support.unlink, support.TESTFN)
+ with open(support.TESTFN, "w") as fp:
+ print('matchfile1', file=fp)
+ print('matchfile2', file=fp)
+
+ filename = os.path.abspath(support.TESTFN)
+ ns = libregrtest._parse_args(['-m', 'match',
+ '--matchfile', filename])
+ self.assertEqual(ns.match_tests,
+ ['match', 'matchfile1', 'matchfile2'])
+
def test_failfast(self):
for opt in '-G', '--failfast':
with self.subTest(opt=opt):
@@ -275,7 +290,6 @@
ns = libregrtest._parse_args([opt])
self.assertTrue(ns.forever)
-
def test_unrecognized_argument(self):
self.checkError(['--xxx'], 'usage:')
@@ -457,7 +471,6 @@
self.fail(msg)
return proc
-
def run_python(self, args, **kw):
args = [sys.executable, '-X', 'faulthandler', '-I', *args]
proc = self.run_command(args, **kw)
@@ -823,6 +836,52 @@
self.check_executed_tests(output, tests, failed=crash_test,
randomize=True)
+ def parse_methods(self, output):
+ regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
+ return [match.group(1) for match in regex.finditer(output)]
+
+ def test_matchfile(self):
+ # Any code which causes a crash
+ code = textwrap.dedent("""
+ import unittest
+
+ class Tests(unittest.TestCase):
+ def test_method1(self):
+ pass
+ def test_method2(self):
+ pass
+ def test_method3(self):
+ pass
+ def test_method4(self):
+ pass
+ """)
+ all_methods = ['test_method1', 'test_method2',
+ 'test_method3', 'test_method4']
+ testname = self.create_test(code=code)
+
+ # by default, all methods should be run
+ output = self.run_tests("-v", testname)
+ methods = self.parse_methods(output)
+ self.assertEqual(methods, all_methods)
+
+ # only run a subset
+ filename = support.TESTFN
+ self.addCleanup(support.unlink, filename)
+
+ subset = [
+ # only match the method name
+ 'test_method1',
+ # match the full identifier
+ '%s.Tests.test_method3' % testname]
+ with open(filename, "w") as fp:
+ for name in subset:
+ print(name, file=fp)
+
+ output = self.run_tests("-v", "--matchfile", filename, testname)
+ methods = self.parse_methods(output)
+ subset = ['test_method1', 'test_method3']
+ self.assertEqual(methods, subset)
+
if __name__ == '__main__':
unittest.main()