Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 1 | """ |
| 2 | Tests of regrtest.py. |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 3 | |
| 4 | Note: test_regrtest cannot be run twice in parallel. |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 5 | """ |
| 6 | |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 7 | import contextlib |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 8 | import faulthandler |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 9 | import io |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 10 | import os.path |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 11 | import platform |
| 12 | import re |
| 13 | import subprocess |
| 14 | import sys |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 15 | import sysconfig |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 16 | import tempfile |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 17 | import textwrap |
Antoine Pitrou | a6a4dc8 | 2017-09-07 18:56:24 +0200 | [diff] [blame] | 18 | import threading |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 19 | import unittest |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 20 | from test import libregrtest |
| 21 | from test import support |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 22 | |
| 23 | |
| 24 | Py_DEBUG = hasattr(sys, 'getobjects') |
| 25 | ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') |
| 26 | ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) |
| 27 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 28 | TEST_INTERRUPTED = textwrap.dedent(""" |
| 29 | from signal import SIGINT |
| 30 | try: |
| 31 | from _testcapi import raise_signal |
| 32 | raise_signal(SIGINT) |
| 33 | except ImportError: |
| 34 | import os |
| 35 | os.kill(os.getpid(), SIGINT) |
| 36 | """) |
| 37 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 38 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 39 | class ParseArgsTestCase(unittest.TestCase): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 40 | """ |
| 41 | Test regrtest's argument parsing, function _parse_args(). |
| 42 | """ |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 43 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 44 | def checkError(self, args, msg): |
| 45 | with support.captured_stderr() as err, self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 46 | libregrtest._parse_args(args) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 47 | self.assertIn(msg, err.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 48 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 49 | def test_help(self): |
| 50 | for opt in '-h', '--help': |
| 51 | with self.subTest(opt=opt): |
| 52 | with support.captured_stdout() as out, \ |
| 53 | self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 54 | libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 55 | self.assertIn('Run Python regression tests.', out.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 56 | |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 57 | @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), |
| 58 | "faulthandler.dump_traceback_later() required") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 59 | def test_timeout(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 60 | ns = libregrtest._parse_args(['--timeout', '4.2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 61 | self.assertEqual(ns.timeout, 4.2) |
| 62 | self.checkError(['--timeout'], 'expected one argument') |
| 63 | self.checkError(['--timeout', 'foo'], 'invalid float value') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 64 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 65 | def test_wait(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 66 | ns = libregrtest._parse_args(['--wait']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 67 | self.assertTrue(ns.wait) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 68 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 69 | def test_slaveargs(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 70 | ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 71 | self.assertEqual(ns.slaveargs, '[[], {}]') |
| 72 | self.checkError(['--slaveargs'], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 73 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 74 | def test_start(self): |
| 75 | for opt in '-S', '--start': |
| 76 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 77 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 78 | self.assertEqual(ns.start, 'foo') |
| 79 | self.checkError([opt], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 80 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 81 | def test_verbose(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 82 | ns = libregrtest._parse_args(['-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 83 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 84 | ns = libregrtest._parse_args(['-vvv']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 85 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 86 | ns = libregrtest._parse_args(['--verbose']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 87 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 88 | ns = libregrtest._parse_args(['--verbose'] * 3) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 89 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 90 | ns = libregrtest._parse_args([]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 91 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 92 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 93 | def test_verbose2(self): |
| 94 | for opt in '-w', '--verbose2': |
| 95 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 96 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 97 | self.assertTrue(ns.verbose2) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 98 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 99 | def test_verbose3(self): |
| 100 | for opt in '-W', '--verbose3': |
| 101 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 102 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 103 | self.assertTrue(ns.verbose3) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 104 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 105 | def test_quiet(self): |
| 106 | for opt in '-q', '--quiet': |
| 107 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 108 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 109 | self.assertTrue(ns.quiet) |
| 110 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 111 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 112 | def test_slow(self): |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 113 | for opt in '-o', '--slowest': |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 114 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 115 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 116 | self.assertTrue(ns.print_slow) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 117 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 118 | def test_header(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 119 | ns = libregrtest._parse_args(['--header']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 120 | self.assertTrue(ns.header) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 121 | |
Victor Stinner | 3d00568 | 2017-05-04 15:21:12 +0200 | [diff] [blame] | 122 | ns = libregrtest._parse_args(['--verbose']) |
| 123 | self.assertTrue(ns.header) |
| 124 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 125 | def test_randomize(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 126 | for opt in '-r', '--randomize': |
| 127 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 128 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 129 | self.assertTrue(ns.randomize) |
| 130 | |
| 131 | def test_randseed(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 132 | ns = libregrtest._parse_args(['--randseed', '12345']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 133 | self.assertEqual(ns.random_seed, 12345) |
| 134 | self.assertTrue(ns.randomize) |
| 135 | self.checkError(['--randseed'], 'expected one argument') |
| 136 | self.checkError(['--randseed', 'foo'], 'invalid int value') |
| 137 | |
| 138 | def test_fromfile(self): |
| 139 | for opt in '-f', '--fromfile': |
| 140 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 141 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 142 | self.assertEqual(ns.fromfile, 'foo') |
| 143 | self.checkError([opt], 'expected one argument') |
| 144 | self.checkError([opt, 'foo', '-s'], "don't go together") |
| 145 | |
| 146 | def test_exclude(self): |
| 147 | for opt in '-x', '--exclude': |
| 148 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 149 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 150 | self.assertTrue(ns.exclude) |
| 151 | |
| 152 | def test_single(self): |
| 153 | for opt in '-s', '--single': |
| 154 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 155 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 156 | self.assertTrue(ns.single) |
| 157 | self.checkError([opt, '-f', 'foo'], "don't go together") |
| 158 | |
| 159 | def test_match(self): |
| 160 | for opt in '-m', '--match': |
| 161 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 162 | ns = libregrtest._parse_args([opt, 'pattern']) |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 163 | self.assertEqual(ns.match_tests, ['pattern']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 164 | self.checkError([opt], 'expected one argument') |
| 165 | |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 166 | ns = libregrtest._parse_args(['-m', 'pattern1', |
| 167 | '-m', 'pattern2']) |
| 168 | self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) |
| 169 | |
| 170 | self.addCleanup(support.unlink, support.TESTFN) |
| 171 | with open(support.TESTFN, "w") as fp: |
| 172 | print('matchfile1', file=fp) |
| 173 | print('matchfile2', file=fp) |
| 174 | |
| 175 | filename = os.path.abspath(support.TESTFN) |
| 176 | ns = libregrtest._parse_args(['-m', 'match', |
| 177 | '--matchfile', filename]) |
| 178 | self.assertEqual(ns.match_tests, |
| 179 | ['match', 'matchfile1', 'matchfile2']) |
| 180 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 181 | def test_failfast(self): |
| 182 | for opt in '-G', '--failfast': |
| 183 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 184 | ns = libregrtest._parse_args([opt, '-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 185 | self.assertTrue(ns.failfast) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 186 | ns = libregrtest._parse_args([opt, '-W']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 187 | self.assertTrue(ns.failfast) |
| 188 | self.checkError([opt], '-G/--failfast needs either -v or -W') |
| 189 | |
| 190 | def test_use(self): |
| 191 | for opt in '-u', '--use': |
| 192 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 193 | ns = libregrtest._parse_args([opt, 'gui,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 194 | self.assertEqual(ns.use_resources, ['gui', 'network']) |
Victor Stinner | 5b392bb | 2017-07-20 15:46:32 +0200 | [diff] [blame] | 195 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 196 | ns = libregrtest._parse_args([opt, 'gui,none,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 197 | self.assertEqual(ns.use_resources, ['network']) |
Victor Stinner | 5b392bb | 2017-07-20 15:46:32 +0200 | [diff] [blame] | 198 | |
| 199 | expected = list(libregrtest.ALL_RESOURCES) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 200 | expected.remove('gui') |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 201 | ns = libregrtest._parse_args([opt, 'all,-gui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 202 | self.assertEqual(ns.use_resources, expected) |
| 203 | self.checkError([opt], 'expected one argument') |
| 204 | self.checkError([opt, 'foo'], 'invalid resource') |
| 205 | |
Victor Stinner | 5b392bb | 2017-07-20 15:46:32 +0200 | [diff] [blame] | 206 | # all + a resource not part of "all" |
| 207 | ns = libregrtest._parse_args([opt, 'all,tzdata']) |
| 208 | self.assertEqual(ns.use_resources, |
| 209 | list(libregrtest.ALL_RESOURCES) + ['tzdata']) |
| 210 | |
| 211 | # test another resource which is not part of "all" |
| 212 | ns = libregrtest._parse_args([opt, 'extralargefile']) |
| 213 | self.assertEqual(ns.use_resources, ['extralargefile']) |
| 214 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 215 | def test_memlimit(self): |
| 216 | for opt in '-M', '--memlimit': |
| 217 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 218 | ns = libregrtest._parse_args([opt, '4G']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 219 | self.assertEqual(ns.memlimit, '4G') |
| 220 | self.checkError([opt], 'expected one argument') |
| 221 | |
| 222 | def test_testdir(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 223 | ns = libregrtest._parse_args(['--testdir', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 224 | self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) |
| 225 | self.checkError(['--testdir'], 'expected one argument') |
| 226 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 227 | def test_runleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 228 | for opt in '-L', '--runleaks': |
| 229 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 230 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 231 | self.assertTrue(ns.runleaks) |
| 232 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 233 | def test_huntrleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 234 | for opt in '-R', '--huntrleaks': |
| 235 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 236 | ns = libregrtest._parse_args([opt, ':']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 237 | self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 238 | ns = libregrtest._parse_args([opt, '6:']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 239 | self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 240 | ns = libregrtest._parse_args([opt, ':3']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 241 | self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 242 | ns = libregrtest._parse_args([opt, '6:3:leaks.log']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 243 | self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) |
| 244 | self.checkError([opt], 'expected one argument') |
| 245 | self.checkError([opt, '6'], |
| 246 | 'needs 2 or 3 colon-separated arguments') |
| 247 | self.checkError([opt, 'foo:'], 'invalid huntrleaks value') |
| 248 | self.checkError([opt, '6:foo'], 'invalid huntrleaks value') |
| 249 | |
| 250 | def test_multiprocess(self): |
| 251 | for opt in '-j', '--multiprocess': |
| 252 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 253 | ns = libregrtest._parse_args([opt, '2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 254 | self.assertEqual(ns.use_mp, 2) |
| 255 | self.checkError([opt], 'expected one argument') |
| 256 | self.checkError([opt, 'foo'], 'invalid int value') |
| 257 | self.checkError([opt, '2', '-T'], "don't go together") |
| 258 | self.checkError([opt, '2', '-l'], "don't go together") |
Xiang Zhang | 270a21f | 2016-12-19 21:01:33 +0800 | [diff] [blame] | 259 | self.checkError([opt, '0', '-T'], "don't go together") |
| 260 | self.checkError([opt, '0', '-l'], "don't go together") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 261 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 262 | def test_coverage(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 263 | for opt in '-T', '--coverage': |
| 264 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 265 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 266 | self.assertTrue(ns.trace) |
| 267 | |
| 268 | def test_coverdir(self): |
| 269 | for opt in '-D', '--coverdir': |
| 270 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 271 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 272 | self.assertEqual(ns.coverdir, |
| 273 | os.path.join(support.SAVEDCWD, 'foo')) |
| 274 | self.checkError([opt], 'expected one argument') |
| 275 | |
| 276 | def test_nocoverdir(self): |
| 277 | for opt in '-N', '--nocoverdir': |
| 278 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 279 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 280 | self.assertIsNone(ns.coverdir) |
| 281 | |
| 282 | def test_threshold(self): |
| 283 | for opt in '-t', '--threshold': |
| 284 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 285 | ns = libregrtest._parse_args([opt, '1000']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 286 | self.assertEqual(ns.threshold, 1000) |
| 287 | self.checkError([opt], 'expected one argument') |
| 288 | self.checkError([opt, 'foo'], 'invalid int value') |
| 289 | |
| 290 | def test_nowindows(self): |
| 291 | for opt in '-n', '--nowindows': |
| 292 | with self.subTest(opt=opt): |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 293 | with contextlib.redirect_stderr(io.StringIO()) as stderr: |
| 294 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 295 | self.assertTrue(ns.nowindows) |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 296 | err = stderr.getvalue() |
| 297 | self.assertIn('the --nowindows (-n) option is deprecated', err) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 298 | |
| 299 | def test_forever(self): |
| 300 | for opt in '-F', '--forever': |
| 301 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 302 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 303 | self.assertTrue(ns.forever) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 304 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 305 | def test_unrecognized_argument(self): |
| 306 | self.checkError(['--xxx'], 'usage:') |
| 307 | |
| 308 | def test_long_option__partial(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 309 | ns = libregrtest._parse_args(['--qui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 310 | self.assertTrue(ns.quiet) |
| 311 | self.assertEqual(ns.verbose, 0) |
| 312 | |
| 313 | def test_two_options(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 314 | ns = libregrtest._parse_args(['--quiet', '--exclude']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 315 | self.assertTrue(ns.quiet) |
| 316 | self.assertEqual(ns.verbose, 0) |
| 317 | self.assertTrue(ns.exclude) |
| 318 | |
| 319 | def test_option_with_empty_string_value(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 320 | ns = libregrtest._parse_args(['--start', '']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 321 | self.assertEqual(ns.start, '') |
| 322 | |
| 323 | def test_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 324 | ns = libregrtest._parse_args(['foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 325 | self.assertEqual(ns.args, ['foo']) |
| 326 | |
| 327 | def test_option_and_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 328 | ns = libregrtest._parse_args(['--quiet', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 329 | self.assertTrue(ns.quiet) |
| 330 | self.assertEqual(ns.verbose, 0) |
| 331 | self.assertEqual(ns.args, ['foo']) |
| 332 | |
Victor Stinner | 1f6b69b | 2016-10-17 18:11:03 +0200 | [diff] [blame] | 333 | def test_arg_option_arg(self): |
Victor Stinner | a506a93 | 2016-10-17 18:13:46 +0200 | [diff] [blame] | 334 | ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop']) |
Victor Stinner | 1f6b69b | 2016-10-17 18:11:03 +0200 | [diff] [blame] | 335 | self.assertEqual(ns.verbose, 1) |
| 336 | self.assertEqual(ns.args, ['test_unaryop', 'test_binop']) |
| 337 | |
| 338 | def test_unknown_option(self): |
| 339 | self.checkError(['--unknown-option'], |
| 340 | 'unrecognized arguments: --unknown-option') |
| 341 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 342 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 343 | class BaseTestCase(unittest.TestCase): |
| 344 | TEST_UNIQUE_ID = 1 |
| 345 | TESTNAME_PREFIX = 'test_regrtest_' |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 346 | TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 347 | |
| 348 | def setUp(self): |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 349 | self.testdir = os.path.realpath(os.path.dirname(__file__)) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 350 | |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 351 | self.tmptestdir = tempfile.mkdtemp() |
| 352 | self.addCleanup(support.rmtree, self.tmptestdir) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 353 | |
| 354 | def create_test(self, name=None, code=''): |
| 355 | if not name: |
| 356 | name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID |
| 357 | BaseTestCase.TEST_UNIQUE_ID += 1 |
| 358 | |
| 359 | # test_regrtest cannot be run twice in parallel because |
| 360 | # of setUp() and create_test() |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 361 | name = self.TESTNAME_PREFIX + name |
| 362 | path = os.path.join(self.tmptestdir, name + '.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 363 | |
| 364 | self.addCleanup(support.unlink, path) |
| 365 | # Use 'x' mode to ensure that we do not override existing tests |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 366 | try: |
| 367 | with open(path, 'x', encoding='utf-8') as fp: |
| 368 | fp.write(code) |
| 369 | except PermissionError as exc: |
| 370 | if not sysconfig.is_python_build(): |
| 371 | self.skipTest("cannot write %s: %s" % (path, exc)) |
| 372 | raise |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 373 | return name |
| 374 | |
| 375 | def regex_search(self, regex, output): |
| 376 | match = re.search(regex, output, re.MULTILINE) |
| 377 | if not match: |
| 378 | self.fail("%r not found in %r" % (regex, output)) |
| 379 | return match |
| 380 | |
| 381 | def check_line(self, output, regex): |
| 382 | regex = re.compile(r'^' + regex, re.MULTILINE) |
| 383 | self.assertRegex(output, regex) |
| 384 | |
| 385 | def parse_executed_tests(self, output): |
Victor Stinner | 3d00568 | 2017-05-04 15:21:12 +0200 | [diff] [blame] | 386 | regex = (r'^[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 387 | % self.TESTNAME_REGEX) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 388 | parser = re.finditer(regex, output, re.MULTILINE) |
| 389 | return list(match.group(1) for match in parser) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 390 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 391 | def check_executed_tests(self, output, tests, skipped=(), failed=(), |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 392 | env_changed=(), omitted=(), |
| 393 | randomize=False, interrupted=False, |
| 394 | fail_env_changed=False): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 395 | if isinstance(tests, str): |
| 396 | tests = [tests] |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 397 | if isinstance(skipped, str): |
| 398 | skipped = [skipped] |
| 399 | if isinstance(failed, str): |
| 400 | failed = [failed] |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 401 | if isinstance(env_changed, str): |
| 402 | env_changed = [env_changed] |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 403 | if isinstance(omitted, str): |
| 404 | omitted = [omitted] |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 405 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 406 | executed = self.parse_executed_tests(output) |
| 407 | if randomize: |
| 408 | self.assertEqual(set(executed), set(tests), output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 409 | else: |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 410 | self.assertEqual(executed, tests, output) |
| 411 | |
| 412 | def plural(count): |
| 413 | return 's' if count != 1 else '' |
| 414 | |
| 415 | def list_regex(line_format, tests): |
| 416 | count = len(tests) |
| 417 | names = ' '.join(sorted(tests)) |
| 418 | regex = line_format % (count, plural(count)) |
| 419 | regex = r'%s:\n %s$' % (regex, names) |
| 420 | return regex |
| 421 | |
| 422 | if skipped: |
| 423 | regex = list_regex('%s test%s skipped', skipped) |
| 424 | self.check_line(output, regex) |
| 425 | |
| 426 | if failed: |
| 427 | regex = list_regex('%s test%s failed', failed) |
| 428 | self.check_line(output, regex) |
| 429 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 430 | if env_changed: |
| 431 | regex = list_regex('%s test%s altered the execution environment', |
| 432 | env_changed) |
| 433 | self.check_line(output, regex) |
| 434 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 435 | if omitted: |
| 436 | regex = list_regex('%s test%s omitted', omitted) |
| 437 | self.check_line(output, regex) |
| 438 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 439 | good = (len(tests) - len(skipped) - len(failed) |
| 440 | - len(omitted) - len(env_changed)) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 441 | if good: |
| 442 | regex = r'%s test%s OK\.$' % (good, plural(good)) |
| 443 | if not skipped and not failed and good > 1: |
| 444 | regex = 'All %s' % regex |
| 445 | self.check_line(output, regex) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 446 | |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 447 | if interrupted: |
| 448 | self.check_line(output, 'Test suite interrupted by signal SIGINT.') |
| 449 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 450 | if failed: |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 451 | result = 'FAILURE' |
| 452 | elif interrupted: |
| 453 | result = 'INTERRUPTED' |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 454 | elif fail_env_changed and env_changed: |
| 455 | result = 'ENV CHANGED' |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 456 | else: |
| 457 | result = 'SUCCESS' |
Victor Stinner | f2abf5c | 2016-08-19 17:54:25 +0200 | [diff] [blame] | 458 | self.check_line(output, 'Tests result: %s' % result) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 459 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 460 | def parse_random_seed(self, output): |
| 461 | match = self.regex_search(r'Using random seed ([0-9]+)', output) |
| 462 | randseed = int(match.group(1)) |
| 463 | self.assertTrue(0 <= randseed <= 10000000, randseed) |
| 464 | return randseed |
| 465 | |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 466 | def run_command(self, args, input=None, exitcode=0, **kw): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 467 | if not input: |
| 468 | input = '' |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 469 | if 'stderr' not in kw: |
| 470 | kw['stderr'] = subprocess.PIPE |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 471 | proc = subprocess.run(args, |
| 472 | universal_newlines=True, |
| 473 | input=input, |
| 474 | stdout=subprocess.PIPE, |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 475 | **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 476 | if proc.returncode != exitcode: |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 477 | msg = ("Command %s failed with exit code %s\n" |
| 478 | "\n" |
| 479 | "stdout:\n" |
| 480 | "---\n" |
| 481 | "%s\n" |
| 482 | "---\n" |
| 483 | % (str(args), proc.returncode, proc.stdout)) |
| 484 | if proc.stderr: |
| 485 | msg += ("\n" |
| 486 | "stderr:\n" |
| 487 | "---\n" |
| 488 | "%s" |
| 489 | "---\n" |
| 490 | % proc.stderr) |
| 491 | self.fail(msg) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 492 | return proc |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 493 | |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 494 | def run_python(self, args, **kw): |
| 495 | args = [sys.executable, '-X', 'faulthandler', '-I', *args] |
| 496 | proc = self.run_command(args, **kw) |
| 497 | return proc.stdout |
| 498 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 499 | |
| 500 | class ProgramsTestCase(BaseTestCase): |
| 501 | """ |
| 502 | Test various ways to run the Python test suite. Use options close |
| 503 | to options used on the buildbot. |
| 504 | """ |
| 505 | |
| 506 | NTEST = 4 |
| 507 | |
| 508 | def setUp(self): |
| 509 | super().setUp() |
| 510 | |
| 511 | # Create NTEST tests doing nothing |
| 512 | self.tests = [self.create_test() for index in range(self.NTEST)] |
| 513 | |
| 514 | self.python_args = ['-Wd', '-E', '-bb'] |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 515 | self.regrtest_args = ['-uall', '-rwW', |
| 516 | '--testdir=%s' % self.tmptestdir] |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 517 | if hasattr(faulthandler, 'dump_traceback_later'): |
| 518 | self.regrtest_args.extend(('--timeout', '3600', '-j4')) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 519 | if sys.platform == 'win32': |
| 520 | self.regrtest_args.append('-n') |
| 521 | |
| 522 | def check_output(self, output): |
| 523 | self.parse_random_seed(output) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 524 | self.check_executed_tests(output, self.tests, randomize=True) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 525 | |
| 526 | def run_tests(self, args): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 527 | output = self.run_python(args) |
| 528 | self.check_output(output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 529 | |
| 530 | def test_script_regrtest(self): |
| 531 | # Lib/test/regrtest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 532 | script = os.path.join(self.testdir, 'regrtest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 533 | |
| 534 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 535 | self.run_tests(args) |
| 536 | |
| 537 | def test_module_test(self): |
| 538 | # -m test |
| 539 | args = [*self.python_args, '-m', 'test', |
| 540 | *self.regrtest_args, *self.tests] |
| 541 | self.run_tests(args) |
| 542 | |
| 543 | def test_module_regrtest(self): |
| 544 | # -m test.regrtest |
| 545 | args = [*self.python_args, '-m', 'test.regrtest', |
| 546 | *self.regrtest_args, *self.tests] |
| 547 | self.run_tests(args) |
| 548 | |
| 549 | def test_module_autotest(self): |
| 550 | # -m test.autotest |
| 551 | args = [*self.python_args, '-m', 'test.autotest', |
| 552 | *self.regrtest_args, *self.tests] |
| 553 | self.run_tests(args) |
| 554 | |
| 555 | def test_module_from_test_autotest(self): |
| 556 | # from test import autotest |
| 557 | code = 'from test import autotest' |
| 558 | args = [*self.python_args, '-c', code, |
| 559 | *self.regrtest_args, *self.tests] |
| 560 | self.run_tests(args) |
| 561 | |
| 562 | def test_script_autotest(self): |
| 563 | # Lib/test/autotest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 564 | script = os.path.join(self.testdir, 'autotest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 565 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 566 | self.run_tests(args) |
| 567 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 568 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 569 | 'run_tests.py script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 570 | def test_tools_script_run_tests(self): |
| 571 | # Tools/scripts/run_tests.py |
| 572 | script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') |
Victor Stinner | 8a34d41 | 2016-03-30 08:51:15 +0200 | [diff] [blame] | 573 | args = [script, *self.regrtest_args, *self.tests] |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 574 | self.run_tests(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 575 | |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 576 | def run_batch(self, *args): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 577 | proc = self.run_command(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 578 | self.check_output(proc.stdout) |
| 579 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 580 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 581 | 'test.bat script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 582 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 583 | def test_tools_buildbot_test(self): |
| 584 | # Tools\buildbot\test.bat |
| 585 | script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') |
Victor Stinner | 732599f | 2016-03-30 08:38:05 +0200 | [diff] [blame] | 586 | test_args = ['--testdir=%s' % self.tmptestdir] |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 587 | if platform.architecture()[0] == '64bit': |
| 588 | test_args.append('-x64') # 64-bit build |
| 589 | if not Py_DEBUG: |
| 590 | test_args.append('+d') # Release build, use python.exe |
| 591 | self.run_batch(script, *test_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 592 | |
| 593 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 594 | def test_pcbuild_rt(self): |
| 595 | # PCbuild\rt.bat |
| 596 | script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 597 | rt_args = ["-q"] # Quick, don't run tests twice |
| 598 | if platform.architecture()[0] == '64bit': |
| 599 | rt_args.append('-x64') # 64-bit build |
| 600 | if Py_DEBUG: |
| 601 | rt_args.append('-d') # Debug build, use python_d.exe |
| 602 | self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 603 | |
| 604 | |
| 605 | class ArgsTestCase(BaseTestCase): |
| 606 | """ |
| 607 | Test arguments of the Python test suite. |
| 608 | """ |
| 609 | |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 610 | def run_tests(self, *testargs, **kw): |
| 611 | cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] |
| 612 | return self.run_python(cmdargs, **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 613 | |
| 614 | def test_failing_test(self): |
| 615 | # test a failing test |
| 616 | code = textwrap.dedent(""" |
| 617 | import unittest |
| 618 | |
| 619 | class FailingTest(unittest.TestCase): |
| 620 | def test_failing(self): |
| 621 | self.fail("bug") |
| 622 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 623 | test_ok = self.create_test('ok') |
| 624 | test_failing = self.create_test('failing', code=code) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 625 | tests = [test_ok, test_failing] |
| 626 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 627 | output = self.run_tests(*tests, exitcode=2) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 628 | self.check_executed_tests(output, tests, failed=test_failing) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 629 | |
| 630 | def test_resources(self): |
| 631 | # test -u command line option |
| 632 | tests = {} |
| 633 | for resource in ('audio', 'network'): |
| 634 | code = 'from test import support\nsupport.requires(%r)' % resource |
| 635 | tests[resource] = self.create_test(resource, code) |
| 636 | test_names = sorted(tests.values()) |
| 637 | |
| 638 | # -u all: 2 resources enabled |
| 639 | output = self.run_tests('-u', 'all', *test_names) |
| 640 | self.check_executed_tests(output, test_names) |
| 641 | |
| 642 | # -u audio: 1 resource enabled |
| 643 | output = self.run_tests('-uaudio', *test_names) |
| 644 | self.check_executed_tests(output, test_names, |
| 645 | skipped=tests['network']) |
| 646 | |
| 647 | # no option: 0 resources enabled |
| 648 | output = self.run_tests(*test_names) |
| 649 | self.check_executed_tests(output, test_names, |
| 650 | skipped=test_names) |
| 651 | |
| 652 | def test_random(self): |
| 653 | # test -r and --randseed command line option |
| 654 | code = textwrap.dedent(""" |
| 655 | import random |
| 656 | print("TESTRANDOM: %s" % random.randint(1, 1000)) |
| 657 | """) |
| 658 | test = self.create_test('random', code) |
| 659 | |
| 660 | # first run to get the output with the random seed |
| 661 | output = self.run_tests('-r', test) |
| 662 | randseed = self.parse_random_seed(output) |
| 663 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 664 | test_random = int(match.group(1)) |
| 665 | |
| 666 | # try to reproduce with the random seed |
| 667 | output = self.run_tests('-r', '--randseed=%s' % randseed, test) |
| 668 | randseed2 = self.parse_random_seed(output) |
| 669 | self.assertEqual(randseed2, randseed) |
| 670 | |
| 671 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 672 | test_random2 = int(match.group(1)) |
| 673 | self.assertEqual(test_random2, test_random) |
| 674 | |
| 675 | def test_fromfile(self): |
| 676 | # test --fromfile |
| 677 | tests = [self.create_test() for index in range(5)] |
| 678 | |
| 679 | # Write the list of files using a format similar to regrtest output: |
| 680 | # [1/2] test_1 |
| 681 | # [2/2] test_2 |
| 682 | filename = support.TESTFN |
| 683 | self.addCleanup(support.unlink, filename) |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 684 | |
| 685 | # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec' |
| 686 | with open(filename, "w") as fp: |
| 687 | previous = None |
| 688 | for index, name in enumerate(tests, 1): |
| 689 | line = ("00:00:%02i [%s/%s] %s" |
| 690 | % (index, index, len(tests), name)) |
| 691 | if previous: |
| 692 | line += " -- %s took 0 sec" % previous |
| 693 | print(line, file=fp) |
| 694 | previous = name |
| 695 | |
| 696 | output = self.run_tests('--fromfile', filename) |
| 697 | self.check_executed_tests(output, tests) |
| 698 | |
| 699 | # test format '[2/7] test_opcodes' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 700 | with open(filename, "w") as fp: |
| 701 | for index, name in enumerate(tests, 1): |
| 702 | print("[%s/%s] %s" % (index, len(tests), name), file=fp) |
| 703 | |
| 704 | output = self.run_tests('--fromfile', filename) |
| 705 | self.check_executed_tests(output, tests) |
| 706 | |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 707 | # test format 'test_opcodes' |
| 708 | with open(filename, "w") as fp: |
| 709 | for name in tests: |
| 710 | print(name, file=fp) |
| 711 | |
| 712 | output = self.run_tests('--fromfile', filename) |
| 713 | self.check_executed_tests(output, tests) |
| 714 | |
Victor Stinner | e8cfec5 | 2017-01-03 01:38:58 +0100 | [diff] [blame] | 715 | # test format 'Lib/test/test_opcodes.py' |
| 716 | with open(filename, "w") as fp: |
| 717 | for name in tests: |
| 718 | print('Lib/test/%s.py' % name, file=fp) |
| 719 | |
| 720 | output = self.run_tests('--fromfile', filename) |
| 721 | self.check_executed_tests(output, tests) |
| 722 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 723 | def test_interrupted(self): |
| 724 | code = TEST_INTERRUPTED |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 725 | test = self.create_test('sigint', code=code) |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 726 | output = self.run_tests(test, exitcode=130) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 727 | self.check_executed_tests(output, test, omitted=test, |
| 728 | interrupted=True) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 729 | |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 730 | def test_slowest(self): |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 731 | # test --slowest |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 732 | tests = [self.create_test() for index in range(3)] |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 733 | output = self.run_tests("--slowest", *tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 734 | self.check_executed_tests(output, tests) |
| 735 | regex = ('10 slowest tests:\n' |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 736 | '(?:- %s: .*\n){%s}' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 737 | % (self.TESTNAME_REGEX, len(tests))) |
| 738 | self.check_line(output, regex) |
| 739 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 740 | def test_slow_interrupted(self): |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 741 | # Issue #25373: test --slowest with an interrupted test |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 742 | code = TEST_INTERRUPTED |
| 743 | test = self.create_test("sigint", code=code) |
| 744 | |
Antoine Pitrou | a6a4dc8 | 2017-09-07 18:56:24 +0200 | [diff] [blame] | 745 | for multiprocessing in (False, True): |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 746 | if multiprocessing: |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 747 | args = ("--slowest", "-j2", test) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 748 | else: |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 749 | args = ("--slowest", test) |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 750 | output = self.run_tests(*args, exitcode=130) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 751 | self.check_executed_tests(output, test, |
| 752 | omitted=test, interrupted=True) |
| 753 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 754 | regex = ('10 slowest tests:\n') |
| 755 | self.check_line(output, regex) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 756 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 757 | def test_coverage(self): |
| 758 | # test --coverage |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 759 | test = self.create_test('coverage') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 760 | output = self.run_tests("--coverage", test) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 761 | self.check_executed_tests(output, [test]) |
R David Murray | 44b548d | 2016-09-08 13:59:53 -0400 | [diff] [blame] | 762 | regex = (r'lines +cov% +module +\(path\)\n' |
| 763 | r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 764 | self.check_line(output, regex) |
| 765 | |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 766 | def test_wait(self): |
| 767 | # test --wait |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 768 | test = self.create_test('wait') |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 769 | output = self.run_tests("--wait", test, input='key') |
| 770 | self.check_line(output, 'Press any key to continue') |
| 771 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 772 | def test_forever(self): |
| 773 | # test --forever |
| 774 | code = textwrap.dedent(""" |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 775 | import builtins |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 776 | import unittest |
| 777 | |
| 778 | class ForeverTester(unittest.TestCase): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 779 | def test_run(self): |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 780 | # Store the state in the builtins module, because the test |
| 781 | # module is reload at each run |
| 782 | if 'RUN' in builtins.__dict__: |
| 783 | builtins.__dict__['RUN'] += 1 |
| 784 | if builtins.__dict__['RUN'] >= 3: |
| 785 | self.fail("fail at the 3rd runs") |
| 786 | else: |
| 787 | builtins.__dict__['RUN'] = 1 |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 788 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 789 | test = self.create_test('forever', code=code) |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 790 | output = self.run_tests('--forever', test, exitcode=2) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 791 | self.check_executed_tests(output, [test]*3, failed=test) |
| 792 | |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 793 | def check_leak(self, code, what): |
| 794 | test = self.create_test('huntrleaks', code=code) |
| 795 | |
| 796 | filename = 'reflog.txt' |
| 797 | self.addCleanup(support.unlink, filename) |
| 798 | output = self.run_tests('--huntrleaks', '3:3:', test, |
| 799 | exitcode=2, |
| 800 | stderr=subprocess.STDOUT) |
| 801 | self.check_executed_tests(output, [test], failed=test) |
| 802 | |
| 803 | line = 'beginning 6 repetitions\n123456\n......\n' |
| 804 | self.check_line(output, re.escape(line)) |
| 805 | |
| 806 | line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) |
| 807 | self.assertIn(line2, output) |
| 808 | |
| 809 | with open(filename) as fp: |
| 810 | reflog = fp.read() |
| 811 | self.assertIn(line2, reflog) |
| 812 | |
| 813 | @unittest.skipUnless(Py_DEBUG, 'need a debug build') |
| 814 | def test_huntrleaks(self): |
| 815 | # test --huntrleaks |
| 816 | code = textwrap.dedent(""" |
| 817 | import unittest |
| 818 | |
| 819 | GLOBAL_LIST = [] |
| 820 | |
| 821 | class RefLeakTest(unittest.TestCase): |
| 822 | def test_leak(self): |
| 823 | GLOBAL_LIST.append(object()) |
| 824 | """) |
| 825 | self.check_leak(code, 'references') |
| 826 | |
Victor Stinner | 46d7535 | 2015-10-03 02:21:35 +0200 | [diff] [blame] | 827 | @unittest.skipUnless(Py_DEBUG, 'need a debug build') |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 828 | def test_huntrleaks_fd_leak(self): |
| 829 | # test --huntrleaks for file descriptor leak |
| 830 | code = textwrap.dedent(""" |
| 831 | import os |
| 832 | import unittest |
| 833 | |
| 834 | class FDLeakTest(unittest.TestCase): |
| 835 | def test_leak(self): |
| 836 | fd = os.open(__file__, os.O_RDONLY) |
Victor Stinner | 302bbbe | 2017-07-25 17:45:53 +0200 | [diff] [blame] | 837 | # bug: never close the file descriptor |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 838 | """) |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 839 | self.check_leak(code, 'file descriptors') |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 840 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 841 | def test_list_tests(self): |
| 842 | # test --list-tests |
| 843 | tests = [self.create_test() for i in range(5)] |
| 844 | output = self.run_tests('--list-tests', *tests) |
| 845 | self.assertEqual(output.rstrip().splitlines(), |
| 846 | tests) |
| 847 | |
Louie Lu | b0c58d3 | 2017-06-16 19:15:28 +0800 | [diff] [blame] | 848 | def test_list_cases(self): |
| 849 | # test --list-cases |
| 850 | code = textwrap.dedent(""" |
| 851 | import unittest |
| 852 | |
| 853 | class Tests(unittest.TestCase): |
| 854 | def test_method1(self): |
| 855 | pass |
| 856 | def test_method2(self): |
| 857 | pass |
| 858 | """) |
| 859 | testname = self.create_test(code=code) |
Victor Stinner | ace56d5 | 2017-06-26 14:18:51 +0200 | [diff] [blame] | 860 | |
| 861 | # Test --list-cases |
Louie Lu | b0c58d3 | 2017-06-16 19:15:28 +0800 | [diff] [blame] | 862 | all_methods = ['%s.Tests.test_method1' % testname, |
| 863 | '%s.Tests.test_method2' % testname] |
| 864 | output = self.run_tests('--list-cases', testname) |
| 865 | self.assertEqual(output.splitlines(), all_methods) |
| 866 | |
Victor Stinner | ace56d5 | 2017-06-26 14:18:51 +0200 | [diff] [blame] | 867 | # Test --list-cases with --match |
| 868 | all_methods = ['%s.Tests.test_method1' % testname] |
| 869 | output = self.run_tests('--list-cases', |
| 870 | '-m', 'test_method1', |
| 871 | testname) |
| 872 | self.assertEqual(output.splitlines(), all_methods) |
| 873 | |
Victor Stinner | 5bad70d | 2017-02-06 12:42:00 +0100 | [diff] [blame] | 874 | def test_crashed(self): |
| 875 | # Any code which causes a crash |
| 876 | code = 'import faulthandler; faulthandler._sigsegv()' |
| 877 | crash_test = self.create_test(name="crash", code=code) |
| 878 | ok_test = self.create_test(name="ok") |
| 879 | |
| 880 | tests = [crash_test, ok_test] |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 881 | output = self.run_tests("-j2", *tests, exitcode=2) |
Victor Stinner | 5bad70d | 2017-02-06 12:42:00 +0100 | [diff] [blame] | 882 | self.check_executed_tests(output, tests, failed=crash_test, |
| 883 | randomize=True) |
| 884 | |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 885 | def parse_methods(self, output): |
| 886 | regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) |
| 887 | return [match.group(1) for match in regex.finditer(output)] |
| 888 | |
| 889 | def test_matchfile(self): |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 890 | code = textwrap.dedent(""" |
| 891 | import unittest |
| 892 | |
| 893 | class Tests(unittest.TestCase): |
| 894 | def test_method1(self): |
| 895 | pass |
| 896 | def test_method2(self): |
| 897 | pass |
| 898 | def test_method3(self): |
| 899 | pass |
| 900 | def test_method4(self): |
| 901 | pass |
| 902 | """) |
| 903 | all_methods = ['test_method1', 'test_method2', |
| 904 | 'test_method3', 'test_method4'] |
| 905 | testname = self.create_test(code=code) |
| 906 | |
| 907 | # by default, all methods should be run |
| 908 | output = self.run_tests("-v", testname) |
| 909 | methods = self.parse_methods(output) |
| 910 | self.assertEqual(methods, all_methods) |
| 911 | |
| 912 | # only run a subset |
| 913 | filename = support.TESTFN |
| 914 | self.addCleanup(support.unlink, filename) |
| 915 | |
| 916 | subset = [ |
| 917 | # only match the method name |
| 918 | 'test_method1', |
| 919 | # match the full identifier |
| 920 | '%s.Tests.test_method3' % testname] |
| 921 | with open(filename, "w") as fp: |
| 922 | for name in subset: |
| 923 | print(name, file=fp) |
| 924 | |
| 925 | output = self.run_tests("-v", "--matchfile", filename, testname) |
| 926 | methods = self.parse_methods(output) |
| 927 | subset = ['test_method1', 'test_method3'] |
| 928 | self.assertEqual(methods, subset) |
| 929 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 930 | def test_env_changed(self): |
| 931 | code = textwrap.dedent(""" |
| 932 | import unittest |
| 933 | |
| 934 | class Tests(unittest.TestCase): |
| 935 | def test_env_changed(self): |
| 936 | open("env_changed", "w").close() |
| 937 | """) |
| 938 | testname = self.create_test(code=code) |
| 939 | |
| 940 | # don't fail by default |
| 941 | output = self.run_tests(testname) |
| 942 | self.check_executed_tests(output, [testname], env_changed=testname) |
| 943 | |
| 944 | # fail with --fail-env-changed |
| 945 | output = self.run_tests("--fail-env-changed", testname, exitcode=3) |
| 946 | self.check_executed_tests(output, [testname], env_changed=testname, |
| 947 | fail_env_changed=True) |
| 948 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 949 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 950 | if __name__ == '__main__': |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 951 | unittest.main() |