Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 1 | """ |
| 2 | Tests of regrtest.py. |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 3 | |
| 4 | Note: test_regrtest cannot be run twice in parallel. |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 5 | """ |
| 6 | |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 7 | import contextlib |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 8 | import faulthandler |
Miss Islington (bot) | 9d54de1 | 2019-07-23 23:30:27 -0700 | [diff] [blame] | 9 | import glob |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 10 | import io |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 11 | import os.path |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 12 | import platform |
| 13 | import re |
| 14 | import subprocess |
| 15 | import sys |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 16 | import sysconfig |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 17 | import tempfile |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 18 | import textwrap |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 19 | import unittest |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 20 | from test import libregrtest |
| 21 | from test import support |
Victor Stinner | 4ffe9c2 | 2018-06-14 14:58:13 +0200 | [diff] [blame] | 22 | from test.libregrtest import utils |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 23 | |
| 24 | |
Victor Stinner | 235e7b2 | 2019-04-25 01:46:31 +0200 | [diff] [blame] | 25 | Py_DEBUG = hasattr(sys, 'gettotalrefcount') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 26 | ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') |
| 27 | ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) |
Victor Stinner | de3195c | 2019-10-03 17:26:25 +0200 | [diff] [blame] | 28 | LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 29 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 30 | TEST_INTERRUPTED = textwrap.dedent(""" |
Victor Stinner | 75120d2 | 2019-04-26 09:28:53 +0200 | [diff] [blame] | 31 | from signal import SIGINT, raise_signal |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 32 | try: |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 33 | raise_signal(SIGINT) |
| 34 | except ImportError: |
| 35 | import os |
| 36 | os.kill(os.getpid(), SIGINT) |
| 37 | """) |
| 38 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 39 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 40 | class ParseArgsTestCase(unittest.TestCase): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 41 | """ |
| 42 | Test regrtest's argument parsing, function _parse_args(). |
| 43 | """ |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 44 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 45 | def checkError(self, args, msg): |
| 46 | with support.captured_stderr() as err, self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 47 | libregrtest._parse_args(args) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 48 | self.assertIn(msg, err.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 49 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 50 | def test_help(self): |
| 51 | for opt in '-h', '--help': |
| 52 | with self.subTest(opt=opt): |
| 53 | with support.captured_stdout() as out, \ |
| 54 | self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 55 | libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 56 | self.assertIn('Run Python regression tests.', out.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 57 | |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 58 | @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), |
| 59 | "faulthandler.dump_traceback_later() required") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 60 | def test_timeout(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 61 | ns = libregrtest._parse_args(['--timeout', '4.2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 62 | self.assertEqual(ns.timeout, 4.2) |
| 63 | self.checkError(['--timeout'], 'expected one argument') |
| 64 | self.checkError(['--timeout', 'foo'], 'invalid float value') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 65 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 66 | def test_wait(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 67 | ns = libregrtest._parse_args(['--wait']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 68 | self.assertTrue(ns.wait) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 69 | |
Victor Stinner | 012f5b9 | 2018-09-07 17:20:42 +0200 | [diff] [blame] | 70 | def test_worker_args(self): |
| 71 | ns = libregrtest._parse_args(['--worker-args', '[[], {}]']) |
| 72 | self.assertEqual(ns.worker_args, '[[], {}]') |
| 73 | self.checkError(['--worker-args'], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 74 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 75 | def test_start(self): |
| 76 | for opt in '-S', '--start': |
| 77 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 78 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 79 | self.assertEqual(ns.start, 'foo') |
| 80 | self.checkError([opt], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 81 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 82 | def test_verbose(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 83 | ns = libregrtest._parse_args(['-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 84 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 85 | ns = libregrtest._parse_args(['-vvv']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 86 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 87 | ns = libregrtest._parse_args(['--verbose']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 88 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 89 | ns = libregrtest._parse_args(['--verbose'] * 3) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 90 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 91 | ns = libregrtest._parse_args([]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 92 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 93 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 94 | def test_verbose2(self): |
| 95 | for opt in '-w', '--verbose2': |
| 96 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 97 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 98 | self.assertTrue(ns.verbose2) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 99 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 100 | def test_verbose3(self): |
| 101 | for opt in '-W', '--verbose3': |
| 102 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 103 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 104 | self.assertTrue(ns.verbose3) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 105 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 106 | def test_quiet(self): |
| 107 | for opt in '-q', '--quiet': |
| 108 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 109 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 110 | self.assertTrue(ns.quiet) |
| 111 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 112 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 113 | def test_slowest(self): |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 114 | for opt in '-o', '--slowest': |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 115 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 116 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 117 | self.assertTrue(ns.print_slow) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 118 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 119 | def test_header(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 120 | ns = libregrtest._parse_args(['--header']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 121 | self.assertTrue(ns.header) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 122 | |
Victor Stinner | 3d00568 | 2017-05-04 15:21:12 +0200 | [diff] [blame] | 123 | ns = libregrtest._parse_args(['--verbose']) |
| 124 | self.assertTrue(ns.header) |
| 125 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 126 | def test_randomize(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 127 | for opt in '-r', '--randomize': |
| 128 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 129 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 130 | self.assertTrue(ns.randomize) |
| 131 | |
| 132 | def test_randseed(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 133 | ns = libregrtest._parse_args(['--randseed', '12345']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 134 | self.assertEqual(ns.random_seed, 12345) |
| 135 | self.assertTrue(ns.randomize) |
| 136 | self.checkError(['--randseed'], 'expected one argument') |
| 137 | self.checkError(['--randseed', 'foo'], 'invalid int value') |
| 138 | |
| 139 | def test_fromfile(self): |
| 140 | for opt in '-f', '--fromfile': |
| 141 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 142 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 143 | self.assertEqual(ns.fromfile, 'foo') |
| 144 | self.checkError([opt], 'expected one argument') |
| 145 | self.checkError([opt, 'foo', '-s'], "don't go together") |
| 146 | |
| 147 | def test_exclude(self): |
| 148 | for opt in '-x', '--exclude': |
| 149 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 150 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 151 | self.assertTrue(ns.exclude) |
| 152 | |
| 153 | def test_single(self): |
| 154 | for opt in '-s', '--single': |
| 155 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 156 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 157 | self.assertTrue(ns.single) |
| 158 | self.checkError([opt, '-f', 'foo'], "don't go together") |
| 159 | |
| 160 | def test_match(self): |
| 161 | for opt in '-m', '--match': |
| 162 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 163 | ns = libregrtest._parse_args([opt, 'pattern']) |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 164 | self.assertEqual(ns.match_tests, ['pattern']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 165 | self.checkError([opt], 'expected one argument') |
| 166 | |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 167 | ns = libregrtest._parse_args(['-m', 'pattern1', |
| 168 | '-m', 'pattern2']) |
| 169 | self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) |
| 170 | |
| 171 | self.addCleanup(support.unlink, support.TESTFN) |
| 172 | with open(support.TESTFN, "w") as fp: |
| 173 | print('matchfile1', file=fp) |
| 174 | print('matchfile2', file=fp) |
| 175 | |
| 176 | filename = os.path.abspath(support.TESTFN) |
| 177 | ns = libregrtest._parse_args(['-m', 'match', |
| 178 | '--matchfile', filename]) |
| 179 | self.assertEqual(ns.match_tests, |
| 180 | ['match', 'matchfile1', 'matchfile2']) |
| 181 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 182 | def test_failfast(self): |
| 183 | for opt in '-G', '--failfast': |
| 184 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 185 | ns = libregrtest._parse_args([opt, '-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 186 | self.assertTrue(ns.failfast) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 187 | ns = libregrtest._parse_args([opt, '-W']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 188 | self.assertTrue(ns.failfast) |
| 189 | self.checkError([opt], '-G/--failfast needs either -v or -W') |
| 190 | |
| 191 | def test_use(self): |
| 192 | for opt in '-u', '--use': |
| 193 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 194 | ns = libregrtest._parse_args([opt, 'gui,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 195 | self.assertEqual(ns.use_resources, ['gui', 'network']) |
Victor Stinner | 5b392bb | 2017-07-20 15:46:32 +0200 | [diff] [blame] | 196 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 197 | ns = libregrtest._parse_args([opt, 'gui,none,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 198 | self.assertEqual(ns.use_resources, ['network']) |
Victor Stinner | 5b392bb | 2017-07-20 15:46:32 +0200 | [diff] [blame] | 199 | |
| 200 | expected = list(libregrtest.ALL_RESOURCES) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 201 | expected.remove('gui') |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 202 | ns = libregrtest._parse_args([opt, 'all,-gui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 203 | self.assertEqual(ns.use_resources, expected) |
| 204 | self.checkError([opt], 'expected one argument') |
| 205 | self.checkError([opt, 'foo'], 'invalid resource') |
| 206 | |
Victor Stinner | 5b392bb | 2017-07-20 15:46:32 +0200 | [diff] [blame] | 207 | # all + a resource not part of "all" |
| 208 | ns = libregrtest._parse_args([opt, 'all,tzdata']) |
| 209 | self.assertEqual(ns.use_resources, |
| 210 | list(libregrtest.ALL_RESOURCES) + ['tzdata']) |
| 211 | |
| 212 | # test another resource which is not part of "all" |
| 213 | ns = libregrtest._parse_args([opt, 'extralargefile']) |
| 214 | self.assertEqual(ns.use_resources, ['extralargefile']) |
| 215 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 216 | def test_memlimit(self): |
| 217 | for opt in '-M', '--memlimit': |
| 218 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 219 | ns = libregrtest._parse_args([opt, '4G']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 220 | self.assertEqual(ns.memlimit, '4G') |
| 221 | self.checkError([opt], 'expected one argument') |
| 222 | |
| 223 | def test_testdir(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 224 | ns = libregrtest._parse_args(['--testdir', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 225 | self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) |
| 226 | self.checkError(['--testdir'], 'expected one argument') |
| 227 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 228 | def test_runleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 229 | for opt in '-L', '--runleaks': |
| 230 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 231 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 232 | self.assertTrue(ns.runleaks) |
| 233 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 234 | def test_huntrleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 235 | for opt in '-R', '--huntrleaks': |
| 236 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 237 | ns = libregrtest._parse_args([opt, ':']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 238 | self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 239 | ns = libregrtest._parse_args([opt, '6:']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 240 | self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 241 | ns = libregrtest._parse_args([opt, ':3']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 242 | self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 243 | ns = libregrtest._parse_args([opt, '6:3:leaks.log']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 244 | self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) |
| 245 | self.checkError([opt], 'expected one argument') |
| 246 | self.checkError([opt, '6'], |
| 247 | 'needs 2 or 3 colon-separated arguments') |
| 248 | self.checkError([opt, 'foo:'], 'invalid huntrleaks value') |
| 249 | self.checkError([opt, '6:foo'], 'invalid huntrleaks value') |
| 250 | |
| 251 | def test_multiprocess(self): |
| 252 | for opt in '-j', '--multiprocess': |
| 253 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 254 | ns = libregrtest._parse_args([opt, '2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 255 | self.assertEqual(ns.use_mp, 2) |
| 256 | self.checkError([opt], 'expected one argument') |
| 257 | self.checkError([opt, 'foo'], 'invalid int value') |
| 258 | self.checkError([opt, '2', '-T'], "don't go together") |
Xiang Zhang | 270a21f | 2016-12-19 21:01:33 +0800 | [diff] [blame] | 259 | self.checkError([opt, '0', '-T'], "don't go together") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 260 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 261 | def test_coverage(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 262 | for opt in '-T', '--coverage': |
| 263 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 264 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 265 | self.assertTrue(ns.trace) |
| 266 | |
| 267 | def test_coverdir(self): |
| 268 | for opt in '-D', '--coverdir': |
| 269 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 270 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 271 | self.assertEqual(ns.coverdir, |
| 272 | os.path.join(support.SAVEDCWD, 'foo')) |
| 273 | self.checkError([opt], 'expected one argument') |
| 274 | |
| 275 | def test_nocoverdir(self): |
| 276 | for opt in '-N', '--nocoverdir': |
| 277 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 278 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 279 | self.assertIsNone(ns.coverdir) |
| 280 | |
| 281 | def test_threshold(self): |
| 282 | for opt in '-t', '--threshold': |
| 283 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 284 | ns = libregrtest._parse_args([opt, '1000']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 285 | self.assertEqual(ns.threshold, 1000) |
| 286 | self.checkError([opt], 'expected one argument') |
| 287 | self.checkError([opt, 'foo'], 'invalid int value') |
| 288 | |
| 289 | def test_nowindows(self): |
| 290 | for opt in '-n', '--nowindows': |
| 291 | with self.subTest(opt=opt): |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 292 | with contextlib.redirect_stderr(io.StringIO()) as stderr: |
| 293 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 294 | self.assertTrue(ns.nowindows) |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 295 | err = stderr.getvalue() |
| 296 | self.assertIn('the --nowindows (-n) option is deprecated', err) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 297 | |
| 298 | def test_forever(self): |
| 299 | for opt in '-F', '--forever': |
| 300 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 301 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 302 | self.assertTrue(ns.forever) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 303 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 304 | def test_unrecognized_argument(self): |
| 305 | self.checkError(['--xxx'], 'usage:') |
| 306 | |
| 307 | def test_long_option__partial(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 308 | ns = libregrtest._parse_args(['--qui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 309 | self.assertTrue(ns.quiet) |
| 310 | self.assertEqual(ns.verbose, 0) |
| 311 | |
| 312 | def test_two_options(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 313 | ns = libregrtest._parse_args(['--quiet', '--exclude']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 314 | self.assertTrue(ns.quiet) |
| 315 | self.assertEqual(ns.verbose, 0) |
| 316 | self.assertTrue(ns.exclude) |
| 317 | |
| 318 | def test_option_with_empty_string_value(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 319 | ns = libregrtest._parse_args(['--start', '']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 320 | self.assertEqual(ns.start, '') |
| 321 | |
| 322 | def test_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 323 | ns = libregrtest._parse_args(['foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 324 | self.assertEqual(ns.args, ['foo']) |
| 325 | |
| 326 | def test_option_and_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 327 | ns = libregrtest._parse_args(['--quiet', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 328 | self.assertTrue(ns.quiet) |
| 329 | self.assertEqual(ns.verbose, 0) |
| 330 | self.assertEqual(ns.args, ['foo']) |
| 331 | |
Victor Stinner | 1f6b69b | 2016-10-17 18:11:03 +0200 | [diff] [blame] | 332 | def test_arg_option_arg(self): |
Victor Stinner | a506a93 | 2016-10-17 18:13:46 +0200 | [diff] [blame] | 333 | ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop']) |
Victor Stinner | 1f6b69b | 2016-10-17 18:11:03 +0200 | [diff] [blame] | 334 | self.assertEqual(ns.verbose, 1) |
| 335 | self.assertEqual(ns.args, ['test_unaryop', 'test_binop']) |
| 336 | |
| 337 | def test_unknown_option(self): |
| 338 | self.checkError(['--unknown-option'], |
| 339 | 'unrecognized arguments: --unknown-option') |
| 340 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 341 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 342 | class BaseTestCase(unittest.TestCase): |
| 343 | TEST_UNIQUE_ID = 1 |
| 344 | TESTNAME_PREFIX = 'test_regrtest_' |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 345 | TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 346 | |
| 347 | def setUp(self): |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 348 | self.testdir = os.path.realpath(os.path.dirname(__file__)) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 349 | |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 350 | self.tmptestdir = tempfile.mkdtemp() |
| 351 | self.addCleanup(support.rmtree, self.tmptestdir) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 352 | |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 353 | def create_test(self, name=None, code=None): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 354 | if not name: |
| 355 | name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID |
| 356 | BaseTestCase.TEST_UNIQUE_ID += 1 |
| 357 | |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 358 | if code is None: |
| 359 | code = textwrap.dedent(""" |
| 360 | import unittest |
| 361 | |
| 362 | class Tests(unittest.TestCase): |
| 363 | def test_empty_test(self): |
| 364 | pass |
| 365 | """) |
| 366 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 367 | # test_regrtest cannot be run twice in parallel because |
| 368 | # of setUp() and create_test() |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 369 | name = self.TESTNAME_PREFIX + name |
| 370 | path = os.path.join(self.tmptestdir, name + '.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 371 | |
| 372 | self.addCleanup(support.unlink, path) |
| 373 | # Use 'x' mode to ensure that we do not override existing tests |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 374 | try: |
| 375 | with open(path, 'x', encoding='utf-8') as fp: |
| 376 | fp.write(code) |
| 377 | except PermissionError as exc: |
| 378 | if not sysconfig.is_python_build(): |
| 379 | self.skipTest("cannot write %s: %s" % (path, exc)) |
| 380 | raise |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 381 | return name |
| 382 | |
| 383 | def regex_search(self, regex, output): |
| 384 | match = re.search(regex, output, re.MULTILINE) |
| 385 | if not match: |
| 386 | self.fail("%r not found in %r" % (regex, output)) |
| 387 | return match |
| 388 | |
| 389 | def check_line(self, output, regex): |
| 390 | regex = re.compile(r'^' + regex, re.MULTILINE) |
| 391 | self.assertRegex(output, regex) |
| 392 | |
| 393 | def parse_executed_tests(self, output): |
Victor Stinner | de3195c | 2019-10-03 17:26:25 +0200 | [diff] [blame] | 394 | regex = (r'^%s\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' |
| 395 | % (LOG_PREFIX, self.TESTNAME_REGEX)) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 396 | parser = re.finditer(regex, output, re.MULTILINE) |
| 397 | return list(match.group(1) for match in parser) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 398 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 399 | def check_executed_tests(self, output, tests, skipped=(), failed=(), |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 400 | env_changed=(), omitted=(), |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 401 | rerun=(), no_test_ran=(), |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 402 | randomize=False, interrupted=False, |
| 403 | fail_env_changed=False): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 404 | if isinstance(tests, str): |
| 405 | tests = [tests] |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 406 | if isinstance(skipped, str): |
| 407 | skipped = [skipped] |
| 408 | if isinstance(failed, str): |
| 409 | failed = [failed] |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 410 | if isinstance(env_changed, str): |
| 411 | env_changed = [env_changed] |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 412 | if isinstance(omitted, str): |
| 413 | omitted = [omitted] |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 414 | if isinstance(rerun, str): |
| 415 | rerun = [rerun] |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 416 | if isinstance(no_test_ran, str): |
| 417 | no_test_ran = [no_test_ran] |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 418 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 419 | executed = self.parse_executed_tests(output) |
| 420 | if randomize: |
| 421 | self.assertEqual(set(executed), set(tests), output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 422 | else: |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 423 | self.assertEqual(executed, tests, output) |
| 424 | |
| 425 | def plural(count): |
| 426 | return 's' if count != 1 else '' |
| 427 | |
| 428 | def list_regex(line_format, tests): |
| 429 | count = len(tests) |
| 430 | names = ' '.join(sorted(tests)) |
| 431 | regex = line_format % (count, plural(count)) |
| 432 | regex = r'%s:\n %s$' % (regex, names) |
| 433 | return regex |
| 434 | |
| 435 | if skipped: |
| 436 | regex = list_regex('%s test%s skipped', skipped) |
| 437 | self.check_line(output, regex) |
| 438 | |
| 439 | if failed: |
| 440 | regex = list_regex('%s test%s failed', failed) |
| 441 | self.check_line(output, regex) |
| 442 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 443 | if env_changed: |
| 444 | regex = list_regex('%s test%s altered the execution environment', |
| 445 | env_changed) |
| 446 | self.check_line(output, regex) |
| 447 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 448 | if omitted: |
| 449 | regex = list_regex('%s test%s omitted', omitted) |
| 450 | self.check_line(output, regex) |
| 451 | |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 452 | if rerun: |
| 453 | regex = list_regex('%s re-run test%s', rerun) |
| 454 | self.check_line(output, regex) |
Victor Stinner | de3195c | 2019-10-03 17:26:25 +0200 | [diff] [blame] | 455 | regex = LOG_PREFIX + r"Re-running failed tests in verbose mode" |
| 456 | self.check_line(output, regex) |
Victor Stinner | 75120d2 | 2019-04-26 09:28:53 +0200 | [diff] [blame] | 457 | for test_name in rerun: |
Victor Stinner | de3195c | 2019-10-03 17:26:25 +0200 | [diff] [blame] | 458 | regex = LOG_PREFIX + f"Re-running {test_name} in verbose mode" |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 459 | self.check_line(output, regex) |
| 460 | |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 461 | if no_test_ran: |
| 462 | regex = list_regex('%s test%s run no tests', no_test_ran) |
| 463 | self.check_line(output, regex) |
| 464 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 465 | good = (len(tests) - len(skipped) - len(failed) |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 466 | - len(omitted) - len(env_changed) - len(no_test_ran)) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 467 | if good: |
| 468 | regex = r'%s test%s OK\.$' % (good, plural(good)) |
| 469 | if not skipped and not failed and good > 1: |
| 470 | regex = 'All %s' % regex |
| 471 | self.check_line(output, regex) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 472 | |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 473 | if interrupted: |
| 474 | self.check_line(output, 'Test suite interrupted by signal SIGINT.') |
| 475 | |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 476 | result = [] |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 477 | if failed: |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 478 | result.append('FAILURE') |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 479 | elif fail_env_changed and env_changed: |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 480 | result.append('ENV CHANGED') |
| 481 | if interrupted: |
| 482 | result.append('INTERRUPTED') |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 483 | if not any((good, result, failed, interrupted, skipped, |
| 484 | env_changed, fail_env_changed)): |
| 485 | result.append("NO TEST RUN") |
| 486 | elif not result: |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 487 | result.append('SUCCESS') |
| 488 | result = ', '.join(result) |
| 489 | if rerun: |
Victor Stinner | 837acc1 | 2019-04-26 09:56:37 +0200 | [diff] [blame] | 490 | self.check_line(output, 'Tests result: FAILURE') |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 491 | result = 'FAILURE then %s' % result |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 492 | |
Victor Stinner | f2abf5c | 2016-08-19 17:54:25 +0200 | [diff] [blame] | 493 | self.check_line(output, 'Tests result: %s' % result) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 494 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 495 | def parse_random_seed(self, output): |
| 496 | match = self.regex_search(r'Using random seed ([0-9]+)', output) |
| 497 | randseed = int(match.group(1)) |
| 498 | self.assertTrue(0 <= randseed <= 10000000, randseed) |
| 499 | return randseed |
| 500 | |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 501 | def run_command(self, args, input=None, exitcode=0, **kw): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 502 | if not input: |
| 503 | input = '' |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 504 | if 'stderr' not in kw: |
| 505 | kw['stderr'] = subprocess.PIPE |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 506 | proc = subprocess.run(args, |
| 507 | universal_newlines=True, |
| 508 | input=input, |
| 509 | stdout=subprocess.PIPE, |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 510 | **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 511 | if proc.returncode != exitcode: |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 512 | msg = ("Command %s failed with exit code %s\n" |
| 513 | "\n" |
| 514 | "stdout:\n" |
| 515 | "---\n" |
| 516 | "%s\n" |
| 517 | "---\n" |
| 518 | % (str(args), proc.returncode, proc.stdout)) |
| 519 | if proc.stderr: |
| 520 | msg += ("\n" |
| 521 | "stderr:\n" |
| 522 | "---\n" |
| 523 | "%s" |
| 524 | "---\n" |
| 525 | % proc.stderr) |
| 526 | self.fail(msg) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 527 | return proc |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 528 | |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 529 | def run_python(self, args, **kw): |
| 530 | args = [sys.executable, '-X', 'faulthandler', '-I', *args] |
| 531 | proc = self.run_command(args, **kw) |
| 532 | return proc.stdout |
| 533 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 534 | |
Miss Islington (bot) | 9d54de1 | 2019-07-23 23:30:27 -0700 | [diff] [blame] | 535 | class CheckActualTests(BaseTestCase): |
| 536 | """ |
| 537 | Check that regrtest appears to find the expected set of tests. |
| 538 | """ |
| 539 | |
| 540 | def test_finds_expected_number_of_tests(self): |
| 541 | args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] |
| 542 | output = self.run_python(args) |
| 543 | rough_number_of_tests_found = len(output.splitlines()) |
| 544 | actual_testsuite_glob = os.path.join(os.path.dirname(__file__), |
| 545 | 'test*.py') |
| 546 | rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob)) |
| 547 | # We're not trying to duplicate test finding logic in here, |
| 548 | # just give a rough estimate of how many there should be and |
| 549 | # be near that. This is a regression test to prevent mishaps |
| 550 | # such as https://bugs.python.org/issue37667 in the future. |
| 551 | # If you need to change the values in here during some |
| 552 | # mythical future test suite reorganization, don't go |
| 553 | # overboard with logic and keep that goal in mind. |
| 554 | self.assertGreater(rough_number_of_tests_found, |
| 555 | rough_counted_test_py_files*9//10, |
| 556 | msg='Unexpectedly low number of tests found in:\n' |
| 557 | f'{", ".join(output.splitlines())}') |
| 558 | |
| 559 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 560 | class ProgramsTestCase(BaseTestCase): |
| 561 | """ |
| 562 | Test various ways to run the Python test suite. Use options close |
| 563 | to options used on the buildbot. |
| 564 | """ |
| 565 | |
| 566 | NTEST = 4 |
| 567 | |
| 568 | def setUp(self): |
| 569 | super().setUp() |
| 570 | |
| 571 | # Create NTEST tests doing nothing |
| 572 | self.tests = [self.create_test() for index in range(self.NTEST)] |
| 573 | |
| 574 | self.python_args = ['-Wd', '-E', '-bb'] |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 575 | self.regrtest_args = ['-uall', '-rwW', |
| 576 | '--testdir=%s' % self.tmptestdir] |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 577 | if hasattr(faulthandler, 'dump_traceback_later'): |
| 578 | self.regrtest_args.extend(('--timeout', '3600', '-j4')) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 579 | if sys.platform == 'win32': |
| 580 | self.regrtest_args.append('-n') |
| 581 | |
| 582 | def check_output(self, output): |
| 583 | self.parse_random_seed(output) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 584 | self.check_executed_tests(output, self.tests, randomize=True) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 585 | |
| 586 | def run_tests(self, args): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 587 | output = self.run_python(args) |
| 588 | self.check_output(output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 589 | |
| 590 | def test_script_regrtest(self): |
| 591 | # Lib/test/regrtest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 592 | script = os.path.join(self.testdir, 'regrtest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 593 | |
| 594 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 595 | self.run_tests(args) |
| 596 | |
| 597 | def test_module_test(self): |
| 598 | # -m test |
| 599 | args = [*self.python_args, '-m', 'test', |
| 600 | *self.regrtest_args, *self.tests] |
| 601 | self.run_tests(args) |
| 602 | |
| 603 | def test_module_regrtest(self): |
| 604 | # -m test.regrtest |
| 605 | args = [*self.python_args, '-m', 'test.regrtest', |
| 606 | *self.regrtest_args, *self.tests] |
| 607 | self.run_tests(args) |
| 608 | |
| 609 | def test_module_autotest(self): |
| 610 | # -m test.autotest |
| 611 | args = [*self.python_args, '-m', 'test.autotest', |
| 612 | *self.regrtest_args, *self.tests] |
| 613 | self.run_tests(args) |
| 614 | |
| 615 | def test_module_from_test_autotest(self): |
| 616 | # from test import autotest |
| 617 | code = 'from test import autotest' |
| 618 | args = [*self.python_args, '-c', code, |
| 619 | *self.regrtest_args, *self.tests] |
| 620 | self.run_tests(args) |
| 621 | |
| 622 | def test_script_autotest(self): |
| 623 | # Lib/test/autotest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 624 | script = os.path.join(self.testdir, 'autotest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 625 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 626 | self.run_tests(args) |
| 627 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 628 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 629 | 'run_tests.py script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 630 | def test_tools_script_run_tests(self): |
| 631 | # Tools/scripts/run_tests.py |
| 632 | script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') |
Victor Stinner | 8a34d41 | 2016-03-30 08:51:15 +0200 | [diff] [blame] | 633 | args = [script, *self.regrtest_args, *self.tests] |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 634 | self.run_tests(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 635 | |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 636 | def run_batch(self, *args): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 637 | proc = self.run_command(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 638 | self.check_output(proc.stdout) |
| 639 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 640 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 641 | 'test.bat script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 642 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 643 | def test_tools_buildbot_test(self): |
| 644 | # Tools\buildbot\test.bat |
| 645 | script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') |
Victor Stinner | 732599f | 2016-03-30 08:38:05 +0200 | [diff] [blame] | 646 | test_args = ['--testdir=%s' % self.tmptestdir] |
Miss Islington (bot) | 84d47bd | 2019-06-07 11:17:52 -0700 | [diff] [blame] | 647 | if platform.machine() == 'ARM64': |
| 648 | test_args.append('-arm64') # ARM 64-bit build |
| 649 | elif platform.architecture()[0] == '64bit': |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 650 | test_args.append('-x64') # 64-bit build |
| 651 | if not Py_DEBUG: |
| 652 | test_args.append('+d') # Release build, use python.exe |
| 653 | self.run_batch(script, *test_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 654 | |
| 655 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 656 | def test_pcbuild_rt(self): |
| 657 | # PCbuild\rt.bat |
| 658 | script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') |
Andrés Delfino | c3b7a6d | 2018-04-30 13:30:53 -0300 | [diff] [blame] | 659 | if not os.path.isfile(script): |
| 660 | self.skipTest(f'File "{script}" does not exist') |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 661 | rt_args = ["-q"] # Quick, don't run tests twice |
Miss Islington (bot) | 84d47bd | 2019-06-07 11:17:52 -0700 | [diff] [blame] | 662 | if platform.machine() == 'ARM64': |
| 663 | rt_args.append('-arm64') # ARM 64-bit build |
| 664 | elif platform.architecture()[0] == '64bit': |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 665 | rt_args.append('-x64') # 64-bit build |
| 666 | if Py_DEBUG: |
| 667 | rt_args.append('-d') # Debug build, use python_d.exe |
| 668 | self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 669 | |
| 670 | |
| 671 | class ArgsTestCase(BaseTestCase): |
| 672 | """ |
| 673 | Test arguments of the Python test suite. |
| 674 | """ |
| 675 | |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 676 | def run_tests(self, *testargs, **kw): |
| 677 | cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] |
| 678 | return self.run_python(cmdargs, **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 679 | |
| 680 | def test_failing_test(self): |
| 681 | # test a failing test |
| 682 | code = textwrap.dedent(""" |
| 683 | import unittest |
| 684 | |
| 685 | class FailingTest(unittest.TestCase): |
| 686 | def test_failing(self): |
| 687 | self.fail("bug") |
| 688 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 689 | test_ok = self.create_test('ok') |
| 690 | test_failing = self.create_test('failing', code=code) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 691 | tests = [test_ok, test_failing] |
| 692 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 693 | output = self.run_tests(*tests, exitcode=2) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 694 | self.check_executed_tests(output, tests, failed=test_failing) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 695 | |
| 696 | def test_resources(self): |
| 697 | # test -u command line option |
| 698 | tests = {} |
| 699 | for resource in ('audio', 'network'): |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 700 | code = textwrap.dedent(""" |
| 701 | from test import support; support.requires(%r) |
| 702 | import unittest |
| 703 | class PassingTest(unittest.TestCase): |
| 704 | def test_pass(self): |
| 705 | pass |
| 706 | """ % resource) |
| 707 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 708 | tests[resource] = self.create_test(resource, code) |
| 709 | test_names = sorted(tests.values()) |
| 710 | |
| 711 | # -u all: 2 resources enabled |
| 712 | output = self.run_tests('-u', 'all', *test_names) |
| 713 | self.check_executed_tests(output, test_names) |
| 714 | |
| 715 | # -u audio: 1 resource enabled |
| 716 | output = self.run_tests('-uaudio', *test_names) |
| 717 | self.check_executed_tests(output, test_names, |
| 718 | skipped=tests['network']) |
| 719 | |
| 720 | # no option: 0 resources enabled |
| 721 | output = self.run_tests(*test_names) |
| 722 | self.check_executed_tests(output, test_names, |
| 723 | skipped=test_names) |
| 724 | |
| 725 | def test_random(self): |
| 726 | # test -r and --randseed command line option |
| 727 | code = textwrap.dedent(""" |
| 728 | import random |
| 729 | print("TESTRANDOM: %s" % random.randint(1, 1000)) |
| 730 | """) |
| 731 | test = self.create_test('random', code) |
| 732 | |
| 733 | # first run to get the output with the random seed |
| 734 | output = self.run_tests('-r', test) |
| 735 | randseed = self.parse_random_seed(output) |
| 736 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 737 | test_random = int(match.group(1)) |
| 738 | |
| 739 | # try to reproduce with the random seed |
| 740 | output = self.run_tests('-r', '--randseed=%s' % randseed, test) |
| 741 | randseed2 = self.parse_random_seed(output) |
| 742 | self.assertEqual(randseed2, randseed) |
| 743 | |
| 744 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 745 | test_random2 = int(match.group(1)) |
| 746 | self.assertEqual(test_random2, test_random) |
| 747 | |
| 748 | def test_fromfile(self): |
| 749 | # test --fromfile |
| 750 | tests = [self.create_test() for index in range(5)] |
| 751 | |
| 752 | # Write the list of files using a format similar to regrtest output: |
| 753 | # [1/2] test_1 |
| 754 | # [2/2] test_2 |
| 755 | filename = support.TESTFN |
| 756 | self.addCleanup(support.unlink, filename) |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 757 | |
| 758 | # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec' |
| 759 | with open(filename, "w") as fp: |
| 760 | previous = None |
| 761 | for index, name in enumerate(tests, 1): |
| 762 | line = ("00:00:%02i [%s/%s] %s" |
| 763 | % (index, index, len(tests), name)) |
| 764 | if previous: |
| 765 | line += " -- %s took 0 sec" % previous |
| 766 | print(line, file=fp) |
| 767 | previous = name |
| 768 | |
| 769 | output = self.run_tests('--fromfile', filename) |
| 770 | self.check_executed_tests(output, tests) |
| 771 | |
| 772 | # test format '[2/7] test_opcodes' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 773 | with open(filename, "w") as fp: |
| 774 | for index, name in enumerate(tests, 1): |
| 775 | print("[%s/%s] %s" % (index, len(tests), name), file=fp) |
| 776 | |
| 777 | output = self.run_tests('--fromfile', filename) |
| 778 | self.check_executed_tests(output, tests) |
| 779 | |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 780 | # test format 'test_opcodes' |
| 781 | with open(filename, "w") as fp: |
| 782 | for name in tests: |
| 783 | print(name, file=fp) |
| 784 | |
| 785 | output = self.run_tests('--fromfile', filename) |
| 786 | self.check_executed_tests(output, tests) |
| 787 | |
Victor Stinner | e8cfec5 | 2017-01-03 01:38:58 +0100 | [diff] [blame] | 788 | # test format 'Lib/test/test_opcodes.py' |
| 789 | with open(filename, "w") as fp: |
| 790 | for name in tests: |
| 791 | print('Lib/test/%s.py' % name, file=fp) |
| 792 | |
| 793 | output = self.run_tests('--fromfile', filename) |
| 794 | self.check_executed_tests(output, tests) |
| 795 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 796 | def test_interrupted(self): |
| 797 | code = TEST_INTERRUPTED |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 798 | test = self.create_test('sigint', code=code) |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 799 | output = self.run_tests(test, exitcode=130) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 800 | self.check_executed_tests(output, test, omitted=test, |
| 801 | interrupted=True) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 802 | |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 803 | def test_slowest(self): |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 804 | # test --slowest |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 805 | tests = [self.create_test() for index in range(3)] |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 806 | output = self.run_tests("--slowest", *tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 807 | self.check_executed_tests(output, tests) |
| 808 | regex = ('10 slowest tests:\n' |
Victor Stinner | 435eaf4 | 2016-08-17 12:22:52 +0200 | [diff] [blame] | 809 | '(?:- %s: .*\n){%s}' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 810 | % (self.TESTNAME_REGEX, len(tests))) |
| 811 | self.check_line(output, regex) |
| 812 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 813 | def test_slowest_interrupted(self): |
Victor Stinner | 6c44619 | 2016-08-17 11:25:43 +0200 | [diff] [blame] | 814 | # Issue #25373: test --slowest with an interrupted test |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 815 | code = TEST_INTERRUPTED |
| 816 | test = self.create_test("sigint", code=code) |
| 817 | |
Antoine Pitrou | a6a4dc8 | 2017-09-07 18:56:24 +0200 | [diff] [blame] | 818 | for multiprocessing in (False, True): |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 819 | with self.subTest(multiprocessing=multiprocessing): |
| 820 | if multiprocessing: |
| 821 | args = ("--slowest", "-j2", test) |
| 822 | else: |
| 823 | args = ("--slowest", test) |
| 824 | output = self.run_tests(*args, exitcode=130) |
| 825 | self.check_executed_tests(output, test, |
| 826 | omitted=test, interrupted=True) |
Victor Stinner | 6368603 | 2016-08-17 16:12:16 +0200 | [diff] [blame] | 827 | |
Victor Stinner | 4d29983 | 2019-04-26 04:08:53 +0200 | [diff] [blame] | 828 | regex = ('10 slowest tests:\n') |
| 829 | self.check_line(output, regex) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 830 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 831 | def test_coverage(self): |
| 832 | # test --coverage |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 833 | test = self.create_test('coverage') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 834 | output = self.run_tests("--coverage", test) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 835 | self.check_executed_tests(output, [test]) |
R David Murray | 44b548d | 2016-09-08 13:59:53 -0400 | [diff] [blame] | 836 | regex = (r'lines +cov% +module +\(path\)\n' |
| 837 | r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 838 | self.check_line(output, regex) |
| 839 | |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 840 | def test_wait(self): |
| 841 | # test --wait |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 842 | test = self.create_test('wait') |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 843 | output = self.run_tests("--wait", test, input='key') |
| 844 | self.check_line(output, 'Press any key to continue') |
| 845 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 846 | def test_forever(self): |
| 847 | # test --forever |
| 848 | code = textwrap.dedent(""" |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 849 | import builtins |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 850 | import unittest |
| 851 | |
| 852 | class ForeverTester(unittest.TestCase): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 853 | def test_run(self): |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 854 | # Store the state in the builtins module, because the test |
| 855 | # module is reload at each run |
| 856 | if 'RUN' in builtins.__dict__: |
| 857 | builtins.__dict__['RUN'] += 1 |
| 858 | if builtins.__dict__['RUN'] >= 3: |
| 859 | self.fail("fail at the 3rd runs") |
| 860 | else: |
| 861 | builtins.__dict__['RUN'] = 1 |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 862 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 863 | test = self.create_test('forever', code=code) |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 864 | output = self.run_tests('--forever', test, exitcode=2) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 865 | self.check_executed_tests(output, [test]*3, failed=test) |
| 866 | |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 867 | def check_leak(self, code, what): |
| 868 | test = self.create_test('huntrleaks', code=code) |
| 869 | |
| 870 | filename = 'reflog.txt' |
| 871 | self.addCleanup(support.unlink, filename) |
| 872 | output = self.run_tests('--huntrleaks', '3:3:', test, |
| 873 | exitcode=2, |
| 874 | stderr=subprocess.STDOUT) |
| 875 | self.check_executed_tests(output, [test], failed=test) |
| 876 | |
| 877 | line = 'beginning 6 repetitions\n123456\n......\n' |
| 878 | self.check_line(output, re.escape(line)) |
| 879 | |
| 880 | line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) |
| 881 | self.assertIn(line2, output) |
| 882 | |
| 883 | with open(filename) as fp: |
| 884 | reflog = fp.read() |
| 885 | self.assertIn(line2, reflog) |
| 886 | |
| 887 | @unittest.skipUnless(Py_DEBUG, 'need a debug build') |
| 888 | def test_huntrleaks(self): |
| 889 | # test --huntrleaks |
| 890 | code = textwrap.dedent(""" |
| 891 | import unittest |
| 892 | |
| 893 | GLOBAL_LIST = [] |
| 894 | |
| 895 | class RefLeakTest(unittest.TestCase): |
| 896 | def test_leak(self): |
| 897 | GLOBAL_LIST.append(object()) |
| 898 | """) |
| 899 | self.check_leak(code, 'references') |
| 900 | |
Victor Stinner | 46d7535 | 2015-10-03 02:21:35 +0200 | [diff] [blame] | 901 | @unittest.skipUnless(Py_DEBUG, 'need a debug build') |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 902 | def test_huntrleaks_fd_leak(self): |
| 903 | # test --huntrleaks for file descriptor leak |
| 904 | code = textwrap.dedent(""" |
| 905 | import os |
| 906 | import unittest |
| 907 | |
| 908 | class FDLeakTest(unittest.TestCase): |
| 909 | def test_leak(self): |
| 910 | fd = os.open(__file__, os.O_RDONLY) |
Victor Stinner | 302bbbe | 2017-07-25 17:45:53 +0200 | [diff] [blame] | 911 | # bug: never close the file descriptor |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 912 | """) |
Victor Stinner | 48b5c42 | 2017-06-27 02:02:04 +0200 | [diff] [blame] | 913 | self.check_leak(code, 'file descriptors') |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 914 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 915 | def test_list_tests(self): |
| 916 | # test --list-tests |
| 917 | tests = [self.create_test() for i in range(5)] |
| 918 | output = self.run_tests('--list-tests', *tests) |
| 919 | self.assertEqual(output.rstrip().splitlines(), |
| 920 | tests) |
| 921 | |
Louie Lu | b0c58d3 | 2017-06-16 19:15:28 +0800 | [diff] [blame] | 922 | def test_list_cases(self): |
| 923 | # test --list-cases |
| 924 | code = textwrap.dedent(""" |
| 925 | import unittest |
| 926 | |
| 927 | class Tests(unittest.TestCase): |
| 928 | def test_method1(self): |
| 929 | pass |
| 930 | def test_method2(self): |
| 931 | pass |
| 932 | """) |
| 933 | testname = self.create_test(code=code) |
Victor Stinner | ace56d5 | 2017-06-26 14:18:51 +0200 | [diff] [blame] | 934 | |
| 935 | # Test --list-cases |
Louie Lu | b0c58d3 | 2017-06-16 19:15:28 +0800 | [diff] [blame] | 936 | all_methods = ['%s.Tests.test_method1' % testname, |
| 937 | '%s.Tests.test_method2' % testname] |
| 938 | output = self.run_tests('--list-cases', testname) |
| 939 | self.assertEqual(output.splitlines(), all_methods) |
| 940 | |
Victor Stinner | ace56d5 | 2017-06-26 14:18:51 +0200 | [diff] [blame] | 941 | # Test --list-cases with --match |
| 942 | all_methods = ['%s.Tests.test_method1' % testname] |
| 943 | output = self.run_tests('--list-cases', |
| 944 | '-m', 'test_method1', |
| 945 | testname) |
| 946 | self.assertEqual(output.splitlines(), all_methods) |
| 947 | |
Victor Stinner | 3cde440 | 2019-04-26 08:40:25 +0200 | [diff] [blame] | 948 | @support.cpython_only |
Victor Stinner | 5bad70d | 2017-02-06 12:42:00 +0100 | [diff] [blame] | 949 | def test_crashed(self): |
| 950 | # Any code which causes a crash |
| 951 | code = 'import faulthandler; faulthandler._sigsegv()' |
| 952 | crash_test = self.create_test(name="crash", code=code) |
Victor Stinner | 5bad70d | 2017-02-06 12:42:00 +0100 | [diff] [blame] | 953 | |
Victor Stinner | 3cde440 | 2019-04-26 08:40:25 +0200 | [diff] [blame] | 954 | tests = [crash_test] |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 955 | output = self.run_tests("-j2", *tests, exitcode=2) |
Victor Stinner | 5bad70d | 2017-02-06 12:42:00 +0100 | [diff] [blame] | 956 | self.check_executed_tests(output, tests, failed=crash_test, |
| 957 | randomize=True) |
| 958 | |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 959 | def parse_methods(self, output): |
| 960 | regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) |
| 961 | return [match.group(1) for match in regex.finditer(output)] |
| 962 | |
| 963 | def test_matchfile(self): |
Victor Stinner | ef8320c | 2017-06-09 10:18:48 +0200 | [diff] [blame] | 964 | code = textwrap.dedent(""" |
| 965 | import unittest |
| 966 | |
| 967 | class Tests(unittest.TestCase): |
| 968 | def test_method1(self): |
| 969 | pass |
| 970 | def test_method2(self): |
| 971 | pass |
| 972 | def test_method3(self): |
| 973 | pass |
| 974 | def test_method4(self): |
| 975 | pass |
| 976 | """) |
| 977 | all_methods = ['test_method1', 'test_method2', |
| 978 | 'test_method3', 'test_method4'] |
| 979 | testname = self.create_test(code=code) |
| 980 | |
| 981 | # by default, all methods should be run |
| 982 | output = self.run_tests("-v", testname) |
| 983 | methods = self.parse_methods(output) |
| 984 | self.assertEqual(methods, all_methods) |
| 985 | |
| 986 | # only run a subset |
| 987 | filename = support.TESTFN |
| 988 | self.addCleanup(support.unlink, filename) |
| 989 | |
| 990 | subset = [ |
| 991 | # only match the method name |
| 992 | 'test_method1', |
| 993 | # match the full identifier |
| 994 | '%s.Tests.test_method3' % testname] |
| 995 | with open(filename, "w") as fp: |
| 996 | for name in subset: |
| 997 | print(name, file=fp) |
| 998 | |
| 999 | output = self.run_tests("-v", "--matchfile", filename, testname) |
| 1000 | methods = self.parse_methods(output) |
| 1001 | subset = ['test_method1', 'test_method3'] |
| 1002 | self.assertEqual(methods, subset) |
| 1003 | |
Victor Stinner | 63f54c6 | 2017-06-26 18:33:19 +0200 | [diff] [blame] | 1004 | def test_env_changed(self): |
| 1005 | code = textwrap.dedent(""" |
| 1006 | import unittest |
| 1007 | |
| 1008 | class Tests(unittest.TestCase): |
| 1009 | def test_env_changed(self): |
| 1010 | open("env_changed", "w").close() |
| 1011 | """) |
| 1012 | testname = self.create_test(code=code) |
| 1013 | |
| 1014 | # don't fail by default |
| 1015 | output = self.run_tests(testname) |
| 1016 | self.check_executed_tests(output, [testname], env_changed=testname) |
| 1017 | |
| 1018 | # fail with --fail-env-changed |
| 1019 | output = self.run_tests("--fail-env-changed", testname, exitcode=3) |
| 1020 | self.check_executed_tests(output, [testname], env_changed=testname, |
| 1021 | fail_env_changed=True) |
| 1022 | |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 1023 | def test_rerun_fail(self): |
Victor Stinner | 837acc1 | 2019-04-26 09:56:37 +0200 | [diff] [blame] | 1024 | # FAILURE then FAILURE |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 1025 | code = textwrap.dedent(""" |
| 1026 | import unittest |
| 1027 | |
| 1028 | class Tests(unittest.TestCase): |
| 1029 | def test_bug(self): |
| 1030 | # test always fail |
| 1031 | self.fail("bug") |
| 1032 | """) |
| 1033 | testname = self.create_test(code=code) |
| 1034 | |
| 1035 | output = self.run_tests("-w", testname, exitcode=2) |
| 1036 | self.check_executed_tests(output, [testname], |
| 1037 | failed=testname, rerun=testname) |
Victor Stinner | 3a8f4fe | 2018-12-14 13:06:50 +0100 | [diff] [blame] | 1038 | |
Victor Stinner | 837acc1 | 2019-04-26 09:56:37 +0200 | [diff] [blame] | 1039 | def test_rerun_success(self): |
| 1040 | # FAILURE then SUCCESS |
| 1041 | code = textwrap.dedent(""" |
| 1042 | import builtins |
| 1043 | import unittest |
| 1044 | |
| 1045 | class Tests(unittest.TestCase): |
| 1046 | failed = False |
| 1047 | |
| 1048 | def test_fail_once(self): |
| 1049 | if not hasattr(builtins, '_test_failed'): |
| 1050 | builtins._test_failed = True |
| 1051 | self.fail("bug") |
| 1052 | """) |
| 1053 | testname = self.create_test(code=code) |
| 1054 | |
| 1055 | output = self.run_tests("-w", testname, exitcode=0) |
| 1056 | self.check_executed_tests(output, [testname], |
| 1057 | rerun=testname) |
| 1058 | |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 1059 | def test_no_tests_ran(self): |
| 1060 | code = textwrap.dedent(""" |
| 1061 | import unittest |
| 1062 | |
| 1063 | class Tests(unittest.TestCase): |
| 1064 | def test_bug(self): |
| 1065 | pass |
| 1066 | """) |
| 1067 | testname = self.create_test(code=code) |
| 1068 | |
| 1069 | output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0) |
| 1070 | self.check_executed_tests(output, [testname], no_test_ran=testname) |
| 1071 | |
Victor Stinner | 3a8f4fe | 2018-12-14 13:06:50 +0100 | [diff] [blame] | 1072 | def test_no_tests_ran_skip(self): |
| 1073 | code = textwrap.dedent(""" |
| 1074 | import unittest |
| 1075 | |
| 1076 | class Tests(unittest.TestCase): |
| 1077 | def test_skipped(self): |
| 1078 | self.skipTest("because") |
| 1079 | """) |
| 1080 | testname = self.create_test(code=code) |
| 1081 | |
| 1082 | output = self.run_tests(testname, exitcode=0) |
| 1083 | self.check_executed_tests(output, [testname]) |
| 1084 | |
Pablo Galindo | 9724348 | 2018-11-29 17:17:44 +0000 | [diff] [blame] | 1085 | def test_no_tests_ran_multiple_tests_nonexistent(self): |
| 1086 | code = textwrap.dedent(""" |
| 1087 | import unittest |
| 1088 | |
| 1089 | class Tests(unittest.TestCase): |
| 1090 | def test_bug(self): |
| 1091 | pass |
| 1092 | """) |
| 1093 | testname = self.create_test(code=code) |
| 1094 | testname2 = self.create_test(code=code) |
| 1095 | |
| 1096 | output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0) |
| 1097 | self.check_executed_tests(output, [testname, testname2], |
| 1098 | no_test_ran=[testname, testname2]) |
| 1099 | |
| 1100 | def test_no_test_ran_some_test_exist_some_not(self): |
| 1101 | code = textwrap.dedent(""" |
| 1102 | import unittest |
| 1103 | |
| 1104 | class Tests(unittest.TestCase): |
| 1105 | def test_bug(self): |
| 1106 | pass |
| 1107 | """) |
| 1108 | testname = self.create_test(code=code) |
| 1109 | other_code = textwrap.dedent(""" |
| 1110 | import unittest |
| 1111 | |
| 1112 | class Tests(unittest.TestCase): |
| 1113 | def test_other_bug(self): |
| 1114 | pass |
| 1115 | """) |
| 1116 | testname2 = self.create_test(code=other_code) |
| 1117 | |
| 1118 | output = self.run_tests(testname, testname2, "-m", "nosuchtest", |
| 1119 | "-m", "test_other_bug", exitcode=0) |
| 1120 | self.check_executed_tests(output, [testname, testname2], |
| 1121 | no_test_ran=[testname]) |
Victor Stinner | c45fc76 | 2018-06-08 09:53:51 +0200 | [diff] [blame] | 1122 | |
Victor Stinner | 75120d2 | 2019-04-26 09:28:53 +0200 | [diff] [blame] | 1123 | @support.cpython_only |
| 1124 | def test_findleaks(self): |
| 1125 | code = textwrap.dedent(r""" |
| 1126 | import _testcapi |
| 1127 | import gc |
| 1128 | import unittest |
| 1129 | |
| 1130 | @_testcapi.with_tp_del |
| 1131 | class Garbage: |
| 1132 | def __tp_del__(self): |
| 1133 | pass |
| 1134 | |
| 1135 | class Tests(unittest.TestCase): |
| 1136 | def test_garbage(self): |
| 1137 | # create an uncollectable object |
| 1138 | obj = Garbage() |
| 1139 | obj.ref_cycle = obj |
| 1140 | obj = None |
| 1141 | """) |
| 1142 | testname = self.create_test(code=code) |
| 1143 | |
| 1144 | output = self.run_tests("--fail-env-changed", testname, exitcode=3) |
| 1145 | self.check_executed_tests(output, [testname], |
| 1146 | env_changed=[testname], |
| 1147 | fail_env_changed=True) |
| 1148 | |
| 1149 | # --findleaks is now basically an alias to --fail-env-changed |
| 1150 | output = self.run_tests("--findleaks", testname, exitcode=3) |
| 1151 | self.check_executed_tests(output, [testname], |
| 1152 | env_changed=[testname], |
| 1153 | fail_env_changed=True) |
| 1154 | |
Victor Stinner | d85c567 | 2019-08-14 16:31:32 +0200 | [diff] [blame] | 1155 | def test_multiprocessing_timeout(self): |
| 1156 | code = textwrap.dedent(r""" |
| 1157 | import time |
| 1158 | import unittest |
| 1159 | try: |
| 1160 | import faulthandler |
| 1161 | except ImportError: |
| 1162 | faulthandler = None |
| 1163 | |
| 1164 | class Tests(unittest.TestCase): |
| 1165 | # test hangs and so should be stopped by the timeout |
| 1166 | def test_sleep(self): |
| 1167 | # we want to test regrtest multiprocessing timeout, |
| 1168 | # not faulthandler timeout |
| 1169 | if faulthandler is not None: |
| 1170 | faulthandler.cancel_dump_traceback_later() |
| 1171 | |
| 1172 | time.sleep(60 * 5) |
| 1173 | """) |
| 1174 | testname = self.create_test(code=code) |
| 1175 | |
| 1176 | output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2) |
| 1177 | self.check_executed_tests(output, [testname], |
| 1178 | failed=testname) |
| 1179 | self.assertRegex(output, |
| 1180 | re.compile('%s timed out' % testname, re.MULTILINE)) |
| 1181 | |
Victor Stinner | 2ec4545 | 2019-06-24 12:51:47 +0200 | [diff] [blame] | 1182 | def test_cleanup(self): |
| 1183 | dirname = os.path.join(self.tmptestdir, "test_python_123") |
| 1184 | os.mkdir(dirname) |
| 1185 | filename = os.path.join(self.tmptestdir, "test_python_456") |
| 1186 | open(filename, "wb").close() |
| 1187 | names = [dirname, filename] |
| 1188 | |
| 1189 | cmdargs = ['-m', 'test', |
| 1190 | '--tempdir=%s' % self.tmptestdir, |
| 1191 | '--cleanup'] |
| 1192 | self.run_python(cmdargs) |
| 1193 | |
| 1194 | for name in names: |
| 1195 | self.assertFalse(os.path.exists(name), name) |
| 1196 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 1197 | |
Victor Stinner | 4ffe9c2 | 2018-06-14 14:58:13 +0200 | [diff] [blame] | 1198 | class TestUtils(unittest.TestCase): |
| 1199 | def test_format_duration(self): |
| 1200 | self.assertEqual(utils.format_duration(0), |
| 1201 | '0 ms') |
| 1202 | self.assertEqual(utils.format_duration(1e-9), |
| 1203 | '1 ms') |
| 1204 | self.assertEqual(utils.format_duration(10e-3), |
| 1205 | '10 ms') |
| 1206 | self.assertEqual(utils.format_duration(1.5), |
Victor Stinner | de3195c | 2019-10-03 17:26:25 +0200 | [diff] [blame] | 1207 | '1.5 sec') |
Victor Stinner | 4ffe9c2 | 2018-06-14 14:58:13 +0200 | [diff] [blame] | 1208 | self.assertEqual(utils.format_duration(1), |
Victor Stinner | de3195c | 2019-10-03 17:26:25 +0200 | [diff] [blame] | 1209 | '1.0 sec') |
Victor Stinner | 4ffe9c2 | 2018-06-14 14:58:13 +0200 | [diff] [blame] | 1210 | self.assertEqual(utils.format_duration(2 * 60), |
| 1211 | '2 min') |
| 1212 | self.assertEqual(utils.format_duration(2 * 60 + 1), |
| 1213 | '2 min 1 sec') |
| 1214 | self.assertEqual(utils.format_duration(3 * 3600), |
| 1215 | '3 hour') |
| 1216 | self.assertEqual(utils.format_duration(3 * 3600 + 2 * 60 + 1), |
| 1217 | '3 hour 2 min') |
| 1218 | self.assertEqual(utils.format_duration(3 * 3600 + 1), |
| 1219 | '3 hour 1 sec') |
| 1220 | |
| 1221 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 1222 | if __name__ == '__main__': |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 1223 | unittest.main() |