Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 1 | """ |
| 2 | Tests of regrtest.py. |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 3 | |
| 4 | Note: test_regrtest cannot be run twice in parallel. |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 5 | """ |
| 6 | |
| 7 | import argparse |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 8 | import contextlib |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 9 | import faulthandler |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 10 | import getopt |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 11 | import io |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 12 | import os.path |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 13 | import platform |
| 14 | import re |
| 15 | import subprocess |
| 16 | import sys |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 17 | import sysconfig |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 18 | import textwrap |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 19 | import unittest |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 20 | from test import libregrtest |
| 21 | from test import support |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 22 | |
| 23 | |
| 24 | Py_DEBUG = hasattr(sys, 'getobjects') |
| 25 | ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') |
| 26 | ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) |
| 27 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 28 | TEST_INTERRUPTED = textwrap.dedent(""" |
| 29 | from signal import SIGINT |
| 30 | try: |
| 31 | from _testcapi import raise_signal |
| 32 | raise_signal(SIGINT) |
| 33 | except ImportError: |
| 34 | import os |
| 35 | os.kill(os.getpid(), SIGINT) |
| 36 | """) |
| 37 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 38 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 39 | class ParseArgsTestCase(unittest.TestCase): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 40 | """ |
| 41 | Test regrtest's argument parsing, function _parse_args(). |
| 42 | """ |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 43 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 44 | def checkError(self, args, msg): |
| 45 | with support.captured_stderr() as err, self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 46 | libregrtest._parse_args(args) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 47 | self.assertIn(msg, err.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 48 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 49 | def test_help(self): |
| 50 | for opt in '-h', '--help': |
| 51 | with self.subTest(opt=opt): |
| 52 | with support.captured_stdout() as out, \ |
| 53 | self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 54 | libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 55 | self.assertIn('Run Python regression tests.', out.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 56 | |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 57 | @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), |
| 58 | "faulthandler.dump_traceback_later() required") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 59 | def test_timeout(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 60 | ns = libregrtest._parse_args(['--timeout', '4.2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 61 | self.assertEqual(ns.timeout, 4.2) |
| 62 | self.checkError(['--timeout'], 'expected one argument') |
| 63 | self.checkError(['--timeout', 'foo'], 'invalid float value') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 64 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 65 | def test_wait(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 66 | ns = libregrtest._parse_args(['--wait']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 67 | self.assertTrue(ns.wait) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 68 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 69 | def test_slaveargs(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 70 | ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 71 | self.assertEqual(ns.slaveargs, '[[], {}]') |
| 72 | self.checkError(['--slaveargs'], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 73 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 74 | def test_start(self): |
| 75 | for opt in '-S', '--start': |
| 76 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 77 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 78 | self.assertEqual(ns.start, 'foo') |
| 79 | self.checkError([opt], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 80 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 81 | def test_verbose(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 82 | ns = libregrtest._parse_args(['-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 83 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 84 | ns = libregrtest._parse_args(['-vvv']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 85 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 86 | ns = libregrtest._parse_args(['--verbose']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 87 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 88 | ns = libregrtest._parse_args(['--verbose'] * 3) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 89 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 90 | ns = libregrtest._parse_args([]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 91 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 92 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 93 | def test_verbose2(self): |
| 94 | for opt in '-w', '--verbose2': |
| 95 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 96 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 97 | self.assertTrue(ns.verbose2) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 98 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 99 | def test_verbose3(self): |
| 100 | for opt in '-W', '--verbose3': |
| 101 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 102 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 103 | self.assertTrue(ns.verbose3) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 104 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 105 | def test_quiet(self): |
| 106 | for opt in '-q', '--quiet': |
| 107 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 108 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 109 | self.assertTrue(ns.quiet) |
| 110 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 111 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 112 | def test_slow(self): |
| 113 | for opt in '-o', '--slow': |
| 114 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 115 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 116 | self.assertTrue(ns.print_slow) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 117 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 118 | def test_header(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 119 | ns = libregrtest._parse_args(['--header']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 120 | self.assertTrue(ns.header) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 121 | |
| 122 | def test_randomize(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 123 | for opt in '-r', '--randomize': |
| 124 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 125 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 126 | self.assertTrue(ns.randomize) |
| 127 | |
| 128 | def test_randseed(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 129 | ns = libregrtest._parse_args(['--randseed', '12345']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 130 | self.assertEqual(ns.random_seed, 12345) |
| 131 | self.assertTrue(ns.randomize) |
| 132 | self.checkError(['--randseed'], 'expected one argument') |
| 133 | self.checkError(['--randseed', 'foo'], 'invalid int value') |
| 134 | |
| 135 | def test_fromfile(self): |
| 136 | for opt in '-f', '--fromfile': |
| 137 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 138 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 139 | self.assertEqual(ns.fromfile, 'foo') |
| 140 | self.checkError([opt], 'expected one argument') |
| 141 | self.checkError([opt, 'foo', '-s'], "don't go together") |
| 142 | |
| 143 | def test_exclude(self): |
| 144 | for opt in '-x', '--exclude': |
| 145 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 146 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 147 | self.assertTrue(ns.exclude) |
| 148 | |
| 149 | def test_single(self): |
| 150 | for opt in '-s', '--single': |
| 151 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 152 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 153 | self.assertTrue(ns.single) |
| 154 | self.checkError([opt, '-f', 'foo'], "don't go together") |
| 155 | |
| 156 | def test_match(self): |
| 157 | for opt in '-m', '--match': |
| 158 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 159 | ns = libregrtest._parse_args([opt, 'pattern']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 160 | self.assertEqual(ns.match_tests, 'pattern') |
| 161 | self.checkError([opt], 'expected one argument') |
| 162 | |
| 163 | def test_failfast(self): |
| 164 | for opt in '-G', '--failfast': |
| 165 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 166 | ns = libregrtest._parse_args([opt, '-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 167 | self.assertTrue(ns.failfast) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 168 | ns = libregrtest._parse_args([opt, '-W']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 169 | self.assertTrue(ns.failfast) |
| 170 | self.checkError([opt], '-G/--failfast needs either -v or -W') |
| 171 | |
| 172 | def test_use(self): |
| 173 | for opt in '-u', '--use': |
| 174 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 175 | ns = libregrtest._parse_args([opt, 'gui,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 176 | self.assertEqual(ns.use_resources, ['gui', 'network']) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 177 | ns = libregrtest._parse_args([opt, 'gui,none,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 178 | self.assertEqual(ns.use_resources, ['network']) |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 179 | expected = list(libregrtest.RESOURCE_NAMES) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 180 | expected.remove('gui') |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 181 | ns = libregrtest._parse_args([opt, 'all,-gui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 182 | self.assertEqual(ns.use_resources, expected) |
| 183 | self.checkError([opt], 'expected one argument') |
| 184 | self.checkError([opt, 'foo'], 'invalid resource') |
| 185 | |
| 186 | def test_memlimit(self): |
| 187 | for opt in '-M', '--memlimit': |
| 188 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 189 | ns = libregrtest._parse_args([opt, '4G']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 190 | self.assertEqual(ns.memlimit, '4G') |
| 191 | self.checkError([opt], 'expected one argument') |
| 192 | |
| 193 | def test_testdir(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 194 | ns = libregrtest._parse_args(['--testdir', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 195 | self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) |
| 196 | self.checkError(['--testdir'], 'expected one argument') |
| 197 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 198 | def test_runleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 199 | for opt in '-L', '--runleaks': |
| 200 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 201 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 202 | self.assertTrue(ns.runleaks) |
| 203 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 204 | def test_huntrleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 205 | for opt in '-R', '--huntrleaks': |
| 206 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 207 | ns = libregrtest._parse_args([opt, ':']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 208 | self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 209 | ns = libregrtest._parse_args([opt, '6:']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 210 | self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 211 | ns = libregrtest._parse_args([opt, ':3']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 212 | self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 213 | ns = libregrtest._parse_args([opt, '6:3:leaks.log']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 214 | self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) |
| 215 | self.checkError([opt], 'expected one argument') |
| 216 | self.checkError([opt, '6'], |
| 217 | 'needs 2 or 3 colon-separated arguments') |
| 218 | self.checkError([opt, 'foo:'], 'invalid huntrleaks value') |
| 219 | self.checkError([opt, '6:foo'], 'invalid huntrleaks value') |
| 220 | |
| 221 | def test_multiprocess(self): |
| 222 | for opt in '-j', '--multiprocess': |
| 223 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 224 | ns = libregrtest._parse_args([opt, '2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 225 | self.assertEqual(ns.use_mp, 2) |
| 226 | self.checkError([opt], 'expected one argument') |
| 227 | self.checkError([opt, 'foo'], 'invalid int value') |
| 228 | self.checkError([opt, '2', '-T'], "don't go together") |
| 229 | self.checkError([opt, '2', '-l'], "don't go together") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 230 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 231 | def test_coverage(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 232 | for opt in '-T', '--coverage': |
| 233 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 234 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 235 | self.assertTrue(ns.trace) |
| 236 | |
| 237 | def test_coverdir(self): |
| 238 | for opt in '-D', '--coverdir': |
| 239 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 240 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 241 | self.assertEqual(ns.coverdir, |
| 242 | os.path.join(support.SAVEDCWD, 'foo')) |
| 243 | self.checkError([opt], 'expected one argument') |
| 244 | |
| 245 | def test_nocoverdir(self): |
| 246 | for opt in '-N', '--nocoverdir': |
| 247 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 248 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 249 | self.assertIsNone(ns.coverdir) |
| 250 | |
| 251 | def test_threshold(self): |
| 252 | for opt in '-t', '--threshold': |
| 253 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 254 | ns = libregrtest._parse_args([opt, '1000']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 255 | self.assertEqual(ns.threshold, 1000) |
| 256 | self.checkError([opt], 'expected one argument') |
| 257 | self.checkError([opt, 'foo'], 'invalid int value') |
| 258 | |
| 259 | def test_nowindows(self): |
| 260 | for opt in '-n', '--nowindows': |
| 261 | with self.subTest(opt=opt): |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 262 | with contextlib.redirect_stderr(io.StringIO()) as stderr: |
| 263 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 264 | self.assertTrue(ns.nowindows) |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 265 | err = stderr.getvalue() |
| 266 | self.assertIn('the --nowindows (-n) option is deprecated', err) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 267 | |
| 268 | def test_forever(self): |
| 269 | for opt in '-F', '--forever': |
| 270 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 271 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 272 | self.assertTrue(ns.forever) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 273 | |
| 274 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 275 | def test_unrecognized_argument(self): |
| 276 | self.checkError(['--xxx'], 'usage:') |
| 277 | |
| 278 | def test_long_option__partial(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 279 | ns = libregrtest._parse_args(['--qui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 280 | self.assertTrue(ns.quiet) |
| 281 | self.assertEqual(ns.verbose, 0) |
| 282 | |
| 283 | def test_two_options(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 284 | ns = libregrtest._parse_args(['--quiet', '--exclude']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 285 | self.assertTrue(ns.quiet) |
| 286 | self.assertEqual(ns.verbose, 0) |
| 287 | self.assertTrue(ns.exclude) |
| 288 | |
| 289 | def test_option_with_empty_string_value(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 290 | ns = libregrtest._parse_args(['--start', '']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 291 | self.assertEqual(ns.start, '') |
| 292 | |
| 293 | def test_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 294 | ns = libregrtest._parse_args(['foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 295 | self.assertEqual(ns.args, ['foo']) |
| 296 | |
| 297 | def test_option_and_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 298 | ns = libregrtest._parse_args(['--quiet', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 299 | self.assertTrue(ns.quiet) |
| 300 | self.assertEqual(ns.verbose, 0) |
| 301 | self.assertEqual(ns.args, ['foo']) |
| 302 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 303 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 304 | class BaseTestCase(unittest.TestCase): |
| 305 | TEST_UNIQUE_ID = 1 |
| 306 | TESTNAME_PREFIX = 'test_regrtest_' |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame^] | 307 | TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 308 | |
| 309 | def setUp(self): |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 310 | self.testdir = os.path.realpath(os.path.dirname(__file__)) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 311 | |
| 312 | # When test_regrtest is interrupted by CTRL+c, it can leave |
| 313 | # temporary test files |
| 314 | remove = [entry.path |
| 315 | for entry in os.scandir(self.testdir) |
| 316 | if (entry.name.startswith(self.TESTNAME_PREFIX) |
| 317 | and entry.name.endswith(".py"))] |
| 318 | for path in remove: |
| 319 | print("WARNING: test_regrtest: remove %s" % path) |
| 320 | support.unlink(path) |
| 321 | |
| 322 | def create_test(self, name=None, code=''): |
| 323 | if not name: |
| 324 | name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID |
| 325 | BaseTestCase.TEST_UNIQUE_ID += 1 |
| 326 | |
| 327 | # test_regrtest cannot be run twice in parallel because |
| 328 | # of setUp() and create_test() |
| 329 | name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) |
| 330 | path = os.path.join(self.testdir, name + '.py') |
| 331 | |
| 332 | self.addCleanup(support.unlink, path) |
| 333 | # Use 'x' mode to ensure that we do not override existing tests |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 334 | try: |
| 335 | with open(path, 'x', encoding='utf-8') as fp: |
| 336 | fp.write(code) |
| 337 | except PermissionError as exc: |
| 338 | if not sysconfig.is_python_build(): |
| 339 | self.skipTest("cannot write %s: %s" % (path, exc)) |
| 340 | raise |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 341 | return name |
| 342 | |
| 343 | def regex_search(self, regex, output): |
| 344 | match = re.search(regex, output, re.MULTILINE) |
| 345 | if not match: |
| 346 | self.fail("%r not found in %r" % (regex, output)) |
| 347 | return match |
| 348 | |
| 349 | def check_line(self, output, regex): |
| 350 | regex = re.compile(r'^' + regex, re.MULTILINE) |
| 351 | self.assertRegex(output, regex) |
| 352 | |
| 353 | def parse_executed_tests(self, output): |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame^] | 354 | regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' |
| 355 | % self.TESTNAME_REGEX) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 356 | parser = re.finditer(regex, output, re.MULTILINE) |
| 357 | return list(match.group(1) for match in parser) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 358 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 359 | def check_executed_tests(self, output, tests, skipped=(), failed=(), |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 360 | omitted=(), randomize=False): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 361 | if isinstance(tests, str): |
| 362 | tests = [tests] |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 363 | if isinstance(skipped, str): |
| 364 | skipped = [skipped] |
| 365 | if isinstance(failed, str): |
| 366 | failed = [failed] |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 367 | if isinstance(omitted, str): |
| 368 | omitted = [omitted] |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 369 | ntest = len(tests) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 370 | nskipped = len(skipped) |
| 371 | nfailed = len(failed) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 372 | nomitted = len(omitted) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 373 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 374 | executed = self.parse_executed_tests(output) |
| 375 | if randomize: |
| 376 | self.assertEqual(set(executed), set(tests), output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 377 | else: |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 378 | self.assertEqual(executed, tests, output) |
| 379 | |
| 380 | def plural(count): |
| 381 | return 's' if count != 1 else '' |
| 382 | |
| 383 | def list_regex(line_format, tests): |
| 384 | count = len(tests) |
| 385 | names = ' '.join(sorted(tests)) |
| 386 | regex = line_format % (count, plural(count)) |
| 387 | regex = r'%s:\n %s$' % (regex, names) |
| 388 | return regex |
| 389 | |
| 390 | if skipped: |
| 391 | regex = list_regex('%s test%s skipped', skipped) |
| 392 | self.check_line(output, regex) |
| 393 | |
| 394 | if failed: |
| 395 | regex = list_regex('%s test%s failed', failed) |
| 396 | self.check_line(output, regex) |
| 397 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 398 | if omitted: |
| 399 | regex = list_regex('%s test%s omitted', omitted) |
| 400 | self.check_line(output, regex) |
| 401 | |
| 402 | good = ntest - nskipped - nfailed - nomitted |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 403 | if good: |
| 404 | regex = r'%s test%s OK\.$' % (good, plural(good)) |
| 405 | if not skipped and not failed and good > 1: |
| 406 | regex = 'All %s' % regex |
| 407 | self.check_line(output, regex) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 408 | |
| 409 | def parse_random_seed(self, output): |
| 410 | match = self.regex_search(r'Using random seed ([0-9]+)', output) |
| 411 | randseed = int(match.group(1)) |
| 412 | self.assertTrue(0 <= randseed <= 10000000, randseed) |
| 413 | return randseed |
| 414 | |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 415 | def run_command(self, args, input=None, exitcode=0, **kw): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 416 | if not input: |
| 417 | input = '' |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 418 | if 'stderr' not in kw: |
| 419 | kw['stderr'] = subprocess.PIPE |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 420 | proc = subprocess.run(args, |
| 421 | universal_newlines=True, |
| 422 | input=input, |
| 423 | stdout=subprocess.PIPE, |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 424 | **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 425 | if proc.returncode != exitcode: |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 426 | msg = ("Command %s failed with exit code %s\n" |
| 427 | "\n" |
| 428 | "stdout:\n" |
| 429 | "---\n" |
| 430 | "%s\n" |
| 431 | "---\n" |
| 432 | % (str(args), proc.returncode, proc.stdout)) |
| 433 | if proc.stderr: |
| 434 | msg += ("\n" |
| 435 | "stderr:\n" |
| 436 | "---\n" |
| 437 | "%s" |
| 438 | "---\n" |
| 439 | % proc.stderr) |
| 440 | self.fail(msg) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 441 | return proc |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 442 | |
| 443 | |
| 444 | def run_python(self, args, **kw): |
| 445 | args = [sys.executable, '-X', 'faulthandler', '-I', *args] |
| 446 | proc = self.run_command(args, **kw) |
| 447 | return proc.stdout |
| 448 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 449 | |
| 450 | class ProgramsTestCase(BaseTestCase): |
| 451 | """ |
| 452 | Test various ways to run the Python test suite. Use options close |
| 453 | to options used on the buildbot. |
| 454 | """ |
| 455 | |
| 456 | NTEST = 4 |
| 457 | |
| 458 | def setUp(self): |
| 459 | super().setUp() |
| 460 | |
| 461 | # Create NTEST tests doing nothing |
| 462 | self.tests = [self.create_test() for index in range(self.NTEST)] |
| 463 | |
| 464 | self.python_args = ['-Wd', '-E', '-bb'] |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 465 | self.regrtest_args = ['-uall', '-rwW'] |
| 466 | if hasattr(faulthandler, 'dump_traceback_later'): |
| 467 | self.regrtest_args.extend(('--timeout', '3600', '-j4')) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 468 | if sys.platform == 'win32': |
| 469 | self.regrtest_args.append('-n') |
| 470 | |
| 471 | def check_output(self, output): |
| 472 | self.parse_random_seed(output) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 473 | self.check_executed_tests(output, self.tests, randomize=True) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 474 | |
| 475 | def run_tests(self, args): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 476 | output = self.run_python(args) |
| 477 | self.check_output(output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 478 | |
| 479 | def test_script_regrtest(self): |
| 480 | # Lib/test/regrtest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 481 | script = os.path.join(self.testdir, 'regrtest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 482 | |
| 483 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 484 | self.run_tests(args) |
| 485 | |
| 486 | def test_module_test(self): |
| 487 | # -m test |
| 488 | args = [*self.python_args, '-m', 'test', |
| 489 | *self.regrtest_args, *self.tests] |
| 490 | self.run_tests(args) |
| 491 | |
| 492 | def test_module_regrtest(self): |
| 493 | # -m test.regrtest |
| 494 | args = [*self.python_args, '-m', 'test.regrtest', |
| 495 | *self.regrtest_args, *self.tests] |
| 496 | self.run_tests(args) |
| 497 | |
| 498 | def test_module_autotest(self): |
| 499 | # -m test.autotest |
| 500 | args = [*self.python_args, '-m', 'test.autotest', |
| 501 | *self.regrtest_args, *self.tests] |
| 502 | self.run_tests(args) |
| 503 | |
| 504 | def test_module_from_test_autotest(self): |
| 505 | # from test import autotest |
| 506 | code = 'from test import autotest' |
| 507 | args = [*self.python_args, '-c', code, |
| 508 | *self.regrtest_args, *self.tests] |
| 509 | self.run_tests(args) |
| 510 | |
| 511 | def test_script_autotest(self): |
| 512 | # Lib/test/autotest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 513 | script = os.path.join(self.testdir, 'autotest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 514 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 515 | self.run_tests(args) |
| 516 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 517 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 518 | 'run_tests.py script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 519 | def test_tools_script_run_tests(self): |
| 520 | # Tools/scripts/run_tests.py |
| 521 | script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') |
| 522 | self.run_tests([script, *self.tests]) |
| 523 | |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 524 | def run_batch(self, *args): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 525 | proc = self.run_command(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 526 | self.check_output(proc.stdout) |
| 527 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 528 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 529 | 'test.bat script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 530 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 531 | def test_tools_buildbot_test(self): |
| 532 | # Tools\buildbot\test.bat |
| 533 | script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 534 | test_args = [] |
| 535 | if platform.architecture()[0] == '64bit': |
| 536 | test_args.append('-x64') # 64-bit build |
| 537 | if not Py_DEBUG: |
| 538 | test_args.append('+d') # Release build, use python.exe |
| 539 | self.run_batch(script, *test_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 540 | |
| 541 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 542 | def test_pcbuild_rt(self): |
| 543 | # PCbuild\rt.bat |
| 544 | script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 545 | rt_args = ["-q"] # Quick, don't run tests twice |
| 546 | if platform.architecture()[0] == '64bit': |
| 547 | rt_args.append('-x64') # 64-bit build |
| 548 | if Py_DEBUG: |
| 549 | rt_args.append('-d') # Debug build, use python_d.exe |
| 550 | self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 551 | |
| 552 | |
| 553 | class ArgsTestCase(BaseTestCase): |
| 554 | """ |
| 555 | Test arguments of the Python test suite. |
| 556 | """ |
| 557 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 558 | def run_tests(self, *args, **kw): |
| 559 | return self.run_python(['-m', 'test', *args], **kw) |
| 560 | |
| 561 | def test_failing_test(self): |
| 562 | # test a failing test |
| 563 | code = textwrap.dedent(""" |
| 564 | import unittest |
| 565 | |
| 566 | class FailingTest(unittest.TestCase): |
| 567 | def test_failing(self): |
| 568 | self.fail("bug") |
| 569 | """) |
| 570 | test_ok = self.create_test() |
| 571 | test_failing = self.create_test(code=code) |
| 572 | tests = [test_ok, test_failing] |
| 573 | |
| 574 | output = self.run_tests(*tests, exitcode=1) |
| 575 | self.check_executed_tests(output, tests, failed=test_failing) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 576 | |
| 577 | def test_resources(self): |
| 578 | # test -u command line option |
| 579 | tests = {} |
| 580 | for resource in ('audio', 'network'): |
| 581 | code = 'from test import support\nsupport.requires(%r)' % resource |
| 582 | tests[resource] = self.create_test(resource, code) |
| 583 | test_names = sorted(tests.values()) |
| 584 | |
| 585 | # -u all: 2 resources enabled |
| 586 | output = self.run_tests('-u', 'all', *test_names) |
| 587 | self.check_executed_tests(output, test_names) |
| 588 | |
| 589 | # -u audio: 1 resource enabled |
| 590 | output = self.run_tests('-uaudio', *test_names) |
| 591 | self.check_executed_tests(output, test_names, |
| 592 | skipped=tests['network']) |
| 593 | |
| 594 | # no option: 0 resources enabled |
| 595 | output = self.run_tests(*test_names) |
| 596 | self.check_executed_tests(output, test_names, |
| 597 | skipped=test_names) |
| 598 | |
| 599 | def test_random(self): |
| 600 | # test -r and --randseed command line option |
| 601 | code = textwrap.dedent(""" |
| 602 | import random |
| 603 | print("TESTRANDOM: %s" % random.randint(1, 1000)) |
| 604 | """) |
| 605 | test = self.create_test('random', code) |
| 606 | |
| 607 | # first run to get the output with the random seed |
| 608 | output = self.run_tests('-r', test) |
| 609 | randseed = self.parse_random_seed(output) |
| 610 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 611 | test_random = int(match.group(1)) |
| 612 | |
| 613 | # try to reproduce with the random seed |
| 614 | output = self.run_tests('-r', '--randseed=%s' % randseed, test) |
| 615 | randseed2 = self.parse_random_seed(output) |
| 616 | self.assertEqual(randseed2, randseed) |
| 617 | |
| 618 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 619 | test_random2 = int(match.group(1)) |
| 620 | self.assertEqual(test_random2, test_random) |
| 621 | |
| 622 | def test_fromfile(self): |
| 623 | # test --fromfile |
| 624 | tests = [self.create_test() for index in range(5)] |
| 625 | |
| 626 | # Write the list of files using a format similar to regrtest output: |
| 627 | # [1/2] test_1 |
| 628 | # [2/2] test_2 |
| 629 | filename = support.TESTFN |
| 630 | self.addCleanup(support.unlink, filename) |
| 631 | with open(filename, "w") as fp: |
| 632 | for index, name in enumerate(tests, 1): |
| 633 | print("[%s/%s] %s" % (index, len(tests), name), file=fp) |
| 634 | |
| 635 | output = self.run_tests('--fromfile', filename) |
| 636 | self.check_executed_tests(output, tests) |
| 637 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 638 | def test_interrupted(self): |
| 639 | code = TEST_INTERRUPTED |
| 640 | test = self.create_test("sigint", code=code) |
| 641 | output = self.run_tests(test, exitcode=1) |
| 642 | self.check_executed_tests(output, test, omitted=test) |
| 643 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 644 | def test_slow(self): |
| 645 | # test --slow |
| 646 | tests = [self.create_test() for index in range(3)] |
| 647 | output = self.run_tests("--slow", *tests) |
| 648 | self.check_executed_tests(output, tests) |
| 649 | regex = ('10 slowest tests:\n' |
| 650 | '(?:%s: [0-9]+\.[0-9]+s\n){%s}' |
| 651 | % (self.TESTNAME_REGEX, len(tests))) |
| 652 | self.check_line(output, regex) |
| 653 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 654 | def test_slow_interrupted(self): |
| 655 | # Issue #25373: test --slow with an interrupted test |
| 656 | code = TEST_INTERRUPTED |
| 657 | test = self.create_test("sigint", code=code) |
| 658 | |
| 659 | for multiprocessing in (False, True): |
| 660 | if multiprocessing: |
| 661 | args = ("--slow", "-j2", test) |
| 662 | else: |
| 663 | args = ("--slow", test) |
| 664 | output = self.run_tests(*args, exitcode=1) |
| 665 | self.check_executed_tests(output, test, omitted=test) |
| 666 | regex = ('10 slowest tests:\n') |
| 667 | self.check_line(output, regex) |
| 668 | self.check_line(output, 'Test suite interrupted by signal SIGINT.') |
| 669 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 670 | def test_coverage(self): |
| 671 | # test --coverage |
| 672 | test = self.create_test() |
| 673 | output = self.run_tests("--coverage", test) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 674 | self.check_executed_tests(output, [test]) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 675 | regex = ('lines +cov% +module +\(path\)\n' |
| 676 | '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') |
| 677 | self.check_line(output, regex) |
| 678 | |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 679 | def test_wait(self): |
| 680 | # test --wait |
| 681 | test = self.create_test() |
| 682 | output = self.run_tests("--wait", test, input='key') |
| 683 | self.check_line(output, 'Press any key to continue') |
| 684 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 685 | def test_forever(self): |
| 686 | # test --forever |
| 687 | code = textwrap.dedent(""" |
| 688 | import unittest |
| 689 | |
| 690 | class ForeverTester(unittest.TestCase): |
| 691 | RUN = 1 |
| 692 | |
| 693 | def test_run(self): |
| 694 | ForeverTester.RUN += 1 |
| 695 | if ForeverTester.RUN > 3: |
| 696 | self.fail("fail at the 3rd runs") |
| 697 | """) |
| 698 | test = self.create_test(code=code) |
| 699 | output = self.run_tests('--forever', test, exitcode=1) |
| 700 | self.check_executed_tests(output, [test]*3, failed=test) |
| 701 | |
Victor Stinner | 46d7535 | 2015-10-03 02:21:35 +0200 | [diff] [blame] | 702 | @unittest.skipUnless(Py_DEBUG, 'need a debug build') |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 703 | def test_huntrleaks_fd_leak(self): |
| 704 | # test --huntrleaks for file descriptor leak |
| 705 | code = textwrap.dedent(""" |
| 706 | import os |
| 707 | import unittest |
| 708 | |
Victor Stinner | 20d2118 | 2015-10-03 21:40:21 +0200 | [diff] [blame] | 709 | # Issue #25306: Disable popups and logs to stderr on assertion |
| 710 | # failures in MSCRT |
| 711 | try: |
| 712 | import msvcrt |
| 713 | msvcrt.CrtSetReportMode |
| 714 | except (ImportError, AttributeError): |
| 715 | # no Windows, o release build |
| 716 | pass |
| 717 | else: |
| 718 | for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: |
| 719 | msvcrt.CrtSetReportMode(m, 0) |
| 720 | |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 721 | class FDLeakTest(unittest.TestCase): |
| 722 | def test_leak(self): |
| 723 | fd = os.open(__file__, os.O_RDONLY) |
| 724 | # bug: never cloes the file descriptor |
| 725 | """) |
| 726 | test = self.create_test(code=code) |
| 727 | |
| 728 | filename = 'reflog.txt' |
| 729 | self.addCleanup(support.unlink, filename) |
| 730 | output = self.run_tests('--huntrleaks', '3:3:', test, |
| 731 | exitcode=1, |
| 732 | stderr=subprocess.STDOUT) |
| 733 | self.check_executed_tests(output, [test], failed=test) |
| 734 | |
| 735 | line = 'beginning 6 repetitions\n123456\n......\n' |
| 736 | self.check_line(output, re.escape(line)) |
| 737 | |
| 738 | line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test |
| 739 | self.check_line(output, re.escape(line2)) |
| 740 | |
| 741 | with open(filename) as fp: |
| 742 | reflog = fp.read() |
| 743 | self.assertEqual(reflog, line2) |
| 744 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 745 | def test_list_tests(self): |
| 746 | # test --list-tests |
| 747 | tests = [self.create_test() for i in range(5)] |
| 748 | output = self.run_tests('--list-tests', *tests) |
| 749 | self.assertEqual(output.rstrip().splitlines(), |
| 750 | tests) |
| 751 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 752 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 753 | if __name__ == '__main__': |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 754 | unittest.main() |