Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 1 | """ |
| 2 | Tests of regrtest.py. |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 3 | |
| 4 | Note: test_regrtest cannot be run twice in parallel. |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 5 | """ |
| 6 | |
| 7 | import argparse |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 8 | import contextlib |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 9 | import faulthandler |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 10 | import getopt |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 11 | import io |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 12 | import os.path |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 13 | import platform |
| 14 | import re |
| 15 | import subprocess |
| 16 | import sys |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 17 | import sysconfig |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 18 | import tempfile |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 19 | import textwrap |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 20 | import unittest |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 21 | from test import libregrtest |
| 22 | from test import support |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 23 | |
| 24 | |
| 25 | Py_DEBUG = hasattr(sys, 'getobjects') |
| 26 | ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') |
| 27 | ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) |
| 28 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 29 | TEST_INTERRUPTED = textwrap.dedent(""" |
| 30 | from signal import SIGINT |
| 31 | try: |
| 32 | from _testcapi import raise_signal |
| 33 | raise_signal(SIGINT) |
| 34 | except ImportError: |
| 35 | import os |
| 36 | os.kill(os.getpid(), SIGINT) |
| 37 | """) |
| 38 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 39 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 40 | class ParseArgsTestCase(unittest.TestCase): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 41 | """ |
| 42 | Test regrtest's argument parsing, function _parse_args(). |
| 43 | """ |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 44 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 45 | def checkError(self, args, msg): |
| 46 | with support.captured_stderr() as err, self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 47 | libregrtest._parse_args(args) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 48 | self.assertIn(msg, err.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 49 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 50 | def test_help(self): |
| 51 | for opt in '-h', '--help': |
| 52 | with self.subTest(opt=opt): |
| 53 | with support.captured_stdout() as out, \ |
| 54 | self.assertRaises(SystemExit): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 55 | libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 56 | self.assertIn('Run Python regression tests.', out.getvalue()) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 57 | |
Charles-François Natali | 833bf1f | 2013-09-08 12:27:33 +0200 | [diff] [blame] | 58 | @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), |
| 59 | "faulthandler.dump_traceback_later() required") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 60 | def test_timeout(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 61 | ns = libregrtest._parse_args(['--timeout', '4.2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 62 | self.assertEqual(ns.timeout, 4.2) |
| 63 | self.checkError(['--timeout'], 'expected one argument') |
| 64 | self.checkError(['--timeout', 'foo'], 'invalid float value') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 65 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 66 | def test_wait(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 67 | ns = libregrtest._parse_args(['--wait']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 68 | self.assertTrue(ns.wait) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 69 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 70 | def test_slaveargs(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 71 | ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 72 | self.assertEqual(ns.slaveargs, '[[], {}]') |
| 73 | self.checkError(['--slaveargs'], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 74 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 75 | def test_start(self): |
| 76 | for opt in '-S', '--start': |
| 77 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 78 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 79 | self.assertEqual(ns.start, 'foo') |
| 80 | self.checkError([opt], 'expected one argument') |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 81 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 82 | def test_verbose(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 83 | ns = libregrtest._parse_args(['-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 84 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 85 | ns = libregrtest._parse_args(['-vvv']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 86 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 87 | ns = libregrtest._parse_args(['--verbose']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 88 | self.assertEqual(ns.verbose, 1) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 89 | ns = libregrtest._parse_args(['--verbose'] * 3) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 90 | self.assertEqual(ns.verbose, 3) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 91 | ns = libregrtest._parse_args([]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 92 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 93 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 94 | def test_verbose2(self): |
| 95 | for opt in '-w', '--verbose2': |
| 96 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 97 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 98 | self.assertTrue(ns.verbose2) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 99 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 100 | def test_verbose3(self): |
| 101 | for opt in '-W', '--verbose3': |
| 102 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 103 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 104 | self.assertTrue(ns.verbose3) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 105 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 106 | def test_quiet(self): |
| 107 | for opt in '-q', '--quiet': |
| 108 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 109 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 110 | self.assertTrue(ns.quiet) |
| 111 | self.assertEqual(ns.verbose, 0) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 112 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 113 | def test_slow(self): |
| 114 | for opt in '-o', '--slow': |
| 115 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 116 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 117 | self.assertTrue(ns.print_slow) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 118 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 119 | def test_header(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 120 | ns = libregrtest._parse_args(['--header']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 121 | self.assertTrue(ns.header) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 122 | |
| 123 | def test_randomize(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 124 | for opt in '-r', '--randomize': |
| 125 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 126 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 127 | self.assertTrue(ns.randomize) |
| 128 | |
| 129 | def test_randseed(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 130 | ns = libregrtest._parse_args(['--randseed', '12345']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 131 | self.assertEqual(ns.random_seed, 12345) |
| 132 | self.assertTrue(ns.randomize) |
| 133 | self.checkError(['--randseed'], 'expected one argument') |
| 134 | self.checkError(['--randseed', 'foo'], 'invalid int value') |
| 135 | |
| 136 | def test_fromfile(self): |
| 137 | for opt in '-f', '--fromfile': |
| 138 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 139 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 140 | self.assertEqual(ns.fromfile, 'foo') |
| 141 | self.checkError([opt], 'expected one argument') |
| 142 | self.checkError([opt, 'foo', '-s'], "don't go together") |
| 143 | |
| 144 | def test_exclude(self): |
| 145 | for opt in '-x', '--exclude': |
| 146 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 147 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 148 | self.assertTrue(ns.exclude) |
| 149 | |
| 150 | def test_single(self): |
| 151 | for opt in '-s', '--single': |
| 152 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 153 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 154 | self.assertTrue(ns.single) |
| 155 | self.checkError([opt, '-f', 'foo'], "don't go together") |
| 156 | |
| 157 | def test_match(self): |
| 158 | for opt in '-m', '--match': |
| 159 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 160 | ns = libregrtest._parse_args([opt, 'pattern']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 161 | self.assertEqual(ns.match_tests, 'pattern') |
| 162 | self.checkError([opt], 'expected one argument') |
| 163 | |
| 164 | def test_failfast(self): |
| 165 | for opt in '-G', '--failfast': |
| 166 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 167 | ns = libregrtest._parse_args([opt, '-v']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 168 | self.assertTrue(ns.failfast) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 169 | ns = libregrtest._parse_args([opt, '-W']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 170 | self.assertTrue(ns.failfast) |
| 171 | self.checkError([opt], '-G/--failfast needs either -v or -W') |
| 172 | |
| 173 | def test_use(self): |
| 174 | for opt in '-u', '--use': |
| 175 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 176 | ns = libregrtest._parse_args([opt, 'gui,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 177 | self.assertEqual(ns.use_resources, ['gui', 'network']) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 178 | ns = libregrtest._parse_args([opt, 'gui,none,network']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 179 | self.assertEqual(ns.use_resources, ['network']) |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 180 | expected = list(libregrtest.RESOURCE_NAMES) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 181 | expected.remove('gui') |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 182 | ns = libregrtest._parse_args([opt, 'all,-gui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 183 | self.assertEqual(ns.use_resources, expected) |
| 184 | self.checkError([opt], 'expected one argument') |
| 185 | self.checkError([opt, 'foo'], 'invalid resource') |
| 186 | |
| 187 | def test_memlimit(self): |
| 188 | for opt in '-M', '--memlimit': |
| 189 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 190 | ns = libregrtest._parse_args([opt, '4G']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 191 | self.assertEqual(ns.memlimit, '4G') |
| 192 | self.checkError([opt], 'expected one argument') |
| 193 | |
| 194 | def test_testdir(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 195 | ns = libregrtest._parse_args(['--testdir', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 196 | self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) |
| 197 | self.checkError(['--testdir'], 'expected one argument') |
| 198 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 199 | def test_runleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 200 | for opt in '-L', '--runleaks': |
| 201 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 202 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 203 | self.assertTrue(ns.runleaks) |
| 204 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 205 | def test_huntrleaks(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 206 | for opt in '-R', '--huntrleaks': |
| 207 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 208 | ns = libregrtest._parse_args([opt, ':']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 209 | self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 210 | ns = libregrtest._parse_args([opt, '6:']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 211 | self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 212 | ns = libregrtest._parse_args([opt, ':3']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 213 | self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 214 | ns = libregrtest._parse_args([opt, '6:3:leaks.log']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 215 | self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) |
| 216 | self.checkError([opt], 'expected one argument') |
| 217 | self.checkError([opt, '6'], |
| 218 | 'needs 2 or 3 colon-separated arguments') |
| 219 | self.checkError([opt, 'foo:'], 'invalid huntrleaks value') |
| 220 | self.checkError([opt, '6:foo'], 'invalid huntrleaks value') |
| 221 | |
| 222 | def test_multiprocess(self): |
| 223 | for opt in '-j', '--multiprocess': |
| 224 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 225 | ns = libregrtest._parse_args([opt, '2']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 226 | self.assertEqual(ns.use_mp, 2) |
| 227 | self.checkError([opt], 'expected one argument') |
| 228 | self.checkError([opt, 'foo'], 'invalid int value') |
| 229 | self.checkError([opt, '2', '-T'], "don't go together") |
| 230 | self.checkError([opt, '2', '-l'], "don't go together") |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 231 | |
Benjamin Peterson | 95536b8 | 2013-09-29 15:18:43 -0400 | [diff] [blame] | 232 | def test_coverage(self): |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 233 | for opt in '-T', '--coverage': |
| 234 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 235 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 236 | self.assertTrue(ns.trace) |
| 237 | |
| 238 | def test_coverdir(self): |
| 239 | for opt in '-D', '--coverdir': |
| 240 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 241 | ns = libregrtest._parse_args([opt, 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 242 | self.assertEqual(ns.coverdir, |
| 243 | os.path.join(support.SAVEDCWD, 'foo')) |
| 244 | self.checkError([opt], 'expected one argument') |
| 245 | |
| 246 | def test_nocoverdir(self): |
| 247 | for opt in '-N', '--nocoverdir': |
| 248 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 249 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 250 | self.assertIsNone(ns.coverdir) |
| 251 | |
| 252 | def test_threshold(self): |
| 253 | for opt in '-t', '--threshold': |
| 254 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 255 | ns = libregrtest._parse_args([opt, '1000']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 256 | self.assertEqual(ns.threshold, 1000) |
| 257 | self.checkError([opt], 'expected one argument') |
| 258 | self.checkError([opt, 'foo'], 'invalid int value') |
| 259 | |
| 260 | def test_nowindows(self): |
| 261 | for opt in '-n', '--nowindows': |
| 262 | with self.subTest(opt=opt): |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 263 | with contextlib.redirect_stderr(io.StringIO()) as stderr: |
| 264 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 265 | self.assertTrue(ns.nowindows) |
Victor Stinner | 14b4662 | 2015-10-11 10:04:26 +0200 | [diff] [blame] | 266 | err = stderr.getvalue() |
| 267 | self.assertIn('the --nowindows (-n) option is deprecated', err) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 268 | |
| 269 | def test_forever(self): |
| 270 | for opt in '-F', '--forever': |
| 271 | with self.subTest(opt=opt): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 272 | ns = libregrtest._parse_args([opt]) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 273 | self.assertTrue(ns.forever) |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 274 | |
| 275 | |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 276 | def test_unrecognized_argument(self): |
| 277 | self.checkError(['--xxx'], 'usage:') |
| 278 | |
| 279 | def test_long_option__partial(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 280 | ns = libregrtest._parse_args(['--qui']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 281 | self.assertTrue(ns.quiet) |
| 282 | self.assertEqual(ns.verbose, 0) |
| 283 | |
| 284 | def test_two_options(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 285 | ns = libregrtest._parse_args(['--quiet', '--exclude']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 286 | self.assertTrue(ns.quiet) |
| 287 | self.assertEqual(ns.verbose, 0) |
| 288 | self.assertTrue(ns.exclude) |
| 289 | |
| 290 | def test_option_with_empty_string_value(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 291 | ns = libregrtest._parse_args(['--start', '']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 292 | self.assertEqual(ns.start, '') |
| 293 | |
| 294 | def test_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 295 | ns = libregrtest._parse_args(['foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 296 | self.assertEqual(ns.args, ['foo']) |
| 297 | |
| 298 | def test_option_and_arg(self): |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 299 | ns = libregrtest._parse_args(['--quiet', 'foo']) |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 300 | self.assertTrue(ns.quiet) |
| 301 | self.assertEqual(ns.verbose, 0) |
| 302 | self.assertEqual(ns.args, ['foo']) |
| 303 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 304 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 305 | class BaseTestCase(unittest.TestCase): |
| 306 | TEST_UNIQUE_ID = 1 |
| 307 | TESTNAME_PREFIX = 'test_regrtest_' |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 308 | TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 309 | |
| 310 | def setUp(self): |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 311 | self.testdir = os.path.realpath(os.path.dirname(__file__)) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 312 | |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 313 | self.tmptestdir = tempfile.mkdtemp() |
| 314 | self.addCleanup(support.rmtree, self.tmptestdir) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 315 | |
| 316 | def create_test(self, name=None, code=''): |
| 317 | if not name: |
| 318 | name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID |
| 319 | BaseTestCase.TEST_UNIQUE_ID += 1 |
| 320 | |
| 321 | # test_regrtest cannot be run twice in parallel because |
| 322 | # of setUp() and create_test() |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 323 | name = self.TESTNAME_PREFIX + name |
| 324 | path = os.path.join(self.tmptestdir, name + '.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 325 | |
| 326 | self.addCleanup(support.unlink, path) |
| 327 | # Use 'x' mode to ensure that we do not override existing tests |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 328 | try: |
| 329 | with open(path, 'x', encoding='utf-8') as fp: |
| 330 | fp.write(code) |
| 331 | except PermissionError as exc: |
| 332 | if not sysconfig.is_python_build(): |
| 333 | self.skipTest("cannot write %s: %s" % (path, exc)) |
| 334 | raise |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 335 | return name |
| 336 | |
| 337 | def regex_search(self, regex, output): |
| 338 | match = re.search(regex, output, re.MULTILINE) |
| 339 | if not match: |
| 340 | self.fail("%r not found in %r" % (regex, output)) |
| 341 | return match |
| 342 | |
| 343 | def check_line(self, output, regex): |
| 344 | regex = re.compile(r'^' + regex, re.MULTILINE) |
| 345 | self.assertRegex(output, regex) |
| 346 | |
| 347 | def parse_executed_tests(self, output): |
Victor Stinner | 69649f2 | 2016-03-23 12:14:10 +0100 | [diff] [blame] | 348 | regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' |
| 349 | % self.TESTNAME_REGEX) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 350 | parser = re.finditer(regex, output, re.MULTILINE) |
| 351 | return list(match.group(1) for match in parser) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 352 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 353 | def check_executed_tests(self, output, tests, skipped=(), failed=(), |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 354 | omitted=(), randomize=False): |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 355 | if isinstance(tests, str): |
| 356 | tests = [tests] |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 357 | if isinstance(skipped, str): |
| 358 | skipped = [skipped] |
| 359 | if isinstance(failed, str): |
| 360 | failed = [failed] |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 361 | if isinstance(omitted, str): |
| 362 | omitted = [omitted] |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 363 | ntest = len(tests) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 364 | nskipped = len(skipped) |
| 365 | nfailed = len(failed) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 366 | nomitted = len(omitted) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 367 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 368 | executed = self.parse_executed_tests(output) |
| 369 | if randomize: |
| 370 | self.assertEqual(set(executed), set(tests), output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 371 | else: |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 372 | self.assertEqual(executed, tests, output) |
| 373 | |
| 374 | def plural(count): |
| 375 | return 's' if count != 1 else '' |
| 376 | |
| 377 | def list_regex(line_format, tests): |
| 378 | count = len(tests) |
| 379 | names = ' '.join(sorted(tests)) |
| 380 | regex = line_format % (count, plural(count)) |
| 381 | regex = r'%s:\n %s$' % (regex, names) |
| 382 | return regex |
| 383 | |
| 384 | if skipped: |
| 385 | regex = list_regex('%s test%s skipped', skipped) |
| 386 | self.check_line(output, regex) |
| 387 | |
| 388 | if failed: |
| 389 | regex = list_regex('%s test%s failed', failed) |
| 390 | self.check_line(output, regex) |
| 391 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 392 | if omitted: |
| 393 | regex = list_regex('%s test%s omitted', omitted) |
| 394 | self.check_line(output, regex) |
| 395 | |
| 396 | good = ntest - nskipped - nfailed - nomitted |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 397 | if good: |
| 398 | regex = r'%s test%s OK\.$' % (good, plural(good)) |
| 399 | if not skipped and not failed and good > 1: |
| 400 | regex = 'All %s' % regex |
| 401 | self.check_line(output, regex) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 402 | |
| 403 | def parse_random_seed(self, output): |
| 404 | match = self.regex_search(r'Using random seed ([0-9]+)', output) |
| 405 | randseed = int(match.group(1)) |
| 406 | self.assertTrue(0 <= randseed <= 10000000, randseed) |
| 407 | return randseed |
| 408 | |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 409 | def run_command(self, args, input=None, exitcode=0, **kw): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 410 | if not input: |
| 411 | input = '' |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 412 | if 'stderr' not in kw: |
| 413 | kw['stderr'] = subprocess.PIPE |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 414 | proc = subprocess.run(args, |
| 415 | universal_newlines=True, |
| 416 | input=input, |
| 417 | stdout=subprocess.PIPE, |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 418 | **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 419 | if proc.returncode != exitcode: |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 420 | msg = ("Command %s failed with exit code %s\n" |
| 421 | "\n" |
| 422 | "stdout:\n" |
| 423 | "---\n" |
| 424 | "%s\n" |
| 425 | "---\n" |
| 426 | % (str(args), proc.returncode, proc.stdout)) |
| 427 | if proc.stderr: |
| 428 | msg += ("\n" |
| 429 | "stderr:\n" |
| 430 | "---\n" |
| 431 | "%s" |
| 432 | "---\n" |
| 433 | % proc.stderr) |
| 434 | self.fail(msg) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 435 | return proc |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 436 | |
| 437 | |
| 438 | def run_python(self, args, **kw): |
| 439 | args = [sys.executable, '-X', 'faulthandler', '-I', *args] |
| 440 | proc = self.run_command(args, **kw) |
| 441 | return proc.stdout |
| 442 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 443 | |
| 444 | class ProgramsTestCase(BaseTestCase): |
| 445 | """ |
| 446 | Test various ways to run the Python test suite. Use options close |
| 447 | to options used on the buildbot. |
| 448 | """ |
| 449 | |
| 450 | NTEST = 4 |
| 451 | |
| 452 | def setUp(self): |
| 453 | super().setUp() |
| 454 | |
| 455 | # Create NTEST tests doing nothing |
| 456 | self.tests = [self.create_test() for index in range(self.NTEST)] |
| 457 | |
| 458 | self.python_args = ['-Wd', '-E', '-bb'] |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 459 | self.regrtest_args = ['-uall', '-rwW', |
| 460 | '--testdir=%s' % self.tmptestdir] |
Victor Stinner | 4967146 | 2015-10-11 10:03:28 +0200 | [diff] [blame] | 461 | if hasattr(faulthandler, 'dump_traceback_later'): |
| 462 | self.regrtest_args.extend(('--timeout', '3600', '-j4')) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 463 | if sys.platform == 'win32': |
| 464 | self.regrtest_args.append('-n') |
| 465 | |
| 466 | def check_output(self, output): |
| 467 | self.parse_random_seed(output) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 468 | self.check_executed_tests(output, self.tests, randomize=True) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 469 | |
| 470 | def run_tests(self, args): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 471 | output = self.run_python(args) |
| 472 | self.check_output(output) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 473 | |
| 474 | def test_script_regrtest(self): |
| 475 | # Lib/test/regrtest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 476 | script = os.path.join(self.testdir, 'regrtest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 477 | |
| 478 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 479 | self.run_tests(args) |
| 480 | |
| 481 | def test_module_test(self): |
| 482 | # -m test |
| 483 | args = [*self.python_args, '-m', 'test', |
| 484 | *self.regrtest_args, *self.tests] |
| 485 | self.run_tests(args) |
| 486 | |
| 487 | def test_module_regrtest(self): |
| 488 | # -m test.regrtest |
| 489 | args = [*self.python_args, '-m', 'test.regrtest', |
| 490 | *self.regrtest_args, *self.tests] |
| 491 | self.run_tests(args) |
| 492 | |
| 493 | def test_module_autotest(self): |
| 494 | # -m test.autotest |
| 495 | args = [*self.python_args, '-m', 'test.autotest', |
| 496 | *self.regrtest_args, *self.tests] |
| 497 | self.run_tests(args) |
| 498 | |
| 499 | def test_module_from_test_autotest(self): |
| 500 | # from test import autotest |
| 501 | code = 'from test import autotest' |
| 502 | args = [*self.python_args, '-c', code, |
| 503 | *self.regrtest_args, *self.tests] |
| 504 | self.run_tests(args) |
| 505 | |
| 506 | def test_script_autotest(self): |
| 507 | # Lib/test/autotest.py |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 508 | script = os.path.join(self.testdir, 'autotest.py') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 509 | args = [*self.python_args, script, *self.regrtest_args, *self.tests] |
| 510 | self.run_tests(args) |
| 511 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 512 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 513 | 'run_tests.py script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 514 | def test_tools_script_run_tests(self): |
| 515 | # Tools/scripts/run_tests.py |
| 516 | script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 517 | args = [script, '--testdir=%s' % self.tmptestdir, *self.tests] |
| 518 | self.run_tests(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 519 | |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 520 | def run_batch(self, *args): |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 521 | proc = self.run_command(args) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 522 | self.check_output(proc.stdout) |
| 523 | |
Victor Stinner | da43ee4 | 2015-11-22 15:18:54 +0100 | [diff] [blame] | 524 | @unittest.skipUnless(sysconfig.is_python_build(), |
| 525 | 'test.bat script is not installed') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 526 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 527 | def test_tools_buildbot_test(self): |
| 528 | # Tools\buildbot\test.bat |
| 529 | script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') |
Victor Stinner | 732599f | 2016-03-30 08:38:05 +0200 | [diff] [blame^] | 530 | test_args = ['--testdir=%s' % self.tmptestdir] |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 531 | if platform.architecture()[0] == '64bit': |
| 532 | test_args.append('-x64') # 64-bit build |
| 533 | if not Py_DEBUG: |
| 534 | test_args.append('+d') # Release build, use python.exe |
| 535 | self.run_batch(script, *test_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 536 | |
| 537 | @unittest.skipUnless(sys.platform == 'win32', 'Windows only') |
| 538 | def test_pcbuild_rt(self): |
| 539 | # PCbuild\rt.bat |
| 540 | script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') |
Victor Stinner | e143c1f | 2015-09-29 01:02:37 +0200 | [diff] [blame] | 541 | rt_args = ["-q"] # Quick, don't run tests twice |
| 542 | if platform.architecture()[0] == '64bit': |
| 543 | rt_args.append('-x64') # 64-bit build |
| 544 | if Py_DEBUG: |
| 545 | rt_args.append('-d') # Debug build, use python_d.exe |
| 546 | self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 547 | |
| 548 | |
| 549 | class ArgsTestCase(BaseTestCase): |
| 550 | """ |
| 551 | Test arguments of the Python test suite. |
| 552 | """ |
| 553 | |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 554 | def run_tests(self, *testargs, **kw): |
| 555 | cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] |
| 556 | return self.run_python(cmdargs, **kw) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 557 | |
| 558 | def test_failing_test(self): |
| 559 | # test a failing test |
| 560 | code = textwrap.dedent(""" |
| 561 | import unittest |
| 562 | |
| 563 | class FailingTest(unittest.TestCase): |
| 564 | def test_failing(self): |
| 565 | self.fail("bug") |
| 566 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 567 | test_ok = self.create_test('ok') |
| 568 | test_failing = self.create_test('failing', code=code) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 569 | tests = [test_ok, test_failing] |
| 570 | |
| 571 | output = self.run_tests(*tests, exitcode=1) |
| 572 | self.check_executed_tests(output, tests, failed=test_failing) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 573 | |
| 574 | def test_resources(self): |
| 575 | # test -u command line option |
| 576 | tests = {} |
| 577 | for resource in ('audio', 'network'): |
| 578 | code = 'from test import support\nsupport.requires(%r)' % resource |
| 579 | tests[resource] = self.create_test(resource, code) |
| 580 | test_names = sorted(tests.values()) |
| 581 | |
| 582 | # -u all: 2 resources enabled |
| 583 | output = self.run_tests('-u', 'all', *test_names) |
| 584 | self.check_executed_tests(output, test_names) |
| 585 | |
| 586 | # -u audio: 1 resource enabled |
| 587 | output = self.run_tests('-uaudio', *test_names) |
| 588 | self.check_executed_tests(output, test_names, |
| 589 | skipped=tests['network']) |
| 590 | |
| 591 | # no option: 0 resources enabled |
| 592 | output = self.run_tests(*test_names) |
| 593 | self.check_executed_tests(output, test_names, |
| 594 | skipped=test_names) |
| 595 | |
| 596 | def test_random(self): |
| 597 | # test -r and --randseed command line option |
| 598 | code = textwrap.dedent(""" |
| 599 | import random |
| 600 | print("TESTRANDOM: %s" % random.randint(1, 1000)) |
| 601 | """) |
| 602 | test = self.create_test('random', code) |
| 603 | |
| 604 | # first run to get the output with the random seed |
| 605 | output = self.run_tests('-r', test) |
| 606 | randseed = self.parse_random_seed(output) |
| 607 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 608 | test_random = int(match.group(1)) |
| 609 | |
| 610 | # try to reproduce with the random seed |
| 611 | output = self.run_tests('-r', '--randseed=%s' % randseed, test) |
| 612 | randseed2 = self.parse_random_seed(output) |
| 613 | self.assertEqual(randseed2, randseed) |
| 614 | |
| 615 | match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) |
| 616 | test_random2 = int(match.group(1)) |
| 617 | self.assertEqual(test_random2, test_random) |
| 618 | |
| 619 | def test_fromfile(self): |
| 620 | # test --fromfile |
| 621 | tests = [self.create_test() for index in range(5)] |
| 622 | |
| 623 | # Write the list of files using a format similar to regrtest output: |
| 624 | # [1/2] test_1 |
| 625 | # [2/2] test_2 |
| 626 | filename = support.TESTFN |
| 627 | self.addCleanup(support.unlink, filename) |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 628 | |
| 629 | # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec' |
| 630 | with open(filename, "w") as fp: |
| 631 | previous = None |
| 632 | for index, name in enumerate(tests, 1): |
| 633 | line = ("00:00:%02i [%s/%s] %s" |
| 634 | % (index, index, len(tests), name)) |
| 635 | if previous: |
| 636 | line += " -- %s took 0 sec" % previous |
| 637 | print(line, file=fp) |
| 638 | previous = name |
| 639 | |
| 640 | output = self.run_tests('--fromfile', filename) |
| 641 | self.check_executed_tests(output, tests) |
| 642 | |
| 643 | # test format '[2/7] test_opcodes' |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 644 | with open(filename, "w") as fp: |
| 645 | for index, name in enumerate(tests, 1): |
| 646 | print("[%s/%s] %s" % (index, len(tests), name), file=fp) |
| 647 | |
| 648 | output = self.run_tests('--fromfile', filename) |
| 649 | self.check_executed_tests(output, tests) |
| 650 | |
Victor Stinner | 5de16e8 | 2016-03-24 09:43:00 +0100 | [diff] [blame] | 651 | # test format 'test_opcodes' |
| 652 | with open(filename, "w") as fp: |
| 653 | for name in tests: |
| 654 | print(name, file=fp) |
| 655 | |
| 656 | output = self.run_tests('--fromfile', filename) |
| 657 | self.check_executed_tests(output, tests) |
| 658 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 659 | def test_interrupted(self): |
| 660 | code = TEST_INTERRUPTED |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 661 | test = self.create_test('sigint', code=code) |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 662 | output = self.run_tests(test, exitcode=1) |
| 663 | self.check_executed_tests(output, test, omitted=test) |
| 664 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 665 | def test_slow(self): |
| 666 | # test --slow |
| 667 | tests = [self.create_test() for index in range(3)] |
| 668 | output = self.run_tests("--slow", *tests) |
| 669 | self.check_executed_tests(output, tests) |
| 670 | regex = ('10 slowest tests:\n' |
| 671 | '(?:%s: [0-9]+\.[0-9]+s\n){%s}' |
| 672 | % (self.TESTNAME_REGEX, len(tests))) |
| 673 | self.check_line(output, regex) |
| 674 | |
Victor Stinner | 3909e58 | 2015-10-11 10:37:25 +0200 | [diff] [blame] | 675 | def test_slow_interrupted(self): |
| 676 | # Issue #25373: test --slow with an interrupted test |
| 677 | code = TEST_INTERRUPTED |
| 678 | test = self.create_test("sigint", code=code) |
| 679 | |
| 680 | for multiprocessing in (False, True): |
| 681 | if multiprocessing: |
| 682 | args = ("--slow", "-j2", test) |
| 683 | else: |
| 684 | args = ("--slow", test) |
| 685 | output = self.run_tests(*args, exitcode=1) |
| 686 | self.check_executed_tests(output, test, omitted=test) |
| 687 | regex = ('10 slowest tests:\n') |
| 688 | self.check_line(output, regex) |
| 689 | self.check_line(output, 'Test suite interrupted by signal SIGINT.') |
| 690 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 691 | def test_coverage(self): |
| 692 | # test --coverage |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 693 | test = self.create_test('coverage') |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 694 | output = self.run_tests("--coverage", test) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 695 | self.check_executed_tests(output, [test]) |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 696 | regex = ('lines +cov% +module +\(path\)\n' |
| 697 | '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') |
| 698 | self.check_line(output, regex) |
| 699 | |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 700 | def test_wait(self): |
| 701 | # test --wait |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 702 | test = self.create_test('wait') |
Victor Stinner | 4277134 | 2015-09-29 14:17:09 +0200 | [diff] [blame] | 703 | output = self.run_tests("--wait", test, input='key') |
| 704 | self.check_line(output, 'Press any key to continue') |
| 705 | |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 706 | def test_forever(self): |
| 707 | # test --forever |
| 708 | code = textwrap.dedent(""" |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 709 | import builtins |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 710 | import unittest |
| 711 | |
| 712 | class ForeverTester(unittest.TestCase): |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 713 | def test_run(self): |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 714 | # Store the state in the builtins module, because the test |
| 715 | # module is reload at each run |
| 716 | if 'RUN' in builtins.__dict__: |
| 717 | builtins.__dict__['RUN'] += 1 |
| 718 | if builtins.__dict__['RUN'] >= 3: |
| 719 | self.fail("fail at the 3rd runs") |
| 720 | else: |
| 721 | builtins.__dict__['RUN'] = 1 |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 722 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 723 | test = self.create_test('forever', code=code) |
Victor Stinner | 9a14214 | 2015-09-30 13:51:17 +0200 | [diff] [blame] | 724 | output = self.run_tests('--forever', test, exitcode=1) |
| 725 | self.check_executed_tests(output, [test]*3, failed=test) |
| 726 | |
Victor Stinner | 46d7535 | 2015-10-03 02:21:35 +0200 | [diff] [blame] | 727 | @unittest.skipUnless(Py_DEBUG, 'need a debug build') |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 728 | def test_huntrleaks_fd_leak(self): |
| 729 | # test --huntrleaks for file descriptor leak |
| 730 | code = textwrap.dedent(""" |
| 731 | import os |
| 732 | import unittest |
| 733 | |
Victor Stinner | 20d2118 | 2015-10-03 21:40:21 +0200 | [diff] [blame] | 734 | # Issue #25306: Disable popups and logs to stderr on assertion |
| 735 | # failures in MSCRT |
| 736 | try: |
| 737 | import msvcrt |
| 738 | msvcrt.CrtSetReportMode |
| 739 | except (ImportError, AttributeError): |
| 740 | # no Windows, o release build |
| 741 | pass |
| 742 | else: |
| 743 | for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: |
| 744 | msvcrt.CrtSetReportMode(m, 0) |
| 745 | |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 746 | class FDLeakTest(unittest.TestCase): |
| 747 | def test_leak(self): |
| 748 | fd = os.open(__file__, os.O_RDONLY) |
| 749 | # bug: never cloes the file descriptor |
| 750 | """) |
Victor Stinner | d6e2502 | 2016-03-30 02:33:52 +0200 | [diff] [blame] | 751 | test = self.create_test('huntrleaks', code=code) |
Victor Stinner | 076fc87 | 2015-10-03 00:20:56 +0200 | [diff] [blame] | 752 | |
| 753 | filename = 'reflog.txt' |
| 754 | self.addCleanup(support.unlink, filename) |
| 755 | output = self.run_tests('--huntrleaks', '3:3:', test, |
| 756 | exitcode=1, |
| 757 | stderr=subprocess.STDOUT) |
| 758 | self.check_executed_tests(output, [test], failed=test) |
| 759 | |
| 760 | line = 'beginning 6 repetitions\n123456\n......\n' |
| 761 | self.check_line(output, re.escape(line)) |
| 762 | |
| 763 | line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test |
| 764 | self.check_line(output, re.escape(line2)) |
| 765 | |
| 766 | with open(filename) as fp: |
| 767 | reflog = fp.read() |
| 768 | self.assertEqual(reflog, line2) |
| 769 | |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 770 | def test_list_tests(self): |
| 771 | # test --list-tests |
| 772 | tests = [self.create_test() for i in range(5)] |
| 773 | output = self.run_tests('--list-tests', *tests) |
| 774 | self.assertEqual(output.rstrip().splitlines(), |
| 775 | tests) |
| 776 | |
Victor Stinner | 6b41510 | 2015-09-28 23:16:17 +0200 | [diff] [blame] | 777 | |
Chris Jerdonek | d6c18dc | 2012-12-27 18:53:12 -0800 | [diff] [blame] | 778 | if __name__ == '__main__': |
Serhiy Storchaka | 64f7c4e | 2013-08-29 12:26:23 +0300 | [diff] [blame] | 779 | unittest.main() |