Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 1 | import argparse |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 2 | import os |
Steve Dower | 08ec6d9 | 2015-10-08 11:34:07 -0700 | [diff] [blame] | 3 | import sys |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 4 | from test import support |
| 5 | |
Victor Stinner | 3844fe5 | 2015-09-26 10:38:01 +0200 | [diff] [blame] | 6 | |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 7 | USAGE = """\ |
| 8 | python -m test [options] [test_name1 [test_name2 ...]] |
| 9 | python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] |
| 10 | """ |
| 11 | |
| 12 | DESCRIPTION = """\ |
| 13 | Run Python regression tests. |
| 14 | |
| 15 | If no arguments or options are provided, finds all files matching |
| 16 | the pattern "test_*" in the Lib/test subdirectory and runs |
| 17 | them in alphabetical order (but see -M and -u, below, for exceptions). |
| 18 | |
| 19 | For more rigorous testing, it is useful to use the following |
| 20 | command line: |
| 21 | |
| 22 | python -E -Wd -m test [options] [test_name1 ...] |
| 23 | """ |
| 24 | |
| 25 | EPILOG = """\ |
| 26 | Additional option details: |
| 27 | |
Serhiy Storchaka | 4a7c03a | 2015-11-02 14:44:29 +0200 | [diff] [blame] | 28 | -r randomizes test execution order. You can use --randseed=int to provide an |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 29 | int seed value for the randomizer; this is useful for reproducing troublesome |
| 30 | test orders. |
| 31 | |
| 32 | -s On the first invocation of regrtest using -s, the first test file found |
| 33 | or the first test file given on the command line is run, and the name of |
| 34 | the next test is recorded in a file named pynexttest. If run from the |
| 35 | Python build directory, pynexttest is located in the 'build' subdirectory, |
| 36 | otherwise it is located in tempfile.gettempdir(). On subsequent runs, |
| 37 | the test in pynexttest is run, and the next test is written to pynexttest. |
| 38 | When the last test has been run, pynexttest is deleted. In this way it |
| 39 | is possible to single step through the test files. This is useful when |
| 40 | doing memory analysis on the Python interpreter, which process tends to |
| 41 | consume too many resources to run the full regression test non-stop. |
| 42 | |
| 43 | -S is used to continue running tests after an aborted run. It will |
| 44 | maintain the order a standard run (ie, this assumes -r is not used). |
| 45 | This is useful after the tests have prematurely stopped for some external |
| 46 | reason and you want to start running from where you left off rather |
| 47 | than starting from the beginning. |
| 48 | |
| 49 | -f reads the names of tests from the file given as f's argument, one |
| 50 | or more test names per line. Whitespace is ignored. Blank lines and |
| 51 | lines beginning with '#' are ignored. This is especially useful for |
| 52 | whittling down failures involving interactions among tests. |
| 53 | |
| 54 | -L causes the leaks(1) command to be run just before exit if it exists. |
| 55 | leaks(1) is available on Mac OS X and presumably on some other |
| 56 | FreeBSD-derived systems. |
| 57 | |
| 58 | -R runs each test several times and examines sys.gettotalrefcount() to |
| 59 | see if the test appears to be leaking references. The argument should |
| 60 | be of the form stab:run:fname where 'stab' is the number of times the |
| 61 | test is run to let gettotalrefcount settle down, 'run' is the number |
| 62 | of times further it is run and 'fname' is the name of the file the |
| 63 | reports are written to. These parameters all have defaults (5, 4 and |
| 64 | "reflog.txt" respectively), and the minimal invocation is '-R :'. |
| 65 | |
| 66 | -M runs tests that require an exorbitant amount of memory. These tests |
| 67 | typically try to ascertain containers keep working when containing more than |
| 68 | 2 billion objects, which only works on 64-bit systems. There are also some |
| 69 | tests that try to exhaust the address space of the process, which only makes |
| 70 | sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, |
| 71 | which is a string in the form of '2.5Gb', determines howmuch memory the |
| 72 | tests will limit themselves to (but they may go slightly over.) The number |
| 73 | shouldn't be more memory than the machine has (including swap memory). You |
| 74 | should also keep in mind that swap memory is generally much, much slower |
| 75 | than RAM, and setting memlimit to all available RAM or higher will heavily |
| 76 | tax the machine. On the other hand, it is no use running these tests with a |
| 77 | limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect |
| 78 | to use more than memlimit memory will be skipped. The big-memory tests |
| 79 | generally run very, very long. |
| 80 | |
| 81 | -u is used to specify which special resource intensive tests to run, |
| 82 | such as those requiring large file support or network connectivity. |
| 83 | The argument is a comma-separated list of words indicating the |
| 84 | resources to test. Currently only the following are defined: |
| 85 | |
| 86 | all - Enable all special resources. |
| 87 | |
| 88 | none - Disable all special resources (this is the default). |
| 89 | |
| 90 | audio - Tests that use the audio device. (There are known |
| 91 | cases of broken audio drivers that can crash Python or |
| 92 | even the Linux kernel.) |
| 93 | |
| 94 | curses - Tests that use curses and will modify the terminal's |
| 95 | state and output modes. |
| 96 | |
| 97 | largefile - It is okay to run some test that may create huge |
| 98 | files. These tests can take a long time and may |
| 99 | consume >2GB of disk space temporarily. |
| 100 | |
| 101 | network - It is okay to run tests that use external network |
| 102 | resource, e.g. testing SSL support for sockets. |
| 103 | |
| 104 | decimal - Test the decimal module against a large suite that |
| 105 | verifies compliance with standards. |
| 106 | |
| 107 | cpu - Used for certain CPU-heavy tests. |
| 108 | |
| 109 | subprocess Run all tests for the subprocess module. |
| 110 | |
| 111 | urlfetch - It is okay to download files required on testing. |
| 112 | |
| 113 | gui - Run tests that require a running GUI. |
| 114 | |
| 115 | To enable all resources except one, use '-uall,-<resource>'. For |
| 116 | example, to run all the tests except for the gui tests, give the |
| 117 | option '-uall,-gui'. |
| 118 | """ |
| 119 | |
| 120 | |
| 121 | RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', |
| 122 | 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') |
| 123 | |
| 124 | class _ArgParser(argparse.ArgumentParser): |
| 125 | |
| 126 | def error(self, message): |
| 127 | super().error(message + "\nPass -h or --help for complete help.") |
| 128 | |
| 129 | |
| 130 | def _create_parser(): |
| 131 | # Set prog to prevent the uninformative "__main__.py" from displaying in |
| 132 | # error messages when using "python -m test ...". |
| 133 | parser = _ArgParser(prog='regrtest.py', |
| 134 | usage=USAGE, |
| 135 | description=DESCRIPTION, |
| 136 | epilog=EPILOG, |
| 137 | add_help=False, |
| 138 | formatter_class=argparse.RawDescriptionHelpFormatter) |
| 139 | |
| 140 | # Arguments with this clause added to its help are described further in |
| 141 | # the epilog's "Additional option details" section. |
| 142 | more_details = ' See the section at bottom for more details.' |
| 143 | |
| 144 | group = parser.add_argument_group('General options') |
| 145 | # We add help explicitly to control what argument group it renders under. |
| 146 | group.add_argument('-h', '--help', action='help', |
| 147 | help='show this help message and exit') |
| 148 | group.add_argument('--timeout', metavar='TIMEOUT', type=float, |
| 149 | help='dump the traceback and exit if a test takes ' |
| 150 | 'more than TIMEOUT seconds; disabled if TIMEOUT ' |
| 151 | 'is negative or equals to zero') |
| 152 | group.add_argument('--wait', action='store_true', |
| 153 | help='wait for user input, e.g., allow a debugger ' |
| 154 | 'to be attached') |
| 155 | group.add_argument('--slaveargs', metavar='ARGS') |
| 156 | group.add_argument('-S', '--start', metavar='START', |
| 157 | help='the name of the test at which to start.' + |
| 158 | more_details) |
| 159 | |
| 160 | group = parser.add_argument_group('Verbosity') |
| 161 | group.add_argument('-v', '--verbose', action='count', |
| 162 | help='run tests in verbose mode with output to stdout') |
| 163 | group.add_argument('-w', '--verbose2', action='store_true', |
| 164 | help='re-run failed tests in verbose mode') |
| 165 | group.add_argument('-W', '--verbose3', action='store_true', |
| 166 | help='display test output on failure') |
| 167 | group.add_argument('-q', '--quiet', action='store_true', |
| 168 | help='no output unless one or more tests fail') |
| 169 | group.add_argument('-o', '--slow', action='store_true', dest='print_slow', |
| 170 | help='print the slowest 10 tests') |
| 171 | group.add_argument('--header', action='store_true', |
| 172 | help='print header with interpreter info') |
| 173 | |
| 174 | group = parser.add_argument_group('Selecting tests') |
| 175 | group.add_argument('-r', '--randomize', action='store_true', |
| 176 | help='randomize test execution order.' + more_details) |
| 177 | group.add_argument('--randseed', metavar='SEED', |
| 178 | dest='random_seed', type=int, |
| 179 | help='pass a random seed to reproduce a previous ' |
| 180 | 'random run') |
| 181 | group.add_argument('-f', '--fromfile', metavar='FILE', |
| 182 | help='read names of tests to run from a file.' + |
| 183 | more_details) |
| 184 | group.add_argument('-x', '--exclude', action='store_true', |
| 185 | help='arguments are tests to *exclude*') |
| 186 | group.add_argument('-s', '--single', action='store_true', |
| 187 | help='single step through a set of tests.' + |
| 188 | more_details) |
| 189 | group.add_argument('-m', '--match', metavar='PAT', |
| 190 | dest='match_tests', |
| 191 | help='match test cases and methods with glob pattern PAT') |
| 192 | group.add_argument('-G', '--failfast', action='store_true', |
| 193 | help='fail as soon as a test fails (only with -v or -W)') |
| 194 | group.add_argument('-u', '--use', metavar='RES1,RES2,...', |
| 195 | action='append', type=resources_list, |
| 196 | help='specify which special resource intensive tests ' |
| 197 | 'to run.' + more_details) |
| 198 | group.add_argument('-M', '--memlimit', metavar='LIMIT', |
| 199 | help='run very large memory-consuming tests.' + |
| 200 | more_details) |
| 201 | group.add_argument('--testdir', metavar='DIR', |
| 202 | type=relative_filename, |
| 203 | help='execute test files in the specified directory ' |
| 204 | '(instead of the Python stdlib test suite)') |
| 205 | |
| 206 | group = parser.add_argument_group('Special runs') |
| 207 | group.add_argument('-l', '--findleaks', action='store_true', |
| 208 | help='if GC is available detect tests that leak memory') |
| 209 | group.add_argument('-L', '--runleaks', action='store_true', |
| 210 | help='run the leaks(1) command just before exit.' + |
| 211 | more_details) |
| 212 | group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', |
| 213 | type=huntrleaks, |
| 214 | help='search for reference leaks (needs debug build, ' |
| 215 | 'very slow).' + more_details) |
| 216 | group.add_argument('-j', '--multiprocess', metavar='PROCESSES', |
| 217 | dest='use_mp', type=int, |
| 218 | help='run PROCESSES processes at once') |
| 219 | group.add_argument('-T', '--coverage', action='store_true', |
| 220 | dest='trace', |
| 221 | help='turn on code coverage tracing using the trace ' |
| 222 | 'module') |
| 223 | group.add_argument('-D', '--coverdir', metavar='DIR', |
| 224 | type=relative_filename, |
| 225 | help='directory where coverage files are put') |
| 226 | group.add_argument('-N', '--nocoverdir', |
| 227 | action='store_const', const=None, dest='coverdir', |
| 228 | help='put coverage files alongside modules') |
| 229 | group.add_argument('-t', '--threshold', metavar='THRESHOLD', |
| 230 | type=int, |
| 231 | help='call gc.set_threshold(THRESHOLD)') |
| 232 | group.add_argument('-n', '--nowindows', action='store_true', |
| 233 | help='suppress error message boxes on Windows') |
| 234 | group.add_argument('-F', '--forever', action='store_true', |
| 235 | help='run the specified tests in a loop, until an ' |
| 236 | 'error happens') |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 237 | group.add_argument('--list-tests', action='store_true', |
| 238 | help="only write the name of tests that will be run, " |
| 239 | "don't execute them") |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 240 | group.add_argument('-P', '--pgo', dest='pgo', action='store_true', |
| 241 | help='enable Profile Guided Optimization training') |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 242 | |
| 243 | parser.add_argument('args', nargs=argparse.REMAINDER, |
| 244 | help=argparse.SUPPRESS) |
| 245 | |
| 246 | return parser |
| 247 | |
| 248 | |
| 249 | def relative_filename(string): |
| 250 | # CWD is replaced with a temporary dir before calling main(), so we |
| 251 | # join it with the saved CWD so it ends up where the user expects. |
| 252 | return os.path.join(support.SAVEDCWD, string) |
| 253 | |
| 254 | |
| 255 | def huntrleaks(string): |
| 256 | args = string.split(':') |
| 257 | if len(args) not in (2, 3): |
| 258 | raise argparse.ArgumentTypeError( |
| 259 | 'needs 2 or 3 colon-separated arguments') |
| 260 | nwarmup = int(args[0]) if args[0] else 5 |
| 261 | ntracked = int(args[1]) if args[1] else 4 |
| 262 | fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' |
| 263 | return nwarmup, ntracked, fname |
| 264 | |
| 265 | |
| 266 | def resources_list(string): |
| 267 | u = [x.lower() for x in string.split(',')] |
| 268 | for r in u: |
| 269 | if r == 'all' or r == 'none': |
| 270 | continue |
| 271 | if r[0] == '-': |
| 272 | r = r[1:] |
| 273 | if r not in RESOURCE_NAMES: |
| 274 | raise argparse.ArgumentTypeError('invalid resource: ' + r) |
| 275 | return u |
| 276 | |
| 277 | |
| 278 | def _parse_args(args, **kwargs): |
| 279 | # Defaults |
| 280 | ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, |
| 281 | exclude=False, single=False, randomize=False, fromfile=None, |
| 282 | findleaks=False, use_resources=None, trace=False, coverdir='coverage', |
| 283 | runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, |
| 284 | random_seed=None, use_mp=None, verbose3=False, forever=False, |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 285 | header=False, failfast=False, match_tests=None, pgo=False) |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 286 | for k, v in kwargs.items(): |
| 287 | if not hasattr(ns, k): |
| 288 | raise TypeError('%r is an invalid keyword argument ' |
| 289 | 'for this function' % k) |
| 290 | setattr(ns, k, v) |
| 291 | if ns.use_resources is None: |
| 292 | ns.use_resources = [] |
| 293 | |
| 294 | parser = _create_parser() |
| 295 | parser.parse_args(args=args, namespace=ns) |
| 296 | |
| 297 | if ns.single and ns.fromfile: |
| 298 | parser.error("-s and -f don't go together!") |
| 299 | if ns.use_mp and ns.trace: |
| 300 | parser.error("-T and -j don't go together!") |
| 301 | if ns.use_mp and ns.findleaks: |
| 302 | parser.error("-l and -j don't go together!") |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 303 | if ns.failfast and not (ns.verbose or ns.verbose3): |
| 304 | parser.error("-G/--failfast needs either -v or -W") |
Brett Cannon | 11faa21 | 2015-10-02 16:20:49 -0700 | [diff] [blame] | 305 | if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): |
| 306 | parser.error("--pgo/-v don't go together!") |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 307 | |
Steve Dower | 12c2945 | 2015-10-08 09:05:36 -0700 | [diff] [blame] | 308 | if ns.nowindows: |
| 309 | print("Warning: the --nowindows (-n) option is deprecated. " |
| 310 | "Use -vv to display assertions in stderr.", file=sys.stderr) |
| 311 | |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 312 | if ns.quiet: |
| 313 | ns.verbose = 0 |
| 314 | if ns.timeout is not None: |
Victor Stinner | 5f9d3ac | 2015-10-03 00:21:12 +0200 | [diff] [blame] | 315 | if ns.timeout <= 0: |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 316 | ns.timeout = None |
| 317 | if ns.use_mp is not None: |
| 318 | if ns.use_mp <= 0: |
| 319 | # Use all cores + extras for tests that like to sleep |
| 320 | ns.use_mp = 2 + (os.cpu_count() or 1) |
Victor Stinner | 98de534 | 2015-09-26 09:43:45 +0200 | [diff] [blame] | 321 | if ns.use: |
| 322 | for a in ns.use: |
| 323 | for r in a: |
| 324 | if r == 'all': |
| 325 | ns.use_resources[:] = RESOURCE_NAMES |
| 326 | continue |
| 327 | if r == 'none': |
| 328 | del ns.use_resources[:] |
| 329 | continue |
| 330 | remove = False |
| 331 | if r[0] == '-': |
| 332 | remove = True |
| 333 | r = r[1:] |
| 334 | if remove: |
| 335 | if r in ns.use_resources: |
| 336 | ns.use_resources.remove(r) |
| 337 | elif r not in ns.use_resources: |
| 338 | ns.use_resources.append(r) |
| 339 | if ns.random_seed is not None: |
| 340 | ns.randomize = True |
| 341 | |
| 342 | return ns |