Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | """ |
| 4 | Run the test suite using a separate process for each test file. |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 5 | |
Siva Chandra | 2d7832e | 2015-05-08 23:08:53 +0000 | [diff] [blame] | 6 | Each test will run with a time limit of 10 minutes by default. |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 7 | |
Siva Chandra | 2d7832e | 2015-05-08 23:08:53 +0000 | [diff] [blame] | 8 | Override the default time limit of 10 minutes by setting |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 9 | the environment variable LLDB_TEST_TIMEOUT. |
| 10 | |
| 11 | E.g., export LLDB_TEST_TIMEOUT=10m |
| 12 | |
| 13 | Override the time limit for individual tests by setting |
| 14 | the environment variable LLDB_[TEST NAME]_TIMEOUT. |
| 15 | |
| 16 | E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m |
| 17 | |
| 18 | Set to "0" to run without time limit. |
| 19 | |
| 20 | E.g., export LLDB_TEST_TIMEOUT=0 |
| 21 | or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0 |
Vince Harron | dcc2b9f | 2015-05-27 04:40:36 +0000 | [diff] [blame] | 22 | |
| 23 | To collect core files for timed out tests, do the following before running dosep.py |
| 24 | |
| 25 | OSX |
| 26 | ulimit -c unlimited |
| 27 | sudo sysctl -w kern.corefile=core.%P |
| 28 | |
| 29 | Linux: |
| 30 | ulimit -c unlimited |
| 31 | echo core.%p | sudo tee /proc/sys/kernel/core_pattern |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 32 | """ |
| 33 | |
Greg Clayton | 2256d0d | 2014-03-24 23:01:57 +0000 | [diff] [blame] | 34 | import multiprocessing |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 35 | import os |
Vince Harron | dcc2b9f | 2015-05-27 04:40:36 +0000 | [diff] [blame] | 36 | import fnmatch |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 37 | import platform |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 38 | import re |
Vince Harron | f8b9a1d | 2015-05-18 19:40:54 +0000 | [diff] [blame] | 39 | import dotest_args |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 40 | import shlex |
| 41 | import subprocess |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 42 | import sys |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 43 | |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 44 | from optparse import OptionParser |
| 45 | |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 46 | def get_timeout_command(): |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 47 | """Search for a suitable timeout command.""" |
Ying Chen | 93190c4 | 2015-07-20 20:04:22 +0000 | [diff] [blame] | 48 | if not sys.platform.startswith("win32"): |
| 49 | try: |
| 50 | subprocess.call("timeout", stderr=subprocess.PIPE) |
| 51 | return "timeout" |
| 52 | except OSError: |
| 53 | pass |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 54 | try: |
Chaoren Lin | 45c17ff | 2015-05-28 23:00:10 +0000 | [diff] [blame] | 55 | subprocess.call("gtimeout", stderr=subprocess.PIPE) |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 56 | return "gtimeout" |
| 57 | except OSError: |
| 58 | pass |
| 59 | return None |
| 60 | |
| 61 | timeout_command = get_timeout_command() |
| 62 | |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 63 | # Status codes for running command with timeout. |
| 64 | eTimedOut, ePassed, eFailed = 124, 0, 1 |
| 65 | |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 66 | output_lock = None |
| 67 | test_counter = None |
| 68 | total_tests = None |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 69 | test_name_len = None |
Pavel Labath | 05ab237 | 2015-07-06 15:57:52 +0000 | [diff] [blame] | 70 | dotest_options = None |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 71 | output_on_success = False |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 72 | |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 73 | def setup_global_variables(lock, counter, total, name_len, options): |
| 74 | global output_lock, test_counter, total_tests, test_name_len |
| 75 | global dotest_options |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 76 | output_lock = lock |
| 77 | test_counter = counter |
| 78 | total_tests = total |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 79 | test_name_len = name_len |
Pavel Labath | 05ab237 | 2015-07-06 15:57:52 +0000 | [diff] [blame] | 80 | dotest_options = options |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 81 | |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 82 | def report_test_failure(name, command, output): |
| 83 | global output_lock |
| 84 | with output_lock: |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 85 | print >> sys.stderr |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 86 | print >> sys.stderr, output |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 87 | print >> sys.stderr, "[%s FAILED]" % name |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 88 | print >> sys.stderr, "Command invoked: %s" % ' '.join(command) |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 89 | update_progress(name) |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 90 | |
| 91 | def report_test_pass(name, output): |
| 92 | global output_lock, output_on_success |
| 93 | with output_lock: |
| 94 | if output_on_success: |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 95 | print >> sys.stderr |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 96 | print >> sys.stderr, output |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 97 | print >> sys.stderr, "[%s PASSED]" % name |
| 98 | update_progress(name) |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 99 | |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 100 | def update_progress(test_name=""): |
| 101 | global output_lock, test_counter, total_tests, test_name_len |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 102 | with output_lock: |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 103 | counter_len = len(str(total_tests)) |
| 104 | sys.stderr.write( |
| 105 | "\r%*d out of %d test suites processed - %-*s" % |
| 106 | (counter_len, test_counter.value, total_tests, |
| 107 | test_name_len.value, test_name)) |
| 108 | if len(test_name) > test_name_len.value: |
| 109 | test_name_len.value = len(test_name) |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 110 | test_counter.value += 1 |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 111 | sys.stdout.flush() |
| 112 | sys.stderr.flush() |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 113 | |
Zachary Turner | c7a7c8a | 2015-05-28 19:56:26 +0000 | [diff] [blame] | 114 | def parse_test_results(output): |
| 115 | passes = 0 |
| 116 | failures = 0 |
| 117 | for result in output: |
| 118 | pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE) |
| 119 | fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE) |
| 120 | error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE) |
| 121 | this_fail_count = 0 |
| 122 | this_error_count = 0 |
| 123 | if pass_count != None: |
| 124 | passes = passes + int(pass_count.group(1)) |
| 125 | if fail_count != None: |
| 126 | failures = failures + int(fail_count.group(1)) |
| 127 | if error_count != None: |
| 128 | failures = failures + int(error_count.group(1)) |
| 129 | pass |
| 130 | return passes, failures |
| 131 | |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 132 | def call_with_timeout(command, timeout, name): |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 133 | """Run command with a timeout if possible.""" |
Vince Harron | dcc2b9f | 2015-05-27 04:40:36 +0000 | [diff] [blame] | 134 | """-s QUIT will create a coredump if they are enabled on your system""" |
Zachary Turner | c7a7c8a | 2015-05-28 19:56:26 +0000 | [diff] [blame] | 135 | process = None |
| 136 | if timeout_command and timeout != "0": |
| 137 | command = [timeout_command, '-s', 'QUIT', timeout] + command |
| 138 | # Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE |
Zachary Turner | dc494d5 | 2015-02-07 00:14:55 +0000 | [diff] [blame] | 139 | if os.name != "nt": |
Chaoren Lin | 9d2b7f9 | 2015-05-29 18:43:46 +0000 | [diff] [blame] | 140 | process = subprocess.Popen(command, stdin=subprocess.PIPE, |
| 141 | stdout=subprocess.PIPE, |
| 142 | stderr=subprocess.PIPE, |
| 143 | close_fds=True) |
Zachary Turner | dc494d5 | 2015-02-07 00:14:55 +0000 | [diff] [blame] | 144 | else: |
Chaoren Lin | 9d2b7f9 | 2015-05-29 18:43:46 +0000 | [diff] [blame] | 145 | process = subprocess.Popen(command, stdin=subprocess.PIPE, |
| 146 | stdout=subprocess.PIPE, |
| 147 | stderr=subprocess.PIPE) |
Zachary Turner | c7a7c8a | 2015-05-28 19:56:26 +0000 | [diff] [blame] | 148 | output = process.communicate() |
| 149 | exit_status = process.returncode |
| 150 | passes, failures = parse_test_results(output) |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 151 | if exit_status == 0: |
| 152 | # stdout does not have any useful information from 'dotest.py', only stderr does. |
| 153 | report_test_pass(name, output[1]) |
| 154 | else: |
| 155 | report_test_failure(name, command, output[1]) |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 156 | return name, exit_status, passes, failures |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 157 | |
Vince Harron | 41657cc | 2015-05-21 18:15:09 +0000 | [diff] [blame] | 158 | def process_dir(root, files, test_root, dotest_argv): |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 159 | """Examine a directory for tests, and invoke any found within it.""" |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 160 | results = [] |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 161 | for name in files: |
Zachary Turner | f6896b0 | 2015-01-05 19:37:03 +0000 | [diff] [blame] | 162 | script_file = os.path.join(test_root, "dotest.py") |
Zachary Turner | f6896b0 | 2015-01-05 19:37:03 +0000 | [diff] [blame] | 163 | command = ([sys.executable, script_file] + |
Vince Harron | 41657cc | 2015-05-21 18:15:09 +0000 | [diff] [blame] | 164 | dotest_argv + |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 165 | ["-p", name, root]) |
| 166 | |
| 167 | timeout_name = os.path.basename(os.path.splitext(name)[0]).upper() |
| 168 | |
Pavel Labath | 05ab237 | 2015-07-06 15:57:52 +0000 | [diff] [blame] | 169 | timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or getDefaultTimeout(dotest_options.lldb_platform_name) |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 170 | |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 171 | results.append(call_with_timeout(command, timeout, name)) |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 172 | |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 173 | # result = (name, status, passes, failures) |
| 174 | timed_out = [name for name, status, _, _ in results |
| 175 | if status == eTimedOut] |
| 176 | passed = [name for name, status, _, _ in results |
| 177 | if status == ePassed] |
| 178 | failed = [name for name, status, _, _ in results |
| 179 | if status != ePassed] |
| 180 | pass_count = sum([result[2] for result in results]) |
| 181 | fail_count = sum([result[3] for result in results]) |
Zachary Turner | c7a7c8a | 2015-05-28 19:56:26 +0000 | [diff] [blame] | 182 | |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 183 | return (timed_out, passed, failed, pass_count, fail_count) |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 184 | |
| 185 | in_q = None |
| 186 | out_q = None |
| 187 | |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 188 | def process_dir_worker(arg_tuple): |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 189 | """Worker thread main loop when in multithreaded mode. |
| 190 | Takes one directory specification at a time and works on it.""" |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 191 | return process_dir(*arg_tuple) |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 192 | |
Vince Harron | 41657cc | 2015-05-21 18:15:09 +0000 | [diff] [blame] | 193 | def walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads): |
Steve Pucci | befe2b1 | 2014-03-07 00:01:11 +0000 | [diff] [blame] | 194 | """Look for matched files and invoke test driver on each one. |
| 195 | In single-threaded mode, each test driver is invoked directly. |
| 196 | In multi-threaded mode, submit each test driver to a worker |
Vince Harron | e06a7a8 | 2015-05-12 23:12:19 +0000 | [diff] [blame] | 197 | queue, and then wait for all to complete. |
| 198 | |
| 199 | test_directory - lldb/test/ directory |
| 200 | test_subdir - lldb/test/ or a subfolder with the tests we're interested in running |
| 201 | """ |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 202 | |
| 203 | # Collect the test files that we'll run. |
| 204 | test_work_items = [] |
Vince Harron | e06a7a8 | 2015-05-12 23:12:19 +0000 | [diff] [blame] | 205 | for root, dirs, files in os.walk(test_subdir, topdown=False): |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 206 | def is_test(name): |
| 207 | # Not interested in symbolically linked files. |
| 208 | if os.path.islink(os.path.join(root, name)): |
| 209 | return False |
| 210 | # Only interested in test files with the "Test*.py" naming pattern. |
| 211 | return name.startswith("Test") and name.endswith(".py") |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 212 | |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 213 | tests = filter(is_test, files) |
| 214 | test_work_items.append((root, tests, test_directory, dotest_argv)) |
| 215 | |
| 216 | global output_lock, test_counter, total_tests, test_name_len |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 217 | output_lock = multiprocessing.RLock() |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 218 | # item = (root, tests, test_directory, dotest_argv) |
| 219 | total_tests = sum([len(item[1]) for item in test_work_items]) |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 220 | test_counter = multiprocessing.Value('i', 0) |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 221 | test_name_len = multiprocessing.Value('i', 0) |
| 222 | print >> sys.stderr, "Testing: %d test suites, %d thread%s" % ( |
| 223 | total_tests, num_threads, (num_threads > 1) * "s") |
| 224 | update_progress() |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 225 | |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 226 | # Run the items, either in a pool (for multicore speedup) or |
| 227 | # calling each individually. |
| 228 | if num_threads > 1: |
Chaoren Lin | ffc63b0 | 2015-08-12 18:02:49 +0000 | [diff] [blame] | 229 | pool = multiprocessing.Pool( |
| 230 | num_threads, |
| 231 | initializer=setup_global_variables, |
| 232 | initargs=(output_lock, test_counter, total_tests, test_name_len, |
| 233 | dotest_options)) |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 234 | test_results = pool.map(process_dir_worker, test_work_items) |
| 235 | else: |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 236 | test_results = map(process_dir_worker, test_work_items) |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 237 | |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 238 | # result = (timed_out, failed, passed, fail_count, pass_count) |
| 239 | timed_out = sum([result[0] for result in test_results], []) |
| 240 | passed = sum([result[1] for result in test_results], []) |
| 241 | failed = sum([result[2] for result in test_results], []) |
| 242 | pass_count = sum([result[3] for result in test_results]) |
| 243 | fail_count = sum([result[4] for result in test_results]) |
Todd Fiala | 3f0a360 | 2014-07-08 06:42:37 +0000 | [diff] [blame] | 244 | |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 245 | return (timed_out, passed, failed, pass_count, fail_count) |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 246 | |
Vince Harron | f8b9a1d | 2015-05-18 19:40:54 +0000 | [diff] [blame] | 247 | def getExpectedTimeouts(platform_name): |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 248 | # returns a set of test filenames that might timeout |
| 249 | # are we running against a remote target? |
Vince Harron | f8b9a1d | 2015-05-18 19:40:54 +0000 | [diff] [blame] | 250 | if platform_name is None: |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 251 | target = sys.platform |
| 252 | remote = False |
Vince Harron | f8b9a1d | 2015-05-18 19:40:54 +0000 | [diff] [blame] | 253 | else: |
| 254 | m = re.search('remote-(\w+)', platform_name) |
| 255 | target = m.group(1) |
| 256 | remote = True |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 257 | |
| 258 | expected_timeout = set() |
| 259 | |
| 260 | if target.startswith("linux"): |
| 261 | expected_timeout |= { |
Vince Harron | de92b52 | 2015-05-13 23:59:03 +0000 | [diff] [blame] | 262 | "TestAttachDenied.py", |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 263 | "TestAttachResume.py", |
Chaoren Lin | 0b8bb3d | 2015-07-22 20:52:17 +0000 | [diff] [blame] | 264 | "TestProcessAttach.py", |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 265 | "TestConnectRemote.py", |
| 266 | "TestCreateAfterAttach.py", |
Tamas Berghammer | 0d0ec9f | 2015-05-19 10:49:40 +0000 | [diff] [blame] | 267 | "TestEvents.py", |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 268 | "TestExitDuringStep.py", |
Tamas Berghammer | 0b11db5 | 2015-06-02 14:45:25 +0000 | [diff] [blame] | 269 | "TestHelloWorld.py", # Times out in ~10% of the times on the build bot |
Oleksiy Vyalov | 18f4c9f | 2015-06-10 01:34:25 +0000 | [diff] [blame] | 270 | "TestMultithreaded.py", |
Vince Harron | e884389 | 2015-06-22 20:54:14 +0000 | [diff] [blame] | 271 | "TestRegisters.py", # ~12/600 dosep runs (build 3120-3122) |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 272 | "TestThreadStepOut.py", |
| 273 | } |
| 274 | elif target.startswith("android"): |
| 275 | expected_timeout |= { |
| 276 | "TestExitDuringStep.py", |
| 277 | "TestHelloWorld.py", |
| 278 | } |
Ed Maste | 4dd8fba | 2015-05-14 16:25:52 +0000 | [diff] [blame] | 279 | elif target.startswith("freebsd"): |
| 280 | expected_timeout |= { |
| 281 | "TestBreakpointConditions.py", |
Ed Maste | 0894813 | 2015-05-28 18:45:30 +0000 | [diff] [blame] | 282 | "TestChangeProcessGroup.py", |
Ed Maste | bfd0563 | 2015-05-27 19:11:29 +0000 | [diff] [blame] | 283 | "TestValueObjectRecursion.py", |
Ed Maste | 4dd8fba | 2015-05-14 16:25:52 +0000 | [diff] [blame] | 284 | "TestWatchpointConditionAPI.py", |
| 285 | } |
Vince Harron | 0f173ac | 2015-05-18 19:36:33 +0000 | [diff] [blame] | 286 | elif target.startswith("darwin"): |
| 287 | expected_timeout |= { |
| 288 | "TestThreadSpecificBreakpoint.py", # times out on MBP Retina, Mid 2012 |
| 289 | } |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 290 | return expected_timeout |
| 291 | |
Pavel Labath | fad30cf | 2015-06-29 14:16:51 +0000 | [diff] [blame] | 292 | def getDefaultTimeout(platform_name): |
| 293 | if os.getenv("LLDB_TEST_TIMEOUT"): |
| 294 | return os.getenv("LLDB_TEST_TIMEOUT") |
| 295 | |
| 296 | if platform_name is None: |
| 297 | platform_name = sys.platform |
| 298 | |
| 299 | if platform_name.startswith("remote-"): |
| 300 | return "10m" |
| 301 | else: |
| 302 | return "4m" |
| 303 | |
Vince Harron | 0b9dbb5 | 2015-05-21 18:18:52 +0000 | [diff] [blame] | 304 | def touch(fname, times=None): |
Greg Clayton | 8c3f9c9 | 2015-08-11 21:01:32 +0000 | [diff] [blame] | 305 | if os.path.exists(fname): |
Vince Harron | 0b9dbb5 | 2015-05-21 18:18:52 +0000 | [diff] [blame] | 306 | os.utime(fname, times) |
| 307 | |
Vince Harron | dcc2b9f | 2015-05-27 04:40:36 +0000 | [diff] [blame] | 308 | def find(pattern, path): |
| 309 | result = [] |
| 310 | for root, dirs, files in os.walk(path): |
| 311 | for name in files: |
| 312 | if fnmatch.fnmatch(name, pattern): |
| 313 | result.append(os.path.join(root, name)) |
| 314 | return result |
| 315 | |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 316 | def main(): |
Vince Harron | d5fa102 | 2015-05-10 15:24:12 +0000 | [diff] [blame] | 317 | # We can't use sys.path[0] to determine the script directory |
| 318 | # because it doesn't work under a debugger |
Vince Harron | e06a7a8 | 2015-05-12 23:12:19 +0000 | [diff] [blame] | 319 | test_directory = os.path.dirname(os.path.realpath(__file__)) |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 320 | parser = OptionParser(usage="""\ |
| 321 | Run lldb test suite using a separate process for each test file. |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 322 | |
Siva Chandra | 2d7832e | 2015-05-08 23:08:53 +0000 | [diff] [blame] | 323 | Each test will run with a time limit of 10 minutes by default. |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 324 | |
Siva Chandra | 2d7832e | 2015-05-08 23:08:53 +0000 | [diff] [blame] | 325 | Override the default time limit of 10 minutes by setting |
Vince Harron | ede5965 | 2015-01-08 02:11:26 +0000 | [diff] [blame] | 326 | the environment variable LLDB_TEST_TIMEOUT. |
| 327 | |
| 328 | E.g., export LLDB_TEST_TIMEOUT=10m |
| 329 | |
| 330 | Override the time limit for individual tests by setting |
| 331 | the environment variable LLDB_[TEST NAME]_TIMEOUT. |
| 332 | |
| 333 | E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m |
| 334 | |
| 335 | Set to "0" to run without time limit. |
| 336 | |
| 337 | E.g., export LLDB_TEST_TIMEOUT=0 |
| 338 | or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0 |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 339 | """) |
| 340 | parser.add_option('-o', '--options', |
| 341 | type='string', action='store', |
| 342 | dest='dotest_options', |
| 343 | help="""The options passed to 'dotest.py' if specified.""") |
| 344 | |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 345 | parser.add_option('-s', '--output-on-success', |
| 346 | action='store_true', |
| 347 | dest='output_on_success', |
| 348 | default=False, |
| 349 | help="""Print full output of 'dotest.py' even when it succeeds.""") |
| 350 | |
Greg Clayton | 2256d0d | 2014-03-24 23:01:57 +0000 | [diff] [blame] | 351 | parser.add_option('-t', '--threads', |
| 352 | type='int', |
| 353 | dest='num_threads', |
Ed Maste | cec2a5b | 2014-11-21 02:41:25 +0000 | [diff] [blame] | 354 | help="""The number of threads to use when running tests separately.""") |
Greg Clayton | 2256d0d | 2014-03-24 23:01:57 +0000 | [diff] [blame] | 355 | |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 356 | opts, args = parser.parse_args() |
Vince Harron | f8b9a1d | 2015-05-18 19:40:54 +0000 | [diff] [blame] | 357 | dotest_option_string = opts.dotest_options |
| 358 | |
Vince Harron | 41657cc | 2015-05-21 18:15:09 +0000 | [diff] [blame] | 359 | is_posix = (os.name == "posix") |
| 360 | dotest_argv = shlex.split(dotest_option_string, posix=is_posix) if dotest_option_string else [] |
Vince Harron | 8994fed | 2015-05-22 19:49:23 +0000 | [diff] [blame] | 361 | |
| 362 | parser = dotest_args.create_parser() |
Pavel Labath | 05ab237 | 2015-07-06 15:57:52 +0000 | [diff] [blame] | 363 | global dotest_options |
Zachary Turner | 38e6417 | 2015-08-10 17:46:11 +0000 | [diff] [blame] | 364 | global output_on_success |
| 365 | output_on_success = opts.output_on_success |
Vince Harron | 8994fed | 2015-05-22 19:49:23 +0000 | [diff] [blame] | 366 | dotest_options = dotest_args.parse_args(parser, dotest_argv) |
| 367 | |
Vince Harron | 41657cc | 2015-05-21 18:15:09 +0000 | [diff] [blame] | 368 | if not dotest_options.s: |
| 369 | # no session log directory, we need to add this to prevent |
| 370 | # every dotest invocation from creating its own directory |
| 371 | import datetime |
| 372 | # The windows platforms don't like ':' in the pathname. |
| 373 | timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") |
| 374 | dotest_argv.append('-s') |
| 375 | dotest_argv.append(timestamp_started) |
Vince Harron | 0b9dbb5 | 2015-05-21 18:18:52 +0000 | [diff] [blame] | 376 | dotest_options.s = timestamp_started |
| 377 | |
| 378 | session_dir = os.path.join(os.getcwd(), dotest_options.s) |
Ed Maste | cec2a5b | 2014-11-21 02:41:25 +0000 | [diff] [blame] | 379 | |
Vince Harron | e06a7a8 | 2015-05-12 23:12:19 +0000 | [diff] [blame] | 380 | # The root directory was specified on the command line |
| 381 | if len(args) == 0: |
| 382 | test_subdir = test_directory |
| 383 | else: |
| 384 | test_subdir = os.path.join(test_directory, args[0]) |
| 385 | |
Vince Harron | dcc2b9f | 2015-05-27 04:40:36 +0000 | [diff] [blame] | 386 | # clean core files in test tree from previous runs (Linux) |
| 387 | cores = find('core.*', test_subdir) |
| 388 | for core in cores: |
| 389 | os.unlink(core) |
| 390 | |
Ed Maste | cec2a5b | 2014-11-21 02:41:25 +0000 | [diff] [blame] | 391 | if opts.num_threads: |
| 392 | num_threads = opts.num_threads |
| 393 | else: |
Greg Clayton | 2256d0d | 2014-03-24 23:01:57 +0000 | [diff] [blame] | 394 | num_threads_str = os.environ.get("LLDB_TEST_THREADS") |
| 395 | if num_threads_str: |
| 396 | num_threads = int(num_threads_str) |
Greg Clayton | 2256d0d | 2014-03-24 23:01:57 +0000 | [diff] [blame] | 397 | else: |
Ed Maste | cec2a5b | 2014-11-21 02:41:25 +0000 | [diff] [blame] | 398 | num_threads = multiprocessing.cpu_count() |
| 399 | if num_threads < 1: |
| 400 | num_threads = 1 |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 401 | |
Daniel Malea | b42556f | 2013-04-19 18:32:53 +0000 | [diff] [blame] | 402 | system_info = " ".join(platform.uname()) |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 403 | (timed_out, passed, failed, pass_count, fail_count) = walk_and_invoke( |
| 404 | test_directory, test_subdir, dotest_argv, num_threads) |
Zachary Turner | c7a7c8a | 2015-05-28 19:56:26 +0000 | [diff] [blame] | 405 | |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 406 | timed_out = set(timed_out) |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 407 | num_test_files = len(passed) + len(failed) |
| 408 | num_test_cases = pass_count + fail_count |
Daniel Malea | b42556f | 2013-04-19 18:32:53 +0000 | [diff] [blame] | 409 | |
Vince Harron | dcc2b9f | 2015-05-27 04:40:36 +0000 | [diff] [blame] | 410 | # move core files into session dir |
| 411 | cores = find('core.*', test_subdir) |
| 412 | for core in cores: |
| 413 | dst = core.replace(test_directory, "")[1:] |
| 414 | dst = dst.replace(os.path.sep, "-") |
| 415 | os.rename(core, os.path.join(session_dir, dst)) |
| 416 | |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 417 | # remove expected timeouts from failures |
Vince Harron | f8b9a1d | 2015-05-18 19:40:54 +0000 | [diff] [blame] | 418 | expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name) |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 419 | for xtime in expected_timeout: |
| 420 | if xtime in timed_out: |
| 421 | timed_out.remove(xtime) |
| 422 | failed.remove(xtime) |
Vince Harron | 0b9dbb5 | 2015-05-21 18:18:52 +0000 | [diff] [blame] | 423 | result = "ExpectedTimeout" |
| 424 | elif xtime in passed: |
| 425 | result = "UnexpectedCompletion" |
| 426 | else: |
| 427 | result = None # failed |
| 428 | |
| 429 | if result: |
| 430 | test_name = os.path.splitext(xtime)[0] |
| 431 | touch(os.path.join(session_dir, "{}-{}".format(result, test_name))) |
Vince Harron | 0638173 | 2015-05-12 23:10:36 +0000 | [diff] [blame] | 432 | |
Chaoren Lin | 5e3ab2b | 2015-06-01 17:49:25 +0000 | [diff] [blame] | 433 | print |
Chaoren Lin | 5a59e46 | 2015-08-12 18:02:51 +0000 | [diff] [blame] | 434 | sys.stdout.write("Ran %d test suites" % num_test_files) |
| 435 | if num_test_files > 0: |
| 436 | sys.stdout.write(" (%d failed) (%f%%)" % ( |
| 437 | len(failed), 100.0 * len(failed) / num_test_files)) |
| 438 | print |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 439 | sys.stdout.write("Ran %d test cases" % num_test_cases) |
| 440 | if num_test_cases > 0: |
Chaoren Lin | 5a59e46 | 2015-08-12 18:02:51 +0000 | [diff] [blame] | 441 | sys.stdout.write(" (%d failed) (%f%%)" % ( |
Chaoren Lin | e80372a | 2015-08-12 18:02:53 +0000 | [diff] [blame^] | 442 | fail_count, 100.0 * fail_count / num_test_cases)) |
Chaoren Lin | 5a59e46 | 2015-08-12 18:02:51 +0000 | [diff] [blame] | 443 | print |
Daniel Malea | cbaef26 | 2013-02-15 21:31:37 +0000 | [diff] [blame] | 444 | if len(failed) > 0: |
Shawn Best | 13491c4 | 2014-10-22 19:29:00 +0000 | [diff] [blame] | 445 | failed.sort() |
Ying Chen | 10ed1a9 | 2015-05-28 23:51:49 +0000 | [diff] [blame] | 446 | print "Failing Tests (%d)" % len(failed) |
Daniel Malea | cbaef26 | 2013-02-15 21:31:37 +0000 | [diff] [blame] | 447 | for f in failed: |
Vince Harron | 17f429f | 2014-12-13 00:08:19 +0000 | [diff] [blame] | 448 | print "%s: LLDB (suite) :: %s (%s)" % ( |
| 449 | "TIMEOUT" if f in timed_out else "FAIL", f, system_info |
| 450 | ) |
Daniel Malea | cbaef26 | 2013-02-15 21:31:37 +0000 | [diff] [blame] | 451 | sys.exit(1) |
| 452 | sys.exit(0) |
Johnny Chen | e8d9dc6 | 2011-10-31 19:04:07 +0000 | [diff] [blame] | 453 | |
| 454 | if __name__ == '__main__': |
| 455 | main() |