blob: c5366a046fb985d55a8428ab99fbe6557c0b80e7 [file] [log] [blame]
Johnny Chene8d9dc62011-10-31 19:04:07 +00001#!/usr/bin/env python
2
3"""
4Run the test suite using a separate process for each test file.
Vince Harronede59652015-01-08 02:11:26 +00005
Siva Chandra2d7832e2015-05-08 23:08:53 +00006Each test will run with a time limit of 10 minutes by default.
Vince Harronede59652015-01-08 02:11:26 +00007
Siva Chandra2d7832e2015-05-08 23:08:53 +00008Override the default time limit of 10 minutes by setting
Vince Harronede59652015-01-08 02:11:26 +00009the environment variable LLDB_TEST_TIMEOUT.
10
11E.g., export LLDB_TEST_TIMEOUT=10m
12
13Override the time limit for individual tests by setting
14the environment variable LLDB_[TEST NAME]_TIMEOUT.
15
16E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m
17
18Set to "0" to run without time limit.
19
20E.g., export LLDB_TEST_TIMEOUT=0
21or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
Vince Harrondcc2b9f2015-05-27 04:40:36 +000022
23To collect core files for timed out tests, do the following before running dosep.py
24
25OSX
26ulimit -c unlimited
27sudo sysctl -w kern.corefile=core.%P
28
29Linux:
30ulimit -c unlimited
31echo core.%p | sudo tee /proc/sys/kernel/core_pattern
Johnny Chene8d9dc62011-10-31 19:04:07 +000032"""
33
Greg Clayton2256d0d2014-03-24 23:01:57 +000034import multiprocessing
Todd Fiala3f0a3602014-07-08 06:42:37 +000035import os
Vince Harrondcc2b9f2015-05-27 04:40:36 +000036import fnmatch
Todd Fiala3f0a3602014-07-08 06:42:37 +000037import platform
Vince Harron06381732015-05-12 23:10:36 +000038import re
Vince Harronf8b9a1d2015-05-18 19:40:54 +000039import dotest_args
Vince Harron17f429f2014-12-13 00:08:19 +000040import shlex
41import subprocess
Todd Fiala3f0a3602014-07-08 06:42:37 +000042import sys
Steve Puccibefe2b12014-03-07 00:01:11 +000043
Johnny Chene8d9dc62011-10-31 19:04:07 +000044from optparse import OptionParser
45
Vince Harron17f429f2014-12-13 00:08:19 +000046def get_timeout_command():
Vince Harronede59652015-01-08 02:11:26 +000047 """Search for a suitable timeout command."""
Ying Chen93190c42015-07-20 20:04:22 +000048 if not sys.platform.startswith("win32"):
49 try:
50 subprocess.call("timeout", stderr=subprocess.PIPE)
51 return "timeout"
52 except OSError:
53 pass
Vince Harron17f429f2014-12-13 00:08:19 +000054 try:
Chaoren Lin45c17ff2015-05-28 23:00:10 +000055 subprocess.call("gtimeout", stderr=subprocess.PIPE)
Vince Harron17f429f2014-12-13 00:08:19 +000056 return "gtimeout"
57 except OSError:
58 pass
59 return None
60
61timeout_command = get_timeout_command()
62
Vince Harron17f429f2014-12-13 00:08:19 +000063# Status codes for running command with timeout.
64eTimedOut, ePassed, eFailed = 124, 0, 1
65
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000066output_lock = None
67test_counter = None
68total_tests = None
Chaoren Linffc63b02015-08-12 18:02:49 +000069test_name_len = None
Pavel Labath05ab2372015-07-06 15:57:52 +000070dotest_options = None
Zachary Turner38e64172015-08-10 17:46:11 +000071output_on_success = False
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000072
Chaoren Linffc63b02015-08-12 18:02:49 +000073def setup_global_variables(lock, counter, total, name_len, options):
74 global output_lock, test_counter, total_tests, test_name_len
75 global dotest_options
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000076 output_lock = lock
77 test_counter = counter
78 total_tests = total
Chaoren Linffc63b02015-08-12 18:02:49 +000079 test_name_len = name_len
Pavel Labath05ab2372015-07-06 15:57:52 +000080 dotest_options = options
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000081
Zachary Turner38e64172015-08-10 17:46:11 +000082def report_test_failure(name, command, output):
83 global output_lock
84 with output_lock:
Chaoren Linffc63b02015-08-12 18:02:49 +000085 print >> sys.stderr
Zachary Turner38e64172015-08-10 17:46:11 +000086 print >> sys.stderr, output
Chaoren Linffc63b02015-08-12 18:02:49 +000087 print >> sys.stderr, "[%s FAILED]" % name
Zachary Turner38e64172015-08-10 17:46:11 +000088 print >> sys.stderr, "Command invoked: %s" % ' '.join(command)
Chaoren Linffc63b02015-08-12 18:02:49 +000089 update_progress(name)
Zachary Turner38e64172015-08-10 17:46:11 +000090
91def report_test_pass(name, output):
92 global output_lock, output_on_success
93 with output_lock:
94 if output_on_success:
Chaoren Linffc63b02015-08-12 18:02:49 +000095 print >> sys.stderr
Zachary Turner38e64172015-08-10 17:46:11 +000096 print >> sys.stderr, output
Chaoren Linffc63b02015-08-12 18:02:49 +000097 print >> sys.stderr, "[%s PASSED]" % name
98 update_progress(name)
Zachary Turner38e64172015-08-10 17:46:11 +000099
Chaoren Linffc63b02015-08-12 18:02:49 +0000100def update_progress(test_name=""):
101 global output_lock, test_counter, total_tests, test_name_len
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000102 with output_lock:
Chaoren Linffc63b02015-08-12 18:02:49 +0000103 counter_len = len(str(total_tests))
104 sys.stderr.write(
105 "\r%*d out of %d test suites processed - %-*s" %
106 (counter_len, test_counter.value, total_tests,
107 test_name_len.value, test_name))
108 if len(test_name) > test_name_len.value:
109 test_name_len.value = len(test_name)
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000110 test_counter.value += 1
Zachary Turner38e64172015-08-10 17:46:11 +0000111 sys.stdout.flush()
112 sys.stderr.flush()
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000113
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000114def parse_test_results(output):
115 passes = 0
116 failures = 0
117 for result in output:
118 pass_count = re.search("^RESULT:.*([0-9]+) passes", result, re.MULTILINE)
119 fail_count = re.search("^RESULT:.*([0-9]+) failures", result, re.MULTILINE)
120 error_count = re.search("^RESULT:.*([0-9]+) errors", result, re.MULTILINE)
121 this_fail_count = 0
122 this_error_count = 0
123 if pass_count != None:
124 passes = passes + int(pass_count.group(1))
125 if fail_count != None:
126 failures = failures + int(fail_count.group(1))
127 if error_count != None:
128 failures = failures + int(error_count.group(1))
129 pass
130 return passes, failures
131
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000132def call_with_timeout(command, timeout, name):
Vince Harronede59652015-01-08 02:11:26 +0000133 """Run command with a timeout if possible."""
Vince Harrondcc2b9f2015-05-27 04:40:36 +0000134 """-s QUIT will create a coredump if they are enabled on your system"""
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000135 process = None
136 if timeout_command and timeout != "0":
137 command = [timeout_command, '-s', 'QUIT', timeout] + command
138 # Specifying a value for close_fds is unsupported on Windows when using subprocess.PIPE
Zachary Turnerdc494d52015-02-07 00:14:55 +0000139 if os.name != "nt":
Chaoren Lin9d2b7f92015-05-29 18:43:46 +0000140 process = subprocess.Popen(command, stdin=subprocess.PIPE,
141 stdout=subprocess.PIPE,
142 stderr=subprocess.PIPE,
143 close_fds=True)
Zachary Turnerdc494d52015-02-07 00:14:55 +0000144 else:
Chaoren Lin9d2b7f92015-05-29 18:43:46 +0000145 process = subprocess.Popen(command, stdin=subprocess.PIPE,
146 stdout=subprocess.PIPE,
147 stderr=subprocess.PIPE)
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000148 output = process.communicate()
149 exit_status = process.returncode
150 passes, failures = parse_test_results(output)
Zachary Turner38e64172015-08-10 17:46:11 +0000151 if exit_status == 0:
152 # stdout does not have any useful information from 'dotest.py', only stderr does.
153 report_test_pass(name, output[1])
154 else:
155 report_test_failure(name, command, output[1])
Chaoren Line80372a2015-08-12 18:02:53 +0000156 return name, exit_status, passes, failures
Johnny Chene8d9dc62011-10-31 19:04:07 +0000157
Vince Harron41657cc2015-05-21 18:15:09 +0000158def process_dir(root, files, test_root, dotest_argv):
Steve Puccibefe2b12014-03-07 00:01:11 +0000159 """Examine a directory for tests, and invoke any found within it."""
Chaoren Line80372a2015-08-12 18:02:53 +0000160 results = []
Steve Puccibefe2b12014-03-07 00:01:11 +0000161 for name in files:
Zachary Turnerf6896b02015-01-05 19:37:03 +0000162 script_file = os.path.join(test_root, "dotest.py")
Zachary Turnerf6896b02015-01-05 19:37:03 +0000163 command = ([sys.executable, script_file] +
Vince Harron41657cc2015-05-21 18:15:09 +0000164 dotest_argv +
Vince Harron17f429f2014-12-13 00:08:19 +0000165 ["-p", name, root])
166
167 timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
168
Pavel Labath05ab2372015-07-06 15:57:52 +0000169 timeout = os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or getDefaultTimeout(dotest_options.lldb_platform_name)
Vince Harron17f429f2014-12-13 00:08:19 +0000170
Chaoren Line80372a2015-08-12 18:02:53 +0000171 results.append(call_with_timeout(command, timeout, name))
Vince Harron17f429f2014-12-13 00:08:19 +0000172
Chaoren Line80372a2015-08-12 18:02:53 +0000173 # result = (name, status, passes, failures)
174 timed_out = [name for name, status, _, _ in results
175 if status == eTimedOut]
176 passed = [name for name, status, _, _ in results
177 if status == ePassed]
178 failed = [name for name, status, _, _ in results
179 if status != ePassed]
180 pass_count = sum([result[2] for result in results])
181 fail_count = sum([result[3] for result in results])
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000182
Chaoren Line80372a2015-08-12 18:02:53 +0000183 return (timed_out, passed, failed, pass_count, fail_count)
Steve Puccibefe2b12014-03-07 00:01:11 +0000184
185in_q = None
186out_q = None
187
Todd Fiala3f0a3602014-07-08 06:42:37 +0000188def process_dir_worker(arg_tuple):
Steve Puccibefe2b12014-03-07 00:01:11 +0000189 """Worker thread main loop when in multithreaded mode.
190 Takes one directory specification at a time and works on it."""
Chaoren Line80372a2015-08-12 18:02:53 +0000191 return process_dir(*arg_tuple)
Steve Puccibefe2b12014-03-07 00:01:11 +0000192
Vince Harron41657cc2015-05-21 18:15:09 +0000193def walk_and_invoke(test_directory, test_subdir, dotest_argv, num_threads):
Steve Puccibefe2b12014-03-07 00:01:11 +0000194 """Look for matched files and invoke test driver on each one.
195 In single-threaded mode, each test driver is invoked directly.
196 In multi-threaded mode, submit each test driver to a worker
Vince Harrone06a7a82015-05-12 23:12:19 +0000197 queue, and then wait for all to complete.
198
199 test_directory - lldb/test/ directory
200 test_subdir - lldb/test/ or a subfolder with the tests we're interested in running
201 """
Todd Fiala3f0a3602014-07-08 06:42:37 +0000202
203 # Collect the test files that we'll run.
204 test_work_items = []
Vince Harrone06a7a82015-05-12 23:12:19 +0000205 for root, dirs, files in os.walk(test_subdir, topdown=False):
Chaoren Linffc63b02015-08-12 18:02:49 +0000206 def is_test(name):
207 # Not interested in symbolically linked files.
208 if os.path.islink(os.path.join(root, name)):
209 return False
210 # Only interested in test files with the "Test*.py" naming pattern.
211 return name.startswith("Test") and name.endswith(".py")
Todd Fiala3f0a3602014-07-08 06:42:37 +0000212
Chaoren Linffc63b02015-08-12 18:02:49 +0000213 tests = filter(is_test, files)
214 test_work_items.append((root, tests, test_directory, dotest_argv))
215
216 global output_lock, test_counter, total_tests, test_name_len
Zachary Turner38e64172015-08-10 17:46:11 +0000217 output_lock = multiprocessing.RLock()
Chaoren Linffc63b02015-08-12 18:02:49 +0000218 # item = (root, tests, test_directory, dotest_argv)
219 total_tests = sum([len(item[1]) for item in test_work_items])
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000220 test_counter = multiprocessing.Value('i', 0)
Chaoren Linffc63b02015-08-12 18:02:49 +0000221 test_name_len = multiprocessing.Value('i', 0)
222 print >> sys.stderr, "Testing: %d test suites, %d thread%s" % (
223 total_tests, num_threads, (num_threads > 1) * "s")
224 update_progress()
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000225
Todd Fiala3f0a3602014-07-08 06:42:37 +0000226 # Run the items, either in a pool (for multicore speedup) or
227 # calling each individually.
228 if num_threads > 1:
Chaoren Linffc63b02015-08-12 18:02:49 +0000229 pool = multiprocessing.Pool(
230 num_threads,
231 initializer=setup_global_variables,
232 initargs=(output_lock, test_counter, total_tests, test_name_len,
233 dotest_options))
Todd Fiala3f0a3602014-07-08 06:42:37 +0000234 test_results = pool.map(process_dir_worker, test_work_items)
235 else:
Chaoren Line80372a2015-08-12 18:02:53 +0000236 test_results = map(process_dir_worker, test_work_items)
Todd Fiala3f0a3602014-07-08 06:42:37 +0000237
Chaoren Line80372a2015-08-12 18:02:53 +0000238 # result = (timed_out, failed, passed, fail_count, pass_count)
239 timed_out = sum([result[0] for result in test_results], [])
240 passed = sum([result[1] for result in test_results], [])
241 failed = sum([result[2] for result in test_results], [])
242 pass_count = sum([result[3] for result in test_results])
243 fail_count = sum([result[4] for result in test_results])
Todd Fiala3f0a3602014-07-08 06:42:37 +0000244
Chaoren Line80372a2015-08-12 18:02:53 +0000245 return (timed_out, passed, failed, pass_count, fail_count)
Johnny Chene8d9dc62011-10-31 19:04:07 +0000246
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000247def getExpectedTimeouts(platform_name):
Vince Harron06381732015-05-12 23:10:36 +0000248 # returns a set of test filenames that might timeout
249 # are we running against a remote target?
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000250 if platform_name is None:
Vince Harron06381732015-05-12 23:10:36 +0000251 target = sys.platform
252 remote = False
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000253 else:
254 m = re.search('remote-(\w+)', platform_name)
255 target = m.group(1)
256 remote = True
Vince Harron06381732015-05-12 23:10:36 +0000257
258 expected_timeout = set()
259
260 if target.startswith("linux"):
261 expected_timeout |= {
Vince Harronde92b522015-05-13 23:59:03 +0000262 "TestAttachDenied.py",
Vince Harron06381732015-05-12 23:10:36 +0000263 "TestAttachResume.py",
Chaoren Lin0b8bb3d2015-07-22 20:52:17 +0000264 "TestProcessAttach.py",
Vince Harron06381732015-05-12 23:10:36 +0000265 "TestConnectRemote.py",
266 "TestCreateAfterAttach.py",
Tamas Berghammer0d0ec9f2015-05-19 10:49:40 +0000267 "TestEvents.py",
Vince Harron06381732015-05-12 23:10:36 +0000268 "TestExitDuringStep.py",
Tamas Berghammer0b11db52015-06-02 14:45:25 +0000269 "TestHelloWorld.py", # Times out in ~10% of the times on the build bot
Oleksiy Vyalov18f4c9f2015-06-10 01:34:25 +0000270 "TestMultithreaded.py",
Vince Harrone8843892015-06-22 20:54:14 +0000271 "TestRegisters.py", # ~12/600 dosep runs (build 3120-3122)
Vince Harron06381732015-05-12 23:10:36 +0000272 "TestThreadStepOut.py",
273 }
274 elif target.startswith("android"):
275 expected_timeout |= {
276 "TestExitDuringStep.py",
277 "TestHelloWorld.py",
278 }
Ed Maste4dd8fba2015-05-14 16:25:52 +0000279 elif target.startswith("freebsd"):
280 expected_timeout |= {
281 "TestBreakpointConditions.py",
Ed Maste08948132015-05-28 18:45:30 +0000282 "TestChangeProcessGroup.py",
Ed Mastebfd05632015-05-27 19:11:29 +0000283 "TestValueObjectRecursion.py",
Ed Maste4dd8fba2015-05-14 16:25:52 +0000284 "TestWatchpointConditionAPI.py",
285 }
Vince Harron0f173ac2015-05-18 19:36:33 +0000286 elif target.startswith("darwin"):
287 expected_timeout |= {
288 "TestThreadSpecificBreakpoint.py", # times out on MBP Retina, Mid 2012
289 }
Vince Harron06381732015-05-12 23:10:36 +0000290 return expected_timeout
291
Pavel Labathfad30cf2015-06-29 14:16:51 +0000292def getDefaultTimeout(platform_name):
293 if os.getenv("LLDB_TEST_TIMEOUT"):
294 return os.getenv("LLDB_TEST_TIMEOUT")
295
296 if platform_name is None:
297 platform_name = sys.platform
298
299 if platform_name.startswith("remote-"):
300 return "10m"
301 else:
302 return "4m"
303
Vince Harron0b9dbb52015-05-21 18:18:52 +0000304def touch(fname, times=None):
Greg Clayton8c3f9c92015-08-11 21:01:32 +0000305 if os.path.exists(fname):
Vince Harron0b9dbb52015-05-21 18:18:52 +0000306 os.utime(fname, times)
307
Vince Harrondcc2b9f2015-05-27 04:40:36 +0000308def find(pattern, path):
309 result = []
310 for root, dirs, files in os.walk(path):
311 for name in files:
312 if fnmatch.fnmatch(name, pattern):
313 result.append(os.path.join(root, name))
314 return result
315
Johnny Chene8d9dc62011-10-31 19:04:07 +0000316def main():
Vince Harrond5fa1022015-05-10 15:24:12 +0000317 # We can't use sys.path[0] to determine the script directory
318 # because it doesn't work under a debugger
Vince Harrone06a7a82015-05-12 23:12:19 +0000319 test_directory = os.path.dirname(os.path.realpath(__file__))
Johnny Chene8d9dc62011-10-31 19:04:07 +0000320 parser = OptionParser(usage="""\
321Run lldb test suite using a separate process for each test file.
Vince Harronede59652015-01-08 02:11:26 +0000322
Siva Chandra2d7832e2015-05-08 23:08:53 +0000323 Each test will run with a time limit of 10 minutes by default.
Vince Harronede59652015-01-08 02:11:26 +0000324
Siva Chandra2d7832e2015-05-08 23:08:53 +0000325 Override the default time limit of 10 minutes by setting
Vince Harronede59652015-01-08 02:11:26 +0000326 the environment variable LLDB_TEST_TIMEOUT.
327
328 E.g., export LLDB_TEST_TIMEOUT=10m
329
330 Override the time limit for individual tests by setting
331 the environment variable LLDB_[TEST NAME]_TIMEOUT.
332
333 E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m
334
335 Set to "0" to run without time limit.
336
337 E.g., export LLDB_TEST_TIMEOUT=0
338 or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
Johnny Chene8d9dc62011-10-31 19:04:07 +0000339""")
340 parser.add_option('-o', '--options',
341 type='string', action='store',
342 dest='dotest_options',
343 help="""The options passed to 'dotest.py' if specified.""")
344
Zachary Turner38e64172015-08-10 17:46:11 +0000345 parser.add_option('-s', '--output-on-success',
346 action='store_true',
347 dest='output_on_success',
348 default=False,
349 help="""Print full output of 'dotest.py' even when it succeeds.""")
350
Greg Clayton2256d0d2014-03-24 23:01:57 +0000351 parser.add_option('-t', '--threads',
352 type='int',
353 dest='num_threads',
Ed Mastecec2a5b2014-11-21 02:41:25 +0000354 help="""The number of threads to use when running tests separately.""")
Greg Clayton2256d0d2014-03-24 23:01:57 +0000355
Johnny Chene8d9dc62011-10-31 19:04:07 +0000356 opts, args = parser.parse_args()
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000357 dotest_option_string = opts.dotest_options
358
Vince Harron41657cc2015-05-21 18:15:09 +0000359 is_posix = (os.name == "posix")
360 dotest_argv = shlex.split(dotest_option_string, posix=is_posix) if dotest_option_string else []
Vince Harron8994fed2015-05-22 19:49:23 +0000361
362 parser = dotest_args.create_parser()
Pavel Labath05ab2372015-07-06 15:57:52 +0000363 global dotest_options
Zachary Turner38e64172015-08-10 17:46:11 +0000364 global output_on_success
365 output_on_success = opts.output_on_success
Vince Harron8994fed2015-05-22 19:49:23 +0000366 dotest_options = dotest_args.parse_args(parser, dotest_argv)
367
Vince Harron41657cc2015-05-21 18:15:09 +0000368 if not dotest_options.s:
369 # no session log directory, we need to add this to prevent
370 # every dotest invocation from creating its own directory
371 import datetime
372 # The windows platforms don't like ':' in the pathname.
373 timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
374 dotest_argv.append('-s')
375 dotest_argv.append(timestamp_started)
Vince Harron0b9dbb52015-05-21 18:18:52 +0000376 dotest_options.s = timestamp_started
377
378 session_dir = os.path.join(os.getcwd(), dotest_options.s)
Ed Mastecec2a5b2014-11-21 02:41:25 +0000379
Vince Harrone06a7a82015-05-12 23:12:19 +0000380 # The root directory was specified on the command line
381 if len(args) == 0:
382 test_subdir = test_directory
383 else:
384 test_subdir = os.path.join(test_directory, args[0])
385
Vince Harrondcc2b9f2015-05-27 04:40:36 +0000386 # clean core files in test tree from previous runs (Linux)
387 cores = find('core.*', test_subdir)
388 for core in cores:
389 os.unlink(core)
390
Ed Mastecec2a5b2014-11-21 02:41:25 +0000391 if opts.num_threads:
392 num_threads = opts.num_threads
393 else:
Greg Clayton2256d0d2014-03-24 23:01:57 +0000394 num_threads_str = os.environ.get("LLDB_TEST_THREADS")
395 if num_threads_str:
396 num_threads = int(num_threads_str)
Greg Clayton2256d0d2014-03-24 23:01:57 +0000397 else:
Ed Mastecec2a5b2014-11-21 02:41:25 +0000398 num_threads = multiprocessing.cpu_count()
399 if num_threads < 1:
400 num_threads = 1
Johnny Chene8d9dc62011-10-31 19:04:07 +0000401
Daniel Maleab42556f2013-04-19 18:32:53 +0000402 system_info = " ".join(platform.uname())
Chaoren Line80372a2015-08-12 18:02:53 +0000403 (timed_out, passed, failed, pass_count, fail_count) = walk_and_invoke(
404 test_directory, test_subdir, dotest_argv, num_threads)
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000405
Vince Harron17f429f2014-12-13 00:08:19 +0000406 timed_out = set(timed_out)
Chaoren Line80372a2015-08-12 18:02:53 +0000407 num_test_files = len(passed) + len(failed)
408 num_test_cases = pass_count + fail_count
Daniel Maleab42556f2013-04-19 18:32:53 +0000409
Vince Harrondcc2b9f2015-05-27 04:40:36 +0000410 # move core files into session dir
411 cores = find('core.*', test_subdir)
412 for core in cores:
413 dst = core.replace(test_directory, "")[1:]
414 dst = dst.replace(os.path.sep, "-")
415 os.rename(core, os.path.join(session_dir, dst))
416
Vince Harron06381732015-05-12 23:10:36 +0000417 # remove expected timeouts from failures
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000418 expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
Vince Harron06381732015-05-12 23:10:36 +0000419 for xtime in expected_timeout:
420 if xtime in timed_out:
421 timed_out.remove(xtime)
422 failed.remove(xtime)
Vince Harron0b9dbb52015-05-21 18:18:52 +0000423 result = "ExpectedTimeout"
424 elif xtime in passed:
425 result = "UnexpectedCompletion"
426 else:
427 result = None # failed
428
429 if result:
430 test_name = os.path.splitext(xtime)[0]
431 touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
Vince Harron06381732015-05-12 23:10:36 +0000432
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000433 print
Chaoren Lin5a59e462015-08-12 18:02:51 +0000434 sys.stdout.write("Ran %d test suites" % num_test_files)
435 if num_test_files > 0:
436 sys.stdout.write(" (%d failed) (%f%%)" % (
437 len(failed), 100.0 * len(failed) / num_test_files))
438 print
Chaoren Line80372a2015-08-12 18:02:53 +0000439 sys.stdout.write("Ran %d test cases" % num_test_cases)
440 if num_test_cases > 0:
Chaoren Lin5a59e462015-08-12 18:02:51 +0000441 sys.stdout.write(" (%d failed) (%f%%)" % (
Chaoren Line80372a2015-08-12 18:02:53 +0000442 fail_count, 100.0 * fail_count / num_test_cases))
Chaoren Lin5a59e462015-08-12 18:02:51 +0000443 print
Daniel Maleacbaef262013-02-15 21:31:37 +0000444 if len(failed) > 0:
Shawn Best13491c42014-10-22 19:29:00 +0000445 failed.sort()
Ying Chen10ed1a92015-05-28 23:51:49 +0000446 print "Failing Tests (%d)" % len(failed)
Daniel Maleacbaef262013-02-15 21:31:37 +0000447 for f in failed:
Vince Harron17f429f2014-12-13 00:08:19 +0000448 print "%s: LLDB (suite) :: %s (%s)" % (
449 "TIMEOUT" if f in timed_out else "FAIL", f, system_info
450 )
Daniel Maleacbaef262013-02-15 21:31:37 +0000451 sys.exit(1)
452 sys.exit(0)
Johnny Chene8d9dc62011-10-31 19:04:07 +0000453
454if __name__ == '__main__':
455 main()