blob: d666fdbe2dfa697081ce24de53be598a7ed0c295 [file] [log] [blame]
Johnny Chene8d9dc62011-10-31 19:04:07 +00001#!/usr/bin/env python
2
3"""
4Run the test suite using a separate process for each test file.
Vince Harronede59652015-01-08 02:11:26 +00005
Siva Chandra2d7832e2015-05-08 23:08:53 +00006Each test will run with a time limit of 10 minutes by default.
Vince Harronede59652015-01-08 02:11:26 +00007
Siva Chandra2d7832e2015-05-08 23:08:53 +00008Override the default time limit of 10 minutes by setting
Vince Harronede59652015-01-08 02:11:26 +00009the environment variable LLDB_TEST_TIMEOUT.
10
11E.g., export LLDB_TEST_TIMEOUT=10m
12
13Override the time limit for individual tests by setting
14the environment variable LLDB_[TEST NAME]_TIMEOUT.
15
16E.g., export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=2m
17
18Set to "0" to run without time limit.
19
20E.g., export LLDB_TEST_TIMEOUT=0
21or export LLDB_TESTCONCURRENTEVENTS_TIMEOUT=0
Vince Harrondcc2b9f2015-05-27 04:40:36 +000022
Chaoren Linb6325d02015-08-12 18:02:54 +000023To collect core files for timed out tests,
24do the following before running dosep.py
Vince Harrondcc2b9f2015-05-27 04:40:36 +000025
26OSX
27ulimit -c unlimited
28sudo sysctl -w kern.corefile=core.%P
29
30Linux:
31ulimit -c unlimited
32echo core.%p | sudo tee /proc/sys/kernel/core_pattern
Johnny Chene8d9dc62011-10-31 19:04:07 +000033"""
34
Todd Fiala68615ce2015-09-15 21:38:04 +000035import asyncore
Vince Harrondcc2b9f2015-05-27 04:40:36 +000036import fnmatch
Todd Fiala8cbeed32015-09-08 22:22:33 +000037import multiprocessing
38import multiprocessing.pool
39import os
Todd Fiala3f0a3602014-07-08 06:42:37 +000040import platform
Todd Fiala8cbeed32015-09-08 22:22:33 +000041import Queue
Vince Harron06381732015-05-12 23:10:36 +000042import re
Todd Fiala8cbeed32015-09-08 22:22:33 +000043import signal
Vince Harron17f429f2014-12-13 00:08:19 +000044import subprocess
Todd Fiala3f0a3602014-07-08 06:42:37 +000045import sys
Todd Fiala8cbeed32015-09-08 22:22:33 +000046import threading
Greg Clayton1827fc22015-09-19 00:39:09 +000047import test_results
Todd Fiala68615ce2015-09-15 21:38:04 +000048import dotest_channels
Todd Fiala8cbeed32015-09-08 22:22:33 +000049import dotest_args
Steve Puccibefe2b12014-03-07 00:01:11 +000050
Chaoren Linb6325d02015-08-12 18:02:54 +000051
Vince Harron17f429f2014-12-13 00:08:19 +000052def get_timeout_command():
Vince Harronede59652015-01-08 02:11:26 +000053 """Search for a suitable timeout command."""
Ying Chen93190c42015-07-20 20:04:22 +000054 if not sys.platform.startswith("win32"):
55 try:
56 subprocess.call("timeout", stderr=subprocess.PIPE)
57 return "timeout"
58 except OSError:
59 pass
Vince Harron17f429f2014-12-13 00:08:19 +000060 try:
Chaoren Lin45c17ff2015-05-28 23:00:10 +000061 subprocess.call("gtimeout", stderr=subprocess.PIPE)
Vince Harron17f429f2014-12-13 00:08:19 +000062 return "gtimeout"
63 except OSError:
64 pass
65 return None
66
67timeout_command = get_timeout_command()
68
Vince Harron17f429f2014-12-13 00:08:19 +000069# Status codes for running command with timeout.
70eTimedOut, ePassed, eFailed = 124, 0, 1
71
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000072output_lock = None
73test_counter = None
74total_tests = None
Chaoren Linffc63b02015-08-12 18:02:49 +000075test_name_len = None
Pavel Labath05ab2372015-07-06 15:57:52 +000076dotest_options = None
Zachary Turner38e64172015-08-10 17:46:11 +000077output_on_success = False
Todd Fiala68615ce2015-09-15 21:38:04 +000078RESULTS_FORMATTER = None
79RUNNER_PROCESS_ASYNC_MAP = None
80RESULTS_LISTENER_CHANNEL = None
Chaoren Linb6325d02015-08-12 18:02:54 +000081
Todd Fiala33896a92015-09-18 21:01:13 +000082"""Contains an optional function pointer that can return the worker index
83 for the given thread/process calling it. Returns a 0-based index."""
84GET_WORKER_INDEX = None
85
Todd Fiala1cc97b42015-09-21 05:42:26 +000086
Todd Fiala33896a92015-09-18 21:01:13 +000087def setup_global_variables(
88 lock, counter, total, name_len, options, worker_index_map):
Chaoren Linffc63b02015-08-12 18:02:49 +000089 global output_lock, test_counter, total_tests, test_name_len
90 global dotest_options
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000091 output_lock = lock
92 test_counter = counter
93 total_tests = total
Chaoren Linffc63b02015-08-12 18:02:49 +000094 test_name_len = name_len
Pavel Labath05ab2372015-07-06 15:57:52 +000095 dotest_options = options
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +000096
Todd Fiala33896a92015-09-18 21:01:13 +000097 if worker_index_map is not None:
98 # We'll use the output lock for this to avoid sharing another lock.
99 # This won't be used much.
100 index_lock = lock
101
102 def get_worker_index_use_pid():
103 """Returns a 0-based, process-unique index for the worker."""
104 pid = os.getpid()
105 with index_lock:
106 if pid not in worker_index_map:
107 worker_index_map[pid] = len(worker_index_map)
108 return worker_index_map[pid]
109
110 global GET_WORKER_INDEX
111 GET_WORKER_INDEX = get_worker_index_use_pid
112
Chaoren Linb6325d02015-08-12 18:02:54 +0000113
Zachary Turner38e64172015-08-10 17:46:11 +0000114def report_test_failure(name, command, output):
115 global output_lock
116 with output_lock:
Greg Clayton1827fc22015-09-19 00:39:09 +0000117 if not (RESULTS_FORMATTER and RESULTS_FORMATTER.is_using_terminal()):
118 print >> sys.stderr
119 print >> sys.stderr, output
120 print >> sys.stderr, "[%s FAILED]" % name
121 print >> sys.stderr, "Command invoked: %s" % ' '.join(command)
Chaoren Linffc63b02015-08-12 18:02:49 +0000122 update_progress(name)
Zachary Turner38e64172015-08-10 17:46:11 +0000123
Chaoren Linb6325d02015-08-12 18:02:54 +0000124
Zachary Turner38e64172015-08-10 17:46:11 +0000125def report_test_pass(name, output):
126 global output_lock, output_on_success
127 with output_lock:
Greg Clayton1827fc22015-09-19 00:39:09 +0000128 if not (RESULTS_FORMATTER and RESULTS_FORMATTER.is_using_terminal()):
129 if output_on_success:
130 print >> sys.stderr
131 print >> sys.stderr, output
132 print >> sys.stderr, "[%s PASSED]" % name
Chaoren Linffc63b02015-08-12 18:02:49 +0000133 update_progress(name)
Zachary Turner38e64172015-08-10 17:46:11 +0000134
Chaoren Linb6325d02015-08-12 18:02:54 +0000135
Chaoren Linffc63b02015-08-12 18:02:49 +0000136def update_progress(test_name=""):
137 global output_lock, test_counter, total_tests, test_name_len
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000138 with output_lock:
Chaoren Linffc63b02015-08-12 18:02:49 +0000139 counter_len = len(str(total_tests))
Greg Clayton1827fc22015-09-19 00:39:09 +0000140 if not (RESULTS_FORMATTER and RESULTS_FORMATTER.is_using_terminal()):
141 sys.stderr.write(
142 "\r%*d out of %d test suites processed - %-*s" %
143 (counter_len, test_counter.value, total_tests,
144 test_name_len.value, test_name))
Chaoren Linffc63b02015-08-12 18:02:49 +0000145 if len(test_name) > test_name_len.value:
146 test_name_len.value = len(test_name)
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000147 test_counter.value += 1
Zachary Turner38e64172015-08-10 17:46:11 +0000148 sys.stdout.flush()
149 sys.stderr.flush()
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +0000150
Chaoren Linb6325d02015-08-12 18:02:54 +0000151
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000152def parse_test_results(output):
153 passes = 0
154 failures = 0
Zachary Turner4cceca72015-08-14 16:45:32 +0000155 unexpected_successes = 0
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000156 for result in output:
Chaoren Linb6325d02015-08-12 18:02:54 +0000157 pass_count = re.search("^RESULT:.*([0-9]+) passes",
158 result, re.MULTILINE)
159 fail_count = re.search("^RESULT:.*([0-9]+) failures",
160 result, re.MULTILINE)
161 error_count = re.search("^RESULT:.*([0-9]+) errors",
162 result, re.MULTILINE)
Zachary Turner4cceca72015-08-14 16:45:32 +0000163 unexpected_success_count = re.search("^RESULT:.*([0-9]+) unexpected successes",
164 result, re.MULTILINE)
Chaoren Linb6325d02015-08-12 18:02:54 +0000165 if pass_count is not None:
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000166 passes = passes + int(pass_count.group(1))
Chaoren Linb6325d02015-08-12 18:02:54 +0000167 if fail_count is not None:
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000168 failures = failures + int(fail_count.group(1))
Zachary Turner4cceca72015-08-14 16:45:32 +0000169 if unexpected_success_count is not None:
170 unexpected_successes = unexpected_successes + int(unexpected_success_count.group(1))
Chaoren Linb6325d02015-08-12 18:02:54 +0000171 if error_count is not None:
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000172 failures = failures + int(error_count.group(1))
173 pass
Zachary Turner4cceca72015-08-14 16:45:32 +0000174 return passes, failures, unexpected_successes
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000175
Chaoren Linb6325d02015-08-12 18:02:54 +0000176
Todd Fiala8cbeed32015-09-08 22:22:33 +0000177def call_with_timeout(command, timeout, name, inferior_pid_events):
Todd Fiala68615ce2015-09-15 21:38:04 +0000178 """Run command with a timeout if possible.
179 -s QUIT will create a coredump if they are enabled on your system
180 """
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000181 process = None
182 if timeout_command and timeout != "0":
183 command = [timeout_command, '-s', 'QUIT', timeout] + command
Todd Fiala68615ce2015-09-15 21:38:04 +0000184
Todd Fiala33896a92015-09-18 21:01:13 +0000185 if GET_WORKER_INDEX is not None:
Todd Fialae83f1402015-09-18 22:45:31 +0000186 try:
187 worker_index = GET_WORKER_INDEX()
188 command.extend([
Todd Fiala40b180e2015-09-18 23:46:30 +0000189 "--event-add-entries", "worker_index={}:int".format(worker_index)])
Todd Fialae83f1402015-09-18 22:45:31 +0000190 except:
191 # Ctrl-C does bad things to multiprocessing.Manager.dict() lookup.
192 pass
193
Chaoren Linb6325d02015-08-12 18:02:54 +0000194 # Specifying a value for close_fds is unsupported on Windows when using
195 # subprocess.PIPE
Zachary Turnerdc494d52015-02-07 00:14:55 +0000196 if os.name != "nt":
Chaoren Linb6325d02015-08-12 18:02:54 +0000197 process = subprocess.Popen(command,
198 stdin=subprocess.PIPE,
199 stdout=subprocess.PIPE,
200 stderr=subprocess.PIPE,
201 close_fds=True)
Zachary Turnerdc494d52015-02-07 00:14:55 +0000202 else:
Chaoren Linb6325d02015-08-12 18:02:54 +0000203 process = subprocess.Popen(command,
204 stdin=subprocess.PIPE,
205 stdout=subprocess.PIPE,
206 stderr=subprocess.PIPE)
Todd Fiala8cbeed32015-09-08 22:22:33 +0000207 inferior_pid = process.pid
208 if inferior_pid_events:
209 inferior_pid_events.put_nowait(('created', inferior_pid))
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000210 output = process.communicate()
Todd Fiala68615ce2015-09-15 21:38:04 +0000211
212 # The inferior should now be entirely wrapped up.
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000213 exit_status = process.returncode
Todd Fiala68615ce2015-09-15 21:38:04 +0000214 if exit_status is None:
215 raise Exception(
216 "no exit status available after the inferior dotest.py "
217 "should have completed")
218
Todd Fiala8cbeed32015-09-08 22:22:33 +0000219 if inferior_pid_events:
220 inferior_pid_events.put_nowait(('destroyed', inferior_pid))
221
Zachary Turner4cceca72015-08-14 16:45:32 +0000222 passes, failures, unexpected_successes = parse_test_results(output)
Zachary Turner38e64172015-08-10 17:46:11 +0000223 if exit_status == 0:
Chaoren Linb6325d02015-08-12 18:02:54 +0000224 # stdout does not have any useful information from 'dotest.py',
225 # only stderr does.
Zachary Turner38e64172015-08-10 17:46:11 +0000226 report_test_pass(name, output[1])
227 else:
Todd Fiala68615ce2015-09-15 21:38:04 +0000228 # TODO need to differentiate a failing test from a run that
229 # was broken out of by a SIGTERM/SIGKILL, reporting those as
230 # an error. If a signal-based completion, need to call that
231 # an error.
Zachary Turner38e64172015-08-10 17:46:11 +0000232 report_test_failure(name, command, output[1])
Zachary Turner4cceca72015-08-14 16:45:32 +0000233 return name, exit_status, passes, failures, unexpected_successes
Johnny Chene8d9dc62011-10-31 19:04:07 +0000234
Chaoren Linb6325d02015-08-12 18:02:54 +0000235
Todd Fiala8cbeed32015-09-08 22:22:33 +0000236def process_dir(root, files, test_root, dotest_argv, inferior_pid_events):
Steve Puccibefe2b12014-03-07 00:01:11 +0000237 """Examine a directory for tests, and invoke any found within it."""
Chaoren Line80372a2015-08-12 18:02:53 +0000238 results = []
Steve Puccibefe2b12014-03-07 00:01:11 +0000239 for name in files:
Zachary Turnerf6896b02015-01-05 19:37:03 +0000240 script_file = os.path.join(test_root, "dotest.py")
Zachary Turnerf6896b02015-01-05 19:37:03 +0000241 command = ([sys.executable, script_file] +
Vince Harron41657cc2015-05-21 18:15:09 +0000242 dotest_argv +
Todd Fialafed95662015-09-03 18:58:44 +0000243 ["--inferior", "-p", name, root])
Vince Harron17f429f2014-12-13 00:08:19 +0000244
245 timeout_name = os.path.basename(os.path.splitext(name)[0]).upper()
246
Chaoren Linb6325d02015-08-12 18:02:54 +0000247 timeout = (os.getenv("LLDB_%s_TIMEOUT" % timeout_name) or
248 getDefaultTimeout(dotest_options.lldb_platform_name))
Vince Harron17f429f2014-12-13 00:08:19 +0000249
Todd Fiala8cbeed32015-09-08 22:22:33 +0000250 results.append(call_with_timeout(
251 command, timeout, name, inferior_pid_events))
Vince Harron17f429f2014-12-13 00:08:19 +0000252
Zachary Turner4cceca72015-08-14 16:45:32 +0000253 # result = (name, status, passes, failures, unexpected_successes)
254 timed_out = [name for name, status, _, _, _ in results
Chaoren Line80372a2015-08-12 18:02:53 +0000255 if status == eTimedOut]
Zachary Turner4cceca72015-08-14 16:45:32 +0000256 passed = [name for name, status, _, _, _ in results
Chaoren Line80372a2015-08-12 18:02:53 +0000257 if status == ePassed]
Zachary Turner4cceca72015-08-14 16:45:32 +0000258 failed = [name for name, status, _, _, _ in results
Chaoren Line80372a2015-08-12 18:02:53 +0000259 if status != ePassed]
Zachary Turner4cceca72015-08-14 16:45:32 +0000260 unexpected_passes = [name for name, _, _, _, unexpected_successes in results
261 if unexpected_successes > 0]
Todd Fialafed95662015-09-03 18:58:44 +0000262
Chaoren Line80372a2015-08-12 18:02:53 +0000263 pass_count = sum([result[2] for result in results])
264 fail_count = sum([result[3] for result in results])
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +0000265
Zachary Turner4cceca72015-08-14 16:45:32 +0000266 return (timed_out, passed, failed, unexpected_passes, pass_count, fail_count)
Steve Puccibefe2b12014-03-07 00:01:11 +0000267
268in_q = None
269out_q = None
270
Chaoren Linb6325d02015-08-12 18:02:54 +0000271
Todd Fiala8cbeed32015-09-08 22:22:33 +0000272def process_dir_worker_multiprocessing(
273 a_output_lock, a_test_counter, a_total_tests, a_test_name_len,
Todd Fiala33896a92015-09-18 21:01:13 +0000274 a_dotest_options, job_queue, result_queue, inferior_pid_events,
275 worker_index_map):
Todd Fiala8cbeed32015-09-08 22:22:33 +0000276 """Worker thread main loop when in multiprocessing mode.
Steve Puccibefe2b12014-03-07 00:01:11 +0000277 Takes one directory specification at a time and works on it."""
Todd Fiala8cbeed32015-09-08 22:22:33 +0000278
279 # Shut off interrupt handling in the child process.
280 signal.signal(signal.SIGINT, signal.SIG_IGN)
Todd Fiala1cc97b42015-09-21 05:42:26 +0000281 signal.signal(signal.SIGHUP, signal.SIG_IGN)
Todd Fiala8cbeed32015-09-08 22:22:33 +0000282
283 # Setup the global state for the worker process.
284 setup_global_variables(
285 a_output_lock, a_test_counter, a_total_tests, a_test_name_len,
Todd Fiala33896a92015-09-18 21:01:13 +0000286 a_dotest_options, worker_index_map)
Todd Fiala8cbeed32015-09-08 22:22:33 +0000287
288 # Keep grabbing entries from the queue until done.
289 while not job_queue.empty():
290 try:
291 job = job_queue.get(block=False)
292 result = process_dir(job[0], job[1], job[2], job[3],
293 inferior_pid_events)
294 result_queue.put(result)
295 except Queue.Empty:
296 # Fine, we're done.
297 pass
Steve Puccibefe2b12014-03-07 00:01:11 +0000298
Chaoren Linb6325d02015-08-12 18:02:54 +0000299
Todd Fiala8cbeed32015-09-08 22:22:33 +0000300def process_dir_worker_multiprocessing_pool(args):
301 return process_dir(*args)
302
303
Todd Fiala68615ce2015-09-15 21:38:04 +0000304def process_dir_worker_threading(job_queue, result_queue, inferior_pid_events):
Todd Fiala8cbeed32015-09-08 22:22:33 +0000305 """Worker thread main loop when in threading mode.
306
307 This one supports the hand-rolled pooling support.
308
309 Takes one directory specification at a time and works on it."""
310
311 # Keep grabbing entries from the queue until done.
312 while not job_queue.empty():
313 try:
314 job = job_queue.get(block=False)
315 result = process_dir(job[0], job[1], job[2], job[3],
316 inferior_pid_events)
317 result_queue.put(result)
318 except Queue.Empty:
319 # Fine, we're done.
320 pass
321
322
323def process_dir_worker_threading_pool(args):
324 return process_dir(*args)
325
326
327def process_dir_mapper_inprocess(args):
328 """Map adapter for running the subprocess-based, non-threaded test runner.
329
330 @param args the process work item tuple
331 @return the test result tuple
332 """
333 return process_dir(*args)
334
335
336def collect_active_pids_from_pid_events(event_queue):
337 """
338 Returns the set of what should be active inferior pids based on
339 the event stream.
340
341 @param event_queue a multiprocessing.Queue containing events of the
342 form:
343 ('created', pid)
344 ('destroyed', pid)
345
346 @return set of inferior dotest.py pids activated but never completed.
347 """
348 active_pid_set = set()
349 while not event_queue.empty():
350 pid_event = event_queue.get_nowait()
351 if pid_event[0] == 'created':
352 active_pid_set.add(pid_event[1])
353 elif pid_event[0] == 'destroyed':
354 active_pid_set.remove(pid_event[1])
355 return active_pid_set
356
357
358def kill_all_worker_processes(workers, inferior_pid_events):
359 """
360 Kills all specified worker processes and their process tree.
361
362 @param workers a list of multiprocess.Process worker objects.
363 @param inferior_pid_events a multiprocess.Queue that contains
364 all inferior create and destroy events. Used to construct
365 the list of child pids still outstanding that need to be killed.
366 """
367 for worker in workers:
368 worker.terminate()
369 worker.join()
370
371 # Add all the child test pids created.
372 active_pid_set = collect_active_pids_from_pid_events(
373 inferior_pid_events)
374 for inferior_pid in active_pid_set:
375 print "killing inferior pid {}".format(inferior_pid)
376 os.kill(inferior_pid, signal.SIGKILL)
377
378
379def kill_all_worker_threads(workers, inferior_pid_events):
380 """
381 Kills all specified worker threads and their process tree.
382
383 @param workers a list of multiprocess.Process worker objects.
384 @param inferior_pid_events a multiprocess.Queue that contains
385 all inferior create and destroy events. Used to construct
386 the list of child pids still outstanding that need to be killed.
387 """
388
389 # Add all the child test pids created.
390 active_pid_set = collect_active_pids_from_pid_events(
391 inferior_pid_events)
392 for inferior_pid in active_pid_set:
393 print "killing inferior pid {}".format(inferior_pid)
394 os.kill(inferior_pid, signal.SIGKILL)
395
396 # We don't have a way to nuke the threads. However, since we killed
397 # all the inferiors, and we drained the job queue, this will be
398 # good enough. Wait cleanly for each worker thread to wrap up.
399 for worker in workers:
400 worker.join()
401
402
403def find_test_files_in_dir_tree(dir_root, found_func):
404 """Calls found_func for all the test files in the given dir hierarchy.
405
406 @param dir_root the path to the directory to start scanning
407 for test files. All files in this directory and all its children
408 directory trees will be searched.
409
410 @param found_func a callable object that will be passed
411 the parent directory (relative to dir_root) and the list of
412 test files from within that directory.
413 """
414 for root, _, files in os.walk(dir_root, topdown=False):
415 def is_test_filename(test_dir, base_filename):
416 """Returns True if the given filename matches the test name format.
417
418 @param test_dir the directory to check. Should be absolute or
419 relative to current working directory.
420
421 @param base_filename the base name of the filename to check for a
422 dherence to the python test case filename format.
423
424 @return True if name matches the python test case filename format.
425 """
426 # Not interested in symbolically linked files.
427 if os.path.islink(os.path.join(test_dir, base_filename)):
428 return False
429 # Only interested in test files with the "Test*.py" naming pattern.
430 return (base_filename.startswith("Test") and
431 base_filename.endswith(".py"))
432
433 tests = [filename for filename in files
434 if is_test_filename(root, filename)]
435 if tests:
436 found_func(root, tests)
437
438
439def initialize_global_vars_common(num_threads, test_work_items):
440 global total_tests, test_counter, test_name_len
Greg Clayton1827fc22015-09-19 00:39:09 +0000441
Todd Fiala8cbeed32015-09-08 22:22:33 +0000442 total_tests = sum([len(item[1]) for item in test_work_items])
443 test_counter = multiprocessing.Value('i', 0)
444 test_name_len = multiprocessing.Value('i', 0)
Greg Clayton1827fc22015-09-19 00:39:09 +0000445 if not (RESULTS_FORMATTER and RESULTS_FORMATTER.is_using_terminal()):
446 print >> sys.stderr, "Testing: %d test suites, %d thread%s" % (
447 total_tests, num_threads, (num_threads > 1) * "s")
Todd Fiala8cbeed32015-09-08 22:22:33 +0000448 update_progress()
449
450
451def initialize_global_vars_multiprocessing(num_threads, test_work_items):
452 # Initialize the global state we'll use to communicate with the
453 # rest of the flat module.
454 global output_lock
455 output_lock = multiprocessing.RLock()
Todd Fiala33896a92015-09-18 21:01:13 +0000456
Todd Fiala8cbeed32015-09-08 22:22:33 +0000457 initialize_global_vars_common(num_threads, test_work_items)
458
459
460def initialize_global_vars_threading(num_threads, test_work_items):
Todd Fiala33896a92015-09-18 21:01:13 +0000461 """Initializes global variables used in threading mode.
462 @param num_threads specifies the number of workers used.
463 @param test_work_items specifies all the work items
464 that will be processed.
465 """
Todd Fiala8cbeed32015-09-08 22:22:33 +0000466 # Initialize the global state we'll use to communicate with the
467 # rest of the flat module.
468 global output_lock
469 output_lock = threading.RLock()
Todd Fiala33896a92015-09-18 21:01:13 +0000470
471 index_lock = threading.RLock()
472 index_map = {}
473
474 def get_worker_index_threading():
475 """Returns a 0-based, thread-unique index for the worker thread."""
476 thread_id = threading.current_thread().ident
477 with index_lock:
478 if thread_id not in index_map:
479 index_map[thread_id] = len(index_map)
480 return index_map[thread_id]
481
482
483 global GET_WORKER_INDEX
484 GET_WORKER_INDEX = get_worker_index_threading
485
Todd Fiala8cbeed32015-09-08 22:22:33 +0000486 initialize_global_vars_common(num_threads, test_work_items)
487
488
Todd Fiala68615ce2015-09-15 21:38:04 +0000489def ctrl_c_loop(main_op_func, done_func, ctrl_c_handler):
490 """Provides a main loop that is Ctrl-C protected.
491
492 The main loop calls the main_op_func() repeatedly until done_func()
493 returns true. The ctrl_c_handler() method is called with a single
494 int parameter that contains the number of times the ctrl_c has been
495 hit (starting with 1). The ctrl_c_handler() should mutate whatever
496 it needs to have the done_func() return True as soon as it is desired
497 to exit the loop.
498 """
499 done = False
500 ctrl_c_count = 0
501
502 while not done:
503 try:
504 # See if we're done. Start with done check since it is
505 # the first thing executed after a Ctrl-C handler in the
506 # following loop.
507 done = done_func()
508 if not done:
509 # Run the main op once.
510 main_op_func()
511
512 except KeyboardInterrupt:
513 ctrl_c_count += 1
514 ctrl_c_handler(ctrl_c_count)
515
516
517def pump_workers_and_asyncore_map(workers, asyncore_map):
518 """Prunes out completed workers and maintains the asyncore loop.
519
520 The asyncore loop contains the optional socket listener
521 and handlers. When all workers are complete, this method
522 takes care of stopping the listener. It also runs the
523 asyncore loop for the given async map for 10 iterations.
524
525 @param workers the list of worker Thread/Process instances.
526
527 @param asyncore_map the asyncore threading-aware map that
528 indicates which channels are in use and still alive.
529 """
530
531 # Check on all the workers, removing them from the workers
532 # list as they complete.
533 dead_workers = []
534 for worker in workers:
535 # This non-blocking join call is what allows us
536 # to still receive keyboard interrupts.
537 worker.join(0.01)
538 if not worker.is_alive():
539 dead_workers.append(worker)
540 # Clear out the completed workers
541 for dead_worker in dead_workers:
542 workers.remove(dead_worker)
543
544 # If there are no more workers and there is a listener,
545 # close the listener.
546 global RESULTS_LISTENER_CHANNEL
547 if len(workers) == 0 and RESULTS_LISTENER_CHANNEL is not None:
548 RESULTS_LISTENER_CHANNEL.close()
549 RESULTS_LISTENER_CHANNEL = None
550
551 # Pump the asyncore map if it isn't empty.
552 if len(asyncore_map) > 0:
553 asyncore.loop(0.1, False, asyncore_map, 10)
554
555
556def handle_ctrl_c(ctrl_c_count, job_queue, workers, inferior_pid_events,
557 stop_all_inferiors_func):
558 """Performs the appropriate ctrl-c action for non-pool parallel test runners
559
560 @param ctrl_c_count starting with 1, indicates the number of times ctrl-c
561 has been intercepted. The value is 1 on the first intercept, 2 on the
562 second, etc.
563
564 @param job_queue a Queue object that contains the work still outstanding
565 (i.e. hasn't been assigned to a worker yet).
566
567 @param workers list of Thread or Process workers.
568
569 @param inferior_pid_events specifies a Queue of inferior process
570 construction and destruction events. Used to build the list of inferior
571 processes that should be killed if we get that far.
572
573 @param stop_all_inferiors_func a callable object that takes the
574 workers and inferior_pid_events parameters (in that order) if a hard
575 stop is to be used on the workers.
576 """
577
578 # Print out which Ctrl-C we're handling.
579 key_name = [
580 "first",
581 "second",
582 "third",
583 "many"]
584
585 if ctrl_c_count < len(key_name):
586 name_index = ctrl_c_count - 1
587 else:
588 name_index = len(key_name) - 1
589 message = "\nHandling {} KeyboardInterrupt".format(key_name[name_index])
590 with output_lock:
591 print message
592
593 if ctrl_c_count == 1:
594 # Remove all outstanding items from the work queue so we stop
595 # doing any more new work.
596 while not job_queue.empty():
597 try:
598 # Just drain it to stop more work from being started.
599 job_queue.get_nowait()
600 except Queue.Empty:
601 pass
602 with output_lock:
603 print "Stopped more work from being started."
604 elif ctrl_c_count == 2:
605 # Try to stop all inferiors, even the ones currently doing work.
606 stop_all_inferiors_func(workers, inferior_pid_events)
607 else:
608 with output_lock:
609 print "All teardown activities kicked off, should finish soon."
610
611
612def workers_and_async_done(workers, async_map):
613 """Returns True if the workers list and asyncore channels are all done.
614
615 @param workers list of workers (threads/processes). These must adhere
616 to the threading Thread or multiprocessing.Process interface.
617
618 @param async_map the threading-aware asyncore channel map to check
619 for live channels.
620
621 @return False if the workers list exists and has any entries in it, or
622 if the async_map exists and has any entries left in it; otherwise, True.
623 """
624 if workers is not None and len(workers) > 0:
625 # We're not done if we still have workers left.
626 return False
627 if async_map is not None and len(async_map) > 0:
628 return False
629 # We're done.
630 return True
631
632
Todd Fiala8cbeed32015-09-08 22:22:33 +0000633def multiprocessing_test_runner(num_threads, test_work_items):
634 """Provides hand-wrapped pooling test runner adapter with Ctrl-C support.
635
636 This concurrent test runner is based on the multiprocessing
637 library, and rolls its own worker pooling strategy so it
638 can handle Ctrl-C properly.
639
640 This test runner is known to have an issue running on
641 Windows platforms.
642
643 @param num_threads the number of worker processes to use.
644
645 @param test_work_items the iterable of test work item tuples
646 to run.
647 """
648
649 # Initialize our global state.
650 initialize_global_vars_multiprocessing(num_threads, test_work_items)
651
652 # Create jobs.
653 job_queue = multiprocessing.Queue(len(test_work_items))
654 for test_work_item in test_work_items:
655 job_queue.put(test_work_item)
656
657 result_queue = multiprocessing.Queue(len(test_work_items))
658
659 # Create queues for started child pids. Terminating
660 # the multiprocess processes does not terminate the
661 # child processes they spawn. We can remove this tracking
662 # if/when we move to having the multiprocess process directly
663 # perform the test logic. The Queue size needs to be able to
664 # hold 2 * (num inferior dotest.py processes started) entries.
665 inferior_pid_events = multiprocessing.Queue(4096)
666
Todd Fiala33896a92015-09-18 21:01:13 +0000667 # Worker dictionary allows each worker to figure out its worker index.
668 manager = multiprocessing.Manager()
669 worker_index_map = manager.dict()
670
Todd Fiala8cbeed32015-09-08 22:22:33 +0000671 # Create workers. We don't use multiprocessing.Pool due to
672 # challenges with handling ^C keyboard interrupts.
673 workers = []
674 for _ in range(num_threads):
675 worker = multiprocessing.Process(
676 target=process_dir_worker_multiprocessing,
677 args=(output_lock,
678 test_counter,
679 total_tests,
680 test_name_len,
681 dotest_options,
682 job_queue,
683 result_queue,
Todd Fiala33896a92015-09-18 21:01:13 +0000684 inferior_pid_events,
685 worker_index_map))
Todd Fiala8cbeed32015-09-08 22:22:33 +0000686 worker.start()
687 workers.append(worker)
688
Todd Fiala68615ce2015-09-15 21:38:04 +0000689 # Main loop: wait for all workers to finish and wait for
690 # the socket handlers to wrap up.
691 ctrl_c_loop(
692 # Main operation of loop
693 lambda: pump_workers_and_asyncore_map(
694 workers, RUNNER_PROCESS_ASYNC_MAP),
Todd Fiala8cbeed32015-09-08 22:22:33 +0000695
Todd Fiala68615ce2015-09-15 21:38:04 +0000696 # Return True when we're done with the main loop.
697 lambda: workers_and_async_done(workers, RUNNER_PROCESS_ASYNC_MAP),
Todd Fiala8cbeed32015-09-08 22:22:33 +0000698
Todd Fiala68615ce2015-09-15 21:38:04 +0000699 # Indicate what we do when we receive one or more Ctrl-Cs.
700 lambda ctrl_c_count: handle_ctrl_c(
701 ctrl_c_count, job_queue, workers, inferior_pid_events,
702 kill_all_worker_processes))
703
704 # Reap the test results.
Todd Fiala8cbeed32015-09-08 22:22:33 +0000705 test_results = []
706 while not result_queue.empty():
707 test_results.append(result_queue.get(block=False))
708 return test_results
709
710
Todd Fiala68615ce2015-09-15 21:38:04 +0000711def map_async_run_loop(future, channel_map, listener_channel):
712 """Blocks until the Pool.map_async completes and the channel completes.
713
714 @param future an AsyncResult instance from a Pool.map_async() call.
715
716 @param channel_map the asyncore dispatch channel map that should be pumped.
717 Optional: may be None.
718
719 @param listener_channel the channel representing a listener that should be
720 closed once the map_async results are available.
721
722 @return the results from the async_result instance.
723 """
724 map_results = None
725
726 done = False
727 while not done:
728 # Check if we need to reap the map results.
729 if map_results is None:
730 if future.ready():
731 # Get the results.
732 map_results = future.get()
733
734 # Close the runner process listener channel if we have
735 # one: no more connections will be incoming.
736 if listener_channel is not None:
737 listener_channel.close()
738
739 # Pump the asyncore loop if we have a listener socket.
740 if channel_map is not None:
741 asyncore.loop(0.01, False, channel_map, 10)
742
743 # Figure out if we're done running.
744 done = map_results is not None
745 if channel_map is not None:
746 # We have a runner process async map. Check if it
747 # is complete.
748 if len(channel_map) > 0:
749 # We still have an asyncore channel running. Not done yet.
750 done = False
751
752 return map_results
753
754
Todd Fiala8cbeed32015-09-08 22:22:33 +0000755def multiprocessing_test_runner_pool(num_threads, test_work_items):
756 # Initialize our global state.
757 initialize_global_vars_multiprocessing(num_threads, test_work_items)
758
Todd Fiala33896a92015-09-18 21:01:13 +0000759 manager = multiprocessing.Manager()
760 worker_index_map = manager.dict()
761
Todd Fiala8cbeed32015-09-08 22:22:33 +0000762 pool = multiprocessing.Pool(
763 num_threads,
764 initializer=setup_global_variables,
765 initargs=(output_lock, test_counter, total_tests, test_name_len,
Todd Fiala33896a92015-09-18 21:01:13 +0000766 dotest_options, worker_index_map))
Todd Fiala68615ce2015-09-15 21:38:04 +0000767
768 # Start the map operation (async mode).
769 map_future = pool.map_async(
770 process_dir_worker_multiprocessing_pool, test_work_items)
771 return map_async_run_loop(
772 map_future, RUNNER_PROCESS_ASYNC_MAP, RESULTS_LISTENER_CHANNEL)
Todd Fiala8cbeed32015-09-08 22:22:33 +0000773
774
775def threading_test_runner(num_threads, test_work_items):
776 """Provides hand-wrapped pooling threading-based test runner adapter
777 with Ctrl-C support.
778
779 This concurrent test runner is based on the threading
780 library, and rolls its own worker pooling strategy so it
781 can handle Ctrl-C properly.
782
783 @param num_threads the number of worker processes to use.
784
785 @param test_work_items the iterable of test work item tuples
786 to run.
787 """
788
789 # Initialize our global state.
790 initialize_global_vars_threading(num_threads, test_work_items)
791
792 # Create jobs.
793 job_queue = Queue.Queue()
794 for test_work_item in test_work_items:
795 job_queue.put(test_work_item)
796
797 result_queue = Queue.Queue()
798
799 # Create queues for started child pids. Terminating
800 # the threading threads does not terminate the
801 # child processes they spawn.
802 inferior_pid_events = Queue.Queue()
803
804 # Create workers. We don't use multiprocessing.pool.ThreadedPool
805 # due to challenges with handling ^C keyboard interrupts.
806 workers = []
807 for _ in range(num_threads):
808 worker = threading.Thread(
809 target=process_dir_worker_threading,
Todd Fiala68615ce2015-09-15 21:38:04 +0000810 args=(job_queue,
Todd Fiala8cbeed32015-09-08 22:22:33 +0000811 result_queue,
812 inferior_pid_events))
813 worker.start()
814 workers.append(worker)
815
Todd Fiala68615ce2015-09-15 21:38:04 +0000816 # Main loop: wait for all workers to finish and wait for
817 # the socket handlers to wrap up.
818 ctrl_c_loop(
819 # Main operation of loop
820 lambda: pump_workers_and_asyncore_map(
821 workers, RUNNER_PROCESS_ASYNC_MAP),
Todd Fiala8cbeed32015-09-08 22:22:33 +0000822
Todd Fiala68615ce2015-09-15 21:38:04 +0000823 # Return True when we're done with the main loop.
824 lambda: workers_and_async_done(workers, RUNNER_PROCESS_ASYNC_MAP),
Todd Fiala8cbeed32015-09-08 22:22:33 +0000825
Todd Fiala68615ce2015-09-15 21:38:04 +0000826 # Indicate what we do when we receive one or more Ctrl-Cs.
827 lambda ctrl_c_count: handle_ctrl_c(
828 ctrl_c_count, job_queue, workers, inferior_pid_events,
829 kill_all_worker_threads))
Todd Fiala8cbeed32015-09-08 22:22:33 +0000830
Todd Fiala68615ce2015-09-15 21:38:04 +0000831 # Reap the test results.
Todd Fiala8cbeed32015-09-08 22:22:33 +0000832 test_results = []
833 while not result_queue.empty():
834 test_results.append(result_queue.get(block=False))
835 return test_results
836
837
838def threading_test_runner_pool(num_threads, test_work_items):
839 # Initialize our global state.
840 initialize_global_vars_threading(num_threads, test_work_items)
841
Todd Fiala68615ce2015-09-15 21:38:04 +0000842 pool = multiprocessing.pool.ThreadPool(num_threads)
843 map_future = pool.map_async(
844 process_dir_worker_threading_pool, test_work_items)
845
846 return map_async_run_loop(
847 map_future, RUNNER_PROCESS_ASYNC_MAP, RESULTS_LISTENER_CHANNEL)
848
849
850def asyncore_run_loop(channel_map):
851 try:
852 asyncore.loop(None, False, channel_map)
853 except:
854 # Swallow it, we're seeing:
855 # error: (9, 'Bad file descriptor')
856 # when the listener channel is closed. Shouldn't be the case.
857 pass
Todd Fiala8cbeed32015-09-08 22:22:33 +0000858
859
860def inprocess_exec_test_runner(test_work_items):
861 # Initialize our global state.
862 initialize_global_vars_multiprocessing(1, test_work_items)
Todd Fiala8cbeed32015-09-08 22:22:33 +0000863
Todd Fiala33896a92015-09-18 21:01:13 +0000864 # We're always worker index 0
865 global GET_WORKER_INDEX
866 GET_WORKER_INDEX = lambda: 0
867
Todd Fiala68615ce2015-09-15 21:38:04 +0000868 # Run the listener and related channel maps in a separate thread.
869 # global RUNNER_PROCESS_ASYNC_MAP
870 global RESULTS_LISTENER_CHANNEL
871 if RESULTS_LISTENER_CHANNEL is not None:
872 socket_thread = threading.Thread(
873 target=lambda: asyncore_run_loop(RUNNER_PROCESS_ASYNC_MAP))
874 socket_thread.start()
875
876 # Do the work.
877 test_results = map(process_dir_mapper_inprocess, test_work_items)
878
879 # If we have a listener channel, shut it down here.
880 if RESULTS_LISTENER_CHANNEL is not None:
881 # Close down the channel.
882 RESULTS_LISTENER_CHANNEL.close()
883 RESULTS_LISTENER_CHANNEL = None
884
885 # Wait for the listener and handlers to complete.
886 socket_thread.join()
887
888 return test_results
Todd Fiala8cbeed32015-09-08 22:22:33 +0000889
890def walk_and_invoke(test_directory, test_subdir, dotest_argv,
891 test_runner_func):
Steve Puccibefe2b12014-03-07 00:01:11 +0000892 """Look for matched files and invoke test driver on each one.
893 In single-threaded mode, each test driver is invoked directly.
894 In multi-threaded mode, submit each test driver to a worker
Vince Harrone06a7a82015-05-12 23:12:19 +0000895 queue, and then wait for all to complete.
896
897 test_directory - lldb/test/ directory
Chaoren Linb6325d02015-08-12 18:02:54 +0000898 test_subdir - lldb/test/ or a subfolder with the tests we're interested in
899 running
Vince Harrone06a7a82015-05-12 23:12:19 +0000900 """
Todd Fiala68615ce2015-09-15 21:38:04 +0000901 # The async_map is important to keep all thread-related asyncore
902 # channels distinct when we call asyncore.loop() later on.
903 global RESULTS_LISTENER_CHANNEL, RUNNER_PROCESS_ASYNC_MAP
904 RUNNER_PROCESS_ASYNC_MAP = {}
905
906 # If we're outputting side-channel test results, create the socket
907 # listener channel and tell the inferior to send results to the
908 # port on which we'll be listening.
909 if RESULTS_FORMATTER is not None:
Todd Fialae83f1402015-09-18 22:45:31 +0000910 forwarding_func = RESULTS_FORMATTER.handle_event
Todd Fiala68615ce2015-09-15 21:38:04 +0000911 RESULTS_LISTENER_CHANNEL = (
912 dotest_channels.UnpicklingForwardingListenerChannel(
913 RUNNER_PROCESS_ASYNC_MAP, "localhost", 0, forwarding_func))
914 dotest_argv.append("--results-port")
915 dotest_argv.append(str(RESULTS_LISTENER_CHANNEL.address[1]))
Todd Fiala3f0a3602014-07-08 06:42:37 +0000916
917 # Collect the test files that we'll run.
918 test_work_items = []
Todd Fiala8cbeed32015-09-08 22:22:33 +0000919 find_test_files_in_dir_tree(
920 test_subdir, lambda testdir, test_files: test_work_items.append([
921 test_subdir, test_files, test_directory, dotest_argv, None]))
Todd Fiala3f0a3602014-07-08 06:42:37 +0000922
Todd Fiala8cbeed32015-09-08 22:22:33 +0000923 # Convert test work items into test results using whatever
924 # was provided as the test run function.
925 test_results = test_runner_func(test_work_items)
Chaoren Linffc63b02015-08-12 18:02:49 +0000926
Todd Fiala8cbeed32015-09-08 22:22:33 +0000927 # Summarize the results and return to caller.
Chaoren Line80372a2015-08-12 18:02:53 +0000928 timed_out = sum([result[0] for result in test_results], [])
929 passed = sum([result[1] for result in test_results], [])
930 failed = sum([result[2] for result in test_results], [])
Zachary Turner4cceca72015-08-14 16:45:32 +0000931 unexpected_successes = sum([result[3] for result in test_results], [])
932 pass_count = sum([result[4] for result in test_results])
933 fail_count = sum([result[5] for result in test_results])
Todd Fiala3f0a3602014-07-08 06:42:37 +0000934
Todd Fiala8cbeed32015-09-08 22:22:33 +0000935 return (timed_out, passed, failed, unexpected_successes, pass_count,
936 fail_count)
Johnny Chene8d9dc62011-10-31 19:04:07 +0000937
Chaoren Linb6325d02015-08-12 18:02:54 +0000938
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000939def getExpectedTimeouts(platform_name):
Vince Harron06381732015-05-12 23:10:36 +0000940 # returns a set of test filenames that might timeout
941 # are we running against a remote target?
Chaoren Linfebef1b2015-08-19 17:22:12 +0000942 host = sys.platform
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000943 if platform_name is None:
Vince Harron06381732015-05-12 23:10:36 +0000944 target = sys.platform
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000945 else:
Todd Fiala68615ce2015-09-15 21:38:04 +0000946 m = re.search(r'remote-(\w+)', platform_name)
Vince Harronf8b9a1d2015-05-18 19:40:54 +0000947 target = m.group(1)
Vince Harron06381732015-05-12 23:10:36 +0000948
949 expected_timeout = set()
950
951 if target.startswith("linux"):
952 expected_timeout |= {
Chaoren Lin0b8bb3d2015-07-22 20:52:17 +0000953 "TestProcessAttach.py",
Vince Harron06381732015-05-12 23:10:36 +0000954 "TestConnectRemote.py",
955 "TestCreateAfterAttach.py",
Tamas Berghammer0d0ec9f2015-05-19 10:49:40 +0000956 "TestEvents.py",
Vince Harron06381732015-05-12 23:10:36 +0000957 "TestExitDuringStep.py",
Chaoren Linb6325d02015-08-12 18:02:54 +0000958
959 # Times out in ~10% of the times on the build bot
960 "TestHelloWorld.py",
Oleksiy Vyalov18f4c9f2015-06-10 01:34:25 +0000961 "TestMultithreaded.py",
Chaoren Linb6325d02015-08-12 18:02:54 +0000962 "TestRegisters.py", # ~12/600 dosep runs (build 3120-3122)
Vince Harron06381732015-05-12 23:10:36 +0000963 "TestThreadStepOut.py",
964 }
965 elif target.startswith("android"):
966 expected_timeout |= {
967 "TestExitDuringStep.py",
968 "TestHelloWorld.py",
969 }
Chaoren Linfebef1b2015-08-19 17:22:12 +0000970 if host.startswith("win32"):
971 expected_timeout |= {
972 "TestEvents.py",
973 "TestThreadStates.py",
974 }
Ed Maste4dd8fba2015-05-14 16:25:52 +0000975 elif target.startswith("freebsd"):
976 expected_timeout |= {
977 "TestBreakpointConditions.py",
Ed Maste08948132015-05-28 18:45:30 +0000978 "TestChangeProcessGroup.py",
Ed Mastebfd05632015-05-27 19:11:29 +0000979 "TestValueObjectRecursion.py",
Ed Maste4dd8fba2015-05-14 16:25:52 +0000980 "TestWatchpointConditionAPI.py",
981 }
Vince Harron0f173ac2015-05-18 19:36:33 +0000982 elif target.startswith("darwin"):
983 expected_timeout |= {
Chaoren Linb6325d02015-08-12 18:02:54 +0000984 # times out on MBP Retina, Mid 2012
985 "TestThreadSpecificBreakpoint.py",
Chaoren Lind9043712015-08-19 17:13:02 +0000986 "TestExitDuringStep.py",
Chaoren Lin99f25be2015-08-20 01:26:57 +0000987 "TestIntegerTypesExpr.py",
Vince Harron0f173ac2015-05-18 19:36:33 +0000988 }
Vince Harron06381732015-05-12 23:10:36 +0000989 return expected_timeout
990
Chaoren Linb6325d02015-08-12 18:02:54 +0000991
Pavel Labathfad30cf2015-06-29 14:16:51 +0000992def getDefaultTimeout(platform_name):
993 if os.getenv("LLDB_TEST_TIMEOUT"):
994 return os.getenv("LLDB_TEST_TIMEOUT")
995
996 if platform_name is None:
997 platform_name = sys.platform
998
999 if platform_name.startswith("remote-"):
1000 return "10m"
1001 else:
1002 return "4m"
1003
Chaoren Linb6325d02015-08-12 18:02:54 +00001004
Vince Harron0b9dbb52015-05-21 18:18:52 +00001005def touch(fname, times=None):
Greg Clayton8c3f9c92015-08-11 21:01:32 +00001006 if os.path.exists(fname):
Vince Harron0b9dbb52015-05-21 18:18:52 +00001007 os.utime(fname, times)
1008
Chaoren Linb6325d02015-08-12 18:02:54 +00001009
Vince Harrondcc2b9f2015-05-27 04:40:36 +00001010def find(pattern, path):
1011 result = []
1012 for root, dirs, files in os.walk(path):
1013 for name in files:
1014 if fnmatch.fnmatch(name, pattern):
1015 result.append(os.path.join(root, name))
1016 return result
1017
Chaoren Linb6325d02015-08-12 18:02:54 +00001018
Todd Fiala8cbeed32015-09-08 22:22:33 +00001019def get_test_runner_strategies(num_threads):
1020 """Returns the test runner strategies by name in a dictionary.
1021
1022 @param num_threads specifies the number of threads/processes
1023 that will be used for concurrent test runners.
1024
1025 @return dictionary with key as test runner strategy name and
1026 value set to a callable object that takes the test work item
1027 and returns a test result tuple.
1028 """
1029 return {
1030 # multiprocessing supports ctrl-c and does not use
1031 # multiprocessing.Pool.
1032 "multiprocessing":
1033 (lambda work_items: multiprocessing_test_runner(
1034 num_threads, work_items)),
1035
1036 # multiprocessing-pool uses multiprocessing.Pool but
1037 # does not support Ctrl-C.
1038 "multiprocessing-pool":
1039 (lambda work_items: multiprocessing_test_runner_pool(
1040 num_threads, work_items)),
1041
1042 # threading uses a hand-rolled worker pool much
1043 # like multiprocessing, but instead uses in-process
1044 # worker threads. This one supports Ctrl-C.
1045 "threading":
1046 (lambda work_items: threading_test_runner(num_threads, work_items)),
1047
1048 # threading-pool uses threading for the workers (in-process)
1049 # and uses the multiprocessing.pool thread-enabled pool.
Todd Fiala68615ce2015-09-15 21:38:04 +00001050 # This does not properly support Ctrl-C.
Todd Fiala8cbeed32015-09-08 22:22:33 +00001051 "threading-pool":
1052 (lambda work_items: threading_test_runner_pool(
1053 num_threads, work_items)),
1054
1055 # serial uses the subprocess-based, single process
1056 # test runner. This provides process isolation but
Todd Fiala68615ce2015-09-15 21:38:04 +00001057 # no concurrent test execution.
Todd Fiala8cbeed32015-09-08 22:22:33 +00001058 "serial":
1059 inprocess_exec_test_runner
1060 }
1061
1062
Todd Fiala68615ce2015-09-15 21:38:04 +00001063def _remove_option(args, option_name, removal_count):
1064 """Removes option and related option arguments from args array.
1065 @param args the array of command line arguments (in/out)
1066 @param option_name the full command line representation of the
1067 option that will be removed (including '--' or '-').
1068 @param the count of elements to remove. A value of 1 will remove
1069 just the found option, while 2 will remove the option and its first
1070 argument.
1071 """
1072 try:
1073 index = args.index(option_name)
1074 # Handle the exact match case.
1075 del args[index:index+removal_count]
1076 return
1077 except ValueError:
1078 # Thanks to argparse not handling options with known arguments
1079 # like other options parsing libraries (see
1080 # https://bugs.python.org/issue9334), we need to support the
1081 # --results-formatter-options={second-level-arguments} (note
1082 # the equal sign to fool the first-level arguments parser into
1083 # not treating the second-level arguments as first-level
1084 # options). We're certainly at risk of getting this wrong
1085 # since now we're forced into the business of trying to figure
1086 # out what is an argument (although I think this
1087 # implementation will suffice).
1088 regex_string = "^" + option_name + "="
1089 regex = re.compile(regex_string)
1090 for index in range(len(args)):
1091 match = regex.match(args[index])
1092 if match:
Todd Fiala68615ce2015-09-15 21:38:04 +00001093 del args[index]
1094 return
1095 print "failed to find regex '{}'".format(regex_string)
1096
1097 # We didn't find the option but we should have.
1098 raise Exception("failed to find option '{}' in args '{}'".format(
1099 option_name, args))
1100
1101
1102def adjust_inferior_options(dotest_argv):
1103 """Adjusts the commandline args array for inferiors.
1104
1105 This method adjusts the inferior dotest commandline options based
1106 on the parallel test runner's options. Some of the inferior options
1107 will need to change to properly handle aggregation functionality.
1108 """
1109 global dotest_options
1110
1111 # If we don't have a session directory, create one.
1112 if not dotest_options.s:
1113 # no session log directory, we need to add this to prevent
1114 # every dotest invocation from creating its own directory
1115 import datetime
1116 # The windows platforms don't like ':' in the pathname.
1117 timestamp_started = datetime.datetime.now().strftime("%F-%H_%M_%S")
1118 dotest_argv.append('-s')
1119 dotest_argv.append(timestamp_started)
1120 dotest_options.s = timestamp_started
1121
1122 # Adjust inferior results formatter options - if the parallel
1123 # test runner is collecting into the user-specified test results,
1124 # we'll have inferiors spawn with the --results-port option and
1125 # strip the original test runner options.
1126 if dotest_options.results_file is not None:
1127 _remove_option(dotest_argv, "--results-file", 2)
1128 if dotest_options.results_port is not None:
1129 _remove_option(dotest_argv, "--results-port", 2)
1130 if dotest_options.results_formatter is not None:
1131 _remove_option(dotest_argv, "--results-formatter", 2)
1132 if dotest_options.results_formatter_options is not None:
1133 _remove_option(dotest_argv, "--results-formatter-options", 2)
1134
Todd Fiala33896a92015-09-18 21:01:13 +00001135 # Remove test runner name if present.
1136 if dotest_options.test_runner_name is not None:
1137 _remove_option(dotest_argv, "--test-runner-name", 2)
1138
1139
Todd Fiala8cbeed32015-09-08 22:22:33 +00001140def main(print_details_on_success, num_threads, test_subdir,
Todd Fiala68615ce2015-09-15 21:38:04 +00001141 test_runner_name, results_formatter):
Todd Fialafed95662015-09-03 18:58:44 +00001142 """Run dotest.py in inferior mode in parallel.
1143
1144 @param print_details_on_success the parsed value of the output-on-success
1145 command line argument. When True, details of a successful dotest inferior
1146 are printed even when everything succeeds. The normal behavior is to
1147 not print any details when all the inferior tests pass.
1148
1149 @param num_threads the parsed value of the num-threads command line
1150 argument.
1151
1152 @param test_subdir optionally specifies a subdir to limit testing
1153 within. May be None if the entire test tree is to be used. This subdir
1154 is assumed to be relative to the lldb/test root of the test hierarchy.
Todd Fiala8cbeed32015-09-08 22:22:33 +00001155
1156 @param test_runner_name if specified, contains the test runner
1157 name which selects the strategy used to run the isolated and
1158 optionally concurrent test runner. Specify None to allow the
1159 system to choose the most appropriate test runner given desired
1160 thread count and OS type.
1161
Todd Fiala68615ce2015-09-15 21:38:04 +00001162 @param results_formatter if specified, provides the TestResultsFormatter
1163 instance that will format and output test result data from the
1164 side-channel test results. When specified, inferior dotest calls
1165 will send test results side-channel data over a socket to the parallel
1166 test runner, which will forward them on to results_formatter.
Todd Fialafed95662015-09-03 18:58:44 +00001167 """
1168
Todd Fiala1cc97b42015-09-21 05:42:26 +00001169 # Do not shut down on sighup.
1170 signal.signal(signal.SIGHUP, signal.SIG_IGN)
1171
Todd Fialafed95662015-09-03 18:58:44 +00001172 dotest_argv = sys.argv[1:]
1173
Greg Claytonb0d148e2015-09-21 17:25:01 +00001174 global output_on_success, RESULTS_FORMATTER
Todd Fialafed95662015-09-03 18:58:44 +00001175 output_on_success = print_details_on_success
Todd Fiala68615ce2015-09-15 21:38:04 +00001176 RESULTS_FORMATTER = results_formatter
Todd Fialafed95662015-09-03 18:58:44 +00001177
Vince Harrond5fa1022015-05-10 15:24:12 +00001178 # We can't use sys.path[0] to determine the script directory
1179 # because it doesn't work under a debugger
Vince Harron8994fed2015-05-22 19:49:23 +00001180 parser = dotest_args.create_parser()
Pavel Labath05ab2372015-07-06 15:57:52 +00001181 global dotest_options
Vince Harron8994fed2015-05-22 19:49:23 +00001182 dotest_options = dotest_args.parse_args(parser, dotest_argv)
1183
Todd Fiala68615ce2015-09-15 21:38:04 +00001184 adjust_inferior_options(dotest_argv)
Vince Harron0b9dbb52015-05-21 18:18:52 +00001185
1186 session_dir = os.path.join(os.getcwd(), dotest_options.s)
Ed Mastecec2a5b2014-11-21 02:41:25 +00001187
Vince Harrone06a7a82015-05-12 23:12:19 +00001188 # The root directory was specified on the command line
Todd Fiala68615ce2015-09-15 21:38:04 +00001189 test_directory = os.path.dirname(os.path.realpath(__file__))
Todd Fialafed95662015-09-03 18:58:44 +00001190 if test_subdir and len(test_subdir) > 0:
1191 test_subdir = os.path.join(test_directory, test_subdir)
Vince Harrone06a7a82015-05-12 23:12:19 +00001192 else:
Todd Fialafed95662015-09-03 18:58:44 +00001193 test_subdir = test_directory
Vince Harrone06a7a82015-05-12 23:12:19 +00001194
Vince Harrondcc2b9f2015-05-27 04:40:36 +00001195 # clean core files in test tree from previous runs (Linux)
1196 cores = find('core.*', test_subdir)
1197 for core in cores:
1198 os.unlink(core)
1199
Daniel Maleab42556f2013-04-19 18:32:53 +00001200 system_info = " ".join(platform.uname())
Todd Fiala8cbeed32015-09-08 22:22:33 +00001201
1202 # Figure out which testrunner strategy we'll use.
1203 runner_strategies_by_name = get_test_runner_strategies(num_threads)
1204
1205 # If the user didn't specify a test runner strategy, determine
1206 # the default now based on number of threads and OS type.
1207 if not test_runner_name:
1208 if num_threads == 1:
1209 # Use the serial runner.
1210 test_runner_name = "serial"
1211 elif os.name == "nt":
1212 # Currently the multiprocessing test runner with ctrl-c
1213 # support isn't running correctly on nt. Use the pool
1214 # support without ctrl-c.
1215 test_runner_name = "multiprocessing-pool"
1216 else:
1217 # For everyone else, use the ctrl-c-enabled
1218 # multiprocessing support.
1219 test_runner_name = "multiprocessing"
1220
1221 if test_runner_name not in runner_strategies_by_name:
1222 raise Exception("specified testrunner name '{}' unknown. "
1223 "Valid choices: {}".format(
1224 test_runner_name,
1225 runner_strategies_by_name.keys()))
1226 test_runner_func = runner_strategies_by_name[test_runner_name]
1227
1228 summary_results = walk_and_invoke(
1229 test_directory, test_subdir, dotest_argv, test_runner_func)
1230
1231 (timed_out, passed, failed, unexpected_successes, pass_count,
1232 fail_count) = summary_results
Zachary Turnerc7a7c8a2015-05-28 19:56:26 +00001233
Vince Harron17f429f2014-12-13 00:08:19 +00001234 timed_out = set(timed_out)
Chaoren Line80372a2015-08-12 18:02:53 +00001235 num_test_files = len(passed) + len(failed)
1236 num_test_cases = pass_count + fail_count
Daniel Maleab42556f2013-04-19 18:32:53 +00001237
Vince Harrondcc2b9f2015-05-27 04:40:36 +00001238 # move core files into session dir
1239 cores = find('core.*', test_subdir)
1240 for core in cores:
1241 dst = core.replace(test_directory, "")[1:]
1242 dst = dst.replace(os.path.sep, "-")
1243 os.rename(core, os.path.join(session_dir, dst))
1244
Vince Harron06381732015-05-12 23:10:36 +00001245 # remove expected timeouts from failures
Vince Harronf8b9a1d2015-05-18 19:40:54 +00001246 expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
Vince Harron06381732015-05-12 23:10:36 +00001247 for xtime in expected_timeout:
1248 if xtime in timed_out:
1249 timed_out.remove(xtime)
1250 failed.remove(xtime)
Vince Harron0b9dbb52015-05-21 18:18:52 +00001251 result = "ExpectedTimeout"
1252 elif xtime in passed:
1253 result = "UnexpectedCompletion"
1254 else:
1255 result = None # failed
1256
1257 if result:
1258 test_name = os.path.splitext(xtime)[0]
1259 touch(os.path.join(session_dir, "{}-{}".format(result, test_name)))
Vince Harron06381732015-05-12 23:10:36 +00001260
Chaoren Lin5e3ab2b2015-06-01 17:49:25 +00001261 print
Chaoren Lin5a59e462015-08-12 18:02:51 +00001262 sys.stdout.write("Ran %d test suites" % num_test_files)
1263 if num_test_files > 0:
1264 sys.stdout.write(" (%d failed) (%f%%)" % (
1265 len(failed), 100.0 * len(failed) / num_test_files))
1266 print
Chaoren Line80372a2015-08-12 18:02:53 +00001267 sys.stdout.write("Ran %d test cases" % num_test_cases)
1268 if num_test_cases > 0:
Chaoren Lin5a59e462015-08-12 18:02:51 +00001269 sys.stdout.write(" (%d failed) (%f%%)" % (
Chaoren Line80372a2015-08-12 18:02:53 +00001270 fail_count, 100.0 * fail_count / num_test_cases))
Chaoren Lin5a59e462015-08-12 18:02:51 +00001271 print
Zachary Turner4cceca72015-08-14 16:45:32 +00001272 exit_code = 0
1273
Daniel Maleacbaef262013-02-15 21:31:37 +00001274 if len(failed) > 0:
Shawn Best13491c42014-10-22 19:29:00 +00001275 failed.sort()
Ying Chen10ed1a92015-05-28 23:51:49 +00001276 print "Failing Tests (%d)" % len(failed)
Daniel Maleacbaef262013-02-15 21:31:37 +00001277 for f in failed:
Vince Harron17f429f2014-12-13 00:08:19 +00001278 print "%s: LLDB (suite) :: %s (%s)" % (
1279 "TIMEOUT" if f in timed_out else "FAIL", f, system_info
1280 )
Zachary Turner4cceca72015-08-14 16:45:32 +00001281 exit_code = 1
1282
1283 if len(unexpected_successes) > 0:
1284 unexpected_successes.sort()
1285 print "\nUnexpected Successes (%d)" % len(unexpected_successes)
1286 for u in unexpected_successes:
1287 print "UNEXPECTED SUCCESS: LLDB (suite) :: %s (%s)" % (u, system_info)
1288
1289 sys.exit(exit_code)
Johnny Chene8d9dc62011-10-31 19:04:07 +00001290
1291if __name__ == '__main__':
Todd Fialafed95662015-09-03 18:58:44 +00001292 sys.stderr.write(
1293 "error: dosep.py no longer supports being called directly. "
1294 "Please call dotest.py directly. The dosep.py-specific arguments "
1295 "have been added under the Parallel processing arguments.\n")
1296 sys.exit(128)