fail performance tests if any jobs fail or timeout
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 9214911..5391ee6 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -353,10 +353,10 @@
return scenarios
-
def finish_qps_workers(jobs):
"""Waits for given jobs to finish and eventually kills them."""
retries = 0
+ num_killed = 0
while any(job.is_running() for job in jobs):
for job in qpsworker_jobs:
if job.is_running():
@@ -365,10 +365,11 @@
print('Killing all QPS workers.')
for job in jobs:
job.kill()
+ num_killed += 1
retries += 1
time.sleep(3)
print('All QPS workers finished.')
-
+ return num_killed
argp = argparse.ArgumentParser(description='Run performance tests.')
argp.add_argument('-l', '--language',
@@ -450,6 +451,7 @@
if not scenarios:
raise Exception('No scenarios to run')
+num_failures = 0
for scenario in scenarios:
if args.dry_run:
print(scenario.name)
@@ -457,8 +459,13 @@
try:
for worker in scenario.workers:
worker.start()
- jobset.run([scenario.jobspec,
- create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
- newline_on_success=True, maxjobs=1)
+ jobset_failures, _ = jobset.run([scenario.jobspec,
+ create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
+ newline_on_success=True, maxjobs=1)
+ num_failures += jobset_failures
finally:
- finish_qps_workers(scenario.workers)
+ # Consider jobs that need to be killed as failures
+ num_failures += finish_qps_workers(scenario.workers)
+
+if num_failures > 0:
+ raise Exception('Failures occured')