fail performance tests if any jobs fail or timeout

1 files changed, 13 insertions, 6 deletions

diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 92149115fe..5391ee6c92 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -353,10 +353,10 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
 
   return scenarios
 
-
 def finish_qps_workers(jobs):
   """Waits for given jobs to finish and eventually kills them."""
   retries = 0
+  num_killed = 0
   while any(job.is_running() for job in jobs):
     for job in qpsworker_jobs:
       if job.is_running():
@@ -365,10 +365,11 @@ def finish_qps_workers(jobs):
       print('Killing all QPS workers.')
       for job in jobs:
         job.kill()
+        num_killed += 1
     retries += 1
     time.sleep(3)
   print('All QPS workers finished.')
-
+  return num_killed
 
 argp = argparse.ArgumentParser(description='Run performance tests.')
 argp.add_argument('-l', '--language',
@@ -450,6 +451,7 @@ scenarios = create_scenarios(languages,
 if not scenarios:
   raise Exception('No scenarios to run')
 
+num_failures = 0
 for scenario in scenarios:
   if args.dry_run:
     print(scenario.name)
@@ -457,8 +459,13 @@ for scenario in scenarios:
     try:
       for worker in scenario.workers:
         worker.start()
-      jobset.run([scenario.jobspec,
-                  create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
-                 newline_on_success=True, maxjobs=1)
+      jobset_failures, _ = jobset.run([scenario.jobspec,
+                                 create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
+                                 newline_on_success=True, maxjobs=1)
+      num_failures += jobset_failures
     finally:
-      finish_qps_workers(scenario.workers)
+      # Consider jobs that need to be killed as failures
+      num_failures += finish_qps_workers(scenario.workers)
+
+if num_failures > 0:
+  raise Exception('Failures occured')