From 497966787a5955f03a4d7b2c0ee8d1948d7bd8e9 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Mon, 17 Oct 2016 10:01:37 -0700 Subject: fail performance tests if any jobs fail or timeout --- tools/run_tests/run_performance_tests.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'tools/run_tests/run_performance_tests.py') diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 92149115fe..5391ee6c92 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -353,10 +353,10 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', return scenarios - def finish_qps_workers(jobs): """Waits for given jobs to finish and eventually kills them.""" retries = 0 + num_killed = 0 while any(job.is_running() for job in jobs): for job in qpsworker_jobs: if job.is_running(): @@ -365,10 +365,11 @@ def finish_qps_workers(jobs): print('Killing all QPS workers.') for job in jobs: job.kill() + num_killed += 1 retries += 1 time.sleep(3) print('All QPS workers finished.') - + return num_killed argp = argparse.ArgumentParser(description='Run performance tests.') argp.add_argument('-l', '--language', @@ -450,6 +451,7 @@ scenarios = create_scenarios(languages, if not scenarios: raise Exception('No scenarios to run') +num_failures = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) @@ -457,8 +459,13 @@ for scenario in scenarios: try: for worker in scenario.workers: worker.start() - jobset.run([scenario.jobspec, - create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], - newline_on_success=True, maxjobs=1) + jobset_failures, _ = jobset.run([scenario.jobspec, + create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], + newline_on_success=True, maxjobs=1) + num_failures += jobset_failures finally: - finish_qps_workers(scenario.workers) + # Consider jobs that need to be killed as failures + num_failures += finish_qps_workers(scenario.workers) + +if num_failures > 0: + raise Exception('Failures occured') -- cgit v1.2.3