diff options
Diffstat (limited to 'tools/run_tests/run_performance_tests.py')
| -rwxr-xr-x | tools/run_tests/run_performance_tests.py | 82 |
1 files changed, 63 insertions, 19 deletions
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 51ed35f760..ada341abf5 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -37,10 +37,12 @@ import json import multiprocessing import os import pipes +import re import subprocess import sys import tempfile import time +import traceback import uuid import performance.scenario_config as scenario_config @@ -82,21 +84,28 @@ def create_qpsworker_job(language, shortname=None, else: host_and_port='localhost:%s' % port + # TODO(jtattermusch): with some care, we can calculate the right timeout + # of a worker from the sum of warmup + benchmark times for all the scenarios jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, - timeout_seconds=15*60) + timeout_seconds=30*60) return QpsWorkerJob(jobspec, language, host_and_port) -def create_scenario_jobspec(scenario_json, workers, remote_host=None): +def create_scenario_jobspec(scenario_json, workers, remote_host=None, + bq_result_table=None): """Runs one scenario using QPS driver.""" # setting QPS_WORKERS env variable here makes sure it works with SSH too. - cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver ' % ','.join(workers) - cmd += '--scenarios_json=%s' % pipes.quote(json.dumps({'scenarios': [scenario_json]})) + cmd = 'QPS_WORKERS="%s" ' % ','.join(workers) + if bq_result_table: + cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table + cmd += 'tools/run_tests/performance/run_qps_driver.sh ' + cmd += '--scenarios_json=%s ' % pipes.quote(json.dumps({'scenarios': [scenario_json]})) + cmd += '--scenario_result_file=scenario_result.json' if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) - cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd) + cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd)) return jobset.JobSpec( cmdline=[cmd], @@ -109,24 +118,29 @@ def create_scenario_jobspec(scenario_json, workers, remote_host=None): def create_quit_jobspec(workers, remote_host=None): """Runs quit using QPS driver.""" # setting QPS_WORKERS env variable here makes sure it works with SSH too. - cmd = 'QPS_WORKERS="%s" bins/opt/qps_driver --quit' % ','.join(workers) + cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver --quit' % ','.join(workers) if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) - cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd) + cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd)) return jobset.JobSpec( cmdline=[cmd], - shortname='qps_driver.quit', + shortname='qps_json_driver.quit', timeout_seconds=3*60, shell=True, verbose_success=True) -def archive_repo(): +def archive_repo(languages): """Archives local version of repo including submodules.""" - # TODO: also archive grpc-go and grpc-java repos + cmdline=['tar', '-cf', '../grpc.tar', '../grpc/'] + if 'java' in languages: + cmdline.append('../grpc-java') + if 'go' in languages: + cmdline.append('../grpc-go') + archive_job = jobset.JobSpec( - cmdline=['tar', '-cf', '../grpc.tar', '../grpc/'], + cmdline=cmdline, shortname='archive_repo', timeout_seconds=3*60) @@ -135,7 +149,7 @@ def archive_repo(): [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', - 'Archive with local repository create successfully.', + 'Archive with local repository created successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to archive local repository.', @@ -221,15 +235,32 @@ def start_qpsworkers(languages, worker_hosts): for worker_idx, worker in enumerate(workers)] -def create_scenarios(languages, workers_by_lang, remote_host=None): +def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', + bq_result_table=None): """Create jobspecs for scenarios to run.""" scenarios = [] for language in languages: for scenario_json in language.scenarios(): - scenario = create_scenario_jobspec(scenario_json, - workers_by_lang[str(language)], - remote_host=remote_host) - scenarios.append(scenario) + if re.search(args.regex, scenario_json['name']): + workers = workers_by_lang[str(language)] + # 'SERVER_LANGUAGE' is an indicator for this script to pick + # a server in different language. It doesn't belong to the Scenario + # schema, so we also need to remove it. + custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None) + if custom_server_lang: + if not workers_by_lang.get(custom_server_lang, []): + print 'Warning: Skipping scenario %s as' % scenario_json['name'] + print('SERVER_LANGUAGE is set to %s yet the language has ' + 'not been selected with -l' % custom_server_lang) + continue + for idx in range(0, scenario_json['num_servers']): + # replace first X workers by workers of a different language + workers[idx] = workers_by_lang[custom_server_lang][idx] + scenario = create_scenario_jobspec(scenario_json, + workers, + remote_host=remote_host, + bq_result_table=bq_result_table) + scenarios.append(scenario) # the very last scenario requests shutting down the workers. all_workers = [worker @@ -268,6 +299,10 @@ argp.add_argument('--remote_worker_host', nargs='+', default=[], help='Worker hosts where to start QPS workers.') +argp.add_argument('-r', '--regex', default='.*', type=str, + help='Regex to select scenarios to run.') +argp.add_argument('--bq_result_table', default=None, type=str, + help='Bigquery "dataset.table" to upload results to.') args = argp.parse_args() @@ -276,6 +311,7 @@ languages = set(scenario_config.LANGUAGES[l] scenario_config.LANGUAGES.iterkeys() if x == 'all' else [x] for x in args.language)) + # Put together set of remote hosts where to run and build remote_hosts = set() if args.remote_worker_host: @@ -285,7 +321,7 @@ if args.remote_driver_host: remote_hosts.add(args.remote_driver_host) if remote_hosts: - archive_repo() + archive_repo(languages=[str(l) for l in languages]) prepare_remote_hosts(remote_hosts) build_local = False @@ -295,6 +331,9 @@ build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build qpsworker_jobs = start_qpsworkers(languages, args.remote_worker_host) +# TODO(jtattermusch): see https://github.com/grpc/grpc/issues/6174 +time.sleep(5) + # get list of worker addresses for each language. worker_addresses = dict([(str(language), []) for language in languages]) for job in qpsworker_jobs: @@ -303,7 +342,9 @@ for job in qpsworker_jobs: try: scenarios = create_scenarios(languages, workers_by_lang=worker_addresses, - remote_host=args.remote_driver_host) + remote_host=args.remote_driver_host, + regex=args.regex, + bq_result_table=args.bq_result_table) if not scenarios: raise Exception('No scenarios to run') @@ -318,5 +359,8 @@ try: jobset.message('FAILED', 'Some of the scenarios failed.', do_newline=True) sys.exit(1) +except: + traceback.print_exc() + raise finally: finish_qps_workers(qpsworker_jobs) |