diff options
Diffstat (limited to 'tools/run_tests/run_performance_tests.py')
| -rwxr-xr-x | tools/run_tests/run_performance_tests.py | 125 |
1 files changed, 99 insertions, 26 deletions
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 5519666e84..14901caf07 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -61,11 +61,11 @@ class QpsWorkerJob: self._spec = spec self.language = language self.host_and_port = host_and_port - self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}) + self._job = jobset.Job(spec, newline_on_success=True, travis=True, add_env={}) def is_running(self): """Polls a job and returns True if given job is still running.""" - return self._job.state(jobset.NoCache()) == jobset._RUNNING + return self._job.state() == jobset._RUNNING def kill(self): return self._job.kill() @@ -73,7 +73,6 @@ class QpsWorkerJob: def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None): - # TODO: support more languages cmdline = language.worker_cmdline() + ['--driver_port=%s' % port] if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) @@ -89,7 +88,7 @@ def create_qpsworker_job(language, shortname=None, jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, - timeout_seconds=30*60) + timeout_seconds=2*60*60) return QpsWorkerJob(jobspec, language, host_and_port) @@ -131,6 +130,36 @@ def create_quit_jobspec(workers, remote_host=None): verbose_success=True) +def create_netperf_jobspec(server_host='localhost', client_host=None, + bq_result_table=None): + """Runs netperf benchmark.""" + cmd = 'NETPERF_SERVER_HOST="%s" ' % server_host + if bq_result_table: + cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table + if client_host: + # If netperf is running remotely, the env variables populated by Jenkins + # won't be available on the client, but we need them for uploading results + # to BigQuery. + jenkins_job_name = os.getenv('JOB_NAME') + if jenkins_job_name: + cmd += 'JOB_NAME="%s" ' % jenkins_job_name + jenkins_build_number = os.getenv('BUILD_NUMBER') + if jenkins_build_number: + cmd += 'BUILD_NUMBER="%s" ' % jenkins_build_number + + cmd += 'tools/run_tests/performance/run_netperf.sh' + if client_host: + user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, client_host) + cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd)) + + return jobset.JobSpec( + cmdline=[cmd], + shortname='netperf', + timeout_seconds=60, + shell=True, + verbose_success=True) + + def archive_repo(languages): """Archives local version of repo including submodules.""" cmdline=['tar', '-cf', '../grpc.tar', '../grpc/'] @@ -244,34 +273,65 @@ def start_qpsworkers(languages, worker_hosts): def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', - bq_result_table=None): + category='all', bq_result_table=None, + netperf=False, netperf_hosts=[]): """Create jobspecs for scenarios to run.""" all_workers = [worker for workers in workers_by_lang.values() for worker in workers] scenarios = [] + + if netperf: + if not netperf_hosts: + netperf_server='localhost' + netperf_client=None + elif len(netperf_hosts) == 1: + netperf_server=netperf_hosts[0] + netperf_client=netperf_hosts[0] + else: + netperf_server=netperf_hosts[0] + netperf_client=netperf_hosts[1] + scenarios.append(create_netperf_jobspec(server_host=netperf_server, + client_host=netperf_client, + bq_result_table=bq_result_table)) + for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json['name']): - workers = workers_by_lang[str(language)] - # 'SERVER_LANGUAGE' is an indicator for this script to pick - # a server in different language. It doesn't belong to the Scenario - # schema, so we also need to remove it. - custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None) - if custom_server_lang: - if not workers_by_lang.get(custom_server_lang, []): - print 'Warning: Skipping scenario %s as' % scenario_json['name'] - print('SERVER_LANGUAGE is set to %s yet the language has ' - 'not been selected with -l' % custom_server_lang) - continue - for idx in range(0, scenario_json['num_servers']): - # replace first X workers by workers of a different language - workers[idx] = workers_by_lang[custom_server_lang][idx] - scenario = create_scenario_jobspec(scenario_json, - workers, - remote_host=remote_host, - bq_result_table=bq_result_table) - scenarios.append(scenario) + if category in scenario_json.get('CATEGORIES', []) or category == 'all': + workers = workers_by_lang[str(language)] + # 'SERVER_LANGUAGE' is an indicator for this script to pick + # a server in different language. + custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) + custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', None) + scenario_json = scenario_config.remove_nonproto_fields(scenario_json) + if custom_server_lang and custom_client_lang: + raise Exception('Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE' + 'in the same scenario') + if custom_server_lang: + if not workers_by_lang.get(custom_server_lang, []): + print 'Warning: Skipping scenario %s as' % scenario_json['name'] + print('SERVER_LANGUAGE is set to %s yet the language has ' + 'not been selected with -l' % custom_server_lang) + continue + for idx in range(0, scenario_json['num_servers']): + # replace first X workers by workers of a different language + workers[idx] = workers_by_lang[custom_server_lang][idx] + if custom_client_lang: + if not workers_by_lang.get(custom_client_lang, []): + print 'Warning: Skipping scenario %s as' % scenario_json['name'] + print('CLIENT_LANGUAGE is set to %s yet the language has ' + 'not been selected with -l' % custom_client_lang) + continue + for idx in range(scenario_json['num_servers'], len(workers)): + # replace all client workers by workers of a different language, + # leave num_server workers as they are server workers. + workers[idx] = workers_by_lang[custom_client_lang][idx] + scenario = create_scenario_jobspec(scenario_json, + workers, + remote_host=remote_host, + bq_result_table=bq_result_table) + scenarios.append(scenario) # the very last scenario requests shutting down the workers. scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) @@ -298,7 +358,7 @@ argp = argparse.ArgumentParser(description='Run performance tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(scenario_config.LANGUAGES.keys()), nargs='+', - default=['all'], + required=True, help='Languages to benchmark.') argp.add_argument('--remote_driver_host', default=None, @@ -311,6 +371,15 @@ argp.add_argument('-r', '--regex', default='.*', type=str, help='Regex to select scenarios to run.') argp.add_argument('--bq_result_table', default=None, type=str, help='Bigquery "dataset.table" to upload results to.') +argp.add_argument('--category', + choices=['smoketest','all','scalable'], + default='all', + help='Select a category of tests to run.') +argp.add_argument('--netperf', + default=False, + action='store_const', + const=True, + help='Run netperf benchmark as one of the scenarios.') args = argp.parse_args() @@ -354,7 +423,11 @@ try: workers_by_lang=worker_addresses, remote_host=args.remote_driver_host, regex=args.regex, - bq_result_table=args.bq_result_table) + category=args.category, + bq_result_table=args.bq_result_table, + netperf=args.netperf, + netperf_hosts=args.remote_worker_host) + if not scenarios: raise Exception('No scenarios to run') |