diff options
Diffstat (limited to 'tools/run_tests')
27 files changed, 163 insertions, 2360 deletions
diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index f10916d192..731bb02d21 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -79,7 +79,10 @@ docker run \ -e HOST_GIT_ROOT=$git_root \ -e LOCAL_GIT_ROOT=$docker_instance_git_root \ -e "BUILD_ID=$BUILD_ID" \ + -e "BUILD_URL=$BUILD_URL" \ + -e "JOB_BASE_NAME=$JOB_BASE_NAME" \ -i $TTY_FLAG \ + -v ~/.config/gcloud:/root/.config/gcloud \ -v "$git_root:$docker_instance_git_root" \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ec8aad6995..a004e8d945 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -41438,7 +41438,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 13, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -42217,7 +42217,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 13, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -43074,7 +43074,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 13, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ @@ -44256,7 +44256,7 @@ { "args": [ "--scenarios_json", - "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 13, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], "boringssl": true, "ci_platforms": [ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 459120d647..8ed675ecc3 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -53,6 +53,7 @@ HISTOGRAM_PARAMS = { # actual target will be slightly higher) OUTSTANDING_REQUESTS={ 'async': 6400, + 'async-1core': 800, 'sync': 1000 } @@ -265,7 +266,7 @@ class CXXLanguage: rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, + unconstrained_client='async-1core', use_generic_payload=True, async_server_threads=1, secure=secure) @@ -752,7 +753,7 @@ class JavaLanguage: yield _ping_pong_scenario( 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, + unconstrained_client='async-1core', use_generic_payload=True, async_server_threads=1, secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index d01e82a76b..4e97128d46 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -222,7 +222,8 @@ class JobResult(object): self.num_failures = 0 self.retries = 0 self.message = '' - + self.cpu_estimated = 1 + self.cpu_measured = 0 class Job(object): """Manages one job.""" @@ -312,7 +313,9 @@ class Job(object): sys = float(m.group(3)) if real > 0.5: cores = (user + sys) / real - measurement = '; cpu_cost=%.01f; estimated=%.01f' % (cores, self._spec.cpu_cost) + self.result.cpu_measured = float('%.01f' % cores) + self.result.cpu_estimated = float('%.01f' % self._spec.cpu_cost) + measurement = '; cpu_cost=%.01f; estimated=%.01f' % (self.result.cpu_measured, self.result.cpu_estimated) if not self._quiet_success: message('PASSED', '%s [time=%.1fsec; retries=%d:%d%s]' % ( self._spec.shortname, elapsed, self._retries, self._timeout_retries, measurement), diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 502efc31f4..b8dd67e060 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -89,6 +89,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc', tree = ET.ElementTree(root) tree.write(xml_report, encoding='UTF-8') + def render_interop_html_report( client_langs, server_langs, test_cases, auth_test_cases, http2_cases, http2_server_cases, resultset, diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py new file mode 100644 index 0000000000..d076d1e5a2 --- /dev/null +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Helper to upload Jenkins test results to BQ""" + +from __future__ import print_function + +import os +import six +import sys +import time +import uuid + +gcp_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../gcp/utils')) +sys.path.append(gcp_utils_dir) +import big_query_utils + +_DATASET_ID = 'jenkins_test_results' +_DESCRIPTION = 'Test results from master job run on Jenkins' +_PROJECT_ID = 'grpc-testing' +_RESULTS_SCHEMA = [ + ('job_name', 'STRING', 'Name of Jenkins job'), + ('build_id', 'INTEGER', 'Build ID of Jenkins job'), + ('build_url', 'STRING', 'URL of Jenkins job'), + ('test_name', 'STRING', 'Individual test name'), + ('language', 'STRING', 'Language of test'), + ('platform', 'STRING', 'Platform used for test'), + ('config', 'STRING', 'Config used for test'), + ('compiler', 'STRING', 'Compiler used for test'), + ('iomgr_platform', 'STRING', 'Iomgr used for test'), + ('result', 'STRING', 'Test result: PASSED, TIMEOUT, FAILED, or SKIPPED'), + ('timestamp', 'TIMESTAMP', 'Timestamp of test run'), + ('elapsed_time', 'FLOAT', 'How long test took to run'), + ('cpu_estimated', 'FLOAT', 'Estimated CPU usage of test'), + ('cpu_measured', 'FLOAT', 'Actual CPU usage of test'), +] + + +def _get_build_metadata(test_results): + """Add Jenkins build metadata to test_results based on environment variables set by Jenkins.""" + build_id = os.getenv('BUILD_ID') + build_url = os.getenv('BUILD_URL') + job_name = os.getenv('JOB_BASE_NAME') + + if build_id: + test_results['build_id'] = build_id + if build_url: + test_results['build_url'] = build_url + if job_name: + test_results['job_name'] = job_name + +def upload_results_to_bq(resultset, bq_table, args, platform): + """Upload test results to a BQ table. + + Args: + resultset: dictionary generated by jobset.run + bq_table: string name of table to create/upload results to in BQ + args: args in run_tests.py, generated by argparse + platform: string name of platform tests were run on + """ + bq = big_query_utils.create_big_query() + big_query_utils.create_table(bq, _PROJECT_ID, _DATASET_ID, bq_table, _RESULTS_SCHEMA, _DESCRIPTION) + + for shortname, results in six.iteritems(resultset): + for result in results: + test_results = {} + _get_build_metadata(test_results) + test_results['compiler'] = args.compiler + test_results['config'] = args.config + test_results['cpu_estimated'] = result.cpu_estimated + test_results['cpu_measured'] = result.cpu_measured + test_results['elapsed_time'] = '%.2f' % result.elapsed_time + test_results['iomgr_platform'] = args.iomgr_platform + # args.language is a list, but will always have one element in the contexts + # this function is used. + test_results['language'] = args.language[0] + test_results['platform'] = platform + test_results['result'] = result.state + test_results['test_name'] = shortname + test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') + + row = big_query_utils.make_row(str(uuid.uuid4()), test_results) + if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, bq_table, [row]): + print('Error uploading result to bigquery.') + sys.exit(1) diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py deleted file mode 100755 index 4eea02118e..0000000000 --- a/tools/run_tests/run_stress_tests.py +++ /dev/null @@ -1,331 +0,0 @@ -#!/usr/bin/env python -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Run stress test in C++""" - -from __future__ import print_function - -import argparse -import atexit -import itertools -import json -import multiprocessing -import os -import re -import subprocess -import sys -import tempfile -import time -import uuid -import six - -import python_utils.dockerjob as dockerjob -import python_utils.jobset as jobset - -# Docker doesn't clean up after itself, so we do it on exit. -atexit.register(lambda: subprocess.call(['stty', 'echo'])) - -ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) -os.chdir(ROOT) - -_DEFAULT_SERVER_PORT = 8080 -_DEFAULT_METRICS_PORT = 8081 -_DEFAULT_TEST_CASES = 'empty_unary:20,large_unary:20,client_streaming:20,server_streaming:20,empty_stream:20' -_DEFAULT_NUM_CHANNELS_PER_SERVER = 5 -_DEFAULT_NUM_STUBS_PER_CHANNEL = 10 - -# 15 mins default -_DEFAULT_TEST_DURATION_SECS = 900 - -class CXXLanguage: - - def __init__(self): - self.client_cwd = None - self.server_cwd = None - self.safename = 'cxx' - - def client_cmd(self, args): - return ['bins/opt/stress_test'] + args - - def server_cmd(self, args): - return ['bins/opt/interop_server'] + args - - def global_env(self): - return {} - - def __str__(self): - return 'c++' - - -_LANGUAGES = {'c++': CXXLanguage(),} - -# languages supported as cloud_to_cloud servers -_SERVERS = ['c++'] - -DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' - - -def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): - """Wraps given cmdline array to create 'docker run' cmdline from it.""" - docker_cmdline = ['docker', 'run', '-i', '--rm=true'] - - # turn environ into -e docker args - if environ: - for k, v in environ.items(): - docker_cmdline += ['-e', '%s=%s' % (k, v)] - - # set working directory - workdir = DOCKER_WORKDIR_ROOT - if cwd: - workdir = os.path.join(workdir, cwd) - docker_cmdline += ['-w', workdir] - - docker_cmdline += docker_args + [image] + cmdline - return docker_cmdline - - -def bash_login_cmdline(cmdline): - """Creates bash -l -c cmdline from args list.""" - # Use login shell: - # * rvm and nvm require it - # * makes error messages clearer if executables are missing - return ['bash', '-l', '-c', ' '.join(cmdline)] - - -def _job_kill_handler(job): - if job._spec.container_name: - dockerjob.docker_kill(job._spec.container_name) - # When the job times out and we decide to kill it, - # we need to wait a before restarting the job - # to prevent "container name already in use" error. - # TODO(jtattermusch): figure out a cleaner way to to this. - time.sleep(2) - - -def cloud_to_cloud_jobspec(language, - test_cases, - server_addresses, - test_duration_secs, - num_channels_per_server, - num_stubs_per_channel, - metrics_port, - docker_image=None): - """Creates jobspec for cloud-to-cloud interop test""" - cmdline = bash_login_cmdline(language.client_cmd([ - '--test_cases=%s' % test_cases, '--server_addresses=%s' % - server_addresses, '--test_duration_secs=%s' % test_duration_secs, - '--num_stubs_per_channel=%s' % num_stubs_per_channel, - '--num_channels_per_server=%s' % num_channels_per_server, - '--metrics_port=%s' % metrics_port - ])) - print(cmdline) - cwd = language.client_cwd - environ = language.global_env() - if docker_image: - container_name = dockerjob.random_name('interop_client_%s' % - language.safename) - cmdline = docker_run_cmdline( - cmdline, - image=docker_image, - environ=environ, - cwd=cwd, - docker_args=['--net=host', '--name', container_name]) - cwd = None - - test_job = jobset.JobSpec(cmdline=cmdline, - cwd=cwd, - environ=environ, - shortname='cloud_to_cloud:%s:%s_server:stress_test' % ( - language, server_name), - timeout_seconds=test_duration_secs * 2, - flake_retries=0, - timeout_retries=0, - kill_handler=_job_kill_handler) - test_job.container_name = container_name - return test_job - - -def server_jobspec(language, docker_image, test_duration_secs): - """Create jobspec for running a server""" - container_name = dockerjob.random_name('interop_server_%s' % - language.safename) - cmdline = bash_login_cmdline(language.server_cmd(['--port=%s' % - _DEFAULT_SERVER_PORT])) - environ = language.global_env() - docker_cmdline = docker_run_cmdline( - cmdline, - image=docker_image, - cwd=language.server_cwd, - environ=environ, - docker_args=['-p', str(_DEFAULT_SERVER_PORT), '--name', container_name]) - - server_job = jobset.JobSpec(cmdline=docker_cmdline, - environ=environ, - shortname='interop_server_%s' % language, - timeout_seconds=test_duration_secs * 3) - server_job.container_name = container_name - return server_job - - -def build_interop_stress_image_jobspec(language, tag=None): - """Creates jobspec for building stress test docker image for a language""" - if not tag: - tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4()) - env = {'INTEROP_IMAGE': tag, - 'BASE_NAME': 'grpc_interop_stress_%s' % language.safename} - build_job = jobset.JobSpec(cmdline=['tools/run_tests/dockerize/build_interop_stress_image.sh'], - environ=env, - shortname='build_docker_%s' % (language), - timeout_seconds=30 * 60) - build_job.tag = tag - return build_job - -argp = argparse.ArgumentParser(description='Run stress tests.') -argp.add_argument('-l', - '--language', - choices=['all'] + sorted(_LANGUAGES), - nargs='+', - default=['all'], - help='Clients to run.') -argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) -argp.add_argument( - '-s', - '--server', - choices=['all'] + sorted(_SERVERS), - action='append', - help='Run cloud_to_cloud servers in a separate docker ' + 'image.', - default=[]) -argp.add_argument( - '--override_server', - action='append', - type=lambda kv: kv.split('='), - help= - 'Use servername=HOST:PORT to explicitly specify a server. E.g. ' - 'csharp=localhost:50000', - default=[]) -argp.add_argument('--test_duration_secs', - help='The duration of the test in seconds', - default=_DEFAULT_TEST_DURATION_SECS) - -args = argp.parse_args() - -servers = set( - s - for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] - for x in args.server)) - -languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( - six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) - -docker_images = {} -# languages for which to build docker images -languages_to_build = set( - _LANGUAGES[k] - for k in set([str(l) for l in languages] + [s for s in servers])) -build_jobs = [] -for l in languages_to_build: - job = build_interop_stress_image_jobspec(l) - docker_images[str(l)] = job.tag - build_jobs.append(job) - -if build_jobs: - jobset.message('START', 'Building interop docker images.', do_newline=True) - num_failures, _ = jobset.run(build_jobs, - newline_on_success=True, - maxjobs=args.jobs) - if num_failures == 0: - jobset.message('SUCCESS', - 'All docker images built successfully.', - do_newline=True) - else: - jobset.message('FAILED', - 'Failed to build interop docker images.', - do_newline=True) - for image in six.itervalues(docker_images): - dockerjob.remove_image(image, skip_nonexistent=True) - sys.exit(1) - -# Start interop servers. -server_jobs = {} -server_addresses = {} -try: - for s in servers: - lang = str(s) - spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), args.test_duration_secs) - job = dockerjob.DockerJob(spec) - server_jobs[lang] = job - server_addresses[lang] = ('localhost', - job.mapped_port(_DEFAULT_SERVER_PORT)) - - jobs = [] - - for server in args.override_server: - server_name = server[0] - (server_host, server_port) = server[1].split(':') - server_addresses[server_name] = (server_host, server_port) - - for server_name, server_address in server_addresses.items(): - (server_host, server_port) = server_address - for language in languages: - test_job = cloud_to_cloud_jobspec( - language, - _DEFAULT_TEST_CASES, - ('%s:%s' % (server_host, server_port)), - args.test_duration_secs, - _DEFAULT_NUM_CHANNELS_PER_SERVER, - _DEFAULT_NUM_STUBS_PER_CHANNEL, - _DEFAULT_METRICS_PORT, - docker_image=docker_images.get(str(language))) - jobs.append(test_job) - - if not jobs: - print('No jobs to run.') - for image in six.itervalues(docker_images): - dockerjob.remove_image(image, skip_nonexistent=True) - sys.exit(1) - - num_failures, resultset = jobset.run(jobs, - newline_on_success=True, - maxjobs=args.jobs) - if num_failures: - jobset.message('FAILED', 'Some tests failed', do_newline=True) - else: - jobset.message('SUCCESS', 'All tests passed', do_newline=True) - -finally: - # Check if servers are still running. - for server, job in server_jobs.items(): - if not job.is_running(): - print('Server "%s" has exited prematurely.' % server) - - dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) - - for image in six.itervalues(docker_images): - print('Removing docker image %s' % image) - dockerjob.remove_image(image) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 62e92c3d1d..22891c5c98 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -60,7 +60,10 @@ import python_utils.jobset as jobset import python_utils.report_utils as report_utils import python_utils.watch_dirs as watch_dirs import python_utils.start_port_server as start_port_server - +try: + from python_utils.upload_test_results import upload_results_to_bq +except (ImportError): + pass # It's ok to not import because this is only necessary to upload results to BQ. _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) @@ -1185,7 +1188,7 @@ argp.add_argument('--build_only', default=False, action='store_const', const=True, - help='Perform all the build steps but dont run any tests.') + help='Perform all the build steps but don\'t run any tests.') argp.add_argument('--measure_cpu_costs', default=False, action='store_const', const=True, help='Measure the cpu costs of tests') argp.add_argument('--update_submodules', default=[], nargs='*', @@ -1200,11 +1203,16 @@ argp.add_argument('--quiet_success', default=False, action='store_const', const=True, - help='Dont print anything when a test passes. Passing tests also will not be reported in XML report. ' + + help='Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. ' + 'Useful when running many iterations of each test (argument -n).') argp.add_argument('--force_default_poller', default=False, action='store_const', const=True, - help='Dont try to iterate over many polling strategies when they exist') + help='Don\'t try to iterate over many polling strategies when they exist') argp.add_argument('--max_time', default=-1, type=int, help='Maximum test runtime in seconds') +argp.add_argument('--bq_result_table', + default='', + type=str, + nargs='?', + help='Upload test results to a specified BQ table.') args = argp.parse_args() if args.force_default_poller: @@ -1503,6 +1511,8 @@ def _build_and_run( finally: for antagonist in antagonists: antagonist.kill() + if args.bq_result_table and resultset: + upload_results_to_bq(resultset, args.bq_result_table, args, platform_string()) if xml_report and resultset: report_utils.render_junit_xml_report(resultset, xml_report, suite_name=args.report_suite_name) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 474b56b5de..884d626469 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -208,7 +208,7 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) extra_args=extra_args, inner_jobs=inner_jobs) - for compiler in ['gcc4.8', 'gcc5.3', + for compiler in ['gcc4.8', 'gcc5.3', 'gcc_musl', 'clang3.5', 'clang3.6', 'clang3.7']: test_jobs += _generate_jobs(languages=['c++'], configs=['dbg'], @@ -267,6 +267,15 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) extra_args=extra_args, inner_jobs=inner_jobs) + test_jobs += _generate_jobs(languages=['python'], + configs=['dbg'], + platforms=['linux'], + arch='default', + compiler='python_alpine', + labels=['portability'], + extra_args=extra_args, + inner_jobs=inner_jobs) + test_jobs += _generate_jobs(languages=['csharp'], configs=['dbg'], platforms=['linux'], @@ -387,6 +396,11 @@ if __name__ == "__main__": const=True, help='Put reports into subdirectories to improve presentation of ' 'results by Internal CI.') + argp.add_argument('--bq_result_table', + default='', + type=str, + nargs='?', + help='Upload test results to a specified BQ table.') args = argp.parse_args() if args.internal_ci: @@ -403,6 +417,10 @@ if __name__ == "__main__": extra_args.append('--quiet_success') if args.max_time > 0: extra_args.extend(('--max_time', '%d' % args.max_time)) + if args.bq_result_table: + extra_args.append('--bq_result_table') + extra_args.append('%s' % args.bq_result_table) + extra_args.append('--measure_cpu_costs') all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \ _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 6be7a39d07..43d05212e8 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -46,7 +46,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7) 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0) ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) - 593e917c176b5bc5aafa57bf9f6030d749d91cd5 third_party/protobuf (v3.1.0-alpha-1-326-g593e917) + a6189acd18b00611c1dc7042299ad75486f08a1a third_party/protobuf (v3.3.0) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) 7691f773af79bf75a62d1863fd0f13ebf9dc51b1 third_party/cares/cares (1.12.0) EOF diff --git a/tools/run_tests/stress_test/README.md b/tools/run_tests/stress_test/README.md deleted file mode 100644 index cc82b875cd..0000000000 --- a/tools/run_tests/stress_test/README.md +++ /dev/null @@ -1,76 +0,0 @@ -Running Stress tests on Google Container Engine
-=======================================
-
-### **Glossary**:
-* GCP: Google Cloud Platform
-* GCE: Google Compute Engine
-* GKE: Google Container Engine
-* GCP console: https://console.cloud.google.com
-
-### **Setup Instructions**
-#### *On GCP:*
-1. Login to GCP with your Google account (for example, your @gmail account) at https://cloud.google.com. If do not have a Google account, you will have to create an account first.
-2. Enable billing on Google cloud platform. Instructions [here](https://cloud.google.com/container-engine/docs/before-you-begin) (see the '*Enable billing*' section).
-3. Create a Project from the [GCP console](https://console.cloud.google.com).i.e Click on the project dropdown box on the top right (to the right of the search box) and click '*Create a project*' option.
-4. Enable the Container Engine API. Instructions [here](https://cloud.google.com/container-engine/docs/before-you-begin) (See the '*Enable the Container Engine API*’ section). Alternatively, you can do the following:
- - Click on the '*Products & Services*' icon on the top left (i.e the icon with three small horizontal bars) and select '*API Manager*'
- - Select the '*Container Engine API*' under '*Google Cloud APIs*' on the main page. Note that you might have to click on '*More*' under '*Google Cloud APIs*' to see the '*Container Engine API*' link
- - Click on the '*Enable*' button. If the API is already enabled, the button's label would be '*Disable*' instead (do NOT click the button if its label is '*Disable*')
-5. Create a Cluster from the GCP console.
- - Go to the Container Engine section from GCP console i.e: Click on the '*Products & Services*' icon on the top left (i.e the icon with three small horizontal bars) and click on '*Container Engine*'
- - Click '*Create Container Cluster*' and follow the instructions.
- - The instructions for 'Name/Zone/MachineType' etc are [here](https://cloud.google.com/container-engine/docs/clusters/operations) (**NOTE**: The page also has instructions to setting up default clusters and configuring `kubectl`. We will be doing that later)
- - For the cluster size, a smaller size of < 10 GCE instances is good enough for our use cases - assuming that we are planning to run a reasonably small number of stress client instances. For the machine type, something like '2 vCPUs 7.5 GB' (available in the drop down box) should be good enough.
- - **IMPORTANT**: Before hitting the '*Create*' button, click on '*More*' link just above the '*Create*' button and Select '*Enabled*' for BigQuery , '*Enabled*' for Cloud Platform and '*Read/Write*' for Cloud User Accounts.
- - Create the cluster by clicking '*Create*' button.
-
-#### *On your machine* (or the machine from which stress tests on GKE are launched):
-1. You need a working gRPC repository on your machine. If you do not have it, clone the grpc repository from github (https://github.com/grpc/grpc) and follow the instructions [here](https://github.com/grpc/grpc/blob/master/INSTALL.md)
-2. Install Docker. Instructions [here](https://docs.docker.com/engine/installation/)
-3. Install Google Cloud SDK. Instructions [here](https://cloud.google.com/sdk/). This installs the `gcloud` tool
-4. Install `kubectl`, Kubernetes command line tool using `gcloud`. i.e
- - `$ gcloud components update kubectl`
- - NOTE: If you are running this from a GCE instance, the command may fail with the following error:
- ```
- You cannot perform this action because this Cloud SDK installation is
- managed by an external package manager. If you would like to get the
- latest version, please see our main download page at:
-
- https://developers.google.com/cloud/sdk/
-
- ERROR: (gcloud.components.update) The component manager is disabled for this installation
- ```
- -- If so, you will have to manually install Cloud SDK by doing the following
- ```shell
- $ # The following installs latest Cloud SDK and updates the PATH
- $ # (Accept the default values when prompted)
- $ curl https://sdk.cloud.google.com | bash
- $ exec -l $SHELL
- $ # Set the defaults. Pick the default GCE credentials when prompted (The service account
- $ # name will have a name similar to: "xxx-compute@developer.gserviceaccount.com")
- $ gcloud init
- ```
-
-5. Install Google python client apis:
- - `‘$ sudo pip install --upgrade google-api-python-client’`
- - **Note**: Do `$ sudo apt-get install python-pip` (or `$ easy_install -U pip`) if you do not have pip
-6. Install the `requests` Python package if you don’t have it already by doing `sudo pip install requests`. More details regarding `requests` package are [here](http://docs.python-requests.org/en/master/user/install/)
-7. Set the `gcloud` defaults: See the instructions [here](https://cloud.google.com/container-engine/docs/before-you-begin) under "*Set gcloud defaults*" section)
- - Make sure you also fetch the cluster credentials for `kubectl` command to use. I.e `$ gcloud container clusters get-credentials CLUSTER_NAME`
-
-### **Launching Stress tests**
-
-The stress tests are launched by the following script (path is relative to GRPC root directory) :
-`tools/run_tests/stress_test/run_stress_tests_on_gke.py`
-
-You can find out more details by using the `--help` flag.
- - `<grpc_root_dir>$ tools/run_tests/stress_test/run_on_gke.py --help`
-
-> **Example**
-> ```bash
-> $ # Change to the grpc root directory
-> $ cd $GRPC_ROOT
-> $ tools/run_tests/stress_test/run_on_gke.py --project_id=sree-gce --config_file=tools/run_tests/stress_test/configs/opt.json
-> ```
-
-> The above runs the stress test on GKE under the project `sree-gce` in the default cluster (that you set by `gcloud` command earlier). The test settings (like number of client instances, servers, the parmeters to pass, test cases etc) are all loaded from the config file `$GRPC_ROOT/tools/run_tests/stress_test/opt.json`
diff --git a/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md b/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md deleted file mode 100644 index 9f079beebc..0000000000 --- a/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md +++ /dev/null @@ -1,25 +0,0 @@ -Stress Test client Specification
-=========================
-This document specifies the features a stress test client should implement in order to work with the stress testing framework. The stress test clients are executed against the existing interop test servers.
-
-**Requirements**
---------------
-**1.** A stress test client should be able to repeatedly execute one or more of the existing 'interop test cases'. It may just be a wrapper around the existing interop test client. The exact command line arguments the client should support are listed in _Table 1_ below.
-
-**2.** The stress test client must implement a metrics server defined by _[metrics.proto](https://github.com/grpc/grpc/blob/master/src/proto/grpc/testing/metrics.proto)_ and must expose _qps_ as a `Long`-valued Gauge. The client can track the overall _qps_ in one Gauge or in multiple Gauges (for example: One per Channel or Stub).
- The framework periodically queries the _qps_ by calling the `GetAllGauges()` method (the framework assumes that all the returned Gauges are _qps_ Gauges and adds them up to determine the final qps) and uses this to determine if the stress test client is running or crashed or stalled.
-> *Note:* In this context, the term _**qps**_ means _interop test cases per second_ (not _messages per second_ or _rpc calls per second_)
-
-
-**Table 1:** Command line arguments that should be supported by the stress test client.
-
->_**Note** The current C++ [stress client](https://github.com/grpc/grpc/blob/master/test/cpp/interop/stress_test.cc) supports more flags than those listed here but those flags will soon be deprecated._
-
-Parameter | Description
-----------------------|---------------------------------
-`--server_addresses` | The stress client should accept a list of server addresses in the following format:<br> ```<name_1>:<port_1>,<name_2>:<port_2>..<name_N>:<port_N>``` <br> _Note:_ `<name>` can be either server name or IP address.<br><br>_Type:_ string <br>_default:_ ```localhost:8080``` <br>_Example:_ ``foo.foobar.com:8080,bar.foobar.com:8080`` <br><br> Currently, the stress test framework only passes one server address to the client.
-`--test_cases` | List of test cases along with the relative weights in the following format:<br> `<testcase_1:w_1>,<testcase_2:w_2>...<testcase_n:w_n>`. <br> The test cases names are the same as those currently used by the interop clients<br><br>_Type:_ string <br>_Example:_ `empty_unary:20,large_unary:10,empty_stream:70` <br>(The stress client would then make `empty_unary` calls 20% of the time, `large_unary` calls 10% of the time and `empty_stream` calls 70% of the time.) <br>_Note:_ The weights need not add up to 100.
-`--test_duration_secs` | The test duration in seconds. A value of -1 means that the test should run forever until forcefully terminated. <br>_Type:_ int <br>_default:_ -1
-`--num_channels_per_server` | Number of channels (i.e connections) to each server. <br> _Type:_ int <br> _default:_ 1 <br><br> _Note:_ Unfortunately, the term `channel` is used differently in `grpc-java` and `C based grpc`. In this context, this really means "number of connections to the server"
-`--num_stubs_per_channel ` | Number of client stubs per each connection to server.<br>_Type:_ int <br>_default:_ 1
-`--metrics_port` | The port at which the stress client exposes [QPS metrics](https://github.com/grpc/grpc/blob/master/src/proto/grpc/testing/metrics.proto). <br>_Type:_ int <br>_default:_ 8081
diff --git a/tools/run_tests/stress_test/cleanup_docker_images.sh b/tools/run_tests/stress_test/cleanup_docker_images.sh deleted file mode 100755 index e424fcfd99..0000000000 --- a/tools/run_tests/stress_test/cleanup_docker_images.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -for img in `docker images | grep \<none\> | awk '{print $3 }'` ; do docker rmi -f $img; done - diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json deleted file mode 100644 index 7ae11ccbf1..0000000000 --- a/tools/run_tests/stress_test/configs/asan.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_cxx_asan" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "asan" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 120, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "cxx_client_asan": { - "baseTemplate": "default", - "stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"], - "metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "cxx_server_asan": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-asan": { - "serverTemplate": "cxx_server_asan", - "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-asan": { - "clientTemplate": "cxx_client_asan", - "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 5, - "serverPodSpec": "stress-server-asan" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8003, - "datasetIdNamePrefix": "stress_test_asan", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/csharp.json b/tools/run_tests/stress_test/configs/csharp.json deleted file mode 100644 index c438e08964..0000000000 --- a/tools/run_tests/stress_test/configs/csharp.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_csharp" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_csharp" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 100, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true", - "deadline_secs": 60 - } - } - }, - "templates": { - "csharp_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "mono", - "/var/local/git/grpc/src/csharp/Grpc.IntegrationTesting.StressClient/bin/Debug/Grpc.IntegrationTesting.StressClient.exe" - ], - "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "csharp_server": { - "baseTemplate": "default", - "stressServerCmd": [ - "mono", - "/var/local/git/grpc/src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/Grpc.IntegrationTesting.Server.exe" - ] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-csharp": { - "serverTemplate": "csharp_server", - "dockerImage": "grpc_stress_csharp", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-csharp": { - "clientTemplate": "csharp_client", - "dockerImage": "grpc_stress_csharp", - "numInstances": 10, - "serverPodSpec": "stress-server-csharp" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 100, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8009, - "datasetIdNamePrefix": "stress_test_csharp", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/go.json b/tools/run_tests/stress_test/configs/go.json deleted file mode 100644 index f1b2b523d3..0000000000 --- a/tools/run_tests/stress_test/configs/go.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_go" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_go" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "go_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "go", - "run", - "/go/src/google.golang.org/grpc/stress/client/main.go" - ], - "metricsClientCmd": [ - "go", - "run", - "/go/src/google.golang.org/grpc/stress/metrics_client/main.go" - ] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "go_server": { - "baseTemplate": "default", - "stressServerCmd": [ - "go", - "run", - "/go/src/google.golang.org/grpc/interop/server/server.go" - ] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "go-stress-server": { - "serverTemplate": "go_server", - "dockerImage": "grpc_stress_go", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "go-stress-client": { - "clientTemplate": "go_client", - "dockerImage": "grpc_stress_go", - "numInstances": 15, - "serverPodSpec": "go-stress-server" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8007, - "datasetIdNamePrefix": "stress_test_go", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/java.json b/tools/run_tests/stress_test/configs/java.json deleted file mode 100644 index 92af63c6b5..0000000000 --- a/tools/run_tests/stress_test/configs/java.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_java" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_java" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 100, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true", - "deadline_secs": 60 - }, - "env": { - "STRESSTEST_CLIENT_OPTS":"-Xmx3g -Xms3g -XX:NewSize=1500m -XX:MaxNewSize=1500m -XX:+UseConcMarkSweepGC" - } - } - }, - "templates": { - "java_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "/var/local/git/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/stresstest-client" - ], - "metricsClientCmd": [ - "/var/local/git/grpc/bins/opt/metrics_client" - ] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080, - "use_tls": "false" - }, - "env": { - "TEST_SERVER_OPTS":"-Xmx3g -Xms3g -XX:NewSize=1500m -XX:MaxNewSize=1500m -XX:+UseConcMarkSweepGC" - } - } - }, - "templates": { - "java_server": { - "baseTemplate": "default", - "stressServerCmd": [ - "/var/local/git/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/test-server" - ] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "java-stress-server": { - "serverTemplate": "java_server", - "dockerImage": "grpc_stress_java", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "java-stress-client": { - "clientTemplate": "java_client", - "dockerImage": "grpc_stress_java", - "numInstances": 10, - "serverPodSpec": "java-stress-server" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 100, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8008, - "datasetIdNamePrefix": "stress_test_java", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/node-cxx.json b/tools/run_tests/stress_test/configs/node-cxx.json deleted file mode 100644 index 094c1236e7..0000000000 --- a/tools/run_tests/stress_test/configs/node-cxx.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_cxx_opt" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "opt" - }, - "grpc_stress_node": { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_node" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "node_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", - "node", - "/var/local/git/grpc/src/node/stress/stress_client.js" - ], - "metricsClientCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", - "node", - "/var/local/git/grpc/src/node/stress/metrics_client.js" - ] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "cxx_server_opt": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-cxx-opt": { - "serverTemplate": "cxx_server_opt", - "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-node": { - "clientTemplate": "node_client", - "dockerImage": "grpc_stress_node", - "numInstances": 20, - "serverPodSpec": "stress-server-cxx-opt" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8006, - "datasetIdNamePrefix": "stress_test_node_cxx_opt", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/node.json b/tools/run_tests/stress_test/configs/node.json deleted file mode 100644 index 85eb9e0003..0000000000 --- a/tools/run_tests/stress_test/configs/node.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_node" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_node" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "node_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", - "node", - "/var/local/git/grpc/src/node/stress/stress_client.js" - ], - "metricsClientCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", - "node", - "/var/local/git/grpc/src/node/stress/metrics_client.js" - ] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "node_server": { - "baseTemplate": "default", - "stressServerCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", - "node", - "/var/local/git/grpc/src/node/interop/interop_server.js" - ] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "node-stress-server": { - "serverTemplate": "node_server", - "dockerImage": "grpc_stress_node", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "node-stress-client": { - "clientTemplate": "node_client", - "dockerImage": "grpc_stress_node", - "numInstances": 15, - "serverPodSpec": "node-stress-server" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8005, - "datasetIdNamePrefix": "stress_test_node", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json deleted file mode 100644 index fcb3678c02..0000000000 --- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_cxx_opt" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "opt" - }, - "grpc_stress_cxx_tsan": { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "tsan" - }, - "grpc_stress_cxx_asan": { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "asan" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "cxx_client_opt": { - "baseTemplate": "default", - "stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"], - "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] - }, - "cxx_client_tsan": { - "baseTemplate": "default", - "stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"], - "metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"] - }, - "cxx_client_asan": { - "baseTemplate": "default", - "stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"], - "metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "cxx_server_opt": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] - }, - "cxx_server_tsan": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"] - }, - "cxx_server_asan": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-opt": { - "serverTemplate": "cxx_server_opt", - "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 1 - }, - "stress-server-tsan": { - "serverTemplate": "cxx_server_tsan", - "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 1 - }, - "stress-server-asan": { - "serverTemplate": "cxx_server_asan", - "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-opt": { - "clientTemplate": "cxx_client_opt", - "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 5, - "serverPodSpec": "stress-server-opt" - }, - "stress-client-tsan": { - "clientTemplate": "cxx_client_tsan", - "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 10, - "serverPodSpec": "stress-server-tsan" - }, - "stress-client-asan": { - "clientTemplate": "cxx_client_asan", - "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 10, - "serverPodSpec": "stress-server-asan" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8004, - "datasetIdNamePrefix": "stress_test_opt_tsan", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json deleted file mode 100644 index 5e0e930d45..0000000000 --- a/tools/run_tests/stress_test/configs/opt.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_cxx_opt" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "opt" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "cxx_client_opt": { - "baseTemplate": "default", - "stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"], - "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "cxx_server_opt": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-opt": { - "serverTemplate": "cxx_server_opt", - "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-opt": { - "clientTemplate": "cxx_client_opt", - "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 15, - "serverPodSpec": "stress-server-opt" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8001, - "datasetIdNamePrefix": "stress_test_opt", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/php-cxx.json b/tools/run_tests/stress_test/configs/php-cxx.json deleted file mode 100644 index 03254b368c..0000000000 --- a/tools/run_tests/stress_test/configs/php-cxx.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_cxx_opt" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "opt" - }, - "grpc_stress_php": { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_php" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081" - } - } - }, - "templates": { - "php_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "/var/local/git/grpc/src/php/bin/stress_client.sh" - ], - "metricsClientCmd": [ - "php", - "/var/local/git/grpc/src/php/tests/interop/metrics_client.php" - ] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "cxx_server_opt": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-cxx-php": { - "serverTemplate": "cxx_server_opt", - "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-php": { - "clientTemplate": "php_client", - "dockerImage": "grpc_stress_php", - "numInstances": 20, - "serverPodSpec": "stress-server-cxx-php" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8010, - "datasetIdNamePrefix": "stress_test_php_cxx_opt", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/python.json b/tools/run_tests/stress_test/configs/python.json deleted file mode 100644 index 4f85de1d5f..0000000000 --- a/tools/run_tests/stress_test/configs/python.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_python" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_python" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - }, - "env": { - "PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", - "LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" - } - } - }, - "templates": { - "python_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "python", - "/var/local/git/grpc/src/python/grpcio/tests/stress/client.py" - ], - "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - }, - "env": { - "PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", - "LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" - } - } - }, - "templates": { - "python_server": { - "baseTemplate": "default", - "stressServerCmd": [ - "python", - "/var/local/git/grpc/src/python/grpcio/tests/interop/server.py" - ] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "python-stress-server": { - "serverTemplate": "python_server", - "dockerImage": "grpc_stress_python", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "python-stress-client": { - "clientTemplate": "python_client", - "dockerImage": "grpc_stress_python", - "numInstances": 5, - "serverPodSpec": "python-stress-server" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8011, - "datasetIdNamePrefix": "stress_test_python", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/ruby.json b/tools/run_tests/stress_test/configs/ruby.json deleted file mode 100644 index 7e2afcbb69..0000000000 --- a/tools/run_tests/stress_test/configs/ruby.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_ruby" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_ruby" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "ruby_client": { - "baseTemplate": "default", - "stressClientCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_ruby.sh", - "ruby", - "/var/local/git/grpc/src/ruby/stress/stress_client.rb" - ], - "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "ruby_server": { - "baseTemplate": "default", - "stressServerCmd": [ - "/var/local/git/grpc/tools/gcp/stress_test/run_ruby.sh", - "ruby", - "/var/local/git/grpc/src/ruby/pb/test/server.rb" - ] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-ruby": { - "serverTemplate": "ruby_server", - "dockerImage": "grpc_stress_ruby", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-ruby": { - "clientTemplate": "ruby_client", - "dockerImage": "grpc_stress_ruby", - "numInstances": 10, - "serverPodSpec": "stress-server-ruby" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8001, - "datasetIdNamePrefix": "stress_test_ruby", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json deleted file mode 100644 index abc759c79d..0000000000 --- a/tools/run_tests/stress_test/configs/tsan.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "dockerImages": { - "grpc_stress_cxx_tsan" : { - "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", - "dockerFileDir": "grpc_interop_stress_cxx", - "buildType": "tsan" - } - }, - - "clientTemplates": { - "baseTemplates": { - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 120, - "clientArgs": { - "num_channels_per_server":5, - "num_stubs_per_channel":10, - "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081 - }, - "metricsPort": 8081, - "metricsArgs": { - "metrics_server_address": "localhost:8081", - "total_only": "true" - } - } - }, - "templates": { - "cxx_client_tsan": { - "baseTemplate": "default", - "stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"], - "metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"] - } - } - }, - - "serverTemplates": { - "baseTemplates":{ - "default": { - "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", - "serverPort": 8080, - "serverArgs": { - "port": 8080 - } - } - }, - "templates": { - "cxx_server_tsan": { - "baseTemplate": "default", - "stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"] - } - } - }, - - "testMatrix": { - "serverPodSpecs": { - "stress-server-tsan": { - "serverTemplate": "cxx_server_tsan", - "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 1 - } - }, - - "clientPodSpecs": { - "stress-client-tsan": { - "clientTemplate": "cxx_client_tsan", - "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 5, - "serverPodSpec": "stress-server-tsan" - } - } - }, - - "globalSettings": { - "buildDockerImages": true, - "pollIntervalSecs": 60, - "testDurationSecs": 7200, - "kubernetesProxyPort": 8002, - "datasetIdNamePrefix": "stress_test_tsan", - "summaryTableId": "summary", - "qpsTableId": "qps", - "podWarmupSecs": 60 - } -} - diff --git a/tools/run_tests/stress_test/print_summary.py b/tools/run_tests/stress_test/print_summary.py deleted file mode 100755 index 6f4ada2f4f..0000000000 --- a/tools/run_tests/stress_test/print_summary.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import argparse -import os -import sys - -stress_test_utils_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../../gcp/stress_test')) -sys.path.append(stress_test_utils_dir) -from stress_test_utils import BigQueryHelper - -argp = argparse.ArgumentParser( - description='Print summary tables', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) -argp.add_argument('--gcp_project_id', - required=True, - help='The Google Cloud Platform Project Id') -argp.add_argument('--dataset_id', type=str, required=True) -argp.add_argument('--run_id', type=str, required=True) -argp.add_argument('--summary_table_id', type=str, default='summary') -argp.add_argument('--qps_table_id', type=str, default='qps') -argp.add_argument('--summary_only', action='store_true', default=True) - -if __name__ == '__main__': - args = argp.parse_args() - bq_helper = BigQueryHelper(args.run_id, '', '', args.gcp_project_id, - args.dataset_id, args.summary_table_id, - args.qps_table_id) - bq_helper.initialize() - if not args.summary_only: - bq_helper.print_qps_records() - bq_helper.print_summary_records() diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py deleted file mode 100755 index b190ebde7a..0000000000 --- a/tools/run_tests/stress_test/run_on_gke.py +++ /dev/null @@ -1,674 +0,0 @@ -#!/usr/bin/env python -# Copyright 2015-2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from __future__ import print_function - -import argparse -import datetime -import json -import os -import subprocess -import sys -import time - -stress_test_utils_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../../gcp/stress_test')) -sys.path.append(stress_test_utils_dir) -from stress_test_utils import BigQueryHelper - -kubernetes_api_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), '../../gcp/utils')) -sys.path.append(kubernetes_api_dir) - -import kubernetes_api - - -class GlobalSettings: - - def __init__(self, gcp_project_id, build_docker_images, - test_poll_interval_secs, test_duration_secs, - kubernetes_proxy_port, dataset_id_prefix, summary_table_id, - qps_table_id, pod_warmup_secs): - self.gcp_project_id = gcp_project_id - self.build_docker_images = build_docker_images - self.test_poll_interval_secs = test_poll_interval_secs - self.test_duration_secs = test_duration_secs - self.kubernetes_proxy_port = kubernetes_proxy_port - self.dataset_id_prefix = dataset_id_prefix - self.summary_table_id = summary_table_id - self.qps_table_id = qps_table_id - self.pod_warmup_secs = pod_warmup_secs - - -class ClientTemplate: - """ Contains all the common settings that are used by a stress client """ - - def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, - wrapper_script_path, poll_interval_secs, client_args_dict, - metrics_args_dict, will_run_forever, env_dict): - self.name = name - self.stress_client_cmd = stress_client_cmd - self.metrics_client_cmd = metrics_client_cmd - self.metrics_port = metrics_port - self.wrapper_script_path = wrapper_script_path - self.poll_interval_secs = poll_interval_secs - self.client_args_dict = client_args_dict - self.metrics_args_dict = metrics_args_dict - self.will_run_forever = will_run_forever - self.env_dict = env_dict - - -class ServerTemplate: - """ Contains all the common settings used by a stress server """ - - def __init__(self, name, server_cmd, wrapper_script_path, server_port, - server_args_dict, will_run_forever, env_dict): - self.name = name - self.server_cmd = server_cmd - self.wrapper_script_path = wrapper_script_path - self.server_port = server_port - self.server_args_dict = server_args_dict - self.will_run_forever = will_run_forever - self.env_dict = env_dict - - -class DockerImage: - """ Represents properties of a Docker image. Provides methods to build the - image and push it to GKE registry - """ - - def __init__(self, gcp_project_id, image_name, build_script_path, - dockerfile_dir, build_type): - """Args: - - image_name: The docker image name - tag_name: The additional tag name. This is the name used when pushing the - docker image to GKE registry - build_script_path: The path to the build script that builds this docker - image - dockerfile_dir: The name of the directory under - '<grpc_root>/tools/dockerfile' that contains the dockerfile - """ - self.image_name = image_name - self.gcp_project_id = gcp_project_id - self.build_script_path = build_script_path - self.dockerfile_dir = dockerfile_dir - self.build_type = build_type - self.tag_name = self._make_tag_name(gcp_project_id, image_name) - - def _make_tag_name(self, project_id, image_name): - return 'gcr.io/%s/%s' % (project_id, image_name) - - def build_image(self): - print('Building docker image: %s (tag: %s)' % (self.image_name, - self.tag_name)) - os.environ['INTEROP_IMAGE'] = self.image_name - os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name - os.environ['BASE_NAME'] = self.dockerfile_dir - os.environ['BUILD_TYPE'] = self.build_type - print('DEBUG: path: ', self.build_script_path) - if subprocess.call(args=[self.build_script_path]) != 0: - print('Error in building the Docker image') - return False - return True - - def push_to_gke_registry(self): - cmd = ['gcloud', 'docker', 'push', self.tag_name] - print('Pushing %s to the GKE registry..' % self.tag_name) - if subprocess.call(args=cmd) != 0: - print('Error in pushing the image %s to the GKE registry' % - self.tag_name) - return False - return True - - -class ServerPodSpec: - """ Contains the information required to launch server pods. """ - - def __init__(self, name, server_template, docker_image, num_instances): - self.name = name - self.template = server_template - self.docker_image = docker_image - self.num_instances = num_instances - - def pod_names(self): - """ Return a list of names of server pods to create. """ - return ['%s-%d' % (self.name, i) for i in range(1, self.num_instances + 1)] - - def server_addresses(self): - """ Return string of server addresses in the following format: - '<server_pod_name_1>:<server_port>,<server_pod_name_2>:<server_port>...' - """ - return ','.join(['%s:%d' % (pod_name, self.template.server_port) - for pod_name in self.pod_names()]) - - -class ClientPodSpec: - """ Contains the information required to launch client pods """ - - def __init__(self, name, client_template, docker_image, num_instances, - server_addresses): - self.name = name - self.template = client_template - self.docker_image = docker_image - self.num_instances = num_instances - self.server_addresses = server_addresses - - def pod_names(self): - """ Return a list of names of client pods to create """ - return ['%s-%d' % (self.name, i) for i in range(1, self.num_instances + 1)] - - # The client args in the template do not have server addresses. This function - # adds the server addresses and returns the updated client args - def get_client_args_dict(self): - args_dict = self.template.client_args_dict.copy() - args_dict['server_addresses'] = self.server_addresses - return args_dict - - -class Gke: - """ Class that has helper methods to interact with GKE """ - - class KubernetesProxy: - """Class to start a proxy on localhost to talk to the Kubernetes API server""" - - def __init__(self, port): - cmd = ['kubectl', 'proxy', '--port=%d' % port] - self.p = subprocess.Popen(args=cmd) - time.sleep(2) - print('\nStarted kubernetes proxy on port: %d' % port) - - def __del__(self): - if self.p is not None: - print('Shutting down Kubernetes proxy..') - self.p.kill() - - def __init__(self, project_id, run_id, dataset_id, summary_table_id, - qps_table_id, kubernetes_port): - self.project_id = project_id - self.run_id = run_id - self.dataset_id = dataset_id - self.summary_table_id = summary_table_id - self.qps_table_id = qps_table_id - - # The environment variables we would like to pass to every pod (both client - # and server) launched in GKE - self.gke_env = { - 'RUN_ID': self.run_id, - 'GCP_PROJECT_ID': self.project_id, - 'DATASET_ID': self.dataset_id, - 'SUMMARY_TABLE_ID': self.summary_table_id, - 'QPS_TABLE_ID': self.qps_table_id - } - - self.kubernetes_port = kubernetes_port - # Start kubernetes proxy - self.kubernetes_proxy = Gke.KubernetesProxy(kubernetes_port) - - def _args_dict_to_str(self, args_dict): - return ' '.join('--%s=%s' % (k, args_dict[k]) for k in args_dict.keys()) - - def launch_servers(self, server_pod_spec): - is_success = True - - # The command to run inside the container is the wrapper script (which then - # launches the actual server) - container_cmd = server_pod_spec.template.wrapper_script_path - - # The parameters to the wrapper script (defined in - # server_pod_spec.template.wrapper_script_path) are are injected into the - # container via environment variables - server_env = self.gke_env.copy() - server_env.update(server_pod_spec.template.env_dict) - server_env.update({ - 'STRESS_TEST_IMAGE_TYPE': 'SERVER', - 'STRESS_TEST_CMD': server_pod_spec.template.server_cmd, - 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( - server_pod_spec.template.server_args_dict), - 'WILL_RUN_FOREVER': str(server_pod_spec.template.will_run_forever) - }) - - for pod_name in server_pod_spec.pod_names(): - server_env['POD_NAME'] = pod_name - print('Creating server: %s' % pod_name) - is_success = kubernetes_api.create_pod_and_service( - 'localhost', - self.kubernetes_port, - 'default', # Use 'default' namespace - pod_name, - server_pod_spec.docker_image.tag_name, - [server_pod_spec.template.server_port], # Ports to expose on the pod - [container_cmd], - [], # Args list is empty since we are passing all args via env variables - server_env, - True # Headless = True for server to that GKE creates a DNS record for pod_name - ) - if not is_success: - print('Error in launching server: %s' % pod_name) - break - - if is_success: - print('Successfully created server(s)') - - return is_success - - def launch_clients(self, client_pod_spec): - is_success = True - - # The command to run inside the container is the wrapper script (which then - # launches the actual stress client) - container_cmd = client_pod_spec.template.wrapper_script_path - - # The parameters to the wrapper script (defined in - # client_pod_spec.template.wrapper_script_path) are are injected into the - # container via environment variables - client_env = self.gke_env.copy() - client_env.update(client_pod_spec.template.env_dict) - client_env.update({ - 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', - 'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, - 'STRESS_TEST_ARGS_STR': self._args_dict_to_str( - client_pod_spec.get_client_args_dict()), - 'METRICS_CLIENT_CMD': client_pod_spec.template.metrics_client_cmd, - 'METRICS_CLIENT_ARGS_STR': self._args_dict_to_str( - client_pod_spec.template.metrics_args_dict), - 'POLL_INTERVAL_SECS': str(client_pod_spec.template.poll_interval_secs), - 'WILL_RUN_FOREVER': str(client_pod_spec.template.will_run_forever) - }) - - for pod_name in client_pod_spec.pod_names(): - client_env['POD_NAME'] = pod_name - print('Creating client: %s' % pod_name) - is_success = kubernetes_api.create_pod_and_service( - 'localhost', - self.kubernetes_port, - 'default', # default namespace, - pod_name, - client_pod_spec.docker_image.tag_name, - [client_pod_spec.template.metrics_port], # Ports to expose on the pod - [container_cmd], - [], # Empty args list since all args are passed via env variables - client_env, - True # Client is a headless service (no need for an external ip) - ) - - if not is_success: - print('Error in launching client %s' % pod_name) - break - - if is_success: - print('Successfully created all client(s)') - - return is_success - - def _delete_pods(self, pod_name_list): - is_success = True - for pod_name in pod_name_list: - print('Deleting %s' % pod_name) - is_success = kubernetes_api.delete_pod_and_service( - 'localhost', - self.kubernetes_port, - 'default', # default namespace - pod_name) - - if not is_success: - print('Error in deleting pod %s' % pod_name) - break - - if is_success: - print('Successfully deleted all pods') - - return is_success - - def delete_servers(self, server_pod_spec): - return self._delete_pods(server_pod_spec.pod_names()) - - def delete_clients(self, client_pod_spec): - return self._delete_pods(client_pod_spec.pod_names()) - - -class Config: - - def __init__(self, config_filename, gcp_project_id): - print('Loading configuration...') - config_dict = self._load_config(config_filename) - - self.global_settings = self._parse_global_settings(config_dict, - gcp_project_id) - self.docker_images_dict = self._parse_docker_images( - config_dict, self.global_settings.gcp_project_id) - self.client_templates_dict = self._parse_client_templates(config_dict) - self.server_templates_dict = self._parse_server_templates(config_dict) - self.server_pod_specs_dict = self._parse_server_pod_specs( - config_dict, self.docker_images_dict, self.server_templates_dict) - self.client_pod_specs_dict = self._parse_client_pod_specs( - config_dict, self.docker_images_dict, self.client_templates_dict, - self.server_pod_specs_dict) - print('Loaded Configuaration.') - - def _parse_global_settings(self, config_dict, gcp_project_id): - global_settings_dict = config_dict['globalSettings'] - return GlobalSettings(gcp_project_id, - global_settings_dict['buildDockerImages'], - global_settings_dict['pollIntervalSecs'], - global_settings_dict['testDurationSecs'], - global_settings_dict['kubernetesProxyPort'], - global_settings_dict['datasetIdNamePrefix'], - global_settings_dict['summaryTableId'], - global_settings_dict['qpsTableId'], - global_settings_dict['podWarmupSecs']) - - def _parse_docker_images(self, config_dict, gcp_project_id): - """Parses the 'dockerImages' section of the config file and returns a - Dictionary of 'DockerImage' objects keyed by docker image names""" - docker_images_dict = {} - - docker_config_dict = config_dict['dockerImages'] - for image_name in docker_config_dict.keys(): - build_script_path = docker_config_dict[image_name]['buildScript'] - dockerfile_dir = docker_config_dict[image_name]['dockerFileDir'] - build_type = docker_config_dict[image_name].get('buildType', 'opt') - docker_images_dict[image_name] = DockerImage(gcp_project_id, image_name, - build_script_path, - dockerfile_dir, build_type) - return docker_images_dict - - def _parse_client_templates(self, config_dict): - """Parses the 'clientTemplates' section of the config file and returns a - Dictionary of 'ClientTemplate' objects keyed by client template names. - - Note: The 'baseTemplates' sub section of the config file contains templates - with default values and the 'templates' sub section contains the actual - client templates (which refer to the base template name to use for default - values). - """ - client_templates_dict = {} - - templates_dict = config_dict['clientTemplates']['templates'] - base_templates_dict = config_dict['clientTemplates'].get('baseTemplates', - {}) - for template_name in templates_dict.keys(): - # temp_dict is a temporary dictionary that merges base template dictionary - # and client template dictionary (with client template dictionary values - # overriding base template values) - temp_dict = {} - - base_template_name = templates_dict[template_name].get('baseTemplate') - if not base_template_name is None: - temp_dict = base_templates_dict[base_template_name].copy() - - temp_dict.update(templates_dict[template_name]) - - # Create and add ClientTemplate object to the final client_templates_dict - stress_client_cmd = ' '.join(temp_dict['stressClientCmd']) - metrics_client_cmd = ' '.join(temp_dict['metricsClientCmd']) - client_templates_dict[template_name] = ClientTemplate( - template_name, stress_client_cmd, metrics_client_cmd, - temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], - temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), - temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1), - temp_dict.get('env', {}).copy()) - - return client_templates_dict - - def _parse_server_templates(self, config_dict): - """Parses the 'serverTemplates' section of the config file and returns a - Dictionary of 'serverTemplate' objects keyed by server template names. - - Note: The 'baseTemplates' sub section of the config file contains templates - with default values and the 'templates' sub section contains the actual - server templates (which refer to the base template name to use for default - values). - """ - server_templates_dict = {} - - templates_dict = config_dict['serverTemplates']['templates'] - base_templates_dict = config_dict['serverTemplates'].get('baseTemplates', - {}) - - for template_name in templates_dict.keys(): - # temp_dict is a temporary dictionary that merges base template dictionary - # and server template dictionary (with server template dictionary values - # overriding base template values) - temp_dict = {} - - base_template_name = templates_dict[template_name].get('baseTemplate') - if not base_template_name is None: - temp_dict = base_templates_dict[base_template_name].copy() - - temp_dict.update(templates_dict[template_name]) - - # Create and add ServerTemplate object to the final server_templates_dict - stress_server_cmd = ' '.join(temp_dict['stressServerCmd']) - server_templates_dict[template_name] = ServerTemplate( - template_name, stress_server_cmd, temp_dict['wrapperScriptPath'], - temp_dict['serverPort'], temp_dict['serverArgs'].copy(), - temp_dict.get('willRunForever', 1), temp_dict.get('env', {}).copy()) - - return server_templates_dict - - def _parse_server_pod_specs(self, config_dict, docker_images_dict, - server_templates_dict): - """Parses the 'serverPodSpecs' sub-section (under 'testMatrix' section) of - the config file and returns a Dictionary of 'ServerPodSpec' objects keyed - by server pod spec names""" - server_pod_specs_dict = {} - - pod_specs_dict = config_dict['testMatrix'].get('serverPodSpecs', {}) - - for pod_name in pod_specs_dict.keys(): - server_template_name = pod_specs_dict[pod_name]['serverTemplate'] - docker_image_name = pod_specs_dict[pod_name]['dockerImage'] - num_instances = pod_specs_dict[pod_name].get('numInstances', 1) - - # Create and add the ServerPodSpec object to the final - # server_pod_specs_dict - server_pod_specs_dict[pod_name] = ServerPodSpec( - pod_name, server_templates_dict[server_template_name], - docker_images_dict[docker_image_name], num_instances) - - return server_pod_specs_dict - - def _parse_client_pod_specs(self, config_dict, docker_images_dict, - client_templates_dict, server_pod_specs_dict): - """Parses the 'clientPodSpecs' sub-section (under 'testMatrix' section) of - the config file and returns a Dictionary of 'ClientPodSpec' objects keyed - by client pod spec names""" - client_pod_specs_dict = {} - - pod_specs_dict = config_dict['testMatrix'].get('clientPodSpecs', {}) - for pod_name in pod_specs_dict.keys(): - client_template_name = pod_specs_dict[pod_name]['clientTemplate'] - docker_image_name = pod_specs_dict[pod_name]['dockerImage'] - num_instances = pod_specs_dict[pod_name]['numInstances'] - - # Get the server addresses from the server pod spec object - server_pod_spec_name = pod_specs_dict[pod_name]['serverPodSpec'] - server_addresses = server_pod_specs_dict[ - server_pod_spec_name].server_addresses() - - client_pod_specs_dict[pod_name] = ClientPodSpec( - pod_name, client_templates_dict[client_template_name], - docker_images_dict[docker_image_name], num_instances, - server_addresses) - - return client_pod_specs_dict - - def _load_config(self, config_filename): - """Opens the config file and converts the Json text to Dictionary""" - if not os.path.isabs(config_filename): - raise Exception('Config objects expects an absolute file path. ' - 'config file name passed: %s' % config_filename) - with open(config_filename) as config_file: - return json.load(config_file) - - -def run_tests(config): - """ The main function that launches the stress tests """ - # Build docker images and push to GKE registry - if config.global_settings.build_docker_images: - for name, docker_image in config.docker_images_dict.iteritems(): - if not (docker_image.build_image() and - docker_image.push_to_gke_registry()): - return False - - # Create a unique id for this run (Note: Using timestamp instead of UUID to - # make it easier to deduce the date/time of the run just by looking at the run - # run id. This is useful in debugging when looking at records in Biq query) - run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id) - print('Run id:', run_id) - print('Dataset id:', dataset_id) - - bq_helper = BigQueryHelper(run_id, '', '', - config.global_settings.gcp_project_id, dataset_id, - config.global_settings.summary_table_id, - config.global_settings.qps_table_id) - bq_helper.initialize() - - gke = Gke(config.global_settings.gcp_project_id, run_id, dataset_id, - config.global_settings.summary_table_id, - config.global_settings.qps_table_id, - config.global_settings.kubernetes_proxy_port) - - is_success = True - - try: - print('Launching servers..') - for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): - if not gke.launch_servers(server_pod_spec): - is_success = False # is_success is checked in the 'finally' block - return False - - print('Launched servers. Waiting for %d seconds for the server pods to be ' - 'fully online') % config.global_settings.pod_warmup_secs - time.sleep(config.global_settings.pod_warmup_secs) - - for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): - if not gke.launch_clients(client_pod_spec): - is_success = False # is_success is checked in the 'finally' block - return False - - print('Launched all clients. Waiting for %d seconds for the client pods to ' - 'be fully online') % config.global_settings.pod_warmup_secs - time.sleep(config.global_settings.pod_warmup_secs) - - start_time = datetime.datetime.now() - end_time = start_time + datetime.timedelta( - seconds=config.global_settings.test_duration_secs) - print('Running the test until %s' % end_time.isoformat()) - - while True: - if datetime.datetime.now() > end_time: - print('Test was run for %d seconds' % - config.global_settings.test_duration_secs) - break - - # Check if either stress server or clients have failed (btw, the bq_helper - # monitors all the rows in the summary table and checks if any of them - # have a failure status) - if bq_helper.check_if_any_tests_failed(): - is_success = False - print('Some tests failed.') - break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables - - # Tests running fine. Wait until next poll time to check the status - print('Sleeping for %d seconds..' % - config.global_settings.test_poll_interval_secs) - time.sleep(config.global_settings.test_poll_interval_secs) - - # Print BiqQuery tables - bq_helper.print_qps_records() - bq_helper.print_summary_records() - - finally: - # If there was a test failure, we should not delete the pods since they - # would contain useful debug information (logs, core dumps etc) - if is_success: - for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): - gke.delete_servers(server_pod_spec) - for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): - gke.delete_clients(client_pod_spec) - - return is_success - - -def tear_down(config): - gke = Gke(config.global_settings.gcp_project_id, '', '', - config.global_settings.summary_table_id, - config.global_settings.qps_table_id, - config.global_settings.kubernetes_proxy_port) - for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): - gke.delete_servers(server_pod_spec) - for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): - gke.delete_clients(client_pod_spec) - - -argp = argparse.ArgumentParser( - description='Launch stress tests in GKE', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) -argp.add_argument('--gcp_project_id', - required=True, - help='The Google Cloud Platform Project Id') -argp.add_argument('--config_file', - required=True, - type=str, - help='The test config file') -argp.add_argument('--tear_down', action='store_true', default=False) - -if __name__ == '__main__': - args = argp.parse_args() - - config_filename = args.config_file - - # Since we will be changing the current working directory to grpc root in the - # next step, we should check if the config filename path is a relative path - # (i.e a path relative to the current working directory) and if so, convert it - # to abosulte path - if not os.path.isabs(config_filename): - config_filename = os.path.abspath(config_filename) - - config = Config(config_filename, args.gcp_project_id) - - # Change current working directory to grpc root - # (This is important because all relative file paths in the config file are - # supposed to interpreted as relative to the GRPC root) - grpc_root = os.path.abspath(os.path.join( - os.path.dirname(sys.argv[0]), '../../..')) - os.chdir(grpc_root) - - # Note that tear_down is only in cases where we want to manually tear down a - # test that for some reason run_tests() could not cleanup - if args.tear_down: - tear_down(config) - sys.exit(1) - - if not run_tests(config): - sys.exit(1) |