aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests
diff options
context:
space:
mode:
Diffstat (limited to 'tools/run_tests')
-rwxr-xr-xtools/run_tests/build_python.sh14
-rw-r--r--tools/run_tests/performance/scenario_config.py70
-rwxr-xr-xtools/run_tests/run_performance_tests.py14
-rwxr-xr-xtools/run_tests/run_tests.py48
-rw-r--r--tools/run_tests/tests.json92
5 files changed, 209 insertions, 29 deletions
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index b786c479f3..54e2fe5347 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -39,6 +39,14 @@ cd $(dirname $0)/../..
PLATFORM=`uname -s`
+function is_msys() {
+ if [ "${PLATFORM/MSYS}" != "$PLATFORM" ]; then
+ echo true
+ else
+ exit 1
+ fi
+}
+
function is_mingw() {
if [ "${PLATFORM/MINGW}" != "$PLATFORM" ]; then
echo true
@@ -108,6 +116,12 @@ VENV=${2:-$(venv $PYTHON)}
VENV_RELATIVE_PYTHON=${3:-$(venv_relative_python)}
TOOLCHAIN=${4:-$(toolchain)}
+if [ $(is_msys) ]; then
+ echo "MSYS doesn't directly provide the right compiler(s);"
+ echo "switch to a MinGW shell."
+ exit 1
+fi
+
ROOT=`pwd`
export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv $CFLAGS"
export GRPC_PYTHON_BUILD_WITH_CYTHON=1
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 47774a1f1e..4e4c16642b 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -232,6 +232,15 @@ class CXXLanguage:
secure=secure,
categories=smoketest_categories + [SCALABLE])
+ yield _ping_pong_scenario(
+ 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' % secstr,
+ rpc_type='STREAMING',
+ client_type='ASYNC_CLIENT',
+ server_type='SYNC_SERVER',
+ unconstrained_client='async',
+ secure=secure,
+ categories=smoketest_categories+[SCALABLE])
+
for rpc_type in ['unary', 'streaming']:
for synchronicity in ['sync', 'async']:
yield _ping_pong_scenario(
@@ -361,7 +370,8 @@ class NodeLanguage:
self.safename = str(self)
def worker_cmdline(self):
- return ['tools/run_tests/performance/run_worker_node.sh']
+ return ['tools/run_tests/performance/run_worker_node.sh',
+ '--benchmark_impl=grpc']
def worker_port_offset(self):
return 200
@@ -664,11 +674,69 @@ class GoLanguage:
def __str__(self):
return 'go'
+class NodeExpressLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_node.sh',
+ '--benchmark_impl=express']
+
+ def worker_port_offset(self):
+ return 700
+
+ def scenarios(self):
+ # TODO(jtattermusch): make this scenario work
+ #yield _ping_pong_scenario(
+ # 'node_generic_async_streaming_ping_pong', rpc_type='STREAMING',
+ # client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+ # use_generic_payload=True)
+
+ # TODO(jtattermusch): make this scenario work
+ #yield _ping_pong_scenario(
+ # 'node_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+ # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+
+ yield _ping_pong_scenario(
+ 'node_protobuf_unary_ping_pong', rpc_type='UNARY',
+ client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+ categories=[SCALABLE, SMOKETEST])
+
+ yield _ping_pong_scenario(
+ 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
+ client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+ unconstrained_client='async',
+ categories=[SCALABLE, SMOKETEST])
+
+ # TODO(jtattermusch): make this scenario work
+ #yield _ping_pong_scenario(
+ # 'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
+ # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+ # unconstrained_client='async')
+
+ # TODO(jtattermusch): make this scenario work
+ #yield _ping_pong_scenario(
+ # 'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
+ # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+ # server_language='c++', server_core_limit=1, async_server_threads=1)
+
+ # TODO(jtattermusch): make this scenario work
+ #yield _ping_pong_scenario(
+ # 'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+ # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+ # server_language='c++', server_core_limit=1, async_server_threads=1)
+
+ def __str__(self):
+ return 'node_express'
+
LANGUAGES = {
'c++' : CXXLanguage(),
'csharp' : CSharpLanguage(),
'node' : NodeLanguage(),
+ 'node_express': NodeExpressLanguage(),
'ruby' : RubyLanguage(),
'java' : JavaLanguage(),
'python' : PythonLanguage(),
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 099ab89ddf..7cb827ecea 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -42,6 +42,7 @@ import os
import performance.scenario_config as scenario_config
import pipes
import re
+import report_utils
import subprocess
import sys
import tempfile
@@ -453,6 +454,7 @@ if not scenarios:
total_scenario_failures = 0
qps_workers_killed = 0
+merged_resultset = {}
for scenario in scenarios:
if args.dry_run:
print(scenario.name)
@@ -460,14 +462,20 @@ for scenario in scenarios:
try:
for worker in scenario.workers:
worker.start()
- scenario_failures, _ = jobset.run([scenario.jobspec,
- create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
- newline_on_success=True, maxjobs=1)
+ scenario_failures, resultset = jobset.run([scenario.jobspec,
+ create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
+ newline_on_success=True, maxjobs=1)
total_scenario_failures += scenario_failures
+ merged_resultset = dict(itertools.chain(merged_resultset.iteritems(),
+ resultset.iteritems()))
finally:
# Consider qps workers that need to be killed as failures
qps_workers_killed += finish_qps_workers(scenario.workers)
+
+report_utils.render_junit_xml_report(merged_resultset, 'report.xml',
+ suite_name='benchmarks')
+
if total_scenario_failures > 0 or qps_workers_killed > 0:
print ("%s scenarios failed and %s qps worker jobs killed" % (total_scenario_failures, qps_workers_killed))
sys.exit(1)
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index db261d98c2..2886870d38 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -842,6 +842,53 @@ class Sanity(object):
def __str__(self):
return 'sanity'
+class NodeExpressLanguage(object):
+ """Dummy Node express test target to enable running express performance
+ benchmarks"""
+
+ def __init__(self):
+ self.platform = platform_string()
+
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default', 'node0.12',
+ 'node4', 'node5', 'node6'])
+ if self.args.compiler == 'default':
+ self.node_version = '4'
+ else:
+ # Take off the word "node"
+ self.node_version = self.args.compiler[4:]
+
+ def test_specs(self):
+ return []
+
+ def pre_build_steps(self):
+ if self.platform == 'windows':
+ return [['tools\\run_tests\\pre_build_node.bat']]
+ else:
+ return [['tools/run_tests/pre_build_node.sh', self.node_version]]
+
+ def make_targets(self):
+ return []
+
+ def make_options(self):
+ return []
+
+ def build_steps(self):
+ return []
+
+ def post_tests_steps(self):
+ return []
+
+ def makefile_name(self):
+ return 'Makefile'
+
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
+
+ def __str__(self):
+ return 'node_express'
# different configurations we can run under
with open('tools/run_tests/configs.json') as f:
@@ -852,6 +899,7 @@ _LANGUAGES = {
'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'),
'node': NodeLanguage(),
+ 'node_express': NodeExpressLanguage(),
'php': PhpLanguage(),
'php7': Php7Language(),
'python': PythonLanguage(),
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 68d99eed51..c3715995d2 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -5156,7 +5156,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_census_test",
"platforms": [
@@ -6193,7 +6193,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_compress_test",
"platforms": [
@@ -7230,7 +7230,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_fake_resolver_test",
"platforms": [
@@ -8254,7 +8254,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_fakesec_test",
"platforms": [
@@ -10179,7 +10179,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_full_test",
"platforms": [
@@ -11165,7 +11165,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_full+pipe_test",
"platforms": [
@@ -12071,7 +12071,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_full+trace_test",
"platforms": [
@@ -13073,7 +13073,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_http_proxy_test",
"platforms": [
@@ -14142,7 +14142,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_load_reporting_test",
"platforms": [
@@ -15190,7 +15190,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_oauth2_test",
"platforms": [
@@ -16222,7 +16222,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_proxy_test",
"platforms": [
@@ -20033,7 +20033,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_ssl_test",
"platforms": [
@@ -21070,7 +21070,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_ssl_cert_test",
"platforms": [
@@ -22070,7 +22070,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_ssl_proxy_test",
"platforms": [
@@ -22995,7 +22995,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_uds_test",
"platforms": [
@@ -24008,7 +24008,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_census_nosec_test",
"platforms": [
@@ -25022,7 +25022,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_compress_nosec_test",
"platforms": [
@@ -26036,7 +26036,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_fake_resolver_nosec_test",
"platforms": [
@@ -27947,7 +27947,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_full_nosec_test",
"platforms": [
@@ -28914,7 +28914,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_full+pipe_nosec_test",
"platforms": [
@@ -29797,7 +29797,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_full+trace_nosec_test",
"platforms": [
@@ -30775,7 +30775,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_http_proxy_nosec_test",
"platforms": [
@@ -31821,7 +31821,7 @@
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_load_reporting_nosec_test",
"platforms": [
@@ -32797,7 +32797,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_proxy_nosec_test",
"platforms": [
@@ -36487,7 +36487,7 @@
"exclude_iomgrs": [
"uv"
],
- "flaky": false,
+ "flaky": true,
"language": "c",
"name": "h2_uds_nosec_test",
"platforms": [
@@ -37319,6 +37319,27 @@
{
"args": [
"--scenarios_json",
+ "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
+ ],
+ "boringssl": true,
+ "ci_platforms": [
+ "linux"
+ ],
+ "cpu_cost": 8,
+ "defaults": "boringssl",
+ "exclude_configs": [],
+ "flaky": false,
+ "language": "c++",
+ "name": "json_run_localhost",
+ "platforms": [
+ "linux"
+ ],
+ "shortname": "json_run_localhost:cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_secure",
+ "timeout_seconds": 180
+ },
+ {
+ "args": [
+ "--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}"
],
"boringssl": true,
@@ -37655,6 +37676,27 @@
{
"args": [
"--scenarios_json",
+ "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
+ ],
+ "boringssl": true,
+ "ci_platforms": [
+ "linux"
+ ],
+ "cpu_cost": 8,
+ "defaults": "boringssl",
+ "exclude_configs": [],
+ "flaky": false,
+ "language": "c++",
+ "name": "json_run_localhost",
+ "platforms": [
+ "linux"
+ ],
+ "shortname": "json_run_localhost:cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_insecure",
+ "timeout_seconds": 180
+ },
+ {
+ "args": [
+ "--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}"
],
"boringssl": true,
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-16 14:20:46 +0000