aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests/performance
diff options
context:
space:
mode:
Diffstat (limited to 'tools/run_tests/performance')
-rwxr-xr-xtools/run_tests/performance/build_performance.sh16
-rwxr-xr-xtools/run_tests/performance/remote_host_prepare.sh7
-rwxr-xr-xtools/run_tests/performance/run_worker_java.sh39
-rwxr-xr-xtools/run_tests/performance/run_worker_ruby.sh36
-rw-r--r--tools/run_tests/performance/scenario_config.py175
5 files changed, 245 insertions, 28 deletions
diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh
index 2c962cba37..8cfe1c48e9 100755
--- a/tools/run_tests/performance/build_performance.sh
+++ b/tools/run_tests/performance/build_performance.sh
@@ -28,6 +28,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+source ~/.rvm/scripts/rvm
set -ex
cd $(dirname $0)/../../..
@@ -41,12 +42,19 @@ CONFIG=${CONFIG:-opt}
# TODO(jtattermusch): not embedding OpenSSL breaks the C# build because
# grpc_csharp_ext needs OpenSSL embedded and some intermediate files from
# this build will be reused.
-make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_driver qps_json_driver -j8
+make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_json_driver -j8
for language in $@
do
- if [ "$language" != "c++" ]
- then
+ case "$language" in
+ "c++")
+ ;; # C++ has already been built.
+ "java")
+ (cd ../grpc-java/ &&
+ ./gradlew -PskipCodegen=true :grpc-benchmarks:installDist)
+ ;;
+ *)
tools/run_tests/run_tests.py -l $language -c $CONFIG --build_only -j 8
- fi
+ ;;
+ esac
done
diff --git a/tools/run_tests/performance/remote_host_prepare.sh b/tools/run_tests/performance/remote_host_prepare.sh
index 18633d1420..17cfa1a599 100755
--- a/tools/run_tests/performance/remote_host_prepare.sh
+++ b/tools/run_tests/performance/remote_host_prepare.sh
@@ -38,7 +38,12 @@ ssh "${USER_AT_HOST}" "rm -rf ~/performance_workspace && mkdir -p ~/performance_
# TODO(jtattermusch): To be sure there are no running processes that would
# mess with the results, be rough and reboot the slave here
# and wait for it to come back online.
-ssh "${USER_AT_HOST}" "killall qps_worker mono node || true"
+# could also kill jenkins.
+ssh "${USER_AT_HOST}" "killall -9 qps_worker mono node ruby || true"
+
+# Kill all java LoadWorker processes. We can't just killall java
+# as one of the processes might be jenkins.
+ssh "${USER_AT_HOST}" 'kill -9 $(jps | grep LoadWorker | cut -f1 -d" ") || true'
# push the current sources to the slave and unpack it.
scp ../grpc.tar "${USER_AT_HOST}:~/performance_workspace"
diff --git a/tools/run_tests/performance/run_worker_java.sh b/tools/run_tests/performance/run_worker_java.sh
new file mode 100755
index 0000000000..d5503a18a4
--- /dev/null
+++ b/tools/run_tests/performance/run_worker_java.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+# Enter repo root
+cd $(dirname $0)/../../..
+
+# Enter the grpc-java repo root (expected to be next to grpc repo root)
+cd ../grpc-java
+
+benchmarks/build/install/grpc-benchmarks/bin/benchmark_worker $@
diff --git a/tools/run_tests/performance/run_worker_ruby.sh b/tools/run_tests/performance/run_worker_ruby.sh
new file mode 100755
index 0000000000..43187345bc
--- /dev/null
+++ b/tools/run_tests/performance/run_worker_ruby.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+source ~/.rvm/scripts/rvm
+set -ex
+
+cd $(dirname $0)/../../..
+
+ruby src/ruby/qps/worker.rb $@
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index bd10b6f032..55657f8d8a 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -31,8 +31,12 @@
SINGLE_MACHINE_CORES=8
WARMUP_SECONDS=5
+JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in.
BENCHMARK_SECONDS=30
+SECURE_SECARGS = {'use_test_ca': True,
+ 'server_host_override': 'foo.test.google.fr'}
+
HISTOGRAM_PARAMS = {
'resolution': 0.01,
'max_possible': 60e9,
@@ -81,8 +85,7 @@ class CXXLanguage:
for secure in [True, False]:
if secure:
secstr = 'secure'
- secargs = {'use_test_ca': True,
- 'server_host_override': 'foo.test.google.fr'}
+ secargs = SECURE_SECARGS
else:
secstr = 'insecure'
secargs = None
@@ -108,7 +111,7 @@ class CXXLanguage:
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 1,
'async_server_threads': 1,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
@@ -125,7 +128,7 @@ class CXXLanguage:
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
@@ -137,7 +140,7 @@ class CXXLanguage:
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
@@ -153,7 +156,7 @@ class CXXLanguage:
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
@@ -181,7 +184,7 @@ class CXXLanguage:
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
@@ -193,13 +196,13 @@ class CXXLanguage:
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'cpp_single_channel_throughput_%s'
+ 'name': 'cpp_protobuf_async_streaming_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -213,21 +216,47 @@ class CXXLanguage:
'load_params': {
'closed_loop': {}
},
- 'payload_config': BIG_GENERIC_PAYLOAD,
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 1,
'async_server_threads': 1,
- 'payload_config': BIG_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'cpp_protobuf_async_ping_pong_%s'
+ 'name': 'cpp_protobuf_sync_unary_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 0,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 0,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_async_unary_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -237,7 +266,7 @@ class CXXLanguage:
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
- 'rpc_type': 'STREAMING',
+ 'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
@@ -247,7 +276,7 @@ class CXXLanguage:
'server_config': {
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 1,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
@@ -270,8 +299,7 @@ class CSharpLanguage:
return 100
def scenarios(self):
- # TODO(jtattermusch): add more scenarios
- secargs = None
+ secargs = SECURE_SECARGS
yield {
'name': 'csharp_generic_async_streaming_ping_pong',
'num_servers': 1,
@@ -293,7 +321,7 @@ class CSharpLanguage:
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': 0,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
@@ -320,7 +348,7 @@ class CSharpLanguage:
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
@@ -346,7 +374,7 @@ class CSharpLanguage:
'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': 0,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
@@ -371,7 +399,7 @@ class CSharpLanguage:
'server_config': {
'server_type': 'SYNC_SERVER',
'security_params': secargs,
- 'core_limit': 0,
+ 'core_limit': 1,
'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
@@ -397,7 +425,7 @@ class NodeLanguage:
def scenarios(self):
# TODO(jtattermusch): add more scenarios
- secargs = None
+ secargs = SECURE_SECARGS
yield {
'name': 'node_protobuf_unary_ping_pong',
'num_servers': 1,
@@ -429,8 +457,109 @@ class NodeLanguage:
return 'node'
+class RubyLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_ruby.sh']
+
+ def worker_port_offset(self):
+ return 300
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ secargs = SECURE_SECARGS
+ yield {
+ 'name': 'ruby_protobuf_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'ruby'
+
+
+class JavaLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_java.sh']
+
+ def worker_port_offset(self):
+ return 400
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ for secure in [True, False]:
+ if secure:
+ secstr = 'secure'
+ secargs = SECURE_SECARGS
+ else:
+ secstr = 'insecure'
+ secargs = None
+
+ yield {
+ 'name': 'java_protobuf_unary_ping_pong_%s' % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': JAVA_WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'java'
+
+
LANGUAGES = {
'c++' : CXXLanguage(),
'csharp' : CSharpLanguage(),
'node' : NodeLanguage(),
+ 'ruby' : RubyLanguage(),
+ 'java' : JavaLanguage(),
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-13 18:39:24 +0000