From 7d764673ef9a18503339747ab8ee002739f6e329 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 19:10:49 -0700 Subject: Backends must be up before some grpclb tests start. --- test/cpp/end2end/grpclb_end2end_test.cc | 87 ++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 570a3d1067..6896b54f1b 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -398,11 +398,40 @@ class GrpclbEnd2endTest : public ::testing::Test { return true; } - void WaitForAllBackends() { + void SendRpcAndCount(int* num_total, int* num_ok, int* num_failure, + int* num_drops) { + const Status status = SendRpc(); + if (status.ok()) { + ++*num_ok; + } else { + if (status.error_message() == "Call dropped by load balancing policy") { + ++*num_drops; + } else { + ++*num_failure; + } + } + ++*num_total; + } + + std::tuple WaitForAllBackends( + int num_requests_multiple_of = 1) { + int num_ok = 0; + int num_failure = 0; + int num_drops = 0; + int num_total = 0; while (!SeenAllBackends()) { - CheckRpcSendOk(); + SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops); + } + while (num_total % num_requests_multiple_of != 0) { + SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops); } ResetBackendCounters(); + gpr_log(GPR_INFO, + "Performed %d warm up requests (a multiple of %d) against the " + "backends. %d succeeded, %d failed, %d dropped.", + num_total, num_requests_multiple_of, num_ok, num_failure, + num_drops); + return std::make_tuple(num_ok, num_failure, num_drops); } void WaitForBackend(size_t backend_idx) { @@ -556,10 +585,8 @@ TEST_F(SingleBalancerTest, Vanilla) { 0); // Make sure that trying to connect works without a call. channel_->GetState(true /* try_to_connect */); - // We need to wait for all backends to come online. WaitForAllBackends(); - // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); @@ -867,6 +894,8 @@ TEST_F(SingleBalancerTest, Drop) { 0, BalancerServiceImpl::BuildResponseForBackends( GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}), 0); + // Wait until all backends are ready. + WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs for each server and drop address. size_t num_drops = 0; for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { @@ -936,6 +965,11 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), 0); + // Wait until all backends are ready. + int num_ok = 0; + int num_failure = 0; + int num_drops = 0; + std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs per server. CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); // Each backend should have gotten 100 requests. @@ -950,11 +984,12 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); const ClientStats client_stats = WaitForLoadReports(); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, client_stats.num_calls_started); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, + client_stats.num_calls_started); + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + (num_ok + num_drops), client_stats.num_calls_finished_known_received); EXPECT_THAT(client_stats.drop_token_counts, ::testing::ElementsAre()); } @@ -965,7 +1000,14 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { 0, BalancerServiceImpl::BuildResponseForBackends( GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}), 0); - + // Wait until all backends are ready. + int num_warmup_ok = 0; + int num_warmup_failure = 0; + int num_warmup_drops = 0; + std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) = + WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */); + const int num_total_warmup_requests = + num_warmup_ok + num_warmup_failure + num_warmup_drops; size_t num_drops = 0; for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { EchoResponse response; @@ -980,7 +1022,6 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { } } EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); - // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress, @@ -993,17 +1034,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); const ClientStats client_stats = WaitForLoadReports(); - EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3), - client_stats.num_calls_started); - EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3), - client_stats.num_calls_finished); + EXPECT_EQ( + kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + client_stats.num_calls_started); + EXPECT_EQ( + kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); - EXPECT_EQ(kNumRpcsPerAddress * num_backends_, + EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok, client_stats.num_calls_finished_known_received); - EXPECT_THAT(client_stats.drop_token_counts, - ::testing::ElementsAre( - ::testing::Pair("load_balancing", kNumRpcsPerAddress), - ::testing::Pair("rate_limiting", kNumRpcsPerAddress * 2))); + // The number of warmup request is a multiple of the number of addresses. + // Therefore, all addresses in the scheduled balancer response are hit the + // same number of times. + const int num_times_drop_addresses_hit = num_warmup_drops / 3; + EXPECT_THAT( + client_stats.drop_token_counts, + ::testing::ElementsAre( + ::testing::Pair("load_balancing", + (kNumRpcsPerAddress + num_times_drop_addresses_hit)), + ::testing::Pair( + "rate_limiting", + (kNumRpcsPerAddress + num_times_drop_addresses_hit) * 2))); } } // namespace -- cgit v1.2.3 From 588e36a3e3b43276f573c466d88c3dd60bd5be16 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 18 Sep 2017 20:01:27 -0700 Subject: Get rid of magic numbers --- test/cpp/end2end/grpclb_end2end_test.cc | 45 ++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 6896b54f1b..77ed155292 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -890,15 +890,22 @@ TEST_F(UpdatesTest, UpdateBalancersDeadUpdate) { TEST_F(SingleBalancerTest, Drop) { const size_t kNumRpcsPerAddress = 100; + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 2; + const int num_of_drop_addresses = num_of_drop_by_rate_limiting_addresses + + num_of_drop_by_load_balancing_addresses; + const int num_total_addresses = num_backends_ + num_of_drop_addresses; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}), + GetBackendPorts(), + {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); // Wait until all backends are ready. WaitForAllBackends(); // Send kNumRpcsPerAddress RPCs for each server and drop address. size_t num_drops = 0; - for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { + for (size_t i = 0; i < kNumRpcsPerAddress * num_total_addresses; ++i) { EchoResponse response; const Status status = SendRpc(&response); if (!status.ok() && @@ -910,7 +917,7 @@ TEST_F(SingleBalancerTest, Drop) { EXPECT_EQ(response.message(), kMessage_); } } - EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); + EXPECT_EQ(kNumRpcsPerAddress * num_of_drop_addresses, num_drops); // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { @@ -925,9 +932,12 @@ TEST_F(SingleBalancerTest, Drop) { TEST_F(SingleBalancerTest, DropAllFirst) { // All registered addresses are marked as "drop". + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 1; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - {}, {{"rate_limiting", 1}, {"load_balancing", 1}}), + {}, {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); const Status status = SendRpc(); EXPECT_FALSE(status.ok()); @@ -938,9 +948,12 @@ TEST_F(SingleBalancerTest, DropAll) { ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}), 0); + const int num_of_drop_by_rate_limiting_addresses = 1; + const int num_of_drop_by_load_balancing_addresses = 1; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - {}, {{"rate_limiting", 1}, {"load_balancing", 1}}), + {}, {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 1000); // First call succeeds. @@ -996,20 +1009,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) { TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { const size_t kNumRpcsPerAddress = 3; + const int num_of_drop_by_rate_limiting_addresses = 2; + const int num_of_drop_by_load_balancing_addresses = 1; + const int num_of_drop_addresses = num_of_drop_by_rate_limiting_addresses + + num_of_drop_by_load_balancing_addresses; + const int num_total_addresses = num_backends_ + num_of_drop_addresses; ScheduleResponseForBalancer( 0, BalancerServiceImpl::BuildResponseForBackends( - GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}), + GetBackendPorts(), + {{"rate_limiting", num_of_drop_by_rate_limiting_addresses}, + {"load_balancing", num_of_drop_by_load_balancing_addresses}}), 0); // Wait until all backends are ready. int num_warmup_ok = 0; int num_warmup_failure = 0; int num_warmup_drops = 0; std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) = - WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */); + WaitForAllBackends(num_total_addresses /* num_requests_multiple_of */); const int num_total_warmup_requests = num_warmup_ok + num_warmup_failure + num_warmup_drops; size_t num_drops = 0; - for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) { + for (size_t i = 0; i < kNumRpcsPerAddress * num_total_addresses; ++i) { EchoResponse response; const Status status = SendRpc(&response); if (!status.ok() && @@ -1021,7 +1041,7 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { EXPECT_EQ(response.message(), kMessage_); } } - EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops); + EXPECT_EQ(kNumRpcsPerAddress * num_of_drop_addresses, num_drops); // Each backend should have gotten 100 requests. for (size_t i = 0; i < backends_.size(); ++i) { EXPECT_EQ(kNumRpcsPerAddress, @@ -1035,10 +1055,10 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { const ClientStats client_stats = WaitForLoadReports(); EXPECT_EQ( - kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + kNumRpcsPerAddress * num_total_addresses + num_total_warmup_requests, client_stats.num_calls_started); EXPECT_EQ( - kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests, + kNumRpcsPerAddress * num_total_addresses + num_total_warmup_requests, client_stats.num_calls_finished); EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send); EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok, @@ -1046,7 +1066,8 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) { // The number of warmup request is a multiple of the number of addresses. // Therefore, all addresses in the scheduled balancer response are hit the // same number of times. - const int num_times_drop_addresses_hit = num_warmup_drops / 3; + const int num_times_drop_addresses_hit = + num_warmup_drops / num_of_drop_addresses; EXPECT_THAT( client_stats.drop_token_counts, ::testing::ElementsAre( -- cgit v1.2.3 From 4917b08a17cd74fbe3c4c4032b9455f00721969e Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Thu, 21 Sep 2017 12:52:50 -0700 Subject: Revert "Revert "Add automated PHP-to-C++ performance testing"" --- tools/gce/linux_performance_worker_init.sh | 5 +++ tools/run_tests/performance/run_worker_php.sh | 43 ++++++++++++++++++++++++++ tools/run_tests/performance/scenario_config.py | 28 +++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100755 tools/run_tests/performance/run_worker_php.sh diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 8f0a0f65c3..88d8de7402 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -128,6 +128,11 @@ ruby -v # Install bundler (prerequisite for gRPC Ruby) gem install bundler +# PHP dependencies +sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev +curl -sS https://getcomposer.org/installer | php +sudo mv composer.phar /usr/local/bin/composer + # Java dependencies - nothing as we already have Java JDK 8 # Go dependencies diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh new file mode 100755 index 0000000000..1da74cd0db --- /dev/null +++ b/tools/run_tests/performance/run_worker_php.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +source ~/.rvm/scripts/rvm +set -ex + +repo=$(dirname $0)/../../.. + +# First set up all dependences needed for PHP QPS test +cd $repo +cd src/php/tests/qps +composer install +# The proxy worker for PHP is implemented in Ruby +cd ../../../.. +ruby src/ruby/qps/proxy-worker.rb $@ + diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index a7560400dd..5efc9f5648 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -800,6 +800,33 @@ class RubyLanguage: return 'ruby' +class PhpLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + return 800 + + def scenarios(self): + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + yield _ping_pong_scenario( + 'php_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', async_server_threads=1) + + def __str__(self): + return 'php' + + class JavaLanguage: def __init__(self): @@ -997,6 +1024,7 @@ LANGUAGES = { 'node' : NodeLanguage(), 'node_express': NodeExpressLanguage(), 'ruby' : RubyLanguage(), + 'php' : PhpLanguage(), 'java' : JavaLanguage(), 'python' : PythonLanguage(), 'go' : GoLanguage(), -- cgit v1.2.3 From 7115862e98059b3fbd2069ae7ad1bf09232a416e Mon Sep 17 00:00:00 2001 From: ZhouyihaiDing Date: Thu, 21 Sep 2017 13:23:35 -0700 Subject: update license --- tools/run_tests/performance/run_worker_php.sh | 35 ++++++++------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/tools/run_tests/performance/run_worker_php.sh b/tools/run_tests/performance/run_worker_php.sh index 1da74cd0db..5d0c4fa4fd 100755 --- a/tools/run_tests/performance/run_worker_php.sh +++ b/tools/run_tests/performance/run_worker_php.sh @@ -1,32 +1,17 @@ #!/bin/bash -# Copyright 2017, Google Inc. -# All rights reserved. +# Copyright 2017 gRPC authors. # -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. +# http://www.apache.org/licenses/LICENSE-2.0 # -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. source ~/.rvm/scripts/rvm set -ex -- cgit v1.2.3