From d5ca876b6de1588a4387a2ef84740d8bae79cfbe Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 6 Dec 2017 12:40:41 -0800 Subject: Fixed ClientLbEnd2endTest.RoundRobinReresolve flakiness --- test/cpp/end2end/client_lb_end2end_test.cc | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index f8bb12fde1..c09a324bf1 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -573,15 +573,28 @@ TEST_F(ClientLbEnd2endTest, RoundRobinReresolve) { CheckRpcSendOk(); } // Kill all servers + gpr_log(GPR_INFO, "****** ABOUT TO KILL SERVERS *******"); for (size_t i = 0; i < servers_.size(); ++i) { servers_[i]->Shutdown(false); } - // Client request should fail. - CheckRpcSendFailure(); + gpr_log(GPR_INFO, "****** SERVERS KILLED *******"); + gpr_log(GPR_INFO, "****** SENDING DOOMED REQUESTS *******"); + // Client requests should fail. Send enough to tickle all subchannels. + for (size_t i = 0; i < servers_.size(); ++i) CheckRpcSendFailure(); + gpr_log(GPR_INFO, "****** DOOMED REQUESTS SENT *******"); // Bring servers back up on the same port (we aren't recreating the channel). + gpr_log(GPR_INFO, "****** RESTARTING SERVERS *******"); StartServers(kNumServers, ports); - // Client request should succeed. - CheckRpcSendOk(); + gpr_log(GPR_INFO, "****** SERVERS RESTARTED *******"); + gpr_log(GPR_INFO, "****** SENDING REQUEST TO SUCCEED *******"); + // Client request should eventually (but still fairly soon) succeed. + bool call_succeeded = false; + for (gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5); + gpr_time_cmp(deadline, gpr_now(GPR_CLOCK_MONOTONIC)) > 0;) { + call_succeeded = SendRpc().ok(); + if (call_succeeded) break; + } + GPR_ASSERT(call_succeeded); } } // namespace -- cgit v1.2.3 From 4d9f7f6cf9600e09d5d6aad920d94204a3a3ef6a Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 6 Dec 2017 16:45:49 -0800 Subject: PR comments --- test/cpp/end2end/client_lb_end2end_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index c09a324bf1..c25f65d37a 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -588,13 +588,13 @@ TEST_F(ClientLbEnd2endTest, RoundRobinReresolve) { gpr_log(GPR_INFO, "****** SERVERS RESTARTED *******"); gpr_log(GPR_INFO, "****** SENDING REQUEST TO SUCCEED *******"); // Client request should eventually (but still fairly soon) succeed. - bool call_succeeded = false; - for (gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5); - gpr_time_cmp(deadline, gpr_now(GPR_CLOCK_MONOTONIC)) > 0;) { - call_succeeded = SendRpc().ok(); - if (call_succeeded) break; + const gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + while (gpr_time_cmp(deadline, now) > 0) { + if (SendRpc().ok()) break; + now = gpr_now(GPR_CLOCK_MONOTONIC); } - GPR_ASSERT(call_succeeded); + GPR_ASSERT(gpr_time_cmp(deadline, now) > 0); } } // namespace -- cgit v1.2.3