Backends must be up before some grpclb tests start.

1 files changed, 69 insertions, 18 deletions

diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc
index 570a3d1067..6896b54f1b 100644
--- a/test/cpp/end2end/grpclb_end2end_test.cc
+++ b/test/cpp/end2end/grpclb_end2end_test.cc
@@ -398,11 +398,40 @@ class GrpclbEnd2endTest : public ::testing::Test {
     return true;
   }
 
-  void WaitForAllBackends() {
+  void SendRpcAndCount(int* num_total, int* num_ok, int* num_failure,
+                       int* num_drops) {
+    const Status status = SendRpc();
+    if (status.ok()) {
+      ++*num_ok;
+    } else {
+      if (status.error_message() == "Call dropped by load balancing policy") {
+        ++*num_drops;
+      } else {
+        ++*num_failure;
+      }
+    }
+    ++*num_total;
+  }
+
+  std::tuple<int, int, int> WaitForAllBackends(
+      int num_requests_multiple_of = 1) {
+    int num_ok = 0;
+    int num_failure = 0;
+    int num_drops = 0;
+    int num_total = 0;
     while (!SeenAllBackends()) {
-      CheckRpcSendOk();
+      SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops);
+    }
+    while (num_total % num_requests_multiple_of != 0) {
+      SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops);
     }
     ResetBackendCounters();
+    gpr_log(GPR_INFO,
+            "Performed %d warm up requests (a multiple of %d) against the "
+            "backends. %d succeeded, %d failed, %d dropped.",
+            num_total, num_requests_multiple_of, num_ok, num_failure,
+            num_drops);
+    return std::make_tuple(num_ok, num_failure, num_drops);
   }
 
   void WaitForBackend(size_t backend_idx) {
@@ -556,10 +585,8 @@ TEST_F(SingleBalancerTest, Vanilla) {
       0);
   // Make sure that trying to connect works without a call.
   channel_->GetState(true /* try_to_connect */);
-
   // We need to wait for all backends to come online.
   WaitForAllBackends();
-
   // Send kNumRpcsPerAddress RPCs per server.
   CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
 
@@ -867,6 +894,8 @@ TEST_F(SingleBalancerTest, Drop) {
       0, BalancerServiceImpl::BuildResponseForBackends(
              GetBackendPorts(), {{"rate_limiting", 1}, {"load_balancing", 2}}),
       0);
+  // Wait until all backends are ready.
+  WaitForAllBackends();
   // Send kNumRpcsPerAddress RPCs for each server and drop address.
   size_t num_drops = 0;
   for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) {
@@ -936,6 +965,11 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) {
   ScheduleResponseForBalancer(
       0, BalancerServiceImpl::BuildResponseForBackends(GetBackendPorts(), {}),
       0);
+  // Wait until all backends are ready.
+  int num_ok = 0;
+  int num_failure = 0;
+  int num_drops = 0;
+  std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
   // Send kNumRpcsPerAddress RPCs per server.
   CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
   // Each backend should have gotten 100 requests.
@@ -950,11 +984,12 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Vanilla) {
   EXPECT_EQ(1U, balancer_servers_[0].service_->response_count());
 
   const ClientStats client_stats = WaitForLoadReports();
-  EXPECT_EQ(kNumRpcsPerAddress * num_backends_, client_stats.num_calls_started);
-  EXPECT_EQ(kNumRpcsPerAddress * num_backends_,
+  EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
+            client_stats.num_calls_started);
+  EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
             client_stats.num_calls_finished);
   EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send);
-  EXPECT_EQ(kNumRpcsPerAddress * num_backends_,
+  EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + (num_ok + num_drops),
             client_stats.num_calls_finished_known_received);
   EXPECT_THAT(client_stats.drop_token_counts, ::testing::ElementsAre());
 }
@@ -965,7 +1000,14 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) {
       0, BalancerServiceImpl::BuildResponseForBackends(
              GetBackendPorts(), {{"rate_limiting", 2}, {"load_balancing", 1}}),
       0);
-
+  // Wait until all backends are ready.
+  int num_warmup_ok = 0;
+  int num_warmup_failure = 0;
+  int num_warmup_drops = 0;
+  std::tie(num_warmup_ok, num_warmup_failure, num_warmup_drops) =
+      WaitForAllBackends(num_backends_ + 3 /* num_requests_multiple_of */);
+  const int num_total_warmup_requests =
+      num_warmup_ok + num_warmup_failure + num_warmup_drops;
   size_t num_drops = 0;
   for (size_t i = 0; i < kNumRpcsPerAddress * (num_backends_ + 3); ++i) {
     EchoResponse response;
@@ -980,7 +1022,6 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) {
     }
   }
   EXPECT_EQ(kNumRpcsPerAddress * 3, num_drops);
-
   // Each backend should have gotten 100 requests.
   for (size_t i = 0; i < backends_.size(); ++i) {
     EXPECT_EQ(kNumRpcsPerAddress,
@@ -993,17 +1034,27 @@ TEST_F(SingleBalancerWithClientLoadReportingTest, Drop) {
   EXPECT_EQ(1U, balancer_servers_[0].service_->response_count());
 
   const ClientStats client_stats = WaitForLoadReports();
-  EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3),
-            client_stats.num_calls_started);
-  EXPECT_EQ(kNumRpcsPerAddress * (num_backends_ + 3),
-            client_stats.num_calls_finished);
+  EXPECT_EQ(
+      kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests,
+      client_stats.num_calls_started);
+  EXPECT_EQ(
+      kNumRpcsPerAddress * (num_backends_ + 3) + num_total_warmup_requests,
+      client_stats.num_calls_finished);
   EXPECT_EQ(0U, client_stats.num_calls_finished_with_client_failed_to_send);
-  EXPECT_EQ(kNumRpcsPerAddress * num_backends_,
+  EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_warmup_ok,
             client_stats.num_calls_finished_known_received);
-  EXPECT_THAT(client_stats.drop_token_counts,
-              ::testing::ElementsAre(
-                  ::testing::Pair("load_balancing", kNumRpcsPerAddress),
-                  ::testing::Pair("rate_limiting", kNumRpcsPerAddress * 2)));
+  // The number of warmup request is a multiple of the number of addresses.
+  // Therefore, all addresses in the scheduled balancer response are hit the
+  // same number of times.
+  const int num_times_drop_addresses_hit = num_warmup_drops / 3;
+  EXPECT_THAT(
+      client_stats.drop_token_counts,
+      ::testing::ElementsAre(
+          ::testing::Pair("load_balancing",
+                          (kNumRpcsPerAddress + num_times_drop_addresses_hit)),
+          ::testing::Pair(
+              "rate_limiting",
+              (kNumRpcsPerAddress + num_times_drop_addresses_hit) * 2)));
 }
 
 }  // namespace