Merge pull request #17435 from yang-g/health_service_shutdown

Add a Shutdown call to HealthCheckServiceInterface

4 files changed, 261 insertions, 56 deletions

diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD
index 446804401a..eb600ffb17 100644
--- a/test/cpp/end2end/BUILD
+++ b/test/cpp/end2end/BUILD
@@ -36,6 +36,18 @@ grpc_cc_library(
 )
 
 grpc_cc_library(
+    name = "test_health_check_service_impl",
+    testonly = True,
+    srcs = ["test_health_check_service_impl.cc"],
+    hdrs = ["test_health_check_service_impl.h"],
+    deps = [
+        "//:grpc",
+        "//:grpc++",
+        "//src/proto/grpc/health/v1:health_proto",
+    ],
+)
+
+grpc_cc_library(
     name = "interceptors_util",
     testonly = True,
     srcs = ["interceptors_util.cc"],
@@ -283,6 +295,7 @@ grpc_cc_test(
         "gtest",
     ],
     deps = [
+        ":test_health_check_service_impl",
         ":test_service_impl",
         "//:gpr",
         "//:grpc",
diff --git a/test/cpp/end2end/health_service_end2end_test.cc b/test/cpp/end2end/health_service_end2end_test.cc
index 89c4bef09c..b96ff53a3e 100644
--- a/test/cpp/end2end/health_service_end2end_test.cc
+++ b/test/cpp/end2end/health_service_end2end_test.cc
@@ -37,6 +37,7 @@
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
+#include "test/cpp/end2end/test_health_check_service_impl.h"
 #include "test/cpp/end2end/test_service_impl.h"
 
 #include <gtest/gtest.h>
@@ -49,62 +50,6 @@ namespace grpc {
 namespace testing {
 namespace {
 
-// A sample sync implementation of the health checking service. This does the
-// same thing as the default one.
-class HealthCheckServiceImpl : public ::grpc::health::v1::Health::Service {
- public:
-  Status Check(ServerContext* context, const HealthCheckRequest* request,
-               HealthCheckResponse* response) override {
-    std::lock_guard<std::mutex> lock(mu_);
-    auto iter = status_map_.find(request->service());
-    if (iter == status_map_.end()) {
-      return Status(StatusCode::NOT_FOUND, "");
-    }
-    response->set_status(iter->second);
-    return Status::OK;
-  }
-
-  Status Watch(ServerContext* context, const HealthCheckRequest* request,
-               ::grpc::ServerWriter<HealthCheckResponse>* writer) override {
-    auto last_state = HealthCheckResponse::UNKNOWN;
-    while (!context->IsCancelled()) {
-      {
-        std::lock_guard<std::mutex> lock(mu_);
-        HealthCheckResponse response;
-        auto iter = status_map_.find(request->service());
-        if (iter == status_map_.end()) {
-          response.set_status(response.SERVICE_UNKNOWN);
-        } else {
-          response.set_status(iter->second);
-        }
-        if (response.status() != last_state) {
-          writer->Write(response, ::grpc::WriteOptions());
-        }
-      }
-      gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
-                                   gpr_time_from_millis(1000, GPR_TIMESPAN)));
-    }
-    return Status::OK;
-  }
-
-  void SetStatus(const grpc::string& service_name,
-                 HealthCheckResponse::ServingStatus status) {
-    std::lock_guard<std::mutex> lock(mu_);
-    status_map_[service_name] = status;
-  }
-
-  void SetAll(HealthCheckResponse::ServingStatus status) {
-    std::lock_guard<std::mutex> lock(mu_);
-    for (auto iter = status_map_.begin(); iter != status_map_.end(); ++iter) {
-      iter->second = status;
-    }
-  }
-
- private:
-  std::mutex mu_;
-  std::map<const grpc::string, HealthCheckResponse::ServingStatus> status_map_;
-};
-
 // A custom implementation of the health checking service interface. This is
 // used to test that it prevents the server from creating a default service and
 // also serves as an example of how to override the default service.
@@ -125,6 +70,8 @@ class CustomHealthCheckService : public HealthCheckServiceInterface {
                           : HealthCheckResponse::NOT_SERVING);
   }
 
+  void Shutdown() override { impl_->Shutdown(); }
+
  private:
   HealthCheckServiceImpl* impl_;  // not owned
 };
@@ -260,6 +207,74 @@ class HealthServiceEnd2endTest : public ::testing::Test {
     context.TryCancel();
   }
 
+  // Verify that after HealthCheckServiceInterface::Shutdown is called
+  // 1. unary client will see NOT_SERVING.
+  // 2. unary client still sees NOT_SERVING after a SetServing(true) is called.
+  // 3. streaming (Watch) client will see an update.
+  // 4. setting a new service to serving after shutdown will add the service
+  // name but return NOT_SERVING to client.
+  // This has to be called last.
+  void VerifyHealthCheckServiceShutdown() {
+    HealthCheckServiceInterface* service = server_->GetHealthCheckService();
+    EXPECT_TRUE(service != nullptr);
+    const grpc::string kHealthyService("healthy_service");
+    const grpc::string kUnhealthyService("unhealthy_service");
+    const grpc::string kNotRegisteredService("not_registered");
+    const grpc::string kNewService("add_after_shutdown");
+    service->SetServingStatus(kHealthyService, true);
+    service->SetServingStatus(kUnhealthyService, false);
+
+    ResetStubs();
+
+    // Start Watch for service.
+    ClientContext context;
+    HealthCheckRequest request;
+    request.set_service(kHealthyService);
+    std::unique_ptr<::grpc::ClientReaderInterface<HealthCheckResponse>> reader =
+        hc_stub_->Watch(&context, request);
+
+    HealthCheckResponse response;
+    EXPECT_TRUE(reader->Read(&response));
+    EXPECT_EQ(response.SERVING, response.status());
+
+    SendHealthCheckRpc("", Status::OK, HealthCheckResponse::SERVING);
+    SendHealthCheckRpc(kHealthyService, Status::OK,
+                       HealthCheckResponse::SERVING);
+    SendHealthCheckRpc(kUnhealthyService, Status::OK,
+                       HealthCheckResponse::NOT_SERVING);
+    SendHealthCheckRpc(kNotRegisteredService,
+                       Status(StatusCode::NOT_FOUND, ""));
+    SendHealthCheckRpc(kNewService, Status(StatusCode::NOT_FOUND, ""));
+
+    // Shutdown health check service.
+    service->Shutdown();
+
+    // Watch client gets another update.
+    EXPECT_TRUE(reader->Read(&response));
+    EXPECT_EQ(response.NOT_SERVING, response.status());
+    // Finish Watch call.
+    context.TryCancel();
+
+    SendHealthCheckRpc("", Status::OK, HealthCheckResponse::NOT_SERVING);
+    SendHealthCheckRpc(kHealthyService, Status::OK,
+                       HealthCheckResponse::NOT_SERVING);
+    SendHealthCheckRpc(kUnhealthyService, Status::OK,
+                       HealthCheckResponse::NOT_SERVING);
+    SendHealthCheckRpc(kNotRegisteredService,
+                       Status(StatusCode::NOT_FOUND, ""));
+
+    // Setting status after Shutdown has no effect.
+    service->SetServingStatus(kHealthyService, true);
+    SendHealthCheckRpc(kHealthyService, Status::OK,
+                       HealthCheckResponse::NOT_SERVING);
+
+    // Adding serving status for a new service after shutdown will return
+    // NOT_SERVING.
+    service->SetServingStatus(kNewService, true);
+    SendHealthCheckRpc(kNewService, Status::OK,
+                       HealthCheckResponse::NOT_SERVING);
+  }
+
   TestServiceImpl echo_test_service_;
   HealthCheckServiceImpl health_check_service_impl_;
   std::unique_ptr<Health::Stub> hc_stub_;
@@ -295,6 +310,13 @@ TEST_F(HealthServiceEnd2endTest, DefaultHealthService) {
                      Status(StatusCode::INVALID_ARGUMENT, ""));
 }
 
+TEST_F(HealthServiceEnd2endTest, DefaultHealthServiceShutdown) {
+  EnableDefaultHealthCheckService(true);
+  EXPECT_TRUE(DefaultHealthCheckServiceEnabled());
+  SetUpServer(true, false, false, nullptr);
+  VerifyHealthCheckServiceShutdown();
+}
+
 // Provide an empty service to disable the default service.
 TEST_F(HealthServiceEnd2endTest, ExplicitlyDisableViaOverride) {
   EnableDefaultHealthCheckService(true);
@@ -326,6 +348,21 @@ TEST_F(HealthServiceEnd2endTest, ExplicitlyOverride) {
   VerifyHealthCheckServiceStreaming();
 }
 
+TEST_F(HealthServiceEnd2endTest, ExplicitlyHealthServiceShutdown) {
+  EnableDefaultHealthCheckService(true);
+  EXPECT_TRUE(DefaultHealthCheckServiceEnabled());
+  std::unique_ptr<HealthCheckServiceInterface> override_service(
+      new CustomHealthCheckService(&health_check_service_impl_));
+  HealthCheckServiceInterface* underlying_service = override_service.get();
+  SetUpServer(false, false, true, std::move(override_service));
+  HealthCheckServiceInterface* service = server_->GetHealthCheckService();
+  EXPECT_TRUE(service == underlying_service);
+
+  ResetStubs();
+
+  VerifyHealthCheckServiceShutdown();
+}
+
 }  // namespace
 }  // namespace testing
 }  // namespace grpc
diff --git a/test/cpp/end2end/test_health_check_service_impl.cc b/test/cpp/end2end/test_health_check_service_impl.cc
new file mode 100644
index 0000000000..0801e30199
--- /dev/null
+++ b/test/cpp/end2end/test_health_check_service_impl.cc
@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test/cpp/end2end/test_health_check_service_impl.h"
+
+#include <grpc/grpc.h>
+
+using grpc::health::v1::HealthCheckRequest;
+using grpc::health::v1::HealthCheckResponse;
+
+namespace grpc {
+namespace testing {
+
+Status HealthCheckServiceImpl::Check(ServerContext* context,
+                                     const HealthCheckRequest* request,
+                                     HealthCheckResponse* response) {
+  std::lock_guard<std::mutex> lock(mu_);
+  auto iter = status_map_.find(request->service());
+  if (iter == status_map_.end()) {
+    return Status(StatusCode::NOT_FOUND, "");
+  }
+  response->set_status(iter->second);
+  return Status::OK;
+}
+
+Status HealthCheckServiceImpl::Watch(
+    ServerContext* context, const HealthCheckRequest* request,
+    ::grpc::ServerWriter<HealthCheckResponse>* writer) {
+  auto last_state = HealthCheckResponse::UNKNOWN;
+  while (!context->IsCancelled()) {
+    {
+      std::lock_guard<std::mutex> lock(mu_);
+      HealthCheckResponse response;
+      auto iter = status_map_.find(request->service());
+      if (iter == status_map_.end()) {
+        response.set_status(response.SERVICE_UNKNOWN);
+      } else {
+        response.set_status(iter->second);
+      }
+      if (response.status() != last_state) {
+        writer->Write(response, ::grpc::WriteOptions());
+      }
+    }
+    gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                                 gpr_time_from_millis(1000, GPR_TIMESPAN)));
+  }
+  return Status::OK;
+}
+
+void HealthCheckServiceImpl::SetStatus(
+    const grpc::string& service_name,
+    HealthCheckResponse::ServingStatus status) {
+  std::lock_guard<std::mutex> lock(mu_);
+  if (shutdown_) {
+    status = HealthCheckResponse::NOT_SERVING;
+  }
+  status_map_[service_name] = status;
+}
+
+void HealthCheckServiceImpl::SetAll(HealthCheckResponse::ServingStatus status) {
+  std::lock_guard<std::mutex> lock(mu_);
+  if (shutdown_) {
+    return;
+  }
+  for (auto iter = status_map_.begin(); iter != status_map_.end(); ++iter) {
+    iter->second = status;
+  }
+}
+
+void HealthCheckServiceImpl::Shutdown() {
+  std::lock_guard<std::mutex> lock(mu_);
+  if (shutdown_) {
+    return;
+  }
+  shutdown_ = true;
+  for (auto iter = status_map_.begin(); iter != status_map_.end(); ++iter) {
+    iter->second = HealthCheckResponse::NOT_SERVING;
+  }
+}
+
+}  // namespace testing
+}  // namespace grpc
diff --git a/test/cpp/end2end/test_health_check_service_impl.h b/test/cpp/end2end/test_health_check_service_impl.h
new file mode 100644
index 0000000000..5d36ce5320
--- /dev/null
+++ b/test/cpp/end2end/test_health_check_service_impl.h
@@ -0,0 +1,58 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#ifndef GRPC_TEST_CPP_END2END_TEST_HEALTH_CHECK_SERVICE_IMPL_H
+#define GRPC_TEST_CPP_END2END_TEST_HEALTH_CHECK_SERVICE_IMPL_H
+
+#include <map>
+#include <mutex>
+
+#include <grpcpp/server_context.h>
+#include <grpcpp/support/status.h>
+
+#include "src/proto/grpc/health/v1/health.grpc.pb.h"
+
+namespace grpc {
+namespace testing {
+
+// A sample sync implementation of the health checking service. This does the
+// same thing as the default one.
+class HealthCheckServiceImpl : public health::v1::Health::Service {
+ public:
+  Status Check(ServerContext* context,
+               const health::v1::HealthCheckRequest* request,
+               health::v1::HealthCheckResponse* response) override;
+  Status Watch(ServerContext* context,
+               const health::v1::HealthCheckRequest* request,
+               ServerWriter<health::v1::HealthCheckResponse>* writer) override;
+  void SetStatus(const grpc::string& service_name,
+                 health::v1::HealthCheckResponse::ServingStatus status);
+  void SetAll(health::v1::HealthCheckResponse::ServingStatus status);
+
+  void Shutdown();
+
+ private:
+  std::mutex mu_;
+  bool shutdown_ = false;
+  std::map<const grpc::string, health::v1::HealthCheckResponse::ServingStatus>
+      status_map_;
+};
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif  // GRPC_TEST_CPP_END2END_TEST_HEALTH_CHECK_SERVICE_IMPL_H