Merge remote-tracking branch 'upstream/master' into fix_gcc_4_4

25 files changed, 163 insertions, 137 deletions

diff --git a/examples/pubsub/main.cc b/examples/pubsub/main.cc
index 6f7737e247..cc5076f0a5 100644
--- a/examples/pubsub/main.cc
+++ b/examples/pubsub/main.cc
@@ -51,14 +51,14 @@
 #include "examples/pubsub/subscriber.h"
 
 DEFINE_int32(server_port, 443, "Server port.");
-DEFINE_string(server_host,
-              "pubsub-staging.googleapis.com", "Server host to connect to");
+DEFINE_string(server_host, "pubsub-staging.googleapis.com",
+              "Server host to connect to");
 DEFINE_string(project_id, "", "GCE project id such as stoked-keyword-656");
 
 // In some distros, gflags is in the namespace google, and in some others,
 // in gflags. This hack is enabling us to find both.
-namespace google { }
-namespace gflags { }
+namespace google {}
+namespace gflags {}
 using namespace google;
 using namespace gflags;
 
@@ -92,32 +92,32 @@ int main(int argc, char** argv) {
   grpc::string topic = ss.str();
 
   ss.str("");
-  ss << FLAGS_project_id << "/"  << kSubscriptionName;
+  ss << FLAGS_project_id << "/" << kSubscriptionName;
   grpc::string subscription_name = ss.str();
 
   // Clean up test topic and subcription if they exist before.
   grpc::string subscription_topic;
-  if (subscriber.GetSubscription(
-      subscription_name, &subscription_topic).IsOk()) {
+  if (subscriber.GetSubscription(subscription_name, &subscription_topic)
+          .IsOk()) {
     subscriber.DeleteSubscription(subscription_name);
   }
 
   if (publisher.GetTopic(topic).IsOk()) publisher.DeleteTopic(topic);
 
   grpc::Status s = publisher.CreateTopic(topic);
-  gpr_log(GPR_INFO, "Create topic returns code %d, %s",
-          s.code(), s.details().c_str());
+  gpr_log(GPR_INFO, "Create topic returns code %d, %s", s.code(),
+          s.details().c_str());
   GPR_ASSERT(s.IsOk());
 
   s = publisher.GetTopic(topic);
-  gpr_log(GPR_INFO, "Get topic returns code %d, %s",
-          s.code(), s.details().c_str());
+  gpr_log(GPR_INFO, "Get topic returns code %d, %s", s.code(),
+          s.details().c_str());
   GPR_ASSERT(s.IsOk());
 
   std::vector<grpc::string> topics;
   s = publisher.ListTopics(FLAGS_project_id, &topics);
-  gpr_log(GPR_INFO, "List topic returns code %d, %s",
-          s.code(), s.details().c_str());
+  gpr_log(GPR_INFO, "List topic returns code %d, %s", s.code(),
+          s.details().c_str());
   bool topic_found = false;
   for (unsigned int i = 0; i < topics.size(); i++) {
     if (topics[i] == topic) topic_found = true;
@@ -127,27 +127,27 @@ int main(int argc, char** argv) {
   GPR_ASSERT(topic_found);
 
   s = subscriber.CreateSubscription(topic, subscription_name);
-  gpr_log(GPR_INFO, "create subscrption returns code %d, %s",
-          s.code(), s.details().c_str());
+  gpr_log(GPR_INFO, "create subscrption returns code %d, %s", s.code(),
+          s.details().c_str());
   GPR_ASSERT(s.IsOk());
 
   s = publisher.Publish(topic, kMessageData);
-  gpr_log(GPR_INFO, "Publish %s returns code %d, %s",
-          kMessageData, s.code(), s.details().c_str());
+  gpr_log(GPR_INFO, "Publish %s returns code %d, %s", kMessageData, s.code(),
+          s.details().c_str());
   GPR_ASSERT(s.IsOk());
 
   grpc::string data;
   s = subscriber.Pull(subscription_name, &data);
   gpr_log(GPR_INFO, "Pull %s", data.c_str());
 
-  s =  subscriber.DeleteSubscription(subscription_name);
-  gpr_log(GPR_INFO, "Delete subscription returns code %d, %s",
-          s.code(), s.details().c_str());
+  s = subscriber.DeleteSubscription(subscription_name);
+  gpr_log(GPR_INFO, "Delete subscription returns code %d, %s", s.code(),
+          s.details().c_str());
   GPR_ASSERT(s.IsOk());
 
   s = publisher.DeleteTopic(topic);
-  gpr_log(GPR_INFO, "Delete topic returns code %d, %s",
-          s.code(), s.details().c_str());
+  gpr_log(GPR_INFO, "Delete topic returns code %d, %s", s.code(),
+          s.details().c_str());
   GPR_ASSERT(s.IsOk());
 
   subscriber.Shutdown();
diff --git a/examples/pubsub/publisher.cc b/examples/pubsub/publisher.cc
index 308f9a77e5..458050af73 100644
--- a/examples/pubsub/publisher.cc
+++ b/examples/pubsub/publisher.cc
@@ -51,12 +51,9 @@ namespace examples {
 namespace pubsub {
 
 Publisher::Publisher(std::shared_ptr<ChannelInterface> channel)
-    : stub_(PublisherService::NewStub(channel)) {
-}
+    : stub_(PublisherService::NewStub(channel)) {}
 
-void Publisher::Shutdown() {
-  stub_.reset();
-}
+void Publisher::Shutdown() { stub_.reset(); }
 
 Status Publisher::CreateTopic(const grpc::string& topic) {
   Topic request;
diff --git a/examples/pubsub/publisher_test.cc b/examples/pubsub/publisher_test.cc
index 62442c7384..ac4921283f 100644
--- a/examples/pubsub/publisher_test.cc
+++ b/examples/pubsub/publisher_test.cc
@@ -31,8 +31,6 @@
  *
  */
 
-#include <google/protobuf/stubs/common.h>
-
 #include <grpc++/channel_arguments.h>
 #include <grpc++/channel_interface.h>
 #include <grpc++/client_context.h>
@@ -84,20 +82,19 @@ class PublisherServiceImpl : public tech::pubsub::PublisherService::Service {
   Status ListTopics(
       ServerContext* context, const ::tech::pubsub::ListTopicsRequest* request,
       ::tech::pubsub::ListTopicsResponse* response) GRPC_OVERRIDE {
-   std::ostringstream ss;
-   ss << "cloud.googleapis.com/project in (/projects/" << kProjectId << ")";
-   EXPECT_EQ(request->query(), ss.str());
-   response->add_topic()->set_name(kTopic);
-   return Status::OK;
- }
-
- Status DeleteTopic(ServerContext* context,
-                    const ::tech::pubsub::DeleteTopicRequest* request,
-                    ::proto2::Empty* response) GRPC_OVERRIDE {
-    EXPECT_EQ(request->topic(), kTopic);
+    std::ostringstream ss;
+    ss << "cloud.googleapis.com/project in (/projects/" << kProjectId << ")";
+    EXPECT_EQ(request->query(), ss.str());
+    response->add_topic()->set_name(kTopic);
     return Status::OK;
- }
+  }
 
+  Status DeleteTopic(ServerContext* context,
+                     const ::tech::pubsub::DeleteTopicRequest* request,
+                     ::proto2::Empty* response) GRPC_OVERRIDE {
+    EXPECT_EQ(request->topic(), kTopic);
+    return Status::OK;
+  }
 };
 
 class PublisherTest : public ::testing::Test {
@@ -107,11 +104,13 @@ class PublisherTest : public ::testing::Test {
     int port = grpc_pick_unused_port_or_die();
     server_address_ << "localhost:" << port;
     ServerBuilder builder;
-    builder.AddListeningPort(server_address_.str(), grpc::InsecureServerCredentials());
+    builder.AddListeningPort(server_address_.str(),
+                             grpc::InsecureServerCredentials());
     builder.RegisterService(&service_);
     server_ = builder.BuildAndStart();
 
-    channel_ = CreateChannel(server_address_.str(), grpc::InsecureCredentials(), ChannelArguments());
+    channel_ = CreateChannel(server_address_.str(), grpc::InsecureCredentials(),
+                             ChannelArguments());
 
     publisher_.reset(new grpc::examples::pubsub::Publisher(channel_));
   }
diff --git a/examples/pubsub/subscriber.cc b/examples/pubsub/subscriber.cc
index 29f6635b7c..d9e0292ba0 100644
--- a/examples/pubsub/subscriber.cc
+++ b/examples/pubsub/subscriber.cc
@@ -49,12 +49,9 @@ namespace examples {
 namespace pubsub {
 
 Subscriber::Subscriber(std::shared_ptr<ChannelInterface> channel)
-    : stub_(SubscriberService::NewStub(channel)) {
-}
+    : stub_(SubscriberService::NewStub(channel)) {}
 
-void Subscriber::Shutdown() {
-  stub_.reset();
-}
+void Subscriber::Shutdown() { stub_.reset(); }
 
 Status Subscriber::CreateSubscription(const grpc::string& topic,
                                       const grpc::string& name) {
diff --git a/examples/pubsub/subscriber_test.cc b/examples/pubsub/subscriber_test.cc
index b8dd1f9486..9ab60ed6a7 100644
--- a/examples/pubsub/subscriber_test.cc
+++ b/examples/pubsub/subscriber_test.cc
@@ -31,8 +31,6 @@
  *
  */
 
-#include <google/protobuf/stubs/common.h>
-
 #include <grpc++/channel_arguments.h>
 #include <grpc++/channel_interface.h>
 #include <grpc++/client_context.h>
@@ -95,7 +93,6 @@ class SubscriberServiceImpl : public tech::pubsub::SubscriberService::Service {
                      proto2::Empty* response) GRPC_OVERRIDE {
     return Status::OK;
   }
-
 };
 
 class SubscriberTest : public ::testing::Test {
@@ -105,11 +102,13 @@ class SubscriberTest : public ::testing::Test {
     int port = grpc_pick_unused_port_or_die();
     server_address_ << "localhost:" << port;
     ServerBuilder builder;
-    builder.AddListeningPort(server_address_.str(), grpc::InsecureServerCredentials());
+    builder.AddListeningPort(server_address_.str(),
+                             grpc::InsecureServerCredentials());
     builder.RegisterService(&service_);
     server_ = builder.BuildAndStart();
 
-    channel_ = CreateChannel(server_address_.str(), grpc::InsecureCredentials(), ChannelArguments());
+    channel_ = CreateChannel(server_address_.str(), grpc::InsecureCredentials(),
+                             ChannelArguments());
 
     subscriber_.reset(new grpc::examples::pubsub::Subscriber(channel_));
   }
@@ -129,17 +128,15 @@ class SubscriberTest : public ::testing::Test {
 };
 
 TEST_F(SubscriberTest, TestSubscriber) {
-  EXPECT_TRUE(subscriber_->CreateSubscription(kTopic,
-                                              kSubscriptionName).IsOk());
+  EXPECT_TRUE(
+      subscriber_->CreateSubscription(kTopic, kSubscriptionName).IsOk());
 
   grpc::string topic;
-  EXPECT_TRUE(subscriber_->GetSubscription(kSubscriptionName,
-                                           &topic).IsOk());
+  EXPECT_TRUE(subscriber_->GetSubscription(kSubscriptionName, &topic).IsOk());
   EXPECT_EQ(topic, kTopic);
 
   grpc::string data;
-  EXPECT_TRUE(subscriber_->Pull(kSubscriptionName,
-                                &data).IsOk());
+  EXPECT_TRUE(subscriber_->Pull(kSubscriptionName, &data).IsOk());
 
   EXPECT_TRUE(subscriber_->DeleteSubscription(kSubscriptionName).IsOk());
 }
diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc
index c2d4cda31a..748417e477 100644
--- a/src/compiler/python_generator.cc
+++ b/src/compiler/python_generator.cc
@@ -309,17 +309,20 @@ bool PrintServerFactory(const grpc::string& package_qualified_service_name,
           make_pair(method->name(), output_message_module_and_class));
     }
     out->Print("method_service_descriptions = {\n");
-    for (auto& name_and_description_constructor :
-         method_description_constructors) {
+    for (auto name_and_description_constructor =
+	   method_description_constructors.begin();
+	 name_and_description_constructor !=
+	   method_description_constructors.end();
+	 name_and_description_constructor++) {
       IndentScope raii_descriptions_indent(out);
-      const grpc::string method_name = name_and_description_constructor.first;
+      const grpc::string method_name = name_and_description_constructor->first;
       auto input_message_module_and_class =
           input_message_modules_and_classes.find(method_name);
       auto output_message_module_and_class =
           output_message_modules_and_classes.find(method_name);
       out->Print("\"$Method$\": utilities.$Constructor$(\n", "Method",
                  method_name, "Constructor",
-                 name_and_description_constructor.second);
+                 name_and_description_constructor->second);
       {
         IndentScope raii_description_arguments_indent(out);
         out->Print("servicer.$Method$,\n", "Method", method_name);
@@ -387,17 +390,20 @@ bool PrintStubFactory(const grpc::string& package_qualified_service_name,
           make_pair(method->name(), output_message_module_and_class));
     }
     out->Print("method_invocation_descriptions = {\n");
-    for (auto& name_and_description_constructor :
-         method_description_constructors) {
+    for (auto name_and_description_constructor =
+	   method_description_constructors.begin();
+	 name_and_description_constructor !=
+	   method_description_constructors.end();
+	 name_and_description_constructor++) {
       IndentScope raii_descriptions_indent(out);
-      const grpc::string method_name = name_and_description_constructor.first;
+      const grpc::string method_name = name_and_description_constructor->first;
       auto input_message_module_and_class =
           input_message_modules_and_classes.find(method_name);
       auto output_message_module_and_class =
           output_message_modules_and_classes.find(method_name);
       out->Print("\"$Method$\": utilities.$Constructor$(\n", "Method",
                  method_name, "Constructor",
-                 name_and_description_constructor.second);
+                 name_and_description_constructor->second);
       {
         IndentScope raii_description_arguments_indent(out);
         out->Print(
diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c
index 8b019e8049..aec626509a 100644
--- a/src/core/iomgr/iocp_windows.c
+++ b/src/core/iomgr/iocp_windows.c
@@ -52,10 +52,11 @@ static OVERLAPPED g_iocp_custom_overlap;
 
 static gpr_event g_shutdown_iocp;
 static gpr_event g_iocp_done;
+static gpr_atm g_orphans = 0;
 
 static HANDLE g_iocp;
 
-static int do_iocp_work() {
+static void do_iocp_work() {
   BOOL success;
   DWORD bytes = 0;
   DWORD flags = 0;
@@ -71,14 +72,14 @@ static int do_iocp_work() {
                                       gpr_time_to_millis(wait_time));
   if (!success && !overlapped) {
     /* The deadline got attained. */
-    return 0;
+    return;
   }
   GPR_ASSERT(completion_key && overlapped);
   if (overlapped == &g_iocp_custom_overlap) {
     if (completion_key == (ULONG_PTR) &g_iocp_kick_token) {
       /* We were awoken from a kick. */
       gpr_log(GPR_DEBUG, "do_iocp_work - got a kick");
-      return 1;
+      return;
     }
     gpr_log(GPR_ERROR, "Unknown custom completion key.");
     abort();
@@ -97,8 +98,13 @@ static int do_iocp_work() {
   }
   success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes,
                                    FALSE, &flags);
-  gpr_log(GPR_DEBUG, "bytes: %u, flags: %u - op %s", bytes, flags,
-          success ? "succeeded" : "failed");
+  gpr_log(GPR_DEBUG, "bytes: %u, flags: %u - op %s %s", bytes, flags,
+          success ? "succeeded" : "failed", socket->orphan ? "orphan" : "");
+  if (socket->orphan) {
+    grpc_winsocket_destroy(socket);
+    gpr_atm_full_fetch_add(&g_orphans, -1);
+    return;
+  }
   info->bytes_transfered = bytes;
   info->wsa_error = success ? 0 : WSAGetLastError();
   GPR_ASSERT(overlapped == &info->overlapped);
@@ -113,12 +119,10 @@ static int do_iocp_work() {
   }
   gpr_mu_unlock(&socket->state_mu);
   if (f) f(opaque, 1);
-
-  return 1;
 }
 
 static void iocp_loop(void *p) {
-  while (!gpr_event_get(&g_shutdown_iocp)) {
+  while (gpr_atm_acq_load(&g_orphans) || !gpr_event_get(&g_shutdown_iocp)) {
     grpc_maybe_call_delayed_callbacks(NULL, 1);
     do_iocp_work();
   }
@@ -138,13 +142,19 @@ void grpc_iocp_init(void) {
   gpr_thd_new(&id, iocp_loop, NULL, NULL);
 }
 
-void grpc_iocp_shutdown(void) {
+void grpc_iocp_kick(void) {
   BOOL success;
-  gpr_event_set(&g_shutdown_iocp, (void *)1);
+
   success = PostQueuedCompletionStatus(g_iocp, 0,
                                        (ULONG_PTR) &g_iocp_kick_token,
                                        &g_iocp_custom_overlap);
   GPR_ASSERT(success);
+}
+
+void grpc_iocp_shutdown(void) {
+  BOOL success;
+  gpr_event_set(&g_shutdown_iocp, (void *)1);
+  grpc_iocp_kick();
   gpr_event_wait(&g_iocp_done, gpr_inf_future);
   success = CloseHandle(g_iocp);
   GPR_ASSERT(success);
@@ -166,6 +176,10 @@ void grpc_iocp_add_socket(grpc_winsocket *socket) {
   GPR_ASSERT(ret == g_iocp);
 }
 
+void grpc_iocp_socket_orphan(grpc_winsocket *socket) {
+  gpr_atm_full_fetch_add(&g_orphans, 1);
+}
+
 static void socket_notify_on_iocp(grpc_winsocket *socket,
                                   void(*cb)(void *, int), void *opaque,
                                   grpc_winsocket_callback_info *info) {
diff --git a/src/core/iomgr/iocp_windows.h b/src/core/iomgr/iocp_windows.h
index 33133193a1..fa3f5eee10 100644
--- a/src/core/iomgr/iocp_windows.h
+++ b/src/core/iomgr/iocp_windows.h
@@ -42,6 +42,7 @@
 void grpc_iocp_init(void);
 void grpc_iocp_shutdown(void);
 void grpc_iocp_add_socket(grpc_winsocket *);
+void grpc_iocp_socket_orphan(grpc_winsocket *);
 
 void grpc_socket_notify_on_write(grpc_winsocket *, void(*cb)(void *, int success),
                                  void *opaque);
diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c
index 058685b295..d0e6706fbd 100644
--- a/src/core/iomgr/iomgr.c
+++ b/src/core/iomgr/iomgr.c
@@ -117,7 +117,16 @@ void grpc_iomgr_shutdown(void) {
       gpr_mu_lock(&g_mu);
     }
     if (g_refs) {
-      if (gpr_cv_wait(&g_rcv, &g_mu, shutdown_deadline) && g_cbs_head == NULL) {
+      int timeout = 0;
+      gpr_timespec short_deadline = gpr_time_add(gpr_now(),
+                                                 gpr_time_from_millis(100));
+      while (gpr_cv_wait(&g_rcv, &g_mu, short_deadline) && g_cbs_head == NULL) {
+        if (gpr_time_cmp(gpr_now(), shutdown_deadline) > 0) {
+          timeout = 1;
+          break;
+        }
+      }
+      if (timeout) {
         gpr_log(GPR_DEBUG,
                 "Failed to free %d iomgr objects before shutdown deadline: "
                 "memory leaks are likely",
diff --git a/src/core/iomgr/socket_windows.c b/src/core/iomgr/socket_windows.c
index 99f38b0e03..22dad41783 100644
--- a/src/core/iomgr/socket_windows.c
+++ b/src/core/iomgr/socket_windows.c
@@ -55,7 +55,7 @@ grpc_winsocket *grpc_winsocket_create(SOCKET socket) {
   return r;
 }
 
-void shutdown_op(grpc_winsocket_callback_info *info) {
+static void shutdown_op(grpc_winsocket_callback_info *info) {
   if (!info->cb) return;
   grpc_iomgr_add_delayed_callback(info->cb, info->opaque, 0);
 }
@@ -68,8 +68,13 @@ void grpc_winsocket_shutdown(grpc_winsocket *socket) {
 
 void grpc_winsocket_orphan(grpc_winsocket *socket) {
   gpr_log(GPR_DEBUG, "grpc_winsocket_orphan");
+  grpc_iocp_socket_orphan(socket);
+  socket->orphan = 1;
   grpc_iomgr_unref();
   closesocket(socket->socket);
+}
+
+void grpc_winsocket_destroy(grpc_winsocket *socket) {
   gpr_mu_destroy(&socket->state_mu);
   gpr_free(socket);
 }
diff --git a/src/core/iomgr/socket_windows.h b/src/core/iomgr/socket_windows.h
index d4776ab10f..cbae91692c 100644
--- a/src/core/iomgr/socket_windows.h
+++ b/src/core/iomgr/socket_windows.h
@@ -57,12 +57,13 @@ typedef struct grpc_winsocket_callback_info {
 typedef struct grpc_winsocket {
   SOCKET socket;
 
-  int added_to_iocp;
-
   grpc_winsocket_callback_info write_info;
   grpc_winsocket_callback_info read_info;
 
   gpr_mu state_mu;
+
+  int added_to_iocp;
+  int orphan;
 } grpc_winsocket;
 
 /* Create a wrapped windows handle.
@@ -71,5 +72,6 @@ grpc_winsocket *grpc_winsocket_create(SOCKET socket);
 
 void grpc_winsocket_shutdown(grpc_winsocket *socket);
 void grpc_winsocket_orphan(grpc_winsocket *socket);
+void grpc_winsocket_destroy(grpc_winsocket *socket);
 
 #endif  /* GRPC_INTERNAL_CORE_IOMGR_SOCKET_WINDOWS_H */
diff --git a/src/cpp/client/insecure_credentials.cc b/src/cpp/client/insecure_credentials.cc
index f3ca430bd4..8945b038de 100644
--- a/src/cpp/client/insecure_credentials.cc
+++ b/src/cpp/client/insecure_credentials.cc
@@ -31,8 +31,6 @@
  *
  */
 
-#include <string>
-
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
 
diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc
index e3c6637623..d6f9acc675 100644
--- a/src/cpp/client/secure_credentials.cc
+++ b/src/cpp/client/secure_credentials.cc
@@ -31,8 +31,6 @@
  *
  */
 
-#include <string>
-
 #include <grpc/grpc_security.h>
 #include <grpc/support/log.h>
 
diff --git a/src/cpp/common/call.cc b/src/cpp/common/call.cc
index 5c26a1ad7c..311e0d01b0 100644
--- a/src/cpp/common/call.cc
+++ b/src/cpp/common/call.cc
@@ -148,7 +148,7 @@ void FillMetadataMap(grpc_metadata_array* arr,
     // TODO(yangg) handle duplicates?
     metadata->insert(std::pair<grpc::string, grpc::string>(
         arr->metadata[i].key,
-        {arr->metadata[i].value, arr->metadata[i].value_length}));
+        grpc::string(arr->metadata[i].value, arr->metadata[i].value_length)));
   }
   grpc_metadata_array_destroy(arr);
   grpc_metadata_array_init(arr);
diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc
index 88f7a9b1a9..49d69a3fb9 100644
--- a/src/cpp/server/secure_server_credentials.cc
+++ b/src/cpp/server/secure_server_credentials.cc
@@ -59,9 +59,12 @@ class SecureServerCredentials GRPC_FINAL : public ServerCredentials {
 std::shared_ptr<ServerCredentials> SslServerCredentials(
     const SslServerCredentialsOptions& options) {
   std::vector<grpc_ssl_pem_key_cert_pair> pem_key_cert_pairs;
-  for (const auto& key_cert_pair : options.pem_key_cert_pairs) {
-    pem_key_cert_pairs.push_back(
-        {key_cert_pair.private_key.c_str(), key_cert_pair.cert_chain.c_str()});
+  for (auto key_cert_pair = options.pem_key_cert_pairs.begin();
+       key_cert_pair != options.pem_key_cert_pairs.end();
+       key_cert_pair++) {
+    grpc_ssl_pem_key_cert_pair p = {key_cert_pair->private_key.c_str(),
+				    key_cert_pair->cert_chain.c_str()};
+    pem_key_cert_pairs.push_back(p);
   }
   grpc_server_credentials* c_creds = grpc_ssl_server_credentials_create(
       options.pem_root_certs.empty() ? nullptr : options.pem_root_certs.c_str(),
diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc
index 5a4ca6915a..8e6a6cf40a 100644
--- a/src/cpp/server/server.cc
+++ b/src/cpp/server/server.cc
@@ -247,8 +247,8 @@ bool Server::Start() {
 
   // Start processing rpcs.
   if (!sync_methods_.empty()) {
-    for (auto& m : sync_methods_) {
-      m.Request(server_);
+    for (auto m = sync_methods_.begin(); m != sync_methods_.end(); m++) {
+      m->Request(server_);
     }
 
     ScheduleCallback();
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 58bf9d937f..c5e115f396 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -86,24 +86,26 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
     thread_pool_owned = true;
   }
   std::unique_ptr<Server> server(new Server(thread_pool_, thread_pool_owned));
-  for (auto* service : services_) {
-    if (!server->RegisterService(service)) {
+  for (auto service = services_.begin(); service != services_.end();
+       service++) {
+    if (!server->RegisterService(*service)) {
       return nullptr;
     }
   }
-  for (auto* service : async_services_) {
-    if (!server->RegisterAsyncService(service)) {
+  for (auto service = async_services_.begin();
+       service != async_services_.end(); service++) {
+    if (!server->RegisterAsyncService(*service)) {
       return nullptr;
     }
   }
   if (generic_service_) {
     server->RegisterAsyncGenericService(generic_service_);
   }
-  for (auto& port : ports_) {
-    int r = server->AddListeningPort(port.addr, port.creds.get());
+  for (auto port = ports_.begin(); port != ports_.end(); port++) {
+    int r = server->AddListeningPort(port->addr, port->creds.get());
     if (!r) return nullptr;
-    if (port.selected_port != nullptr) {
-      *port.selected_port = r;
+    if (port->selected_port != nullptr) {
+      *port->selected_port = r;
     }
   }
   if (!server->Start()) {
diff --git a/src/cpp/server/thread_pool.cc b/src/cpp/server/thread_pool.cc
index 3eb172a3a6..80c96111b1 100644
--- a/src/cpp/server/thread_pool.cc
+++ b/src/cpp/server/thread_pool.cc
@@ -67,8 +67,8 @@ ThreadPool::~ThreadPool() {
     shutdown_ = true;
     cv_.notify_all();
   }
-  for (auto& t : threads_) {
-    t.join();
+  for (auto t = threads_.begin(); t != threads_.end(); t++) {
+    t->join();
   }
 }
 
diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc
index 4c71831dec..233c5b13a7 100644
--- a/test/cpp/end2end/async_end2end_test.cc
+++ b/test/cpp/end2end/async_end2end_test.cc
@@ -595,6 +595,5 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   int result = RUN_ALL_TESTS();
   grpc_shutdown();
-  google::protobuf::ShutdownProtobufLibrary();
   return result;
 }
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index 41c2669533..e2649c2826 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -432,6 +432,5 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   int result = RUN_ALL_TESTS();
   grpc_shutdown();
-  google::protobuf::ShutdownProtobufLibrary();
   return result;
 }
diff --git a/test/cpp/end2end/generic_end2end_test.cc b/test/cpp/end2end/generic_end2end_test.cc
index 711f1b9540..4692188721 100644
--- a/test/cpp/end2end/generic_end2end_test.cc
+++ b/test/cpp/end2end/generic_end2end_test.cc
@@ -83,8 +83,8 @@ bool ParseFromByteBuffer(ByteBuffer* buffer, grpc::protobuf::Message* message) {
   buffer->Dump(&slices);
   grpc::string buf;
   buf.reserve(buffer->Length());
-  for (const Slice& s : slices) {
-    buf.append(reinterpret_cast<const char*>(s.begin()), s.size());
+  for (auto s = slices.begin(); s != slices.end(); s++) {
+    buf.append(reinterpret_cast<const char*>(s->begin()), s->size());
   }
   return message->ParseFromString(buf);
 }
@@ -265,6 +265,5 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   int result = RUN_ALL_TESTS();
   grpc_shutdown();
-  google::protobuf::ShutdownProtobufLibrary();
   return result;
 }
diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc
index 526f37a1fd..f995a63f49 100644
--- a/test/cpp/qps/client_async.cc
+++ b/test/cpp/qps/client_async.cc
@@ -144,7 +144,8 @@ class AsyncClient GRPC_FINAL : public Client {
 
     int t = 0;
     for (int i = 0; i < config.outstanding_rpcs_per_channel(); i++) {
-      for (auto& channel : channels_) {
+      for (auto channel = channels_.begin(); channel != channels_.end();
+	   channel++) {
         auto* cq = cli_cqs_[t].get();
         t = (t + 1) % cli_cqs_.size();
         auto start_req = [cq](TestService::Stub* stub, grpc::ClientContext* ctx,
@@ -152,7 +153,7 @@ class AsyncClient GRPC_FINAL : public Client {
           return stub->AsyncUnaryCall(ctx, request, cq, tag);
         };
 
-        TestService::Stub* stub = channel.get_stub();
+        TestService::Stub* stub = channel->get_stub();
         const SimpleRequest& request = request_;
         new ClientRpcContextUnaryImpl<SimpleRequest, SimpleResponse>(
             stub, request, start_req, check_done);
@@ -165,11 +166,11 @@ class AsyncClient GRPC_FINAL : public Client {
   ~AsyncClient() GRPC_OVERRIDE {
     EndThreads();
 
-    for (auto& cq : cli_cqs_) {
-      cq->Shutdown();
+    for (auto cq = cli_cqs_.begin(); cq != cli_cqs_.end(); cq++) {
+      (*cq)->Shutdown();
       void* got_tag;
       bool ok;
-      while (cq->Next(&got_tag, &ok)) {
+      while ((*cq)->Next(&got_tag, &ok)) {
         delete ClientRpcContext::detag(got_tag);
       }
     }
diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index d29ca1de94..64a53496ae 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -154,19 +154,19 @@ ScenarioResult RunScenario(const ClientConfig& initial_client_config,
   server_mark.mutable_mark();
   ClientArgs client_mark;
   client_mark.mutable_mark();
-  for (auto& server : servers) {
-    GPR_ASSERT(server.stream->Write(server_mark));
+  for (auto server = servers.begin(); server != servers.end(); server++) {
+    GPR_ASSERT(server->stream->Write(server_mark));
   }
-  for (auto& client : clients) {
-    GPR_ASSERT(client.stream->Write(client_mark));
+  for (auto client = clients.begin(); client != clients.end(); client++) {
+    GPR_ASSERT(client->stream->Write(client_mark));
   }
   ServerStatus server_status;
   ClientStatus client_status;
-  for (auto& server : servers) {
-    GPR_ASSERT(server.stream->Read(&server_status));
+  for (auto server = servers.begin(); server != servers.end(); server++) {
+    GPR_ASSERT(server->stream->Read(&server_status));
   }
-  for (auto& client : clients) {
-    GPR_ASSERT(client.stream->Read(&client_status));
+  for (auto client = clients.begin(); client != clients.end(); client++) {
+    GPR_ASSERT(client->stream->Read(&client_status));
   }
 
   // Wait some time
@@ -176,33 +176,33 @@ ScenarioResult RunScenario(const ClientConfig& initial_client_config,
   // Finish a run
   ScenarioResult result;
   gpr_log(GPR_INFO, "Finishing");
-  for (auto& server : servers) {
-    GPR_ASSERT(server.stream->Write(server_mark));
+  for (auto server = servers.begin(); server != servers.end(); server++) {
+    GPR_ASSERT(server->stream->Write(server_mark));
   }
-  for (auto& client : clients) {
-    GPR_ASSERT(client.stream->Write(client_mark));
+  for (auto client = clients.begin(); client != clients.end(); client++) {
+    GPR_ASSERT(client->stream->Write(client_mark));
   }
-  for (auto& server : servers) {
-    GPR_ASSERT(server.stream->Read(&server_status));
+  for (auto server = servers.begin(); server != servers.end(); server++) {
+    GPR_ASSERT(server->stream->Read(&server_status));
     const auto& stats = server_status.stats();
     result.server_resources.push_back(ResourceUsage{
         stats.time_elapsed(), stats.time_user(), stats.time_system()});
   }
-  for (auto& client : clients) {
-    GPR_ASSERT(client.stream->Read(&client_status));
+  for (auto client = clients.begin(); client != clients.end(); client++) {
+    GPR_ASSERT(client->stream->Read(&client_status));
     const auto& stats = client_status.stats();
     result.latencies.MergeProto(stats.latencies());
     result.client_resources.push_back(ResourceUsage{
         stats.time_elapsed(), stats.time_user(), stats.time_system()});
   }
 
-  for (auto& client : clients) {
-    GPR_ASSERT(client.stream->WritesDone());
-    GPR_ASSERT(client.stream->Finish().IsOk());
+  for (auto client = clients.begin(); client != clients.end(); client++) {
+    GPR_ASSERT(client->stream->WritesDone());
+    GPR_ASSERT(client->stream->Finish().IsOk());
   }
-  for (auto& server : servers) {
-    GPR_ASSERT(server.stream->WritesDone());
-    GPR_ASSERT(server.stream->Finish().IsOk());
+  for (auto server = servers.begin(); server != servers.end(); server++) {
+    GPR_ASSERT(server->stream->WritesDone());
+    GPR_ASSERT(server->stream->Finish().IsOk());
   }
   return result;
 }
diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc
index 586b6e7abe..7b81bd35a2 100644
--- a/test/cpp/qps/server_async.cc
+++ b/test/cpp/qps/server_async.cc
@@ -105,8 +105,8 @@ class AsyncQpsServerTest : public Server {
   ~AsyncQpsServerTest() {
     server_->Shutdown();
     srv_cq_.Shutdown();
-    for (auto& thr : threads_) {
-      thr.join();
+    for (auto thr = threads_.begin(); thr != threads_.end(); thr++) {
+      thr->join();
     }
     while (!contexts_.empty()) {
       delete contexts_.front();
diff --git a/test/cpp/qps/stats.h b/test/cpp/qps/stats.h
index ca59390ad7..82dc03e3da 100644
--- a/test/cpp/qps/stats.h
+++ b/test/cpp/qps/stats.h
@@ -43,8 +43,8 @@ namespace testing {
 template <class T, class F>
 double sum(const T& container, F functor) {
   double r = 0;
-  for (auto v : container) {
-    r += functor(v);
+  for (auto v = container.begin(); v != container.end(); v++) {
+    r += functor(*v);
   }
   return r;
 }