Merge remote-tracking branch 'upstream/master' into interop

23 files changed, 331 insertions, 110 deletions

diff --git a/Makefile b/Makefile
index b7b14fba3d..878143e5f2 100644
--- a/Makefile
+++ b/Makefile
@@ -2227,6 +2227,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgpr.$(SHARED_EXT) $(prefix)/lib/libgpr.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf libgpr.$(SHARED_EXT) $(prefix)/lib/libgpr.so.0
 	$(Q) ln -sf libgpr.$(SHARED_EXT) $(prefix)/lib/libgpr.so
 endif
 endif
@@ -2240,6 +2241,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc.$(SHARED_EXT) $(prefix)/lib/libgrpc.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf libgrpc.$(SHARED_EXT) $(prefix)/lib/libgrpc.so.0
 	$(Q) ln -sf libgrpc.$(SHARED_EXT) $(prefix)/lib/libgrpc.so
 endif
 endif
@@ -2253,6 +2255,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.$(SHARED_EXT) $(prefix)/lib/libgrpc_unsecure.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf libgrpc_unsecure.$(SHARED_EXT) $(prefix)/lib/libgrpc_unsecure.so.0
 	$(Q) ln -sf libgrpc_unsecure.$(SHARED_EXT) $(prefix)/lib/libgrpc_unsecure.so
 endif
 endif
@@ -2274,6 +2277,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++.$(SHARED_EXT) $(prefix)/lib/libgrpc++.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf libgrpc++.$(SHARED_EXT) $(prefix)/lib/libgrpc++.so.0
 	$(Q) ln -sf libgrpc++.$(SHARED_EXT) $(prefix)/lib/libgrpc++.so
 endif
 endif
@@ -2287,6 +2291,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.$(SHARED_EXT) $(prefix)/lib/libgrpc++_unsecure.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf libgrpc++_unsecure.$(SHARED_EXT) $(prefix)/lib/libgrpc++_unsecure.so.0
 	$(Q) ln -sf libgrpc++_unsecure.$(SHARED_EXT) $(prefix)/lib/libgrpc++_unsecure.so
 endif
 endif
@@ -2308,6 +2313,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.$(SHARED_EXT) $(prefix)/lib/libgrpc_csharp_ext.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf libgrpc_csharp_ext.$(SHARED_EXT) $(prefix)/lib/libgrpc_csharp_ext.so.0
 	$(Q) ln -sf libgrpc_csharp_ext.$(SHARED_EXT) $(prefix)/lib/libgrpc_csharp_ext.so
 endif
 endif
diff --git a/build.json b/build.json
index b33849434a..e536789a2b 100644
--- a/build.json
+++ b/build.json
@@ -464,6 +464,10 @@
       "name": "grpc++",
       "build": "all",
       "language": "c++",
+      "headers": [
+        "src/cpp/client/secure_credentials.h",
+        "src/cpp/server/secure_server_credentials.h"
+      ],
       "src": [
         "src/cpp/client/secure_credentials.cc",
         "src/cpp/server/secure_server_credentials.cc"
diff --git a/src/core/iomgr/tcp_server.h b/src/core/iomgr/tcp_server.h
index 68ee85c5a7..66bb3ef701 100644
--- a/src/core/iomgr/tcp_server.h
+++ b/src/core/iomgr/tcp_server.h
@@ -71,6 +71,8 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
    up when grpc_tcp_server_destroy is called. */
 int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index);
 
-void grpc_tcp_server_destroy(grpc_tcp_server *server);
+void grpc_tcp_server_destroy(grpc_tcp_server *server,
+                             void (*shutdown_done)(void *shutdown_done_arg),
+                             void *shutdown_done_arg);
 
-#endif  /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */
+#endif /* GRPC_INTERNAL_CORE_IOMGR_TCP_SERVER_H */
diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c
index 90b7eb451d..895f85fc68 100644
--- a/src/core/iomgr/tcp_server_posix.c
+++ b/src/core/iomgr/tcp_server_posix.c
@@ -102,12 +102,18 @@ struct grpc_tcp_server {
   gpr_cv cv;
 
   /* active port count: how many ports are actually still listening */
-  int active_ports;
+  size_t active_ports;
+  /* destroyed port count: how many ports are completely destroyed */
+  size_t destroyed_ports;
 
   /* all listening ports */
   server_port *ports;
   size_t nports;
   size_t port_capacity;
+
+  /* shutdown callback */
+  void (*shutdown_complete)(void *);
+  void *shutdown_complete_arg;
 };
 
 grpc_tcp_server *grpc_tcp_server_create(void) {
@@ -115,6 +121,7 @@ grpc_tcp_server *grpc_tcp_server_create(void) {
   gpr_mu_init(&s->mu);
   gpr_cv_init(&s->cv);
   s->active_ports = 0;
+  s->destroyed_ports = 0;
   s->cb = NULL;
   s->cb_arg = NULL;
   s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP);
@@ -123,29 +130,64 @@ grpc_tcp_server *grpc_tcp_server_create(void) {
   return s;
 }
 
-void grpc_tcp_server_destroy(grpc_tcp_server *s) {
+static void finish_shutdown(grpc_tcp_server *s) {
+  s->shutdown_complete(s->shutdown_complete_arg);
+
+  gpr_mu_destroy(&s->mu);
+  gpr_cv_destroy(&s->cv);
+
+  gpr_free(s->ports);
+  gpr_free(s);
+}
+
+static void destroyed_port(void *server, int success) {
+  grpc_tcp_server *s = server;
+  gpr_mu_lock(&s->mu);
+  s->destroyed_ports++;
+  if (s->destroyed_ports == s->nports) {
+    gpr_mu_unlock(&s->mu);
+    finish_shutdown(s);
+  } else {
+    gpr_mu_unlock(&s->mu);
+  }
+}
+
+static void dont_care_about_shutdown_completion(void *ignored) {}
+
+void grpc_tcp_server_destroy(
+    grpc_tcp_server *s, void (*shutdown_complete)(void *shutdown_complete_arg),
+    void *shutdown_complete_arg) {
   size_t i;
   gpr_mu_lock(&s->mu);
+
+  s->shutdown_complete = shutdown_complete
+                             ? shutdown_complete
+                             : dont_care_about_shutdown_completion;
+  s->shutdown_complete_arg = shutdown_complete_arg;
+
   /* shutdown all fd's */
   for (i = 0; i < s->nports; i++) {
     grpc_fd_shutdown(s->ports[i].emfd);
   }
   /* wait while that happens */
+  /* TODO(ctiller): make this asynchronous also */
   while (s->active_ports) {
     gpr_cv_wait(&s->cv, &s->mu, gpr_inf_future);
   }
   gpr_mu_unlock(&s->mu);
 
   /* delete ALL the things */
-  for (i = 0; i < s->nports; i++) {
-    server_port *sp = &s->ports[i];
-    if (sp->addr.sockaddr.sa_family == AF_UNIX) {
-      unlink_if_unix_domain_socket(&sp->addr.un);
+  if (s->nports) {
+    for (i = 0; i < s->nports; i++) {
+      server_port *sp = &s->ports[i];
+      if (sp->addr.sockaddr.sa_family == AF_UNIX) {
+        unlink_if_unix_domain_socket(&sp->addr.un);
+      }
+      grpc_fd_orphan(sp->emfd, destroyed_port, s);
     }
-    grpc_fd_orphan(sp->emfd, NULL, NULL);
+  } else {
+    finish_shutdown(s);
   }
-  gpr_free(s->ports);
-  gpr_free(s);
 }
 
 /* get max listen queue size on linux */
diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c
index 0c3ab1dc91..a43d5670a4 100644
--- a/src/core/iomgr/tcp_server_windows.c
+++ b/src/core/iomgr/tcp_server_windows.c
@@ -92,7 +92,9 @@ grpc_tcp_server *grpc_tcp_server_create(void) {
   return s;
 }
 
-void grpc_tcp_server_destroy(grpc_tcp_server *s) {
+void grpc_tcp_server_destroy(grpc_tcp_server *s,
+                             void (*shutdown_done)(void *shutdown_done_arg),
+                             void *shutdown_done_arg) {
   size_t i;
   gpr_mu_lock(&s->mu);
   /* shutdown all fd's */
@@ -112,11 +114,15 @@ void grpc_tcp_server_destroy(grpc_tcp_server *s) {
   }
   gpr_free(s->ports);
   gpr_free(s);
+
+  if (shutdown_done) {
+    shutdown_done(shutdown_done_arg);
+  }
 }
 
 /* Prepare a recently-created socket for listening. */
-static int prepare_socket(SOCKET sock,
-                          const struct sockaddr *addr, int addr_len) {
+static int prepare_socket(SOCKET sock, const struct sockaddr *addr,
+                          int addr_len) {
   struct sockaddr_storage sockname_temp;
   socklen_t sockname_len;
 
@@ -147,15 +153,15 @@ static int prepare_socket(SOCKET sock,
   }
 
   sockname_len = sizeof(sockname_temp);
-  if (getsockname(sock, (struct sockaddr *) &sockname_temp, &sockname_len)
-        == SOCKET_ERROR) {
+  if (getsockname(sock, (struct sockaddr *)&sockname_temp, &sockname_len) ==
+      SOCKET_ERROR) {
     char *utf8_message = gpr_format_message(WSAGetLastError());
     gpr_log(GPR_ERROR, "getsockname: %s", utf8_message);
     gpr_free(utf8_message);
     goto error;
   }
 
-  return grpc_sockaddr_get_port((struct sockaddr *) &sockname_temp);
+  return grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
 
 error:
   if (sock != INVALID_SOCKET) closesocket(sock);
@@ -221,8 +227,7 @@ static void on_accept(void *arg, int success) {
     DWORD transfered_bytes = 0;
     DWORD flags;
     BOOL wsa_success = WSAGetOverlappedResult(sock, &info->overlapped,
-                                              &transfered_bytes, FALSE,
-                                              &flags);
+                                              &transfered_bytes, FALSE, &flags);
     if (!wsa_success) {
       char *utf8_message = gpr_format_message(WSAGetLastError());
       gpr_log(GPR_ERROR, "on_accept error: %s", utf8_message);
@@ -257,9 +262,9 @@ static int add_socket_to_server(grpc_tcp_server *s, SOCKET sock,
 
   if (sock == INVALID_SOCKET) return -1;
 
-  status = WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER,
-                    &guid, sizeof(guid), &AcceptEx, sizeof(AcceptEx),
-                    &ioctl_num_bytes, NULL, NULL);
+  status =
+      WSAIoctl(sock, SIO_GET_EXTENSION_FUNCTION_POINTER, &guid, sizeof(guid),
+               &AcceptEx, sizeof(AcceptEx), &ioctl_num_bytes, NULL, NULL);
 
   if (status != 0) {
     char *utf8_message = gpr_format_message(WSAGetLastError());
@@ -307,9 +312,8 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
     for (i = 0; i < s->nports; i++) {
       sockname_len = sizeof(sockname_temp);
       if (0 == getsockname(s->ports[i].socket->socket,
-                           (struct sockaddr *) &sockname_temp,
-                           &sockname_len)) {
-        port = grpc_sockaddr_get_port((struct sockaddr *) &sockname_temp);
+                           (struct sockaddr *)&sockname_temp, &sockname_len)) {
+        port = grpc_sockaddr_get_port((struct sockaddr *)&sockname_temp);
         if (port > 0) {
           allocated_addr = malloc(addr_len);
           memcpy(allocated_addr, addr, addr_len);
@@ -330,7 +334,7 @@ int grpc_tcp_server_add_port(grpc_tcp_server *s, const void *addr,
   if (grpc_sockaddr_is_wildcard(addr, &port)) {
     grpc_sockaddr_make_wildcard6(port, &wildcard);
 
-    addr = (struct sockaddr *) &wildcard;
+    addr = (struct sockaddr *)&wildcard;
     addr_len = sizeof(wildcard);
   }
 
@@ -369,4 +373,4 @@ void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset **pollset,
   gpr_mu_unlock(&s->mu);
 }
 
-#endif  /* GPR_WINSOCK_SOCKET */
+#endif /* GPR_WINSOCK_SOCKET */
diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c
index c155b80b7e..081272724c 100644
--- a/src/core/security/server_secure_chttp2.c
+++ b/src/core/security/server_secure_chttp2.c
@@ -85,10 +85,10 @@ static void on_secure_transport_setup_done(void *statep,
   if (status == GRPC_SECURITY_OK) {
     gpr_mu_lock(&state->mu);
     if (!state->is_shutdown) {
-      grpc_create_chttp2_transport(
-          setup_transport, state->server,
-          grpc_server_get_channel_args(state->server),
-          secure_endpoint, NULL, 0, grpc_mdctx_create(), 0);
+      grpc_create_chttp2_transport(setup_transport, state->server,
+                                   grpc_server_get_channel_args(state->server),
+                                   secure_endpoint, NULL, 0,
+                                   grpc_mdctx_create(), 0);
     } else {
       /* We need to consume this here, because the server may already have gone
        * away. */
@@ -104,7 +104,8 @@ static void on_secure_transport_setup_done(void *statep,
 static void on_accept(void *statep, grpc_endpoint *tcp) {
   grpc_server_secure_state *state = statep;
   state_ref(state);
-  grpc_setup_secure_transport(state->ctx, tcp, on_secure_transport_setup_done, state);
+  grpc_setup_secure_transport(state->ctx, tcp, on_secure_transport_setup_done,
+                              state);
 }
 
 /* Server callback: start listening on our ports */
@@ -120,12 +121,14 @@ static void destroy(grpc_server *server, void *statep) {
   grpc_server_secure_state *state = statep;
   gpr_mu_lock(&state->mu);
   state->is_shutdown = 1;
-  grpc_tcp_server_destroy(state->tcp);
+  grpc_tcp_server_destroy(state->tcp, grpc_server_listener_destroy_done,
+                          server);
   gpr_mu_unlock(&state->mu);
   state_unref(state);
 }
 
-int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, grpc_server_credentials *creds) {
+int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
+                                      grpc_server_credentials *creds) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
   grpc_server_secure_state *state = NULL;
@@ -213,7 +216,7 @@ error:
     grpc_resolved_addresses_destroy(resolved);
   }
   if (tcp) {
-    grpc_tcp_server_destroy(tcp);
+    grpc_tcp_server_destroy(tcp, NULL, NULL);
   }
   if (state) {
     gpr_free(state);
diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c
index 6a1d83ce5d..24f4a05071 100644
--- a/src/core/surface/completion_queue.c
+++ b/src/core/surface/completion_queue.c
@@ -432,3 +432,11 @@ void grpc_cq_dump_pending_ops(grpc_completion_queue *cc) {
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
   return &cc->pollset;
 }
+
+void grpc_cq_hack_spin_pollset(grpc_completion_queue *cc) {
+  gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
+  grpc_pollset_kick(&cc->pollset);
+  grpc_pollset_work(&cc->pollset,
+                    gpr_time_add(gpr_now(), gpr_time_from_millis(100)));
+  gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
+}
diff --git a/src/core/surface/completion_queue.h b/src/core/surface/completion_queue.h
index 3054264cad..3a7cc99dda 100644
--- a/src/core/surface/completion_queue.h
+++ b/src/core/surface/completion_queue.h
@@ -114,4 +114,6 @@ void grpc_cq_dump_pending_ops(grpc_completion_queue *cc);
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
 
-#endif  /* GRPC_INTERNAL_CORE_SURFACE_COMPLETION_QUEUE_H */
+void grpc_cq_hack_spin_pollset(grpc_completion_queue *cc);
+
+#endif /* GRPC_INTERNAL_CORE_SURFACE_COMPLETION_QUEUE_H */
diff --git a/src/core/surface/server.c b/src/core/surface/server.c
index 424734c54c..17cba9a505 100644
--- a/src/core/surface/server.c
+++ b/src/core/surface/server.c
@@ -137,6 +137,7 @@ struct grpc_server {
   size_t cq_count;
 
   gpr_mu mu;
+  gpr_cv cv;
 
   registered_method *registered_methods;
   requested_call_array requested_calls;
@@ -149,6 +150,7 @@ struct grpc_server {
   channel_data root_channel_data;
 
   listener *listeners;
+  int listeners_destroyed;
   gpr_refcount internal_refcount;
 };
 
@@ -263,6 +265,7 @@ static void server_unref(grpc_server *server) {
   if (gpr_unref(&server->internal_refcount)) {
     grpc_channel_args_destroy(server->channel_args);
     gpr_mu_destroy(&server->mu);
+    gpr_cv_destroy(&server->cv);
     gpr_free(server->channel_filters);
     requested_call_array_destroy(&server->requested_calls);
     while ((rm = server->registered_methods) != NULL) {
@@ -589,9 +592,8 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
 }
 
 static const grpc_channel_filter server_surface_filter = {
-    call_op,           channel_op,           sizeof(call_data),
-    init_call_elem,    destroy_call_elem,    sizeof(channel_data),
-    init_channel_elem, destroy_channel_elem, "server",
+    call_op, channel_op, sizeof(call_data), init_call_elem, destroy_call_elem,
+    sizeof(channel_data), init_channel_elem, destroy_channel_elem, "server",
 };
 
 static void addcq(grpc_server *server, grpc_completion_queue *cq) {
@@ -620,6 +622,7 @@ grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
   if (cq) addcq(server, cq);
 
   gpr_mu_init(&server->mu);
+  gpr_cv_init(&server->cv);
 
   server->unregistered_cq = cq;
   /* decremented by grpc_server_destroy */
@@ -733,7 +736,8 @@ grpc_transport_setup_result grpc_server_setup_transport(
   channel = grpc_channel_create_from_filters(filters, num_filters,
                                              s->channel_args, mdctx, 0);
   chand = (channel_data *)grpc_channel_stack_element(
-              grpc_channel_get_channel_stack(channel), 0)->channel_data;
+              grpc_channel_get_channel_stack(channel), 0)
+              ->channel_data;
   chand->server = s;
   server_ref(s);
   chand->channel = channel;
@@ -754,7 +758,7 @@ grpc_transport_setup_result grpc_server_setup_transport(
       method = grpc_mdstr_from_string(mdctx, rm->method);
       hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
       for (probes = 0; chand->registered_methods[(hash + probes) % slots]
-                               .server_registered_method != NULL;
+                           .server_registered_method != NULL;
            probes++)
         ;
       if (probes > max_probes) max_probes = probes;
@@ -781,6 +785,15 @@ grpc_transport_setup_result grpc_server_setup_transport(
   return result;
 }
 
+static int num_listeners(grpc_server *server) {
+  listener *l;
+  int n = 0;
+  for (l = server->listeners; l; l = l->next) {
+    n++;
+  }
+  return n;
+}
+
 static void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
                               void *shutdown_tag) {
   listener *l;
@@ -878,11 +891,6 @@ static void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
   for (l = server->listeners; l; l = l->next) {
     l->destroy(server, l->arg);
   }
-  while (server->listeners) {
-    l = server->listeners;
-    server->listeners = l->next;
-    gpr_free(l);
-  }
 }
 
 void grpc_server_shutdown(grpc_server *server) {
@@ -893,8 +901,18 @@ void grpc_server_shutdown_and_notify(grpc_server *server, void *tag) {
   shutdown_internal(server, 1, tag);
 }
 
+void grpc_server_listener_destroy_done(void *s) {
+  grpc_server *server = s;
+  gpr_mu_lock(&server->mu);
+  server->listeners_destroyed++;
+  gpr_cv_signal(&server->cv);
+  gpr_mu_unlock(&server->mu);
+}
+
 void grpc_server_destroy(grpc_server *server) {
   channel_data *c;
+  listener *l;
+  size_t i;
   gpr_mu_lock(&server->mu);
   if (!server->shutdown) {
     gpr_mu_unlock(&server->mu);
@@ -902,6 +920,23 @@ void grpc_server_destroy(grpc_server *server) {
     gpr_mu_lock(&server->mu);
   }
 
+  while (server->listeners_destroyed != num_listeners(server)) {
+    for (i = 0; i < server->cq_count; i++) {
+      gpr_mu_unlock(&server->mu);
+      grpc_cq_hack_spin_pollset(server->cqs[i]);
+      gpr_mu_lock(&server->mu);
+    }
+
+    gpr_cv_wait(&server->cv, &server->mu,
+                gpr_time_add(gpr_now(), gpr_time_from_millis(100)));
+  }
+
+  while (server->listeners) {
+    l = server->listeners;
+    server->listeners = l->next;
+    gpr_free(l);
+  }
+
   for (c = server->root_channel_data.next; c != &server->root_channel_data;
        c = c->next) {
     shutdown_channel(c);
diff --git a/src/core/surface/server.h b/src/core/surface/server.h
index e33f69b8c7..2cfa38fa43 100644
--- a/src/core/surface/server.h
+++ b/src/core/surface/server.h
@@ -48,9 +48,12 @@ grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
    and when it shuts down, it will call destroy */
 void grpc_server_add_listener(grpc_server *server, void *listener,
                               void (*start)(grpc_server *server, void *arg,
-                                            grpc_pollset **pollsets, size_t npollsets),
+                                            grpc_pollset **pollsets,
+                                            size_t npollsets),
                               void (*destroy)(grpc_server *server, void *arg));
 
+void grpc_server_listener_destroy_done(void *server);
+
 /* Setup a transport - creates a channel stack, binds the transport to the
    server */
 grpc_transport_setup_result grpc_server_setup_transport(
@@ -60,4 +63,4 @@ grpc_transport_setup_result grpc_server_setup_transport(
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
 
-#endif  /* GRPC_INTERNAL_CORE_SURFACE_SERVER_H */
+#endif /* GRPC_INTERNAL_CORE_SURFACE_SERVER_H */
diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c
index 27434b39e2..f3b9219f8b 100644
--- a/src/core/surface/server_chttp2.c
+++ b/src/core/surface/server_chttp2.c
@@ -66,7 +66,8 @@ static void new_transport(void *server, grpc_endpoint *tcp) {
 }
 
 /* Server callback: start listening on our ports */
-static void start(grpc_server *server, void *tcpp, grpc_pollset **pollsets, size_t pollset_count) {
+static void start(grpc_server *server, void *tcpp, grpc_pollset **pollsets,
+                  size_t pollset_count) {
   grpc_tcp_server *tcp = tcpp;
   grpc_tcp_server_start(tcp, pollsets, pollset_count, new_transport, server);
 }
@@ -75,7 +76,7 @@ static void start(grpc_server *server, void *tcpp, grpc_pollset **pollsets, size
    callbacks) */
 static void destroy(grpc_server *server, void *tcpp) {
   grpc_tcp_server *tcp = tcpp;
-  grpc_tcp_server_destroy(tcp);
+  grpc_tcp_server_destroy(tcp, grpc_server_listener_destroy_done, server);
 }
 
 int grpc_server_add_http2_port(grpc_server *server, const char *addr) {
@@ -131,7 +132,7 @@ error:
     grpc_resolved_addresses_destroy(resolved);
   }
   if (tcp) {
-    grpc_tcp_server_destroy(tcp);
+    grpc_tcp_server_destroy(tcp, NULL, NULL);
   }
   return 0;
 }
diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc
index d6f9acc675..0a73b2c0f6 100644
--- a/src/cpp/client/secure_credentials.cc
+++ b/src/cpp/client/secure_credentials.cc
@@ -31,38 +31,23 @@
  *
  */
 
-#include <grpc/grpc_security.h>
 #include <grpc/support/log.h>
 
 #include <grpc++/channel_arguments.h>
-#include <grpc++/config.h>
-#include <grpc++/credentials.h>
 #include "src/cpp/client/channel.h"
+#include "src/cpp/client/secure_credentials.h"
 
 namespace grpc {
 
-class SecureCredentials GRPC_FINAL : public Credentials {
- public:
-  explicit SecureCredentials(grpc_credentials* c_creds) : c_creds_(c_creds) {}
-  ~SecureCredentials() GRPC_OVERRIDE { grpc_credentials_release(c_creds_); }
-  grpc_credentials* GetRawCreds() { return c_creds_; }
-
-  std::shared_ptr<grpc::ChannelInterface> CreateChannel(
-      const string& target, const grpc::ChannelArguments& args) GRPC_OVERRIDE {
-    grpc_channel_args channel_args;
-    args.SetChannelArgs(&channel_args);
-    return std::shared_ptr<ChannelInterface>(new Channel(
-        args.GetSslTargetNameOverride().empty()
-            ? target
-            : args.GetSslTargetNameOverride(),
-        grpc_secure_channel_create(c_creds_, target.c_str(), &channel_args)));
-  }
-
-  SecureCredentials* AsSecureCredentials() GRPC_OVERRIDE { return this; }
-
- private:
-  grpc_credentials* const c_creds_;
-};
+std::shared_ptr<grpc::ChannelInterface> SecureCredentials::CreateChannel(
+    const string& target, const grpc::ChannelArguments& args) {
+  grpc_channel_args channel_args;
+  args.SetChannelArgs(&channel_args);
+  return std::shared_ptr<ChannelInterface>(new Channel(
+      args.GetSslTargetNameOverride().empty() ? target
+                                              : args.GetSslTargetNameOverride(),
+      grpc_secure_channel_create(c_creds_, target.c_str(), &channel_args)));
+}
 
 namespace {
 std::unique_ptr<Credentials> WrapCredentials(grpc_credentials* creds) {
diff --git a/src/cpp/client/secure_credentials.h b/src/cpp/client/secure_credentials.h
new file mode 100644
index 0000000000..77d575813e
--- /dev/null
+++ b/src/cpp/client/secure_credentials.h
@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CPP_CLIENT_SECURE_CREDENTIALS_H
+#define GRPC_INTERNAL_CPP_CLIENT_SECURE_CREDENTIALS_H
+
+#include <grpc/grpc_security.h>
+
+#include <grpc++/config.h>
+#include <grpc++/credentials.h>
+
+namespace grpc {
+
+class SecureCredentials GRPC_FINAL : public Credentials {
+ public:
+  explicit SecureCredentials(grpc_credentials* c_creds) : c_creds_(c_creds) {}
+  ~SecureCredentials() GRPC_OVERRIDE { grpc_credentials_release(c_creds_); }
+  grpc_credentials* GetRawCreds() { return c_creds_; }
+
+  std::shared_ptr<grpc::ChannelInterface> CreateChannel(
+      const string& target, const grpc::ChannelArguments& args) GRPC_OVERRIDE;
+  SecureCredentials* AsSecureCredentials() GRPC_OVERRIDE { return this; }
+
+ private:
+  grpc_credentials* const c_creds_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPC_INTERNAL_CPP_CLIENT_SECURE_CREDENTIALS_H
+
diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc
index 49d69a3fb9..3e262dd74f 100644
--- a/src/cpp/server/secure_server_credentials.cc
+++ b/src/cpp/server/secure_server_credentials.cc
@@ -31,39 +31,22 @@
  *
  */
 
-#include <grpc/grpc_security.h>
-
-#include <grpc++/server_credentials.h>
+#include "src/cpp/server/secure_server_credentials.h"
 
 namespace grpc {
 
-namespace {
-class SecureServerCredentials GRPC_FINAL : public ServerCredentials {
- public:
-  explicit SecureServerCredentials(grpc_server_credentials* creds)
-      : creds_(creds) {}
-  ~SecureServerCredentials() GRPC_OVERRIDE {
-    grpc_server_credentials_release(creds_);
-  }
-
-  int AddPortToServer(const grpc::string& addr,
-                      grpc_server* server) GRPC_OVERRIDE {
-    return grpc_server_add_secure_http2_port(server, addr.c_str(), creds_);
-  }
-
- private:
-  grpc_server_credentials* const creds_;
-};
-}  // namespace
+int SecureServerCredentials::AddPortToServer(
+    const grpc::string& addr, grpc_server* server) {
+  return grpc_server_add_secure_http2_port(server, addr.c_str(), creds_);
+}
 
 std::shared_ptr<ServerCredentials> SslServerCredentials(
     const SslServerCredentialsOptions& options) {
   std::vector<grpc_ssl_pem_key_cert_pair> pem_key_cert_pairs;
   for (auto key_cert_pair = options.pem_key_cert_pairs.begin();
-       key_cert_pair != options.pem_key_cert_pairs.end();
-       key_cert_pair++) {
+       key_cert_pair != options.pem_key_cert_pairs.end(); key_cert_pair++) {
     grpc_ssl_pem_key_cert_pair p = {key_cert_pair->private_key.c_str(),
-				    key_cert_pair->cert_chain.c_str()};
+                                    key_cert_pair->cert_chain.c_str()};
     pem_key_cert_pairs.push_back(p);
   }
   grpc_server_credentials* c_creds = grpc_ssl_server_credentials_create(
diff --git a/src/cpp/server/secure_server_credentials.h b/src/cpp/server/secure_server_credentials.h
new file mode 100644
index 0000000000..b9803f107e
--- /dev/null
+++ b/src/cpp/server/secure_server_credentials.h
@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CPP_SERVER_SECURE_SERVER_CREDENTIALS_H
+#define GRPC_INTERNAL_CPP_SERVER_SECURE_SERVER_CREDENTIALS_H
+
+#include <grpc/grpc_security.h>
+
+#include <grpc++/server_credentials.h>
+
+namespace grpc {
+
+class SecureServerCredentials GRPC_FINAL : public ServerCredentials {
+ public:
+  explicit SecureServerCredentials(grpc_server_credentials* creds)
+      : creds_(creds) {}
+  ~SecureServerCredentials() GRPC_OVERRIDE {
+    grpc_server_credentials_release(creds_);
+  }
+
+  int AddPortToServer(const grpc::string& addr,
+                      grpc_server* server) GRPC_OVERRIDE;
+
+ private:
+  grpc_server_credentials* const creds_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPC_INTERNAL_CPP_SERVER_SECURE_SERVER_CREDENTIALS_H
diff --git a/templates/Makefile.template b/templates/Makefile.template
index 776ee7a49b..2cfbfa36ec 100644
--- a/templates/Makefile.template
+++ b/templates/Makefile.template
@@ -930,6 +930,7 @@ else
 	$(Q) $(INSTALL) -d $(prefix)/lib
 	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/lib${lib.name}.$(SHARED_EXT) $(prefix)/lib/lib${lib.name}.$(SHARED_EXT)
 ifneq ($(SYSTEM),Darwin)
+	$(Q) ln -sf lib${lib.name}.$(SHARED_EXT) $(prefix)/lib/lib${lib.name}.so.${settings.version.major}
 	$(Q) ln -sf lib${lib.name}.$(SHARED_EXT) $(prefix)/lib/lib${lib.name}.so
 endif
 endif
diff --git a/test/core/end2end/tests/cancel_after_invoke.c b/test/core/end2end/tests/cancel_after_invoke.c
index e15fa7fcd9..592dfd415f 100644
--- a/test/core/end2end/tests/cancel_after_invoke.c
+++ b/test/core/end2end/tests/cancel_after_invoke.c
@@ -51,10 +51,11 @@ static void *tag(gpr_intptr t) { return (void *)t; }
 
 static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
                                             const char *test_name,
+                                            cancellation_mode mode,
                                             grpc_channel_args *client_args,
                                             grpc_channel_args *server_args) {
   grpc_end2end_test_fixture f;
-  gpr_log(GPR_INFO, "%s/%s", test_name, config.name);
+  gpr_log(GPR_INFO, "%s/%s/%s", test_name, config.name, mode.name);
   f = config.create_fixture(client_args, server_args);
   config.init_client(&f, client_args);
   config.init_server(&f, server_args);
@@ -109,7 +110,8 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config,
   grpc_op ops[6];
   grpc_op *op;
   grpc_call *c;
-  grpc_end2end_test_fixture f = begin_test(config, __FUNCTION__, NULL, NULL);
+  grpc_end2end_test_fixture f =
+      begin_test(config, __FUNCTION__, mode, NULL, NULL);
   gpr_timespec deadline = five_seconds_time();
   cq_verifier *v_client = cq_verifier_create(f.client_cq);
   grpc_metadata_array initial_metadata_recv;
diff --git a/test/core/end2end/tests/cancel_test_helpers.h b/test/core/end2end/tests/cancel_test_helpers.h
index f2581dc32f..0d680fcfe1 100644
--- a/test/core/end2end/tests/cancel_test_helpers.h
+++ b/test/core/end2end/tests/cancel_test_helpers.h
@@ -35,6 +35,7 @@
 #define GRPC_TEST_CORE_END2END_TESTS_CANCEL_TEST_HELPERS_H
 
 typedef struct {
+  const char *name;
   grpc_call_error (*initiate_cancel)(grpc_call *call);
   grpc_status_code expect_status;
   const char *expect_details;
@@ -45,7 +46,9 @@ static grpc_call_error wait_for_deadline(grpc_call *call) {
 }
 
 static const cancellation_mode cancellation_modes[] = {
-    {grpc_call_cancel, GRPC_STATUS_CANCELLED, ""},
-    {wait_for_deadline, GRPC_STATUS_DEADLINE_EXCEEDED, "Deadline Exceeded"}, };
+    {"cancel", grpc_call_cancel, GRPC_STATUS_CANCELLED, ""},
+    {"deadline", wait_for_deadline, GRPC_STATUS_DEADLINE_EXCEEDED,
+     "Deadline Exceeded"},
+};
 
-#endif  /* GRPC_TEST_CORE_END2END_TESTS_CANCEL_TEST_HELPERS_H */
+#endif /* GRPC_TEST_CORE_END2END_TESTS_CANCEL_TEST_HELPERS_H */
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index 2689c3f38e..6b80ee1ee8 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -60,14 +60,14 @@ static void on_connect(void *arg, grpc_endpoint *tcp) {
 
 static void test_no_op(void) {
   grpc_tcp_server *s = grpc_tcp_server_create();
-  grpc_tcp_server_destroy(s);
+  grpc_tcp_server_destroy(s, NULL, NULL);
 }
 
 static void test_no_op_with_start(void) {
   grpc_tcp_server *s = grpc_tcp_server_create();
   LOG_TEST();
   grpc_tcp_server_start(s, NULL, 0, on_connect, NULL);
-  grpc_tcp_server_destroy(s);
+  grpc_tcp_server_destroy(s, NULL, NULL);
 }
 
 static void test_no_op_with_port(void) {
@@ -80,7 +80,7 @@ static void test_no_op_with_port(void) {
   GPR_ASSERT(
       grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)));
 
-  grpc_tcp_server_destroy(s);
+  grpc_tcp_server_destroy(s, NULL, NULL);
 }
 
 static void test_no_op_with_port_and_start(void) {
@@ -95,7 +95,7 @@ static void test_no_op_with_port_and_start(void) {
 
   grpc_tcp_server_start(s, NULL, 0, on_connect, NULL);
 
-  grpc_tcp_server_destroy(s);
+  grpc_tcp_server_destroy(s, NULL, NULL);
 }
 
 static void test_connect(int n) {
@@ -144,7 +144,7 @@ static void test_connect(int n) {
 
   gpr_mu_unlock(&mu);
 
-  grpc_tcp_server_destroy(s);
+  grpc_tcp_server_destroy(s, NULL, NULL);
 }
 
 int main(int argc, char **argv) {
diff --git a/vsprojects/vs2010/grpc++.vcxproj b/vsprojects/vs2010/grpc++.vcxproj
index 0ee433163c..c1795923c7 100644
--- a/vsprojects/vs2010/grpc++.vcxproj
+++ b/vsprojects/vs2010/grpc++.vcxproj
@@ -104,6 +104,8 @@
     <ClInclude Include="..\..\include\grpc++\thread_pool_interface.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\src\cpp\client\secure_credentials.h" />
+    <ClInclude Include="..\..\src\cpp\server\secure_server_credentials.h" />
     <ClInclude Include="..\..\src\cpp\client\channel.h" />
     <ClInclude Include="..\..\src\cpp\proto\proto_utils.h" />
     <ClInclude Include="..\..\src\cpp\server\thread_pool.h" />
diff --git a/vsprojects/vs2010/grpc++.vcxproj.filters b/vsprojects/vs2010/grpc++.vcxproj.filters
index ed93daee04..77f5383054 100644
--- a/vsprojects/vs2010/grpc++.vcxproj.filters
+++ b/vsprojects/vs2010/grpc++.vcxproj.filters
@@ -161,6 +161,12 @@
     </ClInclude>
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\src\cpp\client\secure_credentials.h">
+      <Filter>src\cpp\client</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\cpp\server\secure_server_credentials.h">
+      <Filter>src\cpp\server</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\cpp\client\channel.h">
       <Filter>src\cpp\client</Filter>
     </ClInclude>
diff --git a/vsprojects/vs2013/grpc++.vcxproj b/vsprojects/vs2013/grpc++.vcxproj
index d545a949cb..b9039c5a8f 100644
--- a/vsprojects/vs2013/grpc++.vcxproj
+++ b/vsprojects/vs2013/grpc++.vcxproj
@@ -106,6 +106,8 @@
     <ClInclude Include="..\..\include\grpc++\thread_pool_interface.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\src\cpp\client\secure_credentials.h" />
+    <ClInclude Include="..\..\src\cpp\server\secure_server_credentials.h" />
     <ClInclude Include="..\..\src\cpp\client\channel.h" />
     <ClInclude Include="..\..\src\cpp\proto\proto_utils.h" />
     <ClInclude Include="..\..\src\cpp\server\thread_pool.h" />
diff --git a/vsprojects/vs2013/grpc++.vcxproj.filters b/vsprojects/vs2013/grpc++.vcxproj.filters
index ed93daee04..77f5383054 100644
--- a/vsprojects/vs2013/grpc++.vcxproj.filters
+++ b/vsprojects/vs2013/grpc++.vcxproj.filters
@@ -161,6 +161,12 @@
     </ClInclude>
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="..\..\src\cpp\client\secure_credentials.h">
+      <Filter>src\cpp\client</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\src\cpp\server\secure_server_credentials.h">
+      <Filter>src\cpp\server</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\cpp\client\channel.h">
       <Filter>src\cpp\client</Filter>
     </ClInclude>