Merge github.com:grpc/grpc into filter-selection

36 files changed, 879 insertions, 199 deletions

diff --git a/src/core/client_config/lb_policies/load_balancer_api.c b/src/core/client_config/lb_policies/load_balancer_api.c
new file mode 100644
index 0000000000..a6b5785fe4
--- /dev/null
+++ b/src/core/client_config/lb_policies/load_balancer_api.c
@@ -0,0 +1,163 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/client_config/lb_policies/load_balancer_api.h"
+#include "third_party/nanopb/pb_decode.h"
+#include "third_party/nanopb/pb_encode.h"
+
+#include <grpc/support/alloc.h>
+
+typedef struct decode_serverlist_arg {
+  int first_pass;
+  int i;
+  size_t num_servers;
+  grpc_grpclb_server **servers;
+} decode_serverlist_arg;
+
+/* invoked once for every Server in ServerList */
+static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field,
+                              void **arg) {
+  decode_serverlist_arg *dec_arg = *arg;
+  if (dec_arg->first_pass != 0) { /* first pass */
+    grpc_grpclb_server server;
+    if (!pb_decode(stream, grpc_lb_v0_Server_fields, &server)) {
+      return false;
+    }
+    dec_arg->num_servers++;
+  } else { /* second pass */
+    grpc_grpclb_server *server = gpr_malloc(sizeof(grpc_grpclb_server));
+    GPR_ASSERT(dec_arg->num_servers > 0);
+    if (dec_arg->i == 0) { /* first iteration of second pass */
+      dec_arg->servers =
+          gpr_malloc(sizeof(grpc_grpclb_server *) * dec_arg->num_servers);
+    }
+    if (!pb_decode(stream, grpc_lb_v0_Server_fields, server)) {
+      return false;
+    }
+    dec_arg->servers[dec_arg->i++] = server;
+  }
+
+  return true;
+}
+
+grpc_grpclb_request *grpc_grpclb_request_create(const char *lb_service_name) {
+  grpc_grpclb_request *req = gpr_malloc(sizeof(grpc_grpclb_request));
+
+  req->has_client_stats = 0; /* TODO(dgq): add support for stats once defined */
+  req->has_initial_request = 1;
+  req->initial_request.has_name = 1;
+  strncpy(req->initial_request.name, lb_service_name,
+          GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH);
+  return req;
+}
+
+gpr_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) {
+  size_t encoded_length;
+  pb_ostream_t sizestream;
+  pb_ostream_t outputstream;
+  gpr_slice slice;
+  memset(&sizestream, 0, sizeof(pb_ostream_t));
+  pb_encode(&sizestream, grpc_lb_v0_LoadBalanceRequest_fields, request);
+  encoded_length = sizestream.bytes_written;
+
+  slice = gpr_slice_malloc(encoded_length);
+  outputstream =
+      pb_ostream_from_buffer(GPR_SLICE_START_PTR(slice), encoded_length);
+  GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v0_LoadBalanceRequest_fields,
+                       request) != 0);
+  return slice;
+}
+
+void grpc_grpclb_request_destroy(grpc_grpclb_request *request) {
+  gpr_free(request);
+}
+
+grpc_grpclb_response *grpc_grpclb_response_parse(gpr_slice encoded_response) {
+  bool status;
+  pb_istream_t stream =
+      pb_istream_from_buffer(GPR_SLICE_START_PTR(encoded_response),
+                             GPR_SLICE_LENGTH(encoded_response));
+  grpc_grpclb_response *res = gpr_malloc(sizeof(grpc_grpclb_response));
+  memset(res, 0, sizeof(*res));
+  status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res);
+  GPR_ASSERT(status == true);
+  return res;
+}
+
+grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist(
+    gpr_slice encoded_response) {
+  grpc_grpclb_serverlist *sl = gpr_malloc(sizeof(grpc_grpclb_serverlist));
+  bool status;
+  decode_serverlist_arg arg;
+  pb_istream_t stream =
+      pb_istream_from_buffer(GPR_SLICE_START_PTR(encoded_response),
+                             GPR_SLICE_LENGTH(encoded_response));
+  pb_istream_t stream_at_start = stream;
+  grpc_grpclb_response *res = gpr_malloc(sizeof(grpc_grpclb_response));
+  memset(res, 0, sizeof(*res));
+  memset(&arg, 0, sizeof(decode_serverlist_arg));
+
+  res->server_list.servers.funcs.decode = decode_serverlist;
+  res->server_list.servers.arg = &arg;
+  arg.first_pass = 1;
+  status = pb_decode(&stream, grpc_lb_v0_LoadBalanceResponse_fields, res);
+  GPR_ASSERT(status == true);
+  GPR_ASSERT(arg.num_servers > 0);
+
+  arg.first_pass = 0;
+  status =
+      pb_decode(&stream_at_start, grpc_lb_v0_LoadBalanceResponse_fields, res);
+  GPR_ASSERT(status == true);
+  GPR_ASSERT(arg.servers != NULL);
+
+  sl->num_servers = arg.num_servers;
+  sl->servers = arg.servers;
+  if (res->server_list.has_expiration_interval) {
+    sl->expiration_interval = res->server_list.expiration_interval;
+  }
+  grpc_grpclb_response_destroy(res);
+  return sl;
+}
+
+void grpc_grpclb_destroy_serverlist(grpc_grpclb_serverlist *serverlist) {
+  size_t i;
+  for (i = 0; i < serverlist->num_servers; i++) {
+    gpr_free(serverlist->servers[i]);
+  }
+  gpr_free(serverlist->servers);
+  gpr_free(serverlist);
+}
+
+void grpc_grpclb_response_destroy(grpc_grpclb_response *response) {
+  gpr_free(response);
+}
diff --git a/src/core/client_config/lb_policies/load_balancer_api.h b/src/core/client_config/lb_policies/load_balancer_api.h
new file mode 100644
index 0000000000..4dbe1d6c22
--- /dev/null
+++ b/src/core/client_config/lb_policies/load_balancer_api.h
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H
+#define GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H
+
+#include <grpc/support/slice_buffer.h>
+
+#include "src/core/client_config/lb_policy_factory.h"
+#include "src/core/proto/grpc/lb/v0/load_balancer.pb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128
+
+typedef grpc_lb_v0_LoadBalanceRequest grpc_grpclb_request;
+typedef grpc_lb_v0_LoadBalanceResponse grpc_grpclb_response;
+typedef grpc_lb_v0_Server grpc_grpclb_server;
+typedef grpc_lb_v0_Duration grpc_grpclb_duration;
+typedef struct grpc_grpclb_serverlist {
+  grpc_grpclb_server **servers;
+  size_t num_servers;
+  grpc_grpclb_duration expiration_interval;
+} grpc_grpclb_serverlist;
+
+/** Create a request for a gRPC LB service under \a lb_service_name */
+grpc_grpclb_request *grpc_grpclb_request_create(const char *lb_service_name);
+
+/** Protocol Buffers v3-encode \a request */
+gpr_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request);
+
+/** Destroy \a request */
+void grpc_grpclb_request_destroy(grpc_grpclb_request *request);
+
+/** Parse (ie, decode) the bytes in \a encoded_response as a \a
+ * grpc_grpclb_response */
+grpc_grpclb_response *grpc_grpclb_response_parse(gpr_slice encoded_response);
+
+/** Destroy \a serverlist */
+void grpc_grpclb_destroy_serverlist(grpc_grpclb_serverlist *serverlist);
+
+/** Parse the list of servers from an encoded \a grpc_grpclb_response */
+grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist(
+    gpr_slice encoded_response);
+
+/** Destroy \a response */
+void grpc_grpclb_response_destroy(grpc_grpclb_response *response);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_INTERNAL_CORE_CLIENT_CONFIG_LB_POLICIES_LOAD_BALANCER_API_H */
diff --git a/src/core/client_config/subchannel_index.c b/src/core/client_config/subchannel_index.c
index f78a7fd588..3f948998f9 100644
--- a/src/core/client_config/subchannel_index.c
+++ b/src/core/client_config/subchannel_index.c
@@ -149,11 +149,13 @@ static const gpr_avl_vtable subchannel_avl_vtable = {
 void grpc_subchannel_index_init(void) {
   g_subchannel_index = gpr_avl_create(&subchannel_avl_vtable);
   gpr_mu_init(&g_mu);
+  gpr_tls_init(&subchannel_index_exec_ctx);
 }
 
 void grpc_subchannel_index_shutdown(void) {
   gpr_mu_destroy(&g_mu);
   gpr_avl_unref(g_subchannel_index);
+  gpr_tls_destroy(&subchannel_index_exec_ctx);
 }
 
 grpc_subchannel *grpc_subchannel_index_find(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/compression/algorithm.c b/src/core/compression/compression_algorithm.c
index 6f3a8eb28e..6f3a8eb28e 100644
--- a/src/core/compression/algorithm.c
+++ b/src/core/compression/compression_algorithm.c
diff --git a/src/core/httpcli/httpcli_security_connector.c b/src/core/httpcli/httpcli_security_connector.c
index 41ad1de6c0..156961a377 100644
--- a/src/core/httpcli/httpcli_security_connector.c
+++ b/src/core/httpcli/httpcli_security_connector.c
@@ -59,7 +59,7 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) {
 }
 
 static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_security_connector *sc,
+                                     grpc_channel_security_connector *sc,
                                      grpc_endpoint *nonsecure_endpoint,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
@@ -78,8 +78,8 @@ static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
             tsi_result_to_string(result));
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, sc, nonsecure_endpoint, cb,
-                               user_data);
+    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
+                               nonsecure_endpoint, cb, user_data);
   }
 }
 
@@ -103,7 +103,7 @@ static void httpcli_ssl_check_peer(grpc_exec_ctx *exec_ctx,
 }
 
 static grpc_security_connector_vtable httpcli_ssl_vtable = {
-    httpcli_ssl_destroy, httpcli_ssl_do_handshake, httpcli_ssl_check_peer};
+    httpcli_ssl_destroy, httpcli_ssl_check_peer};
 
 static grpc_security_status httpcli_ssl_channel_security_connector_create(
     const unsigned char *pem_root_certs, size_t pem_root_certs_size,
@@ -121,7 +121,6 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
   memset(c, 0, sizeof(grpc_httpcli_ssl_channel_security_connector));
 
   gpr_ref_init(&c->base.base.refcount, 1);
-  c->base.base.is_client_side = 1;
   c->base.base.vtable = &httpcli_ssl_vtable;
   if (secure_peer_name != NULL) {
     c->secure_peer_name = gpr_strdup(secure_peer_name);
@@ -136,6 +135,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
     *sc = NULL;
     return GRPC_SECURITY_ERROR;
   }
+  c->base.do_handshake = httpcli_ssl_do_handshake;
   *sc = &c->base;
   return GRPC_SECURITY_OK;
 }
@@ -180,8 +180,8 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
   GPR_ASSERT(httpcli_ssl_channel_security_connector_create(
                  pem_root_certs, pem_root_certs_size, host, &sc) ==
              GRPC_SECURITY_OK);
-  grpc_security_connector_do_handshake(exec_ctx, &sc->base, tcp,
-                                       on_secure_transport_setup_done, c);
+  grpc_channel_security_connector_do_handshake(
+      exec_ctx, sc, tcp, on_secure_transport_setup_done, c);
   GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
 }
 
diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h
index c6b0214dea..6585326f81 100644
--- a/src/core/iomgr/pollset.h
+++ b/src/core/iomgr/pollset.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -83,7 +83,7 @@ void grpc_pollset_destroy(grpc_pollset *pollset);
    pollset
    lock */
 void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker *worker, gpr_timespec now,
+                       grpc_pollset_worker **worker, gpr_timespec now,
                        gpr_timespec deadline);
 
 /* Break one polling thread out of polling work for this pollset.
diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c
index 1063727248..ee7e9f48f4 100644
--- a/src/core/iomgr/pollset_posix.c
+++ b/src/core/iomgr/pollset_posix.c
@@ -246,8 +246,11 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
 }
 
 void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker *worker, gpr_timespec now,
+                       grpc_pollset_worker **worker_hdl, gpr_timespec now,
                        gpr_timespec deadline) {
+  grpc_pollset_worker worker;
+  *worker_hdl = &worker;
+
   /* pollset->mu already held */
   int added_worker = 0;
   int locked = 1;
@@ -255,16 +258,16 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   int keep_polling = 0;
   GPR_TIMER_BEGIN("grpc_pollset_work", 0);
   /* this must happen before we (potentially) drop pollset->mu */
-  worker->next = worker->prev = NULL;
-  worker->reevaluate_polling_on_wakeup = 0;
+  worker.next = worker.prev = NULL;
+  worker.reevaluate_polling_on_wakeup = 0;
   if (pollset->local_wakeup_cache != NULL) {
-    worker->wakeup_fd = pollset->local_wakeup_cache;
-    pollset->local_wakeup_cache = worker->wakeup_fd->next;
+    worker.wakeup_fd = pollset->local_wakeup_cache;
+    pollset->local_wakeup_cache = worker.wakeup_fd->next;
   } else {
-    worker->wakeup_fd = gpr_malloc(sizeof(*worker->wakeup_fd));
-    grpc_wakeup_fd_init(&worker->wakeup_fd->fd);
+    worker.wakeup_fd = gpr_malloc(sizeof(*worker.wakeup_fd));
+    grpc_wakeup_fd_init(&worker.wakeup_fd->fd);
   }
-  worker->kicked_specifically = 0;
+  worker.kicked_specifically = 0;
   /* If there's work waiting for the pollset to be idle, and the
      pollset is idle, then do that work */
   if (!grpc_pollset_has_workers(pollset) &&
@@ -293,13 +296,13 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     keep_polling = 0;
     if (!pollset->kicked_without_pollers) {
       if (!added_worker) {
-        push_front_worker(pollset, worker);
+        push_front_worker(pollset, &worker);
         added_worker = 1;
-        gpr_tls_set(&g_current_thread_worker, (intptr_t)worker);
+        gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
       }
       gpr_tls_set(&g_current_thread_poller, (intptr_t)pollset);
       GPR_TIMER_BEGIN("maybe_work_and_unlock", 0);
-      pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, worker,
+      pollset->vtable->maybe_work_and_unlock(exec_ctx, pollset, &worker,
                                              deadline, now);
       GPR_TIMER_END("maybe_work_and_unlock", 0);
       locked = 0;
@@ -321,10 +324,10 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     /* If we're forced to re-evaluate polling (via grpc_pollset_kick with
        GRPC_POLLSET_REEVALUATE_POLLING_ON_WAKEUP) then we land here and force
        a loop */
-    if (worker->reevaluate_polling_on_wakeup) {
-      worker->reevaluate_polling_on_wakeup = 0;
+    if (worker.reevaluate_polling_on_wakeup) {
+      worker.reevaluate_polling_on_wakeup = 0;
       pollset->kicked_without_pollers = 0;
-      if (queued_work || worker->kicked_specifically) {
+      if (queued_work || worker.kicked_specifically) {
         /* If there's queued work on the list, then set the deadline to be
            immediate so we get back out of the polling loop quickly */
         deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
@@ -333,12 +336,12 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     }
   }
   if (added_worker) {
-    remove_worker(pollset, worker);
+    remove_worker(pollset, &worker);
     gpr_tls_set(&g_current_thread_worker, 0);
   }
   /* release wakeup fd to the local pool */
-  worker->wakeup_fd->next = pollset->local_wakeup_cache;
-  pollset->local_wakeup_cache = worker->wakeup_fd;
+  worker.wakeup_fd->next = pollset->local_wakeup_cache;
+  pollset->local_wakeup_cache = worker.wakeup_fd;
   /* check shutdown conditions */
   if (pollset->shutting_down) {
     if (grpc_pollset_has_workers(pollset)) {
@@ -360,6 +363,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       gpr_mu_lock(&pollset->mu);
     }
   }
+  *worker_hdl = NULL;
   GPR_TIMER_END("grpc_pollset_work", 0);
 }
 
diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c
index 35a956b27f..bbce23b46a 100644
--- a/src/core/iomgr/pollset_windows.c
+++ b/src/core/iomgr/pollset_windows.c
@@ -125,22 +125,25 @@ void grpc_pollset_reset(grpc_pollset *pollset) {
 }
 
 void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                       grpc_pollset_worker *worker, gpr_timespec now,
+                       grpc_pollset_worker **worker_hdl, gpr_timespec now,
                        gpr_timespec deadline) {
+  grpc_pollset_worker worker;
+  *worker_hdl = &worker;
+
   int added_worker = 0;
-  worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next =
-      worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev =
-          worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next =
-              worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL;
-  worker->kicked = 0;
-  worker->pollset = pollset;
-  gpr_cv_init(&worker->cv);
+  worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next =
+      worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev =
+          worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next =
+              worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL;
+  worker.kicked = 0;
+  worker.pollset = pollset;
+  gpr_cv_init(&worker.cv);
   if (!pollset->kicked_without_pollers && !pollset->shutting_down) {
     if (g_active_poller == NULL) {
       grpc_pollset_worker *next_worker;
       /* become poller */
       pollset->is_iocp_worker = 1;
-      g_active_poller = worker;
+      g_active_poller = &worker;
       gpr_mu_unlock(&grpc_polling_mu);
       grpc_iocp_work(exec_ctx, deadline);
       grpc_exec_ctx_flush(exec_ctx);
@@ -167,12 +170,12 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       goto done;
     }
     push_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL,
-                      worker);
+                      &worker);
     push_front_worker(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET,
-                      worker);
+                      &worker);
     added_worker = 1;
-    while (!worker->kicked) {
-      if (gpr_cv_wait(&worker->cv, &grpc_polling_mu, deadline)) {
+    while (!worker.kicked) {
+      if (gpr_cv_wait(&worker.cv, &grpc_polling_mu, deadline)) {
         break;
       }
     }
@@ -186,10 +189,11 @@ done:
     gpr_mu_lock(&grpc_polling_mu);
   }
   if (added_worker) {
-    remove_worker(worker, GRPC_POLLSET_WORKER_LINK_GLOBAL);
-    remove_worker(worker, GRPC_POLLSET_WORKER_LINK_POLLSET);
+    remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_GLOBAL);
+    remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_POLLSET);
   }
-  gpr_cv_destroy(&worker->cv);
+  gpr_cv_destroy(&worker.cv);
+  *worker_hdl = NULL;
 }
 
 void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.c b/src/core/proto/grpc/lb/v0/load_balancer.pb.c
new file mode 100644
index 0000000000..59aae30cff
--- /dev/null
+++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.c
@@ -0,0 +1,119 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.3.5-dev */
+
+#include "src/core/proto/grpc/lb/v0/load_balancer.pb.h"
+
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+
+
+const pb_field_t grpc_lb_v0_Duration_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_Duration, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Duration, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v0_InitialLoadBalanceRequest_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v0_ClientStats_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_InitialLoadBalanceRequest, name, name, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_ClientStats_fields[4] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_ClientStats, total_requests, total_requests, 0),
+    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_ClientStats, client_rpc_errors, total_requests, 0),
+    PB_FIELD(  3, INT64   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_ClientStats, dropped_requests, client_rpc_errors, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_LoadBalanceResponse, initial_response, initial_response, &grpc_lb_v0_InitialLoadBalanceResponse_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_LoadBalanceResponse, server_list, initial_response, &grpc_lb_v0_ServerList_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_InitialLoadBalanceResponse, client_config, client_config, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, load_balancer_delegate, client_config, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v0_Duration_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_ServerList_fields[3] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, grpc_lb_v0_ServerList, servers, servers, &grpc_lb_v0_Server_fields),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_ServerList, expiration_interval, servers, &grpc_lb_v0_Duration_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t grpc_lb_v0_Server_fields[5] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v0_Server, ip_address, ip_address, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Server, port, ip_address, 0),
+    PB_FIELD(  3, BYTES   , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Server, load_balance_token, port, 0),
+    PB_FIELD(  4, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v0_Server, drop_request, load_balance_token, 0),
+    PB_LAST_FIELD
+};
+
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_32BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in 8 or 16 bit
+ * field descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v0_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server)
+#endif
+
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_16BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in the default
+ * 8 bit descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(grpc_lb_v0_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v0_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v0_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v0_ServerList, servers) < 256 && pb_membersize(grpc_lb_v0_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v0_Duration_grpc_lb_v0_LoadBalanceRequest_grpc_lb_v0_InitialLoadBalanceRequest_grpc_lb_v0_ClientStats_grpc_lb_v0_LoadBalanceResponse_grpc_lb_v0_InitialLoadBalanceResponse_grpc_lb_v0_ServerList_grpc_lb_v0_Server)
+#endif
+
+
diff --git a/src/core/proto/grpc/lb/v0/load_balancer.pb.h b/src/core/proto/grpc/lb/v0/load_balancer.pb.h
new file mode 100644
index 0000000000..3599f881bb
--- /dev/null
+++ b/src/core/proto/grpc/lb/v0/load_balancer.pb.h
@@ -0,0 +1,182 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.3.5-dev */
+
+#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED
+#define PB_LOAD_BALANCER_PB_H_INCLUDED
+#include "third_party/nanopb/pb.h"
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Struct definitions */
+typedef struct _grpc_lb_v0_ClientStats {
+    bool has_total_requests;
+    int64_t total_requests;
+    bool has_client_rpc_errors;
+    int64_t client_rpc_errors;
+    bool has_dropped_requests;
+    int64_t dropped_requests;
+} grpc_lb_v0_ClientStats;
+
+typedef struct _grpc_lb_v0_Duration {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} grpc_lb_v0_Duration;
+
+typedef struct _grpc_lb_v0_InitialLoadBalanceRequest {
+    bool has_name;
+    char name[128];
+} grpc_lb_v0_InitialLoadBalanceRequest;
+
+typedef PB_BYTES_ARRAY_T(64) grpc_lb_v0_Server_load_balance_token_t;
+typedef struct _grpc_lb_v0_Server {
+    bool has_ip_address;
+    char ip_address[46];
+    bool has_port;
+    int32_t port;
+    bool has_load_balance_token;
+    grpc_lb_v0_Server_load_balance_token_t load_balance_token;
+    bool has_drop_request;
+    bool drop_request;
+} grpc_lb_v0_Server;
+
+typedef struct _grpc_lb_v0_InitialLoadBalanceResponse {
+    bool has_client_config;
+    char client_config[64];
+    bool has_load_balancer_delegate;
+    char load_balancer_delegate[64];
+    bool has_client_stats_report_interval;
+    grpc_lb_v0_Duration client_stats_report_interval;
+} grpc_lb_v0_InitialLoadBalanceResponse;
+
+typedef struct _grpc_lb_v0_LoadBalanceRequest {
+    bool has_initial_request;
+    grpc_lb_v0_InitialLoadBalanceRequest initial_request;
+    bool has_client_stats;
+    grpc_lb_v0_ClientStats client_stats;
+} grpc_lb_v0_LoadBalanceRequest;
+
+typedef struct _grpc_lb_v0_ServerList {
+    pb_callback_t servers;
+    bool has_expiration_interval;
+    grpc_lb_v0_Duration expiration_interval;
+} grpc_lb_v0_ServerList;
+
+typedef struct _grpc_lb_v0_LoadBalanceResponse {
+    bool has_initial_response;
+    grpc_lb_v0_InitialLoadBalanceResponse initial_response;
+    bool has_server_list;
+    grpc_lb_v0_ServerList server_list;
+} grpc_lb_v0_LoadBalanceResponse;
+
+/* Default values for struct fields */
+
+/* Initializer values for message structs */
+#define grpc_lb_v0_Duration_init_default         {false, 0, false, 0}
+#define grpc_lb_v0_LoadBalanceRequest_init_default {false, grpc_lb_v0_InitialLoadBalanceRequest_init_default, false, grpc_lb_v0_ClientStats_init_default}
+#define grpc_lb_v0_InitialLoadBalanceRequest_init_default {false, ""}
+#define grpc_lb_v0_ClientStats_init_default      {false, 0, false, 0, false, 0}
+#define grpc_lb_v0_LoadBalanceResponse_init_default {false, grpc_lb_v0_InitialLoadBalanceResponse_init_default, false, grpc_lb_v0_ServerList_init_default}
+#define grpc_lb_v0_InitialLoadBalanceResponse_init_default {false, "", false, "", false, grpc_lb_v0_Duration_init_default}
+#define grpc_lb_v0_ServerList_init_default       {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_default}
+#define grpc_lb_v0_Server_init_default           {false, "", false, 0, false, {0, {0}}, false, 0}
+#define grpc_lb_v0_Duration_init_zero            {false, 0, false, 0}
+#define grpc_lb_v0_LoadBalanceRequest_init_zero  {false, grpc_lb_v0_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v0_ClientStats_init_zero}
+#define grpc_lb_v0_InitialLoadBalanceRequest_init_zero {false, ""}
+#define grpc_lb_v0_ClientStats_init_zero         {false, 0, false, 0, false, 0}
+#define grpc_lb_v0_LoadBalanceResponse_init_zero {false, grpc_lb_v0_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v0_ServerList_init_zero}
+#define grpc_lb_v0_InitialLoadBalanceResponse_init_zero {false, "", false, "", false, grpc_lb_v0_Duration_init_zero}
+#define grpc_lb_v0_ServerList_init_zero          {{{NULL}, NULL}, false, grpc_lb_v0_Duration_init_zero}
+#define grpc_lb_v0_Server_init_zero              {false, "", false, 0, false, {0, {0}}, false, 0}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define grpc_lb_v0_ClientStats_total_requests_tag 1
+#define grpc_lb_v0_ClientStats_client_rpc_errors_tag 2
+#define grpc_lb_v0_ClientStats_dropped_requests_tag 3
+#define grpc_lb_v0_Duration_seconds_tag          1
+#define grpc_lb_v0_Duration_nanos_tag            2
+#define grpc_lb_v0_InitialLoadBalanceRequest_name_tag 1
+#define grpc_lb_v0_Server_ip_address_tag         1
+#define grpc_lb_v0_Server_port_tag               2
+#define grpc_lb_v0_Server_load_balance_token_tag 3
+#define grpc_lb_v0_Server_drop_request_tag       4
+#define grpc_lb_v0_InitialLoadBalanceResponse_client_config_tag 1
+#define grpc_lb_v0_InitialLoadBalanceResponse_load_balancer_delegate_tag 2
+#define grpc_lb_v0_InitialLoadBalanceResponse_client_stats_report_interval_tag 3
+#define grpc_lb_v0_LoadBalanceRequest_initial_request_tag 1
+#define grpc_lb_v0_LoadBalanceRequest_client_stats_tag 2
+#define grpc_lb_v0_ServerList_servers_tag        1
+#define grpc_lb_v0_ServerList_expiration_interval_tag 3
+#define grpc_lb_v0_LoadBalanceResponse_initial_response_tag 1
+#define grpc_lb_v0_LoadBalanceResponse_server_list_tag 2
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t grpc_lb_v0_Duration_fields[3];
+extern const pb_field_t grpc_lb_v0_LoadBalanceRequest_fields[3];
+extern const pb_field_t grpc_lb_v0_InitialLoadBalanceRequest_fields[2];
+extern const pb_field_t grpc_lb_v0_ClientStats_fields[4];
+extern const pb_field_t grpc_lb_v0_LoadBalanceResponse_fields[3];
+extern const pb_field_t grpc_lb_v0_InitialLoadBalanceResponse_fields[4];
+extern const pb_field_t grpc_lb_v0_ServerList_fields[3];
+extern const pb_field_t grpc_lb_v0_Server_fields[5];
+
+/* Maximum encoded size of messages (where known) */
+#define grpc_lb_v0_Duration_size                 22
+#define grpc_lb_v0_LoadBalanceRequest_size       169
+#define grpc_lb_v0_InitialLoadBalanceRequest_size 131
+#define grpc_lb_v0_ClientStats_size              33
+#define grpc_lb_v0_LoadBalanceResponse_size      (165 + grpc_lb_v0_ServerList_size)
+#define grpc_lb_v0_InitialLoadBalanceResponse_size 156
+#define grpc_lb_v0_Server_size                   127
+
+/* Message IDs (where set with "msgid" option) */
+#ifdef PB_MSGID
+
+#define LOAD_BALANCER_MESSAGES \
+
+
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/src/core/security/base64.c b/src/core/security/b64.c
index 8367c160c3..c40b528e2f 100644
--- a/src/core/security/base64.c
+++ b/src/core/security/b64.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/security/base64.h"
+#include "src/core/security/b64.h"
 
 #include <stdint.h>
 #include <string.h>
diff --git a/src/core/security/base64.h b/src/core/security/b64.h
index 31ae982691..3e3b521120 100644
--- a/src/core/security/base64.h
+++ b/src/core/security/b64.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c
index 57b367d00f..332d4259d2 100644
--- a/src/core/security/client_auth_filter.c
+++ b/src/core/security/client_auth_filter.c
@@ -310,7 +310,6 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(auth_context != NULL);
 
   /* initialize members */
-  GPR_ASSERT(sc->is_client_side);
   chand->security_connector =
       (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF(
           sc, "client_auth_filter");
diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c
index c58574bd6d..b4fa616fa7 100644
--- a/src/core/security/credentials.c
+++ b/src/core/security/credentials.c
@@ -166,7 +166,7 @@ void grpc_server_credentials_release(grpc_server_credentials *creds) {
 }
 
 grpc_security_status grpc_server_credentials_create_security_connector(
-    grpc_server_credentials *creds, grpc_security_connector **sc) {
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
   if (creds == NULL || creds->vtable->create_security_connector == NULL) {
     gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
     return GRPC_SECURITY_ERROR;
@@ -298,7 +298,7 @@ static grpc_security_status ssl_create_security_connector(
 }
 
 static grpc_security_status ssl_server_create_security_connector(
-    grpc_server_credentials *creds, grpc_security_connector **sc) {
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
   grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
   return grpc_ssl_server_security_connector_create(&c->config, sc);
 }
@@ -894,7 +894,7 @@ static grpc_security_status fake_transport_security_create_security_connector(
 
 static grpc_security_status
 fake_transport_security_server_create_security_connector(
-    grpc_server_credentials *c, grpc_security_connector **sc) {
+    grpc_server_credentials *c, grpc_server_security_connector **sc) {
   *sc = grpc_fake_server_security_connector_create();
   return GRPC_SECURITY_OK;
 }
diff --git a/src/core/security/credentials.h b/src/core/security/credentials.h
index 3cd652cd57..0de4cd9468 100644
--- a/src/core/security/credentials.h
+++ b/src/core/security/credentials.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -234,7 +234,7 @@ grpc_refresh_token_credentials_create_from_auth_refresh_token(
 typedef struct {
   void (*destruct)(grpc_server_credentials *c);
   grpc_security_status (*create_security_connector)(
-      grpc_server_credentials *c, grpc_security_connector **sc);
+      grpc_server_credentials *c, grpc_server_security_connector **sc);
 } grpc_server_credentials_vtable;
 
 struct grpc_server_credentials {
@@ -245,7 +245,7 @@ struct grpc_server_credentials {
 };
 
 grpc_security_status grpc_server_credentials_create_security_connector(
-    grpc_server_credentials *creds, grpc_security_connector **sc);
+    grpc_server_credentials *creds, grpc_server_security_connector **sc);
 
 grpc_server_credentials *grpc_server_credentials_ref(
     grpc_server_credentials *creds);
diff --git a/src/core/security/google_default_credentials.c b/src/core/security/google_default_credentials.c
index f3ac14568a..458d0d3ac3 100644
--- a/src/core/security/google_default_credentials.c
+++ b/src/core/security/google_default_credentials.c
@@ -41,7 +41,7 @@
 
 #include "src/core/httpcli/httpcli.h"
 #include "src/core/support/env.h"
-#include "src/core/support/file.h"
+#include "src/core/support/load_file.h"
 #include "src/core/surface/api_trace.h"
 
 /* -- Constants. -- */
@@ -122,7 +122,7 @@ static int is_stack_running_on_compute_engine(void) {
      called once for the lifetime of the process by the default credentials. */
   gpr_mu_lock(GRPC_POLLSET_MU(&detector.pollset));
   while (!detector.is_done) {
-    grpc_pollset_worker worker;
+    grpc_pollset_worker *worker = NULL;
     grpc_pollset_work(&exec_ctx, &detector.pollset, &worker,
                       gpr_now(GPR_CLOCK_MONOTONIC),
                       gpr_inf_future(GPR_CLOCK_MONOTONIC));
diff --git a/src/core/security/handshake.c b/src/core/security/handshake.c
index a8b2fef629..b5bb6667a7 100644
--- a/src/core/security/handshake.c
+++ b/src/core/security/handshake.c
@@ -33,6 +33,7 @@
 
 #include "src/core/security/handshake.h"
 
+#include <stdbool.h>
 #include <string.h>
 
 #include "src/core/security/security_context.h"
@@ -46,6 +47,7 @@
 typedef struct {
   grpc_security_connector *connector;
   tsi_handshaker *handshaker;
+  bool is_client_side;
   unsigned char *handshake_buffer;
   size_t handshake_buffer_size;
   grpc_endpoint *wrapped_endpoint;
@@ -67,9 +69,11 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void *setup,
                                            bool success);
 
 static void security_connector_remove_handshake(grpc_security_handshake *h) {
+  GPR_ASSERT(!h->is_client_side);
   grpc_security_connector_handshake_list *node;
   grpc_security_connector_handshake_list *tmp;
-  grpc_security_connector *sc = h->connector;
+  grpc_server_security_connector *sc =
+      (grpc_server_security_connector *)h->connector;
   gpr_mu_lock(&sc->mu);
   node = sc->handshaking_handshakes;
   if (node && node->handshake == h) {
@@ -94,7 +98,7 @@ static void security_connector_remove_handshake(grpc_security_handshake *h) {
 static void security_handshake_done(grpc_exec_ctx *exec_ctx,
                                     grpc_security_handshake *h,
                                     int is_success) {
-  if (!h->connector->is_client_side) {
+  if (!h->is_client_side) {
     security_connector_remove_handshake(h);
   }
   if (is_success) {
@@ -290,6 +294,7 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
 void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
                                 tsi_handshaker *handshaker,
                                 grpc_security_connector *connector,
+                                bool is_client_side,
                                 grpc_endpoint *nonsecure_endpoint,
                                 grpc_security_handshake_done_cb cb,
                                 void *user_data) {
@@ -298,6 +303,7 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
   memset(h, 0, sizeof(grpc_security_handshake));
   h->handshaker = handshaker;
   h->connector = GRPC_SECURITY_CONNECTOR_REF(connector, "handshake");
+  h->is_client_side = is_client_side;
   h->handshake_buffer_size = GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE;
   h->handshake_buffer = gpr_malloc(h->handshake_buffer_size);
   h->wrapped_endpoint = nonsecure_endpoint;
@@ -310,13 +316,15 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
   gpr_slice_buffer_init(&h->left_overs);
   gpr_slice_buffer_init(&h->outgoing);
   gpr_slice_buffer_init(&h->incoming);
-  if (!connector->is_client_side) {
+  if (!is_client_side) {
+    grpc_server_security_connector *server_connector =
+        (grpc_server_security_connector *)connector;
     handshake_node = gpr_malloc(sizeof(grpc_security_connector_handshake_list));
     handshake_node->handshake = h;
-    gpr_mu_lock(&connector->mu);
-    handshake_node->next = connector->handshaking_handshakes;
-    connector->handshaking_handshakes = handshake_node;
-    gpr_mu_unlock(&connector->mu);
+    gpr_mu_lock(&server_connector->mu);
+    handshake_node->next = server_connector->handshaking_handshakes;
+    server_connector->handshaking_handshakes = handshake_node;
+    gpr_mu_unlock(&server_connector->mu);
   }
   send_handshake_bytes_to_peer(exec_ctx, h);
 }
diff --git a/src/core/security/handshake.h b/src/core/security/handshake.h
index 44215d16ef..db8b374921 100644
--- a/src/core/security/handshake.h
+++ b/src/core/security/handshake.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,6 +41,7 @@
 void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
                                 tsi_handshaker *handshaker,
                                 grpc_security_connector *connector,
+                                bool is_client_side,
                                 grpc_endpoint *nonsecure_endpoint,
                                 grpc_security_handshake_done_cb cb,
                                 void *user_data);
diff --git a/src/core/security/json_token.c b/src/core/security/json_token.c
index 762f02989a..372e5bfc5a 100644
--- a/src/core/security/json_token.c
+++ b/src/core/security/json_token.c
@@ -39,7 +39,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/security/base64.h"
+#include "src/core/security/b64.h"
 #include "src/core/support/string.h"
 
 #include <openssl/bio.h>
diff --git a/src/core/security/jwt_verifier.c b/src/core/security/jwt_verifier.c
index 042a117f5d..928c6c148d 100644
--- a/src/core/security/jwt_verifier.c
+++ b/src/core/security/jwt_verifier.c
@@ -37,7 +37,7 @@
 #include <string.h>
 
 #include "src/core/httpcli/httpcli.h"
-#include "src/core/security/base64.h"
+#include "src/core/security/b64.h"
 #include "src/core/tsi/ssl_types.h"
 
 #include <grpc/support/alloc.h>
diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c
index b46205323b..33c62a20c2 100644
--- a/src/core/security/security_connector.c
+++ b/src/core/security/security_connector.c
@@ -33,22 +33,23 @@
 
 #include "src/core/security/security_connector.h"
 
+#include <stdbool.h>
 #include <string.h>
 
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
+#include <grpc/support/slice_buffer.h>
+#include <grpc/support/string_util.h>
+
 #include "src/core/security/credentials.h"
 #include "src/core/security/handshake.h"
 #include "src/core/security/secure_endpoint.h"
 #include "src/core/security/security_context.h"
 #include "src/core/support/env.h"
-#include "src/core/support/file.h"
+#include "src/core/support/load_file.h"
 #include "src/core/support/string.h"
 #include "src/core/transport/chttp2/alpn.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/host_port.h>
-#include <grpc/support/log.h>
-#include <grpc/support/slice_buffer.h>
-#include <grpc/support/string_util.h>
 #include "src/core/tsi/fake_transport_security.h"
 #include "src/core/tsi/ssl_transport_security.h"
 
@@ -110,31 +111,39 @@ const tsi_peer_property *tsi_peer_get_property_by_name(const tsi_peer *peer,
   return NULL;
 }
 
-void grpc_security_connector_shutdown(grpc_exec_ctx *exec_ctx,
-                                      grpc_security_connector *connector) {
+void grpc_server_security_connector_shutdown(
+    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector) {
   grpc_security_connector_handshake_list *tmp;
-  if (!connector->is_client_side) {
-    gpr_mu_lock(&connector->mu);
-    while (connector->handshaking_handshakes) {
-      tmp = connector->handshaking_handshakes;
-      grpc_security_handshake_shutdown(
-          exec_ctx, connector->handshaking_handshakes->handshake);
-      connector->handshaking_handshakes = tmp->next;
-      gpr_free(tmp);
-    }
-    gpr_mu_unlock(&connector->mu);
+  gpr_mu_lock(&connector->mu);
+  while (connector->handshaking_handshakes) {
+    tmp = connector->handshaking_handshakes;
+    grpc_security_handshake_shutdown(
+        exec_ctx, connector->handshaking_handshakes->handshake);
+    connector->handshaking_handshakes = tmp->next;
+    gpr_free(tmp);
   }
+  gpr_mu_unlock(&connector->mu);
 }
 
-void grpc_security_connector_do_handshake(grpc_exec_ctx *exec_ctx,
-                                          grpc_security_connector *sc,
-                                          grpc_endpoint *nonsecure_endpoint,
-                                          grpc_security_handshake_done_cb cb,
-                                          void *user_data) {
+void grpc_channel_security_connector_do_handshake(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb,
+    void *user_data) {
+  if (sc == NULL || nonsecure_endpoint == NULL) {
+    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
+  } else {
+    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data);
+  }
+}
+
+void grpc_server_security_connector_do_handshake(
+    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
+    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
+    grpc_security_handshake_done_cb cb, void *user_data) {
   if (sc == NULL || nonsecure_endpoint == NULL) {
     cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   } else {
-    sc->vtable->do_handshake(exec_ctx, sc, nonsecure_endpoint, cb, user_data);
+    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, cb, user_data);
   }
 }
 
@@ -248,7 +257,8 @@ static void fake_channel_destroy(grpc_security_connector *sc) {
 }
 
 static void fake_server_destroy(grpc_security_connector *sc) {
-  gpr_mu_destroy(&sc->mu);
+  grpc_server_security_connector *c = (grpc_server_security_connector *)sc;
+  gpr_mu_destroy(&c->mu);
   gpr_free(sc);
 }
 
@@ -298,49 +308,52 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
 }
 
 static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx,
-                                      grpc_security_connector *sc,
+                                      grpc_channel_security_connector *sc,
                                       grpc_endpoint *nonsecure_endpoint,
                                       grpc_security_handshake_done_cb cb,
                                       void *user_data) {
-  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), sc,
-                             nonsecure_endpoint, cb, user_data);
+  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base,
+                             true, nonsecure_endpoint, cb, user_data);
 }
 
 static void fake_server_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_security_connector *sc,
+                                     grpc_server_security_connector *sc,
+                                     grpc_tcp_server_acceptor *acceptor,
                                      grpc_endpoint *nonsecure_endpoint,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
-  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), sc,
-                             nonsecure_endpoint, cb, user_data);
+  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base,
+                             false, nonsecure_endpoint, cb, user_data);
 }
 
 static grpc_security_connector_vtable fake_channel_vtable = {
-    fake_channel_destroy, fake_channel_do_handshake, fake_check_peer};
+    fake_channel_destroy, fake_check_peer};
 
-static grpc_security_connector_vtable fake_server_vtable = {
-    fake_server_destroy, fake_server_do_handshake, fake_check_peer};
+static grpc_security_connector_vtable fake_server_vtable = {fake_server_destroy,
+                                                            fake_check_peer};
 
 grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
     grpc_call_credentials *request_metadata_creds) {
   grpc_channel_security_connector *c = gpr_malloc(sizeof(*c));
   memset(c, 0, sizeof(*c));
   gpr_ref_init(&c->base.refcount, 1);
-  c->base.is_client_side = 1;
   c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME;
   c->base.vtable = &fake_channel_vtable;
   c->request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds);
   c->check_call_host = fake_channel_check_call_host;
+  c->do_handshake = fake_channel_do_handshake;
   return c;
 }
 
-grpc_security_connector *grpc_fake_server_security_connector_create(void) {
-  grpc_security_connector *c = gpr_malloc(sizeof(grpc_security_connector));
-  memset(c, 0, sizeof(grpc_security_connector));
-  gpr_ref_init(&c->refcount, 1);
-  c->is_client_side = 0;
-  c->vtable = &fake_server_vtable;
-  c->url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME;
+grpc_server_security_connector *grpc_fake_server_security_connector_create(
+    void) {
+  grpc_server_security_connector *c =
+      gpr_malloc(sizeof(grpc_server_security_connector));
+  memset(c, 0, sizeof(*c));
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &fake_server_vtable;
+  c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME;
+  c->do_handshake = fake_server_do_handshake;
   gpr_mu_init(&c->mu);
   return c;
 }
@@ -355,7 +368,7 @@ typedef struct {
 } grpc_ssl_channel_security_connector;
 
 typedef struct {
-  grpc_security_connector base;
+  grpc_server_security_connector base;
   tsi_ssl_handshaker_factory *handshaker_factory;
 } grpc_ssl_server_security_connector;
 
@@ -378,12 +391,12 @@ static void ssl_server_destroy(grpc_security_connector *sc) {
   if (c->handshaker_factory != NULL) {
     tsi_ssl_handshaker_factory_destroy(c->handshaker_factory);
   }
-  gpr_mu_destroy(&sc->mu);
+  gpr_mu_destroy(&c->base.mu);
   gpr_free(sc);
 }
 
 static grpc_security_status ssl_create_handshaker(
-    tsi_ssl_handshaker_factory *handshaker_factory, int is_client,
+    tsi_ssl_handshaker_factory *handshaker_factory, bool is_client,
     const char *peer_name, tsi_handshaker **handshaker) {
   tsi_result result = TSI_OK;
   if (handshaker_factory == NULL) return GRPC_SECURITY_ERROR;
@@ -398,7 +411,7 @@ static grpc_security_status ssl_create_handshaker(
 }
 
 static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_security_connector *sc,
+                                     grpc_channel_security_connector *sc,
                                      grpc_endpoint *nonsecure_endpoint,
                                      grpc_security_handshake_done_cb cb,
                                      void *user_data) {
@@ -406,20 +419,21 @@ static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
       (grpc_ssl_channel_security_connector *)sc;
   tsi_handshaker *handshaker;
   grpc_security_status status = ssl_create_handshaker(
-      c->handshaker_factory, 1,
+      c->handshaker_factory, true,
       c->overridden_target_name != NULL ? c->overridden_target_name
                                         : c->target_name,
       &handshaker);
   if (status != GRPC_SECURITY_OK) {
     cb(exec_ctx, user_data, status, NULL, NULL);
   } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, sc, nonsecure_endpoint, cb,
-                               user_data);
+    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
+                               nonsecure_endpoint, cb, user_data);
   }
 }
 
 static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx,
-                                    grpc_security_connector *sc,
+                                    grpc_server_security_connector *sc,
+                                    grpc_tcp_server_acceptor *acceptor,
                                     grpc_endpoint *nonsecure_endpoint,
                                     grpc_security_handshake_done_cb cb,
                                     void *user_data) {
@@ -427,12 +441,12 @@ static void ssl_server_do_handshake(grpc_exec_ctx *exec_ctx,
       (grpc_ssl_server_security_connector *)sc;
   tsi_handshaker *handshaker;
   grpc_security_status status =
-      ssl_create_handshaker(c->handshaker_factory, 0, NULL, &handshaker);
+      ssl_create_handshaker(c->handshaker_factory, false, NULL, &handshaker);
   if (status != GRPC_SECURITY_OK) {
     cb(exec_ctx, user_data, status, NULL, NULL);
   } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, sc, nonsecure_endpoint, cb,
-                               user_data);
+    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false,
+                               nonsecure_endpoint, cb, user_data);
   }
 }
 
@@ -603,10 +617,10 @@ static void ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
 }
 
 static grpc_security_connector_vtable ssl_channel_vtable = {
-    ssl_channel_destroy, ssl_channel_do_handshake, ssl_channel_check_peer};
+    ssl_channel_destroy, ssl_channel_check_peer};
 
 static grpc_security_connector_vtable ssl_server_vtable = {
-    ssl_server_destroy, ssl_server_do_handshake, ssl_server_check_peer};
+    ssl_server_destroy, ssl_server_check_peer};
 
 static gpr_slice compute_default_pem_root_certs_once(void) {
   gpr_slice result = gpr_empty_slice();
@@ -700,11 +714,11 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
 
   gpr_ref_init(&c->base.base.refcount, 1);
   c->base.base.vtable = &ssl_channel_vtable;
-  c->base.base.is_client_side = 1;
   c->base.base.url_scheme = GRPC_SSL_URL_SCHEME;
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = ssl_channel_check_call_host;
+  c->base.do_handshake = ssl_channel_do_handshake;
   gpr_split_host_port(target_name, &c->target_name, &port);
   gpr_free(port);
   if (overridden_target_name != NULL) {
@@ -735,7 +749,7 @@ error:
 }
 
 grpc_security_status grpc_ssl_server_security_connector_create(
-    const grpc_ssl_server_config *config, grpc_security_connector **sc) {
+    const grpc_ssl_server_config *config, grpc_server_security_connector **sc) {
   size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions();
   const unsigned char **alpn_protocol_strings =
       gpr_malloc(sizeof(const char *) * num_alpn_protocols);
@@ -759,9 +773,9 @@ grpc_security_status grpc_ssl_server_security_connector_create(
   c = gpr_malloc(sizeof(grpc_ssl_server_security_connector));
   memset(c, 0, sizeof(grpc_ssl_server_security_connector));
 
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.url_scheme = GRPC_SSL_URL_SCHEME;
-  c->base.vtable = &ssl_server_vtable;
+  gpr_ref_init(&c->base.base.refcount, 1);
+  c->base.base.url_scheme = GRPC_SSL_URL_SCHEME;
+  c->base.base.vtable = &ssl_server_vtable;
   result = tsi_create_ssl_server_handshaker_factory(
       (const unsigned char **)config->pem_private_keys,
       config->pem_private_keys_sizes,
@@ -774,11 +788,12 @@ grpc_security_status grpc_ssl_server_security_connector_create(
   if (result != TSI_OK) {
     gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.",
             tsi_result_to_string(result));
-    ssl_server_destroy(&c->base);
+    ssl_server_destroy(&c->base.base);
     *sc = NULL;
     goto error;
   }
   gpr_mu_init(&c->base.mu);
+  c->base.do_handshake = ssl_server_do_handshake;
   *sc = &c->base;
   gpr_free((void *)alpn_protocol_strings);
   gpr_free(alpn_protocol_string_lengths);
diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h
index 39df7821f0..1e35d3f9b7 100644
--- a/src/core/security/security_connector.h
+++ b/src/core/security/security_connector.h
@@ -36,6 +36,7 @@
 
 #include <grpc/grpc_security.h>
 #include "src/core/iomgr/endpoint.h"
+#include "src/core/iomgr/tcp_server.h"
 #include "src/core/tsi/transport_security_interface.h"
 
 /* --- status enum. --- */
@@ -68,9 +69,6 @@ typedef void (*grpc_security_handshake_done_cb)(
 
 typedef struct {
   void (*destroy)(grpc_security_connector *sc);
-  void (*do_handshake)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc,
-                       grpc_endpoint *nonsecure_endpoint,
-                       grpc_security_handshake_done_cb cb, void *user_data);
   void (*check_peer)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc,
                      tsi_peer peer, grpc_security_peer_check_cb cb,
                      void *user_data);
@@ -84,13 +82,7 @@ typedef struct grpc_security_connector_handshake_list {
 struct grpc_security_connector {
   const grpc_security_connector_vtable *vtable;
   gpr_refcount refcount;
-  int is_client_side;
   const char *url_scheme;
-  /* Used on server side only. */
-  /* TODO(yangg): Create a grpc_server_security_connector with these. */
-  gpr_mu mu;
-  grpc_security_connector_handshake_list *handshaking_handshakes;
-  const grpc_channel_args *channel_args;
 };
 
 /* Refcounting. */
@@ -113,13 +105,6 @@ grpc_security_connector *grpc_security_connector_ref(
 void grpc_security_connector_unref(grpc_security_connector *policy);
 #endif
 
-/* Handshake. */
-void grpc_security_connector_do_handshake(grpc_exec_ctx *exec_ctx,
-                                          grpc_security_connector *connector,
-                                          grpc_endpoint *nonsecure_endpoint,
-                                          grpc_security_handshake_done_cb cb,
-                                          void *user_data);
-
 /* Check the peer. Callee takes ownership of the peer object.
    The callback will include the resulting auth_context. */
 void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
@@ -128,9 +113,6 @@ void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
                                         grpc_security_peer_check_cb cb,
                                         void *user_data);
 
-void grpc_security_connector_shutdown(grpc_exec_ctx *exec_ctx,
-                                      grpc_security_connector *connector);
-
 /* Util to encapsulate the connector in a channel arg. */
 grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc);
 
@@ -153,12 +135,16 @@ typedef void (*grpc_security_call_host_check_cb)(grpc_exec_ctx *exec_ctx,
                                                  grpc_security_status status);
 
 struct grpc_channel_security_connector {
-  grpc_security_connector base; /* requires is_client_side to be non 0. */
+  grpc_security_connector base;
   grpc_call_credentials *request_metadata_creds;
   void (*check_call_host)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc, const char *host,
                           grpc_auth_context *auth_context,
                           grpc_security_call_host_check_cb cb, void *user_data);
+  void (*do_handshake)(grpc_exec_ctx *exec_ctx,
+                       grpc_channel_security_connector *sc,
+                       grpc_endpoint *nonsecure_endpoint,
+                       grpc_security_handshake_done_cb cb, void *user_data);
 };
 
 /* Checks that the host that will be set for a call is acceptable. */
@@ -167,6 +153,39 @@ void grpc_channel_security_connector_check_call_host(
     const char *host, grpc_auth_context *auth_context,
     grpc_security_call_host_check_cb cb, void *user_data);
 
+/* Handshake. */
+void grpc_channel_security_connector_do_handshake(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
+    grpc_endpoint *nonsecure_endpoint, grpc_security_handshake_done_cb cb,
+    void *user_data);
+
+/* --- server_security_connector object. ---
+
+    A server security connector object represents away to configure the
+    underlying transport security mechanism on the server side.  */
+
+typedef struct grpc_server_security_connector grpc_server_security_connector;
+
+struct grpc_server_security_connector {
+  grpc_security_connector base;
+  gpr_mu mu;
+  grpc_security_connector_handshake_list *handshaking_handshakes;
+  const grpc_channel_args *channel_args;
+  void (*do_handshake)(grpc_exec_ctx *exec_ctx,
+                       grpc_server_security_connector *sc,
+                       grpc_tcp_server_acceptor *acceptor,
+                       grpc_endpoint *nonsecure_endpoint,
+                       grpc_security_handshake_done_cb cb, void *user_data);
+};
+
+void grpc_server_security_connector_do_handshake(
+    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
+    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
+    grpc_security_handshake_done_cb cb, void *user_data);
+
+void grpc_server_security_connector_shutdown(
+    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector);
+
 /* --- Creation security connectors. --- */
 
 /* For TESTING ONLY!
@@ -176,7 +195,8 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
 
 /* For TESTING ONLY!
    Creates a fake connector that emulates real server security.  */
-grpc_security_connector *grpc_fake_server_security_connector_create(void);
+grpc_server_security_connector *grpc_fake_server_security_connector_create(
+    void);
 
 /* Config for ssl clients. */
 typedef struct {
@@ -231,7 +251,7 @@ typedef struct {
   specific error code otherwise.
 */
 grpc_security_status grpc_ssl_server_security_connector_create(
-    const grpc_ssl_server_config *config, grpc_security_connector **sc);
+    const grpc_ssl_server_config *config, grpc_server_security_connector **sc);
 
 /* Util. */
 const tsi_peer_property *tsi_peer_get_property_by_name(const tsi_peer *peer,
diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c
index b4fbd769bd..009ec95682 100644
--- a/src/core/security/server_secure_chttp2.c
+++ b/src/core/security/server_secure_chttp2.c
@@ -55,7 +55,7 @@
 typedef struct grpc_server_secure_state {
   grpc_server *server;
   grpc_tcp_server *tcp;
-  grpc_security_connector *sc;
+  grpc_server_security_connector *sc;
   grpc_server_credentials *creds;
   int is_shutdown;
   gpr_mu mu;
@@ -74,7 +74,7 @@ static void state_unref(grpc_server_secure_state *state) {
     gpr_mu_lock(&state->mu);
     gpr_mu_unlock(&state->mu);
     /* clean up */
-    GRPC_SECURITY_CONNECTOR_UNREF(state->sc, "server");
+    GRPC_SECURITY_CONNECTOR_UNREF(&state->sc->base, "server");
     grpc_server_credentials_unref(state->creds);
     gpr_free(state);
   }
@@ -127,8 +127,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
                       grpc_tcp_server_acceptor *acceptor) {
   grpc_server_secure_state *state = statep;
   state_ref(state);
-  grpc_security_connector_do_handshake(exec_ctx, state->sc, tcp,
-                                       on_secure_handshake_done, state);
+  grpc_server_security_connector_do_handshake(
+      exec_ctx, state->sc, acceptor, tcp, on_secure_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
@@ -145,7 +145,7 @@ static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
     state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg,
                                 success);
   }
-  grpc_security_connector_shutdown(exec_ctx, state->sc);
+  grpc_server_security_connector_shutdown(exec_ctx, state->sc);
   state_unref(state);
 }
 
@@ -173,7 +173,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   int port_num = -1;
   int port_temp;
   grpc_security_status status = GRPC_SECURITY_ERROR;
-  grpc_security_connector *sc = NULL;
+  grpc_server_security_connector *sc = NULL;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
   GRPC_API_TRACE(
@@ -253,7 +253,7 @@ error:
     grpc_tcp_server_unref(&exec_ctx, tcp);
   } else {
     if (sc) {
-      GRPC_SECURITY_CONNECTOR_UNREF(sc, "server");
+      GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "server");
     }
     if (state) {
       gpr_free(state);
diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c
index 1ca6fa1aff..fe51f846b7 100644
--- a/src/core/support/env_linux.c
+++ b/src/core/support/env_linux.c
@@ -43,7 +43,9 @@
 #include "src/core/support/env.h"
 
 #include <dlfcn.h>
+#include <features.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
@@ -60,12 +62,22 @@ char *gpr_getenv(const char *name) {
   const char *names[] = {"secure_getenv", "__secure_getenv", "getenv"};
   for (size_t i = 0; getenv_func == NULL && i < GPR_ARRAY_SIZE(names); i++) {
     getenv_func = (getenv_type)dlsym(RTLD_DEFAULT, names[i]);
+    if (getenv_func != NULL && strstr(names[i], "secure") == NULL) {
+      gpr_log(GPR_DEBUG,
+              "Warning: insecure environment read function '%s' used",
+              names[i]);
+    }
   }
   char *result = getenv_func(name);
   return result == NULL ? result : gpr_strdup(result);
-#else
+#elif __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17)
   char *result = secure_getenv(name);
   return result == NULL ? result : gpr_strdup(result);
+#else
+  gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used",
+          "getenv");
+  char *result = getenv(name);
+  return result == NULL ? result : gpr_strdup(result);
 #endif
 }
 
diff --git a/src/core/support/file.c b/src/core/support/load_file.c
index 8c673dbcc6..650bd62ccb 100644
--- a/src/core/support/file.c
+++ b/src/core/support/load_file.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/support/file.h"
+#include "src/core/support/load_file.h"
 
 #include <errno.h>
 #include <string.h>
diff --git a/src/core/support/file.h b/src/core/support/load_file.h
index d8b7cea44f..746319a50d 100644
--- a/src/core/support/file.h
+++ b/src/core/support/load_file.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CORE_SUPPORT_FILE_H
-#define GRPC_INTERNAL_CORE_SUPPORT_FILE_H
+#ifndef GRPC_INTERNAL_CORE_SUPPORT_LOAD_FILE_H
+#define GRPC_INTERNAL_CORE_SUPPORT_LOAD_FILE_H
 
 #include <stdio.h>
 
@@ -42,22 +42,14 @@
 extern "C" {
 #endif
 
-/* File utility functions */
-
 /* Loads the content of a file into a slice. add_null_terminator will add
    a NULL terminator if non-zero. The success parameter, if not NULL,
    will be set to 1 in case of success and 0 in case of failure. */
 gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
                         int *success);
 
-/* Creates a temporary file from a prefix.
-   If tmp_filename is not NULL, *tmp_filename is assigned the name of the
-   created file and it is the responsibility of the caller to gpr_free it
-   unless an error occurs in which case it will be set to NULL. */
-FILE *gpr_tmpfile(const char *prefix, char **tmp_filename);
-
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* GRPC_INTERNAL_CORE_SUPPORT_FILE_H */
+#endif /* GRPC_INTERNAL_CORE_SUPPORT_LOAD_FILE_H */
diff --git a/src/core/support/tmpfile.h b/src/core/support/tmpfile.h
new file mode 100644
index 0000000000..cecc71e242
--- /dev/null
+++ b/src/core/support/tmpfile.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_SUPPORT_TMPFILE_H
+#define GRPC_INTERNAL_CORE_SUPPORT_TMPFILE_H
+
+#include <stdio.h>
+
+#include <grpc/support/slice.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Creates a temporary file from a prefix.
+   If tmp_filename is not NULL, *tmp_filename is assigned the name of the
+   created file and it is the responsibility of the caller to gpr_free it
+   unless an error occurs in which case it will be set to NULL. */
+FILE *gpr_tmpfile(const char *prefix, char **tmp_filename);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_INTERNAL_CORE_SUPPORT_TMPFILE_H */
diff --git a/src/core/support/file_posix.c b/src/core/support/tmpfile_posix.c
index c11c07148a..b16eeacf9d 100644
--- a/src/core/support/file_posix.c
+++ b/src/core/support/tmpfile_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/support/file.h"
+#include "src/core/support/tmpfile.h"
 
 #include <errno.h>
 #include <stdlib.h>
diff --git a/src/core/support/file_win32.c b/src/core/support/tmpfile_win32.c
index 355744f79a..3000f0029f 100644
--- a/src/core/support/file_win32.c
+++ b/src/core/support/tmpfile_win32.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,8 +44,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/support/file.h"
 #include "src/core/support/string_win32.h"
+#include "src/core/support/tmpfile.h"
 
 FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
   FILE *result = NULL;
diff --git a/src/core/surface/alarm.c b/src/core/surface/alarm.c
index fb496f6c47..8169ede065 100644
--- a/src/core/surface/alarm.c
+++ b/src/core/surface/alarm.c
@@ -64,8 +64,9 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline,
   alarm->tag = tag;
 
   grpc_cq_begin_op(cq, tag);
-  grpc_timer_init(&exec_ctx, &alarm->alarm, deadline, alarm_cb, alarm,
-                  gpr_now(GPR_CLOCK_MONOTONIC));
+  grpc_timer_init(&exec_ctx, &alarm->alarm,
+                  gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC),
+                  alarm_cb, alarm, gpr_now(GPR_CLOCK_MONOTONIC));
   grpc_exec_ctx_finish(&exec_ctx);
   return alarm;
 }
diff --git a/src/core/surface/completion_queue.c b/src/core/surface/completion_queue.c
index 0a80680f02..f9cb852722 100644
--- a/src/core/surface/completion_queue.c
+++ b/src/core/surface/completion_queue.c
@@ -50,7 +50,7 @@
 #include <grpc/support/time.h>
 
 typedef struct {
-  grpc_pollset_worker *worker;
+  grpc_pollset_worker **worker;
   void *tag;
 } plucker;
 
@@ -252,7 +252,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
     pluck_worker = NULL;
     for (i = 0; i < cc->num_pluckers; i++) {
       if (cc->pluckers[i].tag == tag) {
-        pluck_worker = cc->pluckers[i].worker;
+        pluck_worker = *cc->pluckers[i].worker;
         break;
       }
     }
@@ -275,7 +275,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
 grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
                                       gpr_timespec deadline, void *reserved) {
   grpc_event ret;
-  grpc_pollset_worker worker;
+  grpc_pollset_worker *worker = NULL;
   int first_loop = 1;
   gpr_timespec now;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -348,7 +348,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
 }
 
 static int add_plucker(grpc_completion_queue *cc, void *tag,
-                       grpc_pollset_worker *worker) {
+                       grpc_pollset_worker **worker) {
   if (cc->num_pluckers == GRPC_MAX_COMPLETION_QUEUE_PLUCKERS) {
     return 0;
   }
@@ -359,7 +359,7 @@ static int add_plucker(grpc_completion_queue *cc, void *tag,
 }
 
 static void del_plucker(grpc_completion_queue *cc, void *tag,
-                        grpc_pollset_worker *worker) {
+                        grpc_pollset_worker **worker) {
   int i;
   for (i = 0; i < cc->num_pluckers; i++) {
     if (cc->pluckers[i].tag == tag && cc->pluckers[i].worker == worker) {
@@ -376,7 +376,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
   grpc_event ret;
   grpc_cq_completion *c;
   grpc_cq_completion *prev;
-  grpc_pollset_worker worker;
+  grpc_pollset_worker *worker = NULL;
   gpr_timespec now;
   int first_loop = 1;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c
index dab55f853d..cc752227ee 100644
--- a/src/core/surface/secure_channel_create.c
+++ b/src/core/surface/secure_channel_create.c
@@ -123,9 +123,9 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
 static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
                                            bool success) {
   connector *c = arg;
-  grpc_security_connector_do_handshake(exec_ctx, &c->security_connector->base,
-                                       c->connecting_endpoint,
-                                       on_secure_handshake_done, c);
+  grpc_channel_security_connector_do_handshake(exec_ctx, c->security_connector,
+                                               c->connecting_endpoint,
+                                               on_secure_handshake_done, c);
 }
 
 static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
@@ -146,9 +146,8 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
       grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer,
                           &c->initial_string_sent);
     } else {
-      grpc_security_connector_do_handshake(exec_ctx,
-                                           &c->security_connector->base, tcp,
-                                           on_secure_handshake_done, c);
+      grpc_channel_security_connector_do_handshake(
+          exec_ctx, c->security_connector, tcp, on_secure_handshake_done, c);
     }
   } else {
     memset(c->result, 0, sizeof(*c->result));
diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h
index 0e1e2c4265..d76d31be23 100644
--- a/src/core/transport/chttp2/internal.h
+++ b/src/core/transport/chttp2/internal.h
@@ -485,7 +485,8 @@ struct grpc_chttp2_stream {
 
 /** Someone is unlocking the transport mutex: check to see if writes
     are required, and schedule them if so */
-int grpc_chttp2_unlocking_check_writes(grpc_chttp2_transport_global *global,
+int grpc_chttp2_unlocking_check_writes(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_transport_global *global,
                                        grpc_chttp2_transport_writing *writing,
                                        int is_parsing);
 void grpc_chttp2_perform_writes(
@@ -568,8 +569,12 @@ void grpc_chttp2_list_add_writing_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing);
 void grpc_chttp2_list_flush_writing_stalled_by_transport(
-    grpc_chttp2_transport_writing *transport_writing, bool is_window_available);
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing,
+    bool is_window_available);
 
+void grpc_chttp2_list_add_stalled_by_transport(
+    grpc_chttp2_transport_writing *transport_writing,
+    grpc_chttp2_stream_writing *stream_writing);
 int grpc_chttp2_list_pop_stalled_by_transport(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global **stream_global);
diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c
index 2f31a47cb3..b284c78818 100644
--- a/src/core/transport/chttp2/stream_lists.c
+++ b/src/core/transport/chttp2/stream_lists.c
@@ -316,13 +316,16 @@ int grpc_chttp2_list_pop_check_read_ops(
 void grpc_chttp2_list_add_writing_stalled_by_transport(
     grpc_chttp2_transport_writing *transport_writing,
     grpc_chttp2_stream_writing *stream_writing) {
-  stream_list_add(TRANSPORT_FROM_WRITING(transport_writing),
-                  STREAM_FROM_WRITING(stream_writing),
+  grpc_chttp2_stream *stream = STREAM_FROM_WRITING(stream_writing);
+  if (!stream->included[GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT]) {
+    GRPC_CHTTP2_STREAM_REF(&stream->global, "chttp2_writing_stalled");
+  }
+  stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), stream,
                   GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT);
 }
 
 void grpc_chttp2_list_flush_writing_stalled_by_transport(
-    grpc_chttp2_transport_writing *transport_writing,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing,
     bool is_window_available) {
   grpc_chttp2_stream *stream;
   grpc_chttp2_transport *transport = TRANSPORT_FROM_WRITING(transport_writing);
@@ -331,11 +334,22 @@ void grpc_chttp2_list_flush_writing_stalled_by_transport(
     if (is_window_available) {
       grpc_chttp2_list_add_writable_stream(&transport->global, &stream->global);
     } else {
-      stream_list_add(transport, stream, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
+      grpc_chttp2_list_add_stalled_by_transport(transport_writing,
+                                                &stream->writing);
     }
+    GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &stream->global,
+                             "chttp2_writing_stalled");
   }
 }
 
+void grpc_chttp2_list_add_stalled_by_transport(
+    grpc_chttp2_transport_writing *transport_writing,
+    grpc_chttp2_stream_writing *stream_writing) {
+  stream_list_add(TRANSPORT_FROM_WRITING(transport_writing),
+                  STREAM_FROM_WRITING(stream_writing),
+                  GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);
+}
+
 int grpc_chttp2_list_pop_stalled_by_transport(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global **stream_global) {
diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c
index cafecf1046..356fd8174a 100644
--- a/src/core/transport/chttp2/writing.c
+++ b/src/core/transport/chttp2/writing.c
@@ -44,7 +44,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
                             grpc_chttp2_transport_writing *transport_writing);
 
 int grpc_chttp2_unlocking_check_writes(
-    grpc_chttp2_transport_global *transport_global,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_transport_writing *transport_writing, int is_parsing) {
   grpc_chttp2_stream_global *stream_global;
   grpc_chttp2_stream_writing *stream_writing;
@@ -76,8 +76,8 @@ int grpc_chttp2_unlocking_check_writes(
   GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("write", transport_writing, outgoing_window,
                                   transport_global, outgoing_window);
   bool is_window_available = transport_writing->outgoing_window > 0;
-  grpc_chttp2_list_flush_writing_stalled_by_transport(transport_writing,
-                                                      is_window_available);
+  grpc_chttp2_list_flush_writing_stalled_by_transport(
+      exec_ctx, transport_writing, is_window_available);
 
   /* for each grpc_chttp2_stream that's become writable, frame it's data
      (according to available window sizes) and add to the output buffer */
@@ -133,8 +133,8 @@ int grpc_chttp2_unlocking_check_writes(
             GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing");
           }
         } else {
-          grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing,
-                                                            stream_writing);
+          grpc_chttp2_list_add_stalled_by_transport(transport_writing,
+                                                    stream_writing);
         }
       }
       if (stream_global->send_trailing_metadata) {
diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c
index 87bd31531d..c49e1409a4 100644
--- a/src/core/transport/chttp2_transport.c
+++ b/src/core/transport/chttp2_transport.c
@@ -598,7 +598,7 @@ static void lock(grpc_chttp2_transport *t) { gpr_mu_lock(&t->mu); }
 static void unlock(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) {
   GPR_TIMER_BEGIN("unlock", 0);
   if (!t->writing_active && !t->closed &&
-      grpc_chttp2_unlocking_check_writes(&t->global, &t->writing,
+      grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing,
                                          t->parsing_active)) {
     t->writing_active = 1;
     REF_TRANSPORT(t, "writing");