Merge branch 'master' into fix_alarm

127 files changed, 4913 insertions, 1608 deletions

diff --git a/src/core/ext/filters/client_channel/OWNERS b/src/core/ext/filters/client_channel/OWNERS
new file mode 100644
index 0000000000..773bc73179
--- /dev/null
+++ b/src/core/ext/filters/client_channel/OWNERS
@@ -0,0 +1,4 @@
+set noparent
+@markdroth
+@dgquintas
+@ctiller
diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c
index de516ab4c9..58e31d7b45 100644
--- a/src/core/ext/filters/client_channel/client_channel.c
+++ b/src/core/ext/filters/client_channel/client_channel.c
@@ -52,6 +52,9 @@
 
 /* Client channel implementation */
 
+grpc_tracer_flag grpc_client_channel_trace =
+    GRPC_TRACER_INITIALIZER(false, "client_channel");
+
 /*************************************************************************
  * METHOD-CONFIG TABLE
  */
@@ -241,6 +244,10 @@ static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx,
                                          GRPC_ERROR_REF(error));
     }
   }
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p: setting connectivity state to %s", chand,
+            grpc_connectivity_state_name(state));
+  }
   grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, error,
                               reason);
 }
@@ -251,6 +258,10 @@ static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx,
   grpc_connectivity_state publish_state = w->state;
   /* check if the notification is for the latest policy */
   if (w->lb_policy == w->chand->lb_policy) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p: lb_policy=%p state changed to %s", w->chand,
+              w->lb_policy, grpc_connectivity_state_name(w->state));
+    }
     if (publish_state == GRPC_CHANNEL_SHUTDOWN && w->chand->resolver != NULL) {
       publish_state = GRPC_CHANNEL_TRANSIENT_FAILURE;
       grpc_resolver_channel_saw_error_locked(exec_ctx, w->chand->resolver);
@@ -263,7 +274,6 @@ static void on_lb_policy_state_changed_locked(grpc_exec_ctx *exec_ctx,
       watch_lb_policy_locked(exec_ctx, w->chand, w->lb_policy, w->state);
     }
   }
-
   GRPC_CHANNEL_STACK_UNREF(exec_ctx, w->chand->owning_stack, "watch_lb_policy");
   gpr_free(w);
 }
@@ -273,7 +283,6 @@ static void watch_lb_policy_locked(grpc_exec_ctx *exec_ctx, channel_data *chand,
                                    grpc_connectivity_state current_state) {
   lb_policy_connectivity_watcher *w = gpr_malloc(sizeof(*w));
   GRPC_CHANNEL_STACK_REF(chand->owning_stack, "watch_lb_policy");
-
   w->chand = chand;
   GRPC_CLOSURE_INIT(&w->on_changed, on_lb_policy_state_changed_locked, w,
                     grpc_combiner_scheduler(chand->combiner));
@@ -283,6 +292,18 @@ static void watch_lb_policy_locked(grpc_exec_ctx *exec_ctx, channel_data *chand,
                                                &w->on_changed);
 }
 
+static void start_resolving_locked(grpc_exec_ctx *exec_ctx,
+                                   channel_data *chand) {
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p: starting name resolution", chand);
+  }
+  GPR_ASSERT(!chand->started_resolving);
+  chand->started_resolving = true;
+  GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
+  grpc_resolver_next_locked(exec_ctx, chand->resolver, &chand->resolver_result,
+                            &chand->on_resolver_result_changed);
+}
+
 typedef struct {
   char *server_name;
   grpc_server_retry_throttle_data *retry_throttle_data;
@@ -345,8 +366,14 @@ static void parse_retry_throttle_params(const grpc_json *field, void *arg) {
 static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
                                               void *arg, grpc_error *error) {
   channel_data *chand = arg;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p: got resolver result: error=%s", chand,
+            grpc_error_string(error));
+  }
   // Extract the following fields from the resolver result, if non-NULL.
+  bool lb_policy_updated = false;
   char *lb_policy_name = NULL;
+  bool lb_policy_name_changed = false;
   grpc_lb_policy *new_lb_policy = NULL;
   char *service_config_json = NULL;
   grpc_server_retry_throttle_data *retry_throttle_data = NULL;
@@ -394,11 +421,12 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
     // taking a lock on chand->info_mu, because this function is the
     // only thing that modifies its value, and it can only be invoked
     // once at any given time.
-    const bool lb_policy_type_changed =
+    lb_policy_name_changed =
         chand->info_lb_policy_name == NULL ||
         strcmp(chand->info_lb_policy_name, lb_policy_name) != 0;
-    if (chand->lb_policy != NULL && !lb_policy_type_changed) {
+    if (chand->lb_policy != NULL && !lb_policy_name_changed) {
       // Continue using the same LB policy.  Update with new addresses.
+      lb_policy_updated = true;
       grpc_lb_policy_update_locked(exec_ctx, chand->lb_policy, &lb_policy_args);
     } else {
       // Instantiate new LB policy.
@@ -445,6 +473,13 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
     grpc_channel_args_destroy(exec_ctx, chand->resolver_result);
     chand->resolver_result = NULL;
   }
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG,
+            "chand=%p: resolver result: lb_policy_name=\"%s\"%s, "
+            "service_config=\"%s\"",
+            chand, lb_policy_name, lb_policy_name_changed ? " (changed)" : "",
+            service_config_json);
+  }
   // Now swap out fields in chand.  Note that the new values may still
   // be NULL if (e.g.) the resolver failed to return results or the
   // results did not contain the necessary data.
@@ -479,6 +514,10 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
   if (new_lb_policy != NULL || error != GRPC_ERROR_NONE ||
       chand->resolver == NULL) {
     if (chand->lb_policy != NULL) {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG, "chand=%p: unreffing lb_policy=%p", chand,
+                chand->lb_policy);
+      }
       grpc_pollset_set_del_pollset_set(exec_ctx,
                                        chand->lb_policy->interested_parties,
                                        chand->interested_parties);
@@ -489,7 +528,13 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
   // Now that we've swapped out the relevant fields of chand, check for
   // error or shutdown.
   if (error != GRPC_ERROR_NONE || chand->resolver == NULL) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p: shutting down", chand);
+    }
     if (chand->resolver != NULL) {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG, "chand=%p: shutting down resolver", chand);
+      }
       grpc_resolver_shutdown_locked(exec_ctx, chand->resolver);
       GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
       chand->resolver = NULL;
@@ -510,6 +555,9 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
     grpc_error *state_error =
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("No load balancing policy");
     if (new_lb_policy != NULL) {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG, "chand=%p: initializing new LB policy", chand);
+      }
       GRPC_ERROR_UNREF(state_error);
       state = grpc_lb_policy_check_connectivity_locked(exec_ctx, new_lb_policy,
                                                        &state_error);
@@ -524,8 +572,11 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx,
       }
       watch_lb_policy_locked(exec_ctx, chand, new_lb_policy, state);
     }
-    set_channel_connectivity_state_locked(
-        exec_ctx, chand, state, GRPC_ERROR_REF(state_error), "new_lb+resolver");
+    if (!lb_policy_updated) {
+      set_channel_connectivity_state_locked(exec_ctx, chand, state,
+                                            GRPC_ERROR_REF(state_error),
+                                            "new_lb+resolver");
+    }
     grpc_resolver_next_locked(exec_ctx, chand->resolver,
                               &chand->resolver_result,
                               &chand->on_resolver_result_changed);
@@ -772,7 +823,9 @@ typedef struct client_channel_call_data {
   gpr_atm subchannel_call_or_error;
   gpr_arena *arena;
 
-  bool pick_pending;
+  grpc_lb_policy *lb_policy;  // Holds ref while LB pick is pending.
+  grpc_closure lb_pick_closure;
+
   grpc_connected_subchannel *connected_subchannel;
   grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
   grpc_polling_entity *pollent;
@@ -837,8 +890,15 @@ static void waiting_for_pick_batches_add_locked(
 }
 
 static void waiting_for_pick_batches_fail_locked(grpc_exec_ctx *exec_ctx,
-                                                 call_data *calld,
+                                                 grpc_call_element *elem,
                                                  grpc_error *error) {
+  call_data *calld = elem->call_data;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG,
+            "chand=%p calld=%p: failing %" PRIdPTR " pending batches: %s",
+            elem->channel_data, calld, calld->waiting_for_pick_batches_count,
+            grpc_error_string(error));
+  }
   for (size_t i = 0; i < calld->waiting_for_pick_batches_count; ++i) {
     grpc_transport_stream_op_batch_finish_with_failure(
         exec_ctx, calld->waiting_for_pick_batches[i], GRPC_ERROR_REF(error));
@@ -848,14 +908,21 @@ static void waiting_for_pick_batches_fail_locked(grpc_exec_ctx *exec_ctx,
 }
 
 static void waiting_for_pick_batches_resume_locked(grpc_exec_ctx *exec_ctx,
-                                                   call_data *calld) {
+                                                   grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
   if (calld->waiting_for_pick_batches_count == 0) return;
   call_or_error coe = get_call_or_error(calld);
   if (coe.error != GRPC_ERROR_NONE) {
-    waiting_for_pick_batches_fail_locked(exec_ctx, calld,
+    waiting_for_pick_batches_fail_locked(exec_ctx, elem,
                                          GRPC_ERROR_REF(coe.error));
     return;
   }
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: sending %" PRIdPTR
+                       " pending batches to subchannel_call=%p",
+            elem->channel_data, calld, calld->waiting_for_pick_batches_count,
+            coe.subchannel_call);
+  }
   for (size_t i = 0; i < calld->waiting_for_pick_batches_count; ++i) {
     grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call,
                                     calld->waiting_for_pick_batches[i]);
@@ -869,6 +936,10 @@ static void apply_service_config_to_call_locked(grpc_exec_ctx *exec_ctx,
                                                 grpc_call_element *elem) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: applying service config to call",
+            chand, calld);
+  }
   if (chand->retry_throttle_data != NULL) {
     calld->retry_throttle_data =
         grpc_server_retry_throttle_data_ref(chand->retry_throttle_data);
@@ -895,7 +966,9 @@ static void apply_service_config_to_call_locked(grpc_exec_ctx *exec_ctx,
 }
 
 static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx,
-                                          call_data *calld, grpc_error *error) {
+                                          grpc_call_element *elem,
+                                          grpc_error *error) {
+  call_data *calld = elem->call_data;
   grpc_subchannel_call *subchannel_call = NULL;
   const grpc_connected_subchannel_call_args call_args = {
       .pollent = calld->pollent,
@@ -906,13 +979,18 @@ static void create_subchannel_call_locked(grpc_exec_ctx *exec_ctx,
       .context = calld->subchannel_call_context};
   grpc_error *new_error = grpc_connected_subchannel_create_call(
       exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call);
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
+            elem->channel_data, calld, subchannel_call,
+            grpc_error_string(new_error));
+  }
   GPR_ASSERT(set_call_or_error(
       calld, (call_or_error){.subchannel_call = subchannel_call}));
   if (new_error != GRPC_ERROR_NONE) {
     new_error = grpc_error_add_child(new_error, error);
-    waiting_for_pick_batches_fail_locked(exec_ctx, calld, new_error);
+    waiting_for_pick_batches_fail_locked(exec_ctx, elem, new_error);
   } else {
-    waiting_for_pick_batches_resume_locked(exec_ctx, calld);
+    waiting_for_pick_batches_resume_locked(exec_ctx, elem);
   }
   GRPC_ERROR_UNREF(error);
 }
@@ -922,8 +1000,6 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
                                     grpc_error *error) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  GPR_ASSERT(calld->pick_pending);
-  calld->pick_pending = false;
   grpc_polling_entity_del_from_pollset_set(exec_ctx, calld->pollent,
                                            chand->interested_parties);
   call_or_error coe = get_call_or_error(calld);
@@ -935,8 +1011,13 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
                   "Call dropped by load balancing policy")
             : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                   "Failed to create subchannel", &error, 1);
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG,
+              "chand=%p calld=%p: failed to create subchannel: error=%s", chand,
+              calld, grpc_error_string(failure));
+    }
     set_call_or_error(calld, (call_or_error){.error = GRPC_ERROR_REF(failure)});
-    waiting_for_pick_batches_fail_locked(exec_ctx, calld, failure);
+    waiting_for_pick_batches_fail_locked(exec_ctx, elem, failure);
   } else if (coe.error != GRPC_ERROR_NONE) {
     /* already cancelled before subchannel became ready */
     grpc_error *child_errors[] = {error, coe.error};
@@ -950,10 +1031,15 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx,
           grpc_error_set_int(cancellation_error, GRPC_ERROR_INT_GRPC_STATUS,
                              GRPC_STATUS_DEADLINE_EXCEEDED);
     }
-    waiting_for_pick_batches_fail_locked(exec_ctx, calld, cancellation_error);
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG,
+              "chand=%p calld=%p: cancelled before subchannel became ready: %s",
+              chand, calld, grpc_error_string(cancellation_error));
+    }
+    waiting_for_pick_batches_fail_locked(exec_ctx, elem, cancellation_error);
   } else {
     /* Create call on subchannel. */
-    create_subchannel_call_locked(exec_ctx, calld, GRPC_ERROR_REF(error));
+    create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
   }
   GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
   GRPC_ERROR_UNREF(error);
@@ -983,41 +1069,77 @@ typedef struct {
   grpc_closure closure;
 } pick_after_resolver_result_args;
 
-static void continue_picking_after_resolver_result_locked(
-    grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
+                                                   void *arg,
+                                                   grpc_error *error) {
   pick_after_resolver_result_args *args = arg;
   if (args->cancelled) {
     /* cancelled, do nothing */
-  } else if (error != GRPC_ERROR_NONE) {
-    subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_REF(error));
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "call cancelled before resolver result");
+    }
   } else {
-    if (pick_subchannel_locked(exec_ctx, args->elem)) {
-      subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_NONE);
+    channel_data *chand = args->elem->channel_data;
+    call_data *calld = args->elem->call_data;
+    if (error != GRPC_ERROR_NONE) {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data",
+                chand, calld);
+      }
+      subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_REF(error));
+    } else {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick",
+                chand, calld);
+      }
+      if (pick_subchannel_locked(exec_ctx, args->elem)) {
+        subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_NONE);
+      }
     }
   }
   gpr_free(args);
 }
 
-static void cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-                               grpc_error *error) {
+static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
+                                                    grpc_call_element *elem) {
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG,
+            "chand=%p calld=%p: deferring pick pending resolver result", chand,
+            calld);
+  }
+  pick_after_resolver_result_args *args =
+      (pick_after_resolver_result_args *)gpr_zalloc(sizeof(*args));
+  args->elem = elem;
+  GRPC_CLOSURE_INIT(&args->closure, pick_after_resolver_result_done_locked,
+                    args, grpc_combiner_scheduler(chand->combiner));
+  grpc_closure_list_append(&chand->waiting_for_resolver_result_closures,
+                           &args->closure, GRPC_ERROR_NONE);
+}
+
+static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
+                                                     grpc_call_element *elem,
+                                                     grpc_error *error) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  if (chand->lb_policy != NULL) {
-    grpc_lb_policy_cancel_pick_locked(exec_ctx, chand->lb_policy,
-                                      &calld->connected_subchannel,
-                                      GRPC_ERROR_REF(error));
-  }
   // If we don't yet have a resolver result, then a closure for
-  // continue_picking_after_resolver_result_locked() will have been added to
+  // pick_after_resolver_result_done_locked() will have been added to
   // chand->waiting_for_resolver_result_closures, and it may not be invoked
   // until after this call has been destroyed.  We mark the operation as
-  // cancelled, so that when continue_picking_after_resolver_result_locked()
+  // cancelled, so that when pick_after_resolver_result_done_locked()
   // is called, it will be a no-op.  We also immediately invoke
   // subchannel_ready_locked() to propagate the error back to the caller.
   for (grpc_closure *closure = chand->waiting_for_resolver_result_closures.head;
        closure != NULL; closure = closure->next_data.next) {
     pick_after_resolver_result_args *args = closure->cb_arg;
     if (!args->cancelled && args->elem == elem) {
+      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+        gpr_log(GPR_DEBUG,
+                "chand=%p calld=%p: "
+                "cancelling pick waiting for resolver result",
+                chand, calld);
+      }
       args->cancelled = true;
       subchannel_ready_locked(exec_ctx, elem,
                               GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
@@ -1027,24 +1149,21 @@ static void cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   GRPC_ERROR_UNREF(error);
 }
 
-// State for pick callback that holds a reference to the LB policy
-// from which the pick was requested.
-typedef struct {
-  grpc_lb_policy *lb_policy;
-  grpc_call_element *elem;
-  grpc_closure closure;
-} pick_callback_args;
-
 // Callback invoked by grpc_lb_policy_pick_locked() for async picks.
 // Unrefs the LB policy after invoking subchannel_ready_locked().
 static void pick_callback_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                       grpc_error *error) {
-  pick_callback_args *args = arg;
-  GPR_ASSERT(args != NULL);
-  GPR_ASSERT(args->lb_policy != NULL);
-  subchannel_ready_locked(exec_ctx, args->elem, GRPC_ERROR_REF(error));
-  GRPC_LB_POLICY_UNREF(exec_ctx, args->lb_policy, "pick_subchannel");
-  gpr_free(args);
+  grpc_call_element *elem = arg;
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed asynchronously",
+            chand, calld);
+  }
+  GPR_ASSERT(calld->lb_policy != NULL);
+  GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
+  calld->lb_policy = NULL;
+  subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
 }
 
 // Takes a ref to chand->lb_policy and calls grpc_lb_policy_pick_locked().
@@ -1055,23 +1174,44 @@ static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx,
                                        const grpc_lb_policy_pick_args *inputs) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  pick_callback_args *pick_args = gpr_zalloc(sizeof(*pick_args));
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: starting pick on lb_policy=%p",
+            chand, calld, chand->lb_policy);
+  }
+  // Keep a ref to the LB policy in calld while the pick is pending.
   GRPC_LB_POLICY_REF(chand->lb_policy, "pick_subchannel");
-  pick_args->lb_policy = chand->lb_policy;
-  pick_args->elem = elem;
-  GRPC_CLOSURE_INIT(&pick_args->closure, pick_callback_done_locked, pick_args,
+  calld->lb_policy = chand->lb_policy;
+  GRPC_CLOSURE_INIT(&calld->lb_pick_closure, pick_callback_done_locked, elem,
                     grpc_combiner_scheduler(chand->combiner));
   const bool pick_done = grpc_lb_policy_pick_locked(
       exec_ctx, chand->lb_policy, inputs, &calld->connected_subchannel,
-      calld->subchannel_call_context, NULL, &pick_args->closure);
+      calld->subchannel_call_context, NULL, &calld->lb_pick_closure);
   if (pick_done) {
     /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */
-    GRPC_LB_POLICY_UNREF(exec_ctx, chand->lb_policy, "pick_subchannel");
-    gpr_free(pick_args);
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: pick completed synchronously",
+              chand, calld);
+    }
+    GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
+    calld->lb_policy = NULL;
   }
   return pick_done;
 }
 
+static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_element *elem,
+                                        grpc_error *error) {
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  GPR_ASSERT(calld->lb_policy != NULL);
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
+            chand, calld, calld->lb_policy);
+  }
+  grpc_lb_policy_cancel_pick_locked(exec_ctx, calld->lb_policy,
+                                    &calld->connected_subchannel, error);
+}
+
 static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem) {
   GPR_TIMER_BEGIN("pick_subchannel", 0);
@@ -1107,20 +1247,9 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
     pick_done = pick_callback_start_locked(exec_ctx, elem, &inputs);
   } else if (chand->resolver != NULL) {
     if (!chand->started_resolving) {
-      chand->started_resolving = true;
-      GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-      grpc_resolver_next_locked(exec_ctx, chand->resolver,
-                                &chand->resolver_result,
-                                &chand->on_resolver_result_changed);
+      start_resolving_locked(exec_ctx, chand);
     }
-    pick_after_resolver_result_args *args =
-        (pick_after_resolver_result_args *)gpr_zalloc(sizeof(*args));
-    args->elem = elem;
-    GRPC_CLOSURE_INIT(&args->closure,
-                      continue_picking_after_resolver_result_locked, args,
-                      grpc_combiner_scheduler(chand->combiner));
-    grpc_closure_list_append(&chand->waiting_for_resolver_result_closures,
-                             &args->closure, GRPC_ERROR_NONE);
+    pick_after_resolver_result_start_locked(exec_ctx, elem);
   } else {
     subchannel_ready_locked(
         exec_ctx, elem, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected"));
@@ -1133,63 +1262,77 @@ static void start_transport_stream_op_batch_locked(grpc_exec_ctx *exec_ctx,
                                                    void *arg,
                                                    grpc_error *error_ignored) {
   GPR_TIMER_BEGIN("start_transport_stream_op_batch_locked", 0);
-  grpc_transport_stream_op_batch *op = arg;
-  grpc_call_element *elem = op->handler_private.extra_arg;
+  grpc_transport_stream_op_batch *batch = arg;
+  grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
   /* need to recheck that another thread hasn't set the call */
   call_or_error coe = get_call_or_error(calld);
   if (coe.error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
+              chand, calld, grpc_error_string(coe.error));
+    }
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, op, GRPC_ERROR_REF(coe.error));
+        exec_ctx, batch, GRPC_ERROR_REF(coe.error));
     goto done;
   }
   if (coe.subchannel_call != NULL) {
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, op);
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG,
+              "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
+              calld, coe.subchannel_call);
+    }
+    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, batch);
     goto done;
   }
   // Add to waiting-for-pick list.  If we succeed in getting a
   // subchannel call below, we'll handle this batch (along with any
   // other waiting batches) in waiting_for_pick_batches_resume_locked().
-  waiting_for_pick_batches_add_locked(calld, op);
-  /* if this is a cancellation, then we can raise our cancelled flag */
-  if (op->cancel_stream) {
-    grpc_error *error = op->payload->cancel_stream.cancel_error;
+  waiting_for_pick_batches_add_locked(calld, batch);
+  // If this is a cancellation, cancel the pending pick (if any) and
+  // fail any pending batches.
+  if (batch->cancel_stream) {
+    grpc_error *error = batch->payload->cancel_stream.cancel_error;
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: recording cancel_error=%s", chand,
+              calld, grpc_error_string(error));
+    }
     /* Stash a copy of cancel_error in our call data, so that we can use
        it for subsequent operations.  This ensures that if the call is
-       cancelled before any ops are passed down (e.g., if the deadline
+       cancelled before any batches are passed down (e.g., if the deadline
        is in the past when the call starts), we can return the right
-       error to the caller when the first op does get passed down. */
+       error to the caller when the first batch does get passed down. */
     set_call_or_error(calld, (call_or_error){.error = GRPC_ERROR_REF(error)});
-    if (calld->pick_pending) {
-      cancel_pick_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
+    if (calld->lb_policy != NULL) {
+      pick_callback_cancel_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
+    } else {
+      pick_after_resolver_result_cancel_locked(exec_ctx, elem,
+                                               GRPC_ERROR_REF(error));
     }
-    waiting_for_pick_batches_fail_locked(exec_ctx, calld,
-                                         GRPC_ERROR_REF(error));
+    waiting_for_pick_batches_fail_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
     goto done;
   }
   /* if we don't have a subchannel, try to get one */
-  if (!calld->pick_pending && calld->connected_subchannel == NULL &&
-      op->send_initial_metadata) {
-    calld->initial_metadata_payload = op->payload;
-    calld->pick_pending = true;
+  if (batch->send_initial_metadata) {
+    GPR_ASSERT(calld->connected_subchannel == NULL);
+    calld->initial_metadata_payload = batch->payload;
     GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel");
     /* If a subchannel is not available immediately, the polling entity from
        call_data should be provided to channel_data's interested_parties, so
        that IO of the lb_policy and resolver could be done under it. */
     if (pick_subchannel_locked(exec_ctx, elem)) {
       // Pick was returned synchronously.
-      calld->pick_pending = false;
       GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel");
       if (calld->connected_subchannel == NULL) {
         grpc_error *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
             "Call dropped by load balancing policy");
         set_call_or_error(calld,
                           (call_or_error){.error = GRPC_ERROR_REF(error)});
-        waiting_for_pick_batches_fail_locked(exec_ctx, calld, error);
+        waiting_for_pick_batches_fail_locked(exec_ctx, elem, error);
       } else {
         // Create subchannel call.
-        create_subchannel_call_locked(exec_ctx, calld, GRPC_ERROR_NONE);
+        create_subchannel_call_locked(exec_ctx, elem, GRPC_ERROR_NONE);
       }
     } else {
       grpc_polling_entity_add_to_pollset_set(exec_ctx, calld->pollent,
@@ -1232,47 +1375,59 @@ static void on_complete(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
    If it has, we proceed on the fast path. */
 static void cc_start_transport_stream_op_batch(
     grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
-    grpc_transport_stream_op_batch *op) {
+    grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
+  if (GRPC_TRACER_ON(grpc_client_channel_trace) ||
+      GRPC_TRACER_ON(grpc_trace_channel)) {
+    grpc_call_log_op(GPR_INFO, elem, batch);
+  }
   if (chand->deadline_checking_enabled) {
     grpc_deadline_state_client_start_transport_stream_op_batch(exec_ctx, elem,
-                                                               op);
+                                                               batch);
   }
   // Intercept on_complete for recv_trailing_metadata so that we can
   // check retry throttle status.
-  if (op->recv_trailing_metadata) {
-    GPR_ASSERT(op->on_complete != NULL);
-    calld->original_on_complete = op->on_complete;
+  if (batch->recv_trailing_metadata) {
+    GPR_ASSERT(batch->on_complete != NULL);
+    calld->original_on_complete = batch->on_complete;
     GRPC_CLOSURE_INIT(&calld->on_complete, on_complete, elem,
                       grpc_schedule_on_exec_ctx);
-    op->on_complete = &calld->on_complete;
+    batch->on_complete = &calld->on_complete;
   }
   /* try to (atomically) get the call */
   call_or_error coe = get_call_or_error(calld);
   GPR_TIMER_BEGIN("cc_start_transport_stream_op_batch", 0);
   if (coe.error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: failing batch with error: %s",
+              chand, calld, grpc_error_string(coe.error));
+    }
     grpc_transport_stream_op_batch_finish_with_failure(
-        exec_ctx, op, GRPC_ERROR_REF(coe.error));
-    GPR_TIMER_END("cc_start_transport_stream_op_batch", 0);
-    /* early out */
-    return;
+        exec_ctx, batch, GRPC_ERROR_REF(coe.error));
+    goto done;
   }
   if (coe.subchannel_call != NULL) {
-    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, op);
-    GPR_TIMER_END("cc_start_transport_stream_op_batch", 0);
-    /* early out */
-    return;
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG,
+              "chand=%p calld=%p: sending batch to subchannel_call=%p", chand,
+              calld, coe.subchannel_call);
+    }
+    grpc_subchannel_call_process_op(exec_ctx, coe.subchannel_call, batch);
+    goto done;
   }
   /* we failed; lock and figure out what to do */
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG, "chand=%p calld=%p: entering combiner", chand, calld);
+  }
   GRPC_CALL_STACK_REF(calld->owning_call, "start_transport_stream_op_batch");
-  op->handler_private.extra_arg = elem;
+  batch->handler_private.extra_arg = elem;
   GRPC_CLOSURE_SCHED(
-      exec_ctx, GRPC_CLOSURE_INIT(&op->handler_private.closure,
-                                  start_transport_stream_op_batch_locked, op,
+      exec_ctx, GRPC_CLOSURE_INIT(&batch->handler_private.closure,
+                                  start_transport_stream_op_batch_locked, batch,
                                   grpc_combiner_scheduler(chand->combiner)),
       GRPC_ERROR_NONE);
+done:
   GPR_TIMER_END("cc_start_transport_stream_op_batch", 0);
 }
 
@@ -1317,7 +1472,7 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
     GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, coe.subchannel_call,
                                "client_channel_destroy_call");
   }
-  GPR_ASSERT(!calld->pick_pending);
+  GPR_ASSERT(calld->lb_policy == NULL);
   GPR_ASSERT(calld->waiting_for_pick_batches_count == 0);
   if (calld->connected_subchannel != NULL) {
     GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, calld->connected_subchannel,
@@ -1366,11 +1521,7 @@ static void try_to_connect_locked(grpc_exec_ctx *exec_ctx, void *arg,
   } else {
     chand->exit_idle_when_lb_policy_arrives = true;
     if (!chand->started_resolving && chand->resolver != NULL) {
-      GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-      chand->started_resolving = true;
-      grpc_resolver_next_locked(exec_ctx, chand->resolver,
-                                &chand->resolver_result,
-                                &chand->on_resolver_result_changed);
+      start_resolving_locked(exec_ctx, chand);
     }
   }
   GRPC_CHANNEL_STACK_UNREF(exec_ctx, chand->owning_stack, "try_to_connect");
diff --git a/src/core/ext/filters/client_channel/client_channel.h b/src/core/ext/filters/client_channel/client_channel.h
index 63f7c29940..c99f0092e9 100644
--- a/src/core/ext/filters/client_channel/client_channel.h
+++ b/src/core/ext/filters/client_channel/client_channel.h
@@ -23,6 +23,8 @@
 #include "src/core/ext/filters/client_channel/resolver.h"
 #include "src/core/lib/channel/channel_stack.h"
 
+extern grpc_tracer_flag grpc_client_channel_trace;
+
 // Channel arg key for server URI string.
 #define GRPC_ARG_SERVER_URI "grpc.server_uri"
 
diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.c b/src/core/ext/filters/client_channel/client_channel_plugin.c
index 60e77d6268..c32e83d012 100644
--- a/src/core/ext/filters/client_channel/client_channel_plugin.c
+++ b/src/core/ext/filters/client_channel/client_channel_plugin.c
@@ -78,8 +78,9 @@ void grpc_client_channel_init(void) {
       GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, append_filter,
       (void *)&grpc_client_channel_filter);
   grpc_http_connect_register_handshaker_factory();
+  grpc_register_tracer(&grpc_client_channel_trace);
 #ifndef NDEBUG
-  grpc_register_tracer("resolver_refcount", &grpc_trace_resolver_refcount);
+  grpc_register_tracer(&grpc_trace_resolver_refcount);
 #endif
 }
 
diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c
index cfb5ec6f00..ef3512ed83 100644
--- a/src/core/ext/filters/client_channel/http_proxy.c
+++ b/src/core/ext/filters/client_channel/http_proxy.c
@@ -22,6 +22,7 @@
 #include <string.h>
 
 #include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
@@ -29,14 +30,23 @@
 #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
 #include "src/core/ext/filters/client_channel/uri_parser.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/slice/b64.h"
 #include "src/core/lib/support/env.h"
+#include "src/core/lib/support/string.h"
 
-static char* grpc_get_http_proxy_server(grpc_exec_ctx* exec_ctx) {
+/**
+ * Parses the 'http_proxy' env var and returns the proxy hostname to resolve or
+ * NULL on error. Also sets 'user_cred' to user credentials if present in the
+ * 'http_proxy' env var, otherwise leaves it unchanged. It is caller's
+ * responsibility to gpr_free user_cred.
+ */
+static char* get_http_proxy_server(grpc_exec_ctx* exec_ctx, char** user_cred) {
+  GPR_ASSERT(user_cred != NULL);
+  char* proxy_name = NULL;
   char* uri_str = gpr_getenv("http_proxy");
   if (uri_str == NULL) return NULL;
   grpc_uri* uri =
       grpc_uri_parse(exec_ctx, uri_str, false /* suppress_errors */);
-  char* proxy_name = NULL;
   if (uri == NULL || uri->authority == NULL) {
     gpr_log(GPR_ERROR, "cannot parse value of 'http_proxy' env var");
     goto done;
@@ -45,11 +55,27 @@ static char* grpc_get_http_proxy_server(grpc_exec_ctx* exec_ctx) {
     gpr_log(GPR_ERROR, "'%s' scheme not supported in proxy URI", uri->scheme);
     goto done;
   }
-  if (strchr(uri->authority, '@') != NULL) {
-    gpr_log(GPR_ERROR, "userinfo not supported in proxy URI");
-    goto done;
+  /* Split on '@' to separate user credentials from host */
+  char** authority_strs = NULL;
+  size_t authority_nstrs;
+  gpr_string_split(uri->authority, "@", &authority_strs, &authority_nstrs);
+  GPR_ASSERT(authority_nstrs != 0); /* should have at least 1 string */
+  if (authority_nstrs == 1) {
+    /* User cred not present in authority */
+    proxy_name = authority_strs[0];
+  } else if (authority_nstrs == 2) {
+    /* User cred found */
+    *user_cred = authority_strs[0];
+    proxy_name = authority_strs[1];
+    gpr_log(GPR_DEBUG, "userinfo found in proxy URI");
+  } else {
+    /* Bad authority */
+    for (size_t i = 0; i < authority_nstrs; i++) {
+      gpr_free(authority_strs[i]);
+    }
+    proxy_name = NULL;
   }
-  proxy_name = gpr_strdup(uri->authority);
+  gpr_free(authority_strs);
 done:
   gpr_free(uri_str);
   grpc_uri_destroy(uri);
@@ -62,7 +88,8 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
                                   const grpc_channel_args* args,
                                   char** name_to_resolve,
                                   grpc_channel_args** new_args) {
-  *name_to_resolve = grpc_get_http_proxy_server(exec_ctx);
+  char* user_cred = NULL;
+  *name_to_resolve = get_http_proxy_server(exec_ctx, &user_cred);
   if (*name_to_resolve == NULL) return false;
   grpc_uri* uri =
       grpc_uri_parse(exec_ctx, server_uri, false /* suppress_errors */);
@@ -71,19 +98,82 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
             "'http_proxy' environment variable set, but cannot "
             "parse server URI '%s' -- not using proxy",
             server_uri);
-    if (uri != NULL) grpc_uri_destroy(uri);
+    if (uri != NULL) {
+      gpr_free(user_cred);
+      grpc_uri_destroy(uri);
+    }
     return false;
   }
   if (strcmp(uri->scheme, "unix") == 0) {
     gpr_log(GPR_INFO, "not using proxy for Unix domain socket '%s'",
             server_uri);
+    gpr_free(user_cred);
     grpc_uri_destroy(uri);
     return false;
   }
-  grpc_arg new_arg = grpc_channel_arg_string_create(
+  char* no_proxy_str = gpr_getenv("no_proxy");
+  if (no_proxy_str != NULL) {
+    static const char* NO_PROXY_SEPARATOR = ",";
+    bool use_proxy = true;
+    char* server_host;
+    char* server_port;
+    if (!gpr_split_host_port(uri->path[0] == '/' ? uri->path + 1 : uri->path,
+                             &server_host, &server_port)) {
+      gpr_log(GPR_INFO,
+              "unable to split host and port, not checking no_proxy list for "
+              "host '%s'",
+              server_uri);
+    } else {
+      size_t uri_len = strlen(server_host);
+      char** no_proxy_hosts;
+      size_t num_no_proxy_hosts;
+      gpr_string_split(no_proxy_str, NO_PROXY_SEPARATOR, &no_proxy_hosts,
+                       &num_no_proxy_hosts);
+      for (size_t i = 0; i < num_no_proxy_hosts; i++) {
+        char* no_proxy_entry = no_proxy_hosts[i];
+        size_t no_proxy_len = strlen(no_proxy_entry);
+        if (no_proxy_len <= uri_len &&
+            gpr_stricmp(no_proxy_entry, &server_host[uri_len - no_proxy_len]) ==
+                0) {
+          gpr_log(GPR_INFO, "not using proxy for host in no_proxy list '%s'",
+                  server_uri);
+          use_proxy = false;
+          break;
+        }
+      }
+      for (size_t i = 0; i < num_no_proxy_hosts; i++) {
+        gpr_free(no_proxy_hosts[i]);
+      }
+      gpr_free(no_proxy_hosts);
+      gpr_free(server_host);
+      gpr_free(server_port);
+      if (!use_proxy) {
+        grpc_uri_destroy(uri);
+        gpr_free(*name_to_resolve);
+        *name_to_resolve = NULL;
+        return false;
+      }
+    }
+  }
+  grpc_arg args_to_add[2];
+  args_to_add[0] = grpc_channel_arg_string_create(
       GRPC_ARG_HTTP_CONNECT_SERVER,
       uri->path[0] == '/' ? uri->path + 1 : uri->path);
-  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
+  if (user_cred != NULL) {
+    /* Use base64 encoding for user credentials as stated in RFC 7617 */
+    char* encoded_user_cred =
+        grpc_base64_encode(user_cred, strlen(user_cred), 0, 0);
+    char* header;
+    gpr_asprintf(&header, "Proxy-Authorization:Basic %s", encoded_user_cred);
+    gpr_free(encoded_user_cred);
+    args_to_add[1] =
+        grpc_channel_arg_string_create(GRPC_ARG_HTTP_CONNECT_HEADERS, header);
+    *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 2);
+    gpr_free(header);
+  } else {
+    *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 1);
+  }
+  gpr_free(user_cred);
   grpc_uri_destroy(uri);
   return true;
 }
diff --git a/src/core/ext/filters/client_channel/lb_policy.c b/src/core/ext/filters/client_channel/lb_policy.c
index 8d69ba6af5..dd95a135cf 100644
--- a/src/core/ext/filters/client_channel/lb_policy.c
+++ b/src/core/ext/filters/client_channel/lb_policy.c
@@ -22,7 +22,8 @@
 #define WEAK_REF_BITS 16
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_lb_policy_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_lb_policy_refcount =
+    GRPC_TRACER_INITIALIZER(false, "lb_policy_refcount");
 #endif
 
 void grpc_lb_policy_init(grpc_lb_policy *policy,
@@ -53,7 +54,7 @@ static gpr_atm ref_mutate(grpc_lb_policy *c, gpr_atm delta,
 #ifndef NDEBUG
   if (GRPC_TRACER_ON(grpc_trace_lb_policy_refcount)) {
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "LB_POLICY: 0x%p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
+            "LB_POLICY: %p %12s 0x%" PRIxPTR " -> 0x%" PRIxPTR " [%s]", c,
             purpose, old_val, old_val + delta, reason);
   }
 #endif
diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
index 5a5ff2902d..fdb18f687f 100644
--- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
+++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
@@ -123,7 +123,7 @@
 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2
 
-grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb");
 
 /* add lb_token of selected subchannel (address) to the call's initial
  * metadata */
@@ -1705,7 +1705,6 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx,
 static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                               const grpc_lb_policy_args *args) {
   glb_lb_policy *glb_policy = (glb_lb_policy *)policy;
-
   if (glb_policy->updating_lb_channel) {
     if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO,
@@ -1813,9 +1812,11 @@ static void glb_lb_channel_on_connectivity_changed_cb(grpc_exec_ctx *exec_ctx,
         // lb_on_server_status_received will pick up the cancel and reinit
         // lb_call.
         if (glb_policy->pending_update_args != NULL) {
-          const grpc_lb_policy_args *args = glb_policy->pending_update_args;
+          grpc_lb_policy_args *args = glb_policy->pending_update_args;
           glb_policy->pending_update_args = NULL;
           glb_update_locked(exec_ctx, &glb_policy->base, args);
+          grpc_channel_args_destroy(exec_ctx, args->args);
+          gpr_free(args);
         }
       } else if (glb_policy->started_picking && !glb_policy->shutting_down) {
         if (glb_policy->retry_timer_active) {
@@ -1879,9 +1880,9 @@ static bool maybe_add_client_load_reporting_filter(
 
 void grpc_lb_policy_grpclb_init() {
   grpc_register_lb_policy(grpc_glb_lb_factory_create());
-  grpc_register_tracer("glb", &grpc_lb_glb_trace);
+  grpc_register_tracer(&grpc_lb_glb_trace);
 #ifndef NDEBUG
-  grpc_register_tracer("lb_policy_refcount", &grpc_trace_lb_policy_refcount);
+  grpc_register_tracer(&grpc_trace_lb_policy_refcount);
 #endif
   grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL,
                                    GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c
index d0acd7a901..fd0fb41fb9 100644
--- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c
+++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c
@@ -28,7 +28,8 @@
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/transport/connectivity_state.h"
 
-grpc_tracer_flag grpc_lb_pick_first_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_lb_pick_first_trace =
+    GRPC_TRACER_INITIALIZER(false, "pick_first");
 
 typedef struct pending_pick {
   struct pending_pick *next;
@@ -707,7 +708,7 @@ static grpc_lb_policy_factory *pick_first_lb_factory_create() {
 
 void grpc_lb_policy_pick_first_init() {
   grpc_register_lb_policy(pick_first_lb_factory_create());
-  grpc_register_tracer("pick_first", &grpc_lb_pick_first_trace);
+  grpc_register_tracer(&grpc_lb_pick_first_trace);
 }
 
 void grpc_lb_policy_pick_first_shutdown() {}
diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
index 8e9d6b0f47..341763a4d7 100644
--- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
+++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c
@@ -37,7 +37,8 @@
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-grpc_tracer_flag grpc_lb_round_robin_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_lb_round_robin_trace =
+    GRPC_TRACER_INITIALIZER(false, "round_robin");
 
 /** List of entities waiting for a pick.
  *
@@ -141,6 +142,21 @@ struct rr_subchannel_list {
   bool shutting_down;
 };
 
+static rr_subchannel_list *rr_subchannel_list_create(round_robin_lb_policy *p,
+                                                     size_t num_subchannels) {
+  rr_subchannel_list *subchannel_list = gpr_zalloc(sizeof(*subchannel_list));
+  subchannel_list->policy = p;
+  subchannel_list->subchannels =
+      gpr_zalloc(sizeof(subchannel_data) * num_subchannels);
+  subchannel_list->num_subchannels = num_subchannels;
+  gpr_ref_init(&subchannel_list->refcount, 1);
+  if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
+    gpr_log(GPR_INFO, "[RR %p] Created subchannel list %p for %lu subchannels",
+            (void *)p, (void *)subchannel_list, (unsigned long)num_subchannels);
+  }
+  return subchannel_list;
+}
+
 static void rr_subchannel_list_destroy(grpc_exec_ctx *exec_ctx,
                                        rr_subchannel_list *subchannel_list) {
   GPR_ASSERT(subchannel_list->shutting_down);
@@ -158,6 +174,7 @@ static void rr_subchannel_list_destroy(grpc_exec_ctx *exec_ctx,
     if (sd->user_data != NULL) {
       GPR_ASSERT(sd->user_data_vtable != NULL);
       sd->user_data_vtable->destroy(exec_ctx, sd->user_data);
+      sd->user_data = NULL;
     }
   }
   gpr_free(subchannel_list->subchannels);
@@ -169,9 +186,9 @@ static void rr_subchannel_list_ref(rr_subchannel_list *subchannel_list,
   gpr_ref_non_zero(&subchannel_list->refcount);
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
     const gpr_atm count = gpr_atm_acq_load(&subchannel_list->refcount.count);
-    gpr_log(GPR_INFO, "[RR %p] subchannel_list %p REF %lu->%lu",
+    gpr_log(GPR_INFO, "[RR %p] subchannel_list %p REF %lu->%lu (%s)",
             (void *)subchannel_list->policy, (void *)subchannel_list,
-            (unsigned long)(count - 1), (unsigned long)count);
+            (unsigned long)(count - 1), (unsigned long)count, reason);
   }
 }
 
@@ -181,9 +198,9 @@ static void rr_subchannel_list_unref(grpc_exec_ctx *exec_ctx,
   const bool done = gpr_unref(&subchannel_list->refcount);
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
     const gpr_atm count = gpr_atm_acq_load(&subchannel_list->refcount.count);
-    gpr_log(GPR_INFO, "[RR %p] subchannel_list %p UNREF %lu->%lu",
+    gpr_log(GPR_INFO, "[RR %p] subchannel_list %p UNREF %lu->%lu (%s)",
             (void *)subchannel_list->policy, (void *)subchannel_list,
-            (unsigned long)(count + 1), (unsigned long)count);
+            (unsigned long)(count + 1), (unsigned long)count, reason);
   }
   if (done) {
     rr_subchannel_list_destroy(exec_ctx, subchannel_list);
@@ -192,14 +209,27 @@ static void rr_subchannel_list_unref(grpc_exec_ctx *exec_ctx,
 
 /** Mark \a subchannel_list as discarded. Unsubscribes all its subchannels. The
  * watcher's callback will ultimately unref \a subchannel_list.  */
-static void rr_subchannel_list_shutdown(grpc_exec_ctx *exec_ctx,
-                                        rr_subchannel_list *subchannel_list,
-                                        const char *reason) {
+static void rr_subchannel_list_shutdown_and_unref(
+    grpc_exec_ctx *exec_ctx, rr_subchannel_list *subchannel_list,
+    const char *reason) {
+  GPR_ASSERT(!subchannel_list->shutting_down);
+  if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
+    gpr_log(GPR_DEBUG, "[RR %p] Shutting down subchannel_list %p (%s)",
+            (void *)subchannel_list->policy, (void *)subchannel_list, reason);
+  }
   GPR_ASSERT(!subchannel_list->shutting_down);
   subchannel_list->shutting_down = true;
   for (size_t i = 0; i < subchannel_list->num_subchannels; i++) {
     subchannel_data *sd = &subchannel_list->subchannels[i];
     if (sd->subchannel != NULL) {  // if subchannel isn't shutdown, unsubscribe.
+      if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
+        gpr_log(
+            GPR_DEBUG,
+            "[RR %p] Unsubscribing from subchannel %p as part of shutting down "
+            "subchannel_list %p",
+            (void *)subchannel_list->policy, (void *)sd->subchannel,
+            (void *)subchannel_list);
+      }
       grpc_subchannel_notify_on_state_change(exec_ctx, sd->subchannel, NULL,
                                              NULL,
                                              &sd->connectivity_changed_closure);
@@ -228,13 +258,14 @@ static size_t get_next_ready_subchannel_index_locked(
     const size_t index = (i + p->last_ready_subchannel_index + 1) %
                          p->subchannel_list->num_subchannels;
     if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-      gpr_log(GPR_DEBUG,
-              "[RR %p] checking subchannel %p, subchannel_list %p, index %lu: "
-              "state=%d",
-              (void *)p,
-              (void *)p->subchannel_list->subchannels[index].subchannel,
-              (void *)p->subchannel_list, (unsigned long)index,
-              p->subchannel_list->subchannels[index].curr_connectivity_state);
+      gpr_log(
+          GPR_DEBUG,
+          "[RR %p] checking subchannel %p, subchannel_list %p, index %lu: "
+          "state=%s",
+          (void *)p, (void *)p->subchannel_list->subchannels[index].subchannel,
+          (void *)p->subchannel_list, (unsigned long)index,
+          grpc_connectivity_state_name(
+              p->subchannel_list->subchannels[index].curr_connectivity_state));
     }
     if (p->subchannel_list->subchannels[index].curr_connectivity_state ==
         GRPC_CHANNEL_READY) {
@@ -274,7 +305,8 @@ static void update_last_ready_subchannel_index_locked(round_robin_lb_policy *p,
 static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-    gpr_log(GPR_DEBUG, "Destroying Round Robin policy at %p", (void *)pol);
+    gpr_log(GPR_DEBUG, "[RR %p] Destroying Round Robin policy at %p",
+            (void *)pol, (void *)pol);
   }
   grpc_connectivity_state_destroy(exec_ctx, &p->state_tracker);
   gpr_free(p);
@@ -283,7 +315,8 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 static void rr_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-    gpr_log(GPR_DEBUG, "Shutting down Round Robin policy at %p", (void *)pol);
+    gpr_log(GPR_DEBUG, "[RR %p] Shutting down Round Robin policy at %p",
+            (void *)pol, (void *)pol);
   }
   p->shutdown = true;
   pending_pick *pp;
@@ -298,9 +331,18 @@ static void rr_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   grpc_connectivity_state_set(
       exec_ctx, &p->state_tracker, GRPC_CHANNEL_SHUTDOWN,
       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel Shutdown"), "rr_shutdown");
-  rr_subchannel_list_shutdown(exec_ctx, p->subchannel_list,
-                              "sl_shutdown_rr_shutdown");
+  const bool latest_is_current =
+      p->subchannel_list == p->latest_pending_subchannel_list;
+  rr_subchannel_list_shutdown_and_unref(exec_ctx, p->subchannel_list,
+                                        "sl_shutdown_rr_shutdown");
   p->subchannel_list = NULL;
+  if (!latest_is_current && p->latest_pending_subchannel_list != NULL &&
+      !p->latest_pending_subchannel_list->shutting_down) {
+    rr_subchannel_list_shutdown_and_unref(exec_ctx,
+                                          p->latest_pending_subchannel_list,
+                                          "sl_shutdown_pending_rr_shutdown");
+    p->latest_pending_subchannel_list = NULL;
+  }
 }
 
 static void rr_cancel_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
@@ -356,8 +398,8 @@ static void start_picking_locked(grpc_exec_ctx *exec_ctx,
   p->started_picking = true;
   for (size_t i = 0; i < p->subchannel_list->num_subchannels; i++) {
     subchannel_data *sd = &p->subchannel_list->subchannels[i];
-    GRPC_LB_POLICY_WEAK_REF(&p->base, "rr_connectivity");
-    rr_subchannel_list_ref(sd->subchannel_list, "start_picking");
+    GRPC_LB_POLICY_WEAK_REF(&p->base, "start_picking_locked");
+    rr_subchannel_list_ref(sd->subchannel_list, "started_picking");
     grpc_subchannel_notify_on_state_change(
         exec_ctx, sd->subchannel, p->base.interested_parties,
         &sd->pending_connectivity_state_unsafe,
@@ -379,7 +421,7 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
                           grpc_closure *on_complete) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-    gpr_log(GPR_INFO, "Round Robin %p trying to pick", (void *)pol);
+    gpr_log(GPR_INFO, "[RR %p] Trying to pick", (void *)pol);
   }
   if (p->subchannel_list != NULL) {
     const size_t next_ready_index = get_next_ready_subchannel_index_locked(p);
@@ -395,8 +437,8 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
       if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
         gpr_log(
             GPR_DEBUG,
-            "[RR %p] PICKED TARGET <-- SUBCHANNEL %p (CONNECTED %p) (SL %p, "
-            "INDEX %lu)",
+            "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
+            "index %lu)",
             (void *)p, (void *)sd->subchannel, (void *)*target,
             (void *)sd->subchannel_list, (unsigned long)next_ready_index);
       }
@@ -511,38 +553,47 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                            grpc_error *error) {
   subchannel_data *sd = arg;
   round_robin_lb_policy *p = sd->subchannel_list->policy;
+  if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
+    gpr_log(
+        GPR_DEBUG,
+        "[RR %p] connectivity changed for subchannel %p, subchannel_list %p: "
+        "prev_state=%s new_state=%s p->shutdown=%d "
+        "sd->subchannel_list->shutting_down=%d error=%s",
+        (void *)p, (void *)sd->subchannel, (void *)sd->subchannel_list,
+        grpc_connectivity_state_name(sd->prev_connectivity_state),
+        grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe),
+        p->shutdown, sd->subchannel_list->shutting_down,
+        grpc_error_string(error));
+  }
   // If the policy is shutting down, unref and return.
   if (p->shutdown) {
-    rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, "pol_shutdown");
+    rr_subchannel_list_unref(exec_ctx, sd->subchannel_list,
+                             "pol_shutdown+started_picking");
     GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "pol_shutdown");
     return;
   }
-  if (sd->subchannel_list->shutting_down) {
+  if (sd->subchannel_list->shutting_down && error == GRPC_ERROR_CANCELLED) {
     // the subchannel list associated with sd has been discarded. This callback
-    // corresponds to the unsubscription.
-    GPR_ASSERT(error == GRPC_ERROR_CANCELLED);
-    rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, "sl_shutdown");
-    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "sl_shutdown");
+    // corresponds to the unsubscription. The unrefs correspond to the picking
+    // ref (start_picking_locked or update_started_picking).
+    rr_subchannel_list_unref(exec_ctx, sd->subchannel_list,
+                             "sl_shutdown+started_picking");
+    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "sl_shutdown+picking");
     return;
   }
   // Dispose of outdated subchannel lists.
   if (sd->subchannel_list != p->subchannel_list &&
       sd->subchannel_list != p->latest_pending_subchannel_list) {
     // sd belongs to an outdated subchannel_list: get rid of it.
-    rr_subchannel_list_shutdown(exec_ctx, sd->subchannel_list, "sl_oudated");
+    rr_subchannel_list_shutdown_and_unref(exec_ctx, sd->subchannel_list,
+                                          "sl_outdated");
+    GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base, "sl_outdated");
     return;
   }
   // Now that we're inside the combiner, copy the pending connectivity
   // state (which was set by the connectivity state watcher) to
   // curr_connectivity_state, which is what we use inside of the combiner.
   sd->curr_connectivity_state = sd->pending_connectivity_state_unsafe;
-  if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-    gpr_log(GPR_DEBUG,
-            "[RR %p] connectivity changed for subchannel %p: "
-            "prev_state=%d new_state=%d",
-            (void *)p, (void *)sd->subchannel, sd->prev_connectivity_state,
-            sd->curr_connectivity_state);
-  }
   // Update state counters and determine new overall state.
   update_state_counters_locked(sd);
   sd->prev_connectivity_state = sd->curr_connectivity_state;
@@ -556,9 +607,10 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
     if (sd->user_data != NULL) {
       GPR_ASSERT(sd->user_data_vtable != NULL);
       sd->user_data_vtable->destroy(exec_ctx, sd->user_data);
+      sd->user_data = NULL;
     }
     if (new_policy_connectivity_state == GRPC_CHANNEL_SHUTDOWN) {
-      /* the policy is shutting down. Flush all the pending picks... */
+      // the policy is shutting down. Flush all the pending picks...
       pending_pick *pp;
       while ((pp = p->pending_picks)) {
         p->pending_picks = pp->next;
@@ -567,8 +619,9 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
         gpr_free(pp);
       }
     }
-    /* unref the "rr_connectivity" weak ref from start_picking */
-    rr_subchannel_list_unref(exec_ctx, sd->subchannel_list, "sd_shutdown");
+    rr_subchannel_list_unref(exec_ctx, sd->subchannel_list,
+                             "sd_shutdown+started_picking");
+    // unref the "rr_connectivity_update" weak ref from start_picking.
     GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base,
                               "rr_connectivity_sd_shutdown");
   } else {  // sd not in SHUTDOWN
@@ -593,10 +646,10 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
         }
         if (p->subchannel_list != NULL) {
           // dispose of the current subchannel_list
-          rr_subchannel_list_shutdown(exec_ctx, p->subchannel_list,
-                                      "sl_shutdown_rr_update_connectivity");
+          rr_subchannel_list_shutdown_and_unref(exec_ctx, p->subchannel_list,
+                                                "sl_phase_out_shutdown");
         }
-        p->subchannel_list = sd->subchannel_list;
+        p->subchannel_list = p->latest_pending_subchannel_list;
         p->latest_pending_subchannel_list = NULL;
       }
       /* at this point we know there's at least one suitable subchannel. Go
@@ -607,8 +660,8 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
       subchannel_data *selected =
           &p->subchannel_list->subchannels[next_ready_index];
       if (p->pending_picks != NULL) {
-        /* if the selected subchannel is going to be used for the pending
-         * picks, update the last picked pointer */
+        // if the selected subchannel is going to be used for the pending
+        // picks, update the last picked pointer
         update_last_ready_subchannel_index_locked(p, next_ready_index);
       }
       pending_pick *pp;
@@ -622,16 +675,17 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg,
         }
         if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
           gpr_log(GPR_DEBUG,
-                  "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (INDEX %lu)",
-                  (void *)selected->subchannel,
-                  (unsigned long)next_ready_index);
+                  "[RR %p] Fulfilling pending pick. Target <-- subchannel %p "
+                  "(subchannel_list %p, index %lu)",
+                  (void *)p, (void *)selected->subchannel,
+                  (void *)p->subchannel_list, (unsigned long)next_ready_index);
         }
         GRPC_CLOSURE_SCHED(exec_ctx, pp->on_complete, GRPC_ERROR_NONE);
         gpr_free(pp);
       }
     }
-    /* renew notification: reuses the "rr_connectivity" weak ref on the policy
-     * as well as the sd->subchannel_list ref. */
+    /* renew notification: reuses the "rr_connectivity_update" weak ref on the
+     * policy as well as the sd->subchannel_list ref. */
     grpc_subchannel_notify_on_state_change(
         exec_ctx, sd->subchannel, p->base.interested_parties,
         &sd->pending_connectivity_state_unsafe,
@@ -689,8 +743,7 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     } else {
       // otherwise, keep using the current subchannel list (ignore this update).
       gpr_log(GPR_ERROR,
-              "No valid LB addresses channel arg for Round Robin %p update, "
-              "ignoring.",
+              "[RR %p] No valid LB addresses channel arg for update, ignoring.",
               (void *)p);
     }
     return;
@@ -706,24 +759,27 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
         "rr_update_empty");
     if (p->subchannel_list != NULL) {
-      rr_subchannel_list_shutdown(exec_ctx, p->subchannel_list,
-                                  "sl_shutdown_rr_update");
+      rr_subchannel_list_shutdown_and_unref(exec_ctx, p->subchannel_list,
+                                            "sl_shutdown_empty_update");
       p->subchannel_list = NULL;
     }
     return;
   }
   size_t subchannel_index = 0;
-  rr_subchannel_list *subchannel_list = gpr_zalloc(sizeof(*subchannel_list));
-  subchannel_list->policy = p;
-  subchannel_list->subchannels =
-      gpr_zalloc(sizeof(subchannel_data) * num_addrs);
-  subchannel_list->num_subchannels = num_addrs;
-  gpr_ref_init(&subchannel_list->refcount, 1);
-  p->latest_pending_subchannel_list = subchannel_list;
-  if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-    gpr_log(GPR_DEBUG, "Created subchannel list %p for %lu subchannels",
-            (void *)subchannel_list, (unsigned long)num_addrs);
+  rr_subchannel_list *subchannel_list = rr_subchannel_list_create(p, num_addrs);
+  if (p->latest_pending_subchannel_list != NULL && p->started_picking) {
+    if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
+      gpr_log(GPR_DEBUG,
+              "[RR %p] Shutting down latest pending subchannel list %p, about "
+              "to be "
+              "replaced by newer latest %p",
+              (void *)p, (void *)p->latest_pending_subchannel_list,
+              (void *)subchannel_list);
+    }
+    rr_subchannel_list_shutdown_and_unref(
+        exec_ctx, p->latest_pending_subchannel_list, "sl_outdated_dont_smash");
   }
+  p->latest_pending_subchannel_list = subchannel_list;
   grpc_subchannel_args sc_args;
   /* We need to remove the LB addresses in order to be able to compare the
    * subchannel keys of subchannels from a different batch of addresses. */
@@ -747,11 +803,12 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
       char *address_uri =
           grpc_sockaddr_to_uri(&addresses->addresses[i].address);
-      gpr_log(GPR_DEBUG,
-              "index %lu: Created subchannel %p for address uri %s into "
-              "subchannel_list %p",
-              (unsigned long)subchannel_index, (void *)subchannel, address_uri,
-              (void *)subchannel_list);
+      gpr_log(
+          GPR_DEBUG,
+          "[RR %p] index %lu: Created subchannel %p for address uri %s into "
+          "subchannel_list %p",
+          (void *)p, (unsigned long)subchannel_index, (void *)subchannel,
+          address_uri, (void *)subchannel_list);
       gpr_free(address_uri);
     }
     grpc_channel_args_destroy(exec_ctx, new_args);
@@ -790,10 +847,11 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     // The policy isn't picking yet. Save the update for later, disposing of
     // previous version if any.
     if (p->subchannel_list != NULL) {
-      rr_subchannel_list_shutdown(exec_ctx, p->subchannel_list,
-                                  "rr_update_before_started_picking");
+      rr_subchannel_list_shutdown_and_unref(exec_ctx, p->subchannel_list,
+                                            "rr_update_before_started_picking");
     }
     p->subchannel_list = subchannel_list;
+    p->latest_pending_subchannel_list = NULL;
   }
 }
 
@@ -823,7 +881,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
   grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE,
                                "round_robin");
   if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) {
-    gpr_log(GPR_DEBUG, "Created Round Robin %p with %lu subchannels", (void *)p,
+    gpr_log(GPR_DEBUG, "[RR %p] Created with %lu subchannels", (void *)p,
             (unsigned long)p->subchannel_list->num_subchannels);
   }
   return &p->base;
@@ -844,7 +902,7 @@ static grpc_lb_policy_factory *round_robin_lb_factory_create() {
 
 void grpc_lb_policy_round_robin_init() {
   grpc_register_lb_policy(round_robin_lb_factory_create());
-  grpc_register_tracer("round_robin", &grpc_lb_round_robin_trace);
+  grpc_register_tracer(&grpc_lb_round_robin_trace);
 }
 
 void grpc_lb_policy_round_robin_shutdown() {}
diff --git a/src/core/ext/filters/client_channel/resolver.c b/src/core/ext/filters/client_channel/resolver.c
index de9a8ce41b..8401504fcf 100644
--- a/src/core/ext/filters/client_channel/resolver.c
+++ b/src/core/ext/filters/client_channel/resolver.c
@@ -20,7 +20,8 @@
 #include "src/core/lib/iomgr/combiner.h"
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_resolver_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_resolver_refcount =
+    GRPC_TRACER_INITIALIZER(false, "resolver_refcount");
 #endif
 
 void grpc_resolver_init(grpc_resolver *resolver,
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c
index 1ab8295e9e..b696344eab 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.c
@@ -103,10 +103,9 @@ static void fd_node_destroy(grpc_exec_ctx *exec_ctx, fd_node *fdn) {
   grpc_pollset_set_del_fd(exec_ctx, fdn->ev_driver->pollset_set, fdn->grpc_fd);
   /* c-ares library has closed the fd inside grpc_fd. This fd may be picked up
      immediately by another thread, and should not be closed by the following
-     grpc_fd_orphan. To prevent this fd from being closed by grpc_fd_orphan,
-     a fd pointer is provided. */
-  int fd;
-  grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, &fd, "c-ares query finished");
+     grpc_fd_orphan. */
+  grpc_fd_orphan(exec_ctx, fdn->grpc_fd, NULL, NULL, true /* already_closed */,
+                 "c-ares query finished");
   gpr_free(fdn);
 }
 
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
index 244b260dfa..9065e33613 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.c
@@ -236,12 +236,12 @@ static void on_srv_query_done_cb(void *arg, int status, int timeouts,
            srv_it = srv_it->next) {
         if (grpc_ipv6_loopback_available()) {
           grpc_ares_hostbyname_request *hr = create_hostbyname_request(
-              r, srv_it->host, srv_it->port, true /* is_balancer */);
+              r, srv_it->host, htons(srv_it->port), true /* is_balancer */);
           ares_gethostbyname(*channel, hr->host, AF_INET6,
                              on_hostbyname_done_cb, hr);
         }
         grpc_ares_hostbyname_request *hr = create_hostbyname_request(
-            r, srv_it->host, srv_it->port, true /* is_balancer */);
+            r, srv_it->host, htons(srv_it->port), true /* is_balancer */);
         ares_gethostbyname(*channel, hr->host, AF_INET, on_hostbyname_done_cb,
                            hr);
         grpc_ares_ev_driver_start(&exec_ctx, r->ev_driver);
diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c
index af3391a731..5ea75f0554 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c
+++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.c
@@ -132,7 +132,7 @@ static void dns_next_locked(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver,
 
 static void dns_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                       grpc_error *error) {
-  dns_resolver *r = arg;
+  dns_resolver *r = (dns_resolver *)arg;
 
   r->have_retry_timer = false;
   if (error == GRPC_ERROR_NONE) {
@@ -146,7 +146,7 @@ static void dns_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
 
 static void dns_on_resolved_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_error *error) {
-  dns_resolver *r = arg;
+  dns_resolver *r = (dns_resolver *)arg;
   grpc_channel_args *result = NULL;
   GPR_ASSERT(r->resolving);
   r->resolving = false;
@@ -241,7 +241,7 @@ static grpc_resolver *dns_create(grpc_exec_ctx *exec_ctx,
   char *path = args->uri->path;
   if (path[0] == '/') ++path;
   // Create resolver.
-  dns_resolver *r = gpr_zalloc(sizeof(dns_resolver));
+  dns_resolver *r = (dns_resolver *)gpr_zalloc(sizeof(dns_resolver));
   grpc_resolver_init(&r->base, &dns_resolver_vtable, args->combiner);
   r->name_to_resolve = gpr_strdup(path);
   r->default_port = gpr_strdup(default_port);
diff --git a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c
index 7b4fe38272..7ceb8f40a1 100644
--- a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c
+++ b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c
@@ -177,7 +177,8 @@ static grpc_resolver *sockaddr_create(grpc_exec_ctx *exec_ctx,
     return NULL;
   }
   /* Instantiate resolver. */
-  sockaddr_resolver *r = gpr_zalloc(sizeof(sockaddr_resolver));
+  sockaddr_resolver *r =
+      (sockaddr_resolver *)gpr_zalloc(sizeof(sockaddr_resolver));
   r->addresses = addresses;
   r->channel_args = grpc_channel_args_copy(args->args);
   grpc_resolver_init(&r->base, &sockaddr_resolver_vtable, args->combiner);
diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c
index 88157ed738..5788819331 100644
--- a/src/core/ext/filters/client_channel/subchannel.c
+++ b/src/core/ext/filters/client_channel/subchannel.c
@@ -188,6 +188,7 @@ static void subchannel_destroy(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_connector_unref(exec_ctx, c->connector);
   grpc_pollset_set_destroy(exec_ctx, c->pollset_set);
   grpc_subchannel_key_destroy(exec_ctx, c->key);
+  gpr_mu_destroy(&c->mu);
   gpr_free(c);
 }
 
diff --git a/src/core/ext/filters/client_channel/subchannel_index.c b/src/core/ext/filters/client_channel/subchannel_index.c
index e291ca9db9..a33ab950bf 100644
--- a/src/core/ext/filters/client_channel/subchannel_index.c
+++ b/src/core/ext/filters/client_channel/subchannel_index.c
@@ -40,6 +40,8 @@ struct grpc_subchannel_key {
 
 GPR_TLS_DECL(subchannel_index_exec_ctx);
 
+static bool g_force_creation = false;
+
 static void enter_ctx(grpc_exec_ctx *exec_ctx) {
   GPR_ASSERT(gpr_tls_get(&subchannel_index_exec_ctx) == 0);
   gpr_tls_set(&subchannel_index_exec_ctx, (intptr_t)exec_ctx);
@@ -84,6 +86,7 @@ static grpc_subchannel_key *subchannel_key_copy(grpc_subchannel_key *k) {
 
 int grpc_subchannel_key_compare(const grpc_subchannel_key *a,
                                 const grpc_subchannel_key *b) {
+  if (g_force_creation) return false;
   int c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
   if (c != 0) return c;
   if (a->args.filter_count > 0) {
@@ -250,3 +253,7 @@ void grpc_subchannel_index_unregister(grpc_exec_ctx *exec_ctx,
 
   leave_ctx(exec_ctx);
 }
+
+void grpc_subchannel_index_test_only_set_force_creation(bool force_creation) {
+  g_force_creation = force_creation;
+}
diff --git a/src/core/ext/filters/client_channel/subchannel_index.h b/src/core/ext/filters/client_channel/subchannel_index.h
index e303bfaa05..98d882a453 100644
--- a/src/core/ext/filters/client_channel/subchannel_index.h
+++ b/src/core/ext/filters/client_channel/subchannel_index.h
@@ -59,4 +59,16 @@ void grpc_subchannel_index_init(void);
 /** Shutdown the subchannel index (global) */
 void grpc_subchannel_index_shutdown(void);
 
+/** \em TEST ONLY.
+ * If \a force_creation is true, all key comparisons will be false, resulting in
+ * new subchannels always being created. Otherwise, the keys will be compared as
+ * usual.
+ *
+ * This function is *not* threadsafe on purpose: it should *only* be used in
+ * test code.
+ *
+ * Tests using this function \em MUST run tests with and without \a
+ * force_creation set. */
+void grpc_subchannel_index_test_only_set_force_creation(bool force_creation);
+
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_SUBCHANNEL_INDEX_H */
diff --git a/src/core/ext/filters/deadline/deadline_filter.c b/src/core/ext/filters/deadline/deadline_filter.c
index ced025e2e2..6789903c95 100644
--- a/src/core/ext/filters/deadline/deadline_filter.c
+++ b/src/core/ext/filters/deadline/deadline_filter.c
@@ -37,8 +37,8 @@
 // Timer callback.
 static void timer_callback(grpc_exec_ctx* exec_ctx, void* arg,
                            grpc_error* error) {
-  grpc_call_element* elem = arg;
-  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_call_element* elem = (grpc_call_element*)arg;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   if (error != GRPC_ERROR_CANCELLED) {
     grpc_call_element_signal_error(
         exec_ctx, elem,
@@ -57,7 +57,7 @@ static void start_timer_if_needed(grpc_exec_ctx* exec_ctx,
   if (gpr_time_cmp(deadline, gpr_inf_future(GPR_CLOCK_MONOTONIC)) == 0) {
     return;
   }
-  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   grpc_deadline_timer_state cur_state;
   grpc_closure* closure = NULL;
 retry:
@@ -112,7 +112,7 @@ static void cancel_timer_if_needed(grpc_exec_ctx* exec_ctx,
 
 // Callback run when the call is complete.
 static void on_complete(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
-  grpc_deadline_state* deadline_state = arg;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)arg;
   cancel_timer_if_needed(exec_ctx, deadline_state);
   // Invoke the next callback.
   GRPC_CLOSURE_RUN(exec_ctx, deadline_state->next_on_complete,
@@ -145,7 +145,7 @@ static void start_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
 void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               grpc_call_stack* call_stack,
                               gpr_timespec deadline) {
-  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   deadline_state->call_stack = call_stack;
   // Deadline will always be infinite on servers, so the timer will only be
   // set on clients with a finite deadline.
@@ -169,13 +169,13 @@ void grpc_deadline_state_init(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
 
 void grpc_deadline_state_destroy(grpc_exec_ctx* exec_ctx,
                                  grpc_call_element* elem) {
-  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   cancel_timer_if_needed(exec_ctx, deadline_state);
 }
 
 void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                                gpr_timespec new_deadline) {
-  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   cancel_timer_if_needed(exec_ctx, deadline_state);
   start_timer_if_needed(exec_ctx, elem, new_deadline);
 }
@@ -183,7 +183,7 @@ void grpc_deadline_state_reset(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
 void grpc_deadline_state_client_start_transport_stream_op_batch(
     grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     grpc_transport_stream_op_batch* op) {
-  grpc_deadline_state* deadline_state = elem->call_data;
+  grpc_deadline_state* deadline_state = (grpc_deadline_state*)elem->call_data;
   if (op->cancel_stream) {
     cancel_timer_if_needed(exec_ctx, deadline_state);
   } else {
@@ -256,8 +256,8 @@ static void client_start_transport_stream_op_batch(
 // Callback for receiving initial metadata on the server.
 static void recv_initial_metadata_ready(grpc_exec_ctx* exec_ctx, void* arg,
                                         grpc_error* error) {
-  grpc_call_element* elem = arg;
-  server_call_data* calld = elem->call_data;
+  grpc_call_element* elem = (grpc_call_element*)arg;
+  server_call_data* calld = (server_call_data*)elem->call_data;
   // Get deadline from metadata and start the timer if needed.
   start_timer_if_needed(exec_ctx, elem, calld->recv_initial_metadata->deadline);
   // Invoke the next callback.
@@ -269,7 +269,7 @@ static void recv_initial_metadata_ready(grpc_exec_ctx* exec_ctx, void* arg,
 static void server_start_transport_stream_op_batch(
     grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     grpc_transport_stream_op_batch* op) {
-  server_call_data* calld = elem->call_data;
+  server_call_data* calld = (server_call_data*)elem->call_data;
   if (op->cancel_stream) {
     cancel_timer_if_needed(exec_ctx, &calld->base.deadline_state);
   } else {
@@ -341,8 +341,8 @@ static bool maybe_add_deadline_filter(grpc_exec_ctx* exec_ctx,
                                       void* arg) {
   return grpc_deadline_checking_enabled(
              grpc_channel_stack_builder_get_channel_arguments(builder))
-             ? grpc_channel_stack_builder_prepend_filter(builder, arg, NULL,
-                                                         NULL)
+             ? grpc_channel_stack_builder_prepend_filter(
+                   builder, (const grpc_channel_filter*)arg, NULL, NULL)
              : true;
 }
 
diff --git a/src/core/ext/filters/http/http_filters_plugin.c b/src/core/ext/filters/http/http_filters_plugin.c
index 3e4ec01a31..a5c1b92054 100644
--- a/src/core/ext/filters/http/http_filters_plugin.c
+++ b/src/core/ext/filters/http/http_filters_plugin.c
@@ -65,7 +65,7 @@ static bool maybe_add_required_filter(grpc_exec_ctx *exec_ctx,
 }
 
 void grpc_http_filters_init(void) {
-  grpc_register_tracer("compression", &grpc_compression_trace);
+  grpc_register_tracer(&grpc_compression_trace);
   grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL,
                                    GRPC_CHANNEL_INIT_BUILTIN_PRIORITY,
                                    maybe_add_optional_filter, &compress_filter);
diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c
index 35304f8150..7d748b9c32 100644
--- a/src/core/ext/filters/max_age/max_age_filter.c
+++ b/src/core/ext/filters/max_age/max_age_filter.c
@@ -108,7 +108,7 @@ static void decrease_call_count(grpc_exec_ctx* exec_ctx, channel_data* chand) {
 
 static void start_max_idle_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
                                             grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   /* Decrease call_count. If there are no active calls at this time,
      max_idle_timer will start here. If the number of active calls is not 0,
      max_idle_timer will start after all the active calls end. */
@@ -119,7 +119,7 @@ static void start_max_idle_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
 
 static void start_max_age_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
                                            grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   gpr_mu_lock(&chand->max_age_timer_mu);
   chand->max_age_timer_pending = true;
   GRPC_CHANNEL_STACK_REF(chand->channel_stack, "max_age max_age_timer");
@@ -140,7 +140,7 @@ static void start_max_age_timer_after_init(grpc_exec_ctx* exec_ctx, void* arg,
 static void start_max_age_grace_timer_after_goaway_op(grpc_exec_ctx* exec_ctx,
                                                       void* arg,
                                                       grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   gpr_mu_lock(&chand->max_age_timer_mu);
   chand->max_age_grace_timer_pending = true;
   GRPC_CHANNEL_STACK_REF(chand->channel_stack, "max_age max_age_grace_timer");
@@ -156,7 +156,7 @@ static void start_max_age_grace_timer_after_goaway_op(grpc_exec_ctx* exec_ctx,
 
 static void close_max_idle_channel(grpc_exec_ctx* exec_ctx, void* arg,
                                    grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   if (error == GRPC_ERROR_NONE) {
     /* Prevent the max idle timer from being set again */
     gpr_atm_no_barrier_fetch_add(&chand->call_count, 1);
@@ -176,7 +176,7 @@ static void close_max_idle_channel(grpc_exec_ctx* exec_ctx, void* arg,
 
 static void close_max_age_channel(grpc_exec_ctx* exec_ctx, void* arg,
                                   grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   gpr_mu_lock(&chand->max_age_timer_mu);
   chand->max_age_timer_pending = false;
   gpr_mu_unlock(&chand->max_age_timer_mu);
@@ -200,7 +200,7 @@ static void close_max_age_channel(grpc_exec_ctx* exec_ctx, void* arg,
 
 static void force_close_max_age_channel(grpc_exec_ctx* exec_ctx, void* arg,
                                         grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   gpr_mu_lock(&chand->max_age_timer_mu);
   chand->max_age_grace_timer_pending = false;
   gpr_mu_unlock(&chand->max_age_timer_mu);
@@ -220,7 +220,7 @@ static void force_close_max_age_channel(grpc_exec_ctx* exec_ctx, void* arg,
 
 static void channel_connectivity_changed(grpc_exec_ctx* exec_ctx, void* arg,
                                          grpc_error* error) {
-  channel_data* chand = arg;
+  channel_data* chand = (channel_data*)arg;
   if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
     grpc_transport_op* op = grpc_make_transport_op(NULL);
     op->on_connectivity_state_change = &chand->channel_connectivity_changed,
@@ -264,7 +264,7 @@ static int add_random_max_connection_age_jitter(int value) {
 static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   const grpc_call_element_args* args) {
-  channel_data* chand = elem->channel_data;
+  channel_data* chand = (channel_data*)elem->channel_data;
   increase_call_count(exec_ctx, chand);
   return GRPC_ERROR_NONE;
 }
@@ -281,7 +281,7 @@ static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
 static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx,
                                      grpc_channel_element* elem,
                                      grpc_channel_element_args* args) {
-  channel_data* chand = elem->channel_data;
+  channel_data* chand = (channel_data*)elem->channel_data;
   gpr_mu_init(&chand->max_age_timer_mu);
   chand->max_age_timer_pending = false;
   chand->max_age_grace_timer_pending = false;
diff --git a/src/core/ext/filters/message_size/message_size_filter.c b/src/core/ext/filters/message_size/message_size_filter.c
index 9bb565ed6d..846c7df69a 100644
--- a/src/core/ext/filters/message_size/message_size_filter.c
+++ b/src/core/ext/filters/message_size/message_size_filter.c
@@ -60,7 +60,8 @@ static void* message_size_limits_create_from_json(const grpc_json* json) {
       if (max_response_message_bytes == -1) return NULL;
     }
   }
-  message_size_limits* value = gpr_malloc(sizeof(message_size_limits));
+  message_size_limits* value =
+      (message_size_limits*)gpr_malloc(sizeof(message_size_limits));
   value->max_send_size = max_request_message_bytes;
   value->max_recv_size = max_response_message_bytes;
   return value;
@@ -88,8 +89,8 @@ typedef struct channel_data {
 // receive message size.
 static void recv_message_ready(grpc_exec_ctx* exec_ctx, void* user_data,
                                grpc_error* error) {
-  grpc_call_element* elem = user_data;
-  call_data* calld = elem->call_data;
+  grpc_call_element* elem = (grpc_call_element*)user_data;
+  call_data* calld = (call_data*)elem->call_data;
   if (*calld->recv_message != NULL && calld->limits.max_recv_size >= 0 &&
       (*calld->recv_message)->length > (size_t)calld->limits.max_recv_size) {
     char* message_string;
@@ -117,7 +118,7 @@ static void recv_message_ready(grpc_exec_ctx* exec_ctx, void* user_data,
 static void start_transport_stream_op_batch(
     grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     grpc_transport_stream_op_batch* op) {
-  call_data* calld = elem->call_data;
+  call_data* calld = (call_data*)elem->call_data;
   // Check max send message size.
   if (op->send_message && calld->limits.max_send_size >= 0 &&
       op->payload->send_message.send_message->length >
@@ -149,8 +150,8 @@ static void start_transport_stream_op_batch(
 static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   const grpc_call_element_args* args) {
-  channel_data* chand = elem->channel_data;
-  call_data* calld = elem->call_data;
+  channel_data* chand = (channel_data*)elem->channel_data;
+  call_data* calld = (call_data*)elem->call_data;
   calld->next_recv_message_ready = NULL;
   GRPC_CLOSURE_INIT(&calld->recv_message_ready, recv_message_ready, elem,
                     grpc_schedule_on_exec_ctx);
@@ -160,8 +161,9 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
   // size to the receive limit.
   calld->limits = chand->limits;
   if (chand->method_limit_table != NULL) {
-    message_size_limits* limits = grpc_method_config_table_get(
-        exec_ctx, chand->method_limit_table, args->path);
+    message_size_limits* limits =
+        (message_size_limits*)grpc_method_config_table_get(
+            exec_ctx, chand->method_limit_table, args->path);
     if (limits != NULL) {
       if (limits->max_send_size >= 0 &&
           (limits->max_send_size < calld->limits.max_send_size ||
@@ -220,7 +222,7 @@ static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx,
                                      grpc_channel_element* elem,
                                      grpc_channel_element_args* args) {
   GPR_ASSERT(!args->is_last);
-  channel_data* chand = elem->channel_data;
+  channel_data* chand = (channel_data*)elem->channel_data;
   chand->limits = get_message_size_limits(args->channel_args);
   // Get method config table from channel args.
   const grpc_arg* channel_arg =
@@ -243,7 +245,7 @@ static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx,
 // Destructor for channel_data.
 static void destroy_channel_elem(grpc_exec_ctx* exec_ctx,
                                  grpc_channel_element* elem) {
-  channel_data* chand = elem->channel_data;
+  channel_data* chand = (channel_data*)elem->channel_data;
   grpc_slice_hash_table_unref(exec_ctx, chand->method_limit_table);
 }
 
diff --git a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c
index 8b3fff5fa3..b4d2cb4b8c 100644
--- a/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c
+++ b/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.c
@@ -52,8 +52,8 @@ static bool get_user_agent_mdelem(const grpc_metadata_batch* batch,
 // Callback invoked when we receive an initial metadata.
 static void recv_initial_metadata_ready(grpc_exec_ctx* exec_ctx,
                                         void* user_data, grpc_error* error) {
-  grpc_call_element* elem = user_data;
-  call_data* calld = elem->call_data;
+  grpc_call_element* elem = (grpc_call_element*)user_data;
+  call_data* calld = (call_data*)elem->call_data;
 
   if (GRPC_ERROR_NONE == error) {
     grpc_mdelem md;
@@ -75,7 +75,7 @@ static void recv_initial_metadata_ready(grpc_exec_ctx* exec_ctx,
 static void start_transport_stream_op_batch(
     grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     grpc_transport_stream_op_batch* op) {
-  call_data* calld = elem->call_data;
+  call_data* calld = (call_data*)elem->call_data;
 
   // Inject callback for receiving initial metadata
   if (op->recv_initial_metadata) {
@@ -103,7 +103,7 @@ static void start_transport_stream_op_batch(
 static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
                                   grpc_call_element* elem,
                                   const grpc_call_element_args* args) {
-  call_data* calld = elem->call_data;
+  call_data* calld = (call_data*)elem->call_data;
   calld->next_recv_initial_metadata_ready = NULL;
   calld->workaround_active = false;
   GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
diff --git a/src/core/ext/filters/workarounds/workaround_utils.c b/src/core/ext/filters/workarounds/workaround_utils.c
index bc76753a8a..e600fbee67 100644
--- a/src/core/ext/filters/workarounds/workaround_utils.c
+++ b/src/core/ext/filters/workarounds/workaround_utils.c
@@ -33,7 +33,8 @@ grpc_workaround_user_agent_md *grpc_parse_user_agent(grpc_mdelem md) {
   if (NULL != user_agent_md) {
     return user_agent_md;
   }
-  user_agent_md = gpr_malloc(sizeof(grpc_workaround_user_agent_md));
+  user_agent_md = (grpc_workaround_user_agent_md *)gpr_malloc(
+      sizeof(grpc_workaround_user_agent_md));
   for (int i = 0; i < GRPC_MAX_WORKAROUND_ID; i++) {
     if (ua_parser[i]) {
       user_agent_md->workaround_active[i] = ua_parser[i](md);
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c
index 6a8c81445a..78551df9c3 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c
+++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c
@@ -21,10 +21,10 @@
 #include "src/core/lib/transport/metadata.h"
 
 void grpc_chttp2_plugin_init(void) {
-  grpc_register_tracer("http", &grpc_http_trace);
-  grpc_register_tracer("flowctl", &grpc_flowctl_trace);
+  grpc_register_tracer(&grpc_http_trace);
+  grpc_register_tracer(&grpc_flowctl_trace);
 #ifndef NDEBUG
-  grpc_register_tracer("chttp2_refcount", &grpc_trace_chttp2_refcount);
+  grpc_register_tracer(&grpc_trace_chttp2_refcount);
 #endif
 }
 
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
index 6e8eadf7a1..731ebf400f 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c
@@ -33,7 +33,9 @@
 #include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/ext/transport/chttp2/transport/varint.h"
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/compression/stream_compression.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
@@ -73,11 +75,12 @@ static bool g_default_keepalive_permit_without_calls =
     DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS;
 
 #define MAX_CLIENT_STREAM_ID 0x7fffffffu
-grpc_tracer_flag grpc_http_trace = GRPC_TRACER_INITIALIZER(false);
-grpc_tracer_flag grpc_flowctl_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_http_trace = GRPC_TRACER_INITIALIZER(false, "http");
+grpc_tracer_flag grpc_flowctl_trace = GRPC_TRACER_INITIALIZER(false, "flowctl");
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_chttp2_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_chttp2_refcount =
+    GRPC_TRACER_INITIALIZER(false, "chttp2_refcount");
 #endif
 
 static const grpc_transport_vtable vtable;
@@ -280,8 +283,6 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   grpc_slice_buffer_init(&t->outbuf);
   grpc_chttp2_hpack_compressor_init(&t->hpack_compressor);
 
-  GRPC_CLOSURE_INIT(&t->write_action, write_action, t,
-                    grpc_schedule_on_exec_ctx);
   GRPC_CLOSURE_INIT(&t->read_action_locked, read_action_locked, t,
                     grpc_combiner_scheduler(t->combiner));
   GRPC_CLOSURE_INIT(&t->benign_reclaimer_locked, benign_reclaimer_locked, t,
@@ -399,6 +400,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   }
   t->keepalive_permit_without_calls = g_default_keepalive_permit_without_calls;
 
+  t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY;
+
   if (channel_args) {
     for (i = 0; i < channel_args->num_args; i++) {
       if (0 == strcmp(channel_args->args[i].key,
@@ -487,6 +490,23 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
         t->keepalive_permit_without_calls =
             (uint32_t)grpc_channel_arg_get_integer(
                 &channel_args->args[i], (grpc_integer_options){0, 0, 1});
+      } else if (0 == strcmp(channel_args->args[i].key,
+                             GRPC_ARG_OPTIMIZATION_TARGET)) {
+        if (channel_args->args[i].type != GRPC_ARG_STRING) {
+          gpr_log(GPR_ERROR, "%s should be a string",
+                  GRPC_ARG_OPTIMIZATION_TARGET);
+        } else if (0 == strcmp(channel_args->args[i].value.string, "blend")) {
+          t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY;
+        } else if (0 == strcmp(channel_args->args[i].value.string, "latency")) {
+          t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY;
+        } else if (0 ==
+                   strcmp(channel_args->args[i].value.string, "throughput")) {
+          t->opt_target = GRPC_CHTTP2_OPTIMIZE_FOR_THROUGHPUT;
+        } else {
+          gpr_log(GPR_ERROR, "%s value '%s' unknown, assuming 'blend'",
+                  GRPC_ARG_OPTIMIZATION_TARGET,
+                  channel_args->args[i].value.string);
+        }
       } else {
         static const struct {
           const char *channel_arg_name;
@@ -540,6 +560,11 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     }
   }
 
+  GRPC_CLOSURE_INIT(&t->write_action, write_action, t,
+                    t->opt_target == GRPC_CHTTP2_OPTIMIZE_FOR_THROUGHPUT
+                        ? grpc_executor_scheduler
+                        : grpc_schedule_on_exec_ctx);
+
   t->ping_state.pings_before_data_required =
       t->ping_policy.max_pings_without_data;
   t->ping_state.is_delayed_ping_timer_set = false;
@@ -1200,7 +1225,7 @@ static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {}
 
 static void log_metadata(const grpc_metadata_batch *md_batch, uint32_t id,
                          bool is_client, bool is_initial) {
-  for (grpc_linked_mdelem *md = md_batch->list.head; md != md_batch->list.tail;
+  for (grpc_linked_mdelem *md = md_batch->list.head; md != NULL;
        md = md->next) {
     char *key = grpc_slice_to_c_string(GRPC_MDKEY(md->md));
     char *value = grpc_slice_to_c_string(GRPC_MDVALUE(md->md));
@@ -1464,6 +1489,21 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt;
   grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs;
 
+  if (!t->is_client) {
+    if (op->send_initial_metadata) {
+      gpr_timespec deadline =
+          op->payload->send_initial_metadata.send_initial_metadata->deadline;
+      GPR_ASSERT(0 ==
+                 gpr_time_cmp(gpr_inf_future(deadline.clock_type), deadline));
+    }
+    if (op->send_trailing_metadata) {
+      gpr_timespec deadline =
+          op->payload->send_trailing_metadata.send_trailing_metadata->deadline;
+      GPR_ASSERT(0 ==
+                 gpr_time_cmp(gpr_inf_future(deadline.clock_type), deadline));
+    }
+  }
+
   if (GRPC_TRACER_ON(grpc_http_trace)) {
     char *str = grpc_transport_stream_op_batch_string(op);
     gpr_log(GPR_DEBUG, "perform_stream_op[s=%p]: %s", s, str);
@@ -2129,8 +2169,8 @@ static void update_bdp(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
 
 static void update_frame(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                          double bw_dbl, double bdp_dbl) {
-  int32_t bdp = GPR_CLAMP((int32_t)bdp_dbl, 128, INT32_MAX);
-  int32_t target = GPR_MAX((int32_t)bw_dbl / 1000, bdp);
+  int32_t bdp = (int32_t)GPR_CLAMP(bdp_dbl, 128.0, INT32_MAX);
+  int32_t target = (int32_t)GPR_MAX(bw_dbl / 1000, bdp);
   // frame size is bounded [2^14,2^24-1]
   int32_t frame_size = GPR_CLAMP(target, 16384, 16777215);
   int64_t delta = (int64_t)frame_size -
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 9fa72ddbdf..4563b78e75 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -33,6 +33,7 @@
 #include "src/core/ext/transport/chttp2/transport/hpack_parser.h"
 #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 #include "src/core/ext/transport/chttp2/transport/stream_map.h"
+#include "src/core/lib/compression/stream_compression.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/timer.h"
@@ -67,6 +68,11 @@ typedef enum {
 } grpc_chttp2_ping_type;
 
 typedef enum {
+  GRPC_CHTTP2_OPTIMIZE_FOR_LATENCY,
+  GRPC_CHTTP2_OPTIMIZE_FOR_THROUGHPUT,
+} grpc_chttp2_optimization_target;
+
+typedef enum {
   GRPC_CHTTP2_PCL_INITIATE = 0,
   GRPC_CHTTP2_PCL_NEXT,
   GRPC_CHTTP2_PCL_INFLIGHT,
@@ -229,6 +235,8 @@ struct grpc_chttp2_transport {
   /** should we probe bdp? */
   bool enable_bdp_probe;
 
+  grpc_chttp2_optimization_target opt_target;
+
   /** various lists of streams */
   grpc_chttp2_stream_list lists[STREAM_LIST_COUNT];
 
diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c
index 3c8b470b4f..9d46cfa22e 100644
--- a/src/core/ext/transport/chttp2/transport/parsing.c
+++ b/src/core/ext/transport/chttp2/transport/parsing.c
@@ -657,6 +657,10 @@ static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx,
           "ignoring grpc_chttp2_stream with non-client generated index %d",
           t->incoming_stream_id));
       return init_skip_frame_parser(exec_ctx, t, 1);
+    } else if (grpc_chttp2_stream_map_size(&t->stream_map) >=
+               t->settings[GRPC_ACKED_SETTINGS]
+                          [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS]) {
+      return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Max stream count exceeded");
     }
     t->last_new_stream_id = t->incoming_stream_id;
     s = t->incoming_stream =
diff --git a/src/core/ext/transport/inproc/inproc_plugin.c b/src/core/ext/transport/inproc/inproc_plugin.c
new file mode 100644
index 0000000000..6a796a0b19
--- /dev/null
+++ b/src/core/ext/transport/inproc/inproc_plugin.c
@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/ext/transport/inproc/inproc_transport.h"
+#include "src/core/lib/debug/trace.h"
+
+grpc_tracer_flag grpc_inproc_trace = GRPC_TRACER_INITIALIZER(false, "inproc");
+
+void grpc_inproc_plugin_init(void) {
+  grpc_register_tracer(&grpc_inproc_trace);
+  grpc_inproc_transport_init();
+}
+
+void grpc_inproc_plugin_shutdown(void) { grpc_inproc_transport_shutdown(); }
diff --git a/src/core/ext/transport/inproc/inproc_transport.c b/src/core/ext/transport/inproc/inproc_transport.c
new file mode 100644
index 0000000000..14498021eb
--- /dev/null
+++ b/src/core/ext/transport/inproc/inproc_transport.c
@@ -0,0 +1,1280 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/ext/transport/inproc/inproc_transport.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+#include <string.h>
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/channel_stack_type.h"
+#include "src/core/lib/surface/server.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/error_utils.h"
+#include "src/core/lib/transport/transport_impl.h"
+
+#define INPROC_LOG(...)                                          \
+  do {                                                           \
+    if (GRPC_TRACER_ON(grpc_inproc_trace)) gpr_log(__VA_ARGS__); \
+  } while (0)
+
+static const grpc_transport_vtable inproc_vtable;
+static grpc_slice g_empty_slice;
+static grpc_slice g_fake_path_key;
+static grpc_slice g_fake_path_value;
+static grpc_slice g_fake_auth_key;
+static grpc_slice g_fake_auth_value;
+
+typedef struct {
+  gpr_mu mu;
+  gpr_refcount refs;
+} shared_mu;
+
+typedef struct inproc_transport {
+  grpc_transport base;
+  shared_mu *mu;
+  gpr_refcount refs;
+  bool is_client;
+  grpc_connectivity_state_tracker connectivity;
+  void (*accept_stream_cb)(grpc_exec_ctx *exec_ctx, void *user_data,
+                           grpc_transport *transport, const void *server_data);
+  void *accept_stream_data;
+  bool is_closed;
+  struct inproc_transport *other_side;
+  struct inproc_stream *stream_list;
+} inproc_transport;
+
+typedef struct sb_list_entry {
+  grpc_slice_buffer sb;
+  struct sb_list_entry *next;
+} sb_list_entry;
+
+// Specialize grpc_byte_stream for our use case
+typedef struct {
+  grpc_byte_stream base;
+  sb_list_entry *le;
+} inproc_slice_byte_stream;
+
+typedef struct {
+  // TODO (vjpai): Add some inlined elements to avoid alloc in simple cases
+  sb_list_entry *head;
+  sb_list_entry *tail;
+} slice_buffer_list;
+
+static void slice_buffer_list_init(slice_buffer_list *l) {
+  l->head = NULL;
+  l->tail = NULL;
+}
+
+static void sb_list_entry_destroy(grpc_exec_ctx *exec_ctx, sb_list_entry *le) {
+  grpc_slice_buffer_destroy_internal(exec_ctx, &le->sb);
+  gpr_free(le);
+}
+
+static void slice_buffer_list_destroy(grpc_exec_ctx *exec_ctx,
+                                      slice_buffer_list *l) {
+  sb_list_entry *curr = l->head;
+  while (curr != NULL) {
+    sb_list_entry *le = curr;
+    curr = curr->next;
+    sb_list_entry_destroy(exec_ctx, le);
+  }
+  l->head = NULL;
+  l->tail = NULL;
+}
+
+static bool slice_buffer_list_empty(slice_buffer_list *l) {
+  return l->head == NULL;
+}
+
+static void slice_buffer_list_append_entry(slice_buffer_list *l,
+                                           sb_list_entry *next) {
+  next->next = NULL;
+  if (l->tail) {
+    l->tail->next = next;
+    l->tail = next;
+  } else {
+    l->head = next;
+    l->tail = next;
+  }
+}
+
+static grpc_slice_buffer *slice_buffer_list_append(slice_buffer_list *l) {
+  sb_list_entry *next = gpr_malloc(sizeof(*next));
+  grpc_slice_buffer_init(&next->sb);
+  slice_buffer_list_append_entry(l, next);
+  return &next->sb;
+}
+
+static sb_list_entry *slice_buffer_list_pophead(slice_buffer_list *l) {
+  sb_list_entry *ret = l->head;
+  l->head = l->head->next;
+  if (l->head == NULL) {
+    l->tail = NULL;
+  }
+  return ret;
+}
+
+typedef struct inproc_stream {
+  inproc_transport *t;
+  grpc_metadata_batch to_read_initial_md;
+  uint32_t to_read_initial_md_flags;
+  bool to_read_initial_md_filled;
+  slice_buffer_list to_read_message;
+  grpc_metadata_batch to_read_trailing_md;
+  bool to_read_trailing_md_filled;
+  bool reads_needed;
+  bool read_closure_scheduled;
+  grpc_closure read_closure;
+  // Write buffer used only during gap at init time when client-side
+  // stream is set up but server side stream is not yet set up
+  grpc_metadata_batch write_buffer_initial_md;
+  bool write_buffer_initial_md_filled;
+  uint32_t write_buffer_initial_md_flags;
+  gpr_timespec write_buffer_deadline;
+  slice_buffer_list write_buffer_message;
+  grpc_metadata_batch write_buffer_trailing_md;
+  bool write_buffer_trailing_md_filled;
+  grpc_error *write_buffer_cancel_error;
+
+  struct inproc_stream *other_side;
+  bool other_side_closed;               // won't talk anymore
+  bool write_buffer_other_side_closed;  // on hold
+  grpc_stream_refcount *refs;
+  grpc_closure *closure_at_destroy;
+
+  gpr_arena *arena;
+
+  grpc_transport_stream_op_batch *recv_initial_md_op;
+  grpc_transport_stream_op_batch *recv_message_op;
+  grpc_transport_stream_op_batch *recv_trailing_md_op;
+
+  inproc_slice_byte_stream recv_message_stream;
+
+  bool initial_md_sent;
+  bool trailing_md_sent;
+  bool initial_md_recvd;
+  bool trailing_md_recvd;
+
+  bool closed;
+
+  grpc_error *cancel_self_error;
+  grpc_error *cancel_other_error;
+
+  gpr_timespec deadline;
+
+  bool listed;
+  struct inproc_stream *stream_list_prev;
+  struct inproc_stream *stream_list_next;
+} inproc_stream;
+
+static bool inproc_slice_byte_stream_next(grpc_exec_ctx *exec_ctx,
+                                          grpc_byte_stream *bs, size_t max,
+                                          grpc_closure *on_complete) {
+  // Because inproc transport always provides the entire message atomically,
+  // the byte stream always has data available when this function is called.
+  // Thus, this function always returns true (unlike other transports) and
+  // there is never any need to schedule a closure
+  return true;
+}
+
+static grpc_error *inproc_slice_byte_stream_pull(grpc_exec_ctx *exec_ctx,
+                                                 grpc_byte_stream *bs,
+                                                 grpc_slice *slice) {
+  inproc_slice_byte_stream *stream = (inproc_slice_byte_stream *)bs;
+  *slice = grpc_slice_buffer_take_first(&stream->le->sb);
+  return GRPC_ERROR_NONE;
+}
+
+static void inproc_slice_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
+                                             grpc_byte_stream *bs) {
+  inproc_slice_byte_stream *stream = (inproc_slice_byte_stream *)bs;
+  sb_list_entry_destroy(exec_ctx, stream->le);
+}
+
+void inproc_slice_byte_stream_init(inproc_slice_byte_stream *s,
+                                   sb_list_entry *le) {
+  s->base.length = (uint32_t)le->sb.length;
+  s->base.flags = 0;
+  s->base.next = inproc_slice_byte_stream_next;
+  s->base.pull = inproc_slice_byte_stream_pull;
+  s->base.destroy = inproc_slice_byte_stream_destroy;
+  s->le = le;
+}
+
+static void ref_transport(inproc_transport *t) {
+  INPROC_LOG(GPR_DEBUG, "ref_transport %p", t);
+  gpr_ref(&t->refs);
+}
+
+static void really_destroy_transport(grpc_exec_ctx *exec_ctx,
+                                     inproc_transport *t) {
+  INPROC_LOG(GPR_DEBUG, "really_destroy_transport %p", t);
+  grpc_connectivity_state_destroy(exec_ctx, &t->connectivity);
+  if (gpr_unref(&t->mu->refs)) {
+    gpr_free(t->mu);
+  }
+  gpr_free(t);
+}
+
+static void unref_transport(grpc_exec_ctx *exec_ctx, inproc_transport *t) {
+  INPROC_LOG(GPR_DEBUG, "unref_transport %p", t);
+  if (gpr_unref(&t->refs)) {
+    really_destroy_transport(exec_ctx, t);
+  }
+}
+
+#ifndef NDEBUG
+#define STREAM_REF(refs, reason) grpc_stream_ref(refs, reason)
+#define STREAM_UNREF(e, refs, reason) grpc_stream_unref(e, refs, reason)
+#else
+#define STREAM_REF(refs, reason) grpc_stream_ref(refs)
+#define STREAM_UNREF(e, refs, reason) grpc_stream_unref(e, refs)
+#endif
+
+static void ref_stream(inproc_stream *s, const char *reason) {
+  INPROC_LOG(GPR_DEBUG, "ref_stream %p %s", s, reason);
+  STREAM_REF(s->refs, reason);
+}
+
+static void unref_stream(grpc_exec_ctx *exec_ctx, inproc_stream *s,
+                         const char *reason) {
+  INPROC_LOG(GPR_DEBUG, "unref_stream %p %s", s, reason);
+  STREAM_UNREF(exec_ctx, s->refs, reason);
+}
+
+static void really_destroy_stream(grpc_exec_ctx *exec_ctx, inproc_stream *s) {
+  INPROC_LOG(GPR_DEBUG, "really_destroy_stream %p", s);
+
+  slice_buffer_list_destroy(exec_ctx, &s->to_read_message);
+  slice_buffer_list_destroy(exec_ctx, &s->write_buffer_message);
+  GRPC_ERROR_UNREF(s->write_buffer_cancel_error);
+  GRPC_ERROR_UNREF(s->cancel_self_error);
+  GRPC_ERROR_UNREF(s->cancel_other_error);
+
+  unref_transport(exec_ctx, s->t);
+
+  if (s->closure_at_destroy) {
+    GRPC_CLOSURE_SCHED(exec_ctx, s->closure_at_destroy, GRPC_ERROR_NONE);
+  }
+}
+
+static void read_state_machine(grpc_exec_ctx *exec_ctx, void *arg,
+                               grpc_error *error);
+
+static void log_metadata(const grpc_metadata_batch *md_batch, bool is_client,
+                         bool is_initial) {
+  for (grpc_linked_mdelem *md = md_batch->list.head; md != NULL;
+       md = md->next) {
+    char *key = grpc_slice_to_c_string(GRPC_MDKEY(md->md));
+    char *value = grpc_slice_to_c_string(GRPC_MDVALUE(md->md));
+    gpr_log(GPR_INFO, "INPROC:%s:%s: %s: %s", is_initial ? "HDR" : "TRL",
+            is_client ? "CLI" : "SVR", key, value);
+    gpr_free(key);
+    gpr_free(value);
+  }
+}
+
+static grpc_error *fill_in_metadata(grpc_exec_ctx *exec_ctx, inproc_stream *s,
+                                    const grpc_metadata_batch *metadata,
+                                    uint32_t flags, grpc_metadata_batch *out_md,
+                                    uint32_t *outflags, bool *markfilled) {
+  if (GRPC_TRACER_ON(grpc_inproc_trace)) {
+    log_metadata(metadata, s->t->is_client, outflags != NULL);
+  }
+
+  if (outflags != NULL) {
+    *outflags = flags;
+  }
+  if (markfilled != NULL) {
+    *markfilled = true;
+  }
+  grpc_error *error = GRPC_ERROR_NONE;
+  for (grpc_linked_mdelem *elem = metadata->list.head;
+       (elem != NULL) && (error == GRPC_ERROR_NONE); elem = elem->next) {
+    grpc_linked_mdelem *nelem = gpr_arena_alloc(s->arena, sizeof(*nelem));
+    nelem->md = grpc_mdelem_from_slices(
+        exec_ctx, grpc_slice_intern(GRPC_MDKEY(elem->md)),
+        grpc_slice_intern(GRPC_MDVALUE(elem->md)));
+
+    error = grpc_metadata_batch_link_tail(exec_ctx, out_md, nelem);
+  }
+  return error;
+}
+
+static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                       grpc_stream *gs, grpc_stream_refcount *refcount,
+                       const void *server_data, gpr_arena *arena) {
+  INPROC_LOG(GPR_DEBUG, "init_stream %p %p %p", gt, gs, server_data);
+  inproc_transport *t = (inproc_transport *)gt;
+  inproc_stream *s = (inproc_stream *)gs;
+  s->arena = arena;
+
+  s->refs = refcount;
+  // Ref this stream right now
+  ref_stream(s, "inproc_init_stream:init");
+
+  grpc_metadata_batch_init(&s->to_read_initial_md);
+  s->to_read_initial_md_flags = 0;
+  s->to_read_initial_md_filled = false;
+  grpc_metadata_batch_init(&s->to_read_trailing_md);
+  s->to_read_trailing_md_filled = false;
+  grpc_metadata_batch_init(&s->write_buffer_initial_md);
+  s->write_buffer_initial_md_flags = 0;
+  s->write_buffer_initial_md_filled = false;
+  grpc_metadata_batch_init(&s->write_buffer_trailing_md);
+  s->write_buffer_trailing_md_filled = false;
+  slice_buffer_list_init(&s->to_read_message);
+  slice_buffer_list_init(&s->write_buffer_message);
+  s->reads_needed = false;
+  s->read_closure_scheduled = false;
+  GRPC_CLOSURE_INIT(&s->read_closure, read_state_machine, s,
+                    grpc_schedule_on_exec_ctx);
+  s->t = t;
+  s->closure_at_destroy = NULL;
+  s->other_side_closed = false;
+
+  s->initial_md_sent = s->trailing_md_sent = s->initial_md_recvd =
+      s->trailing_md_recvd = false;
+
+  s->closed = false;
+
+  s->cancel_self_error = GRPC_ERROR_NONE;
+  s->cancel_other_error = GRPC_ERROR_NONE;
+  s->write_buffer_cancel_error = GRPC_ERROR_NONE;
+  s->deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  s->write_buffer_deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+
+  s->stream_list_prev = NULL;
+  gpr_mu_lock(&t->mu->mu);
+  s->listed = true;
+  ref_stream(s, "inproc_init_stream:list");
+  s->stream_list_next = t->stream_list;
+  if (t->stream_list) {
+    t->stream_list->stream_list_prev = s;
+  }
+  t->stream_list = s;
+  gpr_mu_unlock(&t->mu->mu);
+
+  if (!server_data) {
+    ref_transport(t);
+    inproc_transport *st = t->other_side;
+    ref_transport(st);
+    s->other_side = NULL;  // will get filled in soon
+    // Pass the client-side stream address to the server-side for a ref
+    ref_stream(s, "inproc_init_stream:clt");  // ref it now on behalf of server
+                                              // side to avoid destruction
+    INPROC_LOG(GPR_DEBUG, "calling accept stream cb %p %p",
+               st->accept_stream_cb, st->accept_stream_data);
+    (*st->accept_stream_cb)(exec_ctx, st->accept_stream_data, &st->base,
+                            (void *)s);
+  } else {
+    // This is the server-side and is being called through accept_stream_cb
+    inproc_stream *cs = (inproc_stream *)server_data;
+    s->other_side = cs;
+    // Ref the server-side stream on behalf of the client now
+    ref_stream(s, "inproc_init_stream:srv");
+
+    // Now we are about to affect the other side, so lock the transport
+    // to make sure that it doesn't get destroyed
+    gpr_mu_lock(&s->t->mu->mu);
+    cs->other_side = s;
+    // Now transfer from the other side's write_buffer if any to the to_read
+    // buffer
+    if (cs->write_buffer_initial_md_filled) {
+      fill_in_metadata(exec_ctx, s, &cs->write_buffer_initial_md,
+                       cs->write_buffer_initial_md_flags,
+                       &s->to_read_initial_md, &s->to_read_initial_md_flags,
+                       &s->to_read_initial_md_filled);
+      s->deadline = gpr_time_min(s->deadline, cs->write_buffer_deadline);
+      grpc_metadata_batch_clear(exec_ctx, &cs->write_buffer_initial_md);
+      cs->write_buffer_initial_md_filled = false;
+    }
+    while (!slice_buffer_list_empty(&cs->write_buffer_message)) {
+      slice_buffer_list_append_entry(
+          &s->to_read_message,
+          slice_buffer_list_pophead(&cs->write_buffer_message));
+    }
+    if (cs->write_buffer_trailing_md_filled) {
+      fill_in_metadata(exec_ctx, s, &cs->write_buffer_trailing_md, 0,
+                       &s->to_read_trailing_md, NULL,
+                       &s->to_read_trailing_md_filled);
+      grpc_metadata_batch_clear(exec_ctx, &cs->write_buffer_trailing_md);
+      cs->write_buffer_trailing_md_filled = false;
+    }
+    if (cs->write_buffer_cancel_error != GRPC_ERROR_NONE) {
+      s->cancel_other_error = cs->write_buffer_cancel_error;
+      cs->write_buffer_cancel_error = GRPC_ERROR_NONE;
+    }
+
+    gpr_mu_unlock(&s->t->mu->mu);
+  }
+  return 0;  // return value is not important
+}
+
+static void close_stream_locked(grpc_exec_ctx *exec_ctx, inproc_stream *s) {
+  if (!s->closed) {
+    // Release the metadata that we would have written out
+    grpc_metadata_batch_destroy(exec_ctx, &s->write_buffer_initial_md);
+    grpc_metadata_batch_destroy(exec_ctx, &s->write_buffer_trailing_md);
+
+    if (s->listed) {
+      inproc_stream *p = s->stream_list_prev;
+      inproc_stream *n = s->stream_list_next;
+      if (p != NULL) {
+        p->stream_list_next = n;
+      } else {
+        s->t->stream_list = n;
+      }
+      if (n != NULL) {
+        n->stream_list_prev = p;
+      }
+      s->listed = false;
+      unref_stream(exec_ctx, s, "close_stream:list");
+    }
+    s->closed = true;
+    unref_stream(exec_ctx, s, "close_stream:closing");
+  }
+}
+
+// This function means that we are done talking/listening to the other side
+static void close_other_side_locked(grpc_exec_ctx *exec_ctx, inproc_stream *s,
+                                    const char *reason) {
+  if (s->other_side != NULL) {
+    // First release the metadata that came from the other side's arena
+    grpc_metadata_batch_destroy(exec_ctx, &s->to_read_initial_md);
+    grpc_metadata_batch_destroy(exec_ctx, &s->to_read_trailing_md);
+
+    unref_stream(exec_ctx, s->other_side, reason);
+    s->other_side_closed = true;
+    s->other_side = NULL;
+  } else if (!s->other_side_closed) {
+    s->write_buffer_other_side_closed = true;
+  }
+}
+
+static void fail_helper_locked(grpc_exec_ctx *exec_ctx, inproc_stream *s,
+                               grpc_error *error) {
+  INPROC_LOG(GPR_DEBUG, "read_state_machine %p fail_helper", s);
+  // If we're failing this side, we need to make sure that
+  // we also send or have already sent trailing metadata
+  if (!s->trailing_md_sent) {
+    // Send trailing md to the other side indicating cancellation
+    s->trailing_md_sent = true;
+
+    grpc_metadata_batch fake_md;
+    grpc_metadata_batch_init(&fake_md);
+
+    inproc_stream *other = s->other_side;
+    grpc_metadata_batch *dest = (other == NULL) ? &s->write_buffer_trailing_md
+                                                : &other->to_read_trailing_md;
+    bool *destfilled = (other == NULL) ? &s->write_buffer_trailing_md_filled
+                                       : &other->to_read_trailing_md_filled;
+    fill_in_metadata(exec_ctx, s, &fake_md, 0, dest, NULL, destfilled);
+    grpc_metadata_batch_destroy(exec_ctx, &fake_md);
+
+    if (other != NULL) {
+      if (other->cancel_other_error == GRPC_ERROR_NONE) {
+        other->cancel_other_error = GRPC_ERROR_REF(error);
+      }
+      if (other->reads_needed) {
+        if (!other->read_closure_scheduled) {
+          GRPC_CLOSURE_SCHED(exec_ctx, &other->read_closure,
+                             GRPC_ERROR_REF(error));
+          other->read_closure_scheduled = true;
+        }
+        other->reads_needed = false;
+      }
+    } else if (s->write_buffer_cancel_error == GRPC_ERROR_NONE) {
+      s->write_buffer_cancel_error = GRPC_ERROR_REF(error);
+    }
+  }
+  if (s->recv_initial_md_op) {
+    grpc_error *err;
+    if (!s->t->is_client) {
+      // If this is a server, provide initial metadata with a path and authority
+      // since it expects that as well as no error yet
+      grpc_metadata_batch fake_md;
+      grpc_metadata_batch_init(&fake_md);
+      grpc_linked_mdelem *path_md = gpr_arena_alloc(s->arena, sizeof(*path_md));
+      path_md->md =
+          grpc_mdelem_from_slices(exec_ctx, g_fake_path_key, g_fake_path_value);
+      GPR_ASSERT(grpc_metadata_batch_link_tail(exec_ctx, &fake_md, path_md) ==
+                 GRPC_ERROR_NONE);
+      grpc_linked_mdelem *auth_md = gpr_arena_alloc(s->arena, sizeof(*auth_md));
+      auth_md->md =
+          grpc_mdelem_from_slices(exec_ctx, g_fake_auth_key, g_fake_auth_value);
+      GPR_ASSERT(grpc_metadata_batch_link_tail(exec_ctx, &fake_md, auth_md) ==
+                 GRPC_ERROR_NONE);
+
+      fill_in_metadata(
+          exec_ctx, s, &fake_md, 0,
+          s->recv_initial_md_op->payload->recv_initial_metadata
+              .recv_initial_metadata,
+          s->recv_initial_md_op->payload->recv_initial_metadata.recv_flags,
+          NULL);
+      grpc_metadata_batch_destroy(exec_ctx, &fake_md);
+      err = GRPC_ERROR_NONE;
+    } else {
+      err = GRPC_ERROR_REF(error);
+    }
+    INPROC_LOG(GPR_DEBUG,
+               "fail_helper %p scheduling initial-metadata-ready %p %p", s,
+               error, err);
+    GRPC_CLOSURE_SCHED(exec_ctx,
+                       s->recv_initial_md_op->payload->recv_initial_metadata
+                           .recv_initial_metadata_ready,
+                       err);
+    // Last use of err so no need to REF and then UNREF it
+
+    if ((s->recv_initial_md_op != s->recv_message_op) &&
+        (s->recv_initial_md_op != s->recv_trailing_md_op)) {
+      INPROC_LOG(GPR_DEBUG,
+                 "fail_helper %p scheduling initial-metadata-on-complete %p",
+                 error, s);
+      GRPC_CLOSURE_SCHED(exec_ctx, s->recv_initial_md_op->on_complete,
+                         GRPC_ERROR_REF(error));
+    }
+    s->recv_initial_md_op = NULL;
+  }
+  if (s->recv_message_op) {
+    INPROC_LOG(GPR_DEBUG, "fail_helper %p scheduling message-ready %p", s,
+               error);
+    GRPC_CLOSURE_SCHED(
+        exec_ctx, s->recv_message_op->payload->recv_message.recv_message_ready,
+        GRPC_ERROR_REF(error));
+    if (s->recv_message_op != s->recv_trailing_md_op) {
+      INPROC_LOG(GPR_DEBUG, "fail_helper %p scheduling message-on-complete %p",
+                 s, error);
+      GRPC_CLOSURE_SCHED(exec_ctx, s->recv_message_op->on_complete,
+                         GRPC_ERROR_REF(error));
+    }
+    s->recv_message_op = NULL;
+  }
+  if (s->recv_trailing_md_op) {
+    INPROC_LOG(GPR_DEBUG,
+               "fail_helper %p scheduling trailing-md-on-complete %p", s,
+               error);
+    GRPC_CLOSURE_SCHED(exec_ctx, s->recv_trailing_md_op->on_complete,
+                       GRPC_ERROR_REF(error));
+    s->recv_trailing_md_op = NULL;
+  }
+  close_other_side_locked(exec_ctx, s, "fail_helper:other_side");
+  close_stream_locked(exec_ctx, s);
+
+  GRPC_ERROR_UNREF(error);
+}
+
+static void read_state_machine(grpc_exec_ctx *exec_ctx, void *arg,
+                               grpc_error *error) {
+  // This function gets called when we have contents in the unprocessed reads
+  // Get what we want based on our ops wanted
+  // Schedule our appropriate closures
+  // and then return to reads_needed state if still needed
+
+  // Since this is a closure directly invoked by the combiner, it should not
+  // unref the error parameter explicitly; the combiner will do that implicitly
+  grpc_error *new_err = GRPC_ERROR_NONE;
+
+  bool needs_close = false;
+
+  INPROC_LOG(GPR_DEBUG, "read_state_machine %p", arg);
+  inproc_stream *s = (inproc_stream *)arg;
+  gpr_mu *mu = &s->t->mu->mu;  // keep aside in case s gets closed
+  gpr_mu_lock(mu);
+  s->read_closure_scheduled = false;
+  // cancellation takes precedence
+  if (s->cancel_self_error != GRPC_ERROR_NONE) {
+    fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(s->cancel_self_error));
+    goto done;
+  } else if (s->cancel_other_error != GRPC_ERROR_NONE) {
+    fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(s->cancel_other_error));
+    goto done;
+  } else if (error != GRPC_ERROR_NONE) {
+    fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(error));
+    goto done;
+  }
+
+  if (s->recv_initial_md_op) {
+    if (!s->to_read_initial_md_filled) {
+      // We entered the state machine on some other kind of read even though
+      // we still haven't satisfied initial md . That's an error.
+      new_err =
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unexpected frame sequencing");
+      INPROC_LOG(GPR_DEBUG,
+                 "read_state_machine %p scheduling on_complete errors for no "
+                 "initial md %p",
+                 s, new_err);
+      fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(new_err));
+      goto done;
+    } else if (s->initial_md_recvd) {
+      new_err =
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Already recvd initial md");
+      INPROC_LOG(
+          GPR_DEBUG,
+          "read_state_machine %p scheduling on_complete errors for already "
+          "recvd initial md %p",
+          s, new_err);
+      fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(new_err));
+      goto done;
+    }
+
+    s->initial_md_recvd = true;
+    new_err = fill_in_metadata(
+        exec_ctx, s, &s->to_read_initial_md, s->to_read_initial_md_flags,
+        s->recv_initial_md_op->payload->recv_initial_metadata
+            .recv_initial_metadata,
+        s->recv_initial_md_op->payload->recv_initial_metadata.recv_flags, NULL);
+    s->recv_initial_md_op->payload->recv_initial_metadata.recv_initial_metadata
+        ->deadline = s->deadline;
+    grpc_metadata_batch_clear(exec_ctx, &s->to_read_initial_md);
+    s->to_read_initial_md_filled = false;
+    INPROC_LOG(GPR_DEBUG,
+               "read_state_machine %p scheduling initial-metadata-ready %p", s,
+               new_err);
+    GRPC_CLOSURE_SCHED(exec_ctx,
+                       s->recv_initial_md_op->payload->recv_initial_metadata
+                           .recv_initial_metadata_ready,
+                       GRPC_ERROR_REF(new_err));
+    if ((s->recv_initial_md_op != s->recv_message_op) &&
+        (s->recv_initial_md_op != s->recv_trailing_md_op)) {
+      INPROC_LOG(
+          GPR_DEBUG,
+          "read_state_machine %p scheduling initial-metadata-on-complete %p", s,
+          new_err);
+      GRPC_CLOSURE_SCHED(exec_ctx, s->recv_initial_md_op->on_complete,
+                         GRPC_ERROR_REF(new_err));
+    }
+    s->recv_initial_md_op = NULL;
+
+    if (new_err != GRPC_ERROR_NONE) {
+      INPROC_LOG(GPR_DEBUG,
+                 "read_state_machine %p scheduling on_complete errors2 %p", s,
+                 new_err);
+      fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(new_err));
+      goto done;
+    }
+  }
+  if (s->to_read_initial_md_filled) {
+    new_err = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unexpected recv frame");
+    fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(new_err));
+    goto done;
+  }
+  if (!slice_buffer_list_empty(&s->to_read_message) && s->recv_message_op) {
+    inproc_slice_byte_stream_init(
+        &s->recv_message_stream,
+        slice_buffer_list_pophead(&s->to_read_message));
+    *s->recv_message_op->payload->recv_message.recv_message =
+        &s->recv_message_stream.base;
+    INPROC_LOG(GPR_DEBUG, "read_state_machine %p scheduling message-ready", s);
+    GRPC_CLOSURE_SCHED(
+        exec_ctx, s->recv_message_op->payload->recv_message.recv_message_ready,
+        GRPC_ERROR_NONE);
+    if (s->recv_message_op != s->recv_trailing_md_op) {
+      INPROC_LOG(GPR_DEBUG,
+                 "read_state_machine %p scheduling message-on-complete %p", s,
+                 new_err);
+      GRPC_CLOSURE_SCHED(exec_ctx, s->recv_message_op->on_complete,
+                         GRPC_ERROR_REF(new_err));
+    }
+    s->recv_message_op = NULL;
+  }
+  if (s->to_read_trailing_md_filled) {
+    if (s->trailing_md_recvd) {
+      new_err =
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Already recvd trailing md");
+      INPROC_LOG(
+          GPR_DEBUG,
+          "read_state_machine %p scheduling on_complete errors for already "
+          "recvd trailing md %p",
+          s, new_err);
+      fail_helper_locked(exec_ctx, s, GRPC_ERROR_REF(new_err));
+      goto done;
+    }
+    if (s->recv_message_op != NULL) {
+      // This message needs to be wrapped up because it will never be
+      // satisfied
+      INPROC_LOG(GPR_DEBUG, "read_state_machine %p scheduling message-ready",
+                 s);
+      GRPC_CLOSURE_SCHED(
+          exec_ctx,
+          s->recv_message_op->payload->recv_message.recv_message_ready,
+          GRPC_ERROR_NONE);
+      if (s->recv_message_op != s->recv_trailing_md_op) {
+        INPROC_LOG(GPR_DEBUG,
+                   "read_state_machine %p scheduling message-on-complete %p", s,
+                   new_err);
+        GRPC_CLOSURE_SCHED(exec_ctx, s->recv_message_op->on_complete,
+                           GRPC_ERROR_REF(new_err));
+      }
+      s->recv_message_op = NULL;
+    }
+    if (s->recv_trailing_md_op != NULL) {
+      // We wanted trailing metadata and we got it
+      s->trailing_md_recvd = true;
+      new_err =
+          fill_in_metadata(exec_ctx, s, &s->to_read_trailing_md, 0,
+                           s->recv_trailing_md_op->payload
+                               ->recv_trailing_metadata.recv_trailing_metadata,
+                           NULL, NULL);
+      grpc_metadata_batch_clear(exec_ctx, &s->to_read_trailing_md);
+      s->to_read_trailing_md_filled = false;
+
+      // We should schedule the recv_trailing_md_op completion if
+      // 1. this stream is the client-side
+      // 2. this stream is the server-side AND has already sent its trailing md
+      //    (If the server hasn't already sent its trailing md, it doesn't have
+      //     a final status, so don't mark this op complete)
+      if (s->t->is_client || s->trailing_md_sent) {
+        INPROC_LOG(
+            GPR_DEBUG,
+            "read_state_machine %p scheduling trailing-md-on-complete %p", s,
+            new_err);
+        GRPC_CLOSURE_SCHED(exec_ctx, s->recv_trailing_md_op->on_complete,
+                           GRPC_ERROR_REF(new_err));
+        s->recv_trailing_md_op = NULL;
+        needs_close = true;
+      } else {
+        INPROC_LOG(GPR_DEBUG,
+                   "read_state_machine %p server needs to delay handling "
+                   "trailing-md-on-complete %p",
+                   s, new_err);
+      }
+    } else {
+      INPROC_LOG(
+          GPR_DEBUG,
+          "read_state_machine %p has trailing md but not yet waiting for it",
+          s);
+    }
+  }
+  if (s->trailing_md_recvd && s->recv_message_op) {
+    // No further message will come on this stream, so finish off the
+    // recv_message_op
+    INPROC_LOG(GPR_DEBUG, "read_state_machine %p scheduling message-ready", s);
+    GRPC_CLOSURE_SCHED(
+        exec_ctx, s->recv_message_op->payload->recv_message.recv_message_ready,
+        GRPC_ERROR_NONE);
+    if (s->recv_message_op != s->recv_trailing_md_op) {
+      INPROC_LOG(GPR_DEBUG,
+                 "read_state_machine %p scheduling message-on-complete %p", s,
+                 new_err);
+      GRPC_CLOSURE_SCHED(exec_ctx, s->recv_message_op->on_complete,
+                         GRPC_ERROR_REF(new_err));
+    }
+    s->recv_message_op = NULL;
+  }
+  if (s->recv_message_op || s->recv_trailing_md_op) {
+    // Didn't get the item we wanted so we still need to get
+    // rescheduled
+    INPROC_LOG(GPR_DEBUG, "read_state_machine %p still needs closure %p %p", s,
+               s->recv_message_op, s->recv_trailing_md_op);
+    s->reads_needed = true;
+  }
+done:
+  if (needs_close) {
+    close_other_side_locked(exec_ctx, s, "read_state_machine");
+    close_stream_locked(exec_ctx, s);
+  }
+  gpr_mu_unlock(mu);
+  GRPC_ERROR_UNREF(new_err);
+}
+
+static grpc_closure do_nothing_closure;
+
+static bool cancel_stream_locked(grpc_exec_ctx *exec_ctx, inproc_stream *s,
+                                 grpc_error *error) {
+  bool ret = false;  // was the cancel accepted
+  INPROC_LOG(GPR_DEBUG, "cancel_stream %p with %s", s,
+             grpc_error_string(error));
+  if (s->cancel_self_error == GRPC_ERROR_NONE) {
+    ret = true;
+    s->cancel_self_error = GRPC_ERROR_REF(error);
+    if (s->reads_needed) {
+      if (!s->read_closure_scheduled) {
+        GRPC_CLOSURE_SCHED(exec_ctx, &s->read_closure,
+                           GRPC_ERROR_REF(s->cancel_self_error));
+        s->read_closure_scheduled = true;
+      }
+      s->reads_needed = false;
+    }
+    // Send trailing md to the other side indicating cancellation, even if we
+    // already have
+    s->trailing_md_sent = true;
+
+    grpc_metadata_batch cancel_md;
+    grpc_metadata_batch_init(&cancel_md);
+
+    inproc_stream *other = s->other_side;
+    grpc_metadata_batch *dest = (other == NULL) ? &s->write_buffer_trailing_md
+                                                : &other->to_read_trailing_md;
+    bool *destfilled = (other == NULL) ? &s->write_buffer_trailing_md_filled
+                                       : &other->to_read_trailing_md_filled;
+    fill_in_metadata(exec_ctx, s, &cancel_md, 0, dest, NULL, destfilled);
+    grpc_metadata_batch_destroy(exec_ctx, &cancel_md);
+
+    if (other != NULL) {
+      if (other->cancel_other_error == GRPC_ERROR_NONE) {
+        other->cancel_other_error = GRPC_ERROR_REF(s->cancel_self_error);
+      }
+      if (other->reads_needed) {
+        if (!other->read_closure_scheduled) {
+          GRPC_CLOSURE_SCHED(exec_ctx, &other->read_closure,
+                             GRPC_ERROR_REF(other->cancel_other_error));
+          other->read_closure_scheduled = true;
+        }
+        other->reads_needed = false;
+      }
+    } else if (s->write_buffer_cancel_error == GRPC_ERROR_NONE) {
+      s->write_buffer_cancel_error = GRPC_ERROR_REF(s->cancel_self_error);
+    }
+
+    // if we are a server and already received trailing md but
+    // couldn't complete that because we hadn't yet sent out trailing
+    // md, now's the chance
+    if (!s->t->is_client && s->trailing_md_recvd && s->recv_trailing_md_op) {
+      INPROC_LOG(GPR_DEBUG,
+                 "cancel_stream %p scheduling trailing-md-on-complete %p", s,
+                 s->cancel_self_error);
+      GRPC_CLOSURE_SCHED(exec_ctx, s->recv_trailing_md_op->on_complete,
+                         GRPC_ERROR_REF(s->cancel_self_error));
+      s->recv_trailing_md_op = NULL;
+    }
+  }
+
+  close_other_side_locked(exec_ctx, s, "cancel_stream:other_side");
+  close_stream_locked(exec_ctx, s);
+
+  GRPC_ERROR_UNREF(error);
+  return ret;
+}
+
+static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                              grpc_stream *gs,
+                              grpc_transport_stream_op_batch *op) {
+  INPROC_LOG(GPR_DEBUG, "perform_stream_op %p %p %p", gt, gs, op);
+  inproc_stream *s = (inproc_stream *)gs;
+  gpr_mu *mu = &s->t->mu->mu;  // save aside in case s gets closed
+  gpr_mu_lock(mu);
+
+  if (GRPC_TRACER_ON(grpc_inproc_trace)) {
+    if (op->send_initial_metadata) {
+      log_metadata(op->payload->send_initial_metadata.send_initial_metadata,
+                   s->t->is_client, true);
+    }
+    if (op->send_trailing_metadata) {
+      log_metadata(op->payload->send_trailing_metadata.send_trailing_metadata,
+                   s->t->is_client, false);
+    }
+  }
+  grpc_error *error = GRPC_ERROR_NONE;
+  grpc_closure *on_complete = op->on_complete;
+  if (on_complete == NULL) {
+    on_complete = &do_nothing_closure;
+  }
+
+  if (op->cancel_stream) {
+    // Call cancel_stream_locked without ref'ing the cancel_error because
+    // this function is responsible to make sure that that field gets unref'ed
+    cancel_stream_locked(exec_ctx, s, op->payload->cancel_stream.cancel_error);
+    // this op can complete without an error
+  } else if (s->cancel_self_error != GRPC_ERROR_NONE) {
+    // already self-canceled so still give it an error
+    error = GRPC_ERROR_REF(s->cancel_self_error);
+  } else {
+    INPROC_LOG(GPR_DEBUG, "perform_stream_op %p%s%s%s%s%s%s", s,
+               op->send_initial_metadata ? " send_initial_metadata" : "",
+               op->send_message ? " send_message" : "",
+               op->send_trailing_metadata ? " send_trailing_metadata" : "",
+               op->recv_initial_metadata ? " recv_initial_metadata" : "",
+               op->recv_message ? " recv_message" : "",
+               op->recv_trailing_metadata ? " recv_trailing_metadata" : "");
+  }
+
+  bool needs_close = false;
+
+  if (error == GRPC_ERROR_NONE &&
+      (op->send_initial_metadata || op->send_message ||
+       op->send_trailing_metadata)) {
+    inproc_stream *other = s->other_side;
+    if (s->t->is_closed) {
+      error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Endpoint already shutdown");
+    }
+    if (error == GRPC_ERROR_NONE && op->send_initial_metadata) {
+      grpc_metadata_batch *dest = (other == NULL) ? &s->write_buffer_initial_md
+                                                  : &other->to_read_initial_md;
+      uint32_t *destflags = (other == NULL) ? &s->write_buffer_initial_md_flags
+                                            : &other->to_read_initial_md_flags;
+      bool *destfilled = (other == NULL) ? &s->write_buffer_initial_md_filled
+                                         : &other->to_read_initial_md_filled;
+      if (*destfilled || s->initial_md_sent) {
+        // The buffer is already in use; that's an error!
+        INPROC_LOG(GPR_DEBUG, "Extra initial metadata %p", s);
+        error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Extra initial metadata");
+      } else {
+        if (!other->closed) {
+          fill_in_metadata(
+              exec_ctx, s,
+              op->payload->send_initial_metadata.send_initial_metadata,
+              op->payload->send_initial_metadata.send_initial_metadata_flags,
+              dest, destflags, destfilled);
+        }
+        if (s->t->is_client) {
+          gpr_timespec *dl =
+              (other == NULL) ? &s->write_buffer_deadline : &other->deadline;
+          *dl = gpr_time_min(*dl, op->payload->send_initial_metadata
+                                      .send_initial_metadata->deadline);
+          s->initial_md_sent = true;
+        }
+      }
+    }
+    if (error == GRPC_ERROR_NONE && op->send_message) {
+      size_t remaining = op->payload->send_message.send_message->length;
+      grpc_slice_buffer *dest = slice_buffer_list_append(
+          (other == NULL) ? &s->write_buffer_message : &other->to_read_message);
+      do {
+        grpc_slice message_slice;
+        grpc_closure unused;
+        GPR_ASSERT(grpc_byte_stream_next(exec_ctx,
+                                         op->payload->send_message.send_message,
+                                         SIZE_MAX, &unused));
+        grpc_byte_stream_pull(exec_ctx, op->payload->send_message.send_message,
+                              &message_slice);
+        remaining -= GRPC_SLICE_LENGTH(message_slice);
+        grpc_slice_buffer_add(dest, message_slice);
+      } while (remaining != 0);
+    }
+    if (error == GRPC_ERROR_NONE && op->send_trailing_metadata) {
+      grpc_metadata_batch *dest = (other == NULL) ? &s->write_buffer_trailing_md
+                                                  : &other->to_read_trailing_md;
+      bool *destfilled = (other == NULL) ? &s->write_buffer_trailing_md_filled
+                                         : &other->to_read_trailing_md_filled;
+      if (*destfilled || s->trailing_md_sent) {
+        // The buffer is already in use; that's an error!
+        INPROC_LOG(GPR_DEBUG, "Extra trailing metadata %p", s);
+        error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Extra trailing metadata");
+      } else {
+        if (!other->closed) {
+          fill_in_metadata(
+              exec_ctx, s,
+              op->payload->send_trailing_metadata.send_trailing_metadata, 0,
+              dest, NULL, destfilled);
+        }
+        s->trailing_md_sent = true;
+        if (!s->t->is_client && s->trailing_md_recvd &&
+            s->recv_trailing_md_op) {
+          INPROC_LOG(GPR_DEBUG,
+                     "perform_stream_op %p scheduling trailing-md-on-complete",
+                     s);
+          GRPC_CLOSURE_SCHED(exec_ctx, s->recv_trailing_md_op->on_complete,
+                             GRPC_ERROR_NONE);
+          s->recv_trailing_md_op = NULL;
+          needs_close = true;
+        }
+      }
+    }
+    if (other != NULL && other->reads_needed) {
+      if (!other->read_closure_scheduled) {
+        GRPC_CLOSURE_SCHED(exec_ctx, &other->read_closure, error);
+        other->read_closure_scheduled = true;
+      }
+      other->reads_needed = false;
+    }
+  }
+  if (error == GRPC_ERROR_NONE &&
+      (op->recv_initial_metadata || op->recv_message ||
+       op->recv_trailing_metadata)) {
+    // If there are any reads, mark it so that the read closure will react to
+    // them
+    if (op->recv_initial_metadata) {
+      s->recv_initial_md_op = op;
+    }
+    if (op->recv_message) {
+      s->recv_message_op = op;
+    }
+    if (op->recv_trailing_metadata) {
+      s->recv_trailing_md_op = op;
+    }
+
+    // We want to initiate the closure if:
+    // 1. There is initial metadata and something ready to take that
+    // 2. There is a message and something ready to take it
+    // 3. There is trailing metadata, even if nothing specifically wants
+    //    that because that can shut down the message as well
+    if ((s->to_read_initial_md_filled && op->recv_initial_metadata) ||
+        ((!slice_buffer_list_empty(&s->to_read_message) ||
+          s->trailing_md_recvd) &&
+         op->recv_message) ||
+        (s->to_read_trailing_md_filled)) {
+      if (!s->read_closure_scheduled) {
+        GRPC_CLOSURE_SCHED(exec_ctx, &s->read_closure, GRPC_ERROR_NONE);
+        s->read_closure_scheduled = true;
+      }
+    } else {
+      s->reads_needed = true;
+    }
+  } else {
+    if (error != GRPC_ERROR_NONE) {
+      // Schedule op's read closures that we didn't push to read state machine
+      if (op->recv_initial_metadata) {
+        INPROC_LOG(
+            GPR_DEBUG,
+            "perform_stream_op error %p scheduling initial-metadata-ready %p",
+            s, error);
+        GRPC_CLOSURE_SCHED(
+            exec_ctx,
+            op->payload->recv_initial_metadata.recv_initial_metadata_ready,
+            GRPC_ERROR_REF(error));
+      }
+      if (op->recv_message) {
+        INPROC_LOG(
+            GPR_DEBUG,
+            "perform_stream_op error %p scheduling recv message-ready %p", s,
+            error);
+        GRPC_CLOSURE_SCHED(exec_ctx,
+                           op->payload->recv_message.recv_message_ready,
+                           GRPC_ERROR_REF(error));
+      }
+    }
+    INPROC_LOG(GPR_DEBUG, "perform_stream_op %p scheduling on_complete %p", s,
+               error);
+    GRPC_CLOSURE_SCHED(exec_ctx, on_complete, GRPC_ERROR_REF(error));
+  }
+  if (needs_close) {
+    close_other_side_locked(exec_ctx, s, "perform_stream_op:other_side");
+    close_stream_locked(exec_ctx, s);
+  }
+  gpr_mu_unlock(mu);
+  GRPC_ERROR_UNREF(error);
+}
+
+static void close_transport_locked(grpc_exec_ctx *exec_ctx,
+                                   inproc_transport *t) {
+  INPROC_LOG(GPR_DEBUG, "close_transport %p %d", t, t->is_closed);
+  grpc_connectivity_state_set(
+      exec_ctx, &t->connectivity, GRPC_CHANNEL_SHUTDOWN,
+      GRPC_ERROR_CREATE_FROM_STATIC_STRING("Closing transport."),
+      "close transport");
+  if (!t->is_closed) {
+    t->is_closed = true;
+    /* Also end all streams on this transport */
+    while (t->stream_list != NULL) {
+      // cancel_stream_locked also adjusts stream list
+      cancel_stream_locked(
+          exec_ctx, t->stream_list,
+          grpc_error_set_int(
+              GRPC_ERROR_CREATE_FROM_STATIC_STRING("Transport closed"),
+              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE));
+    }
+  }
+}
+
+static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                                 grpc_transport_op *op) {
+  inproc_transport *t = (inproc_transport *)gt;
+  INPROC_LOG(GPR_DEBUG, "perform_transport_op %p %p", t, op);
+  gpr_mu_lock(&t->mu->mu);
+  if (op->on_connectivity_state_change) {
+    grpc_connectivity_state_notify_on_state_change(
+        exec_ctx, &t->connectivity, op->connectivity_state,
+        op->on_connectivity_state_change);
+  }
+  if (op->set_accept_stream) {
+    t->accept_stream_cb = op->set_accept_stream_fn;
+    t->accept_stream_data = op->set_accept_stream_user_data;
+  }
+  if (op->on_consumed) {
+    GRPC_CLOSURE_SCHED(exec_ctx, op->on_consumed, GRPC_ERROR_NONE);
+  }
+
+  bool do_close = false;
+  if (op->goaway_error != GRPC_ERROR_NONE) {
+    do_close = true;
+    GRPC_ERROR_UNREF(op->goaway_error);
+  }
+  if (op->disconnect_with_error != GRPC_ERROR_NONE) {
+    do_close = true;
+    GRPC_ERROR_UNREF(op->disconnect_with_error);
+  }
+
+  if (do_close) {
+    close_transport_locked(exec_ctx, t);
+  }
+  gpr_mu_unlock(&t->mu->mu);
+}
+
+static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                           grpc_stream *gs,
+                           grpc_closure *then_schedule_closure) {
+  INPROC_LOG(GPR_DEBUG, "destroy_stream %p %p", gs, then_schedule_closure);
+  inproc_stream *s = (inproc_stream *)gs;
+  s->closure_at_destroy = then_schedule_closure;
+  really_destroy_stream(exec_ctx, s);
+}
+
+static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
+  inproc_transport *t = (inproc_transport *)gt;
+  INPROC_LOG(GPR_DEBUG, "destroy_transport %p", t);
+  gpr_mu_lock(&t->mu->mu);
+  close_transport_locked(exec_ctx, t);
+  gpr_mu_unlock(&t->mu->mu);
+  unref_transport(exec_ctx, t->other_side);
+  unref_transport(exec_ctx, t);
+}
+
+/*******************************************************************************
+ * Main inproc transport functions
+ */
+static void inproc_transports_create(grpc_exec_ctx *exec_ctx,
+                                     grpc_transport **server_transport,
+                                     const grpc_channel_args *server_args,
+                                     grpc_transport **client_transport,
+                                     const grpc_channel_args *client_args) {
+  INPROC_LOG(GPR_DEBUG, "inproc_transports_create");
+  inproc_transport *st = gpr_zalloc(sizeof(*st));
+  inproc_transport *ct = gpr_zalloc(sizeof(*ct));
+  // Share one lock between both sides since both sides get affected
+  st->mu = ct->mu = gpr_malloc(sizeof(*st->mu));
+  gpr_mu_init(&st->mu->mu);
+  gpr_ref_init(&st->mu->refs, 2);
+  st->base.vtable = &inproc_vtable;
+  ct->base.vtable = &inproc_vtable;
+  // Start each side of transport with 2 refs since they each have a ref
+  // to the other
+  gpr_ref_init(&st->refs, 2);
+  gpr_ref_init(&ct->refs, 2);
+  st->is_client = false;
+  ct->is_client = true;
+  grpc_connectivity_state_init(&st->connectivity, GRPC_CHANNEL_READY,
+                               "inproc_server");
+  grpc_connectivity_state_init(&ct->connectivity, GRPC_CHANNEL_READY,
+                               "inproc_client");
+  st->other_side = ct;
+  ct->other_side = st;
+  st->stream_list = NULL;
+  ct->stream_list = NULL;
+  *server_transport = (grpc_transport *)st;
+  *client_transport = (grpc_transport *)ct;
+}
+
+grpc_channel *grpc_inproc_channel_create(grpc_server *server,
+                                         grpc_channel_args *args,
+                                         void *reserved) {
+  GRPC_API_TRACE("grpc_inproc_channel_create(server=%p, args=%p)", 2,
+                 (server, args));
+
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+  const grpc_channel_args *server_args = grpc_server_get_channel_args(server);
+
+  // Add a default authority channel argument for the client
+
+  grpc_arg default_authority_arg;
+  default_authority_arg.type = GRPC_ARG_STRING;
+  default_authority_arg.key = GRPC_ARG_DEFAULT_AUTHORITY;
+  default_authority_arg.value.string = "inproc.authority";
+  grpc_channel_args *client_args =
+      grpc_channel_args_copy_and_add(args, &default_authority_arg, 1);
+
+  grpc_transport *server_transport;
+  grpc_transport *client_transport;
+  inproc_transports_create(&exec_ctx, &server_transport, server_args,
+                           &client_transport, client_args);
+
+  grpc_server_setup_transport(&exec_ctx, server, server_transport, NULL,
+                              server_args);
+  grpc_channel *channel =
+      grpc_channel_create(&exec_ctx, "inproc", client_args,
+                          GRPC_CLIENT_DIRECT_CHANNEL, client_transport);
+
+  // Free up created channel args
+  grpc_channel_args_destroy(&exec_ctx, client_args);
+
+  // Now finish scheduled operations
+  grpc_exec_ctx_finish(&exec_ctx);
+
+  return channel;
+}
+
+/*******************************************************************************
+ * INTEGRATION GLUE
+ */
+
+static void set_pollset(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                        grpc_stream *gs, grpc_pollset *pollset) {
+  // Nothing to do here
+}
+
+static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
+                            grpc_stream *gs, grpc_pollset_set *pollset_set) {
+  // Nothing to do here
+}
+
+static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
+  return gpr_strdup("inproc");
+}
+
+static grpc_endpoint *get_endpoint(grpc_exec_ctx *exec_ctx, grpc_transport *t) {
+  return NULL;
+}
+
+static const grpc_transport_vtable inproc_vtable = {
+    sizeof(inproc_stream), "inproc",
+    init_stream,           set_pollset,
+    set_pollset_set,       perform_stream_op,
+    perform_transport_op,  destroy_stream,
+    destroy_transport,     get_peer,
+    get_endpoint};
+
+/*******************************************************************************
+ * GLOBAL INIT AND DESTROY
+ */
+static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {}
+
+void grpc_inproc_transport_init(void) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GRPC_CLOSURE_INIT(&do_nothing_closure, do_nothing, NULL,
+                    grpc_schedule_on_exec_ctx);
+  g_empty_slice = grpc_slice_from_static_buffer(NULL, 0);
+
+  grpc_slice key_tmp = grpc_slice_from_static_string(":path");
+  g_fake_path_key = grpc_slice_intern(key_tmp);
+  grpc_slice_unref_internal(&exec_ctx, key_tmp);
+
+  g_fake_path_value = grpc_slice_from_static_string("/");
+
+  grpc_slice auth_tmp = grpc_slice_from_static_string(":authority");
+  g_fake_auth_key = grpc_slice_intern(auth_tmp);
+  grpc_slice_unref_internal(&exec_ctx, auth_tmp);
+
+  g_fake_auth_value = grpc_slice_from_static_string("inproc-fail");
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+void grpc_inproc_transport_shutdown(void) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_slice_unref_internal(&exec_ctx, g_empty_slice);
+  grpc_slice_unref_internal(&exec_ctx, g_fake_path_key);
+  grpc_slice_unref_internal(&exec_ctx, g_fake_path_value);
+  grpc_slice_unref_internal(&exec_ctx, g_fake_auth_key);
+  grpc_slice_unref_internal(&exec_ctx, g_fake_auth_value);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
diff --git a/src/core/ext/transport/inproc/inproc_transport.h b/src/core/ext/transport/inproc/inproc_transport.h
new file mode 100644
index 0000000000..37e6d99e99
--- /dev/null
+++ b/src/core/ext/transport/inproc/inproc_transport.h
@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_INPROC_INPROC_TRANSPORT_H
+#define GRPC_CORE_EXT_TRANSPORT_INPROC_INPROC_TRANSPORT_H
+
+#include "src/core/lib/transport/transport_impl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+grpc_channel *grpc_inproc_channel_create(grpc_server *server,
+                                         grpc_channel_args *args,
+                                         void *reserved);
+
+extern grpc_tracer_flag grpc_inproc_trace;
+
+void grpc_inproc_transport_init(void);
+void grpc_inproc_transport_shutdown(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_CORE_EXT_TRANSPORT_INPROC_INPROC_TRANSPORT_H */
diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c
index 106666410a..0f8e33c4be 100644
--- a/src/core/lib/channel/channel_stack.c
+++ b/src/core/lib/channel/channel_stack.c
@@ -23,7 +23,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-grpc_tracer_flag grpc_trace_channel = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_channel = GRPC_TRACER_INITIALIZER(false, "channel");
 
 /* Memory layouts.
 
diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c
index 01529df80a..c369e33073 100644
--- a/src/core/lib/channel/channel_stack_builder.c
+++ b/src/core/lib/channel/channel_stack_builder.c
@@ -24,7 +24,7 @@
 #include <grpc/support/string_util.h>
 
 grpc_tracer_flag grpc_trace_channel_stack_builder =
-    GRPC_TRACER_INITIALIZER(false);
+    GRPC_TRACER_INITIALIZER(false, "channel_stack_builder");
 
 typedef struct filter_node {
   struct filter_node *next;
diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c
new file mode 100644
index 0000000000..df13d53e06
--- /dev/null
+++ b/src/core/lib/compression/stream_compression.c
@@ -0,0 +1,191 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/compression/stream_compression.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/slice/slice_internal.h"
+
+#define OUTPUT_BLOCK_SIZE (1024)
+
+static bool gzip_flate(grpc_stream_compression_context *ctx,
+                       grpc_slice_buffer *in, grpc_slice_buffer *out,
+                       size_t *output_size, size_t max_output_size, int flush,
+                       bool *end_of_context) {
+  GPR_ASSERT(flush == 0 || flush == Z_SYNC_FLUSH || flush == Z_FINISH);
+  /* Full flush is not allowed when inflating. */
+  GPR_ASSERT(!(ctx->flate == inflate && (flush == Z_FINISH)));
+
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  int r;
+  bool eoc = false;
+  size_t original_max_output_size = max_output_size;
+  while (max_output_size > 0 && (in->length > 0 || flush) && !eoc) {
+    size_t slice_size = max_output_size < OUTPUT_BLOCK_SIZE ? max_output_size
+                                                            : OUTPUT_BLOCK_SIZE;
+    grpc_slice slice_out = GRPC_SLICE_MALLOC(slice_size);
+    ctx->zs.avail_out = (uInt)slice_size;
+    ctx->zs.next_out = GRPC_SLICE_START_PTR(slice_out);
+    while (ctx->zs.avail_out > 0 && in->length > 0 && !eoc) {
+      grpc_slice slice = grpc_slice_buffer_take_first(in);
+      ctx->zs.avail_in = (uInt)GRPC_SLICE_LENGTH(slice);
+      ctx->zs.next_in = GRPC_SLICE_START_PTR(slice);
+      r = ctx->flate(&ctx->zs, Z_NO_FLUSH);
+      if (r < 0 && r != Z_BUF_ERROR) {
+        gpr_log(GPR_ERROR, "zlib error (%d)", r);
+        grpc_slice_unref_internal(&exec_ctx, slice_out);
+        grpc_exec_ctx_finish(&exec_ctx);
+        return false;
+      } else if (r == Z_STREAM_END && ctx->flate == inflate) {
+        eoc = true;
+      }
+      if (ctx->zs.avail_in > 0) {
+        grpc_slice_buffer_undo_take_first(
+            in,
+            grpc_slice_sub(slice, GRPC_SLICE_LENGTH(slice) - ctx->zs.avail_in,
+                           GRPC_SLICE_LENGTH(slice)));
+      }
+      grpc_slice_unref_internal(&exec_ctx, slice);
+    }
+    if (flush != 0 && ctx->zs.avail_out > 0 && !eoc) {
+      GPR_ASSERT(in->length == 0);
+      r = ctx->flate(&ctx->zs, flush);
+      if (flush == Z_SYNC_FLUSH) {
+        switch (r) {
+          case Z_OK:
+            /* Maybe flush is not complete; just made some partial progress. */
+            if (ctx->zs.avail_out > 0) {
+              flush = 0;
+            }
+            break;
+          case Z_BUF_ERROR:
+          case Z_STREAM_END:
+            flush = 0;
+            break;
+          default:
+            gpr_log(GPR_ERROR, "zlib error (%d)", r);
+            grpc_slice_unref_internal(&exec_ctx, slice_out);
+            grpc_exec_ctx_finish(&exec_ctx);
+            return false;
+        }
+      } else if (flush == Z_FINISH) {
+        switch (r) {
+          case Z_OK:
+          case Z_BUF_ERROR:
+            /* Wait for the next loop to assign additional output space. */
+            GPR_ASSERT(ctx->zs.avail_out == 0);
+            break;
+          case Z_STREAM_END:
+            flush = 0;
+            break;
+          default:
+            gpr_log(GPR_ERROR, "zlib error (%d)", r);
+            grpc_slice_unref_internal(&exec_ctx, slice_out);
+            grpc_exec_ctx_finish(&exec_ctx);
+            return false;
+        }
+      }
+    }
+
+    if (ctx->zs.avail_out == 0) {
+      grpc_slice_buffer_add(out, slice_out);
+    } else if (ctx->zs.avail_out < slice_size) {
+      slice_out.data.refcounted.length -= ctx->zs.avail_out;
+      grpc_slice_buffer_add(out, slice_out);
+    } else {
+      grpc_slice_unref_internal(&exec_ctx, slice_out);
+    }
+    max_output_size -= (slice_size - ctx->zs.avail_out);
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  if (end_of_context) {
+    *end_of_context = eoc;
+  }
+  if (output_size) {
+    *output_size = original_max_output_size - max_output_size;
+  }
+  return true;
+}
+
+bool grpc_stream_compress(grpc_stream_compression_context *ctx,
+                          grpc_slice_buffer *in, grpc_slice_buffer *out,
+                          size_t *output_size, size_t max_output_size,
+                          grpc_stream_compression_flush flush) {
+  GPR_ASSERT(ctx->flate == deflate);
+  int gzip_flush;
+  switch (flush) {
+    case GRPC_STREAM_COMPRESSION_FLUSH_NONE:
+      gzip_flush = 0;
+      break;
+    case GRPC_STREAM_COMPRESSION_FLUSH_SYNC:
+      gzip_flush = Z_SYNC_FLUSH;
+      break;
+    case GRPC_STREAM_COMPRESSION_FLUSH_FINISH:
+      gzip_flush = Z_FINISH;
+      break;
+    default:
+      gzip_flush = 0;
+  }
+  return gzip_flate(ctx, in, out, output_size, max_output_size, gzip_flush,
+                    NULL);
+}
+
+bool grpc_stream_decompress(grpc_stream_compression_context *ctx,
+                            grpc_slice_buffer *in, grpc_slice_buffer *out,
+                            size_t *output_size, size_t max_output_size,
+                            bool *end_of_context) {
+  GPR_ASSERT(ctx->flate == inflate);
+  return gzip_flate(ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH,
+                    end_of_context);
+}
+
+grpc_stream_compression_context *grpc_stream_compression_context_create(
+    grpc_stream_compression_method method) {
+  grpc_stream_compression_context *ctx =
+      gpr_zalloc(sizeof(grpc_stream_compression_context));
+  int r;
+  if (ctx == NULL) {
+    return NULL;
+  }
+  if (method == GRPC_STREAM_COMPRESSION_DECOMPRESS) {
+    r = inflateInit2(&ctx->zs, 0x1F);
+    ctx->flate = inflate;
+  } else {
+    r = deflateInit2(&ctx->zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8,
+                     Z_DEFAULT_STRATEGY);
+    ctx->flate = deflate;
+  }
+  if (r != Z_OK) {
+    gpr_free(ctx);
+    return NULL;
+  }
+
+  return ctx;
+}
+
+void grpc_stream_compression_context_destroy(
+    grpc_stream_compression_context *ctx) {
+  if (ctx->flate == inflate) {
+    inflateEnd(&ctx->zs);
+  } else {
+    deflateEnd(&ctx->zs);
+  }
+  gpr_free(ctx);
+}
diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h
new file mode 100644
index 0000000000..844dff81a3
--- /dev/null
+++ b/src/core/lib/compression/stream_compression.h
@@ -0,0 +1,90 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_H
+#define GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_H
+
+#include <stdbool.h>
+
+#include <grpc/slice_buffer.h>
+#include <zlib.h>
+
+/* Stream compression/decompression context */
+typedef struct grpc_stream_compression_context {
+  z_stream zs;
+  int (*flate)(z_stream *zs, int flush);
+} grpc_stream_compression_context;
+
+typedef enum grpc_stream_compression_method {
+  GRPC_STREAM_COMPRESSION_COMPRESS = 0,
+  GRPC_STREAM_COMPRESSION_DECOMPRESS,
+  GRPC_STREAM_COMPRESSION_METHOD_COUNT
+} grpc_stream_compression_method;
+
+typedef enum grpc_stream_compression_flush {
+  GRPC_STREAM_COMPRESSION_FLUSH_NONE = 0,
+  GRPC_STREAM_COMPRESSION_FLUSH_SYNC,
+  GRPC_STREAM_COMPRESSION_FLUSH_FINISH,
+  GRPC_STREAM_COMPRESSION_FLUSH_COUNT
+} grpc_stream_compression_flush;
+
+/**
+ * Compress bytes provided in \a in with a given context, with an optional flush
+ * at the end of compression. Emits at most \a max_output_size compressed bytes
+ * into \a out. If all the bytes in input buffer \a in are depleted and \a flush
+ * is not GRPC_STREAM_COMPRESSION_FLUSH_NONE, the corresponding flush method is
+ * executed. The total number of bytes emitted is outputed in \a output_size.
+ *
+ * A SYNC flush indicates that the entire messages in \a in can be decompressed
+ * from \a out. A FINISH flush implies a SYNC flush, and that any further
+ * compression will not be dependent on the state of the current context and any
+ * previous compressed bytes. It allows corresponding decompression context to
+ * be dropped when reaching this boundary.
+ */
+bool grpc_stream_compress(grpc_stream_compression_context *ctx,
+                          grpc_slice_buffer *in, grpc_slice_buffer *out,
+                          size_t *output_size, size_t max_output_size,
+                          grpc_stream_compression_flush flush);
+
+/**
+ * Decompress bytes provided in \a in with a given context. Emits at most \a
+ * max_output_size decompressed bytes into \a out. If decompression process
+ * reached the end of a gzip stream, \a end_of_context is set to true; otherwise
+ * it is set to false. The total number of bytes emitted is outputed in \a
+ * output_size.
+ */
+bool grpc_stream_decompress(grpc_stream_compression_context *ctx,
+                            grpc_slice_buffer *in, grpc_slice_buffer *out,
+                            size_t *output_size, size_t max_output_size,
+                            bool *end_of_context);
+
+/**
+ * Creates a stream compression context. \a pending_bytes_buffer is the input
+ * buffer for compression/decompression operations. \a method specifies whether
+ * the context is for compression or decompression.
+ */
+grpc_stream_compression_context *grpc_stream_compression_context_create(
+    grpc_stream_compression_method method);
+
+/**
+ * Destroys a stream compression context.
+ */
+void grpc_stream_compression_context_destroy(
+    grpc_stream_compression_context *ctx);
+
+#endif
diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c
index 8249b2ebd6..c6c1853e20 100644
--- a/src/core/lib/debug/trace.c
+++ b/src/core/lib/debug/trace.c
@@ -27,7 +27,6 @@
 int grpc_tracer_set_enabled(const char *name, int enabled);
 
 typedef struct tracer {
-  const char *name;
   grpc_tracer_flag *flag;
   struct tracer *next;
 } tracer;
@@ -39,9 +38,8 @@ static tracer *tracers;
 #define TRACER_SET(flag, on) (flag).value = (on)
 #endif
 
-void grpc_register_tracer(const char *name, grpc_tracer_flag *flag) {
+void grpc_register_tracer(grpc_tracer_flag *flag) {
   tracer *t = gpr_malloc(sizeof(*t));
-  t->name = name;
   t->flag = flag;
   t->next = tracers;
   TRACER_SET(*flag, false);
@@ -93,6 +91,14 @@ static void parse(const char *s) {
   gpr_free(strings);
 }
 
+static void list_tracers() {
+  gpr_log(GPR_DEBUG, "available tracers:");
+  tracer *t;
+  for (t = tracers; t; t = t->next) {
+    gpr_log(GPR_DEBUG, "\t%s", t->flag->name);
+  }
+}
+
 void grpc_tracer_init(const char *env_var) {
   char *e = gpr_getenv(env_var);
   if (e != NULL) {
@@ -115,10 +121,18 @@ int grpc_tracer_set_enabled(const char *name, int enabled) {
     for (t = tracers; t; t = t->next) {
       TRACER_SET(*t->flag, enabled);
     }
+  } else if (0 == strcmp(name, "list_tracers")) {
+    list_tracers();
+  } else if (0 == strcmp(name, "refcount")) {
+    for (t = tracers; t; t = t->next) {
+      if (strstr(t->flag->name, "refcount") != NULL) {
+        TRACER_SET(*t->flag, enabled);
+      }
+    }
   } else {
     int found = 0;
     for (t = tracers; t; t = t->next) {
-      if (0 == strcmp(name, t->name)) {
+      if (0 == strcmp(name, t->flag->name)) {
         TRACER_SET(*t->flag, enabled);
         found = 1;
       }
diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h
index 7cc9fb4e41..dd9e6a30fe 100644
--- a/src/core/lib/debug/trace.h
+++ b/src/core/lib/debug/trace.h
@@ -35,19 +35,20 @@ typedef struct {
 #else
   bool value;
 #endif
+  char *name;
 } grpc_tracer_flag;
 
 #ifdef GRPC_THREADSAFE_TRACER
 #define GRPC_TRACER_ON(flag) (gpr_atm_no_barrier_load(&(flag).value) != 0)
-#define GRPC_TRACER_INITIALIZER(on) \
-  { (gpr_atm)(on) }
+#define GRPC_TRACER_INITIALIZER(on, name) \
+  { (gpr_atm)(on), (name) }
 #else
 #define GRPC_TRACER_ON(flag) ((flag).value)
-#define GRPC_TRACER_INITIALIZER(on) \
-  { (on) }
+#define GRPC_TRACER_INITIALIZER(on, name) \
+  { (on), (name) }
 #endif
 
-void grpc_register_tracer(const char *name, grpc_tracer_flag *flag);
+void grpc_register_tracer(grpc_tracer_flag *flag);
 void grpc_tracer_init(const char *env_var_name);
 void grpc_tracer_shutdown(void);
 
diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c
index 71d697c278..9c5e93f4e5 100644
--- a/src/core/lib/http/parser.c
+++ b/src/core/lib/http/parser.c
@@ -25,7 +25,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-grpc_tracer_flag grpc_http1_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_http1_trace = GRPC_TRACER_INITIALIZER(false, "http1");
 
 static char *buf2str(void *buffer, size_t length) {
   char *out = gpr_malloc(length + 1);
diff --git a/src/core/lib/iomgr/closure.c b/src/core/lib/iomgr/closure.c
index e028e72ed6..26f9cbe0fa 100644
--- a/src/core/lib/iomgr/closure.c
+++ b/src/core/lib/iomgr/closure.c
@@ -25,7 +25,7 @@
 #include "src/core/lib/profiling/timers.h"
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_closure = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_closure = GRPC_TRACER_INITIALIZER(false, "closure");
 #endif
 
 #ifndef NDEBUG
diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c
index 7f9c5d837f..c72c37e2b5 100644
--- a/src/core/lib/iomgr/combiner.c
+++ b/src/core/lib/iomgr/combiner.c
@@ -27,7 +27,8 @@
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/profiling/timers.h"
 
-grpc_tracer_flag grpc_combiner_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_combiner_trace =
+    GRPC_TRACER_INITIALIZER(false, "combiner");
 
 #define GRPC_COMBINER_TRACE(fn)                \
   do {                                         \
diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c
index a95929a1fb..3759dda992 100644
--- a/src/core/lib/iomgr/error.c
+++ b/src/core/lib/iomgr/error.c
@@ -36,7 +36,8 @@
 #include "src/core/lib/slice/slice_internal.h"
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_error_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_error_refcount =
+    GRPC_TRACER_INITIALIZER(false, "error_refcount");
 #endif
 
 static const char *error_int_name(grpc_error_ints key) {
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c
index 66ba601adb..dc48d73df9 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.c
+++ b/src/core/lib/iomgr/ev_epoll1_linux.c
@@ -45,6 +45,7 @@
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
+#include "src/core/lib/support/string.h"
 
 static grpc_wakeup_fd global_wakeup_fd;
 static int g_epfd;
@@ -77,8 +78,21 @@ static void fd_global_shutdown(void);
 
 typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state;
 
+static const char *kick_state_string(kick_state st) {
+  switch (st) {
+    case UNKICKED:
+      return "UNKICKED";
+    case KICKED:
+      return "KICKED";
+    case DESIGNATED_POLLER:
+      return "DESIGNATED_POLLER";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
 struct grpc_pollset_worker {
   kick_state kick_state;
+  int kick_state_mutator;  // which line of code last changed kick state
   bool initialized_cv;
   grpc_pollset_worker *next;
   grpc_pollset_worker *prev;
@@ -86,6 +100,12 @@ struct grpc_pollset_worker {
   grpc_closure_list schedule_on_end_work;
 };
 
+#define SET_KICK_STATE(worker, state)        \
+  do {                                       \
+    (worker)->kick_state = (state);          \
+    (worker)->kick_state_mutator = __LINE__; \
+  } while (false)
+
 #define MAX_NEIGHBOURHOODS 1024
 
 typedef struct pollset_neighbourhood {
@@ -100,10 +120,15 @@ struct grpc_pollset {
   bool reassigning_neighbourhood;
   grpc_pollset_worker *root_worker;
   bool kicked_without_poller;
+
+  /* Set to true if the pollset is observed to have no workers available to
+   * poll */
   bool seen_inactive;
-  bool shutting_down;          /* Is the pollset shutting down ? */
-  bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */
+  bool shutting_down;             /* Is the pollset shutting down ? */
   grpc_closure *shutdown_closure; /* Called after after shutdown is complete */
+
+  /* Number of workers who are *about-to* attach themselves to the pollset
+   * worker list */
   int begin_refs;
 
   grpc_pollset *next;
@@ -224,7 +249,7 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
 
   if (!grpc_lfev_is_shutdown(&fd->read_closure)) {
@@ -235,7 +260,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!already_closed) {
     close(fd->fd);
   }
 
@@ -263,29 +288,23 @@ static bool fd_is_shutdown(grpc_fd *fd) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
-
-  /* Note, it is possible that fd_become_readable might be called twice with
-     different 'notifier's when an fd becomes readable and it is in two epoll
-     sets (This can happen briefly during polling island merges). In such cases
-     it does not really matter which notifer is set as the read_notifier_pollset
-     (They would both point to the same polling island anyway) */
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
   /* Use release store to match with acquire load in fd_get_read_notifier */
   gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier);
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 /*******************************************************************************
@@ -410,18 +429,28 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) {
   if (pollset->root_worker != NULL) {
     grpc_pollset_worker *worker = pollset->root_worker;
     do {
-      if (worker->initialized_cv) {
-        worker->kick_state = KICKED;
-        gpr_cv_signal(&worker->cv);
-      } else {
-        worker->kick_state = KICKED;
-        append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd),
-                     "pollset_shutdown");
+      switch (worker->kick_state) {
+        case KICKED:
+          break;
+        case UNKICKED:
+          SET_KICK_STATE(worker, KICKED);
+          if (worker->initialized_cv) {
+            gpr_cv_signal(&worker->cv);
+          }
+          break;
+        case DESIGNATED_POLLER:
+          SET_KICK_STATE(worker, KICKED);
+          append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd),
+                       "pollset_kick_all");
+          break;
       }
 
       worker = worker->next;
     } while (worker != pollset->root_worker);
   }
+  // TODO: sreek.  Check if we need to set 'kicked_without_poller' to true here
+  // in the else case
+
   return error;
 }
 
@@ -437,7 +466,9 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx,
 static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                              grpc_closure *closure) {
   GPR_ASSERT(pollset->shutdown_closure == NULL);
+  GPR_ASSERT(!pollset->shutting_down);
   pollset->shutdown_closure = closure;
+  pollset->shutting_down = true;
   GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset));
   pollset_maybe_finish_shutdown(exec_ctx, pollset);
 }
@@ -510,10 +541,14 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
                          gpr_timespec deadline) {
   if (worker_hdl != NULL) *worker_hdl = worker;
   worker->initialized_cv = false;
-  worker->kick_state = UNKICKED;
+  SET_KICK_STATE(worker, UNKICKED);
   worker->schedule_on_end_work = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
   pollset->begin_refs++;
 
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_ERROR, "PS:%p BEGIN_STARTS:%p", pollset, worker);
+  }
+
   if (pollset->seen_inactive) {
     // pollset has been observed to be inactive, we need to move back to the
     // active list
@@ -529,6 +564,11 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
   retry_lock_neighbourhood:
     gpr_mu_lock(&neighbourhood->mu);
     gpr_mu_lock(&pollset->mu);
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, "PS:%p BEGIN_REORG:%p kick_state=%s is_reassigning=%d",
+              pollset, worker, kick_state_string(worker->kick_state),
+              is_reassigning);
+    }
     if (pollset->seen_inactive) {
       if (neighbourhood != pollset->neighbourhood) {
         gpr_mu_unlock(&neighbourhood->mu);
@@ -539,8 +579,14 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
       pollset->seen_inactive = false;
       if (neighbourhood->active_root == NULL) {
         neighbourhood->active_root = pollset->next = pollset->prev = pollset;
-        if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) {
-          worker->kick_state = DESIGNATED_POLLER;
+        /* TODO: sreek. Why would this worker state be other than UNKICKED
+         * here ? (since the worker isn't added to the pollset yet, there is no
+         * way it can be "found" by other threads to get kicked). */
+
+        /* If there is no designated poller, make this the designated poller */
+        if (worker->kick_state == UNKICKED &&
+            gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) {
+          SET_KICK_STATE(worker, DESIGNATED_POLLER);
         }
       } else {
         pollset->next = neighbourhood->active_root;
@@ -554,24 +600,53 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker,
     }
     gpr_mu_unlock(&neighbourhood->mu);
   }
+
   worker_insert(pollset, worker);
   pollset->begin_refs--;
-  if (worker->kick_state == UNKICKED) {
+  if (worker->kick_state == UNKICKED && !pollset->kicked_without_poller) {
     GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker);
     worker->initialized_cv = true;
     gpr_cv_init(&worker->cv);
-    while (worker->kick_state == UNKICKED &&
-           pollset->shutdown_closure == NULL) {
+    while (worker->kick_state == UNKICKED && !pollset->shutting_down) {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_ERROR, "PS:%p BEGIN_WAIT:%p kick_state=%s shutdown=%d",
+                pollset, worker, kick_state_string(worker->kick_state),
+                pollset->shutting_down);
+      }
+
       if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) &&
           worker->kick_state == UNKICKED) {
-        worker->kick_state = KICKED;
+        /* If gpr_cv_wait returns true (i.e a timeout), pretend that the worker
+           received a kick */
+        SET_KICK_STATE(worker, KICKED);
       }
     }
     *now = gpr_now(now->clock_type);
   }
 
-  return worker->kick_state == DESIGNATED_POLLER &&
-         pollset->shutdown_closure == NULL;
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_ERROR,
+            "PS:%p BEGIN_DONE:%p kick_state=%s shutdown=%d "
+            "kicked_without_poller: %d",
+            pollset, worker, kick_state_string(worker->kick_state),
+            pollset->shutting_down, pollset->kicked_without_poller);
+  }
+
+  /* We release pollset lock in this function at a couple of places:
+   *   1. Briefly when assigning pollset to a neighbourhood
+   *   2. When doing gpr_cv_wait()
+   * It is possible that 'kicked_without_poller' was set to true during (1) and
+   * 'shutting_down' is set to true during (1) or (2). If either of them is
+   * true, this worker cannot do polling */
+  /* TODO(sreek): Perhaps there is a better way to handle kicked_without_poller
+   * case; especially when the worker is the DESIGNATED_POLLER */
+
+  if (pollset->kicked_without_poller) {
+    pollset->kicked_without_poller = false;
+    return false;
+  }
+
+  return worker->kick_state == DESIGNATED_POLLER && !pollset->shutting_down;
 }
 
 static bool check_neighbourhood_for_available_poller(
@@ -591,10 +666,18 @@ static bool check_neighbourhood_for_available_poller(
           case UNKICKED:
             if (gpr_atm_no_barrier_cas(&g_active_poller, 0,
                                        (gpr_atm)inspect_worker)) {
-              inspect_worker->kick_state = DESIGNATED_POLLER;
+              if (GRPC_TRACER_ON(grpc_polling_trace)) {
+                gpr_log(GPR_DEBUG, " .. choose next poller to be %p",
+                        inspect_worker);
+              }
+              SET_KICK_STATE(inspect_worker, DESIGNATED_POLLER);
               if (inspect_worker->initialized_cv) {
                 gpr_cv_signal(&inspect_worker->cv);
               }
+            } else {
+              if (GRPC_TRACER_ON(grpc_polling_trace)) {
+                gpr_log(GPR_DEBUG, " .. beaten to choose next poller");
+              }
             }
             // even if we didn't win the cas, there's a worker, we can stop
             found_worker = true;
@@ -607,9 +690,12 @@ static bool check_neighbourhood_for_available_poller(
             break;
         }
         inspect_worker = inspect_worker->next;
-      } while (inspect_worker != inspect->root_worker);
+      } while (!found_worker && inspect_worker != inspect->root_worker);
     }
     if (!found_worker) {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_DEBUG, " .. mark pollset %p inactive", inspect);
+      }
       inspect->seen_inactive = true;
       if (inspect == neighbourhood->active_root) {
         neighbourhood->active_root =
@@ -627,15 +713,22 @@ static bool check_neighbourhood_for_available_poller(
 static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                        grpc_pollset_worker *worker,
                        grpc_pollset_worker **worker_hdl) {
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_DEBUG, "PS:%p END_WORKER:%p", pollset, worker);
+  }
   if (worker_hdl != NULL) *worker_hdl = NULL;
-  worker->kick_state = KICKED;
+  /* Make sure we appear kicked */
+  SET_KICK_STATE(worker, KICKED);
   grpc_closure_list_move(&worker->schedule_on_end_work,
                          &exec_ctx->closure_list);
   if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) {
     if (worker->next != worker && worker->next->kick_state == UNKICKED) {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_DEBUG, " .. choose next poller to be peer %p", worker);
+      }
       GPR_ASSERT(worker->next->initialized_cv);
       gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next);
-      worker->next->kick_state = DESIGNATED_POLLER;
+      SET_KICK_STATE(worker->next, DESIGNATED_POLLER);
       gpr_cv_signal(&worker->next->cv);
       if (grpc_exec_ctx_has_work(exec_ctx)) {
         gpr_mu_unlock(&pollset->mu);
@@ -644,9 +737,9 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
     } else {
       gpr_atm_no_barrier_store(&g_active_poller, 0);
-      gpr_mu_unlock(&pollset->mu);
       size_t poller_neighbourhood_idx =
           (size_t)(pollset->neighbourhood - g_neighbourhoods);
+      gpr_mu_unlock(&pollset->mu);
       bool found_worker = false;
       bool scan_state[MAX_NEIGHBOURHOODS];
       for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) {
@@ -682,6 +775,9 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (worker->initialized_cv) {
     gpr_cv_destroy(&worker->cv);
   }
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_log(GPR_DEBUG, " .. remove worker");
+  }
   if (EMPTIED == worker_remove(pollset, worker)) {
     pollset_maybe_finish_shutdown(exec_ctx, pollset);
   }
@@ -702,16 +798,18 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     pollset->kicked_without_poller = false;
     return GRPC_ERROR_NONE;
   }
-  gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
   if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) {
+    gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
     gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker);
-    GPR_ASSERT(!pollset->shutdown_closure);
+    GPR_ASSERT(!pollset->shutting_down);
     GPR_ASSERT(!pollset->seen_inactive);
     gpr_mu_unlock(&pollset->mu);
     append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline),
                  err_desc);
     gpr_mu_lock(&pollset->mu);
     gpr_tls_set(&g_current_thread_worker, 0);
+  } else {
+    gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset);
   }
   end_worker(exec_ctx, pollset, &worker, worker_hdl);
   gpr_tls_set(&g_current_thread_pollset, 0);
@@ -720,46 +818,136 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
 static grpc_error *pollset_kick(grpc_pollset *pollset,
                                 grpc_pollset_worker *specific_worker) {
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {
+    gpr_strvec log;
+    gpr_strvec_init(&log);
+    char *tmp;
+    gpr_asprintf(
+        &tmp, "PS:%p KICK:%p curps=%p curworker=%p root=%p", pollset,
+        specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset),
+        (void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker);
+    gpr_strvec_add(&log, tmp);
+    if (pollset->root_worker != NULL) {
+      gpr_asprintf(&tmp, " {kick_state=%s next=%p {kick_state=%s}}",
+                   kick_state_string(pollset->root_worker->kick_state),
+                   pollset->root_worker->next,
+                   kick_state_string(pollset->root_worker->next->kick_state));
+      gpr_strvec_add(&log, tmp);
+    }
+    if (specific_worker != NULL) {
+      gpr_asprintf(&tmp, " worker_kick_state=%s",
+                   kick_state_string(specific_worker->kick_state));
+      gpr_strvec_add(&log, tmp);
+    }
+    tmp = gpr_strvec_flatten(&log, NULL);
+    gpr_strvec_destroy(&log);
+    gpr_log(GPR_ERROR, "%s", tmp);
+    gpr_free(tmp);
+  }
   if (specific_worker == NULL) {
     if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) {
       grpc_pollset_worker *root_worker = pollset->root_worker;
       if (root_worker == NULL) {
         pollset->kicked_without_poller = true;
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. kicked_without_poller");
+        }
         return GRPC_ERROR_NONE;
       }
       grpc_pollset_worker *next_worker = root_worker->next;
-      if (root_worker == next_worker &&
-          root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(
-                             &g_active_poller)) {
-        root_worker->kick_state = KICKED;
+      if (root_worker->kick_state == KICKED) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. already kicked %p", root_worker);
+        }
+        SET_KICK_STATE(root_worker, KICKED);
+        return GRPC_ERROR_NONE;
+      } else if (next_worker->kick_state == KICKED) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. already kicked %p", next_worker);
+        }
+        SET_KICK_STATE(next_worker, KICKED);
+        return GRPC_ERROR_NONE;
+      } else if (root_worker ==
+                     next_worker &&  // only try and wake up a poller if
+                                     // there is no next worker
+                 root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load(
+                                    &g_active_poller)) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. kicked %p", root_worker);
+        }
+        SET_KICK_STATE(root_worker, KICKED);
         return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
       } else if (next_worker->kick_state == UNKICKED) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_ERROR, " .. kicked %p", next_worker);
+        }
         GPR_ASSERT(next_worker->initialized_cv);
-        next_worker->kick_state = KICKED;
+        SET_KICK_STATE(next_worker, KICKED);
         gpr_cv_signal(&next_worker->cv);
         return GRPC_ERROR_NONE;
+      } else if (next_worker->kick_state == DESIGNATED_POLLER) {
+        if (root_worker->kick_state != DESIGNATED_POLLER) {
+          if (GRPC_TRACER_ON(grpc_polling_trace)) {
+            gpr_log(
+                GPR_ERROR,
+                " .. kicked root non-poller %p (initialized_cv=%d) (poller=%p)",
+                root_worker, root_worker->initialized_cv, next_worker);
+          }
+          SET_KICK_STATE(root_worker, KICKED);
+          if (root_worker->initialized_cv) {
+            gpr_cv_signal(&root_worker->cv);
+          }
+          return GRPC_ERROR_NONE;
+        } else {
+          if (GRPC_TRACER_ON(grpc_polling_trace)) {
+            gpr_log(GPR_ERROR, " .. non-root poller %p (root=%p)", next_worker,
+                    root_worker);
+          }
+          SET_KICK_STATE(next_worker, KICKED);
+          return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
+        }
       } else {
+        GPR_ASSERT(next_worker->kick_state == KICKED);
+        SET_KICK_STATE(next_worker, KICKED);
         return GRPC_ERROR_NONE;
       }
     } else {
+      if (GRPC_TRACER_ON(grpc_polling_trace)) {
+        gpr_log(GPR_ERROR, " .. kicked while waking up");
+      }
       return GRPC_ERROR_NONE;
     }
   } else if (specific_worker->kick_state == KICKED) {
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. specific worker already kicked");
+    }
     return GRPC_ERROR_NONE;
   } else if (gpr_tls_get(&g_current_thread_worker) ==
              (intptr_t)specific_worker) {
-    specific_worker->kick_state = KICKED;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. mark %p kicked", specific_worker);
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
     return GRPC_ERROR_NONE;
   } else if (specific_worker ==
              (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) {
-    specific_worker->kick_state = KICKED;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. kick active poller");
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
     return grpc_wakeup_fd_wakeup(&global_wakeup_fd);
   } else if (specific_worker->initialized_cv) {
-    specific_worker->kick_state = KICKED;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. kick waiting worker");
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
     gpr_cv_signal(&specific_worker->cv);
     return GRPC_ERROR_NONE;
   } else {
-    specific_worker->kick_state = KICKED;
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
+      gpr_log(GPR_ERROR, " .. kick non-waiting worker");
+    }
+    SET_KICK_STATE(specific_worker, KICKED);
     return GRPC_ERROR_NONE;
   }
 }
@@ -805,6 +993,7 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
 static void shutdown_engine(void) {
   fd_global_shutdown();
   pollset_global_shutdown();
+  close(g_epfd);
 }
 
 static const grpc_event_engine_vtable vtable = {
@@ -841,8 +1030,11 @@ static const grpc_event_engine_vtable vtable = {
 /* It is possible that GLIBC has epoll but the underlying kernel doesn't.
  * Create a dummy epoll_fd to make sure epoll support is available */
 const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) {
-  /* TODO(ctiller): temporary, until this stabilizes */
-  if (!explicit_request) return NULL;
+  /* TODO(sreek): Temporarily disable this poller unless explicitly requested
+   * via GRPC_POLL_STRATEGY */
+  if (!explicit_request) {
+    return NULL;
+  }
 
   if (!grpc_has_wakeup_fd()) {
     return NULL;
@@ -862,6 +1054,8 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) {
     return NULL;
   }
 
+  gpr_log(GPR_ERROR, "grpc epoll fd: %d", g_epfd);
+
   return &vtable;
 }
 
diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
index 2c91ad357c..f2f3e15704 100644
--- a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c
@@ -57,9 +57,6 @@
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
 
-/* Uncomment the following to enable extra checks on poll_object operations */
-/* #define PO_DEBUG */
-
 /* The maximum number of polling threads per polling island. By default no
    limit */
 static int g_max_pollers_per_pi = INT_MAX;
@@ -92,7 +89,7 @@ typedef enum {
 } poll_obj_type;
 
 typedef struct poll_obj {
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   poll_obj_type obj_type;
 #endif
   gpr_mu mu;
@@ -893,7 +890,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
    * would be holding a lock to it anyway. */
   gpr_mu_lock(&new_fd->po.mu);
   new_fd->po.pi = NULL;
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   new_fd->po.obj_type = POLL_OBJ_FD;
 #endif
 
@@ -934,25 +931,13 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
   gpr_mu_lock(&fd->po.mu);
   fd->on_done_closure = on_done;
 
-  /* If release_fd is not NULL, we should be relinquishing control of the file
-     descriptor fd->fd (but we still own the grpc_fd structure). */
-  if (release_fd != NULL) {
-    *release_fd = fd->fd;
-  } else {
-    close(fd->fd);
-    is_fd_closed = true;
-  }
-
-  fd->orphaned = true;
-
   /* Remove the active status but keep referenced. We want this grpc_fd struct
      to be alive (and not added to freelist) until the end of this function */
   REF_BY(fd, 1, reason);
@@ -967,13 +952,23 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
        before doing this.) */
   if (fd->po.pi != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error);
+    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
     fd->po.pi = NULL;
   }
 
+  /* If release_fd is not NULL, we should be relinquishing control of the file
+     descriptor fd->fd (but we still own the grpc_fd structure). */
+  if (release_fd != NULL) {
+    *release_fd = fd->fd;
+  } else {
+    close(fd->fd);
+  }
+
+  fd->orphaned = true;
+
   GRPC_CLOSURE_SCHED(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error));
 
   gpr_mu_unlock(&fd->po.mu);
@@ -1011,12 +1006,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -1171,7 +1166,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->po.mu);
   *mu = &pollset->po.mu;
   pollset->po.pi = NULL;
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   pollset->po.obj_type = POLL_OBJ_POLLSET;
 #endif
 
@@ -1227,7 +1222,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 
   /* Note, it is possible that fd_become_readable might be called twice with
      different 'notifier's when an fd becomes readable and it is in two epoll
@@ -1239,7 +1234,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx,
@@ -1625,7 +1620,7 @@ static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag,
                             poll_obj_type item_type) {
   GPR_TIMER_BEGIN("add_poll_object", 0);
 
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   GPR_ASSERT(item->obj_type == item_type);
   GPR_ASSERT(bag->obj_type == bag_type);
 #endif
@@ -1784,7 +1779,7 @@ static grpc_pollset_set *pollset_set_create(void) {
   grpc_pollset_set *pss = gpr_malloc(sizeof(*pss));
   gpr_mu_init(&pss->po.mu);
   pss->po.pi = NULL;
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   pss->po.obj_type = POLL_OBJ_POLLSET_SET;
 #endif
   return pss;
diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
index 49be72c03e..07c8eadf4f 100644
--- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
+++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c
@@ -493,8 +493,8 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
+  bool is_fd_closed = already_closed;
   grpc_error *error = GRPC_ERROR_NONE;
   epoll_set *unref_eps = NULL;
 
@@ -505,7 +505,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!is_fd_closed) {
     close(fd->fd);
     is_fd_closed = true;
   }
@@ -560,12 +560,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -696,11 +696,11 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
 }
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c
index 5574838187..770d1fd0a9 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.c
+++ b/src/core/lib/iomgr/ev_epollex_linux.c
@@ -103,6 +103,32 @@ typedef struct pollable {
   grpc_pollset_worker *root_worker;
 } pollable;
 
+static const char *polling_obj_type_string(polling_obj_type t) {
+  switch (t) {
+    case PO_POLLING_GROUP:
+      return "polling_group";
+    case PO_POLLSET_SET:
+      return "pollset_set";
+    case PO_POLLSET:
+      return "pollset";
+    case PO_FD:
+      return "fd";
+    case PO_EMPTY_POLLABLE:
+      return "empty_pollable";
+    case PO_COUNT:
+      return "<invalid:count>";
+  }
+  return "<invalid>";
+}
+
+static char *pollable_desc(pollable *p) {
+  char *out;
+  gpr_asprintf(&out, "type=%s group=%p epfd=%d wakeup=%d",
+               polling_obj_type_string(p->po.type), p->po.group, p->epfd,
+               p->wakeup.read_fd);
+  return out;
+}
+
 static pollable g_empty_pollable;
 
 static void pollable_init(pollable *p, polling_obj_type type);
@@ -354,8 +380,8 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
+  bool is_fd_closed = already_closed;
   grpc_error *error = GRPC_ERROR_NONE;
 
   gpr_mu_lock(&fd->pollable.po.mu);
@@ -366,7 +392,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != NULL) {
     *release_fd = fd->fd;
-  } else {
+  } else if (!is_fd_closed) {
     close(fd->fd);
     is_fd_closed = true;
   }
@@ -412,12 +438,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -472,7 +498,7 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) {
   GPR_ASSERT(epfd != -1);
 
   if (GRPC_TRACER_ON(grpc_polling_trace)) {
-    gpr_log(GPR_DEBUG, "add fd %p to pollable %p", fd, p);
+    gpr_log(GPR_DEBUG, "add fd %p (%d) to pollable %p", fd, fd->fd, p);
   }
 
   gpr_mu_lock(&fd->orphaned_mu);
@@ -537,10 +563,18 @@ static void do_kick_all(grpc_exec_ctx *exec_ctx, void *arg,
       if (worker->pollable != &pollset->pollable) {
         gpr_mu_lock(&worker->pollable->po.mu);
       }
-      if (worker->initialized_cv) {
+      if (worker->initialized_cv && worker != pollset->root_worker) {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_DEBUG, "PS:%p kickall_via_cv %p (pollable %p vs %p)",
+                  pollset, worker, &pollset->pollable, worker->pollable);
+        }
         worker->kicked = true;
         gpr_cv_signal(&worker->cv);
       } else {
+        if (GRPC_TRACER_ON(grpc_polling_trace)) {
+          gpr_log(GPR_DEBUG, "PS:%p kickall_via_wakeup %p (pollable %p vs %p)",
+                  pollset, worker, &pollset->pollable, worker->pollable);
+        }
         append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup),
                      "pollset_shutdown");
       }
@@ -676,7 +710,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 
   /* Note, it is possible that fd_become_readable might be called twice with
      different 'notifier's when an fd becomes readable and it is in two epoll
@@ -688,7 +722,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static grpc_error *fd_become_pollable_locked(grpc_fd *fd) {
@@ -770,7 +804,9 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   int timeout = poll_deadline_to_millis_timeout(deadline, now);
 
   if (GRPC_TRACER_ON(grpc_polling_trace)) {
-    gpr_log(GPR_DEBUG, "PS:%p poll %p for %dms", pollset, p, timeout);
+    char *desc = pollable_desc(p);
+    gpr_log(GPR_DEBUG, "PS:%p poll %p[%s] for %dms", pollset, p, desc, timeout);
+    gpr_free(desc);
   }
 
   if (timeout != 0) {
@@ -985,10 +1021,11 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx,
   static const char *err_desc = "pollset_add_fd";
   grpc_error *error = GRPC_ERROR_NONE;
   if (pollset->current_pollable == &g_empty_pollable) {
-    if (GRPC_TRACER_ON(grpc_polling_trace))
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_DEBUG,
               "PS:%p add fd %p; transition pollable from empty to fd", pollset,
               fd);
+    }
     /* empty pollable --> single fd pollable */
     pollset_kick_all(exec_ctx, pollset);
     pollset->current_pollable = &fd->pollable;
@@ -997,16 +1034,23 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx,
     if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu);
     REF_BY(fd, 2, "pollset_pollable");
   } else if (pollset->current_pollable == &pollset->pollable) {
-    if (GRPC_TRACER_ON(grpc_polling_trace))
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd);
+    }
     append_error(&error, pollable_add_fd(pollset->current_pollable, fd),
                  err_desc);
   } else if (pollset->current_pollable != &fd->pollable) {
     grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable;
-    if (GRPC_TRACER_ON(grpc_polling_trace))
+    if (GRPC_TRACER_ON(grpc_polling_trace)) {
       gpr_log(GPR_DEBUG,
               "PS:%p add fd %p; transition pollable from fd %p to multipoller",
               pollset, fd, had_fd);
+    }
+    /* Introduce a spurious completion.
+       If we do not, then it may be that the fd-specific epoll set consumed
+       a completion without being polled, leading to a missed edge going up. */
+    grpc_lfev_set_ready(exec_ctx, &had_fd->read_closure, "read");
+    grpc_lfev_set_ready(exec_ctx, &had_fd->write_closure, "write");
     pollset_kick_all(exec_ctx, pollset);
     pollset->current_pollable = &pollset->pollable;
     if (append_error(&error, pollable_materialize(&pollset->pollable),
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c
index 255e07010b..070d75e42a 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.c
+++ b/src/core/lib/iomgr/ev_epollsig_linux.c
@@ -54,9 +54,6 @@
     gpr_log(GPR_INFO, __VA_ARGS__);         \
   }
 
-/* Uncomment the following to enable extra checks on poll_object operations */
-/* #define PO_DEBUG */
-
 static int grpc_wakeup_signal = -1;
 static bool is_grpc_wakeup_signal_initialized = false;
 
@@ -85,7 +82,7 @@ typedef enum {
 } poll_obj_type;
 
 typedef struct poll_obj {
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   poll_obj_type obj_type;
 #endif
   gpr_mu mu;
@@ -821,7 +818,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
    * would be holding a lock to it anyway. */
   gpr_mu_lock(&new_fd->po.mu);
   new_fd->po.pi = NULL;
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   new_fd->po.obj_type = POLL_OBJ_FD;
 #endif
 
@@ -857,25 +854,13 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
-  bool is_fd_closed = false;
+                      bool already_closed, const char *reason) {
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
   gpr_mu_lock(&fd->po.mu);
   fd->on_done_closure = on_done;
 
-  /* If release_fd is not NULL, we should be relinquishing control of the file
-     descriptor fd->fd (but we still own the grpc_fd structure). */
-  if (release_fd != NULL) {
-    *release_fd = fd->fd;
-  } else {
-    close(fd->fd);
-    is_fd_closed = true;
-  }
-
-  fd->orphaned = true;
-
   /* Remove the active status but keep referenced. We want this grpc_fd struct
      to be alive (and not added to freelist) until the end of this function */
   REF_BY(fd, 1, reason);
@@ -890,13 +875,23 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
        before doing this.) */
   if (fd->po.pi != NULL) {
     polling_island *pi_latest = polling_island_lock(fd->po.pi);
-    polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error);
+    polling_island_remove_fd_locked(pi_latest, fd, already_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
     unref_pi = fd->po.pi;
     fd->po.pi = NULL;
   }
 
+  /* If release_fd is not NULL, we should be relinquishing control of the file
+     descriptor fd->fd (but we still own the grpc_fd structure). */
+  if (release_fd != NULL) {
+    *release_fd = fd->fd;
+  } else {
+    close(fd->fd);
+  }
+
+  fd->orphaned = true;
+
   GRPC_CLOSURE_SCHED(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error));
 
   gpr_mu_unlock(&fd->po.mu);
@@ -937,12 +932,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure, "read");
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure);
+  grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure, "write");
 }
 
 /*******************************************************************************
@@ -1079,7 +1074,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->po.mu);
   *mu = &pollset->po.mu;
   pollset->po.pi = NULL;
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   pollset->po.obj_type = POLL_OBJ_POLLSET;
 #endif
 
@@ -1119,7 +1114,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  grpc_lfev_set_ready(exec_ctx, &fd->read_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->read_closure, "read");
 
   /* Note, it is possible that fd_become_readable might be called twice with
      different 'notifier's when an fd becomes readable and it is in two epoll
@@ -1131,7 +1126,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  grpc_lfev_set_ready(exec_ctx, &fd->write_closure);
+  grpc_lfev_set_ready(exec_ctx, &fd->write_closure, "write");
 }
 
 static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx,
@@ -1416,7 +1411,7 @@ static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag,
                             poll_obj_type item_type) {
   GPR_TIMER_BEGIN("add_poll_object", 0);
 
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   GPR_ASSERT(item->obj_type == item_type);
   GPR_ASSERT(bag->obj_type == bag_type);
 #endif
@@ -1575,7 +1570,7 @@ static grpc_pollset_set *pollset_set_create(void) {
   grpc_pollset_set *pss = gpr_malloc(sizeof(*pss));
   gpr_mu_init(&pss->po.mu);
   pss->po.pi = NULL;
-#ifdef PO_DEBUG
+#ifndef NDEBUG
   pss->po.obj_type = POLL_OBJ_POLLSET_SET;
 #endif
   return pss;
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index 1f8d7eef26..365aa583bb 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -398,11 +398,14 @@ static int fd_wrapped_fd(grpc_fd *fd) {
 
 static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                       grpc_closure *on_done, int *release_fd,
-                      const char *reason) {
+                      bool already_closed, const char *reason) {
   fd->on_done_closure = on_done;
   fd->released = release_fd != NULL;
-  if (fd->released) {
+  if (release_fd != NULL) {
     *release_fd = fd->fd;
+    fd->released = true;
+  } else if (already_closed) {
+    fd->released = true;
   }
   gpr_mu_lock(&fd->mu);
   REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index 2648df393d..91f8cd5482 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -39,10 +39,11 @@
 #include "src/core/lib/support/env.h"
 
 grpc_tracer_flag grpc_polling_trace =
-    GRPC_TRACER_INITIALIZER(false); /* Disabled by default */
+    GRPC_TRACER_INITIALIZER(false, "polling"); /* Disabled by default */
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_fd_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_fd_refcount =
+    GRPC_TRACER_INITIALIZER(false, "fd_refcount");
 #endif
 
 /** Default poll() function - a pointer so that it can be overridden by some
@@ -120,11 +121,15 @@ void grpc_set_event_engine_test_only(
   g_event_engine = ev_engine;
 }
 
+const grpc_event_engine_vtable *grpc_get_event_engine_test_only() {
+  return g_event_engine;
+}
+
 /* Call this only after calling grpc_event_engine_init() */
 const char *grpc_get_poll_strategy_name() { return g_poll_strategy_name; }
 
 void grpc_event_engine_init(void) {
-  grpc_register_tracer("polling", &grpc_polling_trace);
+  grpc_register_tracer(&grpc_polling_trace);
 
   char *s = gpr_getenv("GRPC_POLL_STRATEGY");
   if (s == NULL) {
@@ -165,8 +170,9 @@ int grpc_fd_wrapped_fd(grpc_fd *fd) {
 }
 
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason) {
-  g_event_engine->fd_orphan(exec_ctx, fd, on_done, release_fd, reason);
+                    int *release_fd, bool already_closed, const char *reason) {
+  g_event_engine->fd_orphan(exec_ctx, fd, on_done, release_fd, already_closed,
+                            reason);
 }
 
 void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 54c4f2ee11..1108e46ef8 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -37,7 +37,7 @@ typedef struct grpc_event_engine_vtable {
   grpc_fd *(*fd_create)(int fd, const char *name);
   int (*fd_wrapped_fd)(grpc_fd *fd);
   void (*fd_orphan)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason);
+                    int *release_fd, bool already_closed, const char *reason);
   void (*fd_shutdown)(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why);
   void (*fd_notify_on_read)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                             grpc_closure *closure);
@@ -104,7 +104,7 @@ int grpc_fd_wrapped_fd(grpc_fd *fd);
    notify_on_write.
    MUST NOT be called with a pollset lock taken */
 void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
-                    int *release_fd, const char *reason);
+                    int *release_fd, bool already_closed, const char *reason);
 
 /* Has grpc_fd_shutdown been called on an fd? */
 bool grpc_fd_is_shutdown(grpc_fd *fd);
@@ -153,7 +153,9 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
 typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int);
 extern grpc_poll_function_type grpc_poll_function;
 
-/* This should be used for testing purposes ONLY */
+/* WARNING: The following two functions should be used for testing purposes
+ * ONLY */
 void grpc_set_event_engine_test_only(const grpc_event_engine_vtable *);
+const grpc_event_engine_vtable *grpc_get_event_engine_test_only();
 
 #endif /* GRPC_CORE_LIB_IOMGR_EV_POSIX_H */
diff --git a/src/core/lib/iomgr/ev_windows.c b/src/core/lib/iomgr/ev_windows.c
index 027609c7e8..c24dfaeaf7 100644
--- a/src/core/lib/iomgr/ev_windows.c
+++ b/src/core/lib/iomgr/ev_windows.c
@@ -23,6 +23,6 @@
 #include "src/core/lib/debug/trace.h"
 
 grpc_tracer_flag grpc_polling_trace =
-    GRPC_TRACER_INITIALIZER(false); /* Disabled by default */
+    GRPC_TRACER_INITIALIZER(false, "polling"); /* Disabled by default */
 
 #endif  // GRPC_WINSOCK_SOCKET
diff --git a/src/core/lib/iomgr/iomgr_posix.c b/src/core/lib/iomgr/iomgr_posix.c
index 43f5d0406e..f5875a247e 100644
--- a/src/core/lib/iomgr/iomgr_posix.c
+++ b/src/core/lib/iomgr/iomgr_posix.c
@@ -28,7 +28,7 @@
 void grpc_iomgr_platform_init(void) {
   grpc_wakeup_fd_global_init();
   grpc_event_engine_init();
-  grpc_register_tracer("tcp", &grpc_tcp_trace);
+  grpc_register_tracer(&grpc_tcp_trace);
 }
 
 void grpc_iomgr_platform_flush(void) {}
diff --git a/src/core/lib/iomgr/iomgr_uv.c b/src/core/lib/iomgr/iomgr_uv.c
index 49d1a03c32..df5d23af3b 100644
--- a/src/core/lib/iomgr/iomgr_uv.c
+++ b/src/core/lib/iomgr/iomgr_uv.c
@@ -21,12 +21,20 @@
 #ifdef GRPC_UV
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/pollset_uv.h"
 #include "src/core/lib/iomgr/tcp_uv.h"
 
+gpr_thd_id g_init_thread;
+
 void grpc_iomgr_platform_init(void) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_pollset_global_init();
-  grpc_register_tracer("tcp", &grpc_tcp_trace);
+  grpc_register_tracer(&grpc_tcp_trace);
+  grpc_executor_set_threading(&exec_ctx, false);
+  g_init_thread = gpr_thd_currentid();
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 void grpc_iomgr_platform_flush(void) {}
 void grpc_iomgr_platform_shutdown(void) { grpc_pollset_global_shutdown(); }
diff --git a/src/core/lib/iomgr/iomgr_uv.h b/src/core/lib/iomgr/iomgr_uv.h
new file mode 100644
index 0000000000..3b4daaa73b
--- /dev/null
+++ b/src/core/lib/iomgr/iomgr_uv.h
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_IOMGR_UV_H
+#define GRPC_CORE_LIB_IOMGR_IOMGR_UV_H
+
+#include "src/core/lib/iomgr/iomgr_internal.h"
+
+#include <grpc/support/thd.h>
+
+/* The thread ID of the thread on which grpc was initialized. Used to verify
+ * that all calls into libuv are made on that same thread */
+extern gpr_thd_id g_init_thread;
+
+#ifdef GRPC_UV_THREAD_CHECK
+#define GRPC_UV_ASSERT_SAME_THREAD() \
+  GPR_ASSERT(gpr_thd_currentid() == g_init_thread)
+#else
+#define GRPC_UV_ASSERT_SAME_THREAD()
+#endif /* GRPC_UV_THREAD_CHECK */
+
+#endif /* GRPC_CORE_LIB_IOMGR_IOMGR_UV_H */
diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c
index c2ceecb3c5..f967b22ba9 100644
--- a/src/core/lib/iomgr/lockfree_event.c
+++ b/src/core/lib/iomgr/lockfree_event.c
@@ -79,12 +79,12 @@ bool grpc_lfev_is_shutdown(gpr_atm *state) {
 }
 
 void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state,
-                         grpc_closure *closure) {
+                         grpc_closure *closure, const char *variable) {
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
-      gpr_log(GPR_DEBUG, "lfev_notify_on: %p curr=%p closure=%p", state,
-              (void *)curr, closure);
+      gpr_log(GPR_ERROR, "lfev_notify_on[%s]: %p curr=%p closure=%p", variable,
+              state, (void *)curr, closure);
     }
     switch (curr) {
       case CLOSURE_NOT_READY: {
@@ -149,7 +149,7 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
-      gpr_log(GPR_DEBUG, "lfev_set_shutdown: %p curr=%p err=%s", state,
+      gpr_log(GPR_ERROR, "lfev_set_shutdown: %p curr=%p err=%s", state,
               (void *)curr, grpc_error_string(shutdown_err));
     }
     switch (curr) {
@@ -193,12 +193,14 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
   GPR_UNREACHABLE_CODE(return false);
 }
 
-void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) {
+void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state,
+                         const char *variable) {
   while (true) {
     gpr_atm curr = gpr_atm_no_barrier_load(state);
 
     if (GRPC_TRACER_ON(grpc_polling_trace)) {
-      gpr_log(GPR_DEBUG, "lfev_set_ready: %p curr=%p", state, (void *)curr);
+      gpr_log(GPR_ERROR, "lfev_set_ready[%s]: %p curr=%p", variable, state,
+              (void *)curr);
     }
 
     switch (curr) {
diff --git a/src/core/lib/iomgr/lockfree_event.h b/src/core/lib/iomgr/lockfree_event.h
index ef3844a07a..6a14a0f3b2 100644
--- a/src/core/lib/iomgr/lockfree_event.h
+++ b/src/core/lib/iomgr/lockfree_event.h
@@ -30,10 +30,11 @@ void grpc_lfev_destroy(gpr_atm *state);
 bool grpc_lfev_is_shutdown(gpr_atm *state);
 
 void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state,
-                         grpc_closure *closure);
+                         grpc_closure *closure, const char *variable);
 /* Returns true on first successful shutdown */
 bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state,
                             grpc_error *shutdown_err);
-void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state);
+void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state,
+                         const char *variable);
 
 #endif /* GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H */
diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c
index 1a54065a91..a79fe89d3e 100644
--- a/src/core/lib/iomgr/pollset_uv.c
+++ b/src/core/lib/iomgr/pollset_uv.c
@@ -28,13 +28,15 @@
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_uv.h"
 
 #include "src/core/lib/debug/trace.h"
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_fd_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_fd_refcount =
+    GRPC_TRACER_INITIALIZER(false, "fd_refcount");
 #endif
 
 struct grpc_pollset {
@@ -69,6 +71,7 @@ void grpc_pollset_global_init(void) {
 }
 
 void grpc_pollset_global_shutdown(void) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   gpr_mu_destroy(&grpc_polling_mu);
   uv_close((uv_handle_t *)dummy_uv_handle, dummy_handle_close_cb);
 }
@@ -78,6 +81,7 @@ static void timer_run_cb(uv_timer_t *timer) {}
 static void timer_close_cb(uv_handle_t *handle) { handle->data = (void *)1; }
 
 void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   *mu = &grpc_polling_mu;
   uv_timer_init(uv_default_loop(), &pollset->timer);
   pollset->shutting_down = 0;
@@ -86,6 +90,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
 void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                            grpc_closure *closure) {
   GPR_ASSERT(!pollset->shutting_down);
+  GRPC_UV_ASSERT_SAME_THREAD();
   pollset->shutting_down = 1;
   if (grpc_pollset_work_run_loop) {
     // Drain any pending UV callbacks without blocking
@@ -98,6 +103,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 }
 
 void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   uv_close((uv_handle_t *)&pollset->timer, timer_close_cb);
   // timer.data is a boolean indicating that the timer has finished closing
   pollset->timer.data = (void *)0;
@@ -112,6 +118,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                               grpc_pollset_worker **worker_hdl,
                               gpr_timespec now, gpr_timespec deadline) {
   uint64_t timeout;
+  GRPC_UV_ASSERT_SAME_THREAD();
   gpr_mu_unlock(&grpc_polling_mu);
   if (grpc_pollset_work_run_loop) {
     if (gpr_time_cmp(deadline, now) >= 0) {
@@ -140,6 +147,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
 grpc_error *grpc_pollset_kick(grpc_pollset *pollset,
                               grpc_pollset_worker *specific_worker) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   uv_timer_start(dummy_uv_handle, dummy_timer_cb, 0, 0);
   return GRPC_ERROR_NONE;
 }
diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c
index 1bfc2a22a8..ea017a6054 100644
--- a/src/core/lib/iomgr/pollset_windows.c
+++ b/src/core/lib/iomgr/pollset_windows.c
@@ -31,7 +31,8 @@
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_fd_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_fd_refcount =
+    GRPC_TRACER_INITIALIZER(false, "fd_refcount");
 #endif
 
 gpr_mu grpc_polling_mu;
diff --git a/src/core/lib/iomgr/resolve_address_uv.c b/src/core/lib/iomgr/resolve_address_uv.c
index 45de289e45..2d438e8b48 100644
--- a/src/core/lib/iomgr/resolve_address_uv.c
+++ b/src/core/lib/iomgr/resolve_address_uv.c
@@ -30,6 +30,7 @@
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -54,7 +55,7 @@ static int retry_named_port_failure(int status, request *r,
         int retry_status;
         uv_getaddrinfo_t *req = gpr_malloc(sizeof(uv_getaddrinfo_t));
         req->data = r;
-        r->port = svc[i][1];
+        r->port = gpr_strdup(svc[i][1]);
         retry_status = uv_getaddrinfo(uv_default_loop(), req, getaddrinfo_cb,
                                       r->host, r->port, r->hints);
         if (retry_status < 0 || getaddrinfo_cb == NULL) {
@@ -114,11 +115,14 @@ static void getaddrinfo_callback(uv_getaddrinfo_t *req, int status,
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_error *error;
   int retry_status;
+  char *port = r->port;
 
   gpr_free(req);
   retry_status = retry_named_port_failure(status, r, getaddrinfo_callback);
   if (retry_status == 0) {
-    // The request is being retried. Nothing should be done here
+    /* The request is being retried. It is using its own port string, so we free
+     * the original one */
+    gpr_free(port);
     return;
   }
   /* Either no retry was attempted, or the retry failed. Either way, the
@@ -127,6 +131,8 @@ static void getaddrinfo_callback(uv_getaddrinfo_t *req, int status,
   GRPC_CLOSURE_SCHED(&exec_ctx, r->on_done, error);
   grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(r->hints);
+  gpr_free(r->host);
+  gpr_free(r->port);
   gpr_free(r);
   uv_freeaddrinfo(res);
 }
@@ -169,6 +175,8 @@ static grpc_error *blocking_resolve_address_impl(
   grpc_error *err;
   int retry_status;
 
+  GRPC_UV_ASSERT_SAME_THREAD();
+
   req.addrinfo = NULL;
 
   err = try_split_host_port(name, default_port, &host, &port);
@@ -216,16 +224,19 @@ static void resolve_address_impl(grpc_exec_ctx *exec_ctx, const char *name,
                                  grpc_pollset_set *interested_parties,
                                  grpc_closure *on_done,
                                  grpc_resolved_addresses **addrs) {
-  uv_getaddrinfo_t *req;
-  request *r;
-  struct addrinfo *hints;
-  char *host;
-  char *port;
+  uv_getaddrinfo_t *req = NULL;
+  request *r = NULL;
+  struct addrinfo *hints = NULL;
+  char *host = NULL;
+  char *port = NULL;
   grpc_error *err;
   int s;
+  GRPC_UV_ASSERT_SAME_THREAD();
   err = try_split_host_port(name, default_port, &host, &port);
   if (err != GRPC_ERROR_NONE) {
     GRPC_CLOSURE_SCHED(exec_ctx, on_done, err);
+    gpr_free(host);
+    gpr_free(port);
     return;
   }
   r = gpr_malloc(sizeof(request));
diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c
index f2cc1be74e..a31d9eef93 100644
--- a/src/core/lib/iomgr/resource_quota.c
+++ b/src/core/lib/iomgr/resource_quota.c
@@ -29,7 +29,8 @@
 
 #include "src/core/lib/iomgr/combiner.h"
 
-grpc_tracer_flag grpc_resource_quota_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_resource_quota_trace =
+    GRPC_TRACER_INITIALIZER(false, "resource_quota");
 
 #define MEMORY_USAGE_ESTIMATION_MAX 65536
 
diff --git a/src/core/lib/iomgr/sockaddr_utils.c b/src/core/lib/iomgr/sockaddr_utils.c
index 99dc2f1c78..3f4145d104 100644
--- a/src/core/lib/iomgr/sockaddr_utils.c
+++ b/src/core/lib/iomgr/sockaddr_utils.c
@@ -220,6 +220,11 @@ const char *grpc_sockaddr_get_uri_scheme(
   return NULL;
 }
 
+int grpc_sockaddr_get_family(const grpc_resolved_address *resolved_addr) {
+  const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr;
+  return addr->sa_family;
+}
+
 int grpc_sockaddr_get_port(const grpc_resolved_address *resolved_addr) {
   const struct sockaddr *addr = (const struct sockaddr *)resolved_addr->addr;
   switch (addr->sa_family) {
diff --git a/src/core/lib/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h
index 7692b969f2..a589a19705 100644
--- a/src/core/lib/iomgr/sockaddr_utils.h
+++ b/src/core/lib/iomgr/sockaddr_utils.h
@@ -75,4 +75,6 @@ char *grpc_sockaddr_to_uri(const grpc_resolved_address *addr);
 /* Returns the URI scheme corresponding to \a addr */
 const char *grpc_sockaddr_get_uri_scheme(const grpc_resolved_address *addr);
 
+int grpc_sockaddr_get_family(const grpc_resolved_address *resolved_addr);
+
 #endif /* GRPC_CORE_LIB_IOMGR_SOCKADDR_UTILS_H */
diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c
index 21e320a6e7..a25fba4527 100644
--- a/src/core/lib/iomgr/tcp_client_posix.c
+++ b/src/core/lib/iomgr/tcp_client_posix.c
@@ -209,7 +209,8 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
 finish:
   if (fd != NULL) {
     grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd);
-    grpc_fd_orphan(exec_ctx, fd, NULL, NULL, "tcp_client_orphan");
+    grpc_fd_orphan(exec_ctx, fd, NULL, NULL, false /* already_closed */,
+                   "tcp_client_orphan");
     fd = NULL;
   }
   done = (--ac->refs == 0);
@@ -295,7 +296,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   }
 
   if (errno != EWOULDBLOCK && errno != EINPROGRESS) {
-    grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, "tcp_client_connect_error");
+    grpc_fd_orphan(exec_ctx, fdobj, NULL, NULL, false /* already_closed */,
+                   "tcp_client_connect_error");
     GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_OS_ERROR(errno, "connect"));
     goto done;
   }
diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c
index ab6832932f..786c456b73 100644
--- a/src/core/lib/iomgr/tcp_client_uv.c
+++ b/src/core/lib/iomgr/tcp_client_uv.c
@@ -26,6 +26,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/tcp_client.h"
 #include "src/core/lib/iomgr/tcp_uv.h"
@@ -48,6 +49,7 @@ typedef struct grpc_uv_tcp_connect {
 static void uv_tcp_connect_cleanup(grpc_exec_ctx *exec_ctx,
                                    grpc_uv_tcp_connect *connect) {
   grpc_resource_quota_unref_internal(exec_ctx, connect->resource_quota);
+  gpr_free(connect->addr_name);
   gpr_free(connect);
 }
 
@@ -105,6 +107,7 @@ static void uv_tc_on_connect(uv_connect_t *req, int status) {
   }
   done = (--connect->refs == 0);
   if (done) {
+    grpc_exec_ctx_flush(&exec_ctx);
     uv_tcp_connect_cleanup(&exec_ctx, connect);
   }
   GRPC_CLOSURE_SCHED(&exec_ctx, closure, error);
@@ -122,6 +125,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   (void)channel_args;
   (void)interested_parties;
 
+  GRPC_UV_ASSERT_SAME_THREAD();
+
   if (channel_args != NULL) {
     for (size_t i = 0; i < channel_args->num_args; i++) {
       if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
@@ -140,6 +145,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   connect->resource_quota = resource_quota;
   uv_tcp_init(uv_default_loop(), connect->tcp_handle);
   connect->connect_req.data = connect;
+  connect->refs = 1;
 
   if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting",
diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c
index 5de2b0f4ee..2f543fd8a9 100644
--- a/src/core/lib/iomgr/tcp_posix.c
+++ b/src/core/lib/iomgr/tcp_posix.c
@@ -59,7 +59,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type;
 typedef size_t msg_iovlen_type;
 #endif
 
-grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false, "tcp");
 
 typedef struct {
   grpc_endpoint base;
@@ -156,7 +156,7 @@ static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
 static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   grpc_fd_orphan(exec_ctx, tcp->em_fd, tcp->release_fd_cb, tcp->release_fd,
-                 "tcp_unref_orphan");
+                 false /* already_closed */, "tcp_unref_orphan");
   grpc_slice_buffer_destroy_internal(exec_ctx, &tcp->last_read_buffer);
   grpc_resource_user_unref(exec_ctx, tcp->resource_user);
   gpr_free(tcp->peer_string);
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index f304642951..0fc5c0fd86 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -166,7 +166,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
       GRPC_CLOSURE_INIT(&sp->destroyed_closure, destroyed_port, s,
                         grpc_schedule_on_exec_ctx);
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
-                     "tcp_listener_shutdown");
+                     false /* already_closed */, "tcp_listener_shutdown");
     }
     gpr_mu_unlock(&s->mu);
   } else {
diff --git a/src/core/lib/iomgr/tcp_server_uv.c b/src/core/lib/iomgr/tcp_server_uv.c
index 2de0ea90e7..3b9332321f 100644
--- a/src/core/lib/iomgr/tcp_server_uv.c
+++ b/src/core/lib/iomgr/tcp_server_uv.c
@@ -20,6 +20,7 @@
 
 #ifdef GRPC_UV
 
+#include <assert.h>
 #include <string.h>
 
 #include <grpc/support/alloc.h>
@@ -27,6 +28,7 @@
 
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/tcp_server.h"
@@ -43,6 +45,8 @@ struct grpc_tcp_listener {
   struct grpc_tcp_listener *next;
 
   bool closed;
+
+  bool has_pending_connection;
 };
 
 struct grpc_tcp_server {
@@ -104,6 +108,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
 }
 
 grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   gpr_ref(&s->refs);
   return s;
 }
@@ -168,6 +173,7 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 }
 
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   if (gpr_unref(&s->refs)) {
     /* Complete shutdown_starting work before destroying. */
     grpc_exec_ctx local_exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -183,18 +189,49 @@ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   }
 }
 
-static void accepted_connection_close_cb(uv_handle_t *handle) {
-  gpr_free(handle);
-}
-
-static void on_connect(uv_stream_t *server, int status) {
-  grpc_tcp_listener *sp = (grpc_tcp_listener *)server->data;
+static void finish_accept(grpc_exec_ctx *exec_ctx, grpc_tcp_listener *sp) {
+  grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
   uv_tcp_t *client;
   grpc_endpoint *ep = NULL;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_resolved_address peer_name;
   char *peer_name_string;
   int err;
+  uv_tcp_t *server = sp->handle;
+
+  client = gpr_malloc(sizeof(uv_tcp_t));
+  uv_tcp_init(uv_default_loop(), client);
+  // UV documentation says this is guaranteed to succeed
+  uv_accept((uv_stream_t *)server, (uv_stream_t *)client);
+  peer_name_string = NULL;
+  memset(&peer_name, 0, sizeof(grpc_resolved_address));
+  peer_name.len = sizeof(struct sockaddr_storage);
+  err = uv_tcp_getpeername(client, (struct sockaddr *)&peer_name.addr,
+                           (int *)&peer_name.len);
+  if (err == 0) {
+    peer_name_string = grpc_sockaddr_to_uri(&peer_name);
+  } else {
+    gpr_log(GPR_INFO, "uv_tcp_getpeername error: %s", uv_strerror(err));
+  }
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    if (peer_name_string) {
+      gpr_log(GPR_DEBUG, "SERVER_CONNECT: %p accepted connection: %s",
+              sp->server, peer_name_string);
+    } else {
+      gpr_log(GPR_DEBUG, "SERVER_CONNECT: %p accepted connection", sp->server);
+    }
+  }
+  ep = grpc_tcp_create(client, sp->server->resource_quota, peer_name_string);
+  acceptor->from_server = sp->server;
+  acceptor->port_index = sp->port_index;
+  acceptor->fd_index = 0;
+  sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
+                           acceptor);
+  gpr_free(peer_name_string);
+}
+
+static void on_connect(uv_stream_t *server, int status) {
+  grpc_tcp_listener *sp = (grpc_tcp_listener *)server->data;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
   if (status < 0) {
     switch (status) {
@@ -207,34 +244,19 @@ static void on_connect(uv_stream_t *server, int status) {
     }
   }
 
-  client = gpr_malloc(sizeof(uv_tcp_t));
-  uv_tcp_init(uv_default_loop(), client);
-  // UV documentation says this is guaranteed to succeed
-  uv_accept((uv_stream_t *)server, (uv_stream_t *)client);
-  // If the server has not been started, we discard incoming connections
-  if (sp->server->on_accept_cb == NULL) {
-    uv_close((uv_handle_t *)client, accepted_connection_close_cb);
+  GPR_ASSERT(!sp->has_pending_connection);
+
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    gpr_log(GPR_DEBUG, "SERVER_CONNECT: %p incoming connection", sp->server);
+  }
+
+  // Create acceptor.
+  if (sp->server->on_accept_cb) {
+    finish_accept(&exec_ctx, sp);
   } else {
-    peer_name_string = NULL;
-    memset(&peer_name, 0, sizeof(grpc_resolved_address));
-    peer_name.len = sizeof(struct sockaddr_storage);
-    err = uv_tcp_getpeername(client, (struct sockaddr *)&peer_name.addr,
-                             (int *)&peer_name.len);
-    if (err == 0) {
-      peer_name_string = grpc_sockaddr_to_uri(&peer_name);
-    } else {
-      gpr_log(GPR_INFO, "uv_tcp_getpeername error: %s", uv_strerror(status));
-    }
-    ep = grpc_tcp_create(client, sp->server->resource_quota, peer_name_string);
-    // Create acceptor.
-    grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
-    acceptor->from_server = sp->server;
-    acceptor->port_index = sp->port_index;
-    acceptor->fd_index = 0;
-    sp->server->on_accept_cb(&exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
-                             acceptor);
-    grpc_exec_ctx_finish(&exec_ctx);
+    sp->has_pending_connection = true;
   }
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static grpc_error *add_socket_to_server(grpc_tcp_server *s, uv_tcp_t *handle,
@@ -281,7 +303,7 @@ static grpc_error *add_socket_to_server(grpc_tcp_server *s, uv_tcp_t *handle,
 
   GPR_ASSERT(port >= 0);
   GPR_ASSERT(!s->on_accept_cb && "must add ports before starting server");
-  sp = gpr_malloc(sizeof(grpc_tcp_listener));
+  sp = gpr_zalloc(sizeof(grpc_tcp_listener));
   sp->next = NULL;
   if (s->head == NULL) {
     s->head = sp;
@@ -315,6 +337,9 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
   unsigned port_index = 0;
   int status;
   grpc_error *error = GRPC_ERROR_NONE;
+  int family;
+
+  GRPC_UV_ASSERT_SAME_THREAD();
 
   if (s->tail != NULL) {
     port_index = s->tail->port_index + 1;
@@ -352,7 +377,18 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
   }
 
   handle = gpr_malloc(sizeof(uv_tcp_t));
-  status = uv_tcp_init(uv_default_loop(), handle);
+
+  family = grpc_sockaddr_get_family(addr);
+  status = uv_tcp_init_ex(uv_default_loop(), handle, (unsigned int)family);
+#if defined(GPR_LINUX) && defined(SO_REUSEPORT)
+  if (family == AF_INET || family == AF_INET6) {
+    int fd;
+    uv_fileno((uv_handle_t *)handle, &fd);
+    int enable = 1;
+    setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(enable));
+  }
+#endif /* GPR_LINUX && SO_REUSEPORT */
+
   if (status == 0) {
     error = add_socket_to_server(s, handle, addr, port_index, &sp);
   } else {
@@ -365,6 +401,18 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
 
   gpr_free(allocated_addr);
 
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    char *port_string;
+    grpc_sockaddr_to_string(&port_string, addr, 0);
+    const char *str = grpc_error_string(error);
+    if (port_string) {
+      gpr_log(GPR_DEBUG, "SERVER %p add_port %s error=%s", s, port_string, str);
+      gpr_free(port_string);
+    } else {
+      gpr_log(GPR_DEBUG, "SERVER %p add_port error=%s", s, str);
+    }
+  }
+
   if (error != GRPC_ERROR_NONE) {
     grpc_error *error_out = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
         "Failed to add port to server", &error, 1);
@@ -384,13 +432,19 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *server,
   grpc_tcp_listener *sp;
   (void)pollsets;
   (void)pollset_count;
+  GRPC_UV_ASSERT_SAME_THREAD();
+  if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+    gpr_log(GPR_DEBUG, "SERVER_START %p", server);
+  }
   GPR_ASSERT(on_accept_cb);
   GPR_ASSERT(!server->on_accept_cb);
   server->on_accept_cb = on_accept_cb;
   server->on_accept_cb_arg = cb_arg;
   for (sp = server->head; sp; sp = sp->next) {
-    GPR_ASSERT(uv_listen((uv_stream_t *)sp->handle, SOMAXCONN, on_connect) ==
-               0);
+    if (sp->has_pending_connection) {
+      finish_accept(exec_ctx, sp);
+      sp->has_pending_connection = false;
+    }
   }
 }
 
diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c
index 7c21b44e76..a05c19b4ac 100644
--- a/src/core/lib/iomgr/tcp_uv.c
+++ b/src/core/lib/iomgr/tcp_uv.c
@@ -30,6 +30,7 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/network_status_tracker.h"
 #include "src/core/lib/iomgr/resource_quota.h"
 #include "src/core/lib/iomgr/tcp_uv.h"
@@ -37,7 +38,7 @@
 #include "src/core/lib/slice/slice_string_helpers.h"
 #include "src/core/lib/support/string.h"
 
-grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false, "tcp");
 
 typedef struct {
   grpc_endpoint base;
@@ -65,7 +66,10 @@ typedef struct {
 } grpc_tcp;
 
 static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
+  grpc_slice_unref_internal(exec_ctx, tcp->read_slice);
   grpc_resource_user_unref(exec_ctx, tcp->resource_user);
+  gpr_free(tcp->handle);
+  gpr_free(tcp->peer_string);
   gpr_free(tcp);
 }
 
@@ -115,13 +119,17 @@ static void uv_close_callback(uv_handle_t *handle) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
+static grpc_slice alloc_read_slice(grpc_exec_ctx *exec_ctx,
+                                   grpc_resource_user *resource_user) {
+  return grpc_resource_user_slice_malloc(exec_ctx, resource_user,
+                                         GRPC_TCP_DEFAULT_READ_SLICE_SIZE);
+}
+
 static void alloc_uv_buf(uv_handle_t *handle, size_t suggested_size,
                          uv_buf_t *buf) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_tcp *tcp = handle->data;
   (void)suggested_size;
-  tcp->read_slice = grpc_resource_user_slice_malloc(
-      &exec_ctx, tcp->resource_user, GRPC_TCP_DEFAULT_READ_SLICE_SIZE);
   buf->base = (char *)GRPC_SLICE_START_PTR(tcp->read_slice);
   buf->len = GRPC_SLICE_LENGTH(tcp->read_slice);
   grpc_exec_ctx_finish(&exec_ctx);
@@ -148,6 +156,7 @@ static void read_callback(uv_stream_t *stream, ssize_t nread,
     // Successful read
     sub = grpc_slice_sub_no_ref(tcp->read_slice, 0, (size_t)nread);
     grpc_slice_buffer_add(tcp->read_slices, sub);
+    tcp->read_slice = alloc_read_slice(&exec_ctx, tcp->resource_user);
     error = GRPC_ERROR_NONE;
     if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       size_t i;
@@ -175,6 +184,7 @@ static void uv_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   grpc_tcp *tcp = (grpc_tcp *)ep;
   int status;
   grpc_error *error = GRPC_ERROR_NONE;
+  GRPC_UV_ASSERT_SAME_THREAD();
   GPR_ASSERT(tcp->read_cb == NULL);
   tcp->read_cb = cb;
   tcp->read_slices = read_slices;
@@ -228,6 +238,7 @@ static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   unsigned int i;
   grpc_slice *slice;
   uv_write_t *write_req;
+  GRPC_UV_ASSERT_SAME_THREAD();
 
   if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     size_t j;
@@ -299,6 +310,10 @@ static void uv_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                                  grpc_error *why) {
   grpc_tcp *tcp = (grpc_tcp *)ep;
   if (!tcp->shutting_down) {
+    if (GRPC_TRACER_ON(grpc_tcp_trace)) {
+      const char *str = grpc_error_string(why);
+      gpr_log(GPR_DEBUG, "TCP %p shutdown why=%s", tcp->handle, str);
+    }
     tcp->shutting_down = true;
     uv_shutdown_t *req = &tcp->shutdown_req;
     uv_shutdown(req, (uv_stream_t *)tcp->handle, shutdown_callback);
@@ -334,6 +349,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle,
                                grpc_resource_quota *resource_quota,
                                char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp));
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
   if (GRPC_TRACER_ON(grpc_tcp_trace)) {
     gpr_log(GPR_DEBUG, "Creating TCP endpoint %p", tcp);
@@ -350,6 +366,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle,
   tcp->peer_string = gpr_strdup(peer_string);
   tcp->shutting_down = false;
   tcp->resource_user = grpc_resource_user_create(resource_quota, peer_string);
+  tcp->read_slice = alloc_read_slice(&exec_ctx, tcp->resource_user);
   /* Tell network status tracking code about the new endpoint */
   grpc_network_status_register_endpoint(&tcp->base);
 
@@ -357,6 +374,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle,
   uv_unref((uv_handle_t *)handle);
 #endif
 
+  grpc_exec_ctx_finish(&exec_ctx);
   return &tcp->base;
 }
 
diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c
index 6704a158ce..2cbb97403b 100644
--- a/src/core/lib/iomgr/tcp_windows.c
+++ b/src/core/lib/iomgr/tcp_windows.c
@@ -48,7 +48,7 @@
 #define GRPC_FIONBIO FIONBIO
 #endif
 
-grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false, "tcp");
 
 static grpc_error *set_non_block(SOCKET sock) {
   int status;
diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c
index bf73d2c685..12efce241f 100644
--- a/src/core/lib/iomgr/timer_generic.c
+++ b/src/core/lib/iomgr/timer_generic.c
@@ -41,44 +41,67 @@
 #define MIN_QUEUE_WINDOW_DURATION 0.01
 #define MAX_QUEUE_WINDOW_DURATION 1
 
-grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false);
-grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false);
-
+grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false, "timer");
+grpc_tracer_flag grpc_timer_check_trace =
+    GRPC_TRACER_INITIALIZER(false, "timer_check");
+
+/* A "timer shard". Contains a 'heap' and a 'list' of timers. All timers with
+ * deadlines earlier than 'queue_deadline" cap are maintained in the heap and
+ * others are maintained in the list (unordered). This helps to keep the number
+ * of elements in the heap low.
+ *
+ * The 'queue_deadline_cap' gets recomputed periodically based on the timer
+ * stats maintained in 'stats' and the relevant timers are then moved from the
+ * 'list' to 'heap'
+ */
 typedef struct {
   gpr_mu mu;
   grpc_time_averaged_stats stats;
   /* All and only timers with deadlines <= this will be in the heap. */
   gpr_atm queue_deadline_cap;
+  /* The deadline of the next timer due in this shard */
   gpr_atm min_deadline;
-  /* Index in the g_shard_queue */
+  /* Index of this timer_shard in the g_shard_queue */
   uint32_t shard_queue_index;
   /* This holds all timers with deadlines < queue_deadline_cap. Timers in this
      list have the top bit of their deadline set to 0. */
   grpc_timer_heap heap;
   /* This holds timers whose deadline is >= queue_deadline_cap. */
   grpc_timer list;
-} shard_type;
+} timer_shard;
+
+/* Array of timer shards. Whenever a timer (grpc_timer *) is added, its address
+ * is hashed to select the timer shard to add the timer to */
+static timer_shard g_shards[NUM_SHARDS];
+
+/* Maintains a sorted list of timer shards (sorted by their min_deadline, i.e
+ * the deadline of the next timer in each shard).
+ * Access to this is protected by g_shared_mutables.mu */
+static timer_shard *g_shard_queue[NUM_SHARDS];
+
+/* Thread local variable that stores the deadline of the next timer the thread
+ * has last-seen. This is an optimization to prevent the thread from checking
+ * shared_mutables.min_timer (which requires acquiring shared_mutables.mu lock,
+ * an expensive operation) */
+GPR_TLS_DECL(g_last_seen_min_timer);
 
 struct shared_mutables {
+  /* The deadline of the next timer due across all timer shards */
   gpr_atm min_timer;
   /* Allow only one run_some_expired_timers at once */
   gpr_spinlock checker_mu;
   bool initialized;
-  /* Protects g_shard_queue */
+  /* Protects g_shard_queue (and the shared_mutables struct itself) */
   gpr_mu mu;
 } GPR_ALIGN_STRUCT(GPR_CACHELINE_SIZE);
 
 static struct shared_mutables g_shared_mutables = {
     .checker_mu = GPR_SPINLOCK_STATIC_INITIALIZER, .initialized = false,
 };
+
 static gpr_clock_type g_clock_type;
-static shard_type g_shards[NUM_SHARDS];
-/* Protected by g_shared_mutables.mu */
-static shard_type *g_shard_queue[NUM_SHARDS];
 static gpr_timespec g_start_time;
 
-GPR_TLS_DECL(g_last_seen_min_timer);
-
 static gpr_atm saturating_add(gpr_atm a, gpr_atm b) {
   if (a > GPR_ATM_MAX - b) {
     return GPR_ATM_MAX;
@@ -122,7 +145,7 @@ static gpr_timespec atm_to_timespec(gpr_atm x) {
   return gpr_time_add(g_start_time, dbl_to_ts((double)x / 1000.0));
 }
 
-static gpr_atm compute_min_deadline(shard_type *shard) {
+static gpr_atm compute_min_deadline(timer_shard *shard) {
   return grpc_timer_heap_is_empty(&shard->heap)
              ? saturating_add(shard->queue_deadline_cap, 1)
              : grpc_timer_heap_top(&shard->heap)->deadline;
@@ -138,11 +161,11 @@ void grpc_timer_list_init(gpr_timespec now) {
   g_shared_mutables.min_timer = timespec_to_atm_round_down(now);
   gpr_tls_init(&g_last_seen_min_timer);
   gpr_tls_set(&g_last_seen_min_timer, 0);
-  grpc_register_tracer("timer", &grpc_timer_trace);
-  grpc_register_tracer("timer_check", &grpc_timer_check_trace);
+  grpc_register_tracer(&grpc_timer_trace);
+  grpc_register_tracer(&grpc_timer_check_trace);
 
   for (i = 0; i < NUM_SHARDS; i++) {
-    shard_type *shard = &g_shards[i];
+    timer_shard *shard = &g_shards[i];
     gpr_mu_init(&shard->mu);
     grpc_time_averaged_stats_init(&shard->stats, 1.0 / ADD_DEADLINE_SCALE, 0.1,
                                   0.5);
@@ -161,7 +184,7 @@ void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {
       exec_ctx, GPR_ATM_MAX, NULL,
       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Timer list shutdown"));
   for (i = 0; i < NUM_SHARDS; i++) {
-    shard_type *shard = &g_shards[i];
+    timer_shard *shard = &g_shards[i];
     gpr_mu_destroy(&shard->mu);
     grpc_timer_heap_destroy(&shard->heap);
   }
@@ -187,7 +210,7 @@ static void list_remove(grpc_timer *timer) {
 }
 
 static void swap_adjacent_shards_in_queue(uint32_t first_shard_queue_index) {
-  shard_type *temp;
+  timer_shard *temp;
   temp = g_shard_queue[first_shard_queue_index];
   g_shard_queue[first_shard_queue_index] =
       g_shard_queue[first_shard_queue_index + 1];
@@ -198,7 +221,7 @@ static void swap_adjacent_shards_in_queue(uint32_t first_shard_queue_index) {
       first_shard_queue_index + 1;
 }
 
-static void note_deadline_change(shard_type *shard) {
+static void note_deadline_change(timer_shard *shard) {
   while (shard->shard_queue_index > 0 &&
          shard->min_deadline <
              g_shard_queue[shard->shard_queue_index - 1]->min_deadline) {
@@ -215,7 +238,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
                      gpr_timespec deadline, grpc_closure *closure,
                      gpr_timespec now) {
   int is_first_timer = 0;
-  shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)];
+  timer_shard *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)];
   GPR_ASSERT(deadline.clock_type == g_clock_type);
   GPR_ASSERT(now.clock_type == g_clock_type);
   timer->closure = closure;
@@ -303,7 +326,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
     return;
   }
 
-  shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)];
+  timer_shard *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)];
   gpr_mu_lock(&shard->mu);
   if (GRPC_TRACER_ON(grpc_timer_trace)) {
     gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer,
@@ -321,12 +344,12 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
   gpr_mu_unlock(&shard->mu);
 }
 
-/* This is called when the queue is empty and "now" has reached the
-   queue_deadline_cap.  We compute a new queue deadline and then scan the map
-   for timers that fall at or under it.  Returns true if the queue is no
-   longer empty.
+/* Rebalances the timer shard by computing a new 'queue_deadline_cap' and moving
+   all relevant timers in shard->list (i.e timers with deadlines earlier than
+   'queue_deadline_cap') into into shard->heap.
+   Returns 'true' if shard->heap has atleast ONE element
    REQUIRES: shard->mu locked */
-static int refill_queue(shard_type *shard, gpr_atm now) {
+static int refill_heap(timer_shard *shard, gpr_atm now) {
   /* Compute the new queue window width and bound by the limits: */
   double computed_deadline_delta =
       grpc_time_averaged_stats_update_average(&shard->stats) *
@@ -363,7 +386,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) {
 /* This pops the next non-cancelled timer with deadline <= now from the
    queue, or returns NULL if there isn't one.
    REQUIRES: shard->mu locked */
-static grpc_timer *pop_one(shard_type *shard, gpr_atm now) {
+static grpc_timer *pop_one(timer_shard *shard, gpr_atm now) {
   grpc_timer *timer;
   for (;;) {
     if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
@@ -373,7 +396,7 @@ static grpc_timer *pop_one(shard_type *shard, gpr_atm now) {
     }
     if (grpc_timer_heap_is_empty(&shard->heap)) {
       if (now < shard->queue_deadline_cap) return NULL;
-      if (!refill_queue(shard, now)) return NULL;
+      if (!refill_heap(shard, now)) return NULL;
     }
     timer = grpc_timer_heap_top(&shard->heap);
     if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
@@ -393,7 +416,7 @@ static grpc_timer *pop_one(shard_type *shard, gpr_atm now) {
 }
 
 /* REQUIRES: shard->mu unlocked */
-static size_t pop_timers(grpc_exec_ctx *exec_ctx, shard_type *shard,
+static size_t pop_timers(grpc_exec_ctx *exec_ctx, timer_shard *shard,
                          gpr_atm now, gpr_atm *new_min_deadline,
                          grpc_error *error) {
   size_t n = 0;
diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c
index 520d4a3252..631f7935d9 100644
--- a/src/core/lib/iomgr/timer_manager.c
+++ b/src/core/lib/iomgr/timer_manager.c
@@ -50,10 +50,13 @@ static completed_thread *g_completed_threads;
 static bool g_kicked;
 // is there a thread waiting until the next timer should fire?
 static bool g_has_timed_waiter;
+// the deadline of the current timed waiter thread (only relevant if
+// g_has_timed_waiter is true)
+static gpr_timespec g_timed_waiter_deadline;
 // generation counter to track which thread is waiting for the next timer
 static uint64_t g_timed_waiter_generation;
 
-static void timer_thread(void *unused);
+static void timer_thread(void *completed_thread_ptr);
 
 static void gc_completed_threads(void) {
   if (g_completed_threads != NULL) {
@@ -78,10 +81,17 @@ static void start_timer_thread_and_unlock(void) {
   if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
     gpr_log(GPR_DEBUG, "Spawn timer thread");
   }
-  gpr_thd_id thd;
   gpr_thd_options opt = gpr_thd_options_default();
   gpr_thd_options_set_joinable(&opt);
-  gpr_thd_new(&thd, timer_thread, NULL, &opt);
+  completed_thread *ct = gpr_malloc(sizeof(*ct));
+  // The call to gpr_thd_new() has to be under the same lock used by
+  // gc_completed_threads(), particularly due to ct->t, which is written here
+  // (internally by gpr_thd_new) and read there. Otherwise it's possible for ct
+  // to leak through g_completed_threads and be freed in gc_completed_threads()
+  // before "&ct->t" is written to, causing a use-after-free.
+  gpr_mu_lock(&g_mu);
+  gpr_thd_new(&ct->t, timer_thread, ct, &opt);
+  gpr_mu_unlock(&g_mu);
 }
 
 void grpc_timer_manager_tick() {
@@ -101,8 +111,7 @@ static void run_some_timers(grpc_exec_ctx *exec_ctx) {
     start_timer_thread_and_unlock();
   } else {
     // if there's no thread waiting with a timeout, kick an existing
-    // waiter
-    // so that the next deadline is not missed
+    // waiter so that the next deadline is not missed
     if (!g_has_timed_waiter) {
       if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
         gpr_log(GPR_DEBUG, "kick untimed waiter");
@@ -132,44 +141,79 @@ static bool wait_until(gpr_timespec next) {
     gpr_mu_unlock(&g_mu);
     return false;
   }
-  // if there's no timed waiter, we should become one: that waiter waits
-  // only until the next timer should expire
-  // all other timers wait forever
-  uint64_t my_timed_waiter_generation = g_timed_waiter_generation - 1;
-  if (!g_has_timed_waiter && gpr_time_cmp(next, inf_future) != 0) {
-    g_has_timed_waiter = true;
-    // we use a generation counter to track the timed waiter so we can
-    // cancel an existing one quickly (and when it actually times out it'll
-    // figure stuff out instead of incurring a wakeup)
-    my_timed_waiter_generation = ++g_timed_waiter_generation;
-    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
-      gpr_timespec wait_time = gpr_time_sub(next, gpr_now(GPR_CLOCK_MONOTONIC));
-      gpr_log(GPR_DEBUG, "sleep for a %" PRId64 ".%09d seconds",
-              wait_time.tv_sec, wait_time.tv_nsec);
+
+  // If g_kicked is true at this point, it means there was a kick from the timer
+  // system that the timer-manager threads here missed. We cannot trust 'next'
+  // here any longer (since there might be an earlier deadline). So if g_kicked
+  // is true at this point, we should quickly exit this and get the next
+  // deadline from the timer system
+
+  if (!g_kicked) {
+    // if there's no timed waiter, we should become one: that waiter waits
+    // only until the next timer should expire. All other timers wait forever
+    //
+    // 'g_timed_waiter_generation' is a global generation counter. The idea here
+    // is that the thread becoming a timed-waiter increments and stores this
+    // global counter locally in 'my_timed_waiter_generation' before going to
+    // sleep. After waking up, if my_timed_waiter_generation ==
+    // g_timed_waiter_generation, it can be sure that it was the timed_waiter
+    // thread (and that no other thread took over while this was asleep)
+    //
+    // Initialize my_timed_waiter_generation to some value that is NOT equal to
+    // g_timed_waiter_generation
+    uint64_t my_timed_waiter_generation = g_timed_waiter_generation - 1;
+
+    /* If there's no timed waiter, we should become one: that waiter waits only
+       until the next timer should expire. All other timer threads wait forever
+       unless their 'next' is earlier than the current timed-waiter's deadline
+       (in which case the thread with earlier 'next' takes over as the new timed
+       waiter) */
+    if (gpr_time_cmp(next, inf_future) != 0) {
+      if (!g_has_timed_waiter ||
+          (gpr_time_cmp(next, g_timed_waiter_deadline) < 0)) {
+        my_timed_waiter_generation = ++g_timed_waiter_generation;
+        g_has_timed_waiter = true;
+        g_timed_waiter_deadline = next;
+
+        if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+          gpr_timespec wait_time =
+              gpr_time_sub(next, gpr_now(GPR_CLOCK_MONOTONIC));
+          gpr_log(GPR_DEBUG, "sleep for a %" PRId64 ".%09d seconds",
+                  wait_time.tv_sec, wait_time.tv_nsec);
+        }
+      } else {  // g_timed_waiter == true && next >= g_timed_waiter_deadline
+        next = inf_future;
+      }
     }
-  } else {
-    next = inf_future;
-    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+
+    if (GRPC_TRACER_ON(grpc_timer_check_trace) &&
+        gpr_time_cmp(next, inf_future) == 0) {
       gpr_log(GPR_DEBUG, "sleep until kicked");
     }
+
+    gpr_cv_wait(&g_cv_wait, &g_mu, next);
+
+    if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
+      gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d",
+              my_timed_waiter_generation == g_timed_waiter_generation,
+              g_kicked);
+    }
+    // if this was the timed waiter, then we need to check timers, and flag
+    // that there's now no timed waiter... we'll look for a replacement if
+    // there's work to do after checking timers (code above)
+    if (my_timed_waiter_generation == g_timed_waiter_generation) {
+      g_has_timed_waiter = false;
+      g_timed_waiter_deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+    }
   }
-  gpr_cv_wait(&g_cv_wait, &g_mu, next);
-  if (GRPC_TRACER_ON(grpc_timer_check_trace)) {
-    gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d",
-            my_timed_waiter_generation == g_timed_waiter_generation, g_kicked);
-  }
-  // if this was the timed waiter, then we need to check timers, and flag
-  // that there's now no timed waiter... we'll look for a replacement if
-  // there's work to do after checking timers (code above)
-  if (my_timed_waiter_generation == g_timed_waiter_generation) {
-    g_has_timed_waiter = false;
-  }
+
   // if this was a kick from the timer system, consume it (and don't stop
   // this thread yet)
   if (g_kicked) {
     grpc_timer_consume_kick();
     g_kicked = false;
   }
+
   gpr_mu_unlock(&g_mu);
   return true;
 }
@@ -208,7 +252,7 @@ static void timer_main_loop(grpc_exec_ctx *exec_ctx) {
   }
 }
 
-static void timer_thread_cleanup(void) {
+static void timer_thread_cleanup(completed_thread *ct) {
   gpr_mu_lock(&g_mu);
   // terminate the thread: drop the waiter count, thread count, and let whomever
   // stopped the threading stuff know that we're done
@@ -217,8 +261,6 @@ static void timer_thread_cleanup(void) {
   if (0 == g_thread_count) {
     gpr_cv_signal(&g_cv_shutdown);
   }
-  completed_thread *ct = gpr_malloc(sizeof(*ct));
-  ct->t = gpr_thd_currentid();
   ct->next = g_completed_threads;
   g_completed_threads = ct;
   gpr_mu_unlock(&g_mu);
@@ -227,14 +269,14 @@ static void timer_thread_cleanup(void) {
   }
 }
 
-static void timer_thread(void *unused) {
+static void timer_thread(void *completed_thread_ptr) {
   // this threads exec_ctx: we try to run things through to completion here
   // since it's easy to spin up new threads
   grpc_exec_ctx exec_ctx =
       GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
   timer_main_loop(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
-  timer_thread_cleanup();
+  timer_thread_cleanup(completed_thread_ptr);
 }
 
 static void start_threads(void) {
@@ -257,6 +299,9 @@ void grpc_timer_manager_init(void) {
   g_waiter_count = 0;
   g_completed_threads = NULL;
 
+  g_has_timed_waiter = false;
+  g_timed_waiter_deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+
   start_threads();
 }
 
@@ -302,6 +347,7 @@ void grpc_kick_poller(void) {
   gpr_mu_lock(&g_mu);
   g_kicked = true;
   g_has_timed_waiter = false;
+  g_timed_waiter_deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
   ++g_timed_waiter_generation;
   gpr_cv_signal(&g_cv_wait);
   gpr_mu_unlock(&g_mu);
diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c
index 4f204cfbf8..70f49bcbe8 100644
--- a/src/core/lib/iomgr/timer_uv.c
+++ b/src/core/lib/iomgr/timer_uv.c
@@ -24,12 +24,14 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/iomgr_uv.h"
 #include "src/core/lib/iomgr/timer.h"
 
 #include <uv.h>
 
-grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false);
-grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false, "timer");
+grpc_tracer_flag grpc_timer_check_trace =
+    GRPC_TRACER_INITIALIZER(false, "timer_check");
 
 static void timer_close_callback(uv_handle_t *handle) { gpr_free(handle); }
 
@@ -42,6 +44,7 @@ static void stop_uv_timer(uv_timer_t *handle) {
 void run_expired_timer(uv_timer_t *handle) {
   grpc_timer *timer = (grpc_timer *)handle->data;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GRPC_UV_ASSERT_SAME_THREAD();
   GPR_ASSERT(timer->pending);
   timer->pending = 0;
   GRPC_CLOSURE_SCHED(&exec_ctx, timer->closure, GRPC_ERROR_NONE);
@@ -54,6 +57,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
                      gpr_timespec now) {
   uint64_t timeout;
   uv_timer_t *uv_timer;
+  GRPC_UV_ASSERT_SAME_THREAD();
   timer->closure = closure;
   if (gpr_time_cmp(deadline, now) <= 0) {
     timer->pending = 0;
@@ -74,6 +78,7 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer,
 }
 
 void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) {
+  GRPC_UV_ASSERT_SAME_THREAD();
   if (timer->pending) {
     timer->pending = 0;
     GRPC_CLOSURE_SCHED(exec_ctx, timer->closure, GRPC_ERROR_CANCELLED);
diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c
index 54e7f417a7..88fa34cb7a 100644
--- a/src/core/lib/iomgr/udp_server.c
+++ b/src/core/lib/iomgr/udp_server.c
@@ -214,7 +214,7 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
                       sp->server->user_data);
       }
       grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL,
-                     "udp_listener_shutdown");
+                     false /* already_closed */, "udp_listener_shutdown");
     }
     gpr_mu_unlock(&s->mu);
   } else {
diff --git a/src/core/lib/security/context/security_context.c b/src/core/lib/security/context/security_context.c
index dffe6d2e91..8fff2c92c5 100644
--- a/src/core/lib/security/context/security_context.c
+++ b/src/core/lib/security/context/security_context.c
@@ -31,7 +31,7 @@
 
 #ifndef NDEBUG
 grpc_tracer_flag grpc_trace_auth_context_refcount =
-    GRPC_TRACER_INITIALIZER(false);
+    GRPC_TRACER_INITIALIZER(false, "auth_context_refcount");
 #endif
 
 /* --- grpc_call --- */
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
index 77d7b04627..09fd60a12c 100644
--- a/src/core/lib/security/credentials/composite/composite_credentials.c
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -32,88 +32,98 @@
 typedef struct {
   grpc_composite_call_credentials *composite_creds;
   size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
   grpc_polling_entity *pollent;
-  grpc_credentials_metadata_cb cb;
+  grpc_auth_metadata_context auth_md_context;
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  grpc_closure internal_on_request_metadata;
 } grpc_composite_call_credentials_metadata_context;
 
 static void composite_call_destruct(grpc_exec_ctx *exec_ctx,
                                     grpc_call_credentials *creds) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
+  for (size_t i = 0; i < c->inner.num_creds; i++) {
     grpc_call_credentials_unref(exec_ctx, c->inner.creds_array[i]);
   }
   gpr_free(c->inner.creds_array);
 }
 
-static void composite_call_md_context_destroy(
-    grpc_exec_ctx *exec_ctx,
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(exec_ctx, ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status,
-                                       const char *error_details) {
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *arg,
+                                       grpc_error *error) {
   grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status, error_details);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
+      (grpc_composite_call_credentials_metadata_context *)arg;
+  if (error == GRPC_ERROR_NONE) {
+    /* See if we need to get some more metadata. */
+    if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+      grpc_call_credentials *inner_creds =
+          ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+      if (grpc_call_credentials_get_request_metadata(
+              exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+              ctx->md_array, &ctx->internal_on_request_metadata, &error)) {
+        // Synchronous response, so call ourselves recursively.
+        composite_call_metadata_cb(exec_ctx, arg, error);
+        GRPC_ERROR_UNREF(error);
+      }
+      return;
     }
+    // We're done!
   }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(
-        exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
-        composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK, NULL);
-  composite_call_md_context_destroy(exec_ctx, ctx);
+  GRPC_CLOSURE_SCHED(exec_ctx, ctx->on_request_metadata, GRPC_ERROR_REF(error));
+  gpr_free(ctx);
 }
 
-static void composite_call_get_request_metadata(
+static bool composite_call_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context auth_md_context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
   grpc_composite_call_credentials_metadata_context *ctx;
-
   ctx = gpr_zalloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
   ctx->composite_creds = c;
   ctx->pollent = pollent;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], ctx->pollent,
-      auth_md_context, composite_call_metadata_cb, ctx);
+  ctx->auth_md_context = auth_md_context;
+  ctx->md_array = md_array;
+  ctx->on_request_metadata = on_request_metadata;
+  GRPC_CLOSURE_INIT(&ctx->internal_on_request_metadata,
+                    composite_call_metadata_cb, ctx, grpc_schedule_on_exec_ctx);
+  while (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    if (grpc_call_credentials_get_request_metadata(
+            exec_ctx, inner_creds, ctx->pollent, ctx->auth_md_context,
+            ctx->md_array, &ctx->internal_on_request_metadata, error)) {
+      if (*error != GRPC_ERROR_NONE) break;
+    } else {
+      break;
+    }
+  }
+  // If we got through all creds synchronously or we got a synchronous
+  // error on one of them, return synchronously.
+  if (ctx->creds_index == ctx->composite_creds->inner.num_creds ||
+      *error != GRPC_ERROR_NONE) {
+    gpr_free(ctx);
+    return true;
+  }
+  // At least one inner cred is returning asynchronously, so we'll
+  // return asynchronously as well.
+  return false;
+}
+
+static void composite_call_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  for (size_t i = 0; i < c->inner.num_creds; ++i) {
+    grpc_call_credentials_cancel_get_request_metadata(
+        exec_ctx, c->inner.creds_array[i], md_array, GRPC_ERROR_REF(error));
+  }
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
+    composite_call_destruct, composite_call_get_request_metadata,
+    composite_call_cancel_get_request_metadata};
 
 static grpc_call_credentials_array get_creds_array(
     grpc_call_credentials **creds_addr) {
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
index b1f1e82076..8a67c9865b 100644
--- a/src/core/lib/security/credentials/credentials.c
+++ b/src/core/lib/security/credentials/credentials.c
@@ -38,13 +38,10 @@
 /* -- Common. -- */
 
 grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
-    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
-    void *user_data) {
+    grpc_call_credentials *creds) {
   grpc_credentials_metadata_request *r =
       gpr_zalloc(sizeof(grpc_credentials_metadata_request));
   r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
   return r;
 }
 
@@ -104,18 +101,25 @@ void grpc_call_credentials_release(grpc_call_credentials *creds) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-void grpc_call_credentials_get_request_metadata(
+bool grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
-    }
+    return true;
+  }
+  return creds->vtable->get_request_metadata(
+      exec_ctx, creds, pollent, context, md_array, on_request_metadata, error);
+}
+
+void grpc_call_credentials_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  if (creds == NULL || creds->vtable->cancel_get_request_metadata == NULL) {
     return;
   }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pollent, context, cb,
-                                      user_data);
+  creds->vtable->cancel_get_request_metadata(exec_ctx, creds, md_array, error);
 }
 
 grpc_security_status grpc_channel_credentials_create_security_connector(
diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h
index c45c2e957c..04a54b0ca8 100644
--- a/src/core/lib/security/credentials/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -138,48 +138,39 @@ grpc_channel_credentials *grpc_channel_credentials_from_arg(
 grpc_channel_credentials *grpc_channel_credentials_find_in_args(
     const grpc_channel_args *args);
 
-/* --- grpc_credentials_md. --- */
+/* --- grpc_credentials_mdelem_array. --- */
 
 typedef struct {
-  grpc_slice key;
-  grpc_slice value;
-} grpc_credentials_md;
+  grpc_mdelem *md;
+  size_t size;
+} grpc_credentials_mdelem_array;
 
-typedef struct {
-  grpc_credentials_md *entries;
-  size_t num_entries;
-  size_t allocated;
-  gpr_refcount refcount;
-} grpc_credentials_md_store;
+/// Takes a new ref to \a md.
+void grpc_credentials_mdelem_array_add(grpc_credentials_mdelem_array *list,
+                                       grpc_mdelem md);
 
-grpc_credentials_md_store *grpc_credentials_md_store_create(
-    size_t initial_capacity);
+/// Appends all elements from \a src to \a dst, taking a new ref to each one.
+void grpc_credentials_mdelem_array_append(grpc_credentials_mdelem_array *dst,
+                                          grpc_credentials_mdelem_array *src);
 
-/* Will ref key and value. */
-void grpc_credentials_md_store_add(grpc_credentials_md_store *store,
-                                   grpc_slice key, grpc_slice value);
-void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store,
-                                            const char *key, const char *value);
-grpc_credentials_md_store *grpc_credentials_md_store_ref(
-    grpc_credentials_md_store *store);
-void grpc_credentials_md_store_unref(grpc_exec_ctx *exec_ctx,
-                                     grpc_credentials_md_store *store);
+void grpc_credentials_mdelem_array_destroy(grpc_exec_ctx *exec_ctx,
+                                           grpc_credentials_mdelem_array *list);
 
 /* --- grpc_call_credentials. --- */
 
-/* error_details must be NULL if status is GRPC_CREDENTIALS_OK. */
-typedef void (*grpc_credentials_metadata_cb)(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems,
-    size_t num_md, grpc_credentials_status status, const char *error_details);
-
 typedef struct {
   void (*destruct)(grpc_exec_ctx *exec_ctx, grpc_call_credentials *c);
-  void (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
+  bool (*get_request_metadata)(grpc_exec_ctx *exec_ctx,
                                grpc_call_credentials *c,
                                grpc_polling_entity *pollent,
                                grpc_auth_metadata_context context,
-                               grpc_credentials_metadata_cb cb,
-                               void *user_data);
+                               grpc_credentials_mdelem_array *md_array,
+                               grpc_closure *on_request_metadata,
+                               grpc_error **error);
+  void (*cancel_get_request_metadata)(grpc_exec_ctx *exec_ctx,
+                                      grpc_call_credentials *c,
+                                      grpc_credentials_mdelem_array *md_array,
+                                      grpc_error *error);
 } grpc_call_credentials_vtable;
 
 struct grpc_call_credentials {
@@ -191,15 +182,29 @@ struct grpc_call_credentials {
 grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds);
 void grpc_call_credentials_unref(grpc_exec_ctx *exec_ctx,
                                  grpc_call_credentials *creds);
-void grpc_call_credentials_get_request_metadata(
+
+/// Returns true if completed synchronously, in which case \a error will
+/// be set to indicate the result.  Otherwise, \a on_request_metadata will
+/// be invoked asynchronously when complete.  \a md_array will be populated
+/// with the resulting metadata once complete.
+bool grpc_call_credentials_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data);
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error);
+
+/// Cancels a pending asynchronous operation started by
+/// grpc_call_credentials_get_request_metadata() with the corresponding
+/// value of \a md_array.
+void grpc_call_credentials_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error);
 
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async);
+    grpc_exec_ctx *exec_ctx, const char *md_key, const char *md_value,
+    bool is_async);
 
 /* --- grpc_server_credentials. --- */
 
@@ -238,14 +243,11 @@ grpc_server_credentials *grpc_find_server_credentials_in_args(
 
 typedef struct {
   grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
   grpc_http_response response;
-  void *user_data;
 } grpc_credentials_metadata_request;
 
 grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
-    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
-    void *user_data);
+    grpc_call_credentials *creds);
 
 void grpc_credentials_metadata_request_destroy(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *r);
diff --git a/src/core/lib/security/credentials/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
index fcfdd52d20..ccd39e610f 100644
--- a/src/core/lib/security/credentials/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -24,65 +24,36 @@
 
 #include "src/core/lib/slice/slice_internal.h"
 
-static void store_ensure_capacity(grpc_credentials_md_store *store) {
-  if (store->num_entries == store->allocated) {
-    store->allocated = (store->allocated == 0) ? 1 : store->allocated * 2;
-    store->entries = gpr_realloc(
-        store->entries, store->allocated * sizeof(grpc_credentials_md));
+static void mdelem_list_ensure_capacity(grpc_credentials_mdelem_array *list,
+                                        size_t additional_space_needed) {
+  size_t target_size = list->size + additional_space_needed;
+  // Find the next power of two greater than the target size (i.e.,
+  // whenever we add more space, we double what we already have).
+  size_t new_size = 2;
+  while (new_size < target_size) {
+    new_size *= 2;
   }
+  list->md = gpr_realloc(list->md, sizeof(grpc_mdelem) * new_size);
 }
 
-grpc_credentials_md_store *grpc_credentials_md_store_create(
-    size_t initial_capacity) {
-  grpc_credentials_md_store *store =
-      gpr_zalloc(sizeof(grpc_credentials_md_store));
-  if (initial_capacity > 0) {
-    store->entries = gpr_malloc(initial_capacity * sizeof(grpc_credentials_md));
-    store->allocated = initial_capacity;
-  }
-  gpr_ref_init(&store->refcount, 1);
-  return store;
-}
-
-void grpc_credentials_md_store_add(grpc_credentials_md_store *store,
-                                   grpc_slice key, grpc_slice value) {
-  if (store == NULL) return;
-  store_ensure_capacity(store);
-  store->entries[store->num_entries].key = grpc_slice_ref_internal(key);
-  store->entries[store->num_entries].value = grpc_slice_ref_internal(value);
-  store->num_entries++;
+void grpc_credentials_mdelem_array_add(grpc_credentials_mdelem_array *list,
+                                       grpc_mdelem md) {
+  mdelem_list_ensure_capacity(list, 1);
+  list->md[list->size++] = GRPC_MDELEM_REF(md);
 }
 
-void grpc_credentials_md_store_add_cstrings(grpc_credentials_md_store *store,
-                                            const char *key,
-                                            const char *value) {
-  if (store == NULL) return;
-  store_ensure_capacity(store);
-  store->entries[store->num_entries].key = grpc_slice_from_copied_string(key);
-  store->entries[store->num_entries].value =
-      grpc_slice_from_copied_string(value);
-  store->num_entries++;
-}
-
-grpc_credentials_md_store *grpc_credentials_md_store_ref(
-    grpc_credentials_md_store *store) {
-  if (store == NULL) return NULL;
-  gpr_ref(&store->refcount);
-  return store;
+void grpc_credentials_mdelem_array_append(grpc_credentials_mdelem_array *dst,
+                                          grpc_credentials_mdelem_array *src) {
+  mdelem_list_ensure_capacity(dst, src->size);
+  for (size_t i = 0; i < src->size; ++i) {
+    dst->md[dst->size++] = GRPC_MDELEM_REF(src->md[i]);
+  }
 }
 
-void grpc_credentials_md_store_unref(grpc_exec_ctx *exec_ctx,
-                                     grpc_credentials_md_store *store) {
-  if (store == NULL) return;
-  if (gpr_unref(&store->refcount)) {
-    if (store->entries != NULL) {
-      size_t i;
-      for (i = 0; i < store->num_entries; i++) {
-        grpc_slice_unref_internal(exec_ctx, store->entries[i].key);
-        grpc_slice_unref_internal(exec_ctx, store->entries[i].value);
-      }
-      gpr_free(store->entries);
-    }
-    gpr_free(store);
+void grpc_credentials_mdelem_array_destroy(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_mdelem_array *list) {
+  for (size_t i = 0; i < list->size; ++i) {
+    GRPC_MDELEM_UNREF(exec_ctx, list->md[i]);
   }
+  gpr_free(list->md);
 }
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
index 67e74f7b92..ac9017850f 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.c
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -98,49 +98,44 @@ const char *grpc_fake_transport_get_expected_targets(
 static void md_only_test_destruct(grpc_exec_ctx *exec_ctx,
                                   grpc_call_credentials *creds) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->md_store);
+  GRPC_MDELEM_UNREF(exec_ctx, c->md);
 }
 
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, grpc_error *error) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK, NULL);
-  grpc_credentials_metadata_request_destroy(exec_ctx, r);
-}
-
-static void md_only_test_get_request_metadata(
+static bool md_only_test_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
+  grpc_credentials_mdelem_array_add(md_array, c->md);
   if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    GRPC_CLOSURE_SCHED(exec_ctx,
-                       GRPC_CLOSURE_CREATE(on_simulated_token_fetch_done,
-                                           cb_arg, grpc_executor_scheduler),
-                       GRPC_ERROR_NONE);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK, NULL);
+    GRPC_CLOSURE_SCHED(exec_ctx, on_request_metadata, GRPC_ERROR_NONE);
+    return false;
   }
+  return true;
+}
+
+static void md_only_test_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
+    md_only_test_destruct, md_only_test_get_request_metadata,
+    md_only_test_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
+    grpc_exec_ctx *exec_ctx, const char *md_key, const char *md_value,
+    bool is_async) {
   grpc_md_only_test_credentials *c =
       gpr_zalloc(sizeof(grpc_md_only_test_credentials));
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
   c->base.vtable = &md_only_test_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->md =
+      grpc_mdelem_from_slices(exec_ctx, grpc_slice_from_copied_string(md_key),
+                              grpc_slice_from_copied_string(md_value));
   c->is_async = is_async;
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
index fae7a6a9c4..aa0f3b6e20 100644
--- a/src/core/lib/security/credentials/fake/fake_credentials.h
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -52,8 +52,8 @@ const char *grpc_fake_transport_get_expected_targets(
 
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
+  grpc_mdelem md;
+  bool is_async;
 } grpc_md_only_test_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
index 4b32c5a047..3de8319d98 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.c
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -30,26 +30,33 @@
 static void iam_destruct(grpc_exec_ctx *exec_ctx,
                          grpc_call_credentials *creds) {
   grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->iam_md);
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &c->md_array);
 }
 
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
                                      grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
+                                     grpc_credentials_mdelem_array *md_array,
+                                     grpc_closure *on_request_metadata,
+                                     grpc_error **error) {
   grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK, NULL);
+  grpc_credentials_mdelem_array_append(md_array, &c->md_array);
+  return true;
 }
 
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
+static void iam_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {
+    iam_destruct, iam_get_request_metadata, iam_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_google_iam_credentials_create(
     const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   GRPC_API_TRACE(
       "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
       "reserved=%p)",
@@ -57,14 +64,22 @@ grpc_call_credentials *grpc_google_iam_credentials_create(
   GPR_ASSERT(reserved == NULL);
   GPR_ASSERT(token != NULL);
   GPR_ASSERT(authority_selector != NULL);
-  c = gpr_zalloc(sizeof(grpc_google_iam_credentials));
+  grpc_google_iam_credentials *c = gpr_zalloc(sizeof(*c));
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
   c->base.vtable = &iam_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  grpc_mdelem md = grpc_mdelem_from_slices(
+      &exec_ctx,
+      grpc_slice_from_static_string(GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY),
+      grpc_slice_from_copied_string(token));
+  grpc_credentials_mdelem_array_add(&c->md_array, md);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  md = grpc_mdelem_from_slices(
+      &exec_ctx,
+      grpc_slice_from_static_string(GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY),
+      grpc_slice_from_copied_string(authority_selector));
+  grpc_credentials_mdelem_array_add(&c->md_array, md);
+  GRPC_MDELEM_UNREF(&exec_ctx, md);
+  grpc_exec_ctx_finish(&exec_ctx);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h
index fff3c6d2ca..5e3cf65bad 100644
--- a/src/core/lib/security/credentials/iam/iam_credentials.h
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -23,7 +23,7 @@
 
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
+  grpc_credentials_mdelem_array md_array;
 } grpc_google_iam_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
index 589a6f9407..02c82e99ba 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.c
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -29,10 +29,8 @@
 
 static void jwt_reset_cache(grpc_exec_ctx *exec_ctx,
                             grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(exec_ctx, c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
+  GRPC_MDELEM_UNREF(exec_ctx, c->cached.jwt_md);
+  c->cached.jwt_md = GRPC_MDNULL;
   if (c->cached.service_url != NULL) {
     gpr_free(c->cached.service_url);
     c->cached.service_url = NULL;
@@ -49,33 +47,34 @@ static void jwt_destruct(grpc_exec_ctx *exec_ctx,
   gpr_mu_destroy(&c->cache_mu);
 }
 
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                      grpc_call_credentials *creds,
                                      grpc_polling_entity *pollent,
                                      grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
+                                     grpc_credentials_mdelem_array *md_array,
+                                     grpc_closure *on_request_metadata,
+                                     grpc_error **error) {
   grpc_service_account_jwt_access_credentials *c =
       (grpc_service_account_jwt_access_credentials *)creds;
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
 
   /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
+  grpc_mdelem jwt_md = GRPC_MDNULL;
   {
     gpr_mu_lock(&c->cache_mu);
     if (c->cached.service_url != NULL &&
         strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
+        !GRPC_MDISNULL(c->cached.jwt_md) &&
         (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
                                    gpr_now(GPR_CLOCK_REALTIME)),
                       refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+      jwt_md = GRPC_MDELEM_REF(c->cached.jwt_md);
     }
     gpr_mu_unlock(&c->cache_mu);
   }
 
-  if (jwt_md == NULL) {
+  if (GRPC_MDISNULL(jwt_md)) {
     char *jwt = NULL;
     /* Generate a new jwt. */
     gpr_mu_lock(&c->cache_mu);
@@ -89,27 +88,33 @@ static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
       c->cached.jwt_expiration =
           gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
       c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      c->cached.jwt_md = grpc_mdelem_from_slices(
+          exec_ctx,
+          grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+          grpc_slice_from_copied_string(md_value));
       gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+      jwt_md = GRPC_MDELEM_REF(c->cached.jwt_md);
     }
     gpr_mu_unlock(&c->cache_mu);
   }
 
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK, NULL);
-    grpc_credentials_md_store_unref(exec_ctx, jwt_md);
+  if (!GRPC_MDISNULL(jwt_md)) {
+    grpc_credentials_mdelem_array_add(md_array, jwt_md);
+    GRPC_MDELEM_UNREF(exec_ctx, jwt_md);
   } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-       "Could not generate JWT.");
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Could not generate JWT.");
   }
+  return true;
+}
+
+static void jwt_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
+static grpc_call_credentials_vtable jwt_vtable = {
+    jwt_destruct, jwt_get_request_metadata, jwt_cancel_get_request_metadata};
 
 grpc_call_credentials *
 grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
@@ -125,6 +130,13 @@ grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
   gpr_ref_init(&c->base.refcount, 1);
   c->base.vtable = &jwt_vtable;
   c->key = key;
+  gpr_timespec max_token_lifetime = grpc_max_auth_token_lifetime();
+  if (gpr_time_cmp(token_lifetime, max_token_lifetime) > 0) {
+    gpr_log(GPR_INFO,
+            "Cropping token lifetime to maximum allowed value (%d secs).",
+            (int)max_token_lifetime.tv_sec);
+    token_lifetime = grpc_max_auth_token_lifetime();
+  }
   c->jwt_lifetime = token_lifetime;
   gpr_mu_init(&c->cache_mu);
   jwt_reset_cache(exec_ctx, c);
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h
index 6e461f1715..07f4022669 100644
--- a/src/core/lib/security/credentials/jwt/jwt_credentials.h
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -29,7 +29,7 @@ typedef struct {
   // the service_url for a more sophisticated one.
   gpr_mu cache_mu;
   struct {
-    grpc_credentials_md_store *jwt_md;
+    grpc_mdelem jwt_md;
     char *service_url;
     gpr_timespec jwt_expiration;
   } cached;
diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
index 8c747085bb..6cd558d123 100644
--- a/src/core/lib/security/credentials/jwt/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -462,6 +462,35 @@ static BIGNUM *bignum_from_base64(grpc_exec_ctx *exec_ctx, const char *b64) {
   return result;
 }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+
+// Provide compatibility across OpenSSL 1.02 and 1.1.
+static int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {
+  /* If the fields n and e in r are NULL, the corresponding input
+   * parameters MUST be non-NULL for n and e.  d may be
+   * left NULL (in case only the public key is used).
+   */
+  if ((r->n == NULL && n == NULL) || (r->e == NULL && e == NULL)) {
+    return 0;
+  }
+
+  if (n != NULL) {
+    BN_free(r->n);
+    r->n = n;
+  }
+  if (e != NULL) {
+    BN_free(r->e);
+    r->e = e;
+  }
+  if (d != NULL) {
+    BN_free(r->d);
+    r->d = d;
+  }
+
+  return 1;
+}
+#endif  // OPENSSL_VERSION_NUMBER < 0x10100000L
+
 static EVP_PKEY *pkey_from_jwk(grpc_exec_ctx *exec_ctx, const grpc_json *json,
                                const char *kty) {
   const grpc_json *key_prop;
@@ -478,21 +507,27 @@ static EVP_PKEY *pkey_from_jwk(grpc_exec_ctx *exec_ctx, const grpc_json *json,
     gpr_log(GPR_ERROR, "Could not create rsa key.");
     goto end;
   }
+  BIGNUM *tmp_n = NULL;
+  BIGNUM *tmp_e = NULL;
   for (key_prop = json->child; key_prop != NULL; key_prop = key_prop->next) {
     if (strcmp(key_prop->key, "n") == 0) {
-      rsa->n =
+      tmp_n =
           bignum_from_base64(exec_ctx, validate_string_field(key_prop, "n"));
-      if (rsa->n == NULL) goto end;
+      if (tmp_n == NULL) goto end;
     } else if (strcmp(key_prop->key, "e") == 0) {
-      rsa->e =
+      tmp_e =
           bignum_from_base64(exec_ctx, validate_string_field(key_prop, "e"));
-      if (rsa->e == NULL) goto end;
+      if (tmp_e == NULL) goto end;
     }
   }
-  if (rsa->e == NULL || rsa->n == NULL) {
+  if (tmp_e == NULL || tmp_n == NULL) {
     gpr_log(GPR_ERROR, "Missing RSA public key field.");
     goto end;
   }
+  if (!RSA_set0_key(rsa, tmp_n, tmp_e, NULL)) {
+    gpr_log(GPR_ERROR, "Cannot set RSA key from inputs.");
+    goto end;
+  }
   result = EVP_PKEY_new();
   EVP_PKEY_set1_RSA(result, rsa); /* uprefs rsa. */
 
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
index 9de561b310..ffa941bb9e 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -107,7 +107,7 @@ static void oauth2_token_fetcher_destruct(grpc_exec_ctx *exec_ctx,
                                           grpc_call_credentials *creds) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->access_token_md);
+  GRPC_MDELEM_UNREF(exec_ctx, c->access_token_md);
   gpr_mu_destroy(&c->mu);
   grpc_httpcli_context_destroy(exec_ctx, &c->httpcli_context);
 }
@@ -115,7 +115,7 @@ static void oauth2_token_fetcher_destruct(grpc_exec_ctx *exec_ctx,
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_exec_ctx *exec_ctx, const grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime) {
+    grpc_mdelem *token_md, gpr_timespec *token_lifetime) {
   char *null_terminated_body = NULL;
   char *new_access_token = NULL;
   grpc_credentials_status status = GRPC_CREDENTIALS_OK;
@@ -184,17 +184,18 @@ grpc_oauth2_token_fetcher_credentials_parse_server_response(
     token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
     token_lifetime->tv_nsec = 0;
     token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(exec_ctx, *token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    if (!GRPC_MDISNULL(*token_md)) GRPC_MDELEM_UNREF(exec_ctx, *token_md);
+    *token_md = grpc_mdelem_from_slices(
+        exec_ctx,
+        grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+        grpc_slice_from_copied_string(new_access_token));
     status = GRPC_CREDENTIALS_OK;
   }
 
 end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(exec_ctx, *token_md);
-    *token_md = NULL;
+  if (status != GRPC_CREDENTIALS_OK && !GRPC_MDISNULL(*token_md)) {
+    GRPC_MDELEM_UNREF(exec_ctx, *token_md);
+    *token_md = GRPC_MDNULL;
   }
   if (null_terminated_body != NULL) gpr_free(null_terminated_body);
   if (new_access_token != NULL) gpr_free(new_access_token);
@@ -205,63 +206,124 @@ end:
 static void on_oauth2_token_fetcher_http_response(grpc_exec_ctx *exec_ctx,
                                                   void *user_data,
                                                   grpc_error *error) {
+  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
   grpc_credentials_metadata_request *r =
       (grpc_credentials_metadata_request *)user_data;
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  grpc_mdelem access_token_md = GRPC_MDNULL;
   gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  GRPC_LOG_IF_ERROR("oauth_fetch", GRPC_ERROR_REF(error));
-
+  grpc_credentials_status status =
+      grpc_oauth2_token_fetcher_credentials_parse_server_response(
+          exec_ctx, &r->response, &access_token_md, &token_lifetime);
+  // Update cache and grab list of pending requests.
   gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      exec_ctx, &r->response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status,
-          "Error occured when fetching oauth2 token.");
-  }
+  c->token_fetch_pending = false;
+  c->access_token_md = GRPC_MDELEM_REF(access_token_md);
+  c->token_expiration =
+      status == GRPC_CREDENTIALS_OK
+          ? gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime)
+          : gpr_inf_past(GPR_CLOCK_REALTIME);
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      c->pending_requests;
+  c->pending_requests = NULL;
   gpr_mu_unlock(&c->mu);
+  // Invoke callbacks for all pending requests.
+  while (pending_request != NULL) {
+    if (status == GRPC_CREDENTIALS_OK) {
+      grpc_credentials_mdelem_array_add(pending_request->md_array,
+                                        access_token_md);
+    } else {
+      error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+          "Error occured when fetching oauth2 token.", &error, 1);
+    }
+    GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata, error);
+    grpc_oauth2_pending_get_request_metadata *prev = pending_request;
+    pending_request = pending_request->next;
+    gpr_free(prev);
+  }
+  GRPC_MDELEM_UNREF(exec_ctx, access_token_md);
+  grpc_call_credentials_unref(exec_ctx, r->creds);
   grpc_credentials_metadata_request_destroy(exec_ctx, r);
 }
 
-static void oauth2_token_fetcher_get_request_metadata(
+static bool oauth2_token_fetcher_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_oauth2_token_fetcher_credentials *c =
       (grpc_oauth2_token_fetcher_credentials *)creds;
+  // Check if we can use the cached token.
   gpr_timespec refresh_threshold = gpr_time_from_seconds(
       GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
+  grpc_mdelem cached_access_token_md = GRPC_MDNULL;
+  gpr_mu_lock(&c->mu);
+  if (!GRPC_MDISNULL(c->access_token_md) &&
+      (gpr_time_cmp(
+           gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+           refresh_threshold) > 0)) {
+    cached_access_token_md = GRPC_MDELEM_REF(c->access_token_md);
+  }
+  if (!GRPC_MDISNULL(cached_access_token_md)) {
     gpr_mu_unlock(&c->mu);
+    grpc_credentials_mdelem_array_add(md_array, cached_access_token_md);
+    GRPC_MDELEM_UNREF(exec_ctx, cached_access_token_md);
+    return true;
   }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK, NULL);
-    grpc_credentials_md_store_unref(exec_ctx, cached_access_token_md);
-  } else {
-    c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pollent, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+  // Couldn't get the token from the cache.
+  // Add request to c->pending_requests and start a new fetch if needed.
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      (grpc_oauth2_pending_get_request_metadata *)gpr_malloc(
+          sizeof(*pending_request));
+  pending_request->md_array = md_array;
+  pending_request->on_request_metadata = on_request_metadata;
+  pending_request->next = c->pending_requests;
+  c->pending_requests = pending_request;
+  bool start_fetch = false;
+  if (!c->token_fetch_pending) {
+    c->token_fetch_pending = true;
+    start_fetch = true;
+  }
+  gpr_mu_unlock(&c->mu);
+  if (start_fetch) {
+    grpc_call_credentials_ref(creds);
+    c->fetch_func(exec_ctx, grpc_credentials_metadata_request_create(creds),
+                  &c->httpcli_context, pollent,
+                  on_oauth2_token_fetcher_http_response,
+                  gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
   }
+  return false;
+}
+
+static void oauth2_token_fetcher_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_mu_lock(&c->mu);
+  grpc_oauth2_pending_get_request_metadata *prev = NULL;
+  grpc_oauth2_pending_get_request_metadata *pending_request =
+      c->pending_requests;
+  while (pending_request != NULL) {
+    if (pending_request->md_array == md_array) {
+      // Remove matching pending request from the list.
+      if (prev != NULL) {
+        prev->next = pending_request->next;
+      } else {
+        c->pending_requests = pending_request->next;
+      }
+      // Invoke the callback immediately with an error.
+      GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
+                         GRPC_ERROR_REF(error));
+      gpr_free(pending_request);
+      break;
+    }
+    prev = pending_request;
+    pending_request = pending_request->next;
+  }
+  gpr_mu_unlock(&c->mu);
+  GRPC_ERROR_UNREF(error);
 }
 
 static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
@@ -280,7 +342,8 @@ static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
 //
 
 static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata,
+    oauth2_token_fetcher_cancel_get_request_metadata};
 
 static void compute_engine_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
@@ -301,7 +364,6 @@ static void compute_engine_fetch_oauth2(
   grpc_httpcli_get(
       exec_ctx, httpcli_context, pollent, resource_quota, &request, deadline,
       GRPC_CLOSURE_CREATE(response_cb, metadata_req, grpc_schedule_on_exec_ctx),
-
       &metadata_req->response);
   grpc_resource_quota_unref_internal(exec_ctx, resource_quota);
 }
@@ -331,7 +393,8 @@ static void refresh_token_destruct(grpc_exec_ctx *exec_ctx,
 }
 
 static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata,
+    oauth2_token_fetcher_cancel_get_request_metadata};
 
 static void refresh_token_fetch_oauth2(
     grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
@@ -416,26 +479,33 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create(
 static void access_token_destruct(grpc_exec_ctx *exec_ctx,
                                   grpc_call_credentials *creds) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(exec_ctx, c->access_token_md);
+  GRPC_MDELEM_UNREF(exec_ctx, c->access_token_md);
 }
 
-static void access_token_get_request_metadata(
+static bool access_token_get_request_metadata(
     grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
     grpc_polling_entity *pollent, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
+    grpc_credentials_mdelem_array *md_array, grpc_closure *on_request_metadata,
+    grpc_error **error) {
   grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK,
-     NULL);
+  grpc_credentials_mdelem_array_add(md_array, c->access_token_md);
+  return true;
+}
+
+static void access_token_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *c,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
+    access_token_destruct, access_token_get_request_metadata,
+    access_token_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_access_token_credentials_create(
     const char *access_token, void *reserved) {
   grpc_access_token_credentials *c =
       gpr_zalloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
   GRPC_API_TRACE(
       "grpc_access_token_credentials_create(access_token=<redacted>, "
       "reserved=%p)",
@@ -444,10 +514,13 @@ grpc_call_credentials *grpc_access_token_credentials_create(
   c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
   c->base.vtable = &access_token_vtable;
   gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
+  char *token_md_value;
   gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  c->access_token_md = grpc_mdelem_from_slices(
+      &exec_ctx, grpc_slice_from_static_string(GRPC_AUTHORIZATION_METADATA_KEY),
+      grpc_slice_from_copied_string(token_md_value));
+  grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(token_md_value);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
index 72093afe9e..9d041a20ea 100644
--- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -58,11 +58,20 @@ typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
                                        grpc_polling_entity *pollent,
                                        grpc_iomgr_cb_func cb,
                                        gpr_timespec deadline);
+
+typedef struct grpc_oauth2_pending_get_request_metadata {
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  struct grpc_oauth2_pending_get_request_metadata *next;
+} grpc_oauth2_pending_get_request_metadata;
+
 typedef struct {
   grpc_call_credentials base;
   gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
+  grpc_mdelem access_token_md;
   gpr_timespec token_expiration;
+  bool token_fetch_pending;
+  grpc_oauth2_pending_get_request_metadata *pending_requests;
   grpc_httpcli_context httpcli_context;
   grpc_fetch_oauth2_func fetch_func;
 } grpc_oauth2_token_fetcher_credentials;
@@ -76,7 +85,7 @@ typedef struct {
 // Access token credentials.
 typedef struct {
   grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
+  grpc_mdelem access_token_md;
 } grpc_access_token_credentials;
 
 // Private constructor for refresh token credentials from an already parsed
@@ -89,6 +98,6 @@ grpc_refresh_token_credentials_create_from_auth_refresh_token(
 grpc_credentials_status
 grpc_oauth2_token_fetcher_credentials_parse_server_response(
     grpc_exec_ctx *exec_ctx, const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+    grpc_mdelem *token_md, gpr_timespec *token_lifetime);
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
index 96ebfb4d0d..73e0c23e0f 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.c
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -31,19 +31,28 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/validate_metadata.h"
 
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
 static void plugin_destruct(grpc_exec_ctx *exec_ctx,
                             grpc_call_credentials *creds) {
   grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  gpr_mu_destroy(&c->mu);
   if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
     c->plugin.destroy(c->plugin.state);
   }
 }
 
+static void pending_request_remove_locked(
+    grpc_plugin_credentials *c,
+    grpc_plugin_credentials_pending_request *pending_request) {
+  if (pending_request->prev == NULL) {
+    c->pending_requests = pending_request->next;
+  } else {
+    pending_request->prev->next = pending_request->next;
+  }
+  if (pending_request->next != NULL) {
+    pending_request->next->prev = pending_request->prev;
+  }
+}
+
 static void plugin_md_request_metadata_ready(void *request,
                                              const grpc_metadata *md,
                                              size_t num_md,
@@ -53,76 +62,117 @@ static void plugin_md_request_metadata_ready(void *request,
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(
       GRPC_EXEC_CTX_FLAG_IS_FINISHED | GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP,
       NULL, NULL);
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-          error_details);
-  } else {
-    size_t i;
-    bool seen_illegal_header = false;
-    grpc_credentials_md *md_array = NULL;
-    for (i = 0; i < num_md; i++) {
-      if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
-                             grpc_validate_header_key_is_legal(md[i].key))) {
-        seen_illegal_header = true;
-        break;
-      } else if (!grpc_is_binary_header(md[i].key) &&
-                 !GRPC_LOG_IF_ERROR(
-                     "validate_metadata_from_plugin",
-                     grpc_validate_header_nonbin_value_is_legal(md[i].value))) {
-        gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
-        seen_illegal_header = true;
-        break;
+  grpc_plugin_credentials_pending_request *r =
+      (grpc_plugin_credentials_pending_request *)request;
+  // Check if the request has been cancelled.
+  // If not, remove it from the pending list, so that it cannot be
+  // cancelled out from under us.
+  gpr_mu_lock(&r->creds->mu);
+  if (!r->cancelled) pending_request_remove_locked(r->creds, r);
+  gpr_mu_unlock(&r->creds->mu);
+  grpc_call_credentials_unref(&exec_ctx, &r->creds->base);
+  // If it has not been cancelled, process it.
+  if (!r->cancelled) {
+    if (status != GRPC_STATUS_OK) {
+      char *msg;
+      gpr_asprintf(&msg, "Getting metadata from plugin failed with error: %s",
+                   error_details);
+      GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata,
+                         GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg));
+      gpr_free(msg);
+    } else {
+      bool seen_illegal_header = false;
+      for (size_t i = 0; i < num_md; ++i) {
+        if (!GRPC_LOG_IF_ERROR("validate_metadata_from_plugin",
+                               grpc_validate_header_key_is_legal(md[i].key))) {
+          seen_illegal_header = true;
+          break;
+        } else if (!grpc_is_binary_header(md[i].key) &&
+                   !GRPC_LOG_IF_ERROR(
+                       "validate_metadata_from_plugin",
+                       grpc_validate_header_nonbin_value_is_legal(
+                           md[i].value))) {
+          gpr_log(GPR_ERROR, "Plugin added invalid metadata value.");
+          seen_illegal_header = true;
+          break;
+        }
       }
-    }
-    if (seen_illegal_header) {
-      r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR,
-            "Illegal metadata");
-    } else if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = grpc_slice_ref_internal(md[i].key);
-        md_array[i].value = grpc_slice_ref_internal(md[i].value);
+      if (seen_illegal_header) {
+        GRPC_CLOSURE_SCHED(
+            &exec_ctx, r->on_request_metadata,
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal metadata"));
+      } else {
+        for (size_t i = 0; i < num_md; ++i) {
+          grpc_mdelem mdelem = grpc_mdelem_from_slices(
+              &exec_ctx, grpc_slice_ref_internal(md[i].key),
+              grpc_slice_ref_internal(md[i].value));
+          grpc_credentials_mdelem_array_add(r->md_array, mdelem);
+          GRPC_MDELEM_UNREF(&exec_ctx, mdelem);
+        }
+        GRPC_CLOSURE_SCHED(&exec_ctx, r->on_request_metadata, GRPC_ERROR_NONE);
       }
-      r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK,
-            NULL);
-      for (i = 0; i < num_md; i++) {
-        grpc_slice_unref_internal(&exec_ctx, md_array[i].key);
-        grpc_slice_unref_internal(&exec_ctx, md_array[i].value);
-      }
-      gpr_free(md_array);
-    } else if (num_md == 0) {
-      r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
     }
   }
   gpr_free(r);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+static bool plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
                                         grpc_call_credentials *creds,
                                         grpc_polling_entity *pollent,
                                         grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
+                                        grpc_credentials_mdelem_array *md_array,
+                                        grpc_closure *on_request_metadata,
+                                        grpc_error **error) {
   grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
   if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_zalloc(sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
+    // Create pending_request object.
+    grpc_plugin_credentials_pending_request *pending_request =
+        (grpc_plugin_credentials_pending_request *)gpr_zalloc(
+            sizeof(*pending_request));
+    pending_request->creds = c;
+    pending_request->md_array = md_array;
+    pending_request->on_request_metadata = on_request_metadata;
+    // Add it to the pending list.
+    gpr_mu_lock(&c->mu);
+    if (c->pending_requests != NULL) {
+      c->pending_requests->prev = pending_request;
+    }
+    pending_request->next = c->pending_requests;
+    c->pending_requests = pending_request;
+    gpr_mu_unlock(&c->mu);
+    // Invoke the plugin.  The callback holds a ref to us.
+    grpc_call_credentials_ref(creds);
     c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK, NULL);
+                           plugin_md_request_metadata_ready, pending_request);
+    return false;
+  }
+  return true;
+}
+
+static void plugin_cancel_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_credentials_mdelem_array *md_array, grpc_error *error) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  gpr_mu_lock(&c->mu);
+  for (grpc_plugin_credentials_pending_request *pending_request =
+           c->pending_requests;
+       pending_request != NULL; pending_request = pending_request->next) {
+    if (pending_request->md_array == md_array) {
+      pending_request->cancelled = true;
+      GRPC_CLOSURE_SCHED(exec_ctx, pending_request->on_request_metadata,
+                         GRPC_ERROR_REF(error));
+      pending_request_remove_locked(c, pending_request);
+      break;
+    }
   }
+  gpr_mu_unlock(&c->mu);
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
+    plugin_destruct, plugin_get_request_metadata,
+    plugin_cancel_get_request_metadata};
 
 grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
     grpc_metadata_credentials_plugin plugin, void *reserved) {
@@ -134,5 +184,6 @@ grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
   c->base.vtable = &plugin_vtable;
   gpr_ref_init(&c->base.refcount, 1);
   c->plugin = plugin;
+  gpr_mu_init(&c->mu);
   return &c->base;
 }
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
index ba3dd76859..57266d589a 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.h
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -21,10 +21,22 @@
 
 #include "src/core/lib/security/credentials/credentials.h"
 
-typedef struct {
+struct grpc_plugin_credentials;
+
+typedef struct grpc_plugin_credentials_pending_request {
+  bool cancelled;
+  struct grpc_plugin_credentials *creds;
+  grpc_credentials_mdelem_array *md_array;
+  grpc_closure *on_request_metadata;
+  struct grpc_plugin_credentials_pending_request *prev;
+  struct grpc_plugin_credentials_pending_request *next;
+} grpc_plugin_credentials_pending_request;
+
+typedef struct grpc_plugin_credentials {
   grpc_call_credentials base;
   grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
+  gpr_mu mu;
+  grpc_plugin_credentials_pending_request *pending_requests;
 } grpc_plugin_credentials;
 
 #endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
index 58112b04b4..531a88434f 100644
--- a/src/core/lib/security/transport/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -49,11 +49,17 @@ typedef struct {
      pollset_set so that work can progress when this call wants work to progress
   */
   grpc_polling_entity *pollent;
-  grpc_transport_stream_op_batch op;
   gpr_atm security_context_set;
   gpr_mu security_context_mu;
+  grpc_credentials_mdelem_array md_array;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
   grpc_auth_metadata_context auth_md_context;
+  grpc_closure closure;
+  // Either 0 (no cancellation and no async operation in flight),
+  // a grpc_closure* (if the lowest bit is 0),
+  // or a grpc_error* (if the lowest bit is 1).
+  gpr_atm cancellation_state;
+  grpc_closure cancel_closure;
 } call_data;
 
 /* We can have a per-channel credentials. */
@@ -62,6 +68,43 @@ typedef struct {
   grpc_auth_context *auth_context;
 } channel_data;
 
+static void decode_cancel_state(gpr_atm cancel_state, grpc_closure **func,
+                                grpc_error **error) {
+  // If the lowest bit is 1, the value is a grpc_error*.
+  // Otherwise, if non-zdero, the value is a grpc_closure*.
+  if (cancel_state & 1) {
+    *error = (grpc_error *)(cancel_state & ~(gpr_atm)1);
+  } else if (cancel_state != 0) {
+    *func = (grpc_closure *)cancel_state;
+  }
+}
+
+static gpr_atm encode_cancel_state_error(grpc_error *error) {
+  // Set the lowest bit to 1 to indicate that it's an error.
+  return (gpr_atm)1 | (gpr_atm)error;
+}
+
+// Returns an error if the call has been cancelled.  Otherwise, sets the
+// cancellation function to be called upon cancellation.
+static grpc_error *set_cancel_func(grpc_call_element *elem,
+                                   grpc_iomgr_cb_func func) {
+  call_data *calld = (call_data *)elem->call_data;
+  // Decode original state.
+  gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state);
+  grpc_error *original_error = GRPC_ERROR_NONE;
+  grpc_closure *original_func = NULL;
+  decode_cancel_state(original_state, &original_func, &original_error);
+  // If error is set, return it.
+  if (original_error != GRPC_ERROR_NONE) return GRPC_ERROR_REF(original_error);
+  // Otherwise, store func.
+  GRPC_CLOSURE_INIT(&calld->cancel_closure, func, elem,
+                    grpc_schedule_on_exec_ctx);
+  GPR_ASSERT(((gpr_atm)&calld->cancel_closure & (gpr_atm)1) == 0);
+  gpr_atm_rel_store(&calld->cancellation_state,
+                    (gpr_atm)&calld->cancel_closure);
+  return GRPC_ERROR_NONE;
+}
+
 static void reset_auth_metadata_context(
     grpc_auth_metadata_context *auth_md_context) {
   if (auth_md_context->service_url != NULL) {
@@ -87,42 +130,30 @@ static void add_error(grpc_error **combined, grpc_error *error) {
   *combined = grpc_error_add_child(*combined, error);
 }
 
-static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *user_data,
-                                    grpc_credentials_md *md_elems,
-                                    size_t num_md,
-                                    grpc_credentials_status status,
-                                    const char *error_details) {
-  grpc_call_element *elem = (grpc_call_element *)user_data;
+static void on_credentials_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                    grpc_error *input_error) {
+  grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
+  grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
-  grpc_transport_stream_op_batch *op = &calld->op;
-  grpc_metadata_batch *mdb;
-  size_t i;
   reset_auth_metadata_context(&calld->auth_md_context);
-  grpc_error *error = GRPC_ERROR_NONE;
-  if (status != GRPC_CREDENTIALS_OK) {
-    error = grpc_error_set_int(
-        GRPC_ERROR_CREATE_FROM_COPIED_STRING(
-            error_details != NULL && strlen(error_details) > 0
-                ? error_details
-                : "Credentials failed to get metadata."),
-        GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAUTHENTICATED);
-  } else {
-    GPR_ASSERT(num_md <= MAX_CREDENTIALS_METADATA_COUNT);
-    GPR_ASSERT(op->send_initial_metadata);
-    mdb = op->payload->send_initial_metadata.send_initial_metadata;
-    for (i = 0; i < num_md; i++) {
-      add_error(&error,
-                grpc_metadata_batch_add_tail(
-                    exec_ctx, mdb, &calld->md_links[i],
-                    grpc_mdelem_from_slices(
-                        exec_ctx, grpc_slice_ref_internal(md_elems[i].key),
-                        grpc_slice_ref_internal(md_elems[i].value))));
+  grpc_error *error = GRPC_ERROR_REF(input_error);
+  if (error == GRPC_ERROR_NONE) {
+    GPR_ASSERT(calld->md_array.size <= MAX_CREDENTIALS_METADATA_COUNT);
+    GPR_ASSERT(batch->send_initial_metadata);
+    grpc_metadata_batch *mdb =
+        batch->payload->send_initial_metadata.send_initial_metadata;
+    for (size_t i = 0; i < calld->md_array.size; ++i) {
+      add_error(&error, grpc_metadata_batch_add_tail(
+                            exec_ctx, mdb, &calld->md_links[i],
+                            GRPC_MDELEM_REF(calld->md_array.md[i])));
     }
   }
   if (error == GRPC_ERROR_NONE) {
-    grpc_call_next_op(exec_ctx, elem, op);
+    grpc_call_next_op(exec_ctx, elem, batch);
   } else {
-    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error);
+    error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS,
+                               GRPC_STATUS_UNAUTHENTICATED);
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch, error);
   }
 }
 
@@ -156,13 +187,21 @@ void build_auth_metadata_context(grpc_security_connector *sc,
   gpr_free(host);
 }
 
+static void cancel_get_request_metadata(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  grpc_call_credentials_cancel_get_request_metadata(
+      exec_ctx, calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
+}
+
 static void send_security_metadata(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *elem,
-                                   grpc_transport_stream_op_batch *op) {
+                                   grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
   grpc_client_security_context *ctx =
-      (grpc_client_security_context *)op->payload
+      (grpc_client_security_context *)batch->payload
           ->context[GRPC_CONTEXT_SECURITY]
           .value;
   grpc_call_credentials *channel_call_creds =
@@ -171,7 +210,7 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
 
   if (channel_call_creds == NULL && !call_creds_has_md) {
     /* Skip sending metadata altogether. */
-    grpc_call_next_op(exec_ctx, elem, op);
+    grpc_call_next_op(exec_ctx, elem, batch);
     return;
   }
 
@@ -180,7 +219,7 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
                                                           ctx->creds, NULL);
     if (calld->creds == NULL) {
       grpc_transport_stream_op_batch_finish_with_failure(
-          exec_ctx, op,
+          exec_ctx, batch,
           grpc_error_set_int(
               GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                   "Incompatible credentials set on channel and call."),
@@ -194,28 +233,42 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
 
   build_auth_metadata_context(&chand->security_connector->base,
                               chand->auth_context, calld);
-  calld->op = *op; /* Copy op (originates from the caller's stack). */
+
+  grpc_error *cancel_error = set_cancel_func(elem, cancel_get_request_metadata);
+  if (cancel_error != GRPC_ERROR_NONE) {
+    grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
+                                                       cancel_error);
+    return;
+  }
   GPR_ASSERT(calld->pollent != NULL);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
-      on_credentials_metadata, elem);
+  GRPC_CLOSURE_INIT(&calld->closure, on_credentials_metadata, batch,
+                    grpc_schedule_on_exec_ctx);
+  grpc_error *error = GRPC_ERROR_NONE;
+  if (grpc_call_credentials_get_request_metadata(
+          exec_ctx, calld->creds, calld->pollent, calld->auth_md_context,
+          &calld->md_array, &calld->closure, &error)) {
+    // Synchronous return; invoke on_credentials_metadata() directly.
+    on_credentials_metadata(exec_ctx, batch, error);
+    GRPC_ERROR_UNREF(error);
+  }
 }
 
-static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
-                            grpc_security_status status) {
-  grpc_call_element *elem = (grpc_call_element *)user_data;
+static void on_host_checked(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  grpc_transport_stream_op_batch *batch = (grpc_transport_stream_op_batch *)arg;
+  grpc_call_element *elem = batch->handler_private.extra_arg;
   call_data *calld = elem->call_data;
 
-  if (status == GRPC_SECURITY_OK) {
-    send_security_metadata(exec_ctx, elem, &calld->op);
+  if (error == GRPC_ERROR_NONE) {
+    send_security_metadata(exec_ctx, elem, batch);
   } else {
     char *error_msg;
     char *host = grpc_slice_to_c_string(calld->host);
     gpr_asprintf(&error_msg, "Invalid host %s set in :authority metadata.",
                  host);
     gpr_free(host);
-    grpc_call_element_signal_error(
-        exec_ctx, elem,
+    grpc_transport_stream_op_batch_finish_with_failure(
+        exec_ctx, batch,
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_msg),
                            GRPC_ERROR_INT_GRPC_STATUS,
                            GRPC_STATUS_UNAUTHENTICATED));
@@ -223,35 +276,64 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
   }
 }
 
-/* Called either:
-     - in response to an API call (or similar) from above, to send something
-     - a network event (or similar) from below, to receive something
-   op contains type and call direction information, in addition to the data
-   that is being sent or received. */
-static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *elem,
-                                    grpc_transport_stream_op_batch *op) {
-  GPR_TIMER_BEGIN("auth_start_transport_op", 0);
+static void cancel_check_call_host(grpc_exec_ctx *exec_ctx, void *arg,
+                                   grpc_error *error) {
+  grpc_call_element *elem = (grpc_call_element *)arg;
+  call_data *calld = (call_data *)elem->call_data;
+  channel_data *chand = (channel_data *)elem->channel_data;
+  grpc_channel_security_connector_cancel_check_call_host(
+      exec_ctx, chand->security_connector, &calld->closure,
+      GRPC_ERROR_REF(error));
+}
+
+static void auth_start_transport_stream_op_batch(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_transport_stream_op_batch *batch) {
+  GPR_TIMER_BEGIN("auth_start_transport_stream_op_batch", 0);
 
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  grpc_linked_mdelem *l;
-  grpc_client_security_context *sec_ctx = NULL;
 
-  if (!op->cancel_stream) {
+  if (batch->cancel_stream) {
+    while (true) {
+      // Decode the original cancellation state.
+      gpr_atm original_state = gpr_atm_acq_load(&calld->cancellation_state);
+      grpc_error *cancel_error = GRPC_ERROR_NONE;
+      grpc_closure *func = NULL;
+      decode_cancel_state(original_state, &func, &cancel_error);
+      // If we had already set a cancellation error, there's nothing
+      // more to do.
+      if (cancel_error != GRPC_ERROR_NONE) break;
+      // If there's a cancel func, call it.
+      // Note that even if the cancel func has been changed by some
+      // other thread between when we decoded it and now, it will just
+      // be a no-op.
+      cancel_error = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
+      if (func != NULL) {
+        GRPC_CLOSURE_SCHED(exec_ctx, func, GRPC_ERROR_REF(cancel_error));
+      }
+      // Encode the new error into cancellation state.
+      if (gpr_atm_full_cas(&calld->cancellation_state, original_state,
+                           encode_cancel_state_error(cancel_error))) {
+        break;  // Success.
+      }
+      // The cas failed, so try again.
+    }
+  } else {
     /* double checked lock over security context to ensure it's set once */
     if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
       gpr_mu_lock(&calld->security_context_mu);
       if (gpr_atm_acq_load(&calld->security_context_set) == 0) {
-        GPR_ASSERT(op->payload->context != NULL);
-        if (op->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) {
-          op->payload->context[GRPC_CONTEXT_SECURITY].value =
+        GPR_ASSERT(batch->payload->context != NULL);
+        if (batch->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) {
+          batch->payload->context[GRPC_CONTEXT_SECURITY].value =
               grpc_client_security_context_create();
-          op->payload->context[GRPC_CONTEXT_SECURITY].destroy =
+          batch->payload->context[GRPC_CONTEXT_SECURITY].destroy =
               grpc_client_security_context_destroy;
         }
-        sec_ctx = op->payload->context[GRPC_CONTEXT_SECURITY].value;
+        grpc_client_security_context *sec_ctx =
+            batch->payload->context[GRPC_CONTEXT_SECURITY].value;
         GRPC_AUTH_CONTEXT_UNREF(sec_ctx->auth_context, "client auth filter");
         sec_ctx->auth_context =
             GRPC_AUTH_CONTEXT_REF(chand->auth_context, "client_auth_filter");
@@ -261,9 +343,9 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
     }
   }
 
-  if (op->send_initial_metadata) {
-    for (l = op->payload->send_initial_metadata.send_initial_metadata->list
-                 .head;
+  if (batch->send_initial_metadata) {
+    for (grpc_linked_mdelem *l = batch->payload->send_initial_metadata
+                                     .send_initial_metadata->list.head;
          l != NULL; l = l->next) {
       grpc_mdelem md = l->md;
       /* Pointer comparison is OK for md_elems created from the same context.
@@ -283,20 +365,34 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
       }
     }
     if (calld->have_host) {
-      char *call_host = grpc_slice_to_c_string(calld->host);
-      calld->op = *op; /* Copy op (originates from the caller's stack). */
-      grpc_channel_security_connector_check_call_host(
-          exec_ctx, chand->security_connector, call_host, chand->auth_context,
-          on_host_checked, elem);
-      gpr_free(call_host);
-      GPR_TIMER_END("auth_start_transport_op", 0);
+      grpc_error *cancel_error = set_cancel_func(elem, cancel_check_call_host);
+      if (cancel_error != GRPC_ERROR_NONE) {
+        grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, batch,
+                                                           cancel_error);
+      } else {
+        char *call_host = grpc_slice_to_c_string(calld->host);
+        batch->handler_private.extra_arg = elem;
+        grpc_error *error = GRPC_ERROR_NONE;
+        if (grpc_channel_security_connector_check_call_host(
+                exec_ctx, chand->security_connector, call_host,
+                chand->auth_context,
+                GRPC_CLOSURE_INIT(&calld->closure, on_host_checked, batch,
+                                  grpc_schedule_on_exec_ctx),
+                &error)) {
+          // Synchronous return; invoke on_host_checked() directly.
+          on_host_checked(exec_ctx, batch, error);
+          GRPC_ERROR_UNREF(error);
+        }
+        gpr_free(call_host);
+      }
+      GPR_TIMER_END("auth_start_transport_stream_op_batch", 0);
       return; /* early exit */
     }
   }
 
   /* pass control down the stack */
-  grpc_call_next_op(exec_ctx, elem, op);
-  GPR_TIMER_END("auth_start_transport_op", 0);
+  grpc_call_next_op(exec_ctx, elem, batch);
+  GPR_TIMER_END("auth_start_transport_stream_op_batch", 0);
 }
 
 /* Constructor for call_data */
@@ -321,6 +417,7 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
                               grpc_closure *ignored) {
   call_data *calld = elem->call_data;
+  grpc_credentials_mdelem_array_destroy(exec_ctx, &calld->md_array);
   grpc_call_credentials_unref(exec_ctx, calld->creds);
   if (calld->have_host) {
     grpc_slice_unref_internal(exec_ctx, calld->host);
@@ -330,6 +427,11 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
   }
   reset_auth_metadata_context(&calld->auth_md_context);
   gpr_mu_destroy(&calld->security_context_mu);
+  gpr_atm cancel_state = gpr_atm_acq_load(&calld->cancellation_state);
+  grpc_error *cancel_error = GRPC_ERROR_NONE;
+  grpc_closure *cancel_func = NULL;
+  decode_cancel_state(cancel_state, &cancel_func, &cancel_error);
+  GRPC_ERROR_UNREF(cancel_error);
 }
 
 /* Constructor for channel_data */
@@ -379,7 +481,15 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
 }
 
 const grpc_channel_filter grpc_client_auth_filter = {
-    auth_start_transport_op, grpc_channel_next_op,       sizeof(call_data),
-    init_call_elem,          set_pollset_or_pollset_set, destroy_call_elem,
-    sizeof(channel_data),    init_channel_elem,          destroy_channel_elem,
-    grpc_call_next_get_peer, grpc_channel_next_get_info, "client-auth"};
+    auth_start_transport_stream_op_batch,
+    grpc_channel_next_op,
+    sizeof(call_data),
+    init_call_elem,
+    set_pollset_or_pollset_set,
+    destroy_call_elem,
+    sizeof(channel_data),
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    grpc_channel_next_get_info,
+    "client-auth"};
diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
index f4ed81db1a..5e41b94ff8 100644
--- a/src/core/lib/security/transport/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -60,7 +60,8 @@ typedef struct {
   gpr_refcount ref;
 } secure_endpoint;
 
-grpc_tracer_flag grpc_trace_secure_endpoint = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_secure_endpoint =
+    GRPC_TRACER_INITIALIZER(false, "secure_endpoint");
 
 static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) {
   secure_endpoint *ep = secure_ep;
diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c
index 3c0c24254b..a7568b995f 100644
--- a/src/core/lib/security/transport/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -45,7 +45,7 @@
 
 #ifndef NDEBUG
 grpc_tracer_flag grpc_trace_security_connector_refcount =
-    GRPC_TRACER_INITIALIZER(false);
+    GRPC_TRACER_INITIALIZER(false, "security_connector_refcount");
 #endif
 
 /* -- Constants. -- */
@@ -136,15 +136,27 @@ void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_channel_security_connector_check_call_host(
+bool grpc_channel_security_connector_check_call_host(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
     const char *host, grpc_auth_context *auth_context,
-    grpc_security_call_host_check_cb cb, void *user_data) {
+    grpc_closure *on_call_host_checked, grpc_error **error) {
   if (sc == NULL || sc->check_call_host == NULL) {
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR);
-  } else {
-    sc->check_call_host(exec_ctx, sc, host, auth_context, cb, user_data);
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "cannot check call host -- no security connector");
+    return true;
   }
+  return sc->check_call_host(exec_ctx, sc, host, auth_context,
+                             on_call_host_checked, error);
+}
+
+void grpc_channel_security_connector_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  if (sc == NULL || sc->cancel_check_call_host == NULL) {
+    GRPC_ERROR_UNREF(error);
+    return;
+  }
+  sc->cancel_check_call_host(exec_ctx, sc, on_call_host_checked, error);
 }
 
 #ifndef NDEBUG
@@ -368,13 +380,19 @@ static void fake_server_check_peer(grpc_exec_ctx *exec_ctx,
   fake_check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked);
 }
 
-static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
+static bool fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
                                          grpc_channel_security_connector *sc,
                                          const char *host,
                                          grpc_auth_context *auth_context,
-                                         grpc_security_call_host_check_cb cb,
-                                         void *user_data) {
-  cb(exec_ctx, user_data, GRPC_SECURITY_OK);
+                                         grpc_closure *on_call_host_checked,
+                                         grpc_error **error) {
+  return true;
+}
+
+static void fake_channel_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static void fake_channel_add_handshakers(
@@ -383,8 +401,7 @@ static void fake_channel_add_handshakers(
   grpc_handshake_manager_add(
       handshake_mgr,
       grpc_security_handshaker_create(
-          exec_ctx, tsi_create_adapter_handshaker(
-                        tsi_create_fake_handshaker(true /* is_client */)),
+          exec_ctx, tsi_create_fake_handshaker(true /* is_client */),
           &sc->base));
 }
 
@@ -394,8 +411,7 @@ static void fake_server_add_handshakers(grpc_exec_ctx *exec_ctx,
   grpc_handshake_manager_add(
       handshake_mgr,
       grpc_security_handshaker_create(
-          exec_ctx, tsi_create_adapter_handshaker(
-                        tsi_create_fake_handshaker(false /* is_client */)),
+          exec_ctx, tsi_create_fake_handshaker(false /* is_client */),
           &sc->base));
 }
 
@@ -415,6 +431,7 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = fake_channel_check_call_host;
+  c->base.cancel_check_call_host = fake_channel_cancel_check_call_host;
   c->base.add_handshakers = fake_channel_add_handshakers;
   c->target = gpr_strdup(target);
   const char *expected_targets = grpc_fake_transport_get_expected_targets(args);
@@ -665,26 +682,35 @@ void tsi_shallow_peer_destruct(tsi_peer *peer) {
   if (peer->properties != NULL) gpr_free(peer->properties);
 }
 
-static void ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
+static bool ssl_channel_check_call_host(grpc_exec_ctx *exec_ctx,
                                         grpc_channel_security_connector *sc,
                                         const char *host,
                                         grpc_auth_context *auth_context,
-                                        grpc_security_call_host_check_cb cb,
-                                        void *user_data) {
+                                        grpc_closure *on_call_host_checked,
+                                        grpc_error **error) {
   grpc_ssl_channel_security_connector *c =
       (grpc_ssl_channel_security_connector *)sc;
   grpc_security_status status = GRPC_SECURITY_ERROR;
   tsi_peer peer = tsi_shallow_peer_from_ssl_auth_context(auth_context);
   if (ssl_host_matches_name(&peer, host)) status = GRPC_SECURITY_OK;
-
   /* If the target name was overridden, then the original target_name was
      'checked' transitively during the previous peer check at the end of the
      handshake. */
   if (c->overridden_target_name != NULL && strcmp(host, c->target_name) == 0) {
     status = GRPC_SECURITY_OK;
   }
-  cb(exec_ctx, user_data, status);
+  if (status != GRPC_SECURITY_OK) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "call host does not match SSL server name");
+  }
   tsi_shallow_peer_destruct(&peer);
+  return true;
+}
+
+static void ssl_channel_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error) {
+  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_security_connector_vtable ssl_channel_vtable = {
@@ -813,6 +839,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = ssl_channel_check_call_host;
+  c->base.cancel_check_call_host = ssl_channel_cancel_check_call_host;
   c->base.add_handshakers = ssl_channel_add_handshakers;
   gpr_split_host_port(target_name, &c->target_name, &port);
   gpr_free(port);
diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h
index 1c0fe40045..4f9b63ad20 100644
--- a/src/core/lib/security/transport/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -117,27 +117,38 @@ grpc_security_connector *grpc_security_connector_find_in_args(
 
 typedef struct grpc_channel_security_connector grpc_channel_security_connector;
 
-typedef void (*grpc_security_call_host_check_cb)(grpc_exec_ctx *exec_ctx,
-                                                 void *user_data,
-                                                 grpc_security_status status);
-
 struct grpc_channel_security_connector {
   grpc_security_connector base;
   grpc_call_credentials *request_metadata_creds;
-  void (*check_call_host)(grpc_exec_ctx *exec_ctx,
+  bool (*check_call_host)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc, const char *host,
                           grpc_auth_context *auth_context,
-                          grpc_security_call_host_check_cb cb, void *user_data);
+                          grpc_closure *on_call_host_checked,
+                          grpc_error **error);
+  void (*cancel_check_call_host)(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_security_connector *sc,
+                                 grpc_closure *on_call_host_checked,
+                                 grpc_error *error);
   void (*add_handshakers)(grpc_exec_ctx *exec_ctx,
                           grpc_channel_security_connector *sc,
                           grpc_handshake_manager *handshake_mgr);
 };
 
-/* Checks that the host that will be set for a call is acceptable. */
-void grpc_channel_security_connector_check_call_host(
+/// Checks that the host that will be set for a call is acceptable.
+/// Returns true if completed synchronously, in which case \a error will
+/// be set to indicate the result.  Otherwise, \a on_call_host_checked
+/// will be invoked when complete.
+bool grpc_channel_security_connector_check_call_host(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
     const char *host, grpc_auth_context *auth_context,
-    grpc_security_call_host_check_cb cb, void *user_data);
+    grpc_closure *on_call_host_checked, grpc_error **error);
+
+/// Cancels a pending asychronous call to
+/// grpc_channel_security_connector_check_call_host() with
+/// \a on_call_host_checked as its callback.
+void grpc_channel_security_connector_cancel_check_call_host(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_closure *on_call_host_checked, grpc_error *error);
 
 /* Registers handshakers with \a handshake_mgr. */
 void grpc_channel_security_connector_add_handshakers(
diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
index 4e6914be7b..9bf3f0ca0f 100644
--- a/src/core/lib/security/transport/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -27,14 +27,9 @@
 #include "src/core/lib/slice/slice_internal.h"
 
 typedef struct call_data {
-  grpc_metadata_batch *recv_initial_metadata;
-  /* Closure to call when finished with the auth_on_recv hook. */
-  grpc_closure *on_done_recv;
-  /* Receive closures are chained: we inject this closure as the on_done_recv
-     up-call on transport_op, and remember to call our on_done_recv member after
-     handling it. */
-  grpc_closure auth_on_recv;
-  grpc_transport_stream_op_batch *transport_op;
+  grpc_transport_stream_op_batch *recv_initial_metadata_batch;
+  grpc_closure *original_recv_initial_metadata_ready;
+  grpc_closure recv_initial_metadata_ready;
   grpc_metadata_array md;
   const grpc_metadata *consumed_md;
   size_t num_consumed_md;
@@ -90,125 +85,96 @@ static void on_md_processing_done(
     grpc_status_code status, const char *error_details) {
   grpc_call_element *elem = user_data;
   call_data *calld = elem->call_data;
+  grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-
   /* TODO(jboeuf): Implement support for response_md. */
   if (response_md != NULL && num_response_md > 0) {
     gpr_log(GPR_INFO,
             "response_md in auth metadata processing not supported for now. "
             "Ignoring...");
   }
-
+  grpc_error *error = GRPC_ERROR_NONE;
   if (status == GRPC_STATUS_OK) {
     calld->consumed_md = consumed_md;
     calld->num_consumed_md = num_consumed_md;
-    /* TODO(ctiller): propagate error */
-    GRPC_LOG_IF_ERROR(
-        "grpc_metadata_batch_filter",
-        grpc_metadata_batch_filter(&exec_ctx, calld->recv_initial_metadata,
-                                   remove_consumed_md, elem,
-                                   "Response metadata filtering error"));
-    for (size_t i = 0; i < calld->md.count; i++) {
-      grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].key);
-      grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].value);
-    }
-    grpc_metadata_array_destroy(&calld->md);
-    GRPC_CLOSURE_SCHED(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE);
+    error = grpc_metadata_batch_filter(
+        &exec_ctx, batch->payload->recv_initial_metadata.recv_initial_metadata,
+        remove_consumed_md, elem, "Response metadata filtering error");
   } else {
-    for (size_t i = 0; i < calld->md.count; i++) {
-      grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].key);
-      grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].value);
-    }
-    grpc_metadata_array_destroy(&calld->md);
-    error_details = error_details != NULL
-                        ? error_details
-                        : "Authentication metadata processing failed.";
-    if (calld->transport_op->send_message) {
-      grpc_byte_stream_destroy(
-          &exec_ctx, calld->transport_op->payload->send_message.send_message);
-      calld->transport_op->payload->send_message.send_message = NULL;
+    if (error_details == NULL) {
+      error_details = "Authentication metadata processing failed.";
     }
-    GRPC_CLOSURE_SCHED(
-        &exec_ctx, calld->on_done_recv,
+    error =
         grpc_error_set_int(GRPC_ERROR_CREATE_FROM_COPIED_STRING(error_details),
-                           GRPC_ERROR_INT_GRPC_STATUS, status));
+                           GRPC_ERROR_INT_GRPC_STATUS, status);
   }
-
+  for (size_t i = 0; i < calld->md.count; i++) {
+    grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].key);
+    grpc_slice_unref_internal(&exec_ctx, calld->md.metadata[i].value);
+  }
+  grpc_metadata_array_destroy(&calld->md);
+  GRPC_CLOSURE_SCHED(&exec_ctx, calld->original_recv_initial_metadata_ready,
+                     error);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void auth_on_recv(grpc_exec_ctx *exec_ctx, void *user_data,
-                         grpc_error *error) {
-  grpc_call_element *elem = user_data;
-  call_data *calld = elem->call_data;
+static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  grpc_call_element *elem = arg;
   channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  grpc_transport_stream_op_batch *batch = calld->recv_initial_metadata_batch;
   if (error == GRPC_ERROR_NONE) {
     if (chand->creds != NULL && chand->creds->processor.process != NULL) {
-      calld->md = metadata_batch_to_md_array(calld->recv_initial_metadata);
+      calld->md = metadata_batch_to_md_array(
+          batch->payload->recv_initial_metadata.recv_initial_metadata);
       chand->creds->processor.process(
           chand->creds->processor.state, calld->auth_context,
           calld->md.metadata, calld->md.count, on_md_processing_done, elem);
       return;
     }
   }
-  GRPC_CLOSURE_SCHED(exec_ctx, calld->on_done_recv, GRPC_ERROR_REF(error));
+  GRPC_CLOSURE_RUN(exec_ctx, calld->original_recv_initial_metadata_ready,
+                   GRPC_ERROR_REF(error));
 }
 
-static void set_recv_ops_md_callbacks(grpc_call_element *elem,
-                                      grpc_transport_stream_op_batch *op) {
+static void auth_start_transport_stream_op_batch(
+    grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
+    grpc_transport_stream_op_batch *batch) {
   call_data *calld = elem->call_data;
-
-  if (op->recv_initial_metadata) {
-    /* substitute our callback for the higher callback */
-    calld->recv_initial_metadata =
-        op->payload->recv_initial_metadata.recv_initial_metadata;
-    calld->on_done_recv =
-        op->payload->recv_initial_metadata.recv_initial_metadata_ready;
-    op->payload->recv_initial_metadata.recv_initial_metadata_ready =
-        &calld->auth_on_recv;
-    calld->transport_op = op;
+  if (batch->recv_initial_metadata) {
+    // Inject our callback.
+    calld->recv_initial_metadata_batch = batch;
+    calld->original_recv_initial_metadata_ready =
+        batch->payload->recv_initial_metadata.recv_initial_metadata_ready;
+    batch->payload->recv_initial_metadata.recv_initial_metadata_ready =
+        &calld->recv_initial_metadata_ready;
   }
-}
-
-/* Called either:
-     - in response to an API call (or similar) from above, to send something
-     - a network event (or similar) from below, to receive something
-   op contains type and call direction information, in addition to the data
-   that is being sent or received. */
-static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
-                                    grpc_call_element *elem,
-                                    grpc_transport_stream_op_batch *op) {
-  set_recv_ops_md_callbacks(elem, op);
-  grpc_call_next_op(exec_ctx, elem, op);
+  grpc_call_next_op(exec_ctx, elem, batch);
 }
 
 /* Constructor for call_data */
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
-  /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
-  grpc_server_security_context *server_ctx = NULL;
-
-  /* initialize members */
-  memset(calld, 0, sizeof(*calld));
-  GRPC_CLOSURE_INIT(&calld->auth_on_recv, auth_on_recv, elem,
+  GRPC_CLOSURE_INIT(&calld->recv_initial_metadata_ready,
+                    recv_initial_metadata_ready, elem,
                     grpc_schedule_on_exec_ctx);
-
+  // Create server security context.  Set its auth context from channel
+  // data and save it in the call context.
+  grpc_server_security_context *server_ctx =
+      grpc_server_security_context_create();
+  server_ctx->auth_context = grpc_auth_context_create(chand->auth_context);
+  calld->auth_context = server_ctx->auth_context;
   if (args->context[GRPC_CONTEXT_SECURITY].value != NULL) {
     args->context[GRPC_CONTEXT_SECURITY].destroy(
         args->context[GRPC_CONTEXT_SECURITY].value);
   }
-
-  server_ctx = grpc_server_security_context_create();
-  server_ctx->auth_context = grpc_auth_context_create(chand->auth_context);
-  calld->auth_context = server_ctx->auth_context;
-
   args->context[GRPC_CONTEXT_SECURITY].value = server_ctx;
   args->context[GRPC_CONTEXT_SECURITY].destroy =
       grpc_server_security_context_destroy;
-
   return GRPC_ERROR_NONE;
 }
 
@@ -221,19 +187,15 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_element *elem,
                                      grpc_channel_element_args *args) {
+  GPR_ASSERT(!args->is_last);
+  channel_data *chand = elem->channel_data;
   grpc_auth_context *auth_context =
       grpc_find_auth_context_in_args(args->channel_args);
-  grpc_server_credentials *creds =
-      grpc_find_server_credentials_in_args(args->channel_args);
-  /* grab pointers to our data from the channel element */
-  channel_data *chand = elem->channel_data;
-
-  GPR_ASSERT(!args->is_last);
   GPR_ASSERT(auth_context != NULL);
-
-  /* initialize members */
   chand->auth_context =
       GRPC_AUTH_CONTEXT_REF(auth_context, "server_auth_filter");
+  grpc_server_credentials *creds =
+      grpc_find_server_credentials_in_args(args->channel_args);
   chand->creds = grpc_server_credentials_ref(creds);
   return GRPC_ERROR_NONE;
 }
@@ -241,14 +203,13 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
 /* Destructor for channel data */
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
                                  grpc_channel_element *elem) {
-  /* grab pointers to our data from the channel element */
   channel_data *chand = elem->channel_data;
   GRPC_AUTH_CONTEXT_UNREF(chand->auth_context, "server_auth_filter");
   grpc_server_credentials_unref(exec_ctx, chand->creds);
 }
 
 const grpc_channel_filter grpc_server_auth_filter = {
-    auth_start_transport_op,
+    auth_start_transport_stream_op_batch,
     grpc_channel_next_op,
     sizeof(call_data),
     init_call_elem,
diff --git a/src/core/lib/support/arena.c b/src/core/lib/support/arena.c
index b433c61b4c..9e0f73ae3d 100644
--- a/src/core/lib/support/arena.c
+++ b/src/core/lib/support/arena.c
@@ -38,7 +38,7 @@ struct gpr_arena {
 
 gpr_arena *gpr_arena_create(size_t initial_size) {
   initial_size = ROUND_UP_TO_ALIGNMENT_SIZE(initial_size);
-  gpr_arena *a = gpr_zalloc(sizeof(gpr_arena) + initial_size);
+  gpr_arena *a = (gpr_arena *)gpr_zalloc(sizeof(gpr_arena) + initial_size);
   a->initial_zone.size_end = initial_size;
   return a;
 }
@@ -64,7 +64,7 @@ void *gpr_arena_alloc(gpr_arena *arena, size_t size) {
     zone *next_z = (zone *)gpr_atm_acq_load(&z->next_atm);
     if (next_z == NULL) {
       size_t next_z_size = (size_t)gpr_atm_no_barrier_load(&arena->size_so_far);
-      next_z = gpr_zalloc(sizeof(zone) + next_z_size);
+      next_z = (zone *)gpr_zalloc(sizeof(zone) + next_z_size);
       next_z->size_begin = z->size_end;
       next_z->size_end = z->size_end + next_z_size;
       if (!gpr_atm_rel_cas(&z->next_atm, (gpr_atm)NULL, (gpr_atm)next_z)) {
diff --git a/src/core/lib/support/atm.c b/src/core/lib/support/atm.c
index caa0bafe33..2f37d62f76 100644
--- a/src/core/lib/support/atm.c
+++ b/src/core/lib/support/atm.c
@@ -21,12 +21,12 @@
 
 gpr_atm gpr_atm_no_barrier_clamped_add(gpr_atm *value, gpr_atm delta,
                                        gpr_atm min, gpr_atm max) {
-  gpr_atm current;
-  gpr_atm new;
+  gpr_atm current_value;
+  gpr_atm new_value;
   do {
-    current = gpr_atm_no_barrier_load(value);
-    new = GPR_CLAMP(current + delta, min, max);
-    if (new == current) break;
-  } while (!gpr_atm_no_barrier_cas(value, current, new));
-  return new;
+    current_value = gpr_atm_no_barrier_load(value);
+    new_value = GPR_CLAMP(current_value + delta, min, max);
+    if (new_value == current_value) break;
+  } while (!gpr_atm_no_barrier_cas(value, current_value, new_value));
+  return new_value;
 }
diff --git a/src/core/lib/support/avl.c b/src/core/lib/support/avl.c
index aa0f665272..a6178fdbce 100644
--- a/src/core/lib/support/avl.c
+++ b/src/core/lib/support/avl.c
@@ -76,7 +76,7 @@ static gpr_avl_node *assert_invariants(gpr_avl_node *n) { return n; }
 
 gpr_avl_node *new_node(void *key, void *value, gpr_avl_node *left,
                        gpr_avl_node *right) {
-  gpr_avl_node *node = gpr_malloc(sizeof(*node));
+  gpr_avl_node *node = (gpr_avl_node *)gpr_malloc(sizeof(*node));
   gpr_ref_init(&node->refs, 1);
   node->key = key;
   node->value = value;
diff --git a/src/core/lib/support/env.h b/src/core/lib/support/env.h
index 18bc08ac62..e2c012a728 100644
--- a/src/core/lib/support/env.h
+++ b/src/core/lib/support/env.h
@@ -36,6 +36,12 @@ char *gpr_getenv(const char *name);
 /* Sets the the environment with the specified name to the specified value. */
 void gpr_setenv(const char *name, const char *value);
 
+/* This is a version of gpr_getenv that does not produce any output if it has to
+   use an insecure version of the function. It is ONLY to be used to solve the
+   problem in which we need to check an env variable to configure the verbosity
+   level of logging. So DO NOT USE THIS. */
+const char *gpr_getenv_silent(const char *name, char **dst);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/core/lib/support/env_linux.c b/src/core/lib/support/env_linux.c
index 0c79a2c401..4c45a977ca 100644
--- a/src/core/lib/support/env_linux.c
+++ b/src/core/lib/support/env_linux.c
@@ -38,7 +38,9 @@
 
 #include "src/core/lib/support/string.h"
 
-char *gpr_getenv(const char *name) {
+const char *gpr_getenv_silent(const char *name, char **dst) {
+  const char *insecure_func_used = NULL;
+  char *result = NULL;
 #if defined(GPR_BACKWARDS_COMPATIBILITY_MODE)
   typedef char *(*getenv_type)(const char *);
   static getenv_type getenv_func = NULL;
@@ -48,22 +50,28 @@ char *gpr_getenv(const char *name) {
   for (size_t i = 0; getenv_func == NULL && i < GPR_ARRAY_SIZE(names); i++) {
     getenv_func = (getenv_type)dlsym(RTLD_DEFAULT, names[i]);
     if (getenv_func != NULL && strstr(names[i], "secure") == NULL) {
-      gpr_log(GPR_DEBUG,
-              "Warning: insecure environment read function '%s' used",
-              names[i]);
+      insecure_func_used = names[i];
     }
   }
-  char *result = getenv_func(name);
-  return result == NULL ? result : gpr_strdup(result);
+  result = getenv_func(name);
 #elif __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17)
-  char *result = secure_getenv(name);
-  return result == NULL ? result : gpr_strdup(result);
+  result = secure_getenv(name);
 #else
-  gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used",
-          "getenv");
-  char *result = getenv(name);
-  return result == NULL ? result : gpr_strdup(result);
+  result = getenv(name);
+  insecure_func_used = "getenv";
 #endif
+  *dst = result == NULL ? result : gpr_strdup(result);
+  return insecure_func_used;
+}
+
+char *gpr_getenv(const char *name) {
+  char *result = NULL;
+  const char *insecure_func_used = gpr_getenv_silent(name, &result);
+  if (insecure_func_used != NULL) {
+    gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used",
+            insecure_func_used);
+  }
+  return result;
 }
 
 void gpr_setenv(const char *name, const char *value) {
diff --git a/src/core/lib/support/env_posix.c b/src/core/lib/support/env_posix.c
index bdbc4da95a..b88822ca02 100644
--- a/src/core/lib/support/env_posix.c
+++ b/src/core/lib/support/env_posix.c
@@ -29,6 +29,11 @@
 #include <grpc/support/string_util.h>
 #include "src/core/lib/support/string.h"
 
+const char *gpr_getenv_silent(const char *name, char **dst) {
+  *dst = gpr_getenv(name);
+  return NULL;
+}
+
 char *gpr_getenv(const char *name) {
   char *result = getenv(name);
   return result == NULL ? result : gpr_strdup(result);
diff --git a/src/core/lib/support/env_windows.c b/src/core/lib/support/env_windows.c
index c1d557e219..652eeb61c6 100644
--- a/src/core/lib/support/env_windows.c
+++ b/src/core/lib/support/env_windows.c
@@ -30,6 +30,11 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+const char *gpr_getenv_silent(const char *name, char **dst) {
+  *dst = gpr_getenv(name);
+  return NULL;
+}
+
 char *gpr_getenv(const char *name) {
   char *result = NULL;
   DWORD size;
diff --git a/src/core/lib/support/log.c b/src/core/lib/support/log.c
index bcc336b8ae..fadb4d9a2c 100644
--- a/src/core/lib/support/log.c
+++ b/src/core/lib/support/log.c
@@ -64,7 +64,8 @@ void gpr_set_log_verbosity(gpr_log_severity min_severity_to_print) {
 }
 
 void gpr_log_verbosity_init() {
-  char *verbosity = gpr_getenv("GRPC_VERBOSITY");
+  char *verbosity = NULL;
+  const char *insecure_getenv = gpr_getenv_silent("GRPC_VERBOSITY", &verbosity);
 
   gpr_atm min_severity_to_print = GPR_LOG_SEVERITY_ERROR;
   if (verbosity != NULL) {
@@ -81,6 +82,11 @@ void gpr_log_verbosity_init() {
       GPR_LOG_VERBOSITY_UNSET) {
     gpr_atm_no_barrier_store(&g_min_severity_to_print, min_severity_to_print);
   }
+
+  if (insecure_getenv != NULL) {
+    gpr_log(GPR_DEBUG, "Warning: insecure environment read function '%s' used",
+            insecure_getenv);
+  }
 }
 
 void gpr_set_log_function(gpr_log_func f) {
diff --git a/src/core/lib/support/log_linux.c b/src/core/lib/support/log_linux.c
index 5c512661a3..61d2346427 100644
--- a/src/core/lib/support/log_linux.c
+++ b/src/core/lib/support/log_linux.c
@@ -64,6 +64,8 @@ void gpr_default_log(gpr_log_func_args *args) {
   time_t timer;
   gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
   struct tm tm;
+  static __thread long tid = 0;
+  if (tid == 0) tid = gettid();
 
   timer = (time_t)now.tv_sec;
   final_slash = strrchr(args->file, '/');
@@ -81,7 +83,7 @@ void gpr_default_log(gpr_log_func_args *args) {
 
   gpr_asprintf(&prefix, "%s%s.%09" PRId32 " %7ld %s:%d]",
                gpr_log_severity_string(args->severity), time_buffer,
-               now.tv_nsec, gettid(), display_file, args->line);
+               now.tv_nsec, tid, display_file, args->line);
 
   fprintf(stderr, "%-60s %s\n", prefix, args->message);
   gpr_free(prefix);
diff --git a/src/core/lib/support/mpscq.c b/src/core/lib/support/mpscq.c
index 58c4c435d3..e9f893988d 100644
--- a/src/core/lib/support/mpscq.c
+++ b/src/core/lib/support/mpscq.c
@@ -31,12 +31,11 @@ void gpr_mpscq_destroy(gpr_mpscq *q) {
   GPR_ASSERT(q->tail == &q->stub);
 }
 
-bool gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) {
+void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) {
   gpr_atm_no_barrier_store(&n->next, (gpr_atm)NULL);
   gpr_mpscq_node *prev =
       (gpr_mpscq_node *)gpr_atm_full_xchg(&q->head, (gpr_atm)n);
   gpr_atm_rel_store(&prev->next, (gpr_atm)n);
-  return prev == &q->stub;
 }
 
 gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) {
@@ -78,25 +77,3 @@ gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty) {
   *empty = false;
   return NULL;
 }
-
-void gpr_locked_mpscq_init(gpr_locked_mpscq *q) {
-  gpr_mpscq_init(&q->queue);
-  q->read_lock = GPR_SPINLOCK_INITIALIZER;
-}
-
-void gpr_locked_mpscq_destroy(gpr_locked_mpscq *q) {
-  gpr_mpscq_destroy(&q->queue);
-}
-
-bool gpr_locked_mpscq_push(gpr_locked_mpscq *q, gpr_mpscq_node *n) {
-  return gpr_mpscq_push(&q->queue, n);
-}
-
-gpr_mpscq_node *gpr_locked_mpscq_pop(gpr_locked_mpscq *q) {
-  if (gpr_spinlock_trylock(&q->read_lock)) {
-    gpr_mpscq_node *n = gpr_mpscq_pop(&q->queue);
-    gpr_spinlock_unlock(&q->read_lock);
-    return n;
-  }
-  return NULL;
-}
diff --git a/src/core/lib/support/mpscq.h b/src/core/lib/support/mpscq.h
index 2f4739d7f8..daa51768f7 100644
--- a/src/core/lib/support/mpscq.h
+++ b/src/core/lib/support/mpscq.h
@@ -22,7 +22,6 @@
 #include <grpc/support/atm.h>
 #include <stdbool.h>
 #include <stddef.h>
-#include "src/core/lib/support/spinlock.h"
 
 // Multiple-producer single-consumer lock free queue, based upon the
 // implementation from Dmitry Vyukov here:
@@ -44,34 +43,11 @@ typedef struct gpr_mpscq {
 void gpr_mpscq_init(gpr_mpscq *q);
 void gpr_mpscq_destroy(gpr_mpscq *q);
 // Push a node
-// Thread safe - can be called from multiple threads concurrently
-// Returns true if this was possibly the first node (may return true
-// sporadically, will not return false sporadically)
-bool gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n);
+void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n);
 // Pop a node (returns NULL if no node is ready - which doesn't indicate that
 // the queue is empty!!)
-// Thread compatible - can only be called from one thread at a time
 gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q);
 // Pop a node; sets *empty to true if the queue is empty, or false if it is not
 gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty);
 
-// An mpscq with a spinlock: it's safe to pop from multiple threads, but doing
-// only one thread will succeed concurrently
-typedef struct gpr_locked_mpscq {
-  gpr_mpscq queue;
-  gpr_spinlock read_lock;
-} gpr_locked_mpscq;
-
-void gpr_locked_mpscq_init(gpr_locked_mpscq *q);
-void gpr_locked_mpscq_destroy(gpr_locked_mpscq *q);
-// Push a node
-// Thread safe - can be called from multiple threads concurrently
-// Returns true if this was possibly the first node (may return true
-// sporadically, will not return false sporadically)
-bool gpr_locked_mpscq_push(gpr_locked_mpscq *q, gpr_mpscq_node *n);
-// Pop a node (returns NULL if no node is ready - which doesn't indicate that
-// the queue is empty!!)
-// Thread safe - can be called from multiple threads concurrently
-gpr_mpscq_node *gpr_locked_mpscq_pop(gpr_locked_mpscq *q);
-
 #endif /* GRPC_CORE_LIB_SUPPORT_MPSCQ_H */
diff --git a/src/core/lib/support/stack_lockfree.c b/src/core/lib/support/stack_lockfree.c
new file mode 100644
index 0000000000..0fb64ed001
--- /dev/null
+++ b/src/core/lib/support/stack_lockfree.c
@@ -0,0 +1,137 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/support/stack_lockfree.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/atm.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+
+/* The lockfree node structure is a single architecture-level
+   word that allows for an atomic CAS to set it up. */
+struct lockfree_node_contents {
+  /* next thing to look at. Actual index for head, next index otherwise */
+  uint16_t index;
+#ifdef GPR_ARCH_64
+  uint16_t pad;
+  uint32_t aba_ctr;
+#else
+#ifdef GPR_ARCH_32
+  uint16_t aba_ctr;
+#else
+#error Unsupported bit width architecture
+#endif
+#endif
+};
+
+/* Use a union to make sure that these are in the same bits as an atm word */
+typedef union lockfree_node {
+  gpr_atm atm;
+  struct lockfree_node_contents contents;
+} lockfree_node;
+
+/* make sure that entries aligned to 8-bytes */
+#define ENTRY_ALIGNMENT_BITS 3
+/* reserve this entry as invalid */
+#define INVALID_ENTRY_INDEX ((1 << 16) - 1)
+
+struct gpr_stack_lockfree {
+  lockfree_node *entries;
+  lockfree_node head; /* An atomic entry describing curr head */
+};
+
+gpr_stack_lockfree *gpr_stack_lockfree_create(size_t entries) {
+  gpr_stack_lockfree *stack;
+  stack = (gpr_stack_lockfree *)gpr_malloc(sizeof(*stack));
+  /* Since we only allocate 16 bits to represent an entry number,
+   * make sure that we are within the desired range */
+  /* Reserve the highest entry number as a dummy */
+  GPR_ASSERT(entries < INVALID_ENTRY_INDEX);
+  stack->entries = (lockfree_node *)gpr_malloc_aligned(
+      entries * sizeof(stack->entries[0]), ENTRY_ALIGNMENT_BITS);
+  /* Clear out all entries */
+  memset(stack->entries, 0, entries * sizeof(stack->entries[0]));
+  memset(&stack->head, 0, sizeof(stack->head));
+
+  GPR_ASSERT(sizeof(stack->entries->atm) == sizeof(stack->entries->contents));
+
+  /* Point the head at reserved dummy entry */
+  stack->head.contents.index = INVALID_ENTRY_INDEX;
+/* Fill in the pad and aba_ctr to avoid confusing memcheck tools */
+#ifdef GPR_ARCH_64
+  stack->head.contents.pad = 0;
+#endif
+  stack->head.contents.aba_ctr = 0;
+  return stack;
+}
+
+void gpr_stack_lockfree_destroy(gpr_stack_lockfree *stack) {
+  gpr_free_aligned(stack->entries);
+  gpr_free(stack);
+}
+
+int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) {
+  lockfree_node head;
+  lockfree_node newhead;
+  lockfree_node curent;
+  lockfree_node newent;
+
+  /* First fill in the entry's index and aba ctr for new head */
+  newhead.contents.index = (uint16_t)entry;
+#ifdef GPR_ARCH_64
+  /* Fill in the pad to avoid confusing memcheck tools */
+  newhead.contents.pad = 0;
+#endif
+
+  /* Also post-increment the aba_ctr */
+  curent.atm = gpr_atm_no_barrier_load(&stack->entries[entry].atm);
+  newhead.contents.aba_ctr = ++curent.contents.aba_ctr;
+  gpr_atm_no_barrier_store(&stack->entries[entry].atm, curent.atm);
+
+  do {
+    /* Atomically get the existing head value for use */
+    head.atm = gpr_atm_no_barrier_load(&(stack->head.atm));
+    /* Point to it */
+    newent.atm = gpr_atm_no_barrier_load(&stack->entries[entry].atm);
+    newent.contents.index = head.contents.index;
+    gpr_atm_no_barrier_store(&stack->entries[entry].atm, newent.atm);
+  } while (!gpr_atm_rel_cas(&(stack->head.atm), head.atm, newhead.atm));
+  /* Use rel_cas above to make sure that entry index is set properly */
+  return head.contents.index == INVALID_ENTRY_INDEX;
+}
+
+int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) {
+  lockfree_node head;
+  lockfree_node newhead;
+
+  do {
+    head.atm = gpr_atm_acq_load(&(stack->head.atm));
+    if (head.contents.index == INVALID_ENTRY_INDEX) {
+      return -1;
+    }
+    newhead.atm =
+        gpr_atm_no_barrier_load(&(stack->entries[head.contents.index].atm));
+
+  } while (!gpr_atm_no_barrier_cas(&(stack->head.atm), head.atm, newhead.atm));
+
+  return head.contents.index;
+}
diff --git a/src/core/lib/support/stack_lockfree.h b/src/core/lib/support/stack_lockfree.h
new file mode 100644
index 0000000000..6324211b72
--- /dev/null
+++ b/src/core/lib/support/stack_lockfree.h
@@ -0,0 +1,38 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_STACK_LOCKFREE_H
+#define GRPC_CORE_LIB_SUPPORT_STACK_LOCKFREE_H
+
+#include <stddef.h>
+
+typedef struct gpr_stack_lockfree gpr_stack_lockfree;
+
+/* This stack must specify the maximum number of entries to track.
+   The current implementation only allows up to 65534 entries */
+gpr_stack_lockfree *gpr_stack_lockfree_create(size_t entries);
+void gpr_stack_lockfree_destroy(gpr_stack_lockfree *stack);
+
+/* Pass in a valid entry number for the next stack entry */
+/* Returns 1 if this is the first element on the stack, 0 otherwise */
+int gpr_stack_lockfree_push(gpr_stack_lockfree *, int entry);
+
+/* Returns -1 on empty or the actual entry number */
+int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack);
+
+#endif /* GRPC_CORE_LIB_SUPPORT_STACK_LOCKFREE_H */
diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c
index 343e48c101..300648627c 100644
--- a/src/core/lib/surface/alarm.c
+++ b/src/core/lib/surface/alarm.c
@@ -23,7 +23,8 @@
 #include "src/core/lib/surface/completion_queue.h"
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_alarm_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_alarm_refcount =
+    GRPC_TRACER_INITIALIZER(false, "alarm_refcount");
 #endif
 
 struct grpc_alarm {
diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c
index f88ffd57aa..56973303da 100644
--- a/src/core/lib/surface/api_trace.c
+++ b/src/core/lib/surface/api_trace.c
@@ -19,4 +19,4 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/debug/trace.h"
 
-grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false, "api");
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index c769866ceb..2365d27307 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -229,8 +229,10 @@ struct grpc_call {
   void *saved_receiving_stream_ready_bctlp;
 };
 
-grpc_tracer_flag grpc_call_error_trace = GRPC_TRACER_INITIALIZER(false);
-grpc_tracer_flag grpc_compression_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_call_error_trace =
+    GRPC_TRACER_INITIALIZER(false, "call_error");
+grpc_tracer_flag grpc_compression_trace =
+    GRPC_TRACER_INITIALIZER(false, "compression");
 
 #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1))
 #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1)
@@ -1504,7 +1506,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           goto done_with_error;
         }
         /* TODO(ctiller): just make these the same variable? */
-        call->metadata_batch[0][0].deadline = call->send_deadline;
+        if (call->is_client) {
+          call->metadata_batch[0][0].deadline = call->send_deadline;
+        }
         stream_op_payload->send_initial_metadata.send_initial_metadata =
             &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */];
         stream_op_payload->send_initial_metadata.send_initial_metadata_flags =
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index b04aee6c73..978d7b4171 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -35,10 +35,13 @@
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/event_string.h"
 
-grpc_tracer_flag grpc_trace_operation_failures = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_operation_failures =
+    GRPC_TRACER_INITIALIZER(false, "op_failure");
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_pending_tags = GRPC_TRACER_INITIALIZER(false);
-grpc_tracer_flag grpc_trace_cq_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_pending_tags =
+    GRPC_TRACER_INITIALIZER(false, "pending_tags");
+grpc_tracer_flag grpc_trace_cq_refcount =
+    GRPC_TRACER_INITIALIZER(false, "cq_refcount");
 #endif
 
 typedef struct {
@@ -189,16 +192,19 @@ static const cq_poller_vtable g_poller_vtable_by_poller_type[] = {
 
 typedef struct cq_vtable {
   grpc_cq_completion_type cq_completion_type;
-  size_t (*size)();
-  void (*begin_op)(grpc_completion_queue *cc, void *tag);
-  void (*end_op)(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, void *tag,
+  size_t data_size;
+  void (*init)(void *data);
+  void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq);
+  void (*destroy)(void *data);
+  void (*begin_op)(grpc_completion_queue *cq, void *tag);
+  void (*end_op)(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq, void *tag,
                  grpc_error *error,
                  void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
                               grpc_cq_completion *storage),
                  void *done_arg, grpc_cq_completion *storage);
-  grpc_event (*next)(grpc_completion_queue *cc, gpr_timespec deadline,
+  grpc_event (*next)(grpc_completion_queue *cq, gpr_timespec deadline,
                      void *reserved);
-  grpc_event (*pluck)(grpc_completion_queue *cc, void *tag,
+  grpc_event (*pluck)(grpc_completion_queue *cq, void *tag,
                       gpr_timespec deadline, void *reserved);
 } cq_vtable;
 
@@ -218,25 +224,28 @@ typedef struct grpc_cq_event_queue {
   gpr_atm num_queue_items;
 } grpc_cq_event_queue;
 
-/* TODO: sreek Refactor this based on the completion_type. Put completion-type
- * specific data in a different structure (and co-allocate memory for it along
- * with completion queue + pollset )*/
-typedef struct cq_data {
-  gpr_mu *mu;
+typedef struct cq_next_data {
+  /** Completed events for completion-queues of type GRPC_CQ_NEXT */
+  grpc_cq_event_queue queue;
 
+  /** Counter of how many things have ever been queued on this completion queue
+      useful for avoiding locks to check the queue */
+  gpr_atm things_queued_ever;
+
+  /* Number of outstanding events (+1 if not shut down) */
+  gpr_atm pending_events;
+
+  int shutdown_called;
+} cq_next_data;
+
+typedef struct cq_pluck_data {
   /** Completed events for completion-queues of type GRPC_CQ_PLUCK */
   grpc_cq_completion completed_head;
   grpc_cq_completion *completed_tail;
 
-  /** Completed events for completion-queues of type GRPC_CQ_NEXT */
-  grpc_cq_event_queue queue;
-
   /** Number of pending events (+1 if we're not shutdown) */
   gpr_refcount pending_events;
 
-  /** Once owning_refs drops to zero, we will destroy the cq */
-  gpr_refcount owning_refs;
-
   /** Counter of how many things have ever been queued on this completion queue
       useful for avoiding locks to check the queue */
   gpr_atm things_queued_ever;
@@ -245,37 +254,45 @@ typedef struct cq_data {
   gpr_atm shutdown;
   int shutdown_called;
 
-  int is_server_cq;
-
   int num_pluckers;
-  int num_polls;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
-  grpc_closure pollset_shutdown_done;
+} cq_pluck_data;
+
+/* Completion queue structure */
+struct grpc_completion_queue {
+  /** Once owning_refs drops to zero, we will destroy the cq */
+  gpr_refcount owning_refs;
+
+  gpr_mu *mu;
+
+  const cq_vtable *vtable;
+  const cq_poller_vtable *poller_vtable;
 
 #ifndef NDEBUG
   void **outstanding_tags;
   size_t outstanding_tag_count;
   size_t outstanding_tag_capacity;
 #endif
-} cq_data;
 
-/* Completion queue structure */
-struct grpc_completion_queue {
-  cq_data data;
-  const cq_vtable *vtable;
-  const cq_poller_vtable *poller_vtable;
+  grpc_closure pollset_shutdown_done;
+  int num_polls;
 };
 
 /* Forward declarations */
-static void cq_finish_shutdown(grpc_exec_ctx *exec_ctx,
-                               grpc_completion_queue *cc);
-
-static size_t cq_size(grpc_completion_queue *cc);
-
-static void cq_begin_op(grpc_completion_queue *cc, void *tag);
+static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx,
+                                    grpc_completion_queue *cq);
+static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx,
+                                     grpc_completion_queue *cq);
+static void cq_shutdown_next(grpc_exec_ctx *exec_ctx,
+                             grpc_completion_queue *cq);
+static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx,
+                              grpc_completion_queue *cq);
+
+static void cq_begin_op_for_next(grpc_completion_queue *cq, void *tag);
+static void cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag);
 
 static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
-                               grpc_completion_queue *cc, void *tag,
+                               grpc_completion_queue *cq, void *tag,
                                grpc_error *error,
                                void (*done)(grpc_exec_ctx *exec_ctx,
                                             void *done_arg,
@@ -283,42 +300,56 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
                                void *done_arg, grpc_cq_completion *storage);
 
 static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
-                                grpc_completion_queue *cc, void *tag,
+                                grpc_completion_queue *cq, void *tag,
                                 grpc_error *error,
                                 void (*done)(grpc_exec_ctx *exec_ctx,
                                              void *done_arg,
                                              grpc_cq_completion *storage),
                                 void *done_arg, grpc_cq_completion *storage);
 
-static grpc_event cq_next(grpc_completion_queue *cc, gpr_timespec deadline,
+static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline,
                           void *reserved);
 
-static grpc_event cq_pluck(grpc_completion_queue *cc, void *tag,
+static grpc_event cq_pluck(grpc_completion_queue *cq, void *tag,
                            gpr_timespec deadline, void *reserved);
 
+static void cq_init_next(void *data);
+static void cq_init_pluck(void *data);
+static void cq_destroy_next(void *data);
+static void cq_destroy_pluck(void *data);
+
 /* Completion queue vtables based on the completion-type */
 static const cq_vtable g_cq_vtable[] = {
     /* GRPC_CQ_NEXT */
-    {.cq_completion_type = GRPC_CQ_NEXT,
-     .size = cq_size,
-     .begin_op = cq_begin_op,
+    {.data_size = sizeof(cq_next_data),
+     .cq_completion_type = GRPC_CQ_NEXT,
+     .init = cq_init_next,
+     .shutdown = cq_shutdown_next,
+     .destroy = cq_destroy_next,
+     .begin_op = cq_begin_op_for_next,
      .end_op = cq_end_op_for_next,
      .next = cq_next,
      .pluck = NULL},
     /* GRPC_CQ_PLUCK */
-    {.cq_completion_type = GRPC_CQ_PLUCK,
-     .size = cq_size,
-     .begin_op = cq_begin_op,
+    {.data_size = sizeof(cq_pluck_data),
+     .cq_completion_type = GRPC_CQ_PLUCK,
+     .init = cq_init_pluck,
+     .shutdown = cq_shutdown_pluck,
+     .destroy = cq_destroy_pluck,
+     .begin_op = cq_begin_op_for_pluck,
      .end_op = cq_end_op_for_pluck,
      .next = NULL,
      .pluck = cq_pluck},
 };
 
-#define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
-#define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
+#define DATA_FROM_CQ(cq) ((void *)(cq + 1))
+#define POLLSET_FROM_CQ(cq) \
+  ((grpc_pollset *)(cq->vtable->data_size + (char *)DATA_FROM_CQ(cq)))
 
-grpc_tracer_flag grpc_cq_pluck_trace = GRPC_TRACER_INITIALIZER(true);
-grpc_tracer_flag grpc_cq_event_timeout_trace = GRPC_TRACER_INITIALIZER(true);
+grpc_tracer_flag grpc_cq_pluck_trace =
+    GRPC_TRACER_INITIALIZER(true, "queue_pluck");
+grpc_tracer_flag grpc_cq_event_timeout_trace =
+    GRPC_TRACER_INITIALIZER(true, "queue_timeout");
 
 #define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event)    \
   if (GRPC_TRACER_ON(grpc_api_trace) &&                 \
@@ -329,7 +360,7 @@ grpc_tracer_flag grpc_cq_event_timeout_trace = GRPC_TRACER_INITIALIZER(true);
     gpr_free(_ev);                                      \
   }
 
-static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc,
+static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cq,
                                      grpc_error *error);
 
 static void cq_event_queue_init(grpc_cq_event_queue *q) {
@@ -342,9 +373,9 @@ static void cq_event_queue_destroy(grpc_cq_event_queue *q) {
   gpr_mpscq_destroy(&q->queue);
 }
 
-static void cq_event_queue_push(grpc_cq_event_queue *q, grpc_cq_completion *c) {
+static bool cq_event_queue_push(grpc_cq_event_queue *q, grpc_cq_completion *c) {
   gpr_mpscq_push(&q->queue, (gpr_mpscq_node *)c);
-  gpr_atm_no_barrier_fetch_add(&q->num_queue_items, 1);
+  return gpr_atm_no_barrier_fetch_add(&q->num_queue_items, 1) == 0;
 }
 
 static grpc_cq_completion *cq_event_queue_pop(grpc_cq_event_queue *q) {
@@ -367,16 +398,10 @@ static long cq_event_queue_num_items(grpc_cq_event_queue *q) {
   return (long)gpr_atm_no_barrier_load(&q->num_queue_items);
 }
 
-static size_t cq_size(grpc_completion_queue *cc) {
-  /* Size of the completion queue and the size of the pollset whose memory is
-     allocated right after that of completion queue */
-  return sizeof(grpc_completion_queue) + cc->poller_vtable->size();
-}
-
 grpc_completion_queue *grpc_completion_queue_create_internal(
     grpc_cq_completion_type completion_type,
     grpc_cq_polling_type polling_type) {
-  grpc_completion_queue *cc;
+  grpc_completion_queue *cq;
 
   GPR_TIMER_BEGIN("grpc_completion_queue_create_internal", 0);
 
@@ -389,158 +414,173 @@ grpc_completion_queue *grpc_completion_queue_create_internal(
   const cq_poller_vtable *poller_vtable =
       &g_poller_vtable_by_poller_type[polling_type];
 
-  cc = gpr_zalloc(sizeof(grpc_completion_queue) + poller_vtable->size());
-  cq_data *cqd = &cc->data;
+  cq = gpr_zalloc(sizeof(grpc_completion_queue) + vtable->data_size +
+                  poller_vtable->size());
 
-  cc->vtable = vtable;
-  cc->poller_vtable = poller_vtable;
+  cq->vtable = vtable;
+  cq->poller_vtable = poller_vtable;
 
-  poller_vtable->init(POLLSET_FROM_CQ(cc), &cc->data.mu);
+  /* One for destroy(), one for pollset_shutdown */
+  gpr_ref_init(&cq->owning_refs, 2);
 
-#ifndef NDEBUG
-  cqd->outstanding_tags = NULL;
-  cqd->outstanding_tag_capacity = 0;
-#endif
+  poller_vtable->init(POLLSET_FROM_CQ(cq), &cq->mu);
+  vtable->init(DATA_FROM_CQ(cq));
+
+  GRPC_CLOSURE_INIT(&cq->pollset_shutdown_done, on_pollset_shutdown_done, cq,
+                    grpc_schedule_on_exec_ctx);
+
+  GPR_TIMER_END("grpc_completion_queue_create_internal", 0);
+
+  return cq;
+}
 
+static void cq_init_next(void *ptr) {
+  cq_next_data *cqd = ptr;
+  /* Initial ref is dropped by grpc_completion_queue_shutdown */
+  gpr_atm_no_barrier_store(&cqd->pending_events, 1);
+  cqd->shutdown_called = false;
+  gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
+  cq_event_queue_init(&cqd->queue);
+}
+
+static void cq_destroy_next(void *ptr) {
+  cq_next_data *cqd = ptr;
+  GPR_ASSERT(cq_event_queue_num_items(&cqd->queue) == 0);
+  cq_event_queue_destroy(&cqd->queue);
+}
+
+static void cq_init_pluck(void *ptr) {
+  cq_pluck_data *cqd = ptr;
   /* Initial ref is dropped by grpc_completion_queue_shutdown */
   gpr_ref_init(&cqd->pending_events, 1);
-  /* One for destroy(), one for pollset_shutdown */
-  gpr_ref_init(&cqd->owning_refs, 2);
   cqd->completed_tail = &cqd->completed_head;
   cqd->completed_head.next = (uintptr_t)cqd->completed_tail;
   gpr_atm_no_barrier_store(&cqd->shutdown, 0);
   cqd->shutdown_called = 0;
-  cqd->is_server_cq = 0;
   cqd->num_pluckers = 0;
-  cqd->num_polls = 0;
   gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0);
-#ifndef NDEBUG
-  cqd->outstanding_tag_count = 0;
-#endif
-  cq_event_queue_init(&cqd->queue);
-  GRPC_CLOSURE_INIT(&cqd->pollset_shutdown_done, on_pollset_shutdown_done, cc,
-                    grpc_schedule_on_exec_ctx);
-
-  GPR_TIMER_END("grpc_completion_queue_create_internal", 0);
+}
 
-  return cc;
+static void cq_destroy_pluck(void *ptr) {
+  cq_pluck_data *cqd = ptr;
+  GPR_ASSERT(cqd->completed_head.next == (uintptr_t)&cqd->completed_head);
 }
 
-grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc) {
-  return cc->vtable->cq_completion_type;
+grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cq) {
+  return cq->vtable->cq_completion_type;
 }
 
-int grpc_get_cq_poll_num(grpc_completion_queue *cc) {
+int grpc_get_cq_poll_num(grpc_completion_queue *cq) {
   int cur_num_polls;
-  gpr_mu_lock(cc->data.mu);
-  cur_num_polls = cc->data.num_polls;
-  gpr_mu_unlock(cc->data.mu);
+  gpr_mu_lock(cq->mu);
+  cur_num_polls = cq->num_polls;
+  gpr_mu_unlock(cq->mu);
   return cur_num_polls;
 }
 
 #ifndef NDEBUG
-void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason,
+void grpc_cq_internal_ref(grpc_completion_queue *cq, const char *reason,
                           const char *file, int line) {
-  cq_data *cqd = &cc->data;
   if (GRPC_TRACER_ON(grpc_trace_cq_refcount)) {
-    gpr_atm val = gpr_atm_no_barrier_load(&cqd->owning_refs.count);
+    gpr_atm val = gpr_atm_no_barrier_load(&cq->owning_refs.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "CQ:%p   ref %" PRIdPTR " -> %" PRIdPTR " %s", cc, val, val + 1,
+            "CQ:%p   ref %" PRIdPTR " -> %" PRIdPTR " %s", cq, val, val + 1,
             reason);
   }
 #else
-void grpc_cq_internal_ref(grpc_completion_queue *cc) {
-  cq_data *cqd = &cc->data;
+void grpc_cq_internal_ref(grpc_completion_queue *cq) {
 #endif
-  gpr_ref(&cqd->owning_refs);
+  gpr_ref(&cq->owning_refs);
 }
 
 static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg,
                                      grpc_error *error) {
-  grpc_completion_queue *cc = arg;
-  GRPC_CQ_INTERNAL_UNREF(exec_ctx, cc, "pollset_destroy");
+  grpc_completion_queue *cq = arg;
+  GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "pollset_destroy");
 }
 
 #ifndef NDEBUG
-void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
+void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq,
                             const char *reason, const char *file, int line) {
-  cq_data *cqd = &cc->data;
   if (GRPC_TRACER_ON(grpc_trace_cq_refcount)) {
-    gpr_atm val = gpr_atm_no_barrier_load(&cqd->owning_refs.count);
+    gpr_atm val = gpr_atm_no_barrier_load(&cq->owning_refs.count);
     gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-            "CQ:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", cc, val, val - 1,
+            "CQ:%p unref %" PRIdPTR " -> %" PRIdPTR " %s", cq, val, val - 1,
             reason);
   }
 #else
 void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx,
-                            grpc_completion_queue *cc) {
-  cq_data *cqd = &cc->data;
+                            grpc_completion_queue *cq) {
 #endif
-  if (gpr_unref(&cqd->owning_refs)) {
-    GPR_ASSERT(cqd->completed_head.next == (uintptr_t)&cqd->completed_head);
-    cc->poller_vtable->destroy(exec_ctx, POLLSET_FROM_CQ(cc));
-    cq_event_queue_destroy(&cqd->queue);
+  if (gpr_unref(&cq->owning_refs)) {
+    cq->vtable->destroy(DATA_FROM_CQ(cq));
+    cq->poller_vtable->destroy(exec_ctx, POLLSET_FROM_CQ(cq));
 #ifndef NDEBUG
-    gpr_free(cqd->outstanding_tags);
+    gpr_free(cq->outstanding_tags);
 #endif
-    gpr_free(cc);
+    gpr_free(cq);
   }
 }
 
-static void cq_begin_op(grpc_completion_queue *cc, void *tag) {
-  cq_data *cqd = &cc->data;
-#ifndef NDEBUG
-  gpr_mu_lock(cqd->mu);
+static void cq_begin_op_for_next(grpc_completion_queue *cq, void *tag) {
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
+  GPR_ASSERT(!cqd->shutdown_called);
+  gpr_atm_no_barrier_fetch_add(&cqd->pending_events, 1);
+}
+
+static void cq_begin_op_for_pluck(grpc_completion_queue *cq, void *tag) {
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
   GPR_ASSERT(!cqd->shutdown_called);
-  if (cqd->outstanding_tag_count == cqd->outstanding_tag_capacity) {
-    cqd->outstanding_tag_capacity =
-        GPR_MAX(4, 2 * cqd->outstanding_tag_capacity);
-    cqd->outstanding_tags =
-        gpr_realloc(cqd->outstanding_tags, sizeof(*cqd->outstanding_tags) *
-                                               cqd->outstanding_tag_capacity);
-  }
-  cqd->outstanding_tags[cqd->outstanding_tag_count++] = tag;
-  gpr_mu_unlock(cqd->mu);
-#endif
   gpr_ref(&cqd->pending_events);
 }
 
-void grpc_cq_begin_op(grpc_completion_queue *cc, void *tag) {
-  cc->vtable->begin_op(cc, tag);
+void grpc_cq_begin_op(grpc_completion_queue *cq, void *tag) {
+#ifndef NDEBUG
+  gpr_mu_lock(cq->mu);
+  if (cq->outstanding_tag_count == cq->outstanding_tag_capacity) {
+    cq->outstanding_tag_capacity = GPR_MAX(4, 2 * cq->outstanding_tag_capacity);
+    cq->outstanding_tags =
+        gpr_realloc(cq->outstanding_tags, sizeof(*cq->outstanding_tags) *
+                                              cq->outstanding_tag_capacity);
+  }
+  cq->outstanding_tags[cq->outstanding_tag_count++] = tag;
+  gpr_mu_unlock(cq->mu);
+#endif
+  cq->vtable->begin_op(cq, tag);
 }
 
 #ifndef NDEBUG
-static void cq_check_tag(grpc_completion_queue *cc, void *tag, bool lock_cq) {
-  cq_data *cqd = &cc->data;
+static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {
   int found = 0;
   if (lock_cq) {
-    gpr_mu_lock(cqd->mu);
+    gpr_mu_lock(cq->mu);
   }
 
-  for (int i = 0; i < (int)cqd->outstanding_tag_count; i++) {
-    if (cqd->outstanding_tags[i] == tag) {
-      cqd->outstanding_tag_count--;
-      GPR_SWAP(void *, cqd->outstanding_tags[i],
-               cqd->outstanding_tags[cqd->outstanding_tag_count]);
+  for (int i = 0; i < (int)cq->outstanding_tag_count; i++) {
+    if (cq->outstanding_tags[i] == tag) {
+      cq->outstanding_tag_count--;
+      GPR_SWAP(void *, cq->outstanding_tags[i],
+               cq->outstanding_tags[cq->outstanding_tag_count]);
       found = 1;
       break;
     }
   }
 
   if (lock_cq) {
-    gpr_mu_unlock(cqd->mu);
+    gpr_mu_unlock(cq->mu);
   }
 
   GPR_ASSERT(found);
 }
 #else
-static void cq_check_tag(grpc_completion_queue *cc, void *tag, bool lock_cq) {}
+static void cq_check_tag(grpc_completion_queue *cq, void *tag, bool lock_cq) {}
 #endif
 
-/* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a completion
+/* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
+ * completion
  * type of GRPC_CQ_NEXT) */
 static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
-                               grpc_completion_queue *cc, void *tag,
+                               grpc_completion_queue *cq, void *tag,
                                grpc_error *error,
                                void (*done)(grpc_exec_ctx *exec_ctx,
                                             void *done_arg,
@@ -553,16 +593,16 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
        error != GRPC_ERROR_NONE)) {
     const char *errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
-        "cq_end_op_for_next(exec_ctx=%p, cc=%p, tag=%p, error=%s, "
+        "cq_end_op_for_next(exec_ctx=%p, cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
-        7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage));
+        7, (exec_ctx, cq, tag, errmsg, done, done_arg, storage));
     if (GRPC_TRACER_ON(grpc_trace_operation_failures) &&
         error != GRPC_ERROR_NONE) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
     }
   }
 
-  cq_data *cqd = &cc->data;
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
   int is_success = (error == GRPC_ERROR_NONE);
 
   storage->tag = tag;
@@ -570,28 +610,42 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
   storage->done_arg = done_arg;
   storage->next = (uintptr_t)(is_success);
 
-  cq_check_tag(cc, tag, true); /* Used in debug builds only */
+  cq_check_tag(cq, tag, true); /* Used in debug builds only */
 
   /* Add the completion to the queue */
-  cq_event_queue_push(&cqd->queue, storage);
+  bool is_first = cq_event_queue_push(&cqd->queue, storage);
   gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1);
-
-  gpr_mu_lock(cqd->mu);
-
-  int shutdown = gpr_unref(&cqd->pending_events);
-  if (!shutdown) {
-    grpc_error *kick_error = cc->poller_vtable->kick(POLLSET_FROM_CQ(cc), NULL);
-    gpr_mu_unlock(cqd->mu);
-
-    if (kick_error != GRPC_ERROR_NONE) {
-      const char *msg = grpc_error_string(kick_error);
-      gpr_log(GPR_ERROR, "Kick failed: %s", msg);
-
-      GRPC_ERROR_UNREF(kick_error);
+  bool will_definitely_shutdown =
+      gpr_atm_no_barrier_load(&cqd->pending_events) == 1;
+
+  if (!will_definitely_shutdown) {
+    /* Only kick if this is the first item queued */
+    if (is_first) {
+      gpr_mu_lock(cq->mu);
+      grpc_error *kick_error =
+          cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL);
+      gpr_mu_unlock(cq->mu);
+
+      if (kick_error != GRPC_ERROR_NONE) {
+        const char *msg = grpc_error_string(kick_error);
+        gpr_log(GPR_ERROR, "Kick failed: %s", msg);
+        GRPC_ERROR_UNREF(kick_error);
+      }
+    }
+    if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+      GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
+      gpr_mu_lock(cq->mu);
+      cq_finish_shutdown_next(exec_ctx, cq);
+      gpr_mu_unlock(cq->mu);
+      GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down");
     }
   } else {
-    cq_finish_shutdown(exec_ctx, cc);
-    gpr_mu_unlock(cqd->mu);
+    GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
+    gpr_atm_rel_store(&cqd->pending_events, 0);
+    gpr_mu_lock(cq->mu);
+    cq_finish_shutdown_next(exec_ctx, cq);
+    gpr_mu_unlock(cq->mu);
+    GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down");
   }
 
   GPR_TIMER_END("cq_end_op_for_next", 0);
@@ -599,16 +653,17 @@ static void cq_end_op_for_next(grpc_exec_ctx *exec_ctx,
   GRPC_ERROR_UNREF(error);
 }
 
-/* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a completion
+/* Queue a GRPC_OP_COMPLETED operation to a completion queue (with a
+ * completion
  * type of GRPC_CQ_PLUCK) */
 static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
-                                grpc_completion_queue *cc, void *tag,
+                                grpc_completion_queue *cq, void *tag,
                                 grpc_error *error,
                                 void (*done)(grpc_exec_ctx *exec_ctx,
                                              void *done_arg,
                                              grpc_cq_completion *storage),
                                 void *done_arg, grpc_cq_completion *storage) {
-  cq_data *cqd = &cc->data;
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
   int is_success = (error == GRPC_ERROR_NONE);
 
   GPR_TIMER_BEGIN("cq_end_op_for_pluck", 0);
@@ -618,9 +673,9 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
        error != GRPC_ERROR_NONE)) {
     const char *errmsg = grpc_error_string(error);
     GRPC_API_TRACE(
-        "cq_end_op_for_pluck(exec_ctx=%p, cc=%p, tag=%p, error=%s, "
+        "cq_end_op_for_pluck(exec_ctx=%p, cq=%p, tag=%p, error=%s, "
         "done=%p, done_arg=%p, storage=%p)",
-        7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage));
+        7, (exec_ctx, cq, tag, errmsg, done, done_arg, storage));
     if (GRPC_TRACER_ON(grpc_trace_operation_failures) &&
         error != GRPC_ERROR_NONE) {
       gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg);
@@ -632,8 +687,8 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
   storage->done_arg = done_arg;
   storage->next = ((uintptr_t)&cqd->completed_head) | ((uintptr_t)(is_success));
 
-  gpr_mu_lock(cqd->mu);
-  cq_check_tag(cc, tag, false); /* Used in debug builds only */
+  gpr_mu_lock(cq->mu);
+  cq_check_tag(cq, tag, false); /* Used in debug builds only */
 
   /* Add to the list of completions */
   gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1);
@@ -652,9 +707,9 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
     }
 
     grpc_error *kick_error =
-        cc->poller_vtable->kick(POLLSET_FROM_CQ(cc), pluck_worker);
+        cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), pluck_worker);
 
-    gpr_mu_unlock(cqd->mu);
+    gpr_mu_unlock(cq->mu);
 
     if (kick_error != GRPC_ERROR_NONE) {
       const char *msg = grpc_error_string(kick_error);
@@ -663,8 +718,8 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
       GRPC_ERROR_UNREF(kick_error);
     }
   } else {
-    cq_finish_shutdown(exec_ctx, cc);
-    gpr_mu_unlock(cqd->mu);
+    cq_finish_shutdown_pluck(exec_ctx, cq);
+    gpr_mu_unlock(cq->mu);
   }
 
   GPR_TIMER_END("cq_end_op_for_pluck", 0);
@@ -672,12 +727,12 @@ static void cq_end_op_for_pluck(grpc_exec_ctx *exec_ctx,
   GRPC_ERROR_UNREF(error);
 }
 
-void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
+void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cq,
                     void *tag, grpc_error *error,
                     void (*done)(grpc_exec_ctx *exec_ctx, void *done_arg,
                                  grpc_cq_completion *storage),
                     void *done_arg, grpc_cq_completion *storage) {
-  cc->vtable->end_op(exec_ctx, cc, tag, error, done, done_arg, storage);
+  cq->vtable->end_op(exec_ctx, cq, tag, error, done, done_arg, storage);
 }
 
 typedef struct {
@@ -692,7 +747,7 @@ typedef struct {
 static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) {
   cq_is_finished_arg *a = arg;
   grpc_completion_queue *cq = a->cq;
-  cq_data *cqd = &cq->data;
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
   GPR_ASSERT(a->stolen_completion == NULL);
 
   gpr_atm current_last_seen_things_queued_ever =
@@ -703,7 +758,8 @@ static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) {
         gpr_atm_no_barrier_load(&cqd->things_queued_ever);
 
     /* Pop a cq_completion from the queue. Returns NULL if the queue is empty
-     * might return NULL in some cases even if the queue is not empty; but that
+     * might return NULL in some cases even if the queue is not empty; but
+     * that
      * is ok and doesn't affect correctness. Might effect the tail latencies a
      * bit) */
     a->stolen_completion = cq_event_queue_pop(&cqd->queue);
@@ -716,58 +772,56 @@ static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) {
 }
 
 #ifndef NDEBUG
-static void dump_pending_tags(grpc_completion_queue *cc) {
+static void dump_pending_tags(grpc_completion_queue *cq) {
   if (!GRPC_TRACER_ON(grpc_trace_pending_tags)) return;
 
-  cq_data *cqd = &cc->data;
-
   gpr_strvec v;
   gpr_strvec_init(&v);
   gpr_strvec_add(&v, gpr_strdup("PENDING TAGS:"));
-  gpr_mu_lock(cqd->mu);
-  for (size_t i = 0; i < cqd->outstanding_tag_count; i++) {
+  gpr_mu_lock(cq->mu);
+  for (size_t i = 0; i < cq->outstanding_tag_count; i++) {
     char *s;
-    gpr_asprintf(&s, " %p", cqd->outstanding_tags[i]);
+    gpr_asprintf(&s, " %p", cq->outstanding_tags[i]);
     gpr_strvec_add(&v, s);
   }
-  gpr_mu_unlock(cqd->mu);
+  gpr_mu_unlock(cq->mu);
   char *out = gpr_strvec_flatten(&v, NULL);
   gpr_strvec_destroy(&v);
   gpr_log(GPR_DEBUG, "%s", out);
   gpr_free(out);
 }
 #else
-static void dump_pending_tags(grpc_completion_queue *cc) {}
+static void dump_pending_tags(grpc_completion_queue *cq) {}
 #endif
 
-static grpc_event cq_next(grpc_completion_queue *cc, gpr_timespec deadline,
+static grpc_event cq_next(grpc_completion_queue *cq, gpr_timespec deadline,
                           void *reserved) {
   grpc_event ret;
   gpr_timespec now;
-  cq_data *cqd = &cc->data;
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
 
   GPR_TIMER_BEGIN("grpc_completion_queue_next", 0);
 
   GRPC_API_TRACE(
       "grpc_completion_queue_next("
-      "cc=%p, "
+      "cq=%p, "
       "deadline=gpr_timespec { tv_sec: %" PRId64
       ", tv_nsec: %d, clock_type: %d }, "
       "reserved=%p)",
-      5, (cc, deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
+      5, (cq, deadline.tv_sec, deadline.tv_nsec, (int)deadline.clock_type,
           reserved));
   GPR_ASSERT(!reserved);
 
-  dump_pending_tags(cc);
+  dump_pending_tags(cq);
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
 
-  GRPC_CQ_INTERNAL_REF(cc, "next");
+  GRPC_CQ_INTERNAL_REF(cq, "next");
 
   cq_is_finished_arg is_finished_arg = {
       .last_seen_things_queued_ever =
           gpr_atm_no_barrier_load(&cqd->things_queued_ever),
-      .cq = cc,
+      .cq = cq,
       .deadline = deadline,
       .stolen_completion = NULL,
       .tag = NULL,
@@ -800,21 +854,24 @@ static grpc_event cq_next(grpc_completion_queue *cc, gpr_timespec deadline,
       /* If c == NULL it means either the queue is empty OR in an transient
          inconsistent state. If it is the latter, we shold do a 0-timeout poll
          so that the thread comes back quickly from poll to make a second
-         attempt at popping. Not doing this can potentially deadlock this thread
+         attempt at popping. Not doing this can potentially deadlock this
+         thread
          forever (if the deadline is infinity) */
       if (cq_event_queue_num_items(&cqd->queue) > 0) {
         iteration_deadline = gpr_time_0(GPR_CLOCK_MONOTONIC);
       }
     }
 
-    if (gpr_atm_no_barrier_load(&cqd->shutdown)) {
+    if (gpr_atm_no_barrier_load(&cqd->pending_events) == 0) {
       /* Before returning, check if the queue has any items left over (since
          gpr_mpscq_pop() can sometimes return NULL even if the queue is not
          empty. If so, keep retrying but do not return GRPC_QUEUE_SHUTDOWN */
       if (cq_event_queue_num_items(&cqd->queue) > 0) {
         /* Go to the beginning of the loop. No point doing a poll because
-           (cc->shutdown == true) is only possible when there is no pending work
-           (i.e cc->pending_events == 0) and any outstanding grpc_cq_completion
+           (cq->shutdown == true) is only possible when there is no pending
+           work
+           (i.e cq->pending_events == 0) and any outstanding
+           grpc_cq_completion
            events are already queued on this cq */
         continue;
       }
@@ -828,16 +885,16 @@ static grpc_event cq_next(grpc_completion_queue *cc, gpr_timespec deadline,
     if (!is_finished_arg.first_loop && gpr_time_cmp(now, deadline) >= 0) {
       memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
-      dump_pending_tags(cc);
+      dump_pending_tags(cq);
       break;
     }
 
     /* The main polling work happens in grpc_pollset_work */
-    gpr_mu_lock(cqd->mu);
-    cqd->num_polls++;
-    grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc),
+    gpr_mu_lock(cq->mu);
+    cq->num_polls++;
+    grpc_error *err = cq->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cq),
                                               NULL, now, iteration_deadline);
-    gpr_mu_unlock(cqd->mu);
+    gpr_mu_unlock(cq->mu);
 
     if (err != GRPC_ERROR_NONE) {
       const char *msg = grpc_error_string(err);
@@ -846,30 +903,74 @@ static grpc_event cq_next(grpc_completion_queue *cc, gpr_timespec deadline,
       GRPC_ERROR_UNREF(err);
       memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
-      dump_pending_tags(cc);
+      dump_pending_tags(cq);
       break;
     }
     is_finished_arg.first_loop = false;
   }
 
-  GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret);
-  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "next");
+  GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, &ret);
+  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cq, "next");
   grpc_exec_ctx_finish(&exec_ctx);
   GPR_ASSERT(is_finished_arg.stolen_completion == NULL);
 
+  if (cq_event_queue_num_items(&cqd->queue) > 0 &&
+      gpr_atm_no_barrier_load(&cqd->pending_events) > 0) {
+    gpr_mu_lock(cq->mu);
+    cq->poller_vtable->kick(POLLSET_FROM_CQ(cq), NULL);
+    gpr_mu_unlock(cq->mu);
+  }
+
   GPR_TIMER_END("grpc_completion_queue_next", 0);
 
   return ret;
 }
 
-grpc_event grpc_completion_queue_next(grpc_completion_queue *cc,
+/* Finishes the completion queue shutdown. This means that there are no more
+   completion events / tags expected from the completion queue
+   - Must be called under completion queue lock
+   - Must be called only once in completion queue's lifetime
+   - grpc_completion_queue_shutdown() MUST have been called before calling
+   this function */
+static void cq_finish_shutdown_next(grpc_exec_ctx *exec_ctx,
+                                    grpc_completion_queue *cq) {
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
+
+  GPR_ASSERT(cqd->shutdown_called);
+  GPR_ASSERT(gpr_atm_no_barrier_load(&cqd->pending_events) == 0);
+
+  cq->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cq),
+                              &cq->pollset_shutdown_done);
+}
+
+static void cq_shutdown_next(grpc_exec_ctx *exec_ctx,
+                             grpc_completion_queue *cq) {
+  cq_next_data *cqd = DATA_FROM_CQ(cq);
+
+  GRPC_CQ_INTERNAL_REF(cq, "shutting_down");
+  gpr_mu_lock(cq->mu);
+  if (cqd->shutdown_called) {
+    gpr_mu_unlock(cq->mu);
+    GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down");
+    GPR_TIMER_END("grpc_completion_queue_shutdown", 0);
+    return;
+  }
+  cqd->shutdown_called = 1;
+  if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) {
+    cq_finish_shutdown_next(exec_ctx, cq);
+  }
+  gpr_mu_unlock(cq->mu);
+  GRPC_CQ_INTERNAL_UNREF(exec_ctx, cq, "shutting_down");
+}
+
+grpc_event grpc_completion_queue_next(grpc_completion_queue *cq,
                                       gpr_timespec deadline, void *reserved) {
-  return cc->vtable->next(cc, deadline, reserved);
+  return cq->vtable->next(cq, deadline, reserved);
 }
 
-static int add_plucker(grpc_completion_queue *cc, void *tag,
+static int add_plucker(grpc_completion_queue *cq, void *tag,
                        grpc_pollset_worker **worker) {
-  cq_data *cqd = &cc->data;
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
   if (cqd->num_pluckers == GRPC_MAX_COMPLETION_QUEUE_PLUCKERS) {
     return 0;
   }
@@ -879,9 +980,9 @@ static int add_plucker(grpc_completion_queue *cc, void *tag,
   return 1;
 }
 
-static void del_plucker(grpc_completion_queue *cc, void *tag,
+static void del_plucker(grpc_completion_queue *cq, void *tag,
                         grpc_pollset_worker **worker) {
-  cq_data *cqd = &cc->data;
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
   for (int i = 0; i < cqd->num_pluckers; i++) {
     if (cqd->pluckers[i].tag == tag && cqd->pluckers[i].worker == worker) {
       cqd->num_pluckers--;
@@ -895,13 +996,13 @@ static void del_plucker(grpc_completion_queue *cc, void *tag,
 static bool cq_is_pluck_finished(grpc_exec_ctx *exec_ctx, void *arg) {
   cq_is_finished_arg *a = arg;
   grpc_completion_queue *cq = a->cq;
-  cq_data *cqd = &cq->data;
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
 
   GPR_ASSERT(a->stolen_completion == NULL);
   gpr_atm current_last_seen_things_queued_ever =
       gpr_atm_no_barrier_load(&cqd->things_queued_ever);
   if (current_last_seen_things_queued_ever != a->last_seen_things_queued_ever) {
-    gpr_mu_lock(cqd->mu);
+    gpr_mu_lock(cq->mu);
     a->last_seen_things_queued_ever =
         gpr_atm_no_barrier_load(&cqd->things_queued_ever);
     grpc_cq_completion *c;
@@ -913,51 +1014,51 @@ static bool cq_is_pluck_finished(grpc_exec_ctx *exec_ctx, void *arg) {
         if (c == cqd->completed_tail) {
           cqd->completed_tail = prev;
         }
-        gpr_mu_unlock(cqd->mu);
+        gpr_mu_unlock(cq->mu);
         a->stolen_completion = c;
         return true;
       }
       prev = c;
     }
-    gpr_mu_unlock(cqd->mu);
+    gpr_mu_unlock(cq->mu);
   }
   return !a->first_loop &&
          gpr_time_cmp(a->deadline, gpr_now(a->deadline.clock_type)) < 0;
 }
 
-static grpc_event cq_pluck(grpc_completion_queue *cc, void *tag,
+static grpc_event cq_pluck(grpc_completion_queue *cq, void *tag,
                            gpr_timespec deadline, void *reserved) {
   grpc_event ret;
   grpc_cq_completion *c;
   grpc_cq_completion *prev;
   grpc_pollset_worker *worker = NULL;
   gpr_timespec now;
-  cq_data *cqd = &cc->data;
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
 
   GPR_TIMER_BEGIN("grpc_completion_queue_pluck", 0);
 
   if (GRPC_TRACER_ON(grpc_cq_pluck_trace)) {
     GRPC_API_TRACE(
         "grpc_completion_queue_pluck("
-        "cc=%p, tag=%p, "
+        "cq=%p, tag=%p, "
         "deadline=gpr_timespec { tv_sec: %" PRId64
         ", tv_nsec: %d, clock_type: %d }, "
         "reserved=%p)",
-        6, (cc, tag, deadline.tv_sec, deadline.tv_nsec,
+        6, (cq, tag, deadline.tv_sec, deadline.tv_nsec,
             (int)deadline.clock_type, reserved));
   }
   GPR_ASSERT(!reserved);
 
-  dump_pending_tags(cc);
+  dump_pending_tags(cq);
 
   deadline = gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC);
 
-  GRPC_CQ_INTERNAL_REF(cc, "pluck");
-  gpr_mu_lock(cqd->mu);
+  GRPC_CQ_INTERNAL_REF(cq, "pluck");
+  gpr_mu_lock(cq->mu);
   cq_is_finished_arg is_finished_arg = {
       .last_seen_things_queued_ever =
           gpr_atm_no_barrier_load(&cqd->things_queued_ever),
-      .cq = cc,
+      .cq = cq,
       .deadline = deadline,
       .stolen_completion = NULL,
       .tag = tag,
@@ -966,7 +1067,7 @@ static grpc_event cq_pluck(grpc_completion_queue *cc, void *tag,
       GRPC_EXEC_CTX_INITIALIZER(0, cq_is_pluck_finished, &is_finished_arg);
   for (;;) {
     if (is_finished_arg.stolen_completion != NULL) {
-      gpr_mu_unlock(cqd->mu);
+      gpr_mu_unlock(cq->mu);
       c = is_finished_arg.stolen_completion;
       is_finished_arg.stolen_completion = NULL;
       ret.type = GRPC_OP_COMPLETE;
@@ -983,7 +1084,7 @@ static grpc_event cq_pluck(grpc_completion_queue *cc, void *tag,
         if (c == cqd->completed_tail) {
           cqd->completed_tail = prev;
         }
-        gpr_mu_unlock(cqd->mu);
+        gpr_mu_unlock(cq->mu);
         ret.type = GRPC_OP_COMPLETE;
         ret.success = c->next & 1u;
         ret.tag = c->tag;
@@ -993,54 +1094,54 @@ static grpc_event cq_pluck(grpc_completion_queue *cc, void *tag,
       prev = c;
     }
     if (gpr_atm_no_barrier_load(&cqd->shutdown)) {
-      gpr_mu_unlock(cqd->mu);
+      gpr_mu_unlock(cq->mu);
       memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_SHUTDOWN;
       break;
     }
-    if (!add_plucker(cc, tag, &worker)) {
+    if (!add_plucker(cq, tag, &worker)) {
       gpr_log(GPR_DEBUG,
               "Too many outstanding grpc_completion_queue_pluck calls: maximum "
               "is %d",
               GRPC_MAX_COMPLETION_QUEUE_PLUCKERS);
-      gpr_mu_unlock(cqd->mu);
+      gpr_mu_unlock(cq->mu);
       memset(&ret, 0, sizeof(ret));
       /* TODO(ctiller): should we use a different result here */
       ret.type = GRPC_QUEUE_TIMEOUT;
-      dump_pending_tags(cc);
+      dump_pending_tags(cq);
       break;
     }
     now = gpr_now(GPR_CLOCK_MONOTONIC);
     if (!is_finished_arg.first_loop && gpr_time_cmp(now, deadline) >= 0) {
-      del_plucker(cc, tag, &worker);
-      gpr_mu_unlock(cqd->mu);
+      del_plucker(cq, tag, &worker);
+      gpr_mu_unlock(cq->mu);
       memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
-      dump_pending_tags(cc);
+      dump_pending_tags(cq);
       break;
     }
 
-    cqd->num_polls++;
-    grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc),
+    cq->num_polls++;
+    grpc_error *err = cq->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cq),
                                               &worker, now, deadline);
     if (err != GRPC_ERROR_NONE) {
-      del_plucker(cc, tag, &worker);
-      gpr_mu_unlock(cqd->mu);
+      del_plucker(cq, tag, &worker);
+      gpr_mu_unlock(cq->mu);
       const char *msg = grpc_error_string(err);
       gpr_log(GPR_ERROR, "Completion queue pluck failed: %s", msg);
 
       GRPC_ERROR_UNREF(err);
       memset(&ret, 0, sizeof(ret));
       ret.type = GRPC_QUEUE_TIMEOUT;
-      dump_pending_tags(cc);
+      dump_pending_tags(cq);
       break;
     }
     is_finished_arg.first_loop = false;
-    del_plucker(cc, tag, &worker);
+    del_plucker(cq, tag, &worker);
   }
 done:
-  GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret);
-  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "pluck");
+  GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, &ret);
+  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cq, "pluck");
   grpc_exec_ctx_finish(&exec_ctx);
   GPR_ASSERT(is_finished_arg.stolen_completion == NULL);
 
@@ -1049,85 +1150,66 @@ done:
   return ret;
 }
 
-grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag,
+grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cq, void *tag,
                                        gpr_timespec deadline, void *reserved) {
-  return cc->vtable->pluck(cc, tag, deadline, reserved);
+  return cq->vtable->pluck(cq, tag, deadline, reserved);
 }
 
-/* Finishes the completion queue shutdown. This means that there are no more
-   completion events / tags expected from the completion queue
-   - Must be called under completion queue lock
-   - Must be called only once in completion queue's lifetime
-   - grpc_completion_queue_shutdown() MUST have been called before calling
-   this function */
-static void cq_finish_shutdown(grpc_exec_ctx *exec_ctx,
-                               grpc_completion_queue *cc) {
-  cq_data *cqd = &cc->data;
+static void cq_finish_shutdown_pluck(grpc_exec_ctx *exec_ctx,
+                                     grpc_completion_queue *cq) {
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
 
   GPR_ASSERT(cqd->shutdown_called);
   GPR_ASSERT(!gpr_atm_no_barrier_load(&cqd->shutdown));
   gpr_atm_no_barrier_store(&cqd->shutdown, 1);
 
-  cc->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cc),
-                              &cqd->pollset_shutdown_done);
+  cq->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cq),
+                              &cq->pollset_shutdown_done);
 }
 
-/* Shutdown simply drops a ref that we reserved at creation time; if we drop
-   to zero here, then enter shutdown mode and wake up any waiters */
-void grpc_completion_queue_shutdown(grpc_completion_queue *cc) {
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0);
-  GRPC_API_TRACE("grpc_completion_queue_shutdown(cc=%p)", 1, (cc));
-  cq_data *cqd = &cc->data;
+static void cq_shutdown_pluck(grpc_exec_ctx *exec_ctx,
+                              grpc_completion_queue *cq) {
+  cq_pluck_data *cqd = DATA_FROM_CQ(cq);
 
-  gpr_mu_lock(cqd->mu);
+  gpr_mu_lock(cq->mu);
   if (cqd->shutdown_called) {
-    gpr_mu_unlock(cqd->mu);
+    gpr_mu_unlock(cq->mu);
     GPR_TIMER_END("grpc_completion_queue_shutdown", 0);
     return;
   }
   cqd->shutdown_called = 1;
   if (gpr_unref(&cqd->pending_events)) {
-    cq_finish_shutdown(&exec_ctx, cc);
+    cq_finish_shutdown_pluck(exec_ctx, cq);
   }
-  gpr_mu_unlock(cqd->mu);
+  gpr_mu_unlock(cq->mu);
+}
+
+/* Shutdown simply drops a ref that we reserved at creation time; if we drop
+   to zero here, then enter shutdown mode and wake up any waiters */
+void grpc_completion_queue_shutdown(grpc_completion_queue *cq) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  GPR_TIMER_BEGIN("grpc_completion_queue_shutdown", 0);
+  GRPC_API_TRACE("grpc_completion_queue_shutdown(cq=%p)", 1, (cq));
+  cq->vtable->shutdown(&exec_ctx, cq);
   grpc_exec_ctx_finish(&exec_ctx);
   GPR_TIMER_END("grpc_completion_queue_shutdown", 0);
 }
 
-void grpc_completion_queue_destroy(grpc_completion_queue *cc) {
-  GRPC_API_TRACE("grpc_completion_queue_destroy(cc=%p)", 1, (cc));
+void grpc_completion_queue_destroy(grpc_completion_queue *cq) {
+  GRPC_API_TRACE("grpc_completion_queue_destroy(cq=%p)", 1, (cq));
   GPR_TIMER_BEGIN("grpc_completion_queue_destroy", 0);
-  grpc_completion_queue_shutdown(cc);
-
-  /* TODO (sreek): This should not ideally be here. Refactor it into the
-   * cq_vtable (perhaps have a create/destroy methods in the cq vtable) */
-  if (cc->vtable->cq_completion_type == GRPC_CQ_NEXT) {
-    GPR_ASSERT(cq_event_queue_num_items(&cc->data.queue) == 0);
-  }
+  grpc_completion_queue_shutdown(cq);
 
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "destroy");
+  GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cq, "destroy");
   grpc_exec_ctx_finish(&exec_ctx);
   GPR_TIMER_END("grpc_completion_queue_destroy", 0);
 }
 
-grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
-  return cc->poller_vtable->can_get_pollset ? POLLSET_FROM_CQ(cc) : NULL;
-}
-
-grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) {
-  return CQ_FROM_POLLSET(ps);
-}
-
-void grpc_cq_mark_server_cq(grpc_completion_queue *cc) {
-  cc->data.is_server_cq = 1;
-}
-
-bool grpc_cq_is_server_cq(grpc_completion_queue *cc) {
-  return cc->data.is_server_cq;
+grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cq) {
+  return cq->poller_vtable->can_get_pollset ? POLLSET_FROM_CQ(cq) : NULL;
 }
 
-bool grpc_cq_can_listen(grpc_completion_queue *cc) {
-  return cc->poller_vtable->can_listen;
+bool grpc_cq_can_listen(grpc_completion_queue *cq) {
+  return cq->poller_vtable->can_listen;
 }
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index 97ea9cae20..af44482513 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -84,10 +84,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
                     void *done_arg, grpc_cq_completion *storage);
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
-grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps);
 
-void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
-bool grpc_cq_is_server_cq(grpc_completion_queue *cc);
 bool grpc_cq_can_listen(grpc_completion_queue *cc);
 
 grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc);
diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c
index 7a5f398c96..d199ac060e 100644
--- a/src/core/lib/surface/init.c
+++ b/src/core/lib/surface/init.c
@@ -121,30 +121,28 @@ void grpc_init(void) {
     grpc_slice_intern_init();
     grpc_mdctx_global_init();
     grpc_channel_init_init();
-    grpc_register_tracer("api", &grpc_api_trace);
-    grpc_register_tracer("channel", &grpc_trace_channel);
-    grpc_register_tracer("connectivity_state", &grpc_connectivity_state_trace);
-    grpc_register_tracer("channel_stack_builder",
-                         &grpc_trace_channel_stack_builder);
-    grpc_register_tracer("http1", &grpc_http1_trace);
-    grpc_register_tracer("queue_pluck", &grpc_cq_pluck_trace);  // default on
-    grpc_register_tracer("combiner", &grpc_combiner_trace);
-    grpc_register_tracer("server_channel", &grpc_server_channel_trace);
-    grpc_register_tracer("bdp_estimator", &grpc_bdp_estimator_trace);
-    grpc_register_tracer("queue_timeout",
-                         &grpc_cq_event_timeout_trace);  // default on
-    grpc_register_tracer("op_failure", &grpc_trace_operation_failures);
-    grpc_register_tracer("resource_quota", &grpc_resource_quota_trace);
-    grpc_register_tracer("call_error", &grpc_call_error_trace);
+    grpc_register_tracer(&grpc_api_trace);
+    grpc_register_tracer(&grpc_trace_channel);
+    grpc_register_tracer(&grpc_connectivity_state_trace);
+    grpc_register_tracer(&grpc_trace_channel_stack_builder);
+    grpc_register_tracer(&grpc_http1_trace);
+    grpc_register_tracer(&grpc_cq_pluck_trace);  // default on
+    grpc_register_tracer(&grpc_combiner_trace);
+    grpc_register_tracer(&grpc_server_channel_trace);
+    grpc_register_tracer(&grpc_bdp_estimator_trace);
+    grpc_register_tracer(&grpc_cq_event_timeout_trace);  // default on
+    grpc_register_tracer(&grpc_trace_operation_failures);
+    grpc_register_tracer(&grpc_resource_quota_trace);
+    grpc_register_tracer(&grpc_call_error_trace);
 #ifndef NDEBUG
-    grpc_register_tracer("pending_tags", &grpc_trace_pending_tags);
-    grpc_register_tracer("alarm_refcount", &grpc_trace_alarm_refcount);
-    grpc_register_tracer("queue_refcount", &grpc_trace_cq_refcount);
-    grpc_register_tracer("closure", &grpc_trace_closure);
-    grpc_register_tracer("error_refcount", &grpc_trace_error_refcount);
-    grpc_register_tracer("stream_refcount", &grpc_trace_stream_refcount);
-    grpc_register_tracer("fd_refcount", &grpc_trace_fd_refcount);
-    grpc_register_tracer("metadata", &grpc_trace_metadata);
+    grpc_register_tracer(&grpc_trace_pending_tags);
+    grpc_register_tracer(&grpc_trace_alarm_refcount);
+    grpc_register_tracer(&grpc_trace_cq_refcount);
+    grpc_register_tracer(&grpc_trace_closure);
+    grpc_register_tracer(&grpc_trace_error_refcount);
+    grpc_register_tracer(&grpc_trace_stream_refcount);
+    grpc_register_tracer(&grpc_trace_fd_refcount);
+    grpc_register_tracer(&grpc_trace_metadata);
 #endif
     grpc_security_pre_init();
     grpc_iomgr_init(&exec_ctx);
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 7dbea581d0..2366c24910 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -37,13 +37,11 @@
 #endif
 
 void grpc_security_pre_init(void) {
-  grpc_register_tracer("secure_endpoint", &grpc_trace_secure_endpoint);
-  grpc_register_tracer("transport_security", &tsi_tracing_enabled);
+  grpc_register_tracer(&grpc_trace_secure_endpoint);
+  grpc_register_tracer(&tsi_tracing_enabled);
 #ifndef NDEBUG
-  grpc_register_tracer("auth_context_refcount",
-                       &grpc_trace_auth_context_refcount);
-  grpc_register_tracer("security_connector_refcount",
-                       &grpc_trace_security_connector_refcount);
+  grpc_register_tracer(&grpc_trace_auth_context_refcount);
+  grpc_register_tracer(&grpc_trace_security_connector_refcount);
 #endif
 }
 
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index 84ddf74ab9..fce7f8dca1 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -32,8 +32,7 @@
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/support/mpscq.h"
-#include "src/core/lib/support/spinlock.h"
+#include "src/core/lib/support/stack_lockfree.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
@@ -59,10 +58,10 @@ typedef struct registered_method registered_method;
 
 typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
 
-grpc_tracer_flag grpc_server_channel_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_server_channel_trace =
+    GRPC_TRACER_INITIALIZER(false, "server_channel");
 
 typedef struct requested_call {
-  gpr_mpscq_node request_link; /* must be first */
   requested_call_type type;
   size_t cq_idx;
   void *tag;
@@ -162,7 +161,7 @@ struct request_matcher {
   grpc_server *server;
   call_data *pending_head;
   call_data *pending_tail;
-  gpr_locked_mpscq *requests_per_cq;
+  gpr_stack_lockfree **requests_per_cq;
 };
 
 struct registered_method {
@@ -207,6 +206,11 @@ struct grpc_server {
   registered_method *registered_methods;
   /** one request matcher for unregistered methods */
   request_matcher unregistered_request_matcher;
+  /** free list of available requested_calls_per_cq indices */
+  gpr_stack_lockfree **request_freelist_per_cq;
+  /** requested call backing data */
+  requested_call **requested_calls_per_cq;
+  int max_requested_calls_per_cq;
 
   gpr_atm shutdown_flag;
   uint8_t shutdown_published;
@@ -306,20 +310,21 @@ static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx,
  * request_matcher
  */
 
-static void request_matcher_init(request_matcher *rm, grpc_server *server) {
+static void request_matcher_init(request_matcher *rm, size_t entries,
+                                 grpc_server *server) {
   memset(rm, 0, sizeof(*rm));
   rm->server = server;
   rm->requests_per_cq =
       gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count);
   for (size_t i = 0; i < server->cq_count; i++) {
-    gpr_locked_mpscq_init(&rm->requests_per_cq[i]);
+    rm->requests_per_cq[i] = gpr_stack_lockfree_create(entries);
   }
 }
 
 static void request_matcher_destroy(request_matcher *rm) {
   for (size_t i = 0; i < rm->server->cq_count; i++) {
-    GPR_ASSERT(gpr_locked_mpscq_pop(&rm->requests_per_cq[i]) == NULL);
-    gpr_locked_mpscq_destroy(&rm->requests_per_cq[i]);
+    GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1);
+    gpr_stack_lockfree_destroy(rm->requests_per_cq[i]);
   }
   gpr_free(rm->requests_per_cq);
 }
@@ -349,17 +354,13 @@ static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx,
                                           grpc_server *server,
                                           request_matcher *rm,
                                           grpc_error *error) {
-  requested_call *rc;
+  int request_id;
   for (size_t i = 0; i < server->cq_count; i++) {
-    /* Here we know:
-       1. no requests are being added (since the server is shut down)
-       2. no other threads are pulling (since the shut down process is single
-          threaded)
-       So, we can ignore the queue lock and just pop, with the guarantee that a
-       NULL returned here truly means that the queue is empty */
-    while ((rc = (requested_call *)gpr_mpscq_pop(
-                &rm->requests_per_cq[i].queue)) != NULL) {
-      fail_call(exec_ctx, server, i, rc, GRPC_ERROR_REF(error));
+    while ((request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[i])) !=
+           -1) {
+      fail_call(exec_ctx, server, i,
+                &server->requested_calls_per_cq[i][request_id],
+                GRPC_ERROR_REF(error));
     }
   }
   GRPC_ERROR_UNREF(error);
@@ -394,7 +395,13 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
   }
   for (i = 0; i < server->cq_count; i++) {
     GRPC_CQ_INTERNAL_UNREF(exec_ctx, server->cqs[i], "server");
+    if (server->started) {
+      gpr_stack_lockfree_destroy(server->request_freelist_per_cq[i]);
+      gpr_free(server->requested_calls_per_cq[i]);
+    }
   }
+  gpr_free(server->request_freelist_per_cq);
+  gpr_free(server->requested_calls_per_cq);
   gpr_free(server->cqs);
   gpr_free(server->pollsets);
   gpr_free(server->shutdown_tags);
@@ -452,7 +459,21 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand,
 
 static void done_request_event(grpc_exec_ctx *exec_ctx, void *req,
                                grpc_cq_completion *c) {
-  gpr_free(req);
+  requested_call *rc = req;
+  grpc_server *server = rc->server;
+
+  if (rc >= server->requested_calls_per_cq[rc->cq_idx] &&
+      rc < server->requested_calls_per_cq[rc->cq_idx] +
+               server->max_requested_calls_per_cq) {
+    GPR_ASSERT(rc - server->requested_calls_per_cq[rc->cq_idx] <= INT_MAX);
+    gpr_stack_lockfree_push(
+        server->request_freelist_per_cq[rc->cq_idx],
+        (int)(rc - server->requested_calls_per_cq[rc->cq_idx]));
+  } else {
+    gpr_free(req);
+  }
+
+  server_unref(exec_ctx, server);
 }
 
 static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
@@ -482,6 +503,10 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
       GPR_UNREACHABLE_CODE(return );
   }
 
+  grpc_call_element *elem =
+      grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
+  channel_data *chand = elem->channel_data;
+  server_ref(chand->server);
   grpc_cq_end_op(exec_ctx, calld->cq_new, rc->tag, GRPC_ERROR_NONE,
                  done_request_event, rc, &rc->completion);
 }
@@ -509,15 +534,15 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg,
 
   for (size_t i = 0; i < server->cq_count; i++) {
     size_t cq_idx = (chand->cq_idx + i) % server->cq_count;
-    requested_call *rc =
-        (requested_call *)gpr_locked_mpscq_pop(&rm->requests_per_cq[cq_idx]);
-    if (rc == NULL) {
+    int request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+    if (request_id == -1) {
       continue;
     } else {
       gpr_mu_lock(&calld->mu_state);
       calld->state = ACTIVATED;
       gpr_mu_unlock(&calld->mu_state);
-      publish_call(exec_ctx, server, calld, cq_idx, rc);
+      publish_call(exec_ctx, server, calld, cq_idx,
+                   &server->requested_calls_per_cq[cq_idx][request_id]);
       return; /* early out */
     }
   }
@@ -951,8 +976,6 @@ static void register_completion_queue(grpc_server *server,
     if (server->cqs[i] == cq) return;
   }
 
-  grpc_cq_mark_server_cq(cq);
-
   GRPC_CQ_INTERNAL_REF(cq, "server");
   n = server->cq_count++;
   server->cqs = gpr_realloc(server->cqs,
@@ -992,6 +1015,8 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   server->root_channel_data.next = server->root_channel_data.prev =
       &server->root_channel_data;
 
+  /* TODO(ctiller): expose a channel_arg for this */
+  server->max_requested_calls_per_cq = 32768;
   server->channel_args = grpc_channel_args_copy(args);
 
   return server;
@@ -1064,15 +1089,29 @@ void grpc_server_start(grpc_server *server) {
   server->started = true;
   server->pollset_count = 0;
   server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
+  server->request_freelist_per_cq =
+      gpr_malloc(sizeof(*server->request_freelist_per_cq) * server->cq_count);
+  server->requested_calls_per_cq =
+      gpr_malloc(sizeof(*server->requested_calls_per_cq) * server->cq_count);
   for (i = 0; i < server->cq_count; i++) {
     if (grpc_cq_can_listen(server->cqs[i])) {
       server->pollsets[server->pollset_count++] =
           grpc_cq_pollset(server->cqs[i]);
     }
+    server->request_freelist_per_cq[i] =
+        gpr_stack_lockfree_create((size_t)server->max_requested_calls_per_cq);
+    for (int j = 0; j < server->max_requested_calls_per_cq; j++) {
+      gpr_stack_lockfree_push(server->request_freelist_per_cq[i], j);
+    }
+    server->requested_calls_per_cq[i] =
+        gpr_malloc((size_t)server->max_requested_calls_per_cq *
+                   sizeof(*server->requested_calls_per_cq[i]));
   }
-  request_matcher_init(&server->unregistered_request_matcher, server);
+  request_matcher_init(&server->unregistered_request_matcher,
+                       (size_t)server->max_requested_calls_per_cq, server);
   for (registered_method *rm = server->registered_methods; rm; rm = rm->next) {
-    request_matcher_init(&rm->request_matcher, server);
+    request_matcher_init(&rm->request_matcher,
+                         (size_t)server->max_requested_calls_per_cq, server);
   }
 
   server_ref(server);
@@ -1116,9 +1155,8 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   chand->channel = channel;
 
   size_t cq_idx;
-  grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
   for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
-    if (s->cqs[cq_idx] == accepting_cq) break;
+    if (grpc_cq_pollset(s->cqs[cq_idx]) == accepting_pollset) break;
   }
   if (cq_idx == s->cq_count) {
     /* completion queue not found: pick a random one to publish new calls to */
@@ -1326,11 +1364,21 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
                                           requested_call *rc) {
   call_data *calld = NULL;
   request_matcher *rm = NULL;
+  int request_id;
   if (gpr_atm_acq_load(&server->shutdown_flag)) {
     fail_call(exec_ctx, server, cq_idx, rc,
               GRPC_ERROR_CREATE_FROM_STATIC_STRING("Server Shutdown"));
     return GRPC_CALL_OK;
   }
+  request_id = gpr_stack_lockfree_pop(server->request_freelist_per_cq[cq_idx]);
+  if (request_id == -1) {
+    /* out of request ids: just fail this one */
+    fail_call(exec_ctx, server, cq_idx, rc,
+              grpc_error_set_int(
+                  GRPC_ERROR_CREATE_FROM_STATIC_STRING("Out of request ids"),
+                  GRPC_ERROR_INT_LIMIT, server->max_requested_calls_per_cq));
+    return GRPC_CALL_OK;
+  }
   switch (rc->type) {
     case BATCH_CALL:
       rm = &server->unregistered_request_matcher;
@@ -1339,13 +1387,15 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
       rm = &rc->data.registered.registered_method->request_matcher;
       break;
   }
-  if (gpr_locked_mpscq_push(&rm->requests_per_cq[cq_idx], &rc->request_link)) {
+  server->requested_calls_per_cq[cq_idx][request_id] = *rc;
+  gpr_free(rc);
+  if (gpr_stack_lockfree_push(rm->requests_per_cq[cq_idx], request_id)) {
     /* this was the first queued request: we need to lock and start
        matching calls */
     gpr_mu_lock(&server->mu_call);
     while ((calld = rm->pending_head) != NULL) {
-      rc = (requested_call *)gpr_locked_mpscq_pop(&rm->requests_per_cq[cq_idx]);
-      if (rc == NULL) break;
+      request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+      if (request_id == -1) break;
       rm->pending_head = calld->pending_next;
       gpr_mu_unlock(&server->mu_call);
       gpr_mu_lock(&calld->mu_state);
@@ -1361,7 +1411,8 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
         GPR_ASSERT(calld->state == PENDING);
         calld->state = ACTIVATED;
         gpr_mu_unlock(&calld->mu_state);
-        publish_call(exec_ctx, server, calld, cq_idx, rc);
+        publish_call(exec_ctx, server, calld, cq_idx,
+                     &server->requested_calls_per_cq[cq_idx][request_id]);
       }
       gpr_mu_lock(&server->mu_call);
     }
@@ -1468,6 +1519,7 @@ static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
   rc->initial_metadata->count = 0;
   GPR_ASSERT(error != GRPC_ERROR_NONE);
 
+  server_ref(server);
   grpc_cq_end_op(exec_ctx, server->cqs[cq_idx], rc->tag, error,
                  done_request_event, rc, &rc->completion);
 }
diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c
index d33e3a48ab..311ae6390d 100644
--- a/src/core/lib/transport/bdp_estimator.c
+++ b/src/core/lib/transport/bdp_estimator.c
@@ -23,7 +23,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
-grpc_tracer_flag grpc_bdp_estimator_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_bdp_estimator_trace =
+    GRPC_TRACER_INITIALIZER(false, "bdp_estimator");
 
 void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) {
   estimator->estimate = 65536;
diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c
index 6fe40af3b2..73a9178ae2 100644
--- a/src/core/lib/transport/connectivity_state.c
+++ b/src/core/lib/transport/connectivity_state.c
@@ -24,7 +24,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-grpc_tracer_flag grpc_connectivity_state_trace = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_connectivity_state_trace =
+    GRPC_TRACER_INITIALIZER(false, "connectivity_state");
 
 const char *grpc_connectivity_state_name(grpc_connectivity_state state) {
   switch (state) {
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 87a2abf344..2fea366072 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -48,7 +48,8 @@
  */
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_metadata = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_metadata =
+    GRPC_TRACER_INITIALIZER(false, "metadata");
 #define DEBUG_ARGS , const char *file, int line
 #define FWD_DEBUG_ARGS , file, line
 #define REF_MD_LOCKED(shard, s) ref_md_locked((shard), (s), __FILE__, __LINE__)
diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c
index 404c240589..2388f19f81 100644
--- a/src/core/lib/transport/static_metadata.c
+++ b/src/core/lib/transport/static_metadata.c
@@ -464,7 +464,8 @@ grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) {
   if (a == -1 || b == -1) return GRPC_MDNULL;
   uint32_t k = (uint32_t)(a * 99 + b);
   uint32_t h = elems_phash(k);
-  return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k
+  return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k &&
+                 elem_idxs[h] != 255
              ? GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[elem_idxs[h]],
                                 GRPC_MDELEM_STORAGE_STATIC)
              : GRPC_MDNULL;
diff --git a/src/core/lib/transport/transport.c b/src/core/lib/transport/transport.c
index 6a9eba110d..7281602d66 100644
--- a/src/core/lib/transport/transport.c
+++ b/src/core/lib/transport/transport.c
@@ -32,7 +32,8 @@
 #include "src/core/lib/transport/transport_impl.h"
 
 #ifndef NDEBUG
-grpc_tracer_flag grpc_trace_stream_refcount = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag grpc_trace_stream_refcount =
+    GRPC_TRACER_INITIALIZER(false, "stream_refcount");
 #endif
 
 #ifndef NDEBUG
diff --git a/src/core/plugin_registry/grpc_cronet_plugin_registry.c b/src/core/plugin_registry/grpc_cronet_plugin_registry.c
index b468c03aa3..322ebea111 100644
--- a/src/core/plugin_registry/grpc_cronet_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_cronet_plugin_registry.c
@@ -26,6 +26,8 @@ extern void grpc_deadline_filter_init(void);
 extern void grpc_deadline_filter_shutdown(void);
 extern void grpc_client_channel_init(void);
 extern void grpc_client_channel_shutdown(void);
+extern void grpc_tsi_gts_init(void);
+extern void grpc_tsi_gts_shutdown(void);
 extern void grpc_load_reporting_plugin_init(void);
 extern void grpc_load_reporting_plugin_shutdown(void);
 
@@ -38,6 +40,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_deadline_filter_shutdown);
   grpc_register_plugin(grpc_client_channel_init,
                        grpc_client_channel_shutdown);
+  grpc_register_plugin(grpc_tsi_gts_init,
+                       grpc_tsi_gts_shutdown);
   grpc_register_plugin(grpc_load_reporting_plugin_init,
                        grpc_load_reporting_plugin_shutdown);
 }
diff --git a/src/core/plugin_registry/grpc_plugin_registry.c b/src/core/plugin_registry/grpc_plugin_registry.c
index a816186c39..fa9974952c 100644
--- a/src/core/plugin_registry/grpc_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_plugin_registry.c
@@ -22,10 +22,14 @@ extern void grpc_http_filters_init(void);
 extern void grpc_http_filters_shutdown(void);
 extern void grpc_chttp2_plugin_init(void);
 extern void grpc_chttp2_plugin_shutdown(void);
+extern void grpc_tsi_gts_init(void);
+extern void grpc_tsi_gts_shutdown(void);
 extern void grpc_deadline_filter_init(void);
 extern void grpc_deadline_filter_shutdown(void);
 extern void grpc_client_channel_init(void);
 extern void grpc_client_channel_shutdown(void);
+extern void grpc_inproc_plugin_init(void);
+extern void grpc_inproc_plugin_shutdown(void);
 extern void grpc_resolver_fake_init(void);
 extern void grpc_resolver_fake_shutdown(void);
 extern void grpc_lb_policy_grpclb_init(void);
@@ -56,10 +60,14 @@ void grpc_register_built_in_plugins(void) {
                        grpc_http_filters_shutdown);
   grpc_register_plugin(grpc_chttp2_plugin_init,
                        grpc_chttp2_plugin_shutdown);
+  grpc_register_plugin(grpc_tsi_gts_init,
+                       grpc_tsi_gts_shutdown);
   grpc_register_plugin(grpc_deadline_filter_init,
                        grpc_deadline_filter_shutdown);
   grpc_register_plugin(grpc_client_channel_init,
                        grpc_client_channel_shutdown);
+  grpc_register_plugin(grpc_inproc_plugin_init,
+                       grpc_inproc_plugin_shutdown);
   grpc_register_plugin(grpc_resolver_fake_init,
                        grpc_resolver_fake_shutdown);
   grpc_register_plugin(grpc_lb_policy_grpclb_init,
diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
index 809d444bf7..7eb599d81a 100644
--- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
@@ -26,6 +26,8 @@ extern void grpc_deadline_filter_init(void);
 extern void grpc_deadline_filter_shutdown(void);
 extern void grpc_client_channel_init(void);
 extern void grpc_client_channel_shutdown(void);
+extern void grpc_inproc_plugin_init(void);
+extern void grpc_inproc_plugin_shutdown(void);
 extern void grpc_resolver_dns_ares_init(void);
 extern void grpc_resolver_dns_ares_shutdown(void);
 extern void grpc_resolver_dns_native_init(void);
@@ -60,6 +62,8 @@ void grpc_register_built_in_plugins(void) {
                        grpc_deadline_filter_shutdown);
   grpc_register_plugin(grpc_client_channel_init,
                        grpc_client_channel_shutdown);
+  grpc_register_plugin(grpc_inproc_plugin_init,
+                       grpc_inproc_plugin_shutdown);
   grpc_register_plugin(grpc_resolver_dns_ares_init,
                        grpc_resolver_dns_ares_shutdown);
   grpc_register_plugin(grpc_resolver_dns_native_init,
diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c
index 1e919c4cce..1280680663 100644
--- a/src/core/tsi/fake_transport_security.c
+++ b/src/core/tsi/fake_transport_security.c
@@ -31,6 +31,7 @@
 #define TSI_FAKE_FRAME_HEADER_SIZE 4
 #define TSI_FAKE_FRAME_INITIAL_ALLOCATED_SIZE 64
 #define TSI_FAKE_DEFAULT_FRAME_SIZE 16384
+#define TSI_FAKE_HANDSHAKER_OUTGOING_BUFFER_INITIAL_SIZE 256
 
 /* --- Structure definitions. ---*/
 
@@ -59,8 +60,10 @@ typedef struct {
   int is_client;
   tsi_fake_handshake_message next_message_to_send;
   int needs_incoming_message;
-  tsi_fake_frame incoming;
-  tsi_fake_frame outgoing;
+  tsi_fake_frame incoming_frame;
+  tsi_fake_frame outgoing_frame;
+  unsigned char *outgoing_bytes_buffer;
+  size_t outgoing_bytes_buffer_size;
   tsi_result result;
 } tsi_fake_handshaker;
 
@@ -116,27 +119,23 @@ static void tsi_fake_frame_reset(tsi_fake_frame *frame, int needs_draining) {
   if (!needs_draining) frame->size = 0;
 }
 
-/* Returns 1 if successful, 0 otherwise. */
-static int tsi_fake_frame_ensure_size(tsi_fake_frame *frame) {
+/* Checks if the frame's allocated size is at least frame->size, and reallocs
+ * more memory if necessary. */
+static void tsi_fake_frame_ensure_size(tsi_fake_frame *frame) {
   if (frame->data == NULL) {
     frame->allocated_size = frame->size;
     frame->data = gpr_malloc(frame->allocated_size);
-    if (frame->data == NULL) return 0;
   } else if (frame->size > frame->allocated_size) {
     unsigned char *new_data = gpr_realloc(frame->data, frame->size);
-    if (new_data == NULL) {
-      gpr_free(frame->data);
-      frame->data = NULL;
-      return 0;
-    }
     frame->data = new_data;
     frame->allocated_size = frame->size;
   }
-  return 1;
 }
 
-/* This method should not be called if frame->needs_framing is not 0.  */
-static tsi_result fill_frame_from_bytes(const unsigned char *incoming_bytes,
+/* Decodes the serialized fake frame contained in incoming_bytes, and fills
+ * frame with the contents of the decoded frame.
+ * This method should not be called if frame->needs_framing is not 0.  */
+static tsi_result tsi_fake_frame_decode(const unsigned char *incoming_bytes,
                                         size_t *incoming_bytes_size,
                                         tsi_fake_frame *frame) {
   size_t available_size = *incoming_bytes_size;
@@ -147,7 +146,6 @@ static tsi_result fill_frame_from_bytes(const unsigned char *incoming_bytes,
   if (frame->data == NULL) {
     frame->allocated_size = TSI_FAKE_FRAME_INITIAL_ALLOCATED_SIZE;
     frame->data = gpr_malloc(frame->allocated_size);
-    if (frame->data == NULL) return TSI_OUT_OF_RESOURCES;
   }
 
   if (frame->offset < TSI_FAKE_FRAME_HEADER_SIZE) {
@@ -165,7 +163,7 @@ static tsi_result fill_frame_from_bytes(const unsigned char *incoming_bytes,
     frame->offset += to_read_size;
     available_size -= to_read_size;
     frame->size = load32_little_endian(frame->data);
-    if (!tsi_fake_frame_ensure_size(frame)) return TSI_OUT_OF_RESOURCES;
+    tsi_fake_frame_ensure_size(frame);
   }
 
   to_read_size = frame->size - frame->offset;
@@ -183,10 +181,12 @@ static tsi_result fill_frame_from_bytes(const unsigned char *incoming_bytes,
   return TSI_OK;
 }
 
-/* This method should not be called if frame->needs_framing is 0.  */
-static tsi_result drain_frame_to_bytes(unsigned char *outgoing_bytes,
-                                       size_t *outgoing_bytes_size,
-                                       tsi_fake_frame *frame) {
+/* Encodes a fake frame into its wire format and places the result in
+ * outgoing_bytes. outgoing_bytes_size indicates the size of the encoded frame.
+ * This method should not be called if frame->needs_framing is 0.  */
+static tsi_result tsi_fake_frame_encode(unsigned char *outgoing_bytes,
+                                        size_t *outgoing_bytes_size,
+                                        tsi_fake_frame *frame) {
   size_t to_write_size = frame->size - frame->offset;
   if (!frame->needs_draining) return TSI_INTERNAL_ERROR;
   if (*outgoing_bytes_size < to_write_size) {
@@ -200,17 +200,20 @@ static tsi_result drain_frame_to_bytes(unsigned char *outgoing_bytes,
   return TSI_OK;
 }
 
-static tsi_result bytes_to_frame(unsigned char *bytes, size_t bytes_size,
-                                 tsi_fake_frame *frame) {
+/* Sets the payload of a fake frame to contain the given data blob, where
+ * data_size indicates the size of data. */
+static tsi_result tsi_fake_frame_set_data(unsigned char *data, size_t data_size,
+                                          tsi_fake_frame *frame) {
   frame->offset = 0;
-  frame->size = bytes_size + TSI_FAKE_FRAME_HEADER_SIZE;
-  if (!tsi_fake_frame_ensure_size(frame)) return TSI_OUT_OF_RESOURCES;
+  frame->size = data_size + TSI_FAKE_FRAME_HEADER_SIZE;
+  tsi_fake_frame_ensure_size(frame);
   store32_little_endian((uint32_t)frame->size, frame->data);
-  memcpy(frame->data + TSI_FAKE_FRAME_HEADER_SIZE, bytes, bytes_size);
+  memcpy(frame->data + TSI_FAKE_FRAME_HEADER_SIZE, data, data_size);
   tsi_fake_frame_reset(frame, 1 /* needs draining */);
   return TSI_OK;
 }
 
+/* Destroys the contents of a fake frame. */
 static void tsi_fake_frame_destruct(tsi_fake_frame *frame) {
   if (frame->data != NULL) gpr_free(frame->data);
 }
@@ -235,7 +238,7 @@ static tsi_result fake_protector_protect(tsi_frame_protector *self,
   if (frame->needs_draining) {
     drained_size = saved_output_size - *num_bytes_written;
     result =
-        drain_frame_to_bytes(protected_output_frames, &drained_size, frame);
+        tsi_fake_frame_encode(protected_output_frames, &drained_size, frame);
     *num_bytes_written += drained_size;
     protected_output_frames += drained_size;
     if (result != TSI_OK) {
@@ -254,15 +257,15 @@ static tsi_result fake_protector_protect(tsi_frame_protector *self,
     size_t written_in_frame_size = 0;
     store32_little_endian((uint32_t)impl->max_frame_size, frame_header);
     written_in_frame_size = TSI_FAKE_FRAME_HEADER_SIZE;
-    result = fill_frame_from_bytes(frame_header, &written_in_frame_size, frame);
+    result = tsi_fake_frame_decode(frame_header, &written_in_frame_size, frame);
     if (result != TSI_INCOMPLETE_DATA) {
-      gpr_log(GPR_ERROR, "fill_frame_from_bytes returned %s",
+      gpr_log(GPR_ERROR, "tsi_fake_frame_decode returned %s",
               tsi_result_to_string(result));
       return result;
     }
   }
   result =
-      fill_frame_from_bytes(unprotected_bytes, unprotected_bytes_size, frame);
+      tsi_fake_frame_decode(unprotected_bytes, unprotected_bytes_size, frame);
   if (result != TSI_OK) {
     if (result == TSI_INCOMPLETE_DATA) result = TSI_OK;
     return result;
@@ -272,7 +275,7 @@ static tsi_result fake_protector_protect(tsi_frame_protector *self,
   if (!frame->needs_draining) return TSI_INTERNAL_ERROR;
   if (frame->offset != 0) return TSI_INTERNAL_ERROR;
   drained_size = saved_output_size - *num_bytes_written;
-  result = drain_frame_to_bytes(protected_output_frames, &drained_size, frame);
+  result = tsi_fake_frame_encode(protected_output_frames, &drained_size, frame);
   *num_bytes_written += drained_size;
   if (result == TSI_INCOMPLETE_DATA) result = TSI_OK;
   return result;
@@ -292,8 +295,8 @@ static tsi_result fake_protector_protect_flush(
     store32_little_endian((uint32_t)frame->size,
                           frame->data); /* Overwrite header. */
   }
-  result = drain_frame_to_bytes(protected_output_frames,
-                                protected_output_frames_size, frame);
+  result = tsi_fake_frame_encode(protected_output_frames,
+                                 protected_output_frames_size, frame);
   if (result == TSI_INCOMPLETE_DATA) result = TSI_OK;
   *still_pending_size = frame->size - frame->offset;
   return result;
@@ -316,7 +319,7 @@ static tsi_result fake_protector_unprotect(
     /* Go past the header if needed. */
     if (frame->offset == 0) frame->offset = TSI_FAKE_FRAME_HEADER_SIZE;
     drained_size = saved_output_size - *num_bytes_written;
-    result = drain_frame_to_bytes(unprotected_bytes, &drained_size, frame);
+    result = tsi_fake_frame_encode(unprotected_bytes, &drained_size, frame);
     unprotected_bytes += drained_size;
     *num_bytes_written += drained_size;
     if (result != TSI_OK) {
@@ -330,7 +333,7 @@ static tsi_result fake_protector_unprotect(
 
   /* Now process the protected_bytes. */
   if (frame->needs_draining) return TSI_INTERNAL_ERROR;
-  result = fill_frame_from_bytes(protected_frames_bytes,
+  result = tsi_fake_frame_decode(protected_frames_bytes,
                                  protected_frames_bytes_size, frame);
   if (result != TSI_OK) {
     if (result == TSI_INCOMPLETE_DATA) result = TSI_OK;
@@ -342,7 +345,7 @@ static tsi_result fake_protector_unprotect(
   if (frame->offset != 0) return TSI_INTERNAL_ERROR;
   frame->offset = TSI_FAKE_FRAME_HEADER_SIZE; /* Go past the header. */
   drained_size = saved_output_size - *num_bytes_written;
-  result = drain_frame_to_bytes(unprotected_bytes, &drained_size, frame);
+  result = tsi_fake_frame_encode(unprotected_bytes, &drained_size, frame);
   *num_bytes_written += drained_size;
   if (result == TSI_INCOMPLETE_DATA) result = TSI_OK;
   return result;
@@ -360,6 +363,72 @@ static const tsi_frame_protector_vtable frame_protector_vtable = {
     fake_protector_unprotect, fake_protector_destroy,
 };
 
+/* --- tsi_handshaker_result methods implementation. ---*/
+
+typedef struct {
+  tsi_handshaker_result base;
+  unsigned char *unused_bytes;
+  size_t unused_bytes_size;
+} fake_handshaker_result;
+
+static tsi_result fake_handshaker_result_extract_peer(
+    const tsi_handshaker_result *self, tsi_peer *peer) {
+  /* Construct a tsi_peer with 1 property: certificate type. */
+  tsi_result result = tsi_construct_peer(1, peer);
+  if (result != TSI_OK) return result;
+  result = tsi_construct_string_peer_property_from_cstring(
+      TSI_CERTIFICATE_TYPE_PEER_PROPERTY, TSI_FAKE_CERTIFICATE_TYPE,
+      &peer->properties[0]);
+  if (result != TSI_OK) tsi_peer_destruct(peer);
+  return result;
+}
+
+static tsi_result fake_handshaker_result_create_frame_protector(
+    const tsi_handshaker_result *self, size_t *max_output_protected_frame_size,
+    tsi_frame_protector **protector) {
+  *protector = tsi_create_fake_frame_protector(max_output_protected_frame_size);
+  return TSI_OK;
+}
+
+static tsi_result fake_handshaker_result_get_unused_bytes(
+    const tsi_handshaker_result *self, unsigned char **bytes,
+    size_t *bytes_size) {
+  fake_handshaker_result *result = (fake_handshaker_result *)self;
+  *bytes_size = result->unused_bytes_size;
+  *bytes = result->unused_bytes;
+  return TSI_OK;
+}
+
+static void fake_handshaker_result_destroy(tsi_handshaker_result *self) {
+  fake_handshaker_result *result = (fake_handshaker_result *)self;
+  gpr_free(result->unused_bytes);
+  gpr_free(self);
+}
+
+static const tsi_handshaker_result_vtable handshaker_result_vtable = {
+    fake_handshaker_result_extract_peer,
+    fake_handshaker_result_create_frame_protector,
+    fake_handshaker_result_get_unused_bytes, fake_handshaker_result_destroy,
+};
+
+static tsi_result fake_handshaker_result_create(
+    const unsigned char *unused_bytes, size_t unused_bytes_size,
+    tsi_handshaker_result **handshaker_result) {
+  if ((unused_bytes_size > 0 && unused_bytes == NULL) ||
+      handshaker_result == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  fake_handshaker_result *result = gpr_zalloc(sizeof(*result));
+  result->base.vtable = &handshaker_result_vtable;
+  if (unused_bytes_size > 0) {
+    result->unused_bytes = gpr_malloc(unused_bytes_size);
+    memcpy(result->unused_bytes, unused_bytes, unused_bytes_size);
+  }
+  result->unused_bytes_size = unused_bytes_size;
+  *handshaker_result = &result->base;
+  return TSI_OK;
+}
+
 /* --- tsi_handshaker methods implementation. ---*/
 
 static tsi_result fake_handshaker_get_bytes_to_send_to_peer(
@@ -370,13 +439,13 @@ static tsi_result fake_handshaker_get_bytes_to_send_to_peer(
     *bytes_size = 0;
     return TSI_OK;
   }
-  if (!impl->outgoing.needs_draining) {
+  if (!impl->outgoing_frame.needs_draining) {
     tsi_fake_handshake_message next_message_to_send =
         impl->next_message_to_send + 2;
     const char *msg_string =
         tsi_fake_handshake_message_to_string(impl->next_message_to_send);
-    result = bytes_to_frame((unsigned char *)msg_string, strlen(msg_string),
-                            &impl->outgoing);
+    result = tsi_fake_frame_set_data((unsigned char *)msg_string,
+                                     strlen(msg_string), &impl->outgoing_frame);
     if (result != TSI_OK) return result;
     if (next_message_to_send > TSI_FAKE_HANDSHAKE_MESSAGE_MAX) {
       next_message_to_send = TSI_FAKE_HANDSHAKE_MESSAGE_MAX;
@@ -388,7 +457,7 @@ static tsi_result fake_handshaker_get_bytes_to_send_to_peer(
     }
     impl->next_message_to_send = next_message_to_send;
   }
-  result = drain_frame_to_bytes(bytes, bytes_size, &impl->outgoing);
+  result = tsi_fake_frame_encode(bytes, bytes_size, &impl->outgoing_frame);
   if (result != TSI_OK) return result;
   if (!impl->is_client &&
       impl->next_message_to_send == TSI_FAKE_HANDSHAKE_MESSAGE_MAX) {
@@ -414,12 +483,12 @@ static tsi_result fake_handshaker_process_bytes_from_peer(
     *bytes_size = 0;
     return TSI_OK;
   }
-  result = fill_frame_from_bytes(bytes, bytes_size, &impl->incoming);
+  result = tsi_fake_frame_decode(bytes, bytes_size, &impl->incoming_frame);
   if (result != TSI_OK) return result;
 
   /* We now have a complete frame. */
   result = tsi_fake_handshake_message_from_string(
-      (const char *)impl->incoming.data + TSI_FAKE_FRAME_HEADER_SIZE,
+      (const char *)impl->incoming_frame.data + TSI_FAKE_FRAME_HEADER_SIZE,
       &received_msg);
   if (result != TSI_OK) {
     impl->result = result;
@@ -434,7 +503,7 @@ static tsi_result fake_handshaker_process_bytes_from_peer(
     gpr_log(GPR_INFO, "%s received %s.", impl->is_client ? "Client" : "Server",
             tsi_fake_handshake_message_to_string(received_msg));
   }
-  tsi_fake_frame_reset(&impl->incoming, 0 /* needs_draining */);
+  tsi_fake_frame_reset(&impl->incoming_frame, 0 /* needs_draining */);
   impl->needs_incoming_message = 0;
   if (impl->next_message_to_send == TSI_FAKE_HANDSHAKE_MESSAGE_MAX) {
     /* We're done. */
@@ -451,40 +520,86 @@ static tsi_result fake_handshaker_get_result(tsi_handshaker *self) {
   return impl->result;
 }
 
-static tsi_result fake_handshaker_extract_peer(tsi_handshaker *self,
-                                               tsi_peer *peer) {
-  tsi_result result = tsi_construct_peer(1, peer);
-  if (result != TSI_OK) return result;
-  result = tsi_construct_string_peer_property_from_cstring(
-      TSI_CERTIFICATE_TYPE_PEER_PROPERTY, TSI_FAKE_CERTIFICATE_TYPE,
-      &peer->properties[0]);
-  if (result != TSI_OK) tsi_peer_destruct(peer);
-  return result;
-}
-
-static tsi_result fake_handshaker_create_frame_protector(
-    tsi_handshaker *self, size_t *max_protected_frame_size,
-    tsi_frame_protector **protector) {
-  *protector = tsi_create_fake_protector(max_protected_frame_size);
-  if (*protector == NULL) return TSI_OUT_OF_RESOURCES;
-  return TSI_OK;
-}
-
 static void fake_handshaker_destroy(tsi_handshaker *self) {
   tsi_fake_handshaker *impl = (tsi_fake_handshaker *)self;
-  tsi_fake_frame_destruct(&impl->incoming);
-  tsi_fake_frame_destruct(&impl->outgoing);
+  tsi_fake_frame_destruct(&impl->incoming_frame);
+  tsi_fake_frame_destruct(&impl->outgoing_frame);
+  gpr_free(impl->outgoing_bytes_buffer);
   gpr_free(self);
 }
 
+static tsi_result fake_handshaker_next(
+    tsi_handshaker *self, const unsigned char *received_bytes,
+    size_t received_bytes_size, unsigned char **bytes_to_send,
+    size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result,
+    tsi_handshaker_on_next_done_cb cb, void *user_data) {
+  /* Sanity check the arguments. */
+  if ((received_bytes_size > 0 && received_bytes == NULL) ||
+      bytes_to_send == NULL || bytes_to_send_size == NULL ||
+      handshaker_result == NULL) {
+    return TSI_INVALID_ARGUMENT;
+  }
+  tsi_fake_handshaker *handshaker = (tsi_fake_handshaker *)self;
+  tsi_result result = TSI_OK;
+
+  /* Decode and process a handshake frame from the peer. */
+  size_t consumed_bytes_size = received_bytes_size;
+  if (received_bytes_size > 0) {
+    result = fake_handshaker_process_bytes_from_peer(self, received_bytes,
+                                                     &consumed_bytes_size);
+    if (result != TSI_OK) return result;
+  }
+
+  /* Create a handshake message to send to the peer and encode it as a fake
+   * frame. */
+  size_t offset = 0;
+  do {
+    size_t sent_bytes_size = handshaker->outgoing_bytes_buffer_size - offset;
+    result = fake_handshaker_get_bytes_to_send_to_peer(
+        self, handshaker->outgoing_bytes_buffer + offset, &sent_bytes_size);
+    offset += sent_bytes_size;
+    if (result == TSI_INCOMPLETE_DATA) {
+      handshaker->outgoing_bytes_buffer_size *= 2;
+      handshaker->outgoing_bytes_buffer =
+          gpr_realloc(handshaker->outgoing_bytes_buffer,
+                      handshaker->outgoing_bytes_buffer_size);
+    }
+  } while (result == TSI_INCOMPLETE_DATA);
+  if (result != TSI_OK) return result;
+  *bytes_to_send = handshaker->outgoing_bytes_buffer;
+  *bytes_to_send_size = offset;
+
+  /* Check if the handshake was completed. */
+  if (fake_handshaker_get_result(self) == TSI_HANDSHAKE_IN_PROGRESS) {
+    *handshaker_result = NULL;
+  } else {
+    /* Calculate the unused bytes. */
+    const unsigned char *unused_bytes = NULL;
+    size_t unused_bytes_size = received_bytes_size - consumed_bytes_size;
+    if (unused_bytes_size > 0) {
+      unused_bytes = received_bytes + consumed_bytes_size;
+    }
+
+    /* Create a handshaker_result containing the unused bytes. */
+    result = fake_handshaker_result_create(unused_bytes, unused_bytes_size,
+                                           handshaker_result);
+    if (result == TSI_OK) {
+      /* Indicate that the handshake has completed and that a handshaker_result
+       * has been created. */
+      self->handshaker_result_created = true;
+    }
+  }
+  return result;
+}
+
 static const tsi_handshaker_vtable handshaker_vtable = {
-    fake_handshaker_get_bytes_to_send_to_peer,
-    fake_handshaker_process_bytes_from_peer,
-    fake_handshaker_get_result,
-    fake_handshaker_extract_peer,
-    fake_handshaker_create_frame_protector,
+    NULL, /* get_bytes_to_send_to_peer -- deprecated */
+    NULL, /* process_bytes_from_peer   -- deprecated */
+    NULL, /* get_result                -- deprecated */
+    NULL, /* extract_peer              -- deprecated */
+    NULL, /* create_frame_protector    -- deprecated */
     fake_handshaker_destroy,
-    NULL,
+    fake_handshaker_next,
 };
 
 tsi_handshaker *tsi_create_fake_handshaker(int is_client) {
@@ -492,6 +607,9 @@ tsi_handshaker *tsi_create_fake_handshaker(int is_client) {
   impl->base.vtable = &handshaker_vtable;
   impl->is_client = is_client;
   impl->result = TSI_HANDSHAKE_IN_PROGRESS;
+  impl->outgoing_bytes_buffer_size =
+      TSI_FAKE_HANDSHAKER_OUTGOING_BUFFER_INITIAL_SIZE;
+  impl->outgoing_bytes_buffer = gpr_malloc(impl->outgoing_bytes_buffer_size);
   if (is_client) {
     impl->needs_incoming_message = 0;
     impl->next_message_to_send = TSI_FAKE_CLIENT_INIT;
@@ -502,7 +620,7 @@ tsi_handshaker *tsi_create_fake_handshaker(int is_client) {
   return &impl->base;
 }
 
-tsi_frame_protector *tsi_create_fake_protector(
+tsi_frame_protector *tsi_create_fake_frame_protector(
     size_t *max_protected_frame_size) {
   tsi_fake_frame_protector *impl = gpr_zalloc(sizeof(*impl));
   impl->max_frame_size = (max_protected_frame_size == NULL)
diff --git a/src/core/tsi/fake_transport_security.h b/src/core/tsi/fake_transport_security.h
index 3d468c477f..934b3cbeb2 100644
--- a/src/core/tsi/fake_transport_security.h
+++ b/src/core/tsi/fake_transport_security.h
@@ -36,7 +36,7 @@ extern "C" {
 tsi_handshaker *tsi_create_fake_handshaker(int is_client);
 
 /* Creates a protector directly without going through the handshake phase. */
-tsi_frame_protector *tsi_create_fake_protector(
+tsi_frame_protector *tsi_create_fake_frame_protector(
     size_t *max_protected_frame_size);
 
 #ifdef __cplusplus
diff --git a/src/core/tsi/gts_transport_security.c b/src/core/tsi/gts_transport_security.c
new file mode 100644
index 0000000000..e2ac685e44
--- /dev/null
+++ b/src/core/tsi/gts_transport_security.c
@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/tsi/gts_transport_security.h"
+
+#include <string.h>
+
+static gts_shared_resource g_gts_resource;
+
+gts_shared_resource *gts_get_shared_resource(void) { return &g_gts_resource; }
+
+void grpc_tsi_gts_init() {
+  memset(&g_gts_resource, 0, sizeof(gts_shared_resource));
+  gpr_mu_init(&g_gts_resource.mu);
+}
+
+void grpc_tsi_gts_shutdown() {
+  gpr_mu_destroy(&g_gts_resource.mu);
+  if (g_gts_resource.cq == NULL) {
+    return;
+  }
+  grpc_completion_queue_destroy(g_gts_resource.cq);
+  grpc_channel_destroy(g_gts_resource.channel);
+  gpr_thd_join(g_gts_resource.thread_id);
+}
diff --git a/src/core/tsi/gts_transport_security.h b/src/core/tsi/gts_transport_security.h
new file mode 100644
index 0000000000..538e1030bc
--- /dev/null
+++ b/src/core/tsi/gts_transport_security.h
@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_TSI_GTS_TRANSPORT_SECURITY_H
+#define GRPC_CORE_TSI_GTS_TRANSPORT_SECURITY_H
+
+#include <grpc/grpc.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
+
+typedef struct gts_shared_resource {
+  gpr_thd_id thread_id;
+  grpc_channel *channel;
+  grpc_completion_queue *cq;
+  gpr_mu mu;
+} gts_shared_resource;
+
+/* This method returns the address of gts_shared_resource object shared by all
+ *    TSI handshakes. */
+gts_shared_resource *gts_get_shared_resource(void);
+
+#endif /* GRPC_CORE_TSI_GTS_TRANSPORT_SECURITY_H */
diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c
index 37d4f038b7..1fd65928f9 100644
--- a/src/core/tsi/ssl_transport_security.c
+++ b/src/core/tsi/ssl_transport_security.c
@@ -411,15 +411,11 @@ static tsi_result do_ssl_read(SSL *ssl, unsigned char *unprotected_bytes,
   GPR_ASSERT(*unprotected_bytes_size <= INT_MAX);
   read_from_ssl =
       SSL_read(ssl, unprotected_bytes, (int)*unprotected_bytes_size);
-  if (read_from_ssl == 0) {
-    gpr_log(GPR_ERROR, "SSL_read returned 0 unexpectedly.");
-    return TSI_INTERNAL_ERROR;
-  }
-  if (read_from_ssl < 0) {
+  if (read_from_ssl <= 0) {
     read_from_ssl = SSL_get_error(ssl, read_from_ssl);
     switch (read_from_ssl) {
-      case SSL_ERROR_WANT_READ:
-        /* We need more data to finish the frame. */
+      case SSL_ERROR_ZERO_RETURN: /* Received a close_notify alert. */
+      case SSL_ERROR_WANT_READ:   /* We need more data to finish the frame. */
         *unprotected_bytes_size = 0;
         return TSI_OK;
       case SSL_ERROR_WANT_WRITE:
diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c
index 08fa43138d..be11d64472 100644
--- a/src/core/tsi/transport_security.c
+++ b/src/core/tsi/transport_security.c
@@ -26,7 +26,7 @@
 
 /* --- Tracing. --- */
 
-grpc_tracer_flag tsi_tracing_enabled = GRPC_TRACER_INITIALIZER(false);
+grpc_tracer_flag tsi_tracing_enabled = GRPC_TRACER_INITIALIZER(false, "tsi");
 
 /* --- tsi_result common implementation. --- */